1 #ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2 #define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
16 namespace Synchronization {
23 struct remove_volatile {
27 struct remove_volatile<volatile T> {
33 return __sync_and_and_fetch(addr, val);
37 ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
38 return __sync_fetch_and_add(addr, val);
41 template<typename T, typename TV = typename remove_volatile<T>::type>
42 ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
43 return __sync_fetch_and_add(addr, val);
46 template<typename T, typename TV = typename remove_volatile<T>::type>
47 ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
48 return __sync_fetch_and_sub(addr, val);
51 template<typename T, typename TV = typename remove_volatile<T>::type>
52 ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
53 return __sync_fetch_and_or(addr, val);
57 ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
58 return __sync_add_and_fetch(addr, val);
62 ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
63 return __sync_sub_and_fetch(addr, val);
66 template<typename T, typename TV = typename remove_volatile<T>::type>
67 ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
68 TV oldval = *expected;
69 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
71 return oldval == gotval;
75 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
78 template<typename T, typename TV = typename remove_volatile<T>::type>
79 ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
84 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
88 ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
97 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
102 return __sync_fetch_and_and(addr, val);
105 template<typename T, typename TV = typename remove_volatile<T>::type>
106 ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
107 return __sync_fetch_and_and(addr, val);
117 __sync_synchronize();
124 return __sync_lock_test_and_set(addr, val);
128 return __sync_or_and_fetch(addr, val);
138 __sync_synchronize();
141 template<typename T, typename TV = typename remove_volatile<T>::type>
142 ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
144 __sync_synchronize();
148 __sync_synchronize();
151 ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
152 __sync_synchronize();
158 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
162 ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
163 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
166 template<typename T, typename TV = typename remove_volatile<T>::type>
167 ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
168 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
171 template<typename T, typename TV = typename remove_volatile<T>::type>
172 ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
173 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
176 template<typename T, typename TV = typename remove_volatile<T>::type>
177 ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
178 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
182 ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
183 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
187 ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
188 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
192 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
195 template<typename T, typename TV = typename remove_volatile<T>::type>
196 ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
197 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
201 ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
202 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
206 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
210 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
214 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
219 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
222 template<typename T, typename TV = typename remove_volatile<T>::type>
224 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
229 __atomic_load(addr, val, __ATOMIC_RELAXED);
234 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
235 __sync_synchronize();
242 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
247 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
251 __atomic_store(addr, val, __ATOMIC_RELAXED);
256 __atomic_store(addr, val, __ATOMIC_RELEASE);
259 template<typename T, typename TV = typename remove_volatile<T>::type>
260 ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
261 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
265 __atomic_thread_fence(__ATOMIC_ACQUIRE);
268 ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
269 __atomic_thread_fence(__ATOMIC_SEQ_CST);
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t