1 #ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2 #define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
16 namespace Synchronization {
23 struct remove_volatile {
27 struct remove_volatile<volatile T> {
33 return __sync_and_and_fetch(addr, val);
37 ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
38 return __sync_fetch_and_add(addr, val);
41 template<typename T, typename TV = typename remove_volatile<T>::type>
42 ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
43 return __sync_fetch_and_add(addr, val);
46 template<typename T, typename TV = typename remove_volatile<T>::type>
47 ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
48 return __sync_fetch_and_sub(addr, val);
51 template<typename T, typename TV = typename remove_volatile<T>::type>
52 ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
53 return __sync_fetch_and_or(addr, val);
57 ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
58 return __sync_add_and_fetch(addr, val);
62 ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
63 return __sync_sub_and_fetch(addr, val);
66 template<typename T, typename TV = typename remove_volatile<T>::type>
67 ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
68 TV oldval = *expected;
69 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
71 return oldval == gotval;
75 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
78 template<typename T, typename TV = typename remove_volatile<T>::type>
79 ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
84 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
88 ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
97 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
102 return __sync_fetch_and_and(addr, val);
105 template<typename T, typename TV = typename remove_volatile<T>::type>
106 ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
107 return __sync_fetch_and_and(addr, val);
117 __sync_synchronize();
124 return __sync_lock_test_and_set(addr, val);
128 return __sync_or_and_fetch(addr, val);
139 __sync_synchronize();
142 template<typename T, typename TV = typename remove_volatile<T>::type>
143 ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
145 __sync_synchronize();
149 __sync_synchronize();
152 ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
153 __sync_synchronize();
159 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
163 ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
164 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
167 template<typename T, typename TV = typename remove_volatile<T>::type>
168 ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
169 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
172 template<typename T, typename TV = typename remove_volatile<T>::type>
173 ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
174 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
177 template<typename T, typename TV = typename remove_volatile<T>::type>
178 ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
179 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
183 ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
184 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
188 ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
189 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
193 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
196 template<typename T, typename TV = typename remove_volatile<T>::type>
197 ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
198 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
202 ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
203 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
207 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
211 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
215 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
220 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
223 template<typename T, typename TV = typename remove_volatile<T>::type>
225 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
230 __atomic_load(addr, val, __ATOMIC_RELAXED);
235 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
236 __sync_synchronize();
243 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
248 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
253 __atomic_store(addr, val, __ATOMIC_RELAXED);
258 __atomic_store(addr, val, __ATOMIC_RELEASE);
261 template<typename T, typename TV = typename remove_volatile<T>::type>
262 ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
263 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
267 __atomic_thread_fence(__ATOMIC_ACQUIRE);
270 ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
271 __atomic_thread_fence(__ATOMIC_SEQ_CST);
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t