1 #ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H 2 #define HALIDE_RUNTIME_RUNTIME_ATOMICS_H 16 namespace Synchronization {
23 struct remove_volatile {
27 struct remove_volatile<volatile T> {
33 return __sync_and_and_fetch(addr, val);
37 ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
38 return __sync_fetch_and_add(addr, val);
41 template<typename T, typename TV = typename remove_volatile<T>::type>
42 ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
43 return __sync_fetch_and_add(addr, val);
46 template<typename T, typename TV = typename remove_volatile<T>::type>
47 ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
48 return __sync_fetch_and_sub(addr, val);
51 template<typename T, typename TV = typename remove_volatile<T>::type>
52 ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
53 return __sync_fetch_and_or(addr, val);
57 ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
58 return __sync_add_and_fetch(addr, val);
62 ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
63 return __sync_sub_and_fetch(addr, val);
66 template<typename T, typename TV = typename remove_volatile<T>::type>
67 ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
68 TV oldval = *expected;
69 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
71 return oldval == gotval;
75 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
78 template<typename T, typename TV = typename remove_volatile<T>::type>
79 ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
84 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
88 ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
97 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
102 return __sync_fetch_and_and(addr, val);
105 template<typename T, typename TV = typename remove_volatile<T>::type>
106 ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
107 return __sync_fetch_and_and(addr, val);
117 __sync_synchronize();
124 return __sync_lock_test_and_set(addr, val);
128 return __sync_or_and_fetch(addr, val);
138 __sync_synchronize();
141 template<typename T, typename TV = typename remove_volatile<T>::type>
142 ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
144 __sync_synchronize();
148 __sync_synchronize();
151 ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
152 __sync_synchronize();
158 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
162 ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
163 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
166 template<typename T, typename TV = typename remove_volatile<T>::type>
167 ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
168 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
171 template<typename T, typename TV = typename remove_volatile<T>::type>
172 ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
173 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
176 template<typename T, typename TV = typename remove_volatile<T>::type>
177 ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
178 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
182 ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
183 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
187 ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
188 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
192 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
195 template<typename T, typename TV = typename remove_volatile<T>::type>
196 ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
197 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
201 ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
202 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
206 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
210 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
214 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
219 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
222 template<typename T, typename TV = typename remove_volatile<T>::type>
224 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
229 __atomic_load(addr, val, __ATOMIC_RELAXED);
234 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
235 __sync_synchronize();
242 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
247 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
251 __atomic_store(addr, val, __ATOMIC_RELAXED);
256 __atomic_store(addr, val, __ATOMIC_RELEASE);
259 template<typename T, typename TV = typename remove_volatile<T>::type>
260 ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
261 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
265 __atomic_thread_fence(__ATOMIC_ACQUIRE);
268 ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
269 __atomic_thread_fence(__ATOMIC_SEQ_CST);
281 #endif // HALIDE_RUNTIME_RUNTIME_ATOMICS_H This file defines the class FunctionDAG, which is our representation of a Halide pipeline, and contains methods to using Halide's bounds tools to query properties of it.
This file declares the routines used by Halide internally in its runtime.
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t