1#ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2#define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
16namespace Synchronization {
23struct remove_volatile {
27struct remove_volatile<volatile T> {
33 return __sync_and_and_fetch(addr, val);
37ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
38 return __sync_fetch_and_add(addr, val);
41template<typename T, typename TV = typename remove_volatile<T>::type>
42ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
43 return __sync_fetch_and_add(addr, val);
46template<typename T, typename TV = typename remove_volatile<T>::type>
47ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
48 return __sync_fetch_and_sub(addr, val);
51template<typename T, typename TV = typename remove_volatile<T>::type>
52ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
53 return __sync_fetch_and_or(addr, val);
57ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
58 return __sync_add_and_fetch(addr, val);
62ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
63 return __sync_sub_and_fetch(addr, val);
66template<typename T, typename TV = typename remove_volatile<T>::type>
67ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
68 TV oldval = *expected;
69 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
71 return oldval == gotval;
75 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
78template<typename T, typename TV = typename remove_volatile<T>::type>
79ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
84 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
88ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
97 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
102 return __sync_fetch_and_and(addr, val);
105template<typename T, typename TV = typename remove_volatile<T>::type>
106ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
107 return __sync_fetch_and_and(addr, val);
117 __sync_synchronize();
124 return __sync_lock_test_and_set(addr, val);
128 return __sync_or_and_fetch(addr, val);
138 __sync_synchronize();
141template<typename T, typename TV = typename remove_volatile<T>::type>
142ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
144 __sync_synchronize();
148 __sync_synchronize();
151ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
152 __sync_synchronize();
158 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
162ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
163 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
166template<typename T, typename TV = typename remove_volatile<T>::type>
167ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
168 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
171template<typename T, typename TV = typename remove_volatile<T>::type>
172ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
173 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
176template<typename T, typename TV = typename remove_volatile<T>::type>
177ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
178 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
182ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
183 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
187ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
188 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
192 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
195template<typename T, typename TV = typename remove_volatile<T>::type>
196ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
197 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
201ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
202 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
206 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
210 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
214 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
219 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
222template<typename T, typename TV = typename remove_volatile<T>::type>
224 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
229 __atomic_load(addr, val, __ATOMIC_RELAXED);
234 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
235 __sync_synchronize();
242 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
247 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
251 __atomic_store(addr, val, __ATOMIC_RELAXED);
256 __atomic_store(addr, val, __ATOMIC_RELEASE);
259template<typename T, typename TV = typename remove_volatile<T>::type>
260ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
261 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
265 __atomic_thread_fence(__ATOMIC_ACQUIRE);
268ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
269 __atomic_thread_fence(__ATOMIC_SEQ_CST);
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t