Halide 17.0.2
Halide compiler and libraries
Loading...
Searching...
No Matches
runtime_atomics.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2#define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
3
4// This file provides an abstraction layer over the __sync/__atomic builtins
5// in Clang; for various reasons, we use __sync for 32-bit targets, and
6// __atomic for 64-bit. At some point it may be desirable/necessary to
7// migrate 32-bit to __atomic as well, at which time this file can
8// likely go away. See https://github.com/halide/Halide/issues/7431 for
9// a discussion of the history and issues as to why we work this way.
10
11#include "HalideRuntime.h"
12
13namespace Halide {
14namespace Runtime {
15namespace Internal {
16namespace Synchronization {
17
18namespace {
19
20// TODO: most of these wrappers should do the remove_volatile for secondary arguments;
21// I've only put it in place for the locations necessary at this time.
22template<class T>
23struct remove_volatile {
24 typedef T type;
25};
26template<class T>
27struct remove_volatile<volatile T> {
28 typedef T type;
29};
30
31#ifdef BITS_32
32ALWAYS_INLINE uintptr_t atomic_and_fetch_release(uintptr_t *addr, uintptr_t val) {
33 return __sync_and_and_fetch(addr, val);
34}
35
36template<typename T>
37ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
38 return __sync_fetch_and_add(addr, val);
39}
40
41template<typename T, typename TV = typename remove_volatile<T>::type>
42ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
43 return __sync_fetch_and_add(addr, val);
44}
45
46template<typename T, typename TV = typename remove_volatile<T>::type>
47ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
48 return __sync_fetch_and_sub(addr, val);
49}
50
51template<typename T, typename TV = typename remove_volatile<T>::type>
52ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
53 return __sync_fetch_and_or(addr, val);
54}
55
56template<typename T>
57ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
58 return __sync_add_and_fetch(addr, val);
59}
60
61template<typename T>
62ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
63 return __sync_sub_and_fetch(addr, val);
64}
65
66template<typename T, typename TV = typename remove_volatile<T>::type>
67ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
68 TV oldval = *expected;
69 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
70 *expected = gotval;
71 return oldval == gotval;
72}
73
74ALWAYS_INLINE bool atomic_cas_strong_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
75 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
76}
77
78template<typename T, typename TV = typename remove_volatile<T>::type>
79ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
81}
82
83ALWAYS_INLINE bool atomic_cas_weak_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
84 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
85}
86
87template<typename T>
88ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
90}
91
92ALWAYS_INLINE bool atomic_cas_weak_relaxed_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
94}
95
96ALWAYS_INLINE bool atomic_cas_weak_acquire_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
97 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
98}
99
100template<typename T>
101ALWAYS_INLINE T atomic_fetch_and_release(T *addr, T val) {
102 return __sync_fetch_and_and(addr, val);
103}
104
105template<typename T, typename TV = typename remove_volatile<T>::type>
106ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
107 return __sync_fetch_and_and(addr, val);
108}
109
110template<typename T>
111ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
112 *val = *addr;
113}
114
115template<typename T>
116ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
117 __sync_synchronize();
118 *val = *addr;
119}
120
121template<typename T>
122ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
123 // Despite the name, this is really just an exchange operation with acquire ordering.
124 return __sync_lock_test_and_set(addr, val);
125}
126
127ALWAYS_INLINE uintptr_t atomic_or_fetch_relaxed(uintptr_t *addr, uintptr_t val) {
128 return __sync_or_and_fetch(addr, val);
129}
130
131ALWAYS_INLINE void atomic_store_relaxed(uintptr_t *addr, uintptr_t *val) {
132 *addr = *val;
133}
134
135template<typename T>
136ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
137 *addr = *val;
138 __sync_synchronize();
139}
140
141template<typename T, typename TV = typename remove_volatile<T>::type>
142ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
143 *addr = *val;
144 __sync_synchronize();
145}
146
147ALWAYS_INLINE void atomic_thread_fence_acquire() {
148 __sync_synchronize();
149}
150
151ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
152 __sync_synchronize();
153}
154
155#else
156
157ALWAYS_INLINE uintptr_t atomic_and_fetch_release(uintptr_t *addr, uintptr_t val) {
158 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
159}
160
161template<typename T>
162ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
163 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
164}
165
166template<typename T, typename TV = typename remove_volatile<T>::type>
167ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
168 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
169}
170
171template<typename T, typename TV = typename remove_volatile<T>::type>
172ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
173 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
174}
175
176template<typename T, typename TV = typename remove_volatile<T>::type>
177ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
178 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
179}
180
181template<typename T>
182ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
183 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
184}
185
186template<typename T>
187ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
188 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
189}
190
191ALWAYS_INLINE bool atomic_cas_strong_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
192 return __atomic_compare_exchange(addr, expected, desired, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
193}
194
195template<typename T, typename TV = typename remove_volatile<T>::type>
196ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
197 return __atomic_compare_exchange(addr, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
198}
199
200template<typename T>
201ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
202 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
203}
204
205ALWAYS_INLINE bool atomic_cas_weak_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
206 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
207}
208
209ALWAYS_INLINE bool atomic_cas_weak_relaxed_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
210 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
211}
212
213ALWAYS_INLINE bool atomic_cas_weak_acquire_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
214 return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
215}
216
217template<typename T>
218ALWAYS_INLINE uintptr_t atomic_fetch_and_release(T *addr, T val) {
219 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
220}
221
222template<typename T, typename TV = typename remove_volatile<T>::type>
223ALWAYS_INLINE uintptr_t atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
224 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
225}
226
227template<typename T>
228ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
229 __atomic_load(addr, val, __ATOMIC_RELAXED);
230}
231
232template<typename T>
233ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
234 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
235 __sync_synchronize();
236 *val = *addr;
237}
238
239template<typename T>
240ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
241 T result;
242 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
243 return result;
244}
245
246ALWAYS_INLINE uintptr_t atomic_or_fetch_relaxed(uintptr_t *addr, uintptr_t val) {
247 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
248}
249
250ALWAYS_INLINE void atomic_store_relaxed(uintptr_t *addr, uintptr_t *val) {
251 __atomic_store(addr, val, __ATOMIC_RELAXED);
252}
253
254template<typename T>
255ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
256 __atomic_store(addr, val, __ATOMIC_RELEASE);
257}
258
259template<typename T, typename TV = typename remove_volatile<T>::type>
260ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
261 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
262}
263
264ALWAYS_INLINE void atomic_thread_fence_acquire() {
265 __atomic_thread_fence(__ATOMIC_ACQUIRE);
266}
267
268ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
269 __atomic_thread_fence(__ATOMIC_SEQ_CST);
270}
271
272#endif
273
274} // namespace
275
276} // namespace Synchronization
277} // namespace Internal
278} // namespace Runtime
279} // namespace Halide
280
281#endif // HALIDE_RUNTIME_RUNTIME_ATOMICS_H
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t
#define ALWAYS_INLINE