Halide 18.0.0
Halide compiler and libraries
Loading...
Searching...
No Matches
runtime_atomics.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2#define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
3
4// This file provides an abstraction layer over the __sync/__atomic builtins
5// in Clang; for various reasons, we use __sync for 32-bit targets, and
6// __atomic for 64-bit. At some point it may be desirable/necessary to
7// migrate 32-bit to __atomic as well, at which time this file can
8// likely go away. See https://github.com/halide/Halide/issues/7431 for
9// a discussion of the history and issues as to why we work this way.
10
11#include "HalideRuntime.h"
12
13namespace Halide {
14namespace Runtime {
15namespace Internal {
16namespace Synchronization {
17
18namespace {
19
20// TODO: most of these wrappers should do the remove_volatile for secondary arguments;
21// I've only put it in place for the locations necessary at this time.
22template<class T>
23struct remove_volatile {
24 typedef T type;
25};
26template<class T>
27struct remove_volatile<volatile T> {
28 typedef T type;
29};
30
31#ifdef BITS_32
33 return __sync_and_and_fetch(addr, val);
34}
35
36template<typename T>
38 return __sync_fetch_and_add(addr, val);
39}
40
43 return __sync_fetch_and_add(addr, val);
44}
45
48 return __sync_fetch_and_sub(addr, val);
49}
50
53 return __sync_fetch_and_or(addr, val);
54}
55
56template<typename T>
58 return __sync_add_and_fetch(addr, val);
59}
60
61template<typename T>
63 return __sync_sub_and_fetch(addr, val);
64}
65
71 return oldval == gotval;
72}
73
76}
77
81}
82
85}
86
87template<typename T>
90}
91
94}
95
98}
99
100template<typename T>
101ALWAYS_INLINE T atomic_fetch_and_release(T *addr, T val) {
102 return __sync_fetch_and_and(addr, val);
103}
104
107 return __sync_fetch_and_and(addr, val);
108}
109
110template<typename T>
111ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
112 *val = *addr;
113}
114
115template<typename T>
116ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
118 *val = *addr;
119}
120
121template<typename T>
122ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
123 // Despite the name, this is really just an exchange operation with acquire ordering.
124 return __sync_lock_test_and_set(addr, val);
125}
126
128 return __sync_or_and_fetch(addr, val);
129}
130
131template<typename T>
132ALWAYS_INLINE void atomic_store_relaxed(T *addr, T *val) {
133 *addr = *val;
134}
135
136template<typename T>
137ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
138 *addr = *val;
140}
141
144 *addr = *val;
146}
147
150}
151
154}
155
156#else
157
159 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
160}
161
162template<typename T>
164 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
165}
166
169 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
170}
171
174 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
175}
176
179 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
180}
181
182template<typename T>
184 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
185}
186
187template<typename T>
189 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
190}
191
194}
195
199}
200
201template<typename T>
204}
205
208}
209
212}
213
216}
217
218template<typename T>
220 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
221}
222
225 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
226}
227
228template<typename T>
229ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
231}
232
233template<typename T>
234ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
237 *val = *addr;
238}
239
240template<typename T>
241ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
242 T result;
243 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
244 return result;
245}
246
248 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
249}
250
251template<typename T>
252ALWAYS_INLINE void atomic_store_relaxed(T *addr, T *val) {
254}
255
256template<typename T>
257ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
259}
260
264}
265
268}
269
272}
273
274#endif
275
276} // namespace
277
278} // namespace Synchronization
279} // namespace Internal
280} // namespace Runtime
281} // namespace Halide
282
283#endif // HALIDE_RUNTIME_RUNTIME_ATOMICS_H
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Internal::ConstantInterval cast(Type t, const Internal::ConstantInterval &a)
Cast operators for ConstantIntervals.
__UINTPTR_TYPE__ uintptr_t
#define ALWAYS_INLINE