Debian Codegraph @

rts/c/atomics64.h

raw
1 // Start of atomics64.h
2
3 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP) || defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)
4
5 SCALAR_FUN_ATTR int64_t atomic_xchg_i64_global(volatile __global int64_t *p, int64_t x);
6 SCALAR_FUN_ATTR int64_t atomic_xchg_i64_shared(volatile __local int64_t *p, int64_t x);
7 SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_global(volatile __global int64_t *p,
8 int64_t cmp, int64_t val);
9 SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_shared(volatile __local int64_t *p,
10 int64_t cmp, int64_t val);
11 SCALAR_FUN_ATTR int64_t atomic_add_i64_global(volatile __global int64_t *p, int64_t x);
12 SCALAR_FUN_ATTR int64_t atomic_add_i64_shared(volatile __local int64_t *p, int64_t x);
13 SCALAR_FUN_ATTR int64_t atomic_smax_i64_global(volatile __global int64_t *p, int64_t x);
14 SCALAR_FUN_ATTR int64_t atomic_smax_i64_shared(volatile __local int64_t *p, int64_t x);
15 SCALAR_FUN_ATTR int64_t atomic_smin_i64_global(volatile __global int64_t *p, int64_t x);
16 SCALAR_FUN_ATTR int64_t atomic_smin_i64_shared(volatile __local int64_t *p, int64_t x);
17 SCALAR_FUN_ATTR uint64_t atomic_umax_i64_global(volatile __global uint64_t *p, uint64_t x);
18 SCALAR_FUN_ATTR uint64_t atomic_umax_i64_shared(volatile __local uint64_t *p, uint64_t x);
19 SCALAR_FUN_ATTR uint64_t atomic_umin_i64_global(volatile __global uint64_t *p, uint64_t x);
20 SCALAR_FUN_ATTR uint64_t atomic_umin_i64_shared(volatile __local uint64_t *p, uint64_t x);
21 SCALAR_FUN_ATTR int64_t atomic_and_i64_global(volatile __global int64_t *p, int64_t x);
22 SCALAR_FUN_ATTR int64_t atomic_and_i64_shared(volatile __local int64_t *p, int64_t x);
23 SCALAR_FUN_ATTR int64_t atomic_or_i64_global(volatile __global int64_t *p, int64_t x);
24 SCALAR_FUN_ATTR int64_t atomic_or_i64_shared(volatile __local int64_t *p, int64_t x);
25 SCALAR_FUN_ATTR int64_t atomic_xor_i64_global(volatile __global int64_t *p, int64_t x);
26 SCALAR_FUN_ATTR int64_t atomic_xor_i64_shared(volatile __local int64_t *p, int64_t x);
27
28 #ifdef FUTHARK_F64_ENABLED
29 SCALAR_FUN_ATTR double atomic_fadd_f64_global(volatile __global double *p, double x);
30 SCALAR_FUN_ATTR double atomic_fadd_f64_shared(volatile __local double *p, double x);
31 #endif
32
33 SCALAR_FUN_ATTR int64_t atomic_xchg_i64_global(volatile __global int64_t *p, int64_t x) {
34 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
35 return atomicExch((unsigned long long*)p, x);
36 #else
37 return atom_xchg(p, x);
38 #endif
39 }
40
41 SCALAR_FUN_ATTR int64_t atomic_xchg_i64_shared(volatile __local int64_t *p, int64_t x) {
42 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
43 return atomicExch((unsigned long long*)p, x);
44 #else
45 return atom_xchg(p, x);
46 #endif
47 }
48
49 SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_global(volatile __global int64_t *p,
50 int64_t cmp, int64_t val) {
51 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
52 return atomicCAS((unsigned long long*)p, cmp, val);
53 #else
54 return atom_cmpxchg(p, cmp, val);
55 #endif
56 }
57
58 SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_shared(volatile __local int64_t *p,
59 int64_t cmp, int64_t val) {
60 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
61 return atomicCAS((unsigned long long*)p, cmp, val);
62 #else
63 return atom_cmpxchg(p, cmp, val);
64 #endif
65 }
66
67 SCALAR_FUN_ATTR int64_t atomic_add_i64_global(volatile __global int64_t *p, int64_t x) {
68 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
69 return atomicAdd((unsigned long long*)p, x);
70 #else
71 return atom_add(p, x);
72 #endif
73 }
74
75 SCALAR_FUN_ATTR int64_t atomic_add_i64_shared(volatile __local int64_t *p, int64_t x) {
76 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
77 return atomicAdd((unsigned long long*)p, x);
78 #else
79 return atom_add(p, x);
80 #endif
81 }
82
83 #ifdef FUTHARK_F64_ENABLED
84
85 SCALAR_FUN_ATTR double atomic_fadd_f64_global(volatile __global double *p, double x) {
86 #if defined(FUTHARK_CUDA) && __CUDA_ARCH__ >= 600 || defined(FUTHARK_HIP)
87 return atomicAdd((double*)p, x);
88 // On OpenCL, use technique from
89 // https://pipinspace.github.io/blog/atomic-float-addition-in-opencl.html
90 #elif defined(cl_nv_pragma_unroll)
91 // use hardware-supported atomic addition on Nvidia GPUs with inline
92 // PTX assembly
93 double ret;
94 asm volatile("atom.global.add.f64 %0,[%1],%2;":"=d"(ret):"l"(p),"d"(x):"memory");
95 return ret;
96 #elif __has_builtin(__builtin_amdgcn_global_atomic_fadd_f64)
97 // use hardware-supported atomic addition on some AMD GPUs
98 return __builtin_amdgcn_global_atomic_fadd_f64(p, x);
99 #else
100 // fallback emulation:
101 // https://forums.developer.nvidia.com/t/atomicadd-float-float-atomicmul-float-float/14639/5
102 union {int64_t i; double f;} old;
103 union {int64_t i; double f;} ret;
104 old.f = x;
105 while (1) {
106 ret.i = atomic_xchg_i64_global((volatile __global int64_t*)p, (int64_t)0);
107 ret.f += old.f;
108 old.i = atomic_xchg_i64_global((volatile __global int64_t*)p, ret.i);
109 if (old.i == 0) {
110 break;
111 }
112 }
113 return ret.f;
114 #endif
115 }
116
117 SCALAR_FUN_ATTR double atomic_fadd_f64_shared(volatile __local double *p, double x) {
118 #if defined(FUTHARK_CUDA) && __CUDA_ARCH__ >= 600 || defined(FUTHARK_HIP)
119 return atomicAdd((double*)p, x);
120 #else
121 union { int64_t i; double f; } old;
122 union { int64_t i; double f; } assumed;
123 old.f = *p;
124 do {
125 assumed.f = old.f;
126 old.f = old.f + x;
127 old.i = atomic_cmpxchg_i64_shared((volatile __local int64_t*)p, assumed.i, old.i);
128 } while (assumed.i != old.i);
129 return old.f;
130 #endif
131 }
132
133 #endif
134
135 SCALAR_FUN_ATTR int64_t atomic_smax_i64_global(volatile __global int64_t *p, int64_t x) {
136 #if defined(FUTHARK_CUDA)
137 return atomicMax((long long*)p, x);
138 #elif defined(FUTHARK_HIP)
139 // Currentely missing in HIP; probably a temporary oversight.
140 int64_t old = *p, assumed;
141 do {
142 assumed = old;
143 old = smax64(old, x);
144 old = atomic_cmpxchg_i64_global((volatile __global int64_t*)p, assumed, old);
145 } while (assumed != old);
146 return old;
147 #else
148 return atom_max(p, x);
149 #endif
150 }
151
152 SCALAR_FUN_ATTR int64_t atomic_smax_i64_shared(volatile __local int64_t *p, int64_t x) {
153 #if defined(FUTHARK_CUDA)
154 return atomicMax((long long*)p, x);
155 #elif defined(FUTHARK_HIP)
156 // Currentely missing in HIP; probably a temporary oversight.
157 int64_t old = *p, assumed;
158 do {
159 assumed = old;
160 old = smax64(old, x);
161 old = atomic_cmpxchg_i64_shared((volatile __local int64_t*)p, assumed, old);
162 } while (assumed != old);
163 return old;
164 #else
165 return atom_max(p, x);
166 #endif
167 }
168
169 SCALAR_FUN_ATTR int64_t atomic_smin_i64_global(volatile __global int64_t *p, int64_t x) {
170 #if defined(FUTHARK_CUDA)
171 return atomicMin((long long*)p, x);
172 #elif defined(FUTHARK_HIP)
173 // Currentely missing in HIP; probably a temporary oversight.
174 int64_t old = *p, assumed;
175 do {
176 assumed = old;
177 old = smin64(old, x);
178 old = atomic_cmpxchg_i64_global((volatile __global int64_t*)p, assumed, old);
179 } while (assumed != old);
180 return old;
181 #else
182 return atom_min(p, x);
183 #endif
184 }
185
186 SCALAR_FUN_ATTR int64_t atomic_smin_i64_shared(volatile __local int64_t *p, int64_t x) {
187 #if defined(FUTHARK_CUDA)
188 return atomicMin((long long*)p, x);
189 #elif defined(FUTHARK_HIP)
190 // Currentely missing in HIP; probably a temporary oversight.
191 int64_t old = *p, assumed;
192 do {
193 assumed = old;
194 old = smin64(old, x);
195 old = atomic_cmpxchg_i64_shared((volatile __local int64_t*)p, assumed, old);
196 } while (assumed != old);
197 return old;
198 #else
199 return atom_min(p, x);
200 #endif
201 }
202
203 SCALAR_FUN_ATTR uint64_t atomic_umax_i64_global(volatile __global uint64_t *p, uint64_t x) {
204 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
205 return atomicMax((unsigned long long*)p, x);
206 #else
207 return atom_max(p, x);
208 #endif
209 }
210
211 SCALAR_FUN_ATTR uint64_t atomic_umax_i64_shared(volatile __local uint64_t *p, uint64_t x) {
212 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
213 return atomicMax((unsigned long long*)p, x);
214 #else
215 return atom_max(p, x);
216 #endif
217 }
218
219 SCALAR_FUN_ATTR uint64_t atomic_umin_i64_global(volatile __global uint64_t *p, uint64_t x) {
220 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
221 return atomicMin((unsigned long long*)p, x);
222 #else
223 return atom_min(p, x);
224 #endif
225 }
226
227 SCALAR_FUN_ATTR uint64_t atomic_umin_i64_shared(volatile __local uint64_t *p, uint64_t x) {
228 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
229 return atomicMin((unsigned long long*)p, x);
230 #else
231 return atom_min(p, x);
232 #endif
233 }
234
235 SCALAR_FUN_ATTR int64_t atomic_and_i64_global(volatile __global int64_t *p, int64_t x) {
236 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
237 return atomicAnd((unsigned long long*)p, x);
238 #else
239 return atom_and(p, x);
240 #endif
241 }
242
243 SCALAR_FUN_ATTR int64_t atomic_and_i64_shared(volatile __local int64_t *p, int64_t x) {
244 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
245 return atomicAnd((unsigned long long*)p, x);
246 #else
247 return atom_and(p, x);
248 #endif
249 }
250
251 SCALAR_FUN_ATTR int64_t atomic_or_i64_global(volatile __global int64_t *p, int64_t x) {
252 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
253 return atomicOr((unsigned long long*)p, x);
254 #else
255 return atom_or(p, x);
256 #endif
257 }
258
259 SCALAR_FUN_ATTR int64_t atomic_or_i64_shared(volatile __local int64_t *p, int64_t x) {
260 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
261 return atomicOr((unsigned long long*)p, x);
262 #else
263 return atom_or(p, x);
264 #endif
265 }
266
267 SCALAR_FUN_ATTR int64_t atomic_xor_i64_global(volatile __global int64_t *p, int64_t x) {
268 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
269 return atomicXor((unsigned long long*)p, x);
270 #else
271 return atom_xor(p, x);
272 #endif
273 }
274
275 SCALAR_FUN_ATTR int64_t atomic_xor_i64_shared(volatile __local int64_t *p, int64_t x) {
276 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
277 return atomicXor((unsigned long long*)p, x);
278 #else
279 return atom_xor(p, x);
280 #endif
281 }
282
283 #endif // defined(FUTHARK_CUDA) || defined(FUTHARK_HIP) || defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)
284
285 // End of atomics64.h