Debian Codegraph @

rts/c/atomics32.h

raw
1 // Start of atomics.h
2
3 SCALAR_FUN_ATTR int32_t atomic_xchg_i32_global(volatile __global int32_t *p, int32_t x);
4 SCALAR_FUN_ATTR int32_t atomic_xchg_i32_shared(volatile __local int32_t *p, int32_t x);
5 SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_global(volatile __global int32_t *p,
6 int32_t cmp, int32_t val);
7 SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_shared(volatile __local int32_t *p,
8 int32_t cmp, int32_t val);
9 SCALAR_FUN_ATTR int32_t atomic_add_i32_global(volatile __global int32_t *p, int32_t x);
10 SCALAR_FUN_ATTR int32_t atomic_add_i32_shared(volatile __local int32_t *p, int32_t x);
11 SCALAR_FUN_ATTR float atomic_fadd_f32_global(volatile __global float *p, float x);
12 SCALAR_FUN_ATTR float atomic_fadd_f32_shared(volatile __local float *p, float x);
13 SCALAR_FUN_ATTR int32_t atomic_smax_i32_global(volatile __global int32_t *p, int32_t x);
14 SCALAR_FUN_ATTR int32_t atomic_smax_i32_shared(volatile __local int32_t *p, int32_t x);
15 SCALAR_FUN_ATTR int32_t atomic_smin_i32_global(volatile __global int32_t *p, int32_t x);
16 SCALAR_FUN_ATTR int32_t atomic_smin_i32_shared(volatile __local int32_t *p, int32_t x);
17 SCALAR_FUN_ATTR uint32_t atomic_umax_i32_global(volatile __global uint32_t *p, uint32_t x);
18 SCALAR_FUN_ATTR uint32_t atomic_umax_i32_shared(volatile __local uint32_t *p, uint32_t x);
19 SCALAR_FUN_ATTR uint32_t atomic_umin_i32_global(volatile __global uint32_t *p, uint32_t x);
20 SCALAR_FUN_ATTR uint32_t atomic_umin_i32_shared(volatile __local uint32_t *p, uint32_t x);
21 SCALAR_FUN_ATTR int32_t atomic_and_i32_global(volatile __global int32_t *p, int32_t x);
22 SCALAR_FUN_ATTR int32_t atomic_and_i32_shared(volatile __local int32_t *p, int32_t x);
23 SCALAR_FUN_ATTR int32_t atomic_or_i32_global(volatile __global int32_t *p, int32_t x);
24 SCALAR_FUN_ATTR int32_t atomic_or_i32_shared(volatile __local int32_t *p, int32_t x);
25 SCALAR_FUN_ATTR int32_t atomic_xor_i32_global(volatile __global int32_t *p, int32_t x);
26 SCALAR_FUN_ATTR int32_t atomic_xor_i32_shared(volatile __local int32_t *p, int32_t x);
27
28 SCALAR_FUN_ATTR int32_t atomic_xchg_i32_global(volatile __global int32_t *p, int32_t x) {
29 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
30 return atomicExch((int32_t*)p, x);
31 #else
32 return atomic_xor(p, x);
33 #endif
34 }
35
36 SCALAR_FUN_ATTR int32_t atomic_xchg_i32_shared(volatile __local int32_t *p, int32_t x) {
37 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
38 return atomicExch((int32_t*)p, x);
39 #else
40 return atomic_xor(p, x);
41 #endif
42 }
43
44 SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_global(volatile __global int32_t *p,
45 int32_t cmp, int32_t val) {
46 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
47 return atomicCAS((int32_t*)p, cmp, val);
48 #else
49 return atomic_cmpxchg(p, cmp, val);
50 #endif
51 }
52
53 SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_shared(volatile __local int32_t *p,
54 int32_t cmp, int32_t val) {
55 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
56 return atomicCAS((int32_t*)p, cmp, val);
57 #else
58 return atomic_cmpxchg(p, cmp, val);
59 #endif
60 }
61
62 SCALAR_FUN_ATTR int32_t atomic_add_i32_global(volatile __global int32_t *p, int32_t x) {
63 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
64 return atomicAdd((int32_t*)p, x);
65 #else
66 return atomic_add(p, x);
67 #endif
68 }
69
70 SCALAR_FUN_ATTR int32_t atomic_add_i32_shared(volatile __local int32_t *p, int32_t x) {
71 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
72 return atomicAdd((int32_t*)p, x);
73 #else
74 return atomic_add(p, x);
75 #endif
76 }
77
78 SCALAR_FUN_ATTR float atomic_fadd_f32_global(volatile __global float *p, float x) {
79 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
80 return atomicAdd((float*)p, x);
81 // On OpenCL, use technique from
82 // https://pipinspace.github.io/blog/atomic-float-addition-in-opencl.html
83 #elif defined(cl_nv_pragma_unroll)
84 // use hardware-supported atomic addition on Nvidia GPUs with inline
85 // PTX assembly
86 float ret;
87 asm volatile("atom.global.add.f32 %0,[%1],%2;":"=f"(ret):"l"(p),"f"(x):"memory");
88 return ret;
89 #elif defined(__opencl_c_ext_fp32_global_atomic_add)
90 // use hardware-supported atomic addition on some Intel GPUs
91 return atomic_fetch_add_explicit((volatile __global atomic_float*)p,
92 x,
93 memory_order_relaxed);
94 #elif __has_builtin(__builtin_amdgcn_global_atomic_fadd_f32)
95 // use hardware-supported atomic addition on some AMD GPUs
96 return __builtin_amdgcn_global_atomic_fadd_f32(p, x);
97 #else
98 // fallback emulation:
99 // https://forums.developer.nvidia.com/t/atomicadd-float-float-atomicmul-float-float/14639/5
100 float old = x;
101 float ret;
102 while ((old=atomic_xchg(p, ret=atomic_xchg(p, 0.0f)+old))!=0.0f);
103 return ret;
104 #endif
105 }
106
107 SCALAR_FUN_ATTR float atomic_fadd_f32_shared(volatile __local float *p, float x) {
108 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
109 return atomicAdd((float*)p, x);
110 #else
111 union { int32_t i; float f; } old;
112 union { int32_t i; float f; } assumed;
113 old.f = *p;
114 do {
115 assumed.f = old.f;
116 old.f = old.f + x;
117 old.i = atomic_cmpxchg_i32_shared((volatile __local int32_t*)p, assumed.i, old.i);
118 } while (assumed.i != old.i);
119 return old.f;
120 #endif
121 }
122
123 SCALAR_FUN_ATTR int32_t atomic_smax_i32_global(volatile __global int32_t *p, int32_t x) {
124 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
125 return atomicMax((int32_t*)p, x);
126 #else
127 return atomic_max(p, x);
128 #endif
129 }
130
131 SCALAR_FUN_ATTR int32_t atomic_smax_i32_shared(volatile __local int32_t *p, int32_t x) {
132 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
133 return atomicMax((int32_t*)p, x);
134 #else
135 return atomic_max(p, x);
136 #endif
137 }
138
139 SCALAR_FUN_ATTR int32_t atomic_smin_i32_global(volatile __global int32_t *p, int32_t x) {
140 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
141 return atomicMin((int32_t*)p, x);
142 #else
143 return atomic_min(p, x);
144 #endif
145 }
146
147 SCALAR_FUN_ATTR int32_t atomic_smin_i32_shared(volatile __local int32_t *p, int32_t x) {
148 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
149 return atomicMin((int32_t*)p, x);
150 #else
151 return atomic_min(p, x);
152 #endif
153 }
154
155 SCALAR_FUN_ATTR uint32_t atomic_umax_i32_global(volatile __global uint32_t *p, uint32_t x) {
156 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
157 return atomicMax((uint32_t*)p, x);
158 #else
159 return atomic_max(p, x);
160 #endif
161 }
162
163 SCALAR_FUN_ATTR uint32_t atomic_umax_i32_shared(volatile __local uint32_t *p, uint32_t x) {
164 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
165 return atomicMax((uint32_t*)p, x);
166 #else
167 return atomic_max(p, x);
168 #endif
169 }
170
171 SCALAR_FUN_ATTR uint32_t atomic_umin_i32_global(volatile __global uint32_t *p, uint32_t x) {
172 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
173 return atomicMin((uint32_t*)p, x);
174 #else
175 return atomic_min(p, x);
176 #endif
177 }
178
179 SCALAR_FUN_ATTR uint32_t atomic_umin_i32_shared(volatile __local uint32_t *p, uint32_t x) {
180 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
181 return atomicMin((uint32_t*)p, x);
182 #else
183 return atomic_min(p, x);
184 #endif
185 }
186
187 SCALAR_FUN_ATTR int32_t atomic_and_i32_global(volatile __global int32_t *p, int32_t x) {
188 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
189 return atomicAnd((int32_t*)p, x);
190 #else
191 return atomic_and(p, x);
192 #endif
193 }
194
195 SCALAR_FUN_ATTR int32_t atomic_and_i32_shared(volatile __local int32_t *p, int32_t x) {
196 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
197 return atomicAnd((int32_t*)p, x);
198 #else
199 return atomic_and(p, x);
200 #endif
201 }
202
203 SCALAR_FUN_ATTR int32_t atomic_or_i32_global(volatile __global int32_t *p, int32_t x) {
204 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
205 return atomicOr((int32_t*)p, x);
206 #else
207 return atomic_or(p, x);
208 #endif
209 }
210
211 SCALAR_FUN_ATTR int32_t atomic_or_i32_shared(volatile __local int32_t *p, int32_t x) {
212 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
213 return atomicOr((int32_t*)p, x);
214 #else
215 return atomic_or(p, x);
216 #endif
217 }
218
219 SCALAR_FUN_ATTR int32_t atomic_xor_i32_global(volatile __global int32_t *p, int32_t x) {
220 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
221 return atomicXor((int32_t*)p, x);
222 #else
223 return atomic_xor(p, x);
224 #endif
225 }
226
227 SCALAR_FUN_ATTR int32_t atomic_xor_i32_shared(volatile __local int32_t *p, int32_t x) {
228 #if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)
229 return atomicXor((int32_t*)p, x);
230 #else
231 return atomic_xor(p, x);
232 #endif
233 }
234
235 // End of atomics.h