Debian Codegraph @

rts/c/ispc_util.h

raw
1 // Start of ispc_util.h.
2
3 // This header file implements various operations that are useful only when
4 // generating ISPC code. This includes wrappers for parts of Futhark's C runtime.
5
6 // Expose gang size
7 export uniform int64_t get_gang_size() {
8 return programCount;
9 }
10
11 // Generate missing overloads for extract on pointers
12 #define make_extract(ty) \
13 static inline uniform ty * uniform extract(uniform ty * varying ptr, uniform int idx) { \
14 int64 c = (int64)ptr; \
15 uniform int64 r = extract(c, idx); \
16 return (uniform ty * uniform)r; \
17 }
18
19 make_extract(int8)
20 make_extract(int16)
21 make_extract(int32)
22 make_extract(int64)
23 make_extract(uint8)
24 make_extract(uint16)
25 make_extract(uint32)
26 make_extract(uint64)
27 make_extract(float16)
28 make_extract(float)
29 make_extract(double)
30 /* make_extract(int8* uniform) */
31 /* make_extract(int16* uniform) */
32 /* make_extract(int32* uniform) */
33 /* make_extract(int64* uniform) */
34 /* make_extract(uint8* uniform) */
35 /* make_extract(uint16* uniform) */
36 /* make_extract(uint32* uniform) */
37 /* make_extract(uint64* uniform) */
38 /* make_extract(float16* uniform) */
39 /* make_extract(float* uniform) */
40 /* make_extract(double* uniform) */
41 make_extract(struct futhark_context)
42 make_extract(struct memblock)
43
44
45 // Handling of atomics
46 // Atomic CAS acts differently in GCC and ISPC, so we emulate it.
47 #define make_atomic_compare_exchange_wrapper(ty) \
48 static inline uniform bool atomic_compare_exchange_wrapper(uniform ty * uniform mem, \
49 uniform ty * uniform old, \
50 const uniform ty val){ \
51 uniform ty actual = atomic_compare_exchange_global(mem, *old, val); \
52 if (actual == *old){ \
53 return 1; \
54 } \
55 *old = actual; \
56 return 0; \
57 } \
58 static inline varying bool atomic_compare_exchange_wrapper(uniform ty * varying mem, \
59 varying ty * uniform old, \
60 const varying ty val){ \
61 varying ty actual = atomic_compare_exchange_global(mem, *old, val); \
62 bool res = 0; \
63 if(actual == *old){ \
64 res = 1; \
65 } else { \
66 *old = actual; \
67 } \
68 return res; \
69 } \
70 static inline varying bool atomic_compare_exchange_wrapper(varying ty * uniform mem, \
71 varying ty * uniform old, \
72 const varying ty val){ \
73 uniform ty * uniform base_mem = (uniform ty * uniform)mem; \
74 uniform ty * uniform base_old = (uniform ty * uniform)old; \
75 bool res = 0; \
76 foreach_active (i) { \
77 uniform ty * uniform curr_mem = base_mem + i; \
78 uniform ty * uniform curr_old = base_old + i; \
79 uniform ty curr_val = extract(val, i); \
80 uniform bool curr = atomic_compare_exchange_wrapper( \
81 curr_mem, curr_old, curr_val); \
82 res = insert(res, i, curr); \
83 } \
84 return res; \
85 } \
86 static inline uniform bool atomic_compare_exchange_wrapper(uniform ty * uniform mem, \
87 uniform ty * uniform old, \
88 const varying ty val){ \
89 uniform ty v = 0; \
90 foreach_active (i) v = extract(val, i); \
91 return atomic_compare_exchange_wrapper(mem, old, v); \
92 }
93
94 make_atomic_compare_exchange_wrapper(int32)
95 make_atomic_compare_exchange_wrapper(int64)
96 make_atomic_compare_exchange_wrapper(uint32)
97 make_atomic_compare_exchange_wrapper(uint64)
98 make_atomic_compare_exchange_wrapper(float)
99 make_atomic_compare_exchange_wrapper(double)
100
101 // This code generates missing overloads for atomic operations on uniform
102 // pointers to varying values.
103 #define make_single_atomic(name, ty) \
104 static inline ty atomic_##name##_global(varying ty * uniform mem, ty val) { \
105 uniform ty * uniform base_mem = (uniform ty * uniform)mem; \
106 ty res = 0; \
107 foreach_active (i) { \
108 uniform ty * uniform curr_mem = base_mem + i; \
109 uniform ty curr_val = extract(val, i); \
110 uniform ty curr = atomic_##name##_global(curr_mem, curr_val); \
111 res = insert(res, i, curr); \
112 } \
113 return res; \
114 }
115
116 #define make_all_atomic(name) \
117 make_single_atomic(name, int32) \
118 make_single_atomic(name, int64) \
119 make_single_atomic(name, uint32) \
120 make_single_atomic(name, uint64)
121
122 make_all_atomic(add)
123 make_all_atomic(subtract)
124 make_all_atomic(and)
125 make_all_atomic(or)
126 make_all_atomic(xor)
127 make_all_atomic(swap)
128
129 // This is a hack to prevent literals (which have unbound variability)
130 // from causing us to pick the wrong overload for atomic operations.
131 static inline varying int32 make_varying(uniform int32 x) { return x; }
132 static inline varying int32 make_varying(varying int32 x) { return x; }
133 static inline varying int64 make_varying(uniform int64 x) { return x; }
134 static inline varying int64 make_varying(varying int64 x) { return x; }
135 static inline varying uint32 make_varying(uniform uint32 x) { return x; }
136 static inline varying uint32 make_varying(varying uint32 x) { return x; }
137 static inline varying uint64 make_varying(uniform uint64 x) { return x; }
138 static inline varying uint64 make_varying(varying uint64 x) { return x; }
139
140 // Redirect atomic operations to the relevant ISPC overloads.
141 #define __atomic_fetch_add(x,y,z) atomic_add_global(x,make_varying(y))
142 #define __atomic_fetch_sub(x,y,z) atomic_sub_global(x,make_varying(y))
143 #define __atomic_fetch_and(x,y,z) atomic_and_global(x,make_varying(y))
144 #define __atomic_fetch_or(x,y,z) atomic_or_global(x,make_varying(y))
145 #define __atomic_fetch_xor(x,y,z) atomic_xor_global(x,make_varying(y))
146 #define __atomic_exchange_n(x,y,z) atomic_swap_global(x,make_varying(y))
147 #define __atomic_compare_exchange_n(x,y,z,h,j,k) atomic_compare_exchange_wrapper(x,y,z)
148
149
150 // Memory allocation handling
151 struct memblock {
152 int32_t * references;
153 uint8_t * mem;
154 int64_t size;
155 const int8_t * desc;
156 };
157
158 static inline void free(void* ptr) {
159 delete ptr;
160 }
161
162 static inline void free(void* uniform ptr) {
163 delete ptr;
164 }
165
166 extern "C" unmasked uniform unsigned char * uniform realloc(uniform unsigned char * uniform ptr, uniform int64_t new_size);
167 extern "C" unmasked uniform char * uniform lexical_realloc_error(uniform struct futhark_context * uniform ctx, uniform int64_t new_size);
168
169 static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx,
170 unsigned char uniform * uniform * uniform ptr,
171 int64_t uniform * uniform old_size,
172 uniform int64_t new_size) {
173 uniform unsigned char * uniform memptr = realloc(*ptr, new_size);
174 if (memptr == NULL) {
175 lexical_realloc_error(ctx, new_size);
176 return FUTHARK_OUT_OF_MEMORY;
177 } else {
178 *ptr = memptr;
179 *old_size = new_size;
180 return FUTHARK_SUCCESS;
181 }
182 }
183
184
185 static inline uniform int lexical_realloc(uniform struct futhark_context *ctx,
186 unsigned char uniform * uniform * uniform ptr,
187 int64_t uniform * uniform old_size,
188 varying int64_t new_size) {
189 return lexical_realloc(ctx, ptr, old_size, reduce_max(new_size));
190 }
191
192 static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx,
193 unsigned char uniform * varying * uniform ptr,
194 int64_t uniform * varying old_size,
195 varying int64_t new_size) {
196 uniform int err = FUTHARK_SUCCESS;
197 foreach_active(i){
198 uniform unsigned char * uniform memptr = realloc(extract(*ptr,i), extract(new_size,i));
199 if (memptr == NULL) {
200 lexical_realloc_error(ctx, extract(new_size,i));
201 err = FUTHARK_OUT_OF_MEMORY;
202 } else {
203 *ptr = (uniform unsigned char * varying)insert((int64_t)*ptr, i, (uniform int64_t) memptr);
204 *old_size = new_size;
205 }
206 }
207 return err;
208 }
209
210 static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx,
211 unsigned char uniform * varying * uniform ptr,
212 int64_t varying * uniform old_size,
213 varying int64_t new_size) {
214 uniform int err = FUTHARK_SUCCESS;
215 foreach_active(i){
216 uniform unsigned char * uniform memptr = realloc(extract(*ptr,i), extract(new_size,i));
217 if (memptr == NULL) {
218 lexical_realloc_error(ctx, extract(new_size,i));
219 err = FUTHARK_OUT_OF_MEMORY;
220 } else {
221 *ptr = (uniform unsigned char * varying)insert((int64_t)*ptr, i, (uniform int64_t) memptr);
222 *old_size = new_size;
223 }
224 }
225 return err;
226 }
227
228 static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx,
229 unsigned char uniform * varying * uniform ptr,
230 size_t varying * uniform old_size,
231 varying int64_t new_size) {
232 return lexical_realloc(ctx, ptr, (varying int64_t * uniform)old_size, new_size);
233 }
234
235 static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx,
236 unsigned char varying * uniform * uniform ptr,
237 size_t varying * uniform old_size,
238 uniform int64_t new_size) {
239 uniform int err = FUTHARK_SUCCESS;
240 uniform unsigned char * uniform memptr = realloc((uniform unsigned char * uniform )*ptr,
241 new_size*programCount);
242 if (memptr == NULL) {
243 lexical_realloc_error(ctx, new_size);
244 err = FUTHARK_OUT_OF_MEMORY;
245 } else {
246 *ptr = (varying unsigned char * uniform)memptr;
247 *old_size = new_size;
248 }
249
250 return err;
251 }
252
253 static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx,
254 unsigned char varying * uniform * uniform ptr,
255 size_t varying * uniform old_size,
256 varying int64_t new_size) {
257 return lexical_realloc(ctx, ptr, old_size, reduce_max(new_size));
258 }
259
260 extern "C" unmasked uniform int memblock_unref(uniform struct futhark_context * uniform ctx,
261 uniform struct memblock * uniform lhs,
262 uniform const char * uniform lhs_desc);
263
264 static uniform int memblock_unref(uniform struct futhark_context * varying ctx,
265 uniform struct memblock * varying lhs,
266 uniform const char * uniform lhs_desc)
267 {
268 uniform int err = 0;
269
270 foreach_active(i) {
271 err |= memblock_unref(extract(ctx,i), extract(lhs,i), lhs_desc);
272 }
273
274 return err;
275 }
276 static uniform int memblock_unref(uniform struct futhark_context * uniform ctx,
277 varying struct memblock * uniform lhs,
278 uniform const char * uniform lhs_desc)
279 {
280 uniform int err = 0;
281
282 varying struct memblock _lhs = *lhs;
283 uniform struct memblock aos[programCount];
284 aos[programIndex] = _lhs;
285
286 foreach_active(i){
287 err |= memblock_unref(ctx,
288 &aos[i],
289 lhs_desc);
290 }
291
292 *lhs = aos[programIndex];
293
294 return err;
295 }
296
297 extern "C" unmasked uniform int memblock_alloc(uniform struct futhark_context * uniform ctx,
298 uniform struct memblock * uniform block,
299 uniform int64_t size,
300 uniform const char * uniform block_desc);
301
302 static uniform int memblock_alloc(uniform struct futhark_context * varying ctx,
303 uniform struct memblock * varying block,
304 varying int64_t size,
305 uniform const char * uniform block_desc) {
306 uniform int err = 0;
307
308 foreach_active(i){
309 err |= memblock_alloc(extract(ctx,i), extract(block,i), extract(size, i), block_desc);
310 }
311
312 return err;
313 }
314
315 static uniform int memblock_alloc(uniform struct futhark_context * uniform ctx,
316 varying struct memblock * uniform block,
317 uniform int64_t size,
318 uniform const char * uniform block_desc) {
319 uniform int err = 0;
320
321 varying struct memblock _block = *block;
322 uniform struct memblock aos[programCount];
323 aos[programIndex] = _block;
324
325 foreach_active(i){
326 err |= memblock_alloc(ctx, &aos[i], size, block_desc);
327 }
328 *block = aos[programIndex];
329
330 return err;
331 }
332
333 static uniform int memblock_alloc(uniform struct futhark_context * uniform ctx,
334 varying struct memblock * uniform block,
335 varying int64_t size,
336 uniform const char * uniform block_desc) {
337 uniform int err = 0;
338
339 varying struct memblock _block = *block;
340 uniform struct memblock aos[programCount];
341 aos[programIndex] = _block;
342 foreach_active(i){
343 err |= memblock_alloc(ctx, &aos[i], extract(size, i), block_desc);
344 }
345 *block = aos[programIndex];
346
347 return err;
348 }
349
350 extern "C" unmasked uniform int memblock_set(uniform struct futhark_context * uniform ctx,
351 uniform struct memblock * uniform lhs,
352 uniform struct memblock * uniform rhs,
353 uniform const char * uniform lhs_desc);
354
355 static uniform int memblock_set (uniform struct futhark_context * uniform ctx,
356 varying struct memblock * uniform lhs,
357 varying struct memblock * uniform rhs,
358 uniform const char * uniform lhs_desc) {
359 uniform int err = 0;
360
361 varying struct memblock _lhs = *lhs;
362 varying struct memblock _rhs = *rhs;
363 uniform struct memblock aos1[programCount];
364 aos1[programIndex] = _lhs;
365
366 uniform struct memblock aos2[programCount];
367 aos2[programIndex] = _rhs;
368
369 foreach_active(i) {
370 err |= memblock_set(ctx,
371 &aos1[i],
372 &aos2[i],
373 lhs_desc);
374 }
375 *lhs = aos1[programIndex];
376 *rhs = aos2[programIndex];
377
378 return err;
379 }
380
381 static uniform int memblock_set (uniform struct futhark_context * uniform ctx,
382 varying struct memblock * uniform lhs,
383 uniform struct memblock * uniform rhs,
384 uniform const char * uniform lhs_desc) {
385 uniform int err = 0;
386
387 varying struct memblock _lhs = *lhs;
388 uniform struct memblock aos1[programCount];
389 aos1[programIndex] = _lhs;
390
391 foreach_active(i) {
392 err |= memblock_set(ctx,
393 &aos1[i],
394 rhs,
395 lhs_desc);
396 }
397 *lhs = aos1[programIndex];
398
399 return err;
400 }
401
402 // End of ispc_util.h.