| 1 |
// Start of ispc_util.h. |
| 2 |
|
| 3 |
// This header file implements various operations that are useful only when |
| 4 |
// generating ISPC code. This includes wrappers for parts of Futhark's C runtime. |
| 5 |
|
| 6 |
// Expose gang size |
| 7 |
export uniform int64_t get_gang_size() { |
| 8 |
return programCount; |
| 9 |
} |
| 10 |
|
| 11 |
// Generate missing overloads for extract on pointers |
| 12 |
#define make_extract(ty) \ |
| 13 |
static inline uniform ty * uniform extract(uniform ty * varying ptr, uniform int idx) { \ |
| 14 |
int64 c = (int64)ptr; \ |
| 15 |
uniform int64 r = extract(c, idx); \ |
| 16 |
return (uniform ty * uniform)r; \ |
| 17 |
} |
| 18 |
|
| 19 |
make_extract(int8) |
| 20 |
make_extract(int16) |
| 21 |
make_extract(int32) |
| 22 |
make_extract(int64) |
| 23 |
make_extract(uint8) |
| 24 |
make_extract(uint16) |
| 25 |
make_extract(uint32) |
| 26 |
make_extract(uint64) |
| 27 |
make_extract(float16) |
| 28 |
make_extract(float) |
| 29 |
make_extract(double) |
| 30 |
/* make_extract(int8* uniform) */ |
| 31 |
/* make_extract(int16* uniform) */ |
| 32 |
/* make_extract(int32* uniform) */ |
| 33 |
/* make_extract(int64* uniform) */ |
| 34 |
/* make_extract(uint8* uniform) */ |
| 35 |
/* make_extract(uint16* uniform) */ |
| 36 |
/* make_extract(uint32* uniform) */ |
| 37 |
/* make_extract(uint64* uniform) */ |
| 38 |
/* make_extract(float16* uniform) */ |
| 39 |
/* make_extract(float* uniform) */ |
| 40 |
/* make_extract(double* uniform) */ |
| 41 |
make_extract(struct futhark_context) |
| 42 |
make_extract(struct memblock) |
| 43 |
|
| 44 |
|
| 45 |
// Handling of atomics |
| 46 |
// Atomic CAS acts differently in GCC and ISPC, so we emulate it. |
| 47 |
#define make_atomic_compare_exchange_wrapper(ty) \ |
| 48 |
static inline uniform bool atomic_compare_exchange_wrapper(uniform ty * uniform mem, \ |
| 49 |
uniform ty * uniform old, \ |
| 50 |
const uniform ty val){ \ |
| 51 |
uniform ty actual = atomic_compare_exchange_global(mem, *old, val); \ |
| 52 |
if (actual == *old){ \ |
| 53 |
return 1; \ |
| 54 |
} \ |
| 55 |
*old = actual; \ |
| 56 |
return 0; \ |
| 57 |
} \ |
| 58 |
static inline varying bool atomic_compare_exchange_wrapper(uniform ty * varying mem, \ |
| 59 |
varying ty * uniform old, \ |
| 60 |
const varying ty val){ \ |
| 61 |
varying ty actual = atomic_compare_exchange_global(mem, *old, val); \ |
| 62 |
bool res = 0; \ |
| 63 |
if(actual == *old){ \ |
| 64 |
res = 1; \ |
| 65 |
} else { \ |
| 66 |
*old = actual; \ |
| 67 |
} \ |
| 68 |
return res; \ |
| 69 |
} \ |
| 70 |
static inline varying bool atomic_compare_exchange_wrapper(varying ty * uniform mem, \ |
| 71 |
varying ty * uniform old, \ |
| 72 |
const varying ty val){ \ |
| 73 |
uniform ty * uniform base_mem = (uniform ty * uniform)mem; \ |
| 74 |
uniform ty * uniform base_old = (uniform ty * uniform)old; \ |
| 75 |
bool res = 0; \ |
| 76 |
foreach_active (i) { \ |
| 77 |
uniform ty * uniform curr_mem = base_mem + i; \ |
| 78 |
uniform ty * uniform curr_old = base_old + i; \ |
| 79 |
uniform ty curr_val = extract(val, i); \ |
| 80 |
uniform bool curr = atomic_compare_exchange_wrapper( \ |
| 81 |
curr_mem, curr_old, curr_val); \ |
| 82 |
res = insert(res, i, curr); \ |
| 83 |
} \ |
| 84 |
return res; \ |
| 85 |
} \ |
| 86 |
static inline uniform bool atomic_compare_exchange_wrapper(uniform ty * uniform mem, \ |
| 87 |
uniform ty * uniform old, \ |
| 88 |
const varying ty val){ \ |
| 89 |
uniform ty v = 0; \ |
| 90 |
foreach_active (i) v = extract(val, i); \ |
| 91 |
return atomic_compare_exchange_wrapper(mem, old, v); \ |
| 92 |
} |
| 93 |
|
| 94 |
make_atomic_compare_exchange_wrapper(int32) |
| 95 |
make_atomic_compare_exchange_wrapper(int64) |
| 96 |
make_atomic_compare_exchange_wrapper(uint32) |
| 97 |
make_atomic_compare_exchange_wrapper(uint64) |
| 98 |
make_atomic_compare_exchange_wrapper(float) |
| 99 |
make_atomic_compare_exchange_wrapper(double) |
| 100 |
|
| 101 |
// This code generates missing overloads for atomic operations on uniform |
| 102 |
// pointers to varying values. |
| 103 |
#define make_single_atomic(name, ty) \ |
| 104 |
static inline ty atomic_##name##_global(varying ty * uniform mem, ty val) { \ |
| 105 |
uniform ty * uniform base_mem = (uniform ty * uniform)mem; \ |
| 106 |
ty res = 0; \ |
| 107 |
foreach_active (i) { \ |
| 108 |
uniform ty * uniform curr_mem = base_mem + i; \ |
| 109 |
uniform ty curr_val = extract(val, i); \ |
| 110 |
uniform ty curr = atomic_##name##_global(curr_mem, curr_val); \ |
| 111 |
res = insert(res, i, curr); \ |
| 112 |
} \ |
| 113 |
return res; \ |
| 114 |
} |
| 115 |
|
| 116 |
#define make_all_atomic(name) \ |
| 117 |
make_single_atomic(name, int32) \ |
| 118 |
make_single_atomic(name, int64) \ |
| 119 |
make_single_atomic(name, uint32) \ |
| 120 |
make_single_atomic(name, uint64) |
| 121 |
|
| 122 |
make_all_atomic(add) |
| 123 |
make_all_atomic(subtract) |
| 124 |
make_all_atomic(and) |
| 125 |
make_all_atomic(or) |
| 126 |
make_all_atomic(xor) |
| 127 |
make_all_atomic(swap) |
| 128 |
|
| 129 |
// This is a hack to prevent literals (which have unbound variability) |
| 130 |
// from causing us to pick the wrong overload for atomic operations. |
| 131 |
static inline varying int32 make_varying(uniform int32 x) { return x; } |
| 132 |
static inline varying int32 make_varying(varying int32 x) { return x; } |
| 133 |
static inline varying int64 make_varying(uniform int64 x) { return x; } |
| 134 |
static inline varying int64 make_varying(varying int64 x) { return x; } |
| 135 |
static inline varying uint32 make_varying(uniform uint32 x) { return x; } |
| 136 |
static inline varying uint32 make_varying(varying uint32 x) { return x; } |
| 137 |
static inline varying uint64 make_varying(uniform uint64 x) { return x; } |
| 138 |
static inline varying uint64 make_varying(varying uint64 x) { return x; } |
| 139 |
|
| 140 |
// Redirect atomic operations to the relevant ISPC overloads. |
| 141 |
#define __atomic_fetch_add(x,y,z) atomic_add_global(x,make_varying(y)) |
| 142 |
#define __atomic_fetch_sub(x,y,z) atomic_sub_global(x,make_varying(y)) |
| 143 |
#define __atomic_fetch_and(x,y,z) atomic_and_global(x,make_varying(y)) |
| 144 |
#define __atomic_fetch_or(x,y,z) atomic_or_global(x,make_varying(y)) |
| 145 |
#define __atomic_fetch_xor(x,y,z) atomic_xor_global(x,make_varying(y)) |
| 146 |
#define __atomic_exchange_n(x,y,z) atomic_swap_global(x,make_varying(y)) |
| 147 |
#define __atomic_compare_exchange_n(x,y,z,h,j,k) atomic_compare_exchange_wrapper(x,y,z) |
| 148 |
|
| 149 |
|
| 150 |
// Memory allocation handling |
| 151 |
struct memblock { |
| 152 |
int32_t * references; |
| 153 |
uint8_t * mem; |
| 154 |
int64_t size; |
| 155 |
const int8_t * desc; |
| 156 |
}; |
| 157 |
|
| 158 |
static inline void free(void* ptr) { |
| 159 |
delete ptr; |
| 160 |
} |
| 161 |
|
| 162 |
static inline void free(void* uniform ptr) { |
| 163 |
delete ptr; |
| 164 |
} |
| 165 |
|
| 166 |
extern "C" unmasked uniform unsigned char * uniform realloc(uniform unsigned char * uniform ptr, uniform int64_t new_size); |
| 167 |
extern "C" unmasked uniform char * uniform lexical_realloc_error(uniform struct futhark_context * uniform ctx, uniform int64_t new_size); |
| 168 |
|
| 169 |
static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx, |
| 170 |
unsigned char uniform * uniform * uniform ptr, |
| 171 |
int64_t uniform * uniform old_size, |
| 172 |
uniform int64_t new_size) { |
| 173 |
uniform unsigned char * uniform memptr = realloc(*ptr, new_size); |
| 174 |
if (memptr == NULL) { |
| 175 |
lexical_realloc_error(ctx, new_size); |
| 176 |
return FUTHARK_OUT_OF_MEMORY; |
| 177 |
} else { |
| 178 |
*ptr = memptr; |
| 179 |
*old_size = new_size; |
| 180 |
return FUTHARK_SUCCESS; |
| 181 |
} |
| 182 |
} |
| 183 |
|
| 184 |
|
| 185 |
static inline uniform int lexical_realloc(uniform struct futhark_context *ctx, |
| 186 |
unsigned char uniform * uniform * uniform ptr, |
| 187 |
int64_t uniform * uniform old_size, |
| 188 |
varying int64_t new_size) { |
| 189 |
return lexical_realloc(ctx, ptr, old_size, reduce_max(new_size)); |
| 190 |
} |
| 191 |
|
| 192 |
static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx, |
| 193 |
unsigned char uniform * varying * uniform ptr, |
| 194 |
int64_t uniform * varying old_size, |
| 195 |
varying int64_t new_size) { |
| 196 |
uniform int err = FUTHARK_SUCCESS; |
| 197 |
foreach_active(i){ |
| 198 |
uniform unsigned char * uniform memptr = realloc(extract(*ptr,i), extract(new_size,i)); |
| 199 |
if (memptr == NULL) { |
| 200 |
lexical_realloc_error(ctx, extract(new_size,i)); |
| 201 |
err = FUTHARK_OUT_OF_MEMORY; |
| 202 |
} else { |
| 203 |
*ptr = (uniform unsigned char * varying)insert((int64_t)*ptr, i, (uniform int64_t) memptr); |
| 204 |
*old_size = new_size; |
| 205 |
} |
| 206 |
} |
| 207 |
return err; |
| 208 |
} |
| 209 |
|
| 210 |
static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx, |
| 211 |
unsigned char uniform * varying * uniform ptr, |
| 212 |
int64_t varying * uniform old_size, |
| 213 |
varying int64_t new_size) { |
| 214 |
uniform int err = FUTHARK_SUCCESS; |
| 215 |
foreach_active(i){ |
| 216 |
uniform unsigned char * uniform memptr = realloc(extract(*ptr,i), extract(new_size,i)); |
| 217 |
if (memptr == NULL) { |
| 218 |
lexical_realloc_error(ctx, extract(new_size,i)); |
| 219 |
err = FUTHARK_OUT_OF_MEMORY; |
| 220 |
} else { |
| 221 |
*ptr = (uniform unsigned char * varying)insert((int64_t)*ptr, i, (uniform int64_t) memptr); |
| 222 |
*old_size = new_size; |
| 223 |
} |
| 224 |
} |
| 225 |
return err; |
| 226 |
} |
| 227 |
|
| 228 |
static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx, |
| 229 |
unsigned char uniform * varying * uniform ptr, |
| 230 |
size_t varying * uniform old_size, |
| 231 |
varying int64_t new_size) { |
| 232 |
return lexical_realloc(ctx, ptr, (varying int64_t * uniform)old_size, new_size); |
| 233 |
} |
| 234 |
|
| 235 |
static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx, |
| 236 |
unsigned char varying * uniform * uniform ptr, |
| 237 |
size_t varying * uniform old_size, |
| 238 |
uniform int64_t new_size) { |
| 239 |
uniform int err = FUTHARK_SUCCESS; |
| 240 |
uniform unsigned char * uniform memptr = realloc((uniform unsigned char * uniform )*ptr, |
| 241 |
new_size*programCount); |
| 242 |
if (memptr == NULL) { |
| 243 |
lexical_realloc_error(ctx, new_size); |
| 244 |
err = FUTHARK_OUT_OF_MEMORY; |
| 245 |
} else { |
| 246 |
*ptr = (varying unsigned char * uniform)memptr; |
| 247 |
*old_size = new_size; |
| 248 |
} |
| 249 |
|
| 250 |
return err; |
| 251 |
} |
| 252 |
|
| 253 |
static inline uniform int lexical_realloc(uniform struct futhark_context * uniform ctx, |
| 254 |
unsigned char varying * uniform * uniform ptr, |
| 255 |
size_t varying * uniform old_size, |
| 256 |
varying int64_t new_size) { |
| 257 |
return lexical_realloc(ctx, ptr, old_size, reduce_max(new_size)); |
| 258 |
} |
| 259 |
|
| 260 |
extern "C" unmasked uniform int memblock_unref(uniform struct futhark_context * uniform ctx, |
| 261 |
uniform struct memblock * uniform lhs, |
| 262 |
uniform const char * uniform lhs_desc); |
| 263 |
|
| 264 |
static uniform int memblock_unref(uniform struct futhark_context * varying ctx, |
| 265 |
uniform struct memblock * varying lhs, |
| 266 |
uniform const char * uniform lhs_desc) |
| 267 |
{ |
| 268 |
uniform int err = 0; |
| 269 |
|
| 270 |
foreach_active(i) { |
| 271 |
err |= memblock_unref(extract(ctx,i), extract(lhs,i), lhs_desc); |
| 272 |
} |
| 273 |
|
| 274 |
return err; |
| 275 |
} |
| 276 |
static uniform int memblock_unref(uniform struct futhark_context * uniform ctx, |
| 277 |
varying struct memblock * uniform lhs, |
| 278 |
uniform const char * uniform lhs_desc) |
| 279 |
{ |
| 280 |
uniform int err = 0; |
| 281 |
|
| 282 |
varying struct memblock _lhs = *lhs; |
| 283 |
uniform struct memblock aos[programCount]; |
| 284 |
aos[programIndex] = _lhs; |
| 285 |
|
| 286 |
foreach_active(i){ |
| 287 |
err |= memblock_unref(ctx, |
| 288 |
&aos[i], |
| 289 |
lhs_desc); |
| 290 |
} |
| 291 |
|
| 292 |
*lhs = aos[programIndex]; |
| 293 |
|
| 294 |
return err; |
| 295 |
} |
| 296 |
|
| 297 |
extern "C" unmasked uniform int memblock_alloc(uniform struct futhark_context * uniform ctx, |
| 298 |
uniform struct memblock * uniform block, |
| 299 |
uniform int64_t size, |
| 300 |
uniform const char * uniform block_desc); |
| 301 |
|
| 302 |
static uniform int memblock_alloc(uniform struct futhark_context * varying ctx, |
| 303 |
uniform struct memblock * varying block, |
| 304 |
varying int64_t size, |
| 305 |
uniform const char * uniform block_desc) { |
| 306 |
uniform int err = 0; |
| 307 |
|
| 308 |
foreach_active(i){ |
| 309 |
err |= memblock_alloc(extract(ctx,i), extract(block,i), extract(size, i), block_desc); |
| 310 |
} |
| 311 |
|
| 312 |
return err; |
| 313 |
} |
| 314 |
|
| 315 |
static uniform int memblock_alloc(uniform struct futhark_context * uniform ctx, |
| 316 |
varying struct memblock * uniform block, |
| 317 |
uniform int64_t size, |
| 318 |
uniform const char * uniform block_desc) { |
| 319 |
uniform int err = 0; |
| 320 |
|
| 321 |
varying struct memblock _block = *block; |
| 322 |
uniform struct memblock aos[programCount]; |
| 323 |
aos[programIndex] = _block; |
| 324 |
|
| 325 |
foreach_active(i){ |
| 326 |
err |= memblock_alloc(ctx, &aos[i], size, block_desc); |
| 327 |
} |
| 328 |
*block = aos[programIndex]; |
| 329 |
|
| 330 |
return err; |
| 331 |
} |
| 332 |
|
| 333 |
static uniform int memblock_alloc(uniform struct futhark_context * uniform ctx, |
| 334 |
varying struct memblock * uniform block, |
| 335 |
varying int64_t size, |
| 336 |
uniform const char * uniform block_desc) { |
| 337 |
uniform int err = 0; |
| 338 |
|
| 339 |
varying struct memblock _block = *block; |
| 340 |
uniform struct memblock aos[programCount]; |
| 341 |
aos[programIndex] = _block; |
| 342 |
foreach_active(i){ |
| 343 |
err |= memblock_alloc(ctx, &aos[i], extract(size, i), block_desc); |
| 344 |
} |
| 345 |
*block = aos[programIndex]; |
| 346 |
|
| 347 |
return err; |
| 348 |
} |
| 349 |
|
| 350 |
extern "C" unmasked uniform int memblock_set(uniform struct futhark_context * uniform ctx, |
| 351 |
uniform struct memblock * uniform lhs, |
| 352 |
uniform struct memblock * uniform rhs, |
| 353 |
uniform const char * uniform lhs_desc); |
| 354 |
|
| 355 |
static uniform int memblock_set (uniform struct futhark_context * uniform ctx, |
| 356 |
varying struct memblock * uniform lhs, |
| 357 |
varying struct memblock * uniform rhs, |
| 358 |
uniform const char * uniform lhs_desc) { |
| 359 |
uniform int err = 0; |
| 360 |
|
| 361 |
varying struct memblock _lhs = *lhs; |
| 362 |
varying struct memblock _rhs = *rhs; |
| 363 |
uniform struct memblock aos1[programCount]; |
| 364 |
aos1[programIndex] = _lhs; |
| 365 |
|
| 366 |
uniform struct memblock aos2[programCount]; |
| 367 |
aos2[programIndex] = _rhs; |
| 368 |
|
| 369 |
foreach_active(i) { |
| 370 |
err |= memblock_set(ctx, |
| 371 |
&aos1[i], |
| 372 |
&aos2[i], |
| 373 |
lhs_desc); |
| 374 |
} |
| 375 |
*lhs = aos1[programIndex]; |
| 376 |
*rhs = aos2[programIndex]; |
| 377 |
|
| 378 |
return err; |
| 379 |
} |
| 380 |
|
| 381 |
static uniform int memblock_set (uniform struct futhark_context * uniform ctx, |
| 382 |
varying struct memblock * uniform lhs, |
| 383 |
uniform struct memblock * uniform rhs, |
| 384 |
uniform const char * uniform lhs_desc) { |
| 385 |
uniform int err = 0; |
| 386 |
|
| 387 |
varying struct memblock _lhs = *lhs; |
| 388 |
uniform struct memblock aos1[programCount]; |
| 389 |
aos1[programIndex] = _lhs; |
| 390 |
|
| 391 |
foreach_active(i) { |
| 392 |
err |= memblock_set(ctx, |
| 393 |
&aos1[i], |
| 394 |
rhs, |
| 395 |
lhs_desc); |
| 396 |
} |
| 397 |
*lhs = aos1[programIndex]; |
| 398 |
|
| 399 |
return err; |
| 400 |
} |
| 401 |
|
| 402 |
// End of ispc_util.h. |