Debian Codegraph @

rts/c/gpu.h

raw
1 // Start of gpu.h
2
3 // Generic functions that use our tiny GPU abstraction layer. The
4 // entire context must be defined before this header is included. In
5 // particular we expect the following functions to be available:
6
7 static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem);
8 static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out);
9 int gpu_launch_kernel(struct futhark_context* ctx,
10 gpu_kernel kernel,
11 const char *name, const char *provenance,
12 const int32_t grid[3],
13 const int32_t block[3],
14 unsigned int shared_mem_bytes,
15 int num_args,
16 void* args[num_args],
17 size_t args_sizes[num_args]);
18 int gpu_memcpy(struct futhark_context* ctx, const char* provenance,
19 gpu_mem dst, int64_t dst_offset,
20 gpu_mem src, int64_t src_offset,
21 int64_t nbytes);
22 int gpu_scalar_from_device(struct futhark_context* ctx, const char* provenance,
23 void *dst,
24 gpu_mem src, size_t offset, size_t size);
25 int gpu_scalar_to_device(struct futhark_context* ctx, const char* provenance,
26 gpu_mem dst, size_t offset, size_t size,
27 void *src);
28 void gpu_create_kernel(struct futhark_context *ctx,
29 gpu_kernel* kernel,
30 const char* name);
31
32 static void gpu_init_log(struct futhark_context *ctx) {
33 if (ctx->cfg->logging) {
34 fprintf(ctx->log, "Default block size: %ld\n", (long)ctx->cfg->gpu.default_block_size);
35 fprintf(ctx->log, "Default grid size: %ld\n", (long)ctx->cfg->gpu.default_grid_size);
36 fprintf(ctx->log, "Default tile size: %ld\n", (long)ctx->cfg->gpu.default_tile_size);
37 fprintf(ctx->log, "Default register tile size: %ld\n", (long)ctx->cfg->gpu.default_reg_tile_size);
38 fprintf(ctx->log, "Default cache: %ld\n", (long)ctx->cfg->gpu.default_cache);
39 fprintf(ctx->log, "Default registers: %ld\n", (long)ctx->cfg->gpu.default_registers);
40 fprintf(ctx->log, "Default threshold: %ld\n", (long)ctx->cfg->gpu.default_threshold);
41 fprintf(ctx->log, "Max thread block size: %ld\n", (long)ctx->max_thread_block_size);
42 fprintf(ctx->log, "Max grid size: %ld\n", (long)ctx->max_grid_size);
43 fprintf(ctx->log, "Max tile size: %ld\n", (long)ctx->max_tile_size);
44 fprintf(ctx->log, "Max threshold: %ld\n", (long)ctx->max_threshold);
45 fprintf(ctx->log, "Max shared memory: %ld\n", (long)ctx->max_shared_memory);
46 fprintf(ctx->log, "Max registers: %ld\n", (long)ctx->max_registers);
47 fprintf(ctx->log, "Max cache: %ld\n", (long)ctx->max_cache);
48 fprintf(ctx->log, "Lockstep width: %ld\n", (long)ctx->lockstep_width);
49 }
50 }
51
52 // Generic GPU command line options.
53
54 void futhark_context_config_set_default_thread_block_size(struct futhark_context_config *cfg, int size) {
55 cfg->gpu.default_block_size = size;
56 cfg->gpu.default_block_size_changed = 1;
57 }
58
59 void futhark_context_config_set_default_group_size(struct futhark_context_config *cfg, int size) {
60 futhark_context_config_set_default_thread_block_size(cfg, size);
61 }
62
63 void futhark_context_config_set_default_grid_size(struct futhark_context_config *cfg, int num) {
64 cfg->gpu.default_grid_size = num;
65 cfg->gpu.default_grid_size_changed = 1;
66 }
67
68 void futhark_context_config_set_default_num_groups(struct futhark_context_config *cfg, int num) {
69 futhark_context_config_set_default_grid_size(cfg, num);
70 }
71
72 void futhark_context_config_set_default_tile_size(struct futhark_context_config *cfg, int size) {
73 cfg->gpu.default_tile_size = size;
74 cfg->gpu.default_tile_size_changed = 1;
75 }
76
77 void futhark_context_config_set_default_reg_tile_size(struct futhark_context_config *cfg, int size) {
78 cfg->gpu.default_reg_tile_size = size;
79 }
80
81 void futhark_context_config_set_default_cache(struct futhark_context_config *cfg, int size) {
82 cfg->gpu.default_cache = size;
83 }
84
85 void futhark_context_config_set_default_registers(struct futhark_context_config *cfg, int size) {
86 cfg->gpu.default_registers = size;
87 }
88
89 void futhark_context_config_set_default_threshold(struct futhark_context_config *cfg, int size) {
90 cfg->gpu.default_threshold = size;
91 }
92
93 int futhark_context_config_set_tuning_param(struct futhark_context_config *cfg,
94 const char *param_name,
95 size_t new_value) {
96 for (int i = 0; i < cfg->num_tuning_params; i++) {
97 if (strcmp(param_name, cfg->tuning_param_names[i]) == 0) {
98 cfg->tuning_params[i] = new_value;
99 return 0;
100 }
101 }
102 if (strcmp(param_name, "default_thread_block_size") == 0 ||
103 strcmp(param_name, "default_group_size") == 0) {
104 cfg->gpu.default_block_size = new_value;
105 return 0;
106 }
107 if (strcmp(param_name, "default_grid_size") == 0 ||
108 strcmp(param_name, "default_num_groups") == 0) {
109 cfg->gpu.default_grid_size = new_value;
110 return 0;
111 }
112 if (strcmp(param_name, "default_threshold") == 0) {
113 cfg->gpu.default_threshold = new_value;
114 return 0;
115 }
116 if (strcmp(param_name, "default_tile_size") == 0) {
117 cfg->gpu.default_tile_size = new_value;
118 return 0;
119 }
120 if (strcmp(param_name, "default_reg_tile_size") == 0) {
121 cfg->gpu.default_reg_tile_size = new_value;
122 return 0;
123 }
124 if (strcmp(param_name, "default_cache") == 0) {
125 cfg->gpu.default_cache = new_value;
126 return 0;
127 }
128 if (strcmp(param_name, "default_shared_memory") == 0) {
129 cfg->gpu.default_shared_memory = new_value;
130 return 0;
131 }
132
133 return 1;
134 }
135
136 // End of GPU command line optiopns.
137
138 // Max number of thead blocks we allow along the second or third
139 // dimension for transpositions.
140 #define MAX_TR_THREAD_BLOCKS 65535
141
142 struct builtin_kernels {
143 // We have a lot of ways to transpose arrays.
144 gpu_kernel map_transpose_1b;
145 gpu_kernel map_transpose_1b_low_height;
146 gpu_kernel map_transpose_1b_low_width;
147 gpu_kernel map_transpose_1b_small;
148 gpu_kernel map_transpose_1b_large;
149 gpu_kernel map_transpose_2b;
150 gpu_kernel map_transpose_2b_low_height;
151 gpu_kernel map_transpose_2b_low_width;
152 gpu_kernel map_transpose_2b_small;
153 gpu_kernel map_transpose_2b_large;
154 gpu_kernel map_transpose_4b;
155 gpu_kernel map_transpose_4b_low_height;
156 gpu_kernel map_transpose_4b_low_width;
157 gpu_kernel map_transpose_4b_small;
158 gpu_kernel map_transpose_4b_large;
159 gpu_kernel map_transpose_8b;
160 gpu_kernel map_transpose_8b_low_height;
161 gpu_kernel map_transpose_8b_low_width;
162 gpu_kernel map_transpose_8b_small;
163 gpu_kernel map_transpose_8b_large;
164
165 // And a few ways of copying.
166 gpu_kernel lmad_copy_1b;
167 gpu_kernel lmad_copy_2b;
168 gpu_kernel lmad_copy_4b;
169 gpu_kernel lmad_copy_8b;
170 };
171
172 struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx) {
173 struct builtin_kernels *kernels = malloc(sizeof(struct builtin_kernels));
174 gpu_create_kernel(ctx, &kernels->map_transpose_1b, "map_transpose_1b");
175 gpu_create_kernel(ctx, &kernels->map_transpose_1b_large, "map_transpose_1b_large");
176 gpu_create_kernel(ctx, &kernels->map_transpose_1b_low_height, "map_transpose_1b_low_height");
177 gpu_create_kernel(ctx, &kernels->map_transpose_1b_low_width, "map_transpose_1b_low_width");
178 gpu_create_kernel(ctx, &kernels->map_transpose_1b_small, "map_transpose_1b_small");
179
180 gpu_create_kernel(ctx, &kernels->map_transpose_2b, "map_transpose_2b");
181 gpu_create_kernel(ctx, &kernels->map_transpose_2b_large, "map_transpose_2b_large");
182 gpu_create_kernel(ctx, &kernels->map_transpose_2b_low_height, "map_transpose_2b_low_height");
183 gpu_create_kernel(ctx, &kernels->map_transpose_2b_low_width, "map_transpose_2b_low_width");
184 gpu_create_kernel(ctx, &kernels->map_transpose_2b_small, "map_transpose_2b_small");
185
186 gpu_create_kernel(ctx, &kernels->map_transpose_4b, "map_transpose_4b");
187 gpu_create_kernel(ctx, &kernels->map_transpose_4b_large, "map_transpose_4b_large");
188 gpu_create_kernel(ctx, &kernels->map_transpose_4b_low_height, "map_transpose_4b_low_height");
189 gpu_create_kernel(ctx, &kernels->map_transpose_4b_low_width, "map_transpose_4b_low_width");
190 gpu_create_kernel(ctx, &kernels->map_transpose_4b_small, "map_transpose_4b_small");
191
192 gpu_create_kernel(ctx, &kernels->map_transpose_8b, "map_transpose_8b");
193 gpu_create_kernel(ctx, &kernels->map_transpose_8b_large, "map_transpose_8b_large");
194 gpu_create_kernel(ctx, &kernels->map_transpose_8b_low_height, "map_transpose_8b_low_height");
195 gpu_create_kernel(ctx, &kernels->map_transpose_8b_low_width, "map_transpose_8b_low_width");
196 gpu_create_kernel(ctx, &kernels->map_transpose_8b_small, "map_transpose_8b_small");
197
198 gpu_create_kernel(ctx, &kernels->lmad_copy_1b, "lmad_copy_1b");
199 gpu_create_kernel(ctx, &kernels->lmad_copy_2b, "lmad_copy_2b");
200 gpu_create_kernel(ctx, &kernels->lmad_copy_4b, "lmad_copy_4b");
201 gpu_create_kernel(ctx, &kernels->lmad_copy_8b, "lmad_copy_8b");
202
203 return kernels;
204 }
205
206 void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels) {
207 gpu_free_kernel(ctx, kernels->map_transpose_1b);
208 gpu_free_kernel(ctx, kernels->map_transpose_1b_large);
209 gpu_free_kernel(ctx, kernels->map_transpose_1b_low_height);
210 gpu_free_kernel(ctx, kernels->map_transpose_1b_low_width);
211 gpu_free_kernel(ctx, kernels->map_transpose_1b_small);
212
213 gpu_free_kernel(ctx, kernels->map_transpose_2b);
214 gpu_free_kernel(ctx, kernels->map_transpose_2b_large);
215 gpu_free_kernel(ctx, kernels->map_transpose_2b_low_height);
216 gpu_free_kernel(ctx, kernels->map_transpose_2b_low_width);
217 gpu_free_kernel(ctx, kernels->map_transpose_2b_small);
218
219 gpu_free_kernel(ctx, kernels->map_transpose_4b);
220 gpu_free_kernel(ctx, kernels->map_transpose_4b_large);
221 gpu_free_kernel(ctx, kernels->map_transpose_4b_low_height);
222 gpu_free_kernel(ctx, kernels->map_transpose_4b_low_width);
223 gpu_free_kernel(ctx, kernels->map_transpose_4b_small);
224
225 gpu_free_kernel(ctx, kernels->map_transpose_8b);
226 gpu_free_kernel(ctx, kernels->map_transpose_8b_large);
227 gpu_free_kernel(ctx, kernels->map_transpose_8b_low_height);
228 gpu_free_kernel(ctx, kernels->map_transpose_8b_low_width);
229 gpu_free_kernel(ctx, kernels->map_transpose_8b_small);
230
231 gpu_free_kernel(ctx, kernels->lmad_copy_1b);
232 gpu_free_kernel(ctx, kernels->lmad_copy_2b);
233 gpu_free_kernel(ctx, kernels->lmad_copy_4b);
234 gpu_free_kernel(ctx, kernels->lmad_copy_8b);
235
236 free(kernels);
237 }
238
239 static int gpu_alloc(struct futhark_context *ctx, FILE *log,
240 size_t min_size, const char *tag,
241 gpu_mem *mem_out, size_t *size_out) {
242 if (min_size < sizeof(int)) {
243 min_size = sizeof(int);
244 }
245
246 // Round up the allocation to be at least divisible by 4, because that is
247 // assumed by the code generator.
248 min_size = (min_size+3) & ~3;
249
250 gpu_mem* memptr;
251 if (free_list_find(&ctx->gpu_free_list, min_size, tag, size_out, (fl_mem*)&memptr) == 0) {
252 // Successfully found a free block. Is it big enough?
253 if (*size_out >= min_size) {
254 if (ctx->cfg->debugging) {
255 fprintf(log, "No need to allocate: Found a block in the free list.\n");
256 }
257 *mem_out = *memptr;
258 free(memptr);
259 return FUTHARK_SUCCESS;
260 } else {
261 if (ctx->cfg->debugging) {
262 fprintf(log, "Found a free block, but it was too small.\n");
263 }
264 int error = gpu_free_actual(ctx, *memptr);
265 free(memptr);
266 if (error != FUTHARK_SUCCESS) {
267 return error;
268 }
269 }
270 }
271
272 *size_out = min_size;
273
274 // We have to allocate a new block from the driver. If the
275 // allocation does not succeed, then we might be in an out-of-memory
276 // situation. We now start freeing things from the free list until
277 // we think we have freed enough that the allocation will succeed.
278 // Since we don't know how far the allocation is from fitting, we
279 // have to check after every deallocation. This might be pretty
280 // expensive. Let's hope that this case is hit rarely.
281
282 if (ctx->cfg->debugging) {
283 fprintf(log, "Actually allocating the desired block.\n");
284 }
285
286 int error = gpu_alloc_actual(ctx, min_size, mem_out);
287
288 while (error == FUTHARK_OUT_OF_MEMORY) {
289 if (ctx->cfg->debugging) {
290 fprintf(log, "Out of GPU memory: releasing entry from the free list...\n");
291 }
292 gpu_mem* memptr;
293 if (free_list_first(&ctx->gpu_free_list, (fl_mem*)&memptr) == 0) {
294 gpu_mem mem = *memptr;
295 free(memptr);
296 error = gpu_free_actual(ctx, mem);
297 if (error != FUTHARK_SUCCESS) {
298 return error;
299 }
300 } else {
301 break;
302 }
303 error = gpu_alloc_actual(ctx, min_size, mem_out);
304 }
305
306 return error;
307 }
308
309 static int gpu_free(struct futhark_context *ctx,
310 gpu_mem mem, size_t size, const char *tag) {
311 gpu_mem* memptr = malloc(sizeof(gpu_mem));
312 *memptr = mem;
313 free_list_insert(&ctx->gpu_free_list, size, (fl_mem)memptr, tag);
314 return FUTHARK_SUCCESS;
315 }
316
317 static int gpu_free_all(struct futhark_context *ctx) {
318 free_list_pack(&ctx->gpu_free_list);
319 gpu_mem* memptr;
320 while (free_list_first(&ctx->gpu_free_list, (fl_mem*)&memptr) == 0) {
321 gpu_mem mem = *memptr;
322 free(memptr);
323 int error = gpu_free_actual(ctx, mem);
324 if (error != FUTHARK_SUCCESS) {
325 return error;
326 }
327 }
328
329 return FUTHARK_SUCCESS;
330 }
331
332 static int gpu_map_transpose(struct futhark_context* ctx,
333 const char* provenance,
334 gpu_kernel kernel_default,
335 gpu_kernel kernel_low_height,
336 gpu_kernel kernel_low_width,
337 gpu_kernel kernel_small,
338 gpu_kernel kernel_large,
339 const char *name, size_t elem_size,
340 gpu_mem dst, int64_t dst_offset,
341 gpu_mem src, int64_t src_offset,
342 int64_t k, int64_t n, int64_t m) {
343 int64_t mulx = TR_BLOCK_DIM / n;
344 int64_t muly = TR_BLOCK_DIM / m;
345 int32_t mulx32 = mulx;
346 int32_t muly32 = muly;
347 int32_t k32 = k;
348 int32_t n32 = n;
349 int32_t m32 = m;
350
351 gpu_kernel kernel = kernel_default;
352 int32_t grid[3];
353 int32_t block[3];
354
355 void* args[11];
356 size_t args_sizes[11] = {
357 sizeof(gpu_mem), sizeof(int64_t),
358 sizeof(gpu_mem), sizeof(int64_t),
359 sizeof(int32_t),
360 sizeof(int32_t),
361 sizeof(int32_t),
362 sizeof(int32_t),
363 sizeof(int32_t)
364 };
365
366 args[0] = &dst;
367 args[1] = &dst_offset;
368 args[2] = &src;
369 args[3] = &src_offset;
370 args[7] = &mulx;
371 args[8] = &muly;
372
373 if (dst_offset + k * n * m <= 2147483647L &&
374 src_offset + k * n * m <= 2147483647L) {
375 if (m <= TR_BLOCK_DIM/2 && n <= TR_BLOCK_DIM/2) {
376 if (ctx->logging) { fprintf(ctx->log, "Using small kernel\n"); }
377 kernel = kernel_small;
378 grid[0] = ((k * n * m) + (TR_BLOCK_DIM*TR_BLOCK_DIM) - 1) / (TR_BLOCK_DIM*TR_BLOCK_DIM);
379 grid[1] = 1;
380 grid[2] = 1;
381 block[0] = TR_BLOCK_DIM*TR_BLOCK_DIM;
382 block[1] = 1;
383 block[2] = 1;
384 } else if (m <= TR_BLOCK_DIM/2 && TR_BLOCK_DIM < n) {
385 if (ctx->logging) { fprintf(ctx->log, "Using low-width kernel\n"); }
386 kernel = kernel_low_width;
387 int64_t x_elems = m;
388 int64_t y_elems = (n + muly - 1) / muly;
389 grid[0] = (x_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;
390 grid[1] = (y_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;
391 grid[2] = k;
392 block[0] = TR_BLOCK_DIM;
393 block[1] = TR_BLOCK_DIM;
394 block[2] = 1;
395 } else if (n <= TR_BLOCK_DIM/2 && TR_BLOCK_DIM < m) {
396 if (ctx->logging) { fprintf(ctx->log, "Using low-height kernel\n"); }
397 kernel = kernel_low_height;
398 int64_t x_elems = (m + mulx - 1) / mulx;
399 int64_t y_elems = n;
400 grid[0] = (x_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;
401 grid[1] = (y_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;
402 grid[2] = k;
403 block[0] = TR_BLOCK_DIM;
404 block[1] = TR_BLOCK_DIM;
405 block[2] = 1;
406 } else {
407 if (ctx->logging) { fprintf(ctx->log, "Using default kernel\n"); }
408 kernel = kernel_default;
409 grid[0] = (m+TR_TILE_DIM-1)/TR_TILE_DIM;
410 grid[1] = (n+TR_TILE_DIM-1)/TR_TILE_DIM;
411 grid[2] = k;
412 block[0] = TR_TILE_DIM;
413 block[1] = TR_TILE_DIM/TR_ELEMS_PER_THREAD;
414 block[2] = 1;
415 }
416 args[4] = &k32;
417 args[5] = &m32;
418 args[6] = &n32;
419 args[7] = &mulx32;
420 args[8] = &muly32;
421 } else {
422 if (ctx->logging) { fprintf(ctx->log, "Using large kernel\n"); }
423 kernel = kernel_large;
424 grid[0] = (m+TR_TILE_DIM-1)/TR_TILE_DIM;
425 grid[1] = (n+TR_TILE_DIM-1)/TR_TILE_DIM;
426 grid[2] = k;
427 block[0] = TR_TILE_DIM;
428 block[1] = TR_TILE_DIM/TR_ELEMS_PER_THREAD;
429 block[2] = 1;
430 args[4] = &k;
431 args[5] = &m;
432 args[6] = &n;
433 args[7] = &mulx;
434 args[8] = &muly;
435 args_sizes[4] = sizeof(int64_t);
436 args_sizes[5] = sizeof(int64_t);
437 args_sizes[6] = sizeof(int64_t);
438 args_sizes[7] = sizeof(int64_t);
439 args_sizes[8] = sizeof(int64_t);
440 }
441
442 // Cap the number of thead blocks we launch and figure out how many
443 // repeats we need alongside each dimension.
444 int32_t repeat_1 = grid[1] / MAX_TR_THREAD_BLOCKS;
445 int32_t repeat_2 = grid[2] / MAX_TR_THREAD_BLOCKS;
446 grid[1] = repeat_1 > 0 ? MAX_TR_THREAD_BLOCKS : grid[1];
447 grid[2] = repeat_2 > 0 ? MAX_TR_THREAD_BLOCKS : grid[2];
448 args[9] = &repeat_1;
449 args[10] = &repeat_2;
450 args_sizes[9] = sizeof(repeat_1);
451 args_sizes[10] = sizeof(repeat_2);
452
453 if (ctx->logging) {
454 fprintf(ctx->log, "\n");
455 }
456
457 return gpu_launch_kernel(ctx, kernel, name, provenance, grid, block,
458 TR_TILE_DIM*(TR_TILE_DIM+1)*elem_size,
459 sizeof(args)/sizeof(args[0]), args, args_sizes);
460 }
461
462 #define GEN_MAP_TRANSPOSE_GPU2GPU(NAME, ELEM_TYPE) \
463 static int map_transpose_gpu2gpu_##NAME \
464 (struct futhark_context* ctx, const char* provenance, \
465 gpu_mem dst, int64_t dst_offset, \
466 gpu_mem src, int64_t src_offset, \
467 int64_t k, int64_t m, int64_t n) \
468 { \
469 return \
470 gpu_map_transpose \
471 (ctx, \
472 provenance, \
473 ctx->kernels->map_transpose_##NAME, \
474 ctx->kernels->map_transpose_##NAME##_low_height, \
475 ctx->kernels->map_transpose_##NAME##_low_width, \
476 ctx->kernels->map_transpose_##NAME##_small, \
477 ctx->kernels->map_transpose_##NAME##_large, \
478 "map_transpose_" #NAME, sizeof(ELEM_TYPE), \
479 dst, dst_offset, src, src_offset, \
480 k, n, m); \
481 }
482
483 static int gpu_lmad_copy(struct futhark_context* ctx, const char* provenance,
484 gpu_kernel kernel, int r,
485 gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],
486 gpu_mem src, int64_t src_offset, int64_t src_strides[r],
487 int64_t shape[r]) {
488 if (r > 8) {
489 set_error(ctx, strdup("Futhark runtime limitation:\nCannot copy array of greater than rank 8.\n"));
490 return 1;
491 }
492
493 int64_t n = 1;
494 for (int i = 0; i < r; i++) { n *= shape[i]; }
495
496 void* args[6+(8*3)];
497 size_t args_sizes[6+(8*3)];
498
499 args[0] = &dst;
500 args_sizes[0] = sizeof(gpu_mem);
501 args[1] = &dst_offset;
502 args_sizes[1] = sizeof(dst_offset);
503 args[2] = &src;
504 args_sizes[2] = sizeof(gpu_mem);
505 args[3] = &src_offset;
506 args_sizes[3] = sizeof(src_offset);
507 args[4] = &n;
508 args_sizes[4] = sizeof(n);
509 args[5] = &r;
510 args_sizes[5] = sizeof(r);
511
512 int64_t zero = 0;
513
514 for (int i = 0; i < 8; i++) {
515 args_sizes[6+i*3] = sizeof(int64_t);
516 args_sizes[6+i*3+1] = sizeof(int64_t);
517 args_sizes[6+i*3+2] = sizeof(int64_t);
518 if (i < r) {
519 args[6+i*3] = &shape[i];
520 args[6+i*3+1] = &dst_strides[i];
521 args[6+i*3+2] = &src_strides[i];
522 } else {
523 args[6+i*3] = &zero;
524 args[6+i*3+1] = &zero;
525 args[6+i*3+2] = &zero;
526 }
527 }
528 const size_t w = 256; // XXX: hardcoded thread block size.
529
530 return gpu_launch_kernel(ctx, kernel, "copy_lmad_dev_to_dev", provenance,
531 (const int32_t[3]) {(n+w-1)/w,1,1},
532 (const int32_t[3]) {w,1,1},
533 0, 6+(8*3), args, args_sizes);
534 }
535
536 #define GEN_LMAD_COPY_ELEMENTS_GPU2GPU(NAME, ELEM_TYPE) \
537 static int lmad_copy_elements_gpu2gpu_##NAME \
538 (struct futhark_context* ctx, const char* provenance, \
539 int r, \
540 gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r], \
541 gpu_mem src, int64_t src_offset, int64_t src_strides[r], \
542 int64_t shape[r]) { \
543 return gpu_lmad_copy(ctx, provenance, \
544 ctx->kernels->lmad_copy_##NAME, \
545 r, \
546 dst, dst_offset, dst_strides, \
547 src, src_offset, src_strides, \
548 shape); \
549 } \
550
551 #define GEN_LMAD_COPY_GPU2GPU(NAME, ELEM_TYPE) \
552 static int lmad_copy_gpu2gpu_##NAME \
553 (struct futhark_context* ctx, const char* provenance, \
554 int r, \
555 gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r], \
556 gpu_mem src, int64_t src_offset, int64_t src_strides[r], \
557 int64_t shape[r]) { \
558 log_copy(ctx, "GPU to GPU", provenance, r, dst_offset, dst_strides, \
559 src_offset, src_strides, shape); \
560 int64_t size = 1; \
561 for (int i = 0; i < r; i++) { size *= shape[i]; } \
562 if (size == 0) { return FUTHARK_SUCCESS; } \
563 int64_t k, n, m; \
564 if (lmad_map_tr(&k, &n, &m, \
565 r, dst_strides, src_strides, shape)) { \
566 log_transpose(ctx, k, n, m); \
567 return map_transpose_gpu2gpu_##NAME \
568 (ctx, provenance, dst, dst_offset, src, src_offset, k, n, m); \
569 } else if (lmad_memcpyable(r, dst_strides, src_strides, shape)) { \
570 if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");} \
571 return gpu_memcpy(ctx, \
572 provenance, \
573 dst, dst_offset*sizeof(ELEM_TYPE), \
574 src, src_offset*sizeof(ELEM_TYPE), \
575 size * sizeof(ELEM_TYPE)); \
576 } else { \
577 if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");} \
578 return lmad_copy_elements_gpu2gpu_##NAME \
579 (ctx, provenance, \
580 r, \
581 dst, dst_offset, dst_strides, \
582 src, src_offset, src_strides, \
583 shape); \
584 } \
585 }
586
587 static int
588 lmad_copy_elements_host2gpu(struct futhark_context *ctx, const char *provenance,
589 size_t elem_size,
590 int r,
591 gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],
592 unsigned char* src, int64_t src_offset, int64_t src_strides[r],
593 int64_t shape[r]) {
594 (void)ctx; (void)provenance; (void)elem_size; (void)r;
595 (void)dst; (void)dst_offset; (void)dst_strides;
596 (void)src; (void)src_offset; (void)src_strides;
597 (void)shape;
598 set_error(ctx, strdup("Futhark runtime limitation:\nCannot copy unstructured array from host to GPU.\n"));
599 return 1;
600 }
601
602 static int
603 lmad_copy_elements_gpu2host (struct futhark_context *ctx, const char *provenance,
604 size_t elem_size,
605 int r,
606 unsigned char* dst, int64_t dst_offset, int64_t dst_strides[r],
607 gpu_mem src, int64_t src_offset, int64_t src_strides[r],
608 int64_t shape[r]) {
609 (void)ctx; (void)provenance; (void)elem_size; (void)r;
610 (void)dst; (void)dst_offset; (void)dst_strides;
611 (void)src; (void)src_offset; (void)src_strides;
612 (void)shape;
613 set_error(ctx, strdup("Futhark runtime limitation:\nCannot copy unstructured array from GPU to host.\n"));
614 return 1;
615 }
616
617 static int lmad_copy_host2gpu(struct futhark_context* ctx, const char* provenance,
618 size_t elem_size, bool sync,
619 int r,
620 gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],
621 unsigned char* src, int64_t src_offset, int64_t src_strides[r],
622 int64_t shape[r]) {
623 log_copy(ctx, "Host to GPU", provenance, r, dst_offset, dst_strides,
624 src_offset, src_strides, shape);
625 int64_t size = elem_size;
626 for (int i = 0; i < r; i++) { size *= shape[i]; }
627 if (size == 0) { return FUTHARK_SUCCESS; }
628 int64_t k, n, m;
629 if (lmad_memcpyable(r, dst_strides, src_strides, shape)) {
630 if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");}
631 return memcpy_host2gpu(ctx, provenance,
632 sync,
633 dst, dst_offset*elem_size,
634 src, src_offset*elem_size,
635 size);
636 } else {
637 if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");}
638 int error;
639 error = lmad_copy_elements_host2gpu
640 (ctx, provenance,
641 elem_size, r,
642 dst, dst_offset, dst_strides,
643 src, src_offset, src_strides,
644 shape);
645 if (error == 0 && sync) {
646 error = futhark_context_sync(ctx);
647 }
648 return error;
649 }
650 }
651
652 static int lmad_copy_gpu2host(struct futhark_context* ctx, const char* provenance,
653 size_t elem_size, bool sync,
654 int r,
655 unsigned char* dst, int64_t dst_offset, int64_t dst_strides[r],
656 gpu_mem src, int64_t src_offset, int64_t src_strides[r],
657 int64_t shape[r]) {
658 log_copy(ctx, "Host to GPU", provenance, r, dst_offset, dst_strides,
659 src_offset, src_strides, shape);
660 int64_t size = elem_size;
661 for (int i = 0; i < r; i++) { size *= shape[i]; }
662 if (size == 0) { return FUTHARK_SUCCESS; }
663 int64_t k, n, m;
664 if (lmad_memcpyable(r, dst_strides, src_strides, shape)) {
665 if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");}
666 return memcpy_gpu2host(ctx, provenance,
667 sync,
668 dst, dst_offset*elem_size,
669 src, src_offset*elem_size,
670 size);
671 } else {
672 if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");}
673 int error;
674 error = lmad_copy_elements_gpu2host
675 (ctx, provenance, elem_size, r,
676 dst, dst_offset, dst_strides,
677 src, src_offset, src_strides,
678 shape);
679 if (error == 0 && sync) {
680 error = futhark_context_sync(ctx);
681 }
682 return error;
683 }
684 }
685
686 GEN_MAP_TRANSPOSE_GPU2GPU(1b, uint8_t)
687 GEN_MAP_TRANSPOSE_GPU2GPU(2b, uint16_t)
688 GEN_MAP_TRANSPOSE_GPU2GPU(4b, uint32_t)
689 GEN_MAP_TRANSPOSE_GPU2GPU(8b, uint64_t)
690
691 GEN_LMAD_COPY_ELEMENTS_GPU2GPU(1b, uint8_t)
692 GEN_LMAD_COPY_ELEMENTS_GPU2GPU(2b, uint16_t)
693 GEN_LMAD_COPY_ELEMENTS_GPU2GPU(4b, uint32_t)
694 GEN_LMAD_COPY_ELEMENTS_GPU2GPU(8b, uint64_t)
695
696 GEN_LMAD_COPY_GPU2GPU(1b, uint8_t)
697 GEN_LMAD_COPY_GPU2GPU(2b, uint16_t)
698 GEN_LMAD_COPY_GPU2GPU(4b, uint32_t)
699 GEN_LMAD_COPY_GPU2GPU(8b, uint64_t)
700
701 // End of gpu.h