Debian Codegraph @

rts/c/backends/cuda.h

raw
1 // Start of backends/cuda.h.
2
3 // Forward declarations.
4 // Invoked by setup_opencl() after the platform and device has been
5 // found, but before the program is loaded. Its intended use is to
6 // tune constants based on the selected platform and device.
7 static void set_tuning_params(struct futhark_context* ctx);
8 static char* get_failure_msg(int failure_idx, int64_t args[]);
9
10 #define CUDA_SUCCEED_FATAL(x) cuda_api_succeed_fatal(x, #x, __FILE__, __LINE__)
11 #define CUDA_SUCCEED_NONFATAL(x) cuda_api_succeed_nonfatal(x, #x, __FILE__, __LINE__)
12 #define NVRTC_SUCCEED_FATAL(x) nvrtc_api_succeed_fatal(x, #x, __FILE__, __LINE__)
13 #define NVRTC_SUCCEED_NONFATAL(x) nvrtc_api_succeed_nonfatal(x, #x, __FILE__, __LINE__)
14 // Take care not to override an existing error.
15 #define CUDA_SUCCEED_OR_RETURN(e) { \
16 char *serror = CUDA_SUCCEED_NONFATAL(e); \
17 if (serror) { \
18 if (!ctx->error) { \
19 ctx->error = serror; \
20 } else { \
21 free(serror); \
22 } \
23 return bad; \
24 } \
25 }
26
27 // CUDA_SUCCEED_OR_RETURN returns the value of the variable 'bad' in
28 // scope. By default, it will be this one. Create a local variable
29 // of some other type if needed. This is a bit of a hack, but it
30 // saves effort in the code generator.
31 static const int bad = 1;
32
33 static inline void cuda_api_succeed_fatal(CUresult res, const char *call,
34 const char *file, int line) {
35 if (res != CUDA_SUCCESS) {
36 const char *err_str;
37 cuGetErrorString(res, &err_str);
38 if (err_str == NULL) { err_str = "Unknown"; }
39 futhark_panic(-1, "%s:%d: CUDA call\n %s\nfailed with error code %d (%s)\n",
40 file, line, call, res, err_str);
41 }
42 }
43
44 static char* cuda_api_succeed_nonfatal(CUresult res, const char *call,
45 const char *file, int line) {
46 if (res != CUDA_SUCCESS) {
47 const char *err_str;
48 cuGetErrorString(res, &err_str);
49 if (err_str == NULL) { err_str = "Unknown"; }
50 return msgprintf("%s:%d: CUDA call\n %s\nfailed with error code %d (%s)\n",
51 file, line, call, res, err_str);
52 } else {
53 return NULL;
54 }
55 }
56
57 static inline void nvrtc_api_succeed_fatal(nvrtcResult res, const char *call,
58 const char *file, int line) {
59 if (res != NVRTC_SUCCESS) {
60 const char *err_str = nvrtcGetErrorString(res);
61 futhark_panic(-1, "%s:%d: NVRTC call\n %s\nfailed with error code %d (%s)\n",
62 file, line, call, res, err_str);
63 }
64 }
65
66 static char* nvrtc_api_succeed_nonfatal(nvrtcResult res, const char *call,
67 const char *file, int line) {
68 if (res != NVRTC_SUCCESS) {
69 const char *err_str = nvrtcGetErrorString(res);
70 return msgprintf("%s:%d: NVRTC call\n %s\nfailed with error code %d (%s)\n",
71 file, line, call, res, err_str);
72 } else {
73 return NULL;
74 }
75 }
76
77 struct futhark_context_config {
78 int in_use;
79 int debugging;
80 int profiling;
81 int logging;
82 char* cache_fname;
83 int num_tuning_params;
84 int64_t *tuning_params;
85 const char** tuning_param_names;
86 const char** tuning_param_vars;
87 const char** tuning_param_classes;
88 // Uniform fields above.
89
90 char* program;
91 int num_nvrtc_opts;
92 char* *nvrtc_opts;
93
94 char* preferred_device;
95 int preferred_device_num;
96
97 int unified_memory;
98
99 char* dump_ptx_to;
100 char* load_ptx_from;
101
102 struct gpu_config gpu;
103 };
104
105 static void backend_context_config_setup(struct futhark_context_config *cfg) {
106 cfg->num_nvrtc_opts = 0;
107 cfg->nvrtc_opts = (char**) malloc(sizeof(char*));
108 cfg->nvrtc_opts[0] = NULL;
109
110 cfg->program = strconcat(gpu_program);
111
112 cfg->preferred_device_num = 0;
113 cfg->preferred_device = strdup("");
114
115 cfg->dump_ptx_to = NULL;
116 cfg->load_ptx_from = NULL;
117
118 cfg->unified_memory = 2;
119
120 cfg->gpu = gpu_config_initial;
121 cfg->gpu.default_block_size = 256;
122 cfg->gpu.default_tile_size = 32;
123 cfg->gpu.default_reg_tile_size = 2;
124 cfg->gpu.default_threshold = 32*1024;
125 }
126
127 static void backend_context_config_teardown(struct futhark_context_config* cfg) {
128 for (int i = 0; i < cfg->num_nvrtc_opts; i++) {
129 free(cfg->nvrtc_opts[i]);
130 }
131 free(cfg->nvrtc_opts);
132 free(cfg->dump_ptx_to);
133 free(cfg->load_ptx_from);
134 free(cfg->preferred_device);
135 free(cfg->program);
136 }
137
138 void futhark_context_config_add_nvrtc_option(struct futhark_context_config *cfg, const char *opt) {
139 cfg->nvrtc_opts[cfg->num_nvrtc_opts] = strdup(opt);
140 cfg->num_nvrtc_opts++;
141 cfg->nvrtc_opts = (char **) realloc(cfg->nvrtc_opts, (cfg->num_nvrtc_opts + 1) * sizeof(char *));
142 cfg->nvrtc_opts[cfg->num_nvrtc_opts] = NULL;
143 }
144
145 void futhark_context_config_set_device(struct futhark_context_config *cfg, const char *s) {
146 int x = 0;
147 if (*s == '#') {
148 s++;
149 while (isdigit(*s)) {
150 x = x * 10 + (*s++)-'0';
151 }
152 // Skip trailing spaces.
153 while (isspace(*s)) {
154 s++;
155 }
156 }
157 free(cfg->preferred_device);
158 cfg->preferred_device = strdup(s);
159 cfg->preferred_device_num = x;
160 }
161
162 const char* futhark_context_config_get_program(struct futhark_context_config *cfg) {
163 return cfg->program;
164 }
165
166 void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) {
167 free(cfg->program);
168 cfg->program = strdup(s);
169 }
170
171 void futhark_context_config_dump_ptx_to(struct futhark_context_config *cfg, const char *path) {
172 free(cfg->dump_ptx_to);
173 cfg->dump_ptx_to = strdup(path);
174 }
175
176 void futhark_context_config_load_ptx_from(struct futhark_context_config *cfg, const char *path) {
177 free(cfg->load_ptx_from);
178 cfg->load_ptx_from = strdup(path);
179 }
180
181 void futhark_context_config_set_unified_memory(struct futhark_context_config* cfg, int flag) {
182 cfg->unified_memory = flag;
183 }
184
185 // A record of something that happened.
186 struct profiling_record {
187 cudaEvent_t *events; // Points to two events.
188 const char *name;
189 };
190
191 struct futhark_context {
192 struct futhark_context_config* cfg;
193 int detail_memory;
194 int debugging;
195 int profiling;
196 int profiling_paused;
197 int logging;
198 lock_t lock;
199 char *error;
200 lock_t error_lock;
201 FILE *log;
202 struct constants *constants;
203 struct free_list free_list;
204 struct event_list event_list;
205 int64_t peak_mem_usage_default;
206 int64_t cur_mem_usage_default;
207 struct program* program;
208 bool program_initialised;
209 // Uniform fields above.
210
211 CUdeviceptr global_failure;
212 CUdeviceptr global_failure_args;
213 struct tuning_params tuning_params;
214 // True if a potentially failing kernel has been enqueued.
215 int32_t failure_is_an_option;
216 int total_runs;
217 long int total_runtime;
218 int64_t peak_mem_usage_device;
219 int64_t cur_mem_usage_device;
220
221 CUdevice dev;
222 CUcontext cu_ctx;
223 CUmodule module;
224 CUstream stream;
225
226 struct free_list gpu_free_list;
227
228 size_t max_thread_block_size;
229 size_t max_grid_size;
230 size_t max_tile_size;
231 size_t max_threshold;
232 size_t max_shared_memory;
233 size_t max_bespoke;
234 size_t max_registers;
235 size_t max_cache;
236
237 size_t lockstep_width;
238
239 struct builtin_kernels* kernels;
240 };
241
242 #define CU_DEV_ATTR(x) (CU_DEVICE_ATTRIBUTE_##x)
243 #define device_query(dev,attrib) _device_query(dev, CU_DEV_ATTR(attrib))
244 static int _device_query(CUdevice dev, CUdevice_attribute attrib) {
245 int val;
246 CUDA_SUCCEED_FATAL(cuDeviceGetAttribute(&val, attrib, dev));
247 return val;
248 }
249
250 #define CU_FUN_ATTR(x) (CU_FUNC_ATTRIBUTE_##x)
251 #define function_query(fn,attrib) _function_query(dev, CU_FUN_ATTR(attrib))
252 static int _function_query(CUfunction dev, CUfunction_attribute attrib) {
253 int val;
254 CUDA_SUCCEED_FATAL(cuFuncGetAttribute(&val, attrib, dev));
255 return val;
256 }
257
258 static int cuda_device_setup(struct futhark_context *ctx) {
259 struct futhark_context_config *cfg = ctx->cfg;
260 char name[256];
261 int count, chosen = -1, best_cc = -1;
262 int cc_major_best = 0, cc_minor_best = 0;
263 int cc_major = 0, cc_minor = 0;
264 CUdevice dev;
265
266 CUDA_SUCCEED_FATAL(cuDeviceGetCount(&count));
267 if (count == 0) { return 1; }
268
269 int num_device_matches = 0;
270
271 // XXX: Current device selection policy is to choose the device with the
272 // highest compute capability (if no preferred device is set).
273 // This should maybe be changed, since greater compute capability is not
274 // necessarily an indicator of better performance.
275 for (int i = 0; i < count; i++) {
276 CUDA_SUCCEED_FATAL(cuDeviceGet(&dev, i));
277
278 cc_major = device_query(dev, COMPUTE_CAPABILITY_MAJOR);
279 cc_minor = device_query(dev, COMPUTE_CAPABILITY_MINOR);
280
281 CUDA_SUCCEED_FATAL(cuDeviceGetName(name, sizeof(name) - 1, dev));
282 name[sizeof(name) - 1] = 0;
283
284 if (cfg->logging) {
285 fprintf(ctx->log, "Device #%d: name=\"%s\", compute capability=%d.%d\n",
286 i, name, cc_major, cc_minor);
287 }
288
289 if (device_query(dev, COMPUTE_MODE) == CU_COMPUTEMODE_PROHIBITED) {
290 if (cfg->logging) {
291 fprintf(ctx->log, "Device #%d is compute-prohibited, ignoring\n", i);
292 }
293 continue;
294 }
295
296 if (best_cc == -1 || cc_major > cc_major_best ||
297 (cc_major == cc_major_best && cc_minor > cc_minor_best)) {
298 best_cc = i;
299 cc_major_best = cc_major;
300 cc_minor_best = cc_minor;
301 }
302
303 if (strstr(name, cfg->preferred_device) != NULL &&
304 num_device_matches++ == cfg->preferred_device_num) {
305 chosen = i;
306 break;
307 }
308 }
309
310 if (chosen == -1) { chosen = best_cc; }
311 if (chosen == -1) { return 1; }
312
313 if (cfg->logging) {
314 fprintf(ctx->log, "Using device #%d\n", chosen);
315 }
316
317 CUDA_SUCCEED_FATAL(cuDeviceGet(&ctx->dev, chosen));
318 return 0;
319 }
320
321 static const char *cuda_nvrtc_get_arch(CUdevice dev) {
322 static struct {
323 int major;
324 int minor;
325 const char *arch_str;
326 } const x[] = {
327 { 3, 0, "compute_30" },
328 { 3, 2, "compute_32" },
329 { 3, 5, "compute_35" },
330 { 3, 7, "compute_37" },
331 { 5, 0, "compute_50" },
332 { 5, 2, "compute_52" },
333 { 5, 3, "compute_53" },
334 { 6, 0, "compute_60" },
335 { 6, 1, "compute_61" },
336 { 6, 2, "compute_62" },
337 { 7, 0, "compute_70" },
338 { 7, 2, "compute_72" },
339 { 7, 5, "compute_75" },
340 { 8, 0, "compute_80" },
341 { 8, 6, "compute_80" },
342 { 8, 7, "compute_80" },
343 { 8, 9, "compute_80" },
344 { 9, 0, "compute_90" },
345 { 10, 0, "compute_100" },
346 { 10, 1, "compute_101" },
347 { 12, 0, "compute_120" }
348 };
349
350 int major = device_query(dev, COMPUTE_CAPABILITY_MAJOR);
351 int minor = device_query(dev, COMPUTE_CAPABILITY_MINOR);
352
353 int chosen = -1;
354 int num_archs = sizeof(x)/sizeof(x[0]);
355 for (int i = 0; i < num_archs; i++) {
356 if (x[i].major < major || (x[i].major == major && x[i].minor <= minor)) {
357 chosen = i;
358 } else {
359 break;
360 }
361 }
362
363 if (chosen == -1) {
364 futhark_panic(-1, "Unsupported compute capability %d.%d\n", major, minor);
365 }
366
367 if (x[chosen].major != major || x[chosen].minor != minor) {
368 fprintf(stderr,
369 "Warning: device compute capability is %d.%d, but newest supported by Futhark is %d.%d.\n",
370 major, minor, x[chosen].major, x[chosen].minor);
371 }
372
373 return x[chosen].arch_str;
374 }
375
376 static void cuda_nvrtc_mk_build_options(struct futhark_context *ctx, const char *extra_opts[],
377 char*** opts_out, size_t *n_opts) {
378 int arch_set = 0, num_extra_opts;
379 struct futhark_context_config *cfg = ctx->cfg;
380
381 char** macro_names;
382 int64_t* macro_vals;
383 int num_macros = gpu_macros(ctx, &macro_names, &macro_vals);
384
385 // nvrtc cannot handle multiple -arch options. Hence, if one of the
386 // extra_opts is -arch, we have to be careful not to do our usual
387 // automatic generation.
388 for (num_extra_opts = 0; extra_opts[num_extra_opts] != NULL; num_extra_opts++) {
389 if (strstr(extra_opts[num_extra_opts], "-arch")
390 == extra_opts[num_extra_opts] ||
391 strstr(extra_opts[num_extra_opts], "--gpu-architecture")
392 == extra_opts[num_extra_opts]) {
393 arch_set = 1;
394 }
395 }
396
397 size_t i = 0, n_opts_alloc = 20 + num_macros + num_extra_opts + cfg->num_tuning_params;
398 char **opts = (char**) malloc(n_opts_alloc * sizeof(char *));
399 if (!arch_set) {
400 opts[i++] = strdup("-arch");
401 opts[i++] = strdup(cuda_nvrtc_get_arch(ctx->dev));
402 }
403 opts[i++] = strdup("-default-device");
404 if (cfg->debugging) {
405 opts[i++] = strdup("-G");
406 opts[i++] = strdup("-lineinfo");
407 } else {
408 opts[i++] = strdup("--disable-warnings");
409 }
410 opts[i++] = msgprintf("-D%s=%d",
411 "max_thread_block_size",
412 (int)ctx->max_thread_block_size);
413 opts[i++] = msgprintf("-D%s=%d",
414 "max_shared_memory",
415 (int)ctx->max_shared_memory);
416 opts[i++] = msgprintf("-D%s=%d",
417 "max_registers",
418 (int)ctx->max_registers);
419
420 for (int j = 0; j < num_macros; j++) {
421 opts[i++] = msgprintf("-D%s=%zu", macro_names[j], macro_vals[j]);
422 }
423
424 for (int j = 0; j < cfg->num_tuning_params; j++) {
425 opts[i++] = msgprintf("-D%s=%zu", cfg->tuning_param_vars[j],
426 cfg->tuning_params[j]);
427 }
428 opts[i++] = msgprintf("-DLOCKSTEP_WIDTH=%zu", ctx->lockstep_width);
429 opts[i++] = msgprintf("-DMAX_THREADS_PER_BLOCK=%zu", ctx->max_thread_block_size);
430
431 // Time for the best lines of the code in the entire compiler.
432 if (getenv("CUDA_HOME") != NULL) {
433 opts[i++] = msgprintf("-I%s/include", getenv("CUDA_HOME"));
434 }
435 if (getenv("CUDA_ROOT") != NULL) {
436 opts[i++] = msgprintf("-I%s/include", getenv("CUDA_ROOT"));
437 }
438 if (getenv("CUDA_PATH") != NULL) {
439 opts[i++] = msgprintf("-I%s/include", getenv("CUDA_PATH"));
440 }
441 opts[i++] = msgprintf("-I/usr/local/cuda/include");
442 opts[i++] = msgprintf("-I/usr/include");
443
444 for (int j = 0; extra_opts[j] != NULL; j++) {
445 opts[i++] = strdup(extra_opts[j]);
446 }
447
448 opts[i++] = msgprintf("-DTR_BLOCK_DIM=%d", TR_BLOCK_DIM);
449 opts[i++] = msgprintf("-DTR_TILE_DIM=%d", TR_TILE_DIM);
450 opts[i++] = msgprintf("-DTR_ELEMS_PER_THREAD=%d", TR_ELEMS_PER_THREAD);
451
452 free(macro_names);
453 free(macro_vals);
454
455 *n_opts = i;
456 *opts_out = opts;
457 }
458
459 static char* cuda_nvrtc_build(const char *src, const char *opts[], size_t n_opts,
460 char **ptx) {
461 nvrtcProgram prog;
462 char *problem = NULL;
463
464 problem = NVRTC_SUCCEED_NONFATAL(nvrtcCreateProgram(&prog, src, "futhark-cuda", 0, NULL, NULL));
465
466 if (problem) {
467 return problem;
468 }
469
470 nvrtcResult res = nvrtcCompileProgram(prog, n_opts, opts);
471 if (res != NVRTC_SUCCESS) {
472 size_t log_size;
473 if (nvrtcGetProgramLogSize(prog, &log_size) == NVRTC_SUCCESS) {
474 char *log = (char*) malloc(log_size);
475 if (nvrtcGetProgramLog(prog, log) == NVRTC_SUCCESS) {
476 problem = msgprintf("NVRTC compilation failed.\n\n%s\n", log);
477 } else {
478 problem = msgprintf("Could not retrieve compilation log\n");
479 }
480 free(log);
481 }
482 return problem;
483 }
484
485 size_t ptx_size;
486 NVRTC_SUCCEED_FATAL(nvrtcGetPTXSize(prog, &ptx_size));
487 *ptx = (char*) malloc(ptx_size);
488 NVRTC_SUCCEED_FATAL(nvrtcGetPTX(prog, *ptx));
489
490 NVRTC_SUCCEED_FATAL(nvrtcDestroyProgram(&prog));
491
492 return NULL;
493 }
494
495 static void cuda_load_ptx_from_cache(struct futhark_context_config *cfg,
496 const char *src,
497 const char *opts[], size_t n_opts,
498 struct cache_hash *h, const char *cache_fname,
499 char **ptx) {
500 if (cfg->logging) {
501 fprintf(stderr, "Restoring cache from from %s...\n", cache_fname);
502 }
503 cache_hash_init(h);
504 for (size_t i = 0; i < n_opts; i++) {
505 cache_hash(h, opts[i], strlen(opts[i]));
506 }
507 cache_hash(h, src, strlen(src));
508 size_t ptxsize;
509 errno = 0;
510 if (cache_restore(cache_fname, h, (unsigned char**)ptx, &ptxsize) != 0) {
511 if (cfg->logging) {
512 fprintf(stderr, "Failed to restore cache (errno: %s)\n", strerror(errno));
513 }
514 }
515 }
516
517 static void cuda_size_setup(struct futhark_context *ctx)
518 {
519 struct futhark_context_config *cfg = ctx->cfg;
520 if (cfg->gpu.default_block_size > ctx->max_thread_block_size) {
521 if (cfg->gpu.default_block_size_changed) {
522 fprintf(stderr,
523 "Note: Device limits default block size to %zu (down from %zu).\n",
524 ctx->max_thread_block_size, cfg->gpu.default_block_size);
525 }
526 cfg->gpu.default_block_size = ctx->max_thread_block_size;
527 }
528 if (cfg->gpu.default_grid_size > ctx->max_grid_size) {
529 if (cfg->gpu.default_grid_size_changed) {
530 fprintf(stderr,
531 "Note: Device limits default grid size to %zu (down from %zu).\n",
532 ctx->max_grid_size, cfg->gpu.default_grid_size);
533 }
534 cfg->gpu.default_grid_size = ctx->max_grid_size;
535 }
536 if (cfg->gpu.default_tile_size > ctx->max_tile_size) {
537 if (cfg->gpu.default_tile_size_changed) {
538 fprintf(stderr,
539 "Note: Device limits default tile size to %zu (down from %zu).\n",
540 ctx->max_tile_size, cfg->gpu.default_tile_size);
541 }
542 cfg->gpu.default_tile_size = ctx->max_tile_size;
543 }
544
545 if (!cfg->gpu.default_grid_size_changed) {
546 cfg->gpu.default_grid_size =
547 (device_query(ctx->dev, MULTIPROCESSOR_COUNT) *
548 device_query(ctx->dev, MAX_THREADS_PER_MULTIPROCESSOR))
549 / cfg->gpu.default_block_size;
550 }
551
552 for (int i = 0; i < cfg->num_tuning_params; i++) {
553 const char *size_class = cfg->tuning_param_classes[i];
554 int64_t *size_value = &cfg->tuning_params[i];
555 const char* size_name = cfg->tuning_param_names[i];
556 int64_t max_value = 0, default_value = 0;
557
558 if (strstr(size_class, "thread_block_size") == size_class) {
559 max_value = ctx->max_thread_block_size;
560 default_value = cfg->gpu.default_block_size;
561 } else if (strstr(size_class, "grid_size") == size_class) {
562 max_value = ctx->max_grid_size;
563 default_value = cfg->gpu.default_grid_size;
564 // XXX: as a quick and dirty hack, use twice as many threads for
565 // histograms by default. We really should just be smarter
566 // about sizes somehow.
567 if (strstr(size_name, ".seghist_") != NULL) {
568 default_value *= 2;
569 }
570 } else if (strstr(size_class, "tile_size") == size_class) {
571 max_value = ctx->max_tile_size;
572 default_value = cfg->gpu.default_tile_size;
573 } else if (strstr(size_class, "reg_tile_size") == size_class) {
574 max_value = 0; // No limit.
575 default_value = cfg->gpu.default_reg_tile_size;
576 } else if (strstr(size_class, "shared_memory") == size_class) {
577 max_value = ctx->max_shared_memory;
578 default_value = ctx->max_shared_memory;
579 } else if (strstr(size_class, "cache") == size_class) {
580 max_value = ctx->max_cache;
581 default_value = ctx->max_cache;
582 } else if (strstr(size_class, "threshold") == size_class) {
583 // Threshold can be as large as it takes.
584 default_value = cfg->gpu.default_threshold;
585 } else {
586 // Bespoke sizes have no limit or default.
587 }
588
589 if (*size_value == 0) {
590 *size_value = default_value;
591 } else if (max_value > 0 && *size_value > max_value) {
592 fprintf(stderr, "Note: Device limits %s to %zu (down from %zu)\n",
593 size_name, max_value, *size_value);
594 *size_value = max_value;
595 }
596 }
597 }
598
599 static char* cuda_module_setup(struct futhark_context *ctx,
600 const char *src,
601 const char *extra_opts[],
602 const char* cache_fname) {
603 char *ptx = NULL;
604 struct futhark_context_config *cfg = ctx->cfg;
605
606 if (cfg->load_ptx_from) {
607 ptx = slurp_file(cfg->load_ptx_from, NULL);
608 }
609
610 char **opts;
611 size_t n_opts;
612 cuda_nvrtc_mk_build_options(ctx, extra_opts, &opts, &n_opts);
613
614 if (cfg->logging) {
615 fprintf(stderr, "NVRTC compile options:\n");
616 for (size_t j = 0; j < n_opts; j++) {
617 fprintf(stderr, "\t%s\n", opts[j]);
618 }
619 fprintf(stderr, "\n");
620 }
621
622 struct cache_hash h;
623 int loaded_ptx_from_cache = 0;
624 if (cache_fname != NULL) {
625 cuda_load_ptx_from_cache(cfg, src, (const char**)opts, n_opts, &h, cache_fname, &ptx);
626
627 if (ptx != NULL) {
628 if (cfg->logging) {
629 fprintf(stderr, "Restored PTX from cache; now loading module...\n");
630 }
631 if (cuModuleLoadData(&ctx->module, ptx) == CUDA_SUCCESS) {
632 if (cfg->logging) {
633 fprintf(stderr, "Success!\n");
634 }
635 loaded_ptx_from_cache = 1;
636 } else {
637 if (cfg->logging) {
638 fprintf(stderr, "Failed!\n");
639 }
640 free(ptx);
641 ptx = NULL;
642 }
643 }
644 }
645
646 if (ptx == NULL) {
647 char* problem = cuda_nvrtc_build(src, (const char**)opts, n_opts, &ptx);
648 if (problem != NULL) {
649 return problem;
650 }
651 }
652
653 if (cfg->dump_ptx_to != NULL) {
654 dump_file(cfg->dump_ptx_to, ptx, strlen(ptx));
655 }
656
657 if (!loaded_ptx_from_cache) {
658 CUDA_SUCCEED_FATAL(cuModuleLoadData(&ctx->module, ptx));
659 }
660
661 if (cache_fname != NULL && !loaded_ptx_from_cache) {
662 if (cfg->logging) {
663 fprintf(stderr, "Caching PTX in %s...\n", cache_fname);
664 }
665 errno = 0;
666 if (cache_store(cache_fname, &h, (const unsigned char*)ptx, strlen(ptx)) != 0) {
667 fprintf(stderr, "Failed to cache PTX: %s\n", strerror(errno));
668 }
669 }
670
671 for (size_t i = 0; i < n_opts; i++) {
672 free((char *)opts[i]);
673 }
674 free(opts);
675 free(ptx);
676
677 return NULL;
678 }
679
680 struct cuda_event {
681 cudaEvent_t start;
682 cudaEvent_t end;
683 };
684
685 static struct cuda_event* cuda_event_new(struct futhark_context* ctx) {
686 if (ctx->profiling && !ctx->profiling_paused) {
687 struct cuda_event* e = malloc(sizeof(struct cuda_event));
688 cudaEventCreate(&e->start);
689 cudaEventCreate(&e->end);
690 return e;
691 } else {
692 return NULL;
693 }
694 }
695
696 static int cuda_event_report(struct str_builder* sb, struct cuda_event* e) {
697 float ms;
698 CUresult err;
699 if ((err = cuEventElapsedTime(&ms, e->start, e->end)) != CUDA_SUCCESS) {
700 return err;
701 }
702
703 // CUDA provides milisecond resolution, but we want microseconds.
704 str_builder(sb, ",\"duration\":%f", ms*1000);
705
706 if ((err = cuEventDestroy(e->start)) != CUDA_SUCCESS) {
707 return 1;
708 }
709
710 if ((err = cuEventDestroy(e->end)) != CUDA_SUCCESS) {
711 return 1;
712 }
713
714 free(e);
715
716 return 0;
717 }
718
719 int futhark_context_sync(struct futhark_context* ctx) {
720 CUDA_SUCCEED_OR_RETURN(cuCtxPushCurrent(ctx->cu_ctx));
721 CUDA_SUCCEED_OR_RETURN(cuCtxSynchronize());
722 if (ctx->failure_is_an_option) {
723 // Check for any delayed error.
724 int32_t failure_idx;
725 CUDA_SUCCEED_OR_RETURN(
726 cuMemcpyDtoH(&failure_idx,
727 ctx->global_failure,
728 sizeof(int32_t)));
729 ctx->failure_is_an_option = 0;
730
731 if (failure_idx >= 0) {
732 // We have to clear global_failure so that the next entry point
733 // is not considered a failure from the start.
734 int32_t no_failure = -1;
735 CUDA_SUCCEED_OR_RETURN(
736 cuMemcpyHtoD(ctx->global_failure,
737 &no_failure,
738 sizeof(int32_t)));
739
740 int64_t args[max_failure_args+1];
741 CUDA_SUCCEED_OR_RETURN(
742 cuMemcpyDtoH(&args,
743 ctx->global_failure_args,
744 sizeof(args)));
745
746 ctx->error = get_failure_msg(failure_idx, args);
747
748 return FUTHARK_PROGRAM_ERROR;
749 }
750 }
751
752 CUDA_SUCCEED_OR_RETURN(cuCtxPopCurrent(&ctx->cu_ctx));
753 return 0;
754 }
755
756 struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);
757 void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);
758
759 int backend_context_setup(struct futhark_context* ctx) {
760 ctx->failure_is_an_option = 0;
761 ctx->total_runs = 0;
762 ctx->total_runtime = 0;
763 ctx->peak_mem_usage_device = 0;
764 ctx->cur_mem_usage_device = 0;
765 ctx->kernels = NULL;
766
767 CUDA_SUCCEED_FATAL(cuInit(0));
768 if (cuda_device_setup(ctx) != 0) {
769 futhark_panic(-1, "No suitable CUDA device found.\n");
770 }
771 CUDA_SUCCEED_FATAL(cuCtxCreate(&ctx->cu_ctx, 0, ctx->dev));
772
773 free_list_init(&ctx->gpu_free_list);
774
775 if (ctx->cfg->unified_memory == 2) {
776 ctx->cfg->unified_memory = device_query(ctx->dev, MANAGED_MEMORY);
777 }
778
779 if (ctx->cfg->logging) {
780 if (ctx->cfg->unified_memory) {
781 fprintf(ctx->log, "Using managed memory\n");
782 } else {
783 fprintf(ctx->log, "Using unmanaged memory\n");
784 }
785 }
786
787 ctx->max_thread_block_size = device_query(ctx->dev, MAX_THREADS_PER_BLOCK);
788 ctx->max_grid_size = device_query(ctx->dev, MAX_GRID_DIM_X);
789 ctx->max_tile_size = sqrt(ctx->max_thread_block_size);
790 ctx->max_threshold = 1U<<31; // No limit.
791 ctx->max_bespoke = 1U<<31; // No limit.
792
793 if (ctx->cfg->gpu.default_registers != 0) {
794 ctx->max_registers = ctx->cfg->gpu.default_registers;
795 } else {
796 ctx->max_registers = device_query(ctx->dev, MAX_REGISTERS_PER_BLOCK);
797 }
798
799 if (ctx->cfg->gpu.default_shared_memory != 0) {
800 ctx->max_shared_memory = ctx->cfg->gpu.default_shared_memory;
801 } else {
802 // MAX_SHARED_MEMORY_PER_BLOCK gives bogus numbers (48KiB); probably
803 // for backwards compatibility. Add _OPTIN and you seem to get the
804 // right number.
805 ctx->max_shared_memory = device_query(ctx->dev, MAX_SHARED_MEMORY_PER_BLOCK_OPTIN);
806 #if CUDART_VERSION >= 12000
807 ctx->max_shared_memory -= device_query(ctx->dev, RESERVED_SHARED_MEMORY_PER_BLOCK);
808 #endif
809 }
810
811 if (ctx->cfg->gpu.default_cache != 0) {
812 ctx->max_cache = ctx->cfg->gpu.default_cache;
813 } else {
814 ctx->max_cache = device_query(ctx->dev, L2_CACHE_SIZE);
815 }
816
817 ctx->lockstep_width = device_query(ctx->dev, WARP_SIZE);
818 CUDA_SUCCEED_FATAL(cuStreamCreate(&ctx->stream, CU_STREAM_DEFAULT));
819 cuda_size_setup(ctx);
820
821 gpu_init_log(ctx);
822
823 ctx->error = cuda_module_setup(ctx,
824 ctx->cfg->program,
825 (const char**)ctx->cfg->nvrtc_opts,
826 ctx->cfg->cache_fname);
827
828 if (ctx->error != NULL) {
829 futhark_panic(1, "During CUDA initialisation:\n%s\n", ctx->error);
830 }
831
832 int32_t no_error = -1;
833 CUDA_SUCCEED_FATAL(cuMemAlloc(&ctx->global_failure, sizeof(no_error)));
834 CUDA_SUCCEED_FATAL(cuMemcpyHtoD(ctx->global_failure, &no_error, sizeof(no_error)));
835 // The +1 is to avoid zero-byte allocations.
836 CUDA_SUCCEED_FATAL(cuMemAlloc(&ctx->global_failure_args, sizeof(int64_t)*(max_failure_args+1)));
837
838 if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) {
839 return 1;
840 }
841
842 return 0;
843 }
844
845 void backend_context_teardown(struct futhark_context* ctx) {
846 if (ctx->kernels != NULL) {
847 free_builtin_kernels(ctx, ctx->kernels);
848 cuMemFree(ctx->global_failure);
849 cuMemFree(ctx->global_failure_args);
850 CUDA_SUCCEED_FATAL(gpu_free_all(ctx));
851 CUDA_SUCCEED_FATAL(cuStreamDestroy(ctx->stream));
852 CUDA_SUCCEED_FATAL(cuModuleUnload(ctx->module));
853 CUDA_SUCCEED_FATAL(cuCtxDestroy(ctx->cu_ctx));
854 }
855 free_list_destroy(&ctx->gpu_free_list);
856 }
857
858 // GPU ABSTRACTION LAYER
859
860 // Types.
861
862 typedef CUfunction gpu_kernel;
863 typedef CUdeviceptr gpu_mem;
864
865 static void gpu_create_kernel(struct futhark_context *ctx,
866 gpu_kernel* kernel,
867 const char* name) {
868 if (ctx->debugging) {
869 fprintf(ctx->log, "Creating kernel %s.\n", name);
870 }
871 CUDA_SUCCEED_FATAL(cuModuleGetFunction(kernel, ctx->module, name));
872 // Unless the below is set, the kernel is limited to 48KiB of memory.
873 CUDA_SUCCEED_FATAL(cuFuncSetAttribute(*kernel,
874 cudaFuncAttributeMaxDynamicSharedMemorySize,
875 ctx->max_shared_memory));
876 }
877
878 static void gpu_free_kernel(struct futhark_context *ctx,
879 gpu_kernel kernel) {
880 (void)ctx;
881 (void)kernel;
882 }
883
884 static int gpu_scalar_to_device(struct futhark_context* ctx,
885 const char *provenance,
886 gpu_mem dst, size_t offset, size_t size,
887 void *src) {
888 struct cuda_event *event = cuda_event_new(ctx);
889 if (event != NULL) {
890 add_event(ctx,
891 "copy_scalar_to_dev",
892 provenance,
893 NULL,
894 event,
895 (event_report_fn)cuda_event_report);
896 CUDA_SUCCEED_FATAL(cuEventRecord(event->start, ctx->stream));
897 }
898 CUDA_SUCCEED_OR_RETURN(cuMemcpyHtoD(dst + offset, src, size));
899 if (event != NULL) {
900 CUDA_SUCCEED_FATAL(cuEventRecord(event->end, ctx->stream));
901 }
902 return FUTHARK_SUCCESS;
903 }
904
905 static int gpu_scalar_from_device(struct futhark_context* ctx,
906 const char *provenance,
907 void *dst,
908 gpu_mem src, size_t offset, size_t size) {
909 struct cuda_event *event = cuda_event_new(ctx);
910 if (event != NULL) {
911 add_event(ctx,
912 "copy_scalar_from_dev",
913 provenance,
914 NULL,
915 event,
916 (event_report_fn)cuda_event_report);
917 CUDA_SUCCEED_FATAL(cuEventRecord(event->start, ctx->stream));
918 }
919 CUDA_SUCCEED_OR_RETURN(cuMemcpyDtoH(dst, src + offset, size));
920 if (event != NULL) {
921 CUDA_SUCCEED_FATAL(cuEventRecord(event->end, ctx->stream));
922 }
923 return FUTHARK_SUCCESS;
924 }
925
926 static int gpu_memcpy(struct futhark_context* ctx,
927 const char *provenance,
928 gpu_mem dst, int64_t dst_offset,
929 gpu_mem src, int64_t src_offset,
930 int64_t nbytes) {
931 struct cuda_event *event = cuda_event_new(ctx);
932 if (event != NULL) {
933 add_event(ctx,
934 "copy_dev_to_dev",
935 provenance,
936 NULL,
937 event,
938 (event_report_fn)cuda_event_report);
939 CUDA_SUCCEED_FATAL(cuEventRecord(event->start, ctx->stream));
940 }
941 CUDA_SUCCEED_OR_RETURN(cuMemcpyAsync(dst+dst_offset, src+src_offset, nbytes, ctx->stream));
942 if (event != NULL) {
943 CUDA_SUCCEED_FATAL(cuEventRecord(event->end, ctx->stream));
944 }
945 return FUTHARK_SUCCESS;
946 }
947
948 static int memcpy_host2gpu(struct futhark_context* ctx,
949 const char *provenance,
950 bool sync,
951 gpu_mem dst, int64_t dst_offset,
952 const unsigned char* src, int64_t src_offset,
953 int64_t nbytes) {
954 if (nbytes > 0) {
955 struct cuda_event *event = cuda_event_new(ctx);
956 if (event != NULL) {
957 add_event(ctx,
958 "copy_host_to_dev",
959 provenance,
960 NULL,
961 event,
962 (event_report_fn)cuda_event_report);
963 CUDA_SUCCEED_FATAL(cuEventRecord(event->start, ctx->stream));
964 }
965 if (sync) {
966 CUDA_SUCCEED_OR_RETURN
967 (cuMemcpyHtoD(dst + dst_offset, src + src_offset, nbytes));
968 } else {
969 CUDA_SUCCEED_OR_RETURN
970 (cuMemcpyHtoDAsync(dst + dst_offset, src + src_offset, nbytes, ctx->stream));
971 }
972 if (event != NULL) {
973 CUDA_SUCCEED_FATAL(cuEventRecord(event->end, ctx->stream));
974 }
975 }
976 return FUTHARK_SUCCESS;
977 }
978
979 static int memcpy_gpu2host(struct futhark_context* ctx,
980 const char *provenance,
981 bool sync,
982 unsigned char* dst, int64_t dst_offset,
983 gpu_mem src, int64_t src_offset,
984 int64_t nbytes) {
985 if (nbytes > 0) {
986 struct cuda_event *event = cuda_event_new(ctx);
987 if (event != NULL) {
988 add_event(ctx,
989 "copy_dev_to_host",
990 provenance,
991 NULL,
992 event,
993 (event_report_fn)cuda_event_report);
994 CUDA_SUCCEED_FATAL(cuEventRecord(event->start, ctx->stream));
995 }
996 if (sync) {
997 CUDA_SUCCEED_OR_RETURN
998 (cuMemcpyDtoH(dst + dst_offset, src + src_offset, nbytes));
999 } else {
1000 CUDA_SUCCEED_OR_RETURN
1001 (cuMemcpyDtoHAsync(dst + dst_offset, src + src_offset, nbytes, ctx->stream));
1002 }
1003 if (event != NULL) {
1004 CUDA_SUCCEED_FATAL(cuEventRecord(event->end, ctx->stream));
1005 }
1006 if (sync &&
1007 ctx->failure_is_an_option &&
1008 futhark_context_sync(ctx) != 0) {
1009 return 1;
1010 }
1011 }
1012 return FUTHARK_SUCCESS;
1013 }
1014
1015 static int gpu_launch_kernel(struct futhark_context* ctx,
1016 gpu_kernel kernel,
1017 const char *name, const char *provenance,
1018 const int32_t grid[3],
1019 const int32_t block[3],
1020 unsigned int shared_mem_bytes,
1021 int num_args,
1022 void* args[num_args],
1023 size_t args_sizes[num_args]) {
1024 (void) args_sizes;
1025
1026 if (shared_mem_bytes > ctx->max_shared_memory) {
1027 set_error(ctx, msgprintf("Kernel %s with %d bytes of memory exceeds device limit of %d\n",
1028 name, shared_mem_bytes, (int)ctx->max_shared_memory));
1029 return 1;
1030 }
1031
1032 int64_t time_start = 0, time_end = 0;
1033 if (ctx->debugging) {
1034 time_start = get_wall_time();
1035 }
1036
1037 struct cuda_event *event = cuda_event_new(ctx);
1038
1039 if (event != NULL) {
1040 CUDA_SUCCEED_FATAL(cuEventRecord(event->start, ctx->stream));
1041
1042 struct kvs *kvs = kvs_new();
1043 kvs_printf(kvs, "kernel", "\"%s\"", name);
1044 kvs_printf(kvs, "grid", "[%d,%d,%d]", grid[0], grid[1], grid[2]);
1045 kvs_printf(kvs, "block", "[%d,%d,%d]", block[0], block[1], block[2]);
1046 kvs_printf(kvs, "shared memory", "%d", shared_mem_bytes);
1047
1048 add_event(ctx,
1049 name,
1050 provenance,
1051 kvs,
1052 event,
1053 (event_report_fn)cuda_event_report);
1054 }
1055
1056 CUDA_SUCCEED_OR_RETURN
1057 (cuLaunchKernel(kernel,
1058 grid[0], grid[1], grid[2],
1059 block[0], block[1], block[2],
1060 shared_mem_bytes, ctx->stream,
1061 args, NULL));
1062
1063 if (event != NULL) {
1064 CUDA_SUCCEED_FATAL(cuEventRecord(event->end, ctx->stream));
1065 }
1066
1067 if (ctx->debugging) {
1068 CUDA_SUCCEED_FATAL(cuCtxSynchronize());
1069 time_end = get_wall_time();
1070 long int time_diff = time_end - time_start;
1071 fprintf(ctx->log, " runtime: %ldus\n", time_diff);
1072 }
1073 if (ctx->logging) {
1074 fprintf(ctx->log, "\n");
1075 }
1076
1077 return FUTHARK_SUCCESS;
1078 }
1079
1080 static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) {
1081 CUresult res;
1082 if (ctx->cfg->unified_memory) {
1083 res = cuMemAllocManaged(mem_out, size, CU_MEM_ATTACH_GLOBAL);
1084 } else {
1085 res = cuMemAlloc(mem_out, size);
1086 }
1087
1088 if (res == CUDA_ERROR_OUT_OF_MEMORY) {
1089 return FUTHARK_OUT_OF_MEMORY;
1090 }
1091
1092 CUDA_SUCCEED_OR_RETURN(res);
1093 return FUTHARK_SUCCESS;
1094 }
1095
1096 static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) {
1097 (void)ctx;
1098 CUDA_SUCCEED_OR_RETURN(cuMemFree(mem));
1099 return FUTHARK_SUCCESS;
1100 }
1101
1102 // End of backends/cuda.h.