Debian Codegraph @

rts/c/backends/hip.h

raw
1 // Start of backends/hip.h.
2
3 // Forward declarations.
4 // Invoked by setup_opencl() after the platform and device has been
5 // found, but before the program is loaded. Its intended use is to
6 // tune constants based on the selected platform and device.
7 static void set_tuning_params(struct futhark_context* ctx);
8 static char* get_failure_msg(int failure_idx, int64_t args[]);
9
10 #define HIP_SUCCEED_FATAL(x) hip_api_succeed_fatal(x, #x, __FILE__, __LINE__)
11 #define HIP_SUCCEED_NONFATAL(x) hip_api_succeed_nonfatal(x, #x, __FILE__, __LINE__)
12 #define HIPRTC_SUCCEED_FATAL(x) hiprtc_api_succeed_fatal(x, #x, __FILE__, __LINE__)
13 #define HIPRTC_SUCCEED_NONFATAL(x) hiprtc_api_succeed_nonfatal(x, #x, __FILE__, __LINE__)
14 // Take care not to override an existing error.
15 #define HIP_SUCCEED_OR_RETURN(e) { \
16 char *serror = HIP_SUCCEED_NONFATAL(e); \
17 if (serror) { \
18 if (!ctx->error) { \
19 ctx->error = serror; \
20 } else { \
21 free(serror); \
22 } \
23 return bad; \
24 } \
25 }
26
27 // HIP_SUCCEED_OR_RETURN returns the value of the variable 'bad' in
28 // scope. By default, it will be this one. Create a local variable
29 // of some other type if needed. This is a bit of a hack, but it
30 // saves effort in the code generator.
31 static const int bad = 1;
32
33 static inline void hip_api_succeed_fatal(hipError_t res, const char *call,
34 const char *file, int line) {
35 if (res != hipSuccess) {
36 const char *err_str = hipGetErrorString(res);
37 if (err_str == NULL) { err_str = "Unknown"; }
38 futhark_panic(-1, "%s:%d: HIP call\n %s\nfailed with error code %d (%s)\n",
39 file, line, call, res, err_str);
40 }
41 }
42
43 static char* hip_api_succeed_nonfatal(hipError_t res, const char *call,
44 const char *file, int line) {
45 if (res != hipSuccess) {
46 const char *err_str = hipGetErrorString(res);
47 if (err_str == NULL) { err_str = "Unknown"; }
48 return msgprintf("%s:%d: HIP call\n %s\nfailed with error code %d (%s)\n",
49 file, line, call, res, err_str);
50 } else {
51 return NULL;
52 }
53 }
54
55 static inline void hiprtc_api_succeed_fatal(hiprtcResult res, const char *call,
56 const char *file, int line) {
57 if (res != HIPRTC_SUCCESS) {
58 const char *err_str = hiprtcGetErrorString(res);
59 futhark_panic(-1, "%s:%d: HIPRTC call\n %s\nfailed with error code %d (%s)\n",
60 file, line, call, res, err_str);
61 }
62 }
63
64 static char* hiprtc_api_succeed_nonfatal(hiprtcResult res, const char *call,
65 const char *file, int line) {
66 if (res != HIPRTC_SUCCESS) {
67 const char *err_str = hiprtcGetErrorString(res);
68 return msgprintf("%s:%d: HIPRTC call\n %s\nfailed with error code %d (%s)\n",
69 file, line, call, res, err_str);
70 } else {
71 return NULL;
72 }
73 }
74
75 struct futhark_context_config {
76 int in_use;
77 int debugging;
78 int profiling;
79 int logging;
80 char* cache_fname;
81 int num_tuning_params;
82 int64_t *tuning_params;
83 const char** tuning_param_names;
84 const char** tuning_param_vars;
85 const char** tuning_param_classes;
86 // Uniform fields above.
87
88 char* program;
89 int num_build_opts;
90 char* *build_opts;
91
92 int unified_memory;
93
94 char* preferred_device;
95 int preferred_device_num;
96
97 struct gpu_config gpu;
98 };
99
100 static void backend_context_config_setup(struct futhark_context_config *cfg) {
101 cfg->num_build_opts = 0;
102 cfg->build_opts = (char**) malloc(sizeof(char*));
103 cfg->build_opts[0] = NULL;
104 cfg->preferred_device_num = 0;
105 cfg->preferred_device = strdup("");
106 cfg->program = strconcat(gpu_program);
107
108 cfg->unified_memory = 0;
109
110 cfg->gpu = gpu_config_initial;
111 cfg->gpu.default_block_size = 256;
112 cfg->gpu.default_tile_size = 32;
113 cfg->gpu.default_reg_tile_size = 2;
114 cfg->gpu.default_threshold = 32*1024;
115 }
116
117 static void backend_context_config_teardown(struct futhark_context_config* cfg) {
118 for (int i = 0; i < cfg->num_build_opts; i++) {
119 free(cfg->build_opts[i]);
120 }
121
122 free(cfg->build_opts);
123 free(cfg->preferred_device);
124 free(cfg->program);
125 }
126
127 void futhark_context_config_add_build_option(struct futhark_context_config *cfg, const char *opt) {
128 cfg->build_opts[cfg->num_build_opts] = strdup(opt);
129 cfg->num_build_opts++;
130 cfg->build_opts = (char **) realloc(cfg->build_opts, (cfg->num_build_opts + 1) * sizeof(char *));
131 cfg->build_opts[cfg->num_build_opts] = NULL;
132 }
133
134 void futhark_context_config_set_device(struct futhark_context_config *cfg, const char *s) {
135 int x = 0;
136 if (*s == '#') {
137 s++;
138 while (isdigit(*s)) {
139 x = x * 10 + (*s++)-'0';
140 }
141 // Skip trailing spaces.
142 while (isspace(*s)) {
143 s++;
144 }
145 }
146 free(cfg->preferred_device);
147 cfg->preferred_device = strdup(s);
148 cfg->preferred_device_num = x;
149 }
150
151
152 const char* futhark_context_config_get_program(struct futhark_context_config *cfg) {
153 return cfg->program;
154 }
155
156 void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) {
157 free(cfg->program);
158 cfg->program = strdup(s);
159 }
160
161 void futhark_context_config_set_unified_memory(struct futhark_context_config* cfg, int flag) {
162 cfg->unified_memory = flag;
163 }
164
165 struct futhark_context {
166 struct futhark_context_config* cfg;
167 int detail_memory;
168 int debugging;
169 int profiling;
170 int profiling_paused;
171 int logging;
172 lock_t lock;
173 char *error;
174 lock_t error_lock;
175 FILE *log;
176 struct constants *constants;
177 struct free_list free_list;
178 struct event_list event_list;
179 int64_t peak_mem_usage_default;
180 int64_t cur_mem_usage_default;
181 bool program_initialised;
182 // Uniform fields above.
183
184 void* global_failure;
185 void* global_failure_args;
186 struct tuning_params tuning_params;
187 // True if a potentially failing kernel has been enqueued.
188 int32_t failure_is_an_option;
189 int total_runs;
190 long int total_runtime;
191 int64_t peak_mem_usage_device;
192 int64_t cur_mem_usage_device;
193 struct program* program;
194
195 hipDevice_t dev;
196 int dev_id;
197 hipModule_t module;
198 hipStream_t stream;
199
200 struct free_list gpu_free_list;
201
202 size_t max_thread_block_size;
203 size_t max_grid_size;
204 size_t max_tile_size;
205 size_t max_threshold;
206 size_t max_shared_memory;
207 size_t max_bespoke;
208 size_t max_registers;
209 size_t max_cache;
210
211 size_t lockstep_width;
212
213 struct builtin_kernels* kernels;
214 };
215
216 static int device_query(int dev_id, hipDeviceAttribute_t attr) {
217 int val;
218 HIP_SUCCEED_FATAL(hipDeviceGetAttribute(&val, attr, dev_id));
219 return val;
220 }
221
222 static int function_query(hipFunction_t f, hipFunction_attribute attr) {
223 int val;
224 HIP_SUCCEED_FATAL(hipFuncGetAttribute(&val, attr, f));
225 return val;
226 }
227
228 static int hip_device_setup(struct futhark_context *ctx) {
229 struct futhark_context_config *cfg = ctx->cfg;
230 int count, chosen = -1;
231 hipDevice_t dev;
232
233 HIP_SUCCEED_FATAL(hipGetDeviceCount(&count));
234 if (count == 0) { return 1; }
235
236 int num_device_matches = 0;
237
238 for (int i = 0; i < count; i++) {
239 hipDeviceProp_t prop;
240 hipGetDeviceProperties(&prop, i);
241
242 if (cfg->logging) {
243 fprintf(ctx->log, "Device #%d: name=\"%s\"\n", i, prop.name);
244 }
245
246 if (strstr(prop.name, cfg->preferred_device) != NULL &&
247 num_device_matches++ == cfg->preferred_device_num) {
248 chosen = i;
249 break;
250 }
251 }
252
253 if (chosen == -1) { return 1; }
254
255 if (cfg->logging) {
256 fprintf(ctx->log, "Using device #%d\n", chosen);
257 }
258
259 ctx->dev_id = chosen;
260 HIP_SUCCEED_FATAL(hipDeviceGet(&ctx->dev, ctx->dev_id));
261 return 0;
262 }
263
264 static void hip_load_code_from_cache(struct futhark_context_config *cfg,
265 const char *src,
266 const char *opts[], size_t n_opts,
267 struct cache_hash *h, const char *cache_fname,
268 char **code, size_t *code_size) {
269 if (cfg->logging) {
270 fprintf(stderr, "Restoring cache from from %s...\n", cache_fname);
271 }
272 cache_hash_init(h);
273 for (size_t i = 0; i < n_opts; i++) {
274 cache_hash(h, opts[i], strlen(opts[i]));
275 }
276 cache_hash(h, src, strlen(src));
277 errno = 0;
278 if (cache_restore(cache_fname, h, (unsigned char**)code, code_size) != 0) {
279 if (cfg->logging) {
280 fprintf(stderr, "Failed to restore cache (errno: %s)\n", strerror(errno));
281 }
282 }
283 }
284
285 static void hip_size_setup(struct futhark_context *ctx) {
286 struct futhark_context_config *cfg = ctx->cfg;
287 if (cfg->gpu.default_block_size > ctx->max_thread_block_size) {
288 if (cfg->gpu.default_block_size_changed) {
289 fprintf(stderr,
290 "Note: Device limits default block size to %zu (down from %zu).\n",
291 ctx->max_thread_block_size, cfg->gpu.default_block_size);
292 }
293 cfg->gpu.default_block_size = ctx->max_thread_block_size;
294 }
295 if (cfg->gpu.default_grid_size > ctx->max_grid_size) {
296 if (cfg->gpu.default_grid_size_changed) {
297 fprintf(stderr,
298 "Note: Device limits default grid size to %zu (down from %zu).\n",
299 ctx->max_grid_size, cfg->gpu.default_grid_size);
300 }
301 cfg->gpu.default_grid_size = ctx->max_grid_size;
302 }
303 if (cfg->gpu.default_tile_size > ctx->max_tile_size) {
304 if (cfg->gpu.default_tile_size_changed) {
305 fprintf(stderr,
306 "Note: Device limits default tile size to %zu (down from %zu).\n",
307 ctx->max_tile_size, cfg->gpu.default_tile_size);
308 }
309 cfg->gpu.default_tile_size = ctx->max_tile_size;
310 }
311
312 if (!cfg->gpu.default_grid_size_changed) {
313 cfg->gpu.default_grid_size =
314 (device_query(ctx->dev, hipDeviceAttributePhysicalMultiProcessorCount) *
315 device_query(ctx->dev, hipDeviceAttributeMaxThreadsPerMultiProcessor))
316 / cfg->gpu.default_block_size;
317 }
318
319 for (int i = 0; i < cfg->num_tuning_params; i++) {
320 const char *size_class = cfg->tuning_param_classes[i];
321 int64_t *size_value = &cfg->tuning_params[i];
322 const char* size_name = cfg->tuning_param_names[i];
323 int64_t max_value = 0, default_value = 0;
324
325 if (strstr(size_class, "thread_block_size") == size_class) {
326 max_value = ctx->max_thread_block_size;
327 default_value = cfg->gpu.default_block_size;
328 } else if (strstr(size_class, "grid_size") == size_class) {
329 max_value = ctx->max_grid_size;
330 default_value = cfg->gpu.default_grid_size;
331 // XXX: as a quick and dirty hack, use twice as many threads for
332 // histograms by default. We really should just be smarter
333 // about sizes somehow.
334 if (strstr(size_name, ".seghist_") != NULL) {
335 default_value *= 2;
336 }
337 } else if (strstr(size_class, "tile_size") == size_class) {
338 max_value = ctx->max_tile_size;
339 default_value = cfg->gpu.default_tile_size;
340 } else if (strstr(size_class, "reg_tile_size") == size_class) {
341 max_value = 0; // No limit.
342 default_value = cfg->gpu.default_reg_tile_size;
343 } else if (strstr(size_class, "shared_memory") == size_class) {
344 max_value = ctx->max_shared_memory;
345 default_value = ctx->max_shared_memory;
346 } else if (strstr(size_class, "cache") == size_class) {
347 max_value = ctx->max_cache;
348 default_value = ctx->max_cache;
349 } else if (strstr(size_class, "threshold") == size_class) {
350 // Threshold can be as large as it takes.
351 default_value = cfg->gpu.default_threshold;
352 } else {
353 // Bespoke sizes have no limit or default.
354 }
355
356 if (*size_value == 0) {
357 *size_value = default_value;
358 } else if (max_value > 0 && *size_value > max_value) {
359 fprintf(stderr, "Note: Device limits %s to %zu (down from %zu)\n",
360 size_name, max_value, *size_value);
361 *size_value = max_value;
362 }
363 }
364 }
365
366 static char* hiprtc_build(const char *src, const char *opts[], size_t n_opts,
367 char **code, size_t *code_size) {
368 hiprtcProgram prog;
369 char *problem = NULL;
370
371 problem = HIPRTC_SUCCEED_NONFATAL(hiprtcCreateProgram(&prog, src, "futhark-hip", 0, NULL, NULL));
372
373 if (problem) {
374 return problem;
375 }
376
377 hiprtcResult res = hiprtcCompileProgram(prog, n_opts, opts);
378 if (res != HIPRTC_SUCCESS) {
379 size_t log_size;
380 if (hiprtcGetProgramLogSize(prog, &log_size) == HIPRTC_SUCCESS) {
381 char *log = (char*) malloc(log_size+1);
382 log[log_size] = 0; // HIPRTC does not zero-terminate.
383 if (hiprtcGetProgramLog(prog, log) == HIPRTC_SUCCESS) {
384 problem = msgprintf("HIPRTC compilation failed.\n\n%s\n", log);
385 } else {
386 problem = msgprintf("Could not retrieve compilation log\n");
387 }
388 free(log);
389 }
390 return problem;
391 }
392
393 HIPRTC_SUCCEED_FATAL(hiprtcGetCodeSize(prog, code_size));
394 *code = (char*) malloc(*code_size);
395 HIPRTC_SUCCEED_FATAL(hiprtcGetCode(prog, *code));
396 HIPRTC_SUCCEED_FATAL(hiprtcDestroyProgram(&prog));
397 return NULL;
398 }
399
400 static void hiprtc_mk_build_options(struct futhark_context *ctx, const char *extra_opts[],
401 char*** opts_out, size_t *n_opts) {
402 int arch_set = 0, num_extra_opts;
403 struct futhark_context_config *cfg = ctx->cfg;
404
405 char** macro_names;
406 int64_t* macro_vals;
407 int num_macros = gpu_macros(ctx, &macro_names, &macro_vals);
408
409 for (num_extra_opts = 0; extra_opts[num_extra_opts] != NULL; num_extra_opts++) {
410 if (strstr(extra_opts[num_extra_opts], "--gpu-architecture")
411 == extra_opts[num_extra_opts]) {
412 arch_set = 1;
413 }
414 }
415
416 size_t i = 0, n_opts_alloc = 20 + num_macros + num_extra_opts + cfg->num_tuning_params;
417 char **opts = (char**) malloc(n_opts_alloc * sizeof(char *));
418 if (!arch_set) {
419 hipDeviceProp_t props;
420 HIP_SUCCEED_FATAL(hipGetDeviceProperties(&props, ctx->dev_id));
421 opts[i++] = msgprintf("--gpu-architecture=%s", props.gcnArchName);
422 }
423 if (cfg->debugging) {
424 opts[i++] = strdup("-G");
425 opts[i++] = strdup("-lineinfo");
426 }
427 opts[i++] = msgprintf("-D%s=%d",
428 "max_thread_block_size",
429 (int)ctx->max_thread_block_size);
430 opts[i++] = msgprintf("-D%s=%d",
431 "max_shared_memory",
432 (int)ctx->max_shared_memory);
433 opts[i++] = msgprintf("-D%s=%d",
434 "max_registers",
435 (int)ctx->max_registers);
436
437 for (int j = 0; j < num_macros; j++) {
438 opts[i++] = msgprintf("-D%s=%zu", macro_names[j], macro_vals[j]);
439 }
440
441 for (int j = 0; j < cfg->num_tuning_params; j++) {
442 opts[i++] = msgprintf("-D%s=%zu", cfg->tuning_param_vars[j],
443 cfg->tuning_params[j]);
444 }
445 opts[i++] = msgprintf("-DLOCKSTEP_WIDTH=%zu", ctx->lockstep_width);
446 opts[i++] = msgprintf("-DMAX_THREADS_PER_BLOCK=%zu", ctx->max_thread_block_size);
447
448 for (int j = 0; extra_opts[j] != NULL; j++) {
449 opts[i++] = strdup(extra_opts[j]);
450 }
451
452 opts[i++] = msgprintf("-DTR_BLOCK_DIM=%d", TR_BLOCK_DIM);
453 opts[i++] = msgprintf("-DTR_TILE_DIM=%d", TR_TILE_DIM);
454 opts[i++] = msgprintf("-DTR_ELEMS_PER_THREAD=%d", TR_ELEMS_PER_THREAD);
455
456 free(macro_names);
457 free(macro_vals);
458
459 *n_opts = i;
460 *opts_out = opts;
461 }
462
463 static char* hip_module_setup(struct futhark_context *ctx,
464 const char *src,
465 const char *extra_opts[],
466 const char* cache_fname) {
467 char *code = NULL;
468 size_t code_size = 0;
469 struct futhark_context_config *cfg = ctx->cfg;
470
471 char **opts;
472 size_t n_opts;
473 hiprtc_mk_build_options(ctx, extra_opts, &opts, &n_opts);
474
475 if (cfg->logging) {
476 fprintf(stderr, "HIPRTC build options:\n");
477 for (size_t j = 0; j < n_opts; j++) {
478 fprintf(stderr, "\t%s\n", opts[j]);
479 }
480 fprintf(stderr, "\n");
481 }
482
483 struct cache_hash h;
484 int loaded_code_from_cache = 0;
485 if (cache_fname != NULL) {
486 hip_load_code_from_cache(cfg, src, (const char**)opts, n_opts, &h, cache_fname, &code, &code_size);
487
488 if (code != NULL) {
489 if (cfg->logging) {
490 fprintf(stderr, "Restored compiled code from cache; now loading module...\n");
491 }
492 if (hipModuleLoadData(&ctx->module, code) == hipSuccess) {
493 if (cfg->logging) {
494 fprintf(stderr, "Success!\n");
495 }
496 loaded_code_from_cache = 1;
497 } else {
498 if (cfg->logging) {
499 fprintf(stderr, "Failed!\n");
500 }
501 free(code);
502 code = NULL;
503 }
504 }
505 }
506
507 if (code == NULL) {
508 char* problem = hiprtc_build(src, (const char**)opts, n_opts, &code, &code_size);
509 if (problem != NULL) {
510 return problem;
511 }
512 }
513
514 if (!loaded_code_from_cache) {
515 HIP_SUCCEED_FATAL(hipModuleLoadData(&ctx->module, code));
516 }
517
518 if (cache_fname != NULL && !loaded_code_from_cache) {
519 if (cfg->logging) {
520 fprintf(stderr, "Caching compiled code in %s...\n", cache_fname);
521 }
522 errno = 0;
523 if (cache_store(cache_fname, &h, (const unsigned char*)code, code_size) != 0) {
524 fprintf(stderr, "Failed to cache compiled code: %s\n", strerror(errno));
525 }
526 }
527
528 for (size_t i = 0; i < n_opts; i++) {
529 free((char *)opts[i]);
530 }
531 free(opts);
532 free(code);
533
534 return NULL;
535 }
536
537 struct hip_event {
538 hipEvent_t start;
539 hipEvent_t end;
540 };
541
542 static struct hip_event* hip_event_new(struct futhark_context* ctx) {
543 if (ctx->profiling && !ctx->profiling_paused) {
544 struct hip_event* e = malloc(sizeof(struct hip_event));
545 hipEventCreate(&e->start);
546 hipEventCreate(&e->end);
547 return e;
548 } else {
549 return NULL;
550 }
551 }
552
553 static int hip_event_report(struct str_builder* sb, struct hip_event* e) {
554 float ms;
555 hipError_t err;
556 if ((err = hipEventElapsedTime(&ms, e->start, e->end)) != hipSuccess) {
557 return err;
558 }
559
560 // HIP provides milisecond resolution, but we want microseconds.
561 str_builder(sb, ",\"duration\":%f", ms*1000);
562
563 if ((err = hipEventDestroy(e->start)) != hipSuccess) {
564 return 1;
565 }
566
567 if ((err = hipEventDestroy(e->end)) != hipSuccess) {
568 return 1;
569 }
570
571 free(e);
572
573 return 0;
574 }
575
576 int futhark_context_sync(struct futhark_context* ctx) {
577 HIP_SUCCEED_OR_RETURN(hipStreamSynchronize(ctx->stream));
578 if (ctx->failure_is_an_option) {
579 // Check for any delayed error.
580 int32_t failure_idx;
581 HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(&failure_idx,
582 ctx->global_failure,
583 sizeof(int32_t)));
584 ctx->failure_is_an_option = 0;
585
586 if (failure_idx >= 0) {
587 // We have to clear global_failure so that the next entry point
588 // is not considered a failure from the start.
589 int32_t no_failure = -1;
590 HIP_SUCCEED_OR_RETURN(hipMemcpyHtoD(ctx->global_failure,
591 &no_failure,
592 sizeof(int32_t)));
593
594 int64_t args[max_failure_args+1];
595 HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(&args,
596 ctx->global_failure_args,
597 sizeof(args)));
598
599 ctx->error = get_failure_msg(failure_idx, args);
600
601 return FUTHARK_PROGRAM_ERROR;
602 }
603 }
604 return 0;
605 }
606
607 struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);
608 void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);
609
610 int backend_context_setup(struct futhark_context* ctx) {
611 ctx->failure_is_an_option = 0;
612 ctx->total_runs = 0;
613 ctx->total_runtime = 0;
614 ctx->peak_mem_usage_device = 0;
615 ctx->cur_mem_usage_device = 0;
616 ctx->kernels = NULL;
617
618 HIP_SUCCEED_FATAL(hipInit(0));
619 if (hip_device_setup(ctx) != 0) {
620 futhark_panic(-1, "No suitable HIP device found.\n");
621 }
622
623 free_list_init(&ctx->gpu_free_list);
624
625 if (ctx->cfg->unified_memory == 2) {
626 ctx->cfg->unified_memory = device_query(ctx->dev, hipDeviceAttributeManagedMemory);
627 }
628
629 if (ctx->cfg->logging) {
630 if (ctx->cfg->unified_memory) {
631 fprintf(ctx->log, "Using managed memory\n");
632 } else {
633 fprintf(ctx->log, "Using unmanaged memory\n");
634 }
635 }
636
637 ctx->max_thread_block_size = device_query(ctx->dev, hipDeviceAttributeMaxThreadsPerBlock);
638 ctx->max_grid_size = device_query(ctx->dev, hipDeviceAttributeMaxGridDimX);
639 ctx->max_tile_size = sqrt(ctx->max_thread_block_size);
640 ctx->max_threshold = 1U<<31; // No limit.
641 ctx->max_bespoke = 0;
642 ctx->max_registers = device_query(ctx->dev, hipDeviceAttributeMaxRegistersPerBlock);
643
644 if (ctx->cfg->gpu.default_shared_memory != 0) {
645 ctx->max_shared_memory = ctx->cfg->gpu.default_shared_memory;
646 } else {
647 ctx->max_shared_memory = device_query(ctx->dev, hipDeviceAttributeMaxSharedMemoryPerBlock);
648 }
649
650 if (ctx->cfg->gpu.default_cache != 0) {
651 ctx->max_cache = ctx->cfg->gpu.default_cache;
652 } else {
653 ctx->max_cache = device_query(ctx->dev, hipDeviceAttributeL2CacheSize);
654 }
655
656 // FIXME: in principle we should query hipDeviceAttributeWarpSize
657 // from the device, which will provide 64 on AMD GPUs.
658 // Unfortunately, we currently do nasty implicit intra-warp
659 // synchronisation in codegen, which does not work when this is 64.
660 // Once our codegen properly synchronises intra-warp operations, we
661 // can use the actual hardware lockstep width instead.
662 ctx->lockstep_width = 32;
663 HIP_SUCCEED_FATAL(hipStreamCreate(&ctx->stream));
664 hip_size_setup(ctx);
665
666 gpu_init_log(ctx);
667
668 ctx->error = hip_module_setup(ctx,
669 ctx->cfg->program,
670 (const char**)ctx->cfg->build_opts,
671 ctx->cfg->cache_fname);
672
673 if (ctx->error != NULL) {
674 futhark_panic(1, "During HIP initialisation:\n%s\n", ctx->error);
675 }
676
677 int32_t no_error = -1;
678 HIP_SUCCEED_FATAL(hipMalloc(&ctx->global_failure, sizeof(no_error)));
679 HIP_SUCCEED_FATAL(hipMemcpyHtoD(ctx->global_failure, &no_error, sizeof(no_error)));
680 // The +1 is to avoid zero-byte allocations.
681 HIP_SUCCEED_FATAL(hipMalloc(&ctx->global_failure_args, sizeof(int64_t)*(max_failure_args+1)));
682
683 if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) {
684 return 1;
685 }
686
687 return 0;
688 }
689
690 void backend_context_teardown(struct futhark_context* ctx) {
691 if (ctx->kernels != NULL) {
692 free_builtin_kernels(ctx, ctx->kernels);
693 hipFree(ctx->global_failure);
694 hipFree(ctx->global_failure_args);
695 HIP_SUCCEED_FATAL(gpu_free_all(ctx));
696 HIP_SUCCEED_FATAL(hipStreamDestroy(ctx->stream));
697 HIP_SUCCEED_FATAL(hipModuleUnload(ctx->module));
698 }
699 free_list_destroy(&ctx->gpu_free_list);
700 }
701
702 // GPU ABSTRACTION LAYER
703
704 typedef hipFunction_t gpu_kernel;
705 typedef hipDeviceptr_t gpu_mem;
706
707 static void gpu_create_kernel(struct futhark_context *ctx,
708 gpu_kernel* kernel,
709 const char* name) {
710 if (ctx->debugging) {
711 fprintf(ctx->log, "Creating kernel %s.\n", name);
712 }
713 HIP_SUCCEED_FATAL(hipModuleGetFunction(kernel, ctx->module, name));
714 }
715
716 static void gpu_free_kernel(struct futhark_context *ctx,
717 gpu_kernel kernel) {
718 (void)ctx;
719 (void)kernel;
720 }
721
722 static int gpu_scalar_to_device(struct futhark_context* ctx,
723 const char *provenance,
724 gpu_mem dst, size_t offset, size_t size,
725 void *src) {
726 struct hip_event *event = hip_event_new(ctx);
727 if (event != NULL) {
728 add_event(ctx,
729 "copy_scalar_to_dev",
730 provenance,
731 NULL,
732 event,
733 (event_report_fn)hip_event_report);
734 HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream));
735 }
736 HIP_SUCCEED_OR_RETURN(hipMemcpyHtoD((unsigned char*)dst + offset, src, size));
737 if (event != NULL) {
738 HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream));
739 }
740 return FUTHARK_SUCCESS;
741 }
742
743 static int gpu_scalar_from_device(struct futhark_context* ctx,
744 const char *provenance,
745 void *dst,
746 gpu_mem src, size_t offset, size_t size) {
747 struct hip_event *event = hip_event_new(ctx);
748 if (event != NULL) {
749 add_event(ctx,
750 "copy_scalar_from_dev",
751 provenance,
752 NULL,
753 event,
754 (event_report_fn)hip_event_report);
755 HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream));
756 }
757 HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(dst, (unsigned char*)src + offset, size));
758 if (event != NULL) {
759 HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream));
760 }
761 return FUTHARK_SUCCESS;
762 }
763
764 static int gpu_memcpy(struct futhark_context* ctx,
765 const char *provenance,
766 gpu_mem dst, int64_t dst_offset,
767 gpu_mem src, int64_t src_offset,
768 int64_t nbytes) {
769 struct hip_event *event = hip_event_new(ctx);
770 if (event != NULL) {
771 add_event(ctx,
772 "copy_dev_to_dev",
773 provenance,
774 NULL,
775 event,
776 (event_report_fn)hip_event_report);
777 HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream));
778 }
779 HIP_SUCCEED_OR_RETURN(hipMemcpyWithStream((unsigned char*)dst+dst_offset, (unsigned char*)src+src_offset,
780 nbytes, hipMemcpyDeviceToDevice, ctx->stream));
781 if (event != NULL) {
782 HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream));
783 }
784 return FUTHARK_SUCCESS;
785 }
786
787 static int memcpy_host2gpu(struct futhark_context* ctx,
788 const char *provenance,
789 bool sync,
790 gpu_mem dst, int64_t dst_offset,
791 const unsigned char* src, int64_t src_offset,
792 int64_t nbytes) {
793 if (nbytes > 0) {
794 struct hip_event *event = hip_event_new(ctx);
795 if (event != NULL) {
796 add_event(ctx,
797 "copy_host_to_dev",
798 provenance,
799 NULL,
800 event,
801 (event_report_fn)hip_event_report);
802 HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream));
803 }
804 if (sync) {
805 HIP_SUCCEED_OR_RETURN
806 (hipMemcpyHtoD((unsigned char*)dst + dst_offset,
807 (unsigned char*)src + src_offset, nbytes));
808 } else {
809 HIP_SUCCEED_OR_RETURN
810 (hipMemcpyHtoDAsync((unsigned char*)dst + dst_offset,
811 (unsigned char*)src + src_offset,
812 nbytes, ctx->stream));
813 }
814 if (event != NULL) {
815 HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream));
816 }
817 }
818 return FUTHARK_SUCCESS;
819 }
820
821 static int memcpy_gpu2host(struct futhark_context* ctx,
822 const char *provenance,
823 bool sync,
824 unsigned char* dst, int64_t dst_offset,
825 gpu_mem src, int64_t src_offset,
826 int64_t nbytes) {
827 if (nbytes > 0) {
828 struct hip_event *event = hip_event_new(ctx);
829 if (event != NULL) {
830 add_event(ctx,
831 "copy_dev_to_host",
832 provenance,
833 NULL,
834 event,
835 (event_report_fn)hip_event_report);
836 HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream));
837 }
838 if (sync) {
839 HIP_SUCCEED_OR_RETURN
840 (hipMemcpyDtoH(dst + dst_offset,
841 (unsigned char*)src + src_offset,
842 nbytes));
843 } else {
844 HIP_SUCCEED_OR_RETURN
845 (hipMemcpyDtoHAsync(dst + dst_offset,
846 (unsigned char*)src + src_offset,
847 nbytes, ctx->stream));
848 }
849 if (event != NULL) {
850 HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream));
851 }
852 if (sync &&
853 ctx->failure_is_an_option &&
854 futhark_context_sync(ctx) != 0) {
855 return 1;
856 }
857 }
858 return FUTHARK_SUCCESS;
859 }
860
861 static int gpu_launch_kernel(struct futhark_context* ctx,
862 gpu_kernel kernel,
863 const char *name, const char *provenance,
864 const int32_t grid[3],
865 const int32_t block[3],
866 unsigned int shared_mem_bytes,
867 int num_args,
868 void* args[num_args],
869 size_t args_sizes[num_args]) {
870 (void) args_sizes;
871
872 if (shared_mem_bytes > ctx->max_shared_memory) {
873 set_error(ctx, msgprintf("Kernel %s with %d bytes of memory exceeds device limit of %d\n",
874 name, shared_mem_bytes, (int)ctx->max_shared_memory));
875 return 1;
876 }
877
878 int64_t time_start = 0, time_end = 0;
879 if (ctx->debugging) {
880 time_start = get_wall_time();
881 }
882
883 struct hip_event *event = hip_event_new(ctx);
884
885 if (event != NULL) {
886 HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream));
887
888 struct kvs *kvs = kvs_new();
889 kvs_printf(kvs, "kernel", "\"%s\"", name);
890 kvs_printf(kvs, "grid", "[%d,%d,%d]", grid[0], grid[1], grid[2]);
891 kvs_printf(kvs, "block", "[%d,%d,%d]", block[0], block[1], block[2]);
892 kvs_printf(kvs, "shared memory", "%d", shared_mem_bytes);
893
894 add_event(ctx,
895 name,
896 provenance,
897 kvs,
898 event,
899 (event_report_fn)hip_event_report);
900 }
901
902 HIP_SUCCEED_OR_RETURN
903 (hipModuleLaunchKernel(kernel,
904 grid[0], grid[1], grid[2],
905 block[0], block[1], block[2],
906 shared_mem_bytes, ctx->stream,
907 args, NULL));
908
909 if (event != NULL) {
910 HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream));
911 }
912
913 if (ctx->debugging) {
914 HIP_SUCCEED_FATAL(hipStreamSynchronize(ctx->stream));
915 time_end = get_wall_time();
916 long int time_diff = time_end - time_start;
917 fprintf(ctx->log, " runtime: %ldus\n", time_diff);
918 }
919 if (ctx->logging) {
920 fprintf(ctx->log, "\n");
921 }
922
923 return FUTHARK_SUCCESS;
924 }
925
926 static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) {
927 hipError_t res;
928
929 if (ctx->cfg->unified_memory) {
930 res = hipMallocManaged(mem_out, size, hipMemAttachGlobal);
931 } else {
932 res = hipMalloc(mem_out, size);
933 }
934
935 if (res == hipErrorOutOfMemory) {
936 return FUTHARK_OUT_OF_MEMORY;
937 }
938 HIP_SUCCEED_OR_RETURN(res);
939 return FUTHARK_SUCCESS;
940 }
941
942 static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) {
943 (void)ctx;
944 HIP_SUCCEED_OR_RETURN(hipFree(mem));
945 return FUTHARK_SUCCESS;
946 }
947
948 // End of backends/hip.h.