Debian Codegraph @

rts/c/backends/opencl.h

raw
1 // Start of backends/opencl.h
2
3 // Note [32-bit transpositions]
4 //
5 // Transposition kernels are much slower when they have to use 64-bit
6 // arithmetic. I observed about 0.67x slowdown on an A100 GPU when
7 // transposing four-byte elements (much less when transposing 8-byte
8 // elements). Unfortunately, 64-bit arithmetic is a requirement for
9 // large arrays (see #1953 for what happens otherwise). We generate
10 // both 32- and 64-bit index arithmetic versions of transpositions,
11 // and dynamically pick between them at runtime. This is an
12 // unfortunate code bloat, and it would be preferable if we could
13 // simply optimise the 64-bit version to make this distinction
14 // unnecessary. Fortunately these kernels are quite small.
15
16 // Forward declarations.
17 struct opencl_device_option;
18 // Invoked by setup_opencl() after the platform and device has been
19 // found, but before the program is loaded. Its intended use is to
20 // tune constants based on the selected platform and device.
21 static void post_opencl_setup(struct futhark_context*, struct opencl_device_option*);
22 static void set_tuning_params(struct futhark_context* ctx);
23 static char* get_failure_msg(int failure_idx, int64_t args[]);
24
25 #define OPENCL_SUCCEED_FATAL(e) opencl_succeed_fatal(e, #e, __FILE__, __LINE__)
26 #define OPENCL_SUCCEED_NONFATAL(e) opencl_succeed_nonfatal(e, #e, __FILE__, __LINE__)
27 // Take care not to override an existing error.
28 #define OPENCL_SUCCEED_OR_RETURN(e) { \
29 char *serror = OPENCL_SUCCEED_NONFATAL(e); \
30 if (serror) { \
31 if (!ctx->error) { \
32 ctx->error = serror; \
33 } else { \
34 free(serror); \
35 } \
36 return bad; \
37 } \
38 }
39
40 // OPENCL_SUCCEED_OR_RETURN returns the value of the variable 'bad' in
41 // scope. By default, it will be this one. Create a local variable
42 // of some other type if needed. This is a bit of a hack, but it
43 // saves effort in the code generator.
44 static const int bad = 1;
45
46 static const char* opencl_error_string(cl_int err) {
47 switch (err) {
48 case CL_SUCCESS: return "Success!";
49 case CL_DEVICE_NOT_FOUND: return "Device not found.";
50 case CL_DEVICE_NOT_AVAILABLE: return "Device not available";
51 case CL_COMPILER_NOT_AVAILABLE: return "Compiler not available";
52 case CL_MEM_OBJECT_ALLOCATION_FAILURE: return "Memory object allocation failure";
53 case CL_OUT_OF_RESOURCES: return "Out of resources";
54 case CL_OUT_OF_HOST_MEMORY: return "Out of host memory";
55 case CL_PROFILING_INFO_NOT_AVAILABLE: return "Profiling information not available";
56 case CL_MEM_COPY_OVERLAP: return "Memory copy overlap";
57 case CL_IMAGE_FORMAT_MISMATCH: return "Image format mismatch";
58 case CL_IMAGE_FORMAT_NOT_SUPPORTED: return "Image format not supported";
59 case CL_BUILD_PROGRAM_FAILURE: return "Program build failure";
60 case CL_MAP_FAILURE: return "Map failure";
61 case CL_INVALID_VALUE: return "Invalid value";
62 case CL_INVALID_DEVICE_TYPE: return "Invalid device type";
63 case CL_INVALID_PLATFORM: return "Invalid platform";
64 case CL_INVALID_DEVICE: return "Invalid device";
65 case CL_INVALID_CONTEXT: return "Invalid context";
66 case CL_INVALID_QUEUE_PROPERTIES: return "Invalid queue properties";
67 case CL_INVALID_COMMAND_QUEUE: return "Invalid command queue";
68 case CL_INVALID_HOST_PTR: return "Invalid host pointer";
69 case CL_INVALID_MEM_OBJECT: return "Invalid memory object";
70 case CL_INVALID_IMAGE_FORMAT_DESCRIPTOR: return "Invalid image format descriptor";
71 case CL_INVALID_IMAGE_SIZE: return "Invalid image size";
72 case CL_INVALID_SAMPLER: return "Invalid sampler";
73 case CL_INVALID_BINARY: return "Invalid binary";
74 case CL_INVALID_BUILD_OPTIONS: return "Invalid build options";
75 case CL_INVALID_PROGRAM: return "Invalid program";
76 case CL_INVALID_PROGRAM_EXECUTABLE: return "Invalid program executable";
77 case CL_INVALID_KERNEL_NAME: return "Invalid kernel name";
78 case CL_INVALID_KERNEL_DEFINITION: return "Invalid kernel definition";
79 case CL_INVALID_KERNEL: return "Invalid kernel";
80 case CL_INVALID_ARG_INDEX: return "Invalid argument index";
81 case CL_INVALID_ARG_VALUE: return "Invalid argument value";
82 case CL_INVALID_ARG_SIZE: return "Invalid argument size";
83 case CL_INVALID_KERNEL_ARGS: return "Invalid kernel arguments";
84 case CL_INVALID_WORK_DIMENSION: return "Invalid work dimension";
85 case CL_INVALID_WORK_GROUP_SIZE: return "Invalid work group size";
86 case CL_INVALID_WORK_ITEM_SIZE: return "Invalid work item size";
87 case CL_INVALID_GLOBAL_OFFSET: return "Invalid global offset";
88 case CL_INVALID_EVENT_WAIT_LIST: return "Invalid event wait list";
89 case CL_INVALID_EVENT: return "Invalid event";
90 case CL_INVALID_OPERATION: return "Invalid operation";
91 case CL_INVALID_GL_OBJECT: return "Invalid OpenGL object";
92 case CL_INVALID_BUFFER_SIZE: return "Invalid buffer size";
93 case CL_INVALID_MIP_LEVEL: return "Invalid mip-map level";
94 default: return "Unknown";
95 }
96 }
97
98 static void opencl_succeed_fatal(cl_int ret,
99 const char *call,
100 const char *file,
101 int line) {
102 if (ret != CL_SUCCESS) {
103 futhark_panic(-1, "%s:%d: OpenCL call\n %s\nfailed with error code %d (%s)\n",
104 file, line, call, ret, opencl_error_string(ret));
105 }
106 }
107
108 static char* opencl_succeed_nonfatal(cl_int ret,
109 const char *call,
110 const char *file,
111 int line) {
112 if (ret != CL_SUCCESS) {
113 return msgprintf("%s:%d: OpenCL call\n %s\nfailed with error code %d (%s)\n",
114 file, line, call, ret, opencl_error_string(ret));
115 } else {
116 return NULL;
117 }
118 }
119
120 struct futhark_context_config {
121 int in_use;
122 int debugging;
123 int profiling;
124 int logging;
125 char *cache_fname;
126 int num_tuning_params;
127 int64_t *tuning_params;
128 const char** tuning_param_names;
129 const char** tuning_param_vars;
130 const char** tuning_param_classes;
131 // Uniform fields above.
132
133 char* program;
134 int preferred_device_num;
135 char* preferred_platform;
136 char* preferred_device;
137 int ignore_blacklist;
138
139 int unified_memory;
140
141 char* dump_binary_to;
142 char* load_binary_from;
143
144 int num_build_opts;
145 char* *build_opts;
146
147 cl_command_queue queue;
148 int queue_set;
149
150 struct gpu_config gpu;
151 };
152
153 static void backend_context_config_setup(struct futhark_context_config* cfg) {
154 cfg->num_build_opts = 0;
155 cfg->build_opts = (char**) malloc(sizeof(const char*));
156 cfg->build_opts[0] = NULL;
157 cfg->preferred_device_num = 0;
158 cfg->preferred_platform = strdup("");
159 cfg->preferred_device = strdup("");
160 cfg->ignore_blacklist = 0;
161 cfg->dump_binary_to = NULL;
162 cfg->load_binary_from = NULL;
163 cfg->program = strconcat(gpu_program);
164
165 cfg->unified_memory = 2;
166
167 cfg->gpu = gpu_config_initial;
168
169 cfg->queue_set = 0;
170 }
171
172 static void backend_context_config_teardown(struct futhark_context_config* cfg) {
173 for (int i = 0; i < cfg->num_build_opts; i++) {
174 free(cfg->build_opts[i]);
175 }
176 free(cfg->build_opts);
177 free(cfg->dump_binary_to);
178 free(cfg->load_binary_from);
179 free(cfg->preferred_device);
180 free(cfg->preferred_platform);
181 free(cfg->program);
182 }
183
184 void futhark_context_config_add_build_option(struct futhark_context_config* cfg, const char *opt) {
185 cfg->build_opts[cfg->num_build_opts] = strdup(opt);
186 cfg->num_build_opts++;
187 cfg->build_opts = (char**) realloc(cfg->build_opts, (cfg->num_build_opts+1) * sizeof(char*));
188 cfg->build_opts[cfg->num_build_opts] = NULL;
189 }
190
191 void futhark_context_config_set_device(struct futhark_context_config *cfg, const char* s) {
192 int x = 0;
193 if (*s == '#') {
194 s++;
195 while (isdigit(*s)) {
196 x = x * 10 + (*s++)-'0';
197 }
198 // Skip trailing spaces.
199 while (isspace(*s)) {
200 s++;
201 }
202 }
203 free(cfg->preferred_device);
204 cfg->preferred_device = strdup(s);
205 cfg->preferred_device_num = x;
206 cfg->ignore_blacklist = 1;
207 }
208
209 void futhark_context_config_set_platform(struct futhark_context_config *cfg, const char *s) {
210 free(cfg->preferred_platform);
211 cfg->preferred_platform = strdup(s);
212 cfg->ignore_blacklist = 1;
213 }
214
215 void futhark_context_config_set_command_queue(struct futhark_context_config *cfg, cl_command_queue q) {
216 cfg->queue = q;
217 cfg->queue_set = 1;
218 }
219
220 struct opencl_device_option {
221 cl_platform_id platform;
222 cl_device_id device;
223 cl_device_type device_type;
224 char *platform_name;
225 char *device_name;
226 };
227
228 static char* opencl_platform_info(cl_platform_id platform,
229 cl_platform_info param) {
230 size_t req_bytes;
231 char *info;
232
233 OPENCL_SUCCEED_FATAL(clGetPlatformInfo(platform, param, 0, NULL, &req_bytes));
234
235 info = (char*) malloc(req_bytes);
236
237 OPENCL_SUCCEED_FATAL(clGetPlatformInfo(platform, param, req_bytes, info, NULL));
238
239 return info;
240 }
241
242 static char* opencl_device_info(cl_device_id device,
243 cl_device_info param) {
244 size_t req_bytes;
245 char *info;
246
247 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device, param, 0, NULL, &req_bytes));
248
249 info = (char*) malloc(req_bytes);
250
251 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device, param, req_bytes, info, NULL));
252
253 return info;
254 }
255
256 static int is_blacklisted(const char *platform_name, const char *device_name,
257 const struct futhark_context_config *cfg) {
258 if (strcmp(cfg->preferred_platform, "") != 0 ||
259 strcmp(cfg->preferred_device, "") != 0) {
260 return 0;
261 } else if (strstr(platform_name, "Apple") != NULL &&
262 strstr(device_name, "Intel(R) Core(TM)") != NULL) {
263 return 1;
264 } else {
265 return 0;
266 }
267 }
268
269 static void opencl_all_device_options(struct opencl_device_option **devices_out,
270 size_t *num_devices_out) {
271 size_t num_devices = 0, num_devices_added = 0;
272
273 cl_platform_id *all_platforms;
274 cl_uint *platform_num_devices;
275
276 cl_uint num_platforms;
277
278 // Find the number of platforms.
279 OPENCL_SUCCEED_FATAL(clGetPlatformIDs(0, NULL, &num_platforms));
280
281 // Make room for them.
282 all_platforms = calloc(num_platforms, sizeof(cl_platform_id));
283 platform_num_devices = calloc(num_platforms, sizeof(cl_uint));
284
285 // Fetch all the platforms.
286 OPENCL_SUCCEED_FATAL(clGetPlatformIDs(num_platforms, all_platforms, NULL));
287
288 // Count the number of devices for each platform, as well as the
289 // total number of devices.
290 for (cl_uint i = 0; i < num_platforms; i++) {
291 if (clGetDeviceIDs(all_platforms[i], CL_DEVICE_TYPE_ALL,
292 0, NULL, &platform_num_devices[i]) == CL_SUCCESS) {
293 num_devices += platform_num_devices[i];
294 } else {
295 platform_num_devices[i] = 0;
296 }
297 }
298
299 // Make room for all the device options.
300 struct opencl_device_option *devices =
301 calloc(num_devices, sizeof(struct opencl_device_option));
302
303 // Loop through the platforms, getting information about their devices.
304 for (cl_uint i = 0; i < num_platforms; i++) {
305 cl_platform_id platform = all_platforms[i];
306 cl_uint num_platform_devices = platform_num_devices[i];
307
308 if (num_platform_devices == 0) {
309 continue;
310 }
311
312 char *platform_name = opencl_platform_info(platform, CL_PLATFORM_NAME);
313 cl_device_id *platform_devices =
314 calloc(num_platform_devices, sizeof(cl_device_id));
315
316 // Fetch all the devices.
317 OPENCL_SUCCEED_FATAL(clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL,
318 num_platform_devices, platform_devices, NULL));
319
320 // Loop through the devices, adding them to the devices array.
321 for (cl_uint i = 0; i < num_platform_devices; i++) {
322 char *device_name = opencl_device_info(platform_devices[i], CL_DEVICE_NAME);
323 devices[num_devices_added].platform = platform;
324 devices[num_devices_added].device = platform_devices[i];
325 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(platform_devices[i], CL_DEVICE_TYPE,
326 sizeof(cl_device_type),
327 &devices[num_devices_added].device_type,
328 NULL));
329 // We don't want the structs to share memory, so copy the platform name.
330 // Each device name is already unique.
331 devices[num_devices_added].platform_name = strclone(platform_name);
332 devices[num_devices_added].device_name = device_name;
333 num_devices_added++;
334 }
335 free(platform_devices);
336 free(platform_name);
337 }
338 free(all_platforms);
339 free(platform_num_devices);
340
341 *devices_out = devices;
342 *num_devices_out = num_devices;
343 }
344
345 void futhark_context_config_select_device_interactively(struct futhark_context_config *cfg) {
346 struct opencl_device_option *devices;
347 size_t num_devices;
348
349 opencl_all_device_options(&devices, &num_devices);
350
351 printf("Choose OpenCL device:\n");
352 const char *cur_platform = "";
353 for (size_t i = 0; i < num_devices; i++) {
354 struct opencl_device_option device = devices[i];
355 if (strcmp(cur_platform, device.platform_name) != 0) {
356 printf("Platform: %s\n", device.platform_name);
357 cur_platform = device.platform_name;
358 }
359 printf("[%d] %s\n", (int)i, device.device_name);
360 }
361
362 int selection;
363 printf("Choice: ");
364 if (scanf("%d", &selection) == 1) {
365 cfg->preferred_platform = "";
366 cfg->preferred_device = "";
367 cfg->preferred_device_num = selection;
368 cfg->ignore_blacklist = 1;
369 }
370
371 // Free all the platform and device names.
372 for (size_t j = 0; j < num_devices; j++) {
373 free(devices[j].platform_name);
374 free(devices[j].device_name);
375 }
376 free(devices);
377 }
378
379 void futhark_context_config_list_devices(struct futhark_context_config *cfg) {
380 (void)cfg;
381 struct opencl_device_option *devices;
382 size_t num_devices;
383
384 opencl_all_device_options(&devices, &num_devices);
385
386 const char *cur_platform = "";
387 for (size_t i = 0; i < num_devices; i++) {
388 struct opencl_device_option device = devices[i];
389 if (strcmp(cur_platform, device.platform_name) != 0) {
390 printf("Platform: %s\n", device.platform_name);
391 cur_platform = device.platform_name;
392 }
393 printf("[%d]: %s\n", (int)i, device.device_name);
394 }
395
396 // Free all the platform and device names.
397 for (size_t j = 0; j < num_devices; j++) {
398 free(devices[j].platform_name);
399 free(devices[j].device_name);
400 }
401 free(devices);
402 }
403
404 const char* futhark_context_config_get_program(struct futhark_context_config *cfg) {
405 return cfg->program;
406 }
407
408 void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) {
409 free(cfg->program);
410 cfg->program = strdup(s);
411 }
412
413 void futhark_context_config_dump_binary_to(struct futhark_context_config *cfg, const char *path) {
414 free(cfg->dump_binary_to);
415 cfg->dump_binary_to = strdup(path);
416 }
417
418 void futhark_context_config_load_binary_from(struct futhark_context_config *cfg, const char *path) {
419 free(cfg->load_binary_from);
420 cfg->load_binary_from = strdup(path);
421 }
422
423 void futhark_context_config_set_unified_memory(struct futhark_context_config* cfg, int flag) {
424 cfg->unified_memory = flag;
425 }
426
427 struct futhark_context {
428 struct futhark_context_config* cfg;
429 int detail_memory;
430 int debugging;
431 int profiling;
432 int profiling_paused;
433 int logging;
434 lock_t lock;
435 char *error;
436 lock_t error_lock;
437 FILE *log;
438 struct constants *constants;
439 struct free_list free_list;
440 struct event_list event_list;
441 int64_t peak_mem_usage_default;
442 int64_t cur_mem_usage_default;
443 struct program* program;
444 bool program_initialised;
445 // Uniform fields above.
446
447 cl_mem global_failure;
448 cl_mem global_failure_args;
449 struct tuning_params tuning_params;
450 // True if a potentially failing kernel has been enqueued.
451 cl_int failure_is_an_option;
452 int total_runs;
453 long int total_runtime;
454 int64_t peak_mem_usage_device;
455 int64_t cur_mem_usage_device;
456
457 cl_device_id device;
458 cl_context ctx;
459 cl_command_queue queue;
460 cl_program clprogram;
461
462 struct free_list gpu_free_list;
463
464 size_t max_thread_block_size;
465 size_t max_grid_size;
466 size_t max_tile_size;
467 size_t max_threshold;
468 size_t max_shared_memory;
469 size_t max_bespoke;
470 size_t max_registers;
471 size_t max_cache;
472
473 size_t lockstep_width;
474
475 struct builtin_kernels* kernels;
476 };
477
478 static cl_build_status build_gpu_program(cl_program program, cl_device_id device, const char* options, char** log) {
479 cl_int clBuildProgram_error = clBuildProgram(program, 1, &device, options, NULL, NULL);
480
481 // Avoid termination due to CL_BUILD_PROGRAM_FAILURE
482 if (clBuildProgram_error != CL_SUCCESS &&
483 clBuildProgram_error != CL_BUILD_PROGRAM_FAILURE) {
484 OPENCL_SUCCEED_FATAL(clBuildProgram_error);
485 }
486
487 cl_build_status build_status;
488 OPENCL_SUCCEED_FATAL(clGetProgramBuildInfo(program,
489 device,
490 CL_PROGRAM_BUILD_STATUS,
491 sizeof(cl_build_status),
492 &build_status,
493 NULL));
494
495 if (build_status != CL_BUILD_SUCCESS) {
496 char *build_log;
497 size_t ret_val_size;
498 OPENCL_SUCCEED_FATAL(clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &ret_val_size));
499
500 build_log = (char*) malloc(ret_val_size+1);
501 OPENCL_SUCCEED_FATAL(clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, ret_val_size, build_log, NULL));
502
503 // The spec technically does not say whether the build log is
504 // zero-terminated, so let's be careful.
505 build_log[ret_val_size] = '\0';
506 *log = build_log;
507 }
508
509 return build_status;
510 }
511
512 static char* mk_compile_opts(struct futhark_context *ctx,
513 const char *extra_build_opts[],
514 struct opencl_device_option device_option) {
515 int compile_opts_size = 1024;
516
517 for (int i = 0; i < ctx->cfg->num_tuning_params; i++) {
518 compile_opts_size += strlen(ctx->cfg->tuning_param_names[i]) + 20;
519 }
520
521 char** macro_names;
522 int64_t* macro_vals;
523 int num_macros = gpu_macros(ctx, &macro_names, &macro_vals);
524
525 for (int i = 0; extra_build_opts[i] != NULL; i++) {
526 compile_opts_size += strlen(extra_build_opts[i] + 1);
527 }
528
529 for (int i = 0; i < num_macros; i++) {
530 compile_opts_size += strlen(macro_names[i]) + 1 + 20;
531 }
532
533 char *compile_opts = (char*) malloc(compile_opts_size);
534
535 int w = snprintf(compile_opts, compile_opts_size,
536 "-DLOCKSTEP_WIDTH=%d ",
537 (int)ctx->lockstep_width);
538
539 w += snprintf(compile_opts+w, compile_opts_size-w,
540 "-D%s=%d ",
541 "max_thread_block_size",
542 (int)ctx->max_thread_block_size);
543
544 w += snprintf(compile_opts+w, compile_opts_size-w,
545 "-D%s=%d ",
546 "max_shared_memory",
547 (int)ctx->max_shared_memory);
548
549 w += snprintf(compile_opts+w, compile_opts_size-w,
550 "-D%s=%d ",
551 "max_registers",
552 (int)ctx->max_registers);
553
554 for (int i = 0; i < ctx->cfg->num_tuning_params; i++) {
555 w += snprintf(compile_opts+w, compile_opts_size-w,
556 "-D%s=%d ",
557 ctx->cfg->tuning_param_vars[i],
558 (int)ctx->cfg->tuning_params[i]);
559 }
560
561 for (int i = 0; extra_build_opts[i] != NULL; i++) {
562 w += snprintf(compile_opts+w, compile_opts_size-w,
563 "%s ", extra_build_opts[i]);
564 }
565
566 for (int i = 0; i < num_macros; i++) {
567 w += snprintf(compile_opts+w, compile_opts_size-w,
568 "-D%s=%zu ", macro_names[i], macro_vals[i]);
569 }
570
571 w += snprintf(compile_opts+w, compile_opts_size-w,
572 "-DTR_BLOCK_DIM=%d -DTR_TILE_DIM=%d -DTR_ELEMS_PER_THREAD=%d ",
573 TR_BLOCK_DIM, TR_TILE_DIM, TR_ELEMS_PER_THREAD);
574
575 // Oclgrind claims to support cl_khr_fp16, but this is not actually
576 // the case.
577 if (strcmp(device_option.platform_name, "Oclgrind") == 0) {
578 w += snprintf(compile_opts+w, compile_opts_size-w, "-DEMULATE_F16 ");
579 }
580
581 // By default, OpenCL allows imprecise (but faster) division and
582 // square root operations. For equivalence with other backends, ask
583 // for correctly rounded ones here.
584 w += snprintf(compile_opts+w, compile_opts_size-w,
585 "-cl-fp32-correctly-rounded-divide-sqrt");
586
587 free(macro_names);
588 free(macro_vals);
589
590 return compile_opts;
591 }
592
593 static cl_event* opencl_event_new(struct futhark_context* ctx) {
594 if (ctx->profiling && !ctx->profiling_paused) {
595 return malloc(sizeof(cl_event));
596 } else {
597 return NULL;
598 }
599 }
600
601 static int opencl_event_report(struct str_builder* sb, cl_event* e) {
602 cl_int err;
603 cl_ulong start_t, end_t;
604
605 assert(e != NULL);
606 OPENCL_SUCCEED_FATAL(clGetEventProfilingInfo(*e,
607 CL_PROFILING_COMMAND_START,
608 sizeof(start_t),
609 &start_t,
610 NULL));
611 OPENCL_SUCCEED_FATAL(clGetEventProfilingInfo(*e,
612 CL_PROFILING_COMMAND_END,
613 sizeof(end_t),
614 &end_t,
615 NULL));
616
617 // OpenCL provides nanosecond resolution, but we want microseconds.
618 str_builder(sb, ",\"duration\":%f", (end_t - start_t)/1000.0);
619
620 OPENCL_SUCCEED_FATAL(clReleaseEvent(*e));
621
622 free(e);
623
624 return 0;
625 }
626
627 int futhark_context_sync(struct futhark_context* ctx) {
628 // Check for any delayed error.
629 cl_int failure_idx = -1;
630 if (ctx->failure_is_an_option) {
631 OPENCL_SUCCEED_OR_RETURN(
632 clEnqueueReadBuffer(ctx->queue,
633 ctx->global_failure,
634 CL_FALSE,
635 0, sizeof(cl_int), &failure_idx,
636 0, NULL, NULL));
637 ctx->failure_is_an_option = 0;
638 }
639
640 OPENCL_SUCCEED_OR_RETURN(clFinish(ctx->queue));
641
642 if (failure_idx >= 0) {
643 // We have to clear global_failure so that the next entry point
644 // is not considered a failure from the start.
645 cl_int no_failure = -1;
646 OPENCL_SUCCEED_OR_RETURN(
647 clEnqueueWriteBuffer(ctx->queue, ctx->global_failure, CL_TRUE,
648 0, sizeof(cl_int), &no_failure,
649 0, NULL, NULL));
650
651 int64_t args[max_failure_args+1];
652 OPENCL_SUCCEED_OR_RETURN(
653 clEnqueueReadBuffer(ctx->queue,
654 ctx->global_failure_args,
655 CL_TRUE,
656 0, sizeof(args), &args,
657 0, NULL, NULL));
658
659 ctx->error = get_failure_msg(failure_idx, args);
660
661 return FUTHARK_PROGRAM_ERROR;
662 }
663 return 0;
664 }
665
666
667 // We take as input several strings representing the program, because
668 // C does not guarantee that the compiler supports particularly large
669 // literals. Notably, Visual C has a limit of 2048 characters. The
670 // array must be NULL-terminated.
671 static void setup_opencl_with_command_queue(struct futhark_context *ctx,
672 cl_command_queue queue,
673 const char* extra_build_opts[],
674 const char* cache_fname) {
675 int error;
676
677 free_list_init(&ctx->gpu_free_list);
678 ctx->queue = queue;
679
680 OPENCL_SUCCEED_FATAL(clGetCommandQueueInfo(ctx->queue, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx->ctx, NULL));
681
682 // Fill out the device info. This is redundant work if we are
683 // called from setup_opencl() (which is the common case), but I
684 // doubt it matters much.
685 struct opencl_device_option device_option;
686 OPENCL_SUCCEED_FATAL(clGetCommandQueueInfo(ctx->queue, CL_QUEUE_DEVICE,
687 sizeof(cl_device_id),
688 &device_option.device,
689 NULL));
690 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_PLATFORM,
691 sizeof(cl_platform_id),
692 &device_option.platform,
693 NULL));
694 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_TYPE,
695 sizeof(cl_device_type),
696 &device_option.device_type,
697 NULL));
698 device_option.platform_name = opencl_platform_info(device_option.platform, CL_PLATFORM_NAME);
699 device_option.device_name = opencl_device_info(device_option.device, CL_DEVICE_NAME);
700
701 ctx->device = device_option.device;
702
703 if (f64_required) {
704 cl_uint supported;
705 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE,
706 sizeof(cl_uint), &supported, NULL));
707 if (!supported) {
708 futhark_panic(1, "Program uses double-precision floats, but this is not supported on the chosen device: %s\n",
709 device_option.device_name);
710 }
711 }
712
713 bool is_amd = strstr(device_option.platform_name, "AMD") != NULL;
714 bool is_nvidia = strstr(device_option.platform_name, "NVIDIA CUDA") != NULL;
715
716 size_t max_thread_block_size;
717 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_MAX_WORK_GROUP_SIZE,
718 sizeof(size_t), &max_thread_block_size, NULL));
719
720 size_t max_tile_size = sqrt(max_thread_block_size);
721
722 cl_ulong max_shared_memory;
723 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_LOCAL_MEM_SIZE,
724 sizeof(size_t), &max_shared_memory, NULL));
725
726 // Futhark reserves 4 bytes for bookkeeping information.
727 max_shared_memory -= 4;
728
729 // The OpenCL implementation may reserve some local memory bytes for
730 // various purposes. In principle, we should use
731 // clGetKernelWorkGroupInfo() to figure out for each kernel how much
732 // is actually available, but our current code generator design
733 // makes this infeasible. Instead, we have this nasty hack where we
734 // arbitrarily subtract some bytes, based on empirical measurements
735 // (but which might be arbitrarily wrong). Fortunately, we rarely
736 // try to really push the local memory usage.
737 if (is_nvidia) {
738 max_shared_memory -= 12;
739 } else if (is_amd) {
740 max_shared_memory -= 16;
741 }
742
743 // Make sure this function is defined.
744 post_opencl_setup(ctx, &device_option);
745
746 if (max_thread_block_size < ctx->cfg->gpu.default_block_size) {
747 if (ctx->cfg->gpu.default_block_size_changed) {
748 fprintf(stderr, "Note: Device limits default group size to %zu (down from %zu).\n",
749 max_thread_block_size, ctx->cfg->gpu.default_block_size);
750 }
751 ctx->cfg->gpu.default_block_size = max_thread_block_size;
752 }
753
754 if (max_tile_size < ctx->cfg->gpu.default_tile_size) {
755 if (ctx->cfg->gpu.default_tile_size_changed) {
756 fprintf(stderr, "Note: Device limits default tile size to %zu (down from %zu).\n",
757 max_tile_size, ctx->cfg->gpu.default_tile_size);
758 }
759 ctx->cfg->gpu.default_tile_size = max_tile_size;
760 }
761
762 // Some of the code generated by Futhark will use the L2 cache size
763 // to make very precise decisions about execution. OpenCL does not
764 // specify whether CL_DEVICE_GLOBAL_MEM_CACHE_SIZE is L1 or L2 cache
765 // (or maybe something else entirely). NVIDIA's implementation
766 // reports L2, but AMDs reports L1 (and provides no way to query for
767 // the L2 size). That means it is time to hack.
768
769 cl_ulong l2_cache_size;
770 if (ctx->cfg->gpu.default_cache != 0) {
771 l2_cache_size = ctx->cfg->gpu.default_cache;
772 } else {
773 cl_ulong opencl_cache_size;
774 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_GLOBAL_MEM_CACHE_SIZE,
775 sizeof(opencl_cache_size), &opencl_cache_size, NULL));
776
777 if (is_amd) {
778 // We multiply the L1 cache size with the number of compute units
779 // times 4 (number of SIMD units with GCN). Empirically this
780 // doesn't get us the right result, but it gets us fairly close.
781 cl_ulong compute_units;
782 OPENCL_SUCCEED_FATAL(clGetDeviceInfo(device_option.device, CL_DEVICE_MAX_COMPUTE_UNITS,
783 sizeof(compute_units), &compute_units, NULL));
784 l2_cache_size = opencl_cache_size * compute_units * 4;
785 } else {
786 l2_cache_size = opencl_cache_size;
787 }
788
789 if (l2_cache_size == 0) {
790 // Some code assumes nonzero cache.
791 l2_cache_size = 1024*1024;
792 }
793 }
794
795 ctx->max_thread_block_size = max_thread_block_size;
796 ctx->max_tile_size = max_tile_size; // No limit.
797 ctx->max_threshold = ctx->max_grid_size = 1U<<31; // No limit.
798
799 if (ctx->cfg->gpu.default_cache != 0) {
800 ctx->max_cache = ctx->cfg->gpu.default_cache;
801 } else {
802 ctx->max_cache = l2_cache_size;
803 }
804
805 if (ctx->cfg->gpu.default_registers != 0) {
806 ctx->max_registers = ctx->cfg->gpu.default_registers;
807 } else {
808 ctx->max_registers = 1<<16; // I cannot find a way to query for this.
809 }
810
811 if (ctx->cfg->gpu.default_shared_memory != 0) {
812 ctx->max_shared_memory = ctx->cfg->gpu.default_shared_memory;
813 } else {
814 ctx->max_shared_memory = max_shared_memory;
815 }
816
817 // Now we go through all the sizes, clamp them to the valid range,
818 // or set them to the default.
819 for (int i = 0; i < ctx->cfg->num_tuning_params; i++) {
820 const char *size_class = ctx->cfg->tuning_param_classes[i];
821 int64_t *size_value = &ctx->cfg->tuning_params[i];
822 const char* size_name = ctx->cfg->tuning_param_names[i];
823 int64_t max_value = 0, default_value = 0;
824
825 if (strstr(size_class, "thread_block_size") == size_class) {
826 max_value = max_thread_block_size;
827 default_value = ctx->cfg->gpu.default_block_size;
828 } else if (strstr(size_class, "grid_size") == size_class) {
829 max_value = max_thread_block_size; // Futhark assumes this constraint.
830 default_value = ctx->cfg->gpu.default_grid_size;
831 // XXX: as a quick and dirty hack, use twice as many threads for
832 // histograms by default. We really should just be smarter
833 // about sizes somehow.
834 if (strstr(size_name, ".seghist_") != NULL) {
835 default_value *= 2;
836 }
837 } else if (strstr(size_class, "tile_size") == size_class) {
838 max_value = sqrt(max_thread_block_size);
839 default_value = ctx->cfg->gpu.default_tile_size;
840 } else if (strstr(size_class, "reg_tile_size") == size_class) {
841 max_value = 0; // No limit.
842 default_value = ctx->cfg->gpu.default_reg_tile_size;
843 } else if (strstr(size_class, "shared_memory") == size_class) {
844 max_value = ctx->max_shared_memory;
845 default_value = ctx->max_shared_memory;
846 } else if (strstr(size_class, "cache") == size_class) {
847 max_value = ctx->max_cache;
848 default_value = ctx->max_cache;
849 } else if (strstr(size_class, "threshold") == size_class) {
850 // Threshold can be as large as it takes.
851 default_value = ctx->cfg->gpu.default_threshold;
852 } else {
853 // Bespoke sizes have no limit or default.
854 }
855 if (*size_value == 0) {
856 *size_value = default_value;
857 } else if (max_value > 0 && *size_value > max_value) {
858 fprintf(stderr, "Note: Device limits %s to %d (down from %d)\n",
859 size_name, (int)max_value, (int)*size_value);
860 *size_value = max_value;
861 }
862 }
863
864 if (ctx->lockstep_width == 0) {
865 ctx->lockstep_width = 1;
866 }
867
868 gpu_init_log(ctx);
869
870 char *compile_opts = mk_compile_opts(ctx, extra_build_opts, device_option);
871
872 if (ctx->cfg->logging) {
873 fprintf(stderr, "OpenCL compiler options: %s\n", compile_opts);
874 }
875
876 const char* opencl_src = ctx->cfg->program;
877 cl_program prog;
878 error = CL_SUCCESS;
879
880 struct cache_hash h;
881
882 int loaded_from_cache = 0;
883 if (ctx->cfg->load_binary_from == NULL) {
884 size_t src_size = 0;
885
886 if (cache_fname != NULL) {
887 if (ctx->cfg->logging) {
888 fprintf(stderr, "Restoring cache from from %s...\n", cache_fname);
889 }
890 cache_hash_init(&h);
891 cache_hash(&h, opencl_src, strlen(opencl_src));
892 cache_hash(&h, compile_opts, strlen(compile_opts));
893
894 unsigned char *buf;
895 size_t bufsize;
896 errno = 0;
897 if (cache_restore(cache_fname, &h, &buf, &bufsize) != 0) {
898 if (ctx->cfg->logging) {
899 fprintf(stderr, "Failed to restore cache (errno: %s)\n", strerror(errno));
900 }
901 } else {
902 if (ctx->cfg->logging) {
903 fprintf(stderr, "Cache restored; loading OpenCL binary...\n");
904 }
905
906 cl_int status = 0;
907 prog = clCreateProgramWithBinary(ctx->ctx, 1, &device_option.device,
908 &bufsize, (const unsigned char**)&buf,
909 &status, &error);
910 if (status == CL_SUCCESS) {
911 loaded_from_cache = 1;
912 if (ctx->cfg->logging) {
913 fprintf(stderr, "Loading succeeded.\n");
914 }
915 } else {
916 if (ctx->cfg->logging) {
917 fprintf(stderr, "Loading failed.\n");
918 }
919 }
920 }
921 }
922
923 if (!loaded_from_cache) {
924 if (ctx->cfg->logging) {
925 fprintf(stderr, "Creating OpenCL program...\n");
926 }
927
928 const char* src_ptr[] = {opencl_src};
929 prog = clCreateProgramWithSource(ctx->ctx, 1, src_ptr, &src_size, &error);
930 OPENCL_SUCCEED_FATAL(error);
931 }
932 } else {
933 if (ctx->cfg->logging) {
934 fprintf(stderr, "Loading OpenCL binary from %s...\n", ctx->cfg->load_binary_from);
935 }
936 size_t binary_size;
937 unsigned char *fut_opencl_bin =
938 (unsigned char*) slurp_file(ctx->cfg->load_binary_from, &binary_size);
939 assert(fut_opencl_bin != NULL);
940 const unsigned char *binaries[1] = { fut_opencl_bin };
941 cl_int status = 0;
942
943 prog = clCreateProgramWithBinary(ctx->ctx, 1, &device_option.device,
944 &binary_size, binaries,
945 &status, &error);
946
947 OPENCL_SUCCEED_FATAL(status);
948 OPENCL_SUCCEED_FATAL(error);
949 }
950
951 if (ctx->cfg->logging) {
952 fprintf(stderr, "Building OpenCL program...\n");
953 }
954 char* build_log;
955 cl_build_status status =
956 build_gpu_program(prog, device_option.device, compile_opts, &build_log);
957 free(compile_opts);
958
959 if (status != CL_BUILD_SUCCESS) {
960 ctx->error = msgprintf("Compilation of OpenCL program failed.\nBuild log:\n%s",
961 build_log);
962 // We are giving up on initialising this OpenCL context. That also
963 // means we need to free all the OpenCL bits we have managed to
964 // allocate thus far, as futhark_context_free() will not touch
965 // these unless initialisation was completely successful.
966 (void)clReleaseProgram(prog);
967 (void)clReleaseCommandQueue(ctx->queue);
968 (void)clReleaseContext(ctx->ctx);
969 free(build_log);
970 return;
971 }
972
973 size_t binary_size = 0;
974 unsigned char *binary = NULL;
975 int store_in_cache = cache_fname != NULL && !loaded_from_cache;
976 if (store_in_cache || ctx->cfg->dump_binary_to != NULL) {
977 OPENCL_SUCCEED_FATAL(clGetProgramInfo(prog, CL_PROGRAM_BINARY_SIZES,
978 sizeof(size_t), &binary_size, NULL));
979 binary = (unsigned char*) malloc(binary_size);
980 OPENCL_SUCCEED_FATAL(clGetProgramInfo(prog, CL_PROGRAM_BINARIES,
981 sizeof(unsigned char*), &binary, NULL));
982 }
983
984 if (store_in_cache) {
985 if (ctx->cfg->logging) {
986 fprintf(stderr, "Caching OpenCL binary in %s...\n", cache_fname);
987 }
988 if (cache_store(cache_fname, &h, binary, binary_size) != 0) {
989 printf("Failed to cache binary: %s\n", strerror(errno));
990 }
991 }
992
993 if (ctx->cfg->dump_binary_to != NULL) {
994 if (ctx->cfg->logging) {
995 fprintf(stderr, "Dumping OpenCL binary to %s...\n", ctx->cfg->dump_binary_to);
996 }
997 dump_file(ctx->cfg->dump_binary_to, binary, binary_size);
998 }
999
1000 ctx->clprogram = prog;
1001 }
1002
1003 static struct opencl_device_option get_preferred_device(struct futhark_context *ctx,
1004 const struct futhark_context_config *cfg) {
1005 struct opencl_device_option *devices;
1006 size_t num_devices;
1007
1008 opencl_all_device_options(&devices, &num_devices);
1009
1010 int num_device_matches = 0;
1011
1012 for (size_t i = 0; i < num_devices; i++) {
1013 struct opencl_device_option device = devices[i];
1014 if (strstr(device.platform_name, cfg->preferred_platform) != NULL &&
1015 strstr(device.device_name, cfg->preferred_device) != NULL &&
1016 (cfg->ignore_blacklist ||
1017 !is_blacklisted(device.platform_name, device.device_name, cfg)) &&
1018 num_device_matches++ == cfg->preferred_device_num) {
1019 // Free all the platform and device names, except the ones we have chosen.
1020 for (size_t j = 0; j < num_devices; j++) {
1021 if (j != i) {
1022 free(devices[j].platform_name);
1023 free(devices[j].device_name);
1024 }
1025 }
1026 free(devices);
1027 return device;
1028 }
1029 }
1030
1031 ctx->error = strdup("Could not find acceptable OpenCL device.\n");
1032 struct opencl_device_option device;
1033 return device;
1034 }
1035
1036 static void setup_opencl(struct futhark_context *ctx,
1037 const char *extra_build_opts[],
1038 const char* cache_fname) {
1039 struct opencl_device_option device_option = get_preferred_device(ctx, ctx->cfg);
1040
1041 if (ctx->error != NULL) {
1042 return;
1043 }
1044
1045 if (ctx->cfg->logging) {
1046 fprintf(stderr, "Using platform: %s\n", device_option.platform_name);
1047 fprintf(stderr, "Using device: %s\n", device_option.device_name);
1048 }
1049
1050 // Note that NVIDIA's OpenCL requires the platform property
1051 cl_context_properties properties[] = {
1052 CL_CONTEXT_PLATFORM,
1053 (cl_context_properties)device_option.platform,
1054 0
1055 };
1056
1057 cl_int clCreateContext_error;
1058 ctx->ctx = clCreateContext(properties, 1, &device_option.device, NULL, NULL, &clCreateContext_error);
1059 OPENCL_SUCCEED_FATAL(clCreateContext_error);
1060
1061 cl_int clCreateCommandQueue_error;
1062 cl_command_queue queue =
1063 clCreateCommandQueue(ctx->ctx,
1064 device_option.device,
1065 ctx->cfg->profiling ? CL_QUEUE_PROFILING_ENABLE : 0,
1066 &clCreateCommandQueue_error);
1067 OPENCL_SUCCEED_FATAL(clCreateCommandQueue_error);
1068
1069 setup_opencl_with_command_queue(ctx, queue, extra_build_opts, cache_fname);
1070 }
1071
1072 struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);
1073 void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);
1074
1075 int backend_context_setup(struct futhark_context* ctx) {
1076 ctx->lockstep_width = 0; // Real value set later.
1077 ctx->failure_is_an_option = 0;
1078 ctx->total_runs = 0;
1079 ctx->total_runtime = 0;
1080 ctx->peak_mem_usage_device = 0;
1081 ctx->cur_mem_usage_device = 0;
1082 ctx->kernels = NULL;
1083
1084 if (ctx->cfg->queue_set) {
1085 setup_opencl_with_command_queue(ctx, ctx->cfg->queue, (const char**)ctx->cfg->build_opts, ctx->cfg->cache_fname);
1086 } else {
1087 setup_opencl(ctx, (const char**)ctx->cfg->build_opts, ctx->cfg->cache_fname);
1088 }
1089
1090 if (ctx->error != NULL) {
1091 return 1;
1092 }
1093
1094 cl_int error;
1095 cl_int no_error = -1;
1096 ctx->global_failure =
1097 clCreateBuffer(ctx->ctx,
1098 CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR,
1099 sizeof(cl_int), &no_error, &error);
1100 OPENCL_SUCCEED_OR_RETURN(error);
1101
1102 // The +1 is to avoid zero-byte allocations.
1103 ctx->global_failure_args =
1104 clCreateBuffer(ctx->ctx,
1105 CL_MEM_READ_WRITE,
1106 sizeof(int64_t)*(max_failure_args+1), NULL, &error);
1107 OPENCL_SUCCEED_OR_RETURN(error);
1108
1109 if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) {
1110 return 1;
1111 }
1112
1113 return FUTHARK_SUCCESS;
1114 }
1115
1116 static int gpu_free_all(struct futhark_context *ctx);
1117
1118 void backend_context_teardown(struct futhark_context* ctx) {
1119 if (ctx->kernels != NULL) {
1120 free_builtin_kernels(ctx, ctx->kernels);
1121 OPENCL_SUCCEED_FATAL(clReleaseMemObject(ctx->global_failure));
1122 OPENCL_SUCCEED_FATAL(clReleaseMemObject(ctx->global_failure_args));
1123 (void)gpu_free_all(ctx);
1124 (void)clReleaseProgram(ctx->clprogram);
1125 (void)clReleaseCommandQueue(ctx->queue);
1126 (void)clReleaseContext(ctx->ctx);
1127 }
1128 free_list_destroy(&ctx->gpu_free_list);
1129 }
1130
1131 cl_command_queue futhark_context_get_command_queue(struct futhark_context* ctx) {
1132 return ctx->queue;
1133 }
1134
1135 // GPU ABSTRACTION LAYER
1136
1137 // Types.
1138
1139 typedef cl_kernel gpu_kernel;
1140 typedef cl_mem gpu_mem;
1141
1142 static void gpu_create_kernel(struct futhark_context *ctx,
1143 gpu_kernel* kernel,
1144 const char* name) {
1145 if (ctx->debugging) {
1146 fprintf(ctx->log, "Creating kernel %s.\n", name);
1147 }
1148 cl_int error;
1149 *kernel = clCreateKernel(ctx->clprogram, name, &error);
1150 OPENCL_SUCCEED_FATAL(error);
1151 }
1152
1153 static void gpu_free_kernel(struct futhark_context *ctx,
1154 gpu_kernel kernel) {
1155 (void)ctx;
1156 clReleaseKernel(kernel);
1157 }
1158
1159 static int gpu_scalar_to_device(struct futhark_context* ctx,
1160 const char *provenance,
1161 gpu_mem dst, size_t offset, size_t size,
1162 void *src) {
1163 cl_event* event = opencl_event_new(ctx);
1164 if (event != NULL) {
1165 add_event(ctx,
1166 "copy_scalar_to_dev",
1167 provenance,
1168 NULL,
1169 event,
1170 (event_report_fn)opencl_event_report);
1171 }
1172 OPENCL_SUCCEED_OR_RETURN
1173 (clEnqueueWriteBuffer
1174 (ctx->queue, dst, CL_TRUE,
1175 offset, size, src, 0, NULL, event));
1176 return 0;
1177 }
1178
1179 static int gpu_scalar_from_device(struct futhark_context* ctx,
1180 const char *provenance,
1181 void *dst,
1182 gpu_mem src, size_t offset, size_t size) {
1183 cl_event* event = opencl_event_new(ctx);
1184 if (event != NULL) {
1185 add_event(ctx,
1186 "copy_scalar_from_dev",
1187 provenance,
1188 NULL,
1189 event,
1190 (event_report_fn)opencl_event_report);
1191 }
1192 OPENCL_SUCCEED_OR_RETURN
1193 (clEnqueueReadBuffer
1194 (ctx->queue, src, ctx->failure_is_an_option ? CL_FALSE : CL_TRUE,
1195 offset, size, dst, 0, NULL, event));
1196 return 0;
1197 }
1198
1199 static int gpu_memcpy(struct futhark_context* ctx, const char *provenance,
1200 gpu_mem dst, int64_t dst_offset,
1201 gpu_mem src, int64_t src_offset,
1202 int64_t nbytes) {
1203 if (nbytes > 0) {
1204 cl_event* event = opencl_event_new(ctx);
1205 if (event != NULL) {
1206 add_event(ctx,
1207 "copy_dev_to_dev",
1208 provenance,
1209 NULL,
1210 event,
1211 (event_report_fn)opencl_event_report);
1212 }
1213 // OpenCL swaps the usual order of operands for memcpy()-like
1214 // functions. The order below is not a typo.
1215 OPENCL_SUCCEED_OR_RETURN
1216 (clEnqueueCopyBuffer
1217 (ctx->queue, src, dst, src_offset, dst_offset, nbytes,
1218 0, NULL, event));
1219 if (ctx->debugging) {
1220 OPENCL_SUCCEED_FATAL(clFinish(ctx->queue));
1221 }
1222 }
1223 return FUTHARK_SUCCESS;
1224 }
1225
1226 static int memcpy_host2gpu(struct futhark_context* ctx, const char *provenance,
1227 bool sync,
1228 gpu_mem dst, int64_t dst_offset,
1229 const unsigned char* src, int64_t src_offset,
1230 int64_t nbytes) {
1231 if (nbytes > 0) {
1232 cl_event* event = opencl_event_new(ctx);
1233 if (event != NULL) {
1234 add_event(ctx,
1235 "copy_host_to_dev",
1236 provenance,
1237 NULL,
1238 event,
1239 (event_report_fn)opencl_event_report);
1240 }
1241 OPENCL_SUCCEED_OR_RETURN
1242 (clEnqueueWriteBuffer(ctx->queue,
1243 dst,
1244 sync ? CL_TRUE : CL_FALSE,
1245 (size_t)dst_offset, (size_t)nbytes,
1246 src + src_offset,
1247 0, NULL, event));
1248 if (ctx->debugging) {
1249 OPENCL_SUCCEED_FATAL(clFinish(ctx->queue));
1250 }
1251 }
1252 return FUTHARK_SUCCESS;
1253 }
1254
1255 static int memcpy_gpu2host(struct futhark_context* ctx, const char *provenance,
1256 bool sync,
1257 unsigned char* dst, int64_t dst_offset,
1258 gpu_mem src, int64_t src_offset,
1259 int64_t nbytes) {
1260 if (nbytes > 0) {
1261 cl_event* event = opencl_event_new(ctx);
1262 if (event != NULL) {
1263 add_event(ctx,
1264 "copy_dev_to_host",
1265 provenance,
1266 NULL,
1267 event,
1268 (event_report_fn)opencl_event_report);
1269 }
1270 OPENCL_SUCCEED_OR_RETURN
1271 (clEnqueueReadBuffer(ctx->queue, src,
1272 ctx->failure_is_an_option ? CL_FALSE
1273 : sync ? CL_TRUE : CL_FALSE,
1274 src_offset, nbytes,
1275 dst + dst_offset,
1276 0, NULL, event));
1277 if (sync &&
1278 ctx->failure_is_an_option &&
1279 futhark_context_sync(ctx) != 0) {
1280 return 1;
1281 }
1282 }
1283 return FUTHARK_SUCCESS;
1284 }
1285
1286 static int gpu_launch_kernel(struct futhark_context* ctx,
1287 gpu_kernel kernel,
1288 const char *name,
1289 const char *provenance,
1290 const int32_t grid[3],
1291 const int32_t block[3],
1292 unsigned int shared_mem_bytes,
1293 int num_args,
1294 void* args[num_args],
1295 size_t args_sizes[num_args]) {
1296 if (shared_mem_bytes > ctx->max_shared_memory) {
1297 set_error(ctx, msgprintf("Kernel %s with %d bytes of memory exceeds device limit of %d\n",
1298 name, shared_mem_bytes, (int)ctx->max_shared_memory));
1299 return 1;
1300 }
1301
1302 int64_t time_start = 0, time_end = 0;
1303
1304 cl_event* event = opencl_event_new(ctx);
1305 if (event != NULL) {
1306
1307 struct kvs *kvs = kvs_new();
1308 kvs_printf(kvs, "kernel", "\"%s\"", name);
1309 kvs_printf(kvs, "grid", "[%d,%d,%d]", grid[0], grid[1], grid[2]);
1310 kvs_printf(kvs, "block", "[%d,%d,%d]", block[0], block[1], block[2]);
1311 kvs_printf(kvs, "shared memory", "%d", shared_mem_bytes);
1312
1313 add_event(ctx,
1314 name,
1315 provenance,
1316 kvs,
1317 event,
1318 (event_report_fn)opencl_event_report);
1319 }
1320
1321 if (ctx->debugging) {
1322 time_start = get_wall_time();
1323 }
1324
1325 // Some implementations do not work with 0-byte shared memory.
1326 if (shared_mem_bytes == 0) {
1327 shared_mem_bytes = 4;
1328 }
1329
1330 OPENCL_SUCCEED_OR_RETURN
1331 (clSetKernelArg(kernel, 0, shared_mem_bytes, NULL));
1332 for (int i = 0; i < num_args; i++) {
1333 OPENCL_SUCCEED_OR_RETURN
1334 (clSetKernelArg(kernel, i+1, args_sizes[i], args[i]));
1335 }
1336
1337 const size_t global_work_size[3] =
1338 {(size_t)grid[0]*block[0],
1339 (size_t)grid[1]*block[1],
1340 (size_t)grid[2]*block[2]};
1341 const size_t local_work_size[3] =
1342 {block[0],
1343 block[1],
1344 block[2]};
1345
1346 OPENCL_SUCCEED_OR_RETURN
1347 (clEnqueueNDRangeKernel(ctx->queue,
1348 kernel,
1349 3, NULL, global_work_size, local_work_size,
1350 0, NULL, event));
1351
1352 if (ctx->debugging) {
1353 OPENCL_SUCCEED_FATAL(clFinish(ctx->queue));
1354 time_end = get_wall_time();
1355 long int time_diff = time_end - time_start;
1356 fprintf(ctx->log, " runtime: %ldus\n", time_diff);
1357 }
1358 if (ctx->logging) {
1359 fprintf(ctx->log, "\n");
1360 }
1361
1362 return FUTHARK_SUCCESS;
1363 }
1364
1365 // Allocate memory from driver. The problem is that OpenCL may perform
1366 // lazy allocation, so we cannot know whether an allocation succeeded
1367 // until the first time we try to use it. Hence we immediately
1368 // perform a write to see if the allocation succeeded. This is slow,
1369 // but the assumption is that this operation will be rare (most things
1370 // will go through the free list).
1371 static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) {
1372 int error;
1373 *mem_out = clCreateBuffer(ctx->ctx, CL_MEM_READ_WRITE, size, NULL, &error);
1374
1375 OPENCL_SUCCEED_OR_RETURN(error);
1376
1377 int x = 2;
1378 error = clEnqueueWriteBuffer(ctx->queue, *mem_out,
1379 CL_TRUE,
1380 0, sizeof(x), &x,
1381 0, NULL, NULL);
1382
1383 // No need to wait for completion here. clWaitForEvents() cannot
1384 // return mem object allocation failures. This implies that the
1385 // buffer is faulted onto the device on enqueue. (Observation by
1386 // Andreas Kloeckner.)
1387
1388 if (error == CL_MEM_OBJECT_ALLOCATION_FAILURE) {
1389 return FUTHARK_OUT_OF_MEMORY;
1390 }
1391 OPENCL_SUCCEED_OR_RETURN(error);
1392 return FUTHARK_SUCCESS;
1393 }
1394
1395 static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) {
1396 (void)ctx;
1397 OPENCL_SUCCEED_OR_RETURN(clReleaseMemObject(mem));
1398 return FUTHARK_SUCCESS;
1399 }
1400
1401 // End of backends/opencl.h