| 1 |
// Start of backends/hip.h. |
| 2 |
|
| 3 |
// Forward declarations. |
| 4 |
// Invoked by setup_opencl() after the platform and device has been |
| 5 |
// found, but before the program is loaded. Its intended use is to |
| 6 |
// tune constants based on the selected platform and device. |
| 7 |
static void set_tuning_params(struct futhark_context* ctx); |
| 8 |
static char* get_failure_msg(int failure_idx, int64_t args[]); |
| 9 |
|
| 10 |
#define HIP_SUCCEED_FATAL(x) hip_api_succeed_fatal(x, #x, __FILE__, __LINE__) |
| 11 |
#define HIP_SUCCEED_NONFATAL(x) hip_api_succeed_nonfatal(x, #x, __FILE__, __LINE__) |
| 12 |
#define HIPRTC_SUCCEED_FATAL(x) hiprtc_api_succeed_fatal(x, #x, __FILE__, __LINE__) |
| 13 |
#define HIPRTC_SUCCEED_NONFATAL(x) hiprtc_api_succeed_nonfatal(x, #x, __FILE__, __LINE__) |
| 14 |
// Take care not to override an existing error. |
| 15 |
#define HIP_SUCCEED_OR_RETURN(e) { \ |
| 16 |
char *serror = HIP_SUCCEED_NONFATAL(e); \ |
| 17 |
if (serror) { \ |
| 18 |
if (!ctx->error) { \ |
| 19 |
ctx->error = serror; \ |
| 20 |
} else { \ |
| 21 |
free(serror); \ |
| 22 |
} \ |
| 23 |
return bad; \ |
| 24 |
} \ |
| 25 |
} |
| 26 |
|
| 27 |
// HIP_SUCCEED_OR_RETURN returns the value of the variable 'bad' in |
| 28 |
// scope. By default, it will be this one. Create a local variable |
| 29 |
// of some other type if needed. This is a bit of a hack, but it |
| 30 |
// saves effort in the code generator. |
| 31 |
static const int bad = 1; |
| 32 |
|
| 33 |
static inline void hip_api_succeed_fatal(hipError_t res, const char *call, |
| 34 |
const char *file, int line) { |
| 35 |
if (res != hipSuccess) { |
| 36 |
const char *err_str = hipGetErrorString(res); |
| 37 |
if (err_str == NULL) { err_str = "Unknown"; } |
| 38 |
futhark_panic(-1, "%s:%d: HIP call\n %s\nfailed with error code %d (%s)\n", |
| 39 |
file, line, call, res, err_str); |
| 40 |
} |
| 41 |
} |
| 42 |
|
| 43 |
static char* hip_api_succeed_nonfatal(hipError_t res, const char *call, |
| 44 |
const char *file, int line) { |
| 45 |
if (res != hipSuccess) { |
| 46 |
const char *err_str = hipGetErrorString(res); |
| 47 |
if (err_str == NULL) { err_str = "Unknown"; } |
| 48 |
return msgprintf("%s:%d: HIP call\n %s\nfailed with error code %d (%s)\n", |
| 49 |
file, line, call, res, err_str); |
| 50 |
} else { |
| 51 |
return NULL; |
| 52 |
} |
| 53 |
} |
| 54 |
|
| 55 |
static inline void hiprtc_api_succeed_fatal(hiprtcResult res, const char *call, |
| 56 |
const char *file, int line) { |
| 57 |
if (res != HIPRTC_SUCCESS) { |
| 58 |
const char *err_str = hiprtcGetErrorString(res); |
| 59 |
futhark_panic(-1, "%s:%d: HIPRTC call\n %s\nfailed with error code %d (%s)\n", |
| 60 |
file, line, call, res, err_str); |
| 61 |
} |
| 62 |
} |
| 63 |
|
| 64 |
static char* hiprtc_api_succeed_nonfatal(hiprtcResult res, const char *call, |
| 65 |
const char *file, int line) { |
| 66 |
if (res != HIPRTC_SUCCESS) { |
| 67 |
const char *err_str = hiprtcGetErrorString(res); |
| 68 |
return msgprintf("%s:%d: HIPRTC call\n %s\nfailed with error code %d (%s)\n", |
| 69 |
file, line, call, res, err_str); |
| 70 |
} else { |
| 71 |
return NULL; |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
struct futhark_context_config { |
| 76 |
int in_use; |
| 77 |
int debugging; |
| 78 |
int profiling; |
| 79 |
int logging; |
| 80 |
char* cache_fname; |
| 81 |
int num_tuning_params; |
| 82 |
int64_t *tuning_params; |
| 83 |
const char** tuning_param_names; |
| 84 |
const char** tuning_param_vars; |
| 85 |
const char** tuning_param_classes; |
| 86 |
// Uniform fields above. |
| 87 |
|
| 88 |
char* program; |
| 89 |
int num_build_opts; |
| 90 |
char* *build_opts; |
| 91 |
|
| 92 |
int unified_memory; |
| 93 |
|
| 94 |
char* preferred_device; |
| 95 |
int preferred_device_num; |
| 96 |
|
| 97 |
struct gpu_config gpu; |
| 98 |
}; |
| 99 |
|
| 100 |
static void backend_context_config_setup(struct futhark_context_config *cfg) { |
| 101 |
cfg->num_build_opts = 0; |
| 102 |
cfg->build_opts = (char**) malloc(sizeof(char*)); |
| 103 |
cfg->build_opts[0] = NULL; |
| 104 |
cfg->preferred_device_num = 0; |
| 105 |
cfg->preferred_device = strdup(""); |
| 106 |
cfg->program = strconcat(gpu_program); |
| 107 |
|
| 108 |
cfg->unified_memory = 0; |
| 109 |
|
| 110 |
cfg->gpu = gpu_config_initial; |
| 111 |
cfg->gpu.default_block_size = 256; |
| 112 |
cfg->gpu.default_tile_size = 32; |
| 113 |
cfg->gpu.default_reg_tile_size = 2; |
| 114 |
cfg->gpu.default_threshold = 32*1024; |
| 115 |
} |
| 116 |
|
| 117 |
static void backend_context_config_teardown(struct futhark_context_config* cfg) { |
| 118 |
for (int i = 0; i < cfg->num_build_opts; i++) { |
| 119 |
free(cfg->build_opts[i]); |
| 120 |
} |
| 121 |
|
| 122 |
free(cfg->build_opts); |
| 123 |
free(cfg->preferred_device); |
| 124 |
free(cfg->program); |
| 125 |
} |
| 126 |
|
| 127 |
void futhark_context_config_add_build_option(struct futhark_context_config *cfg, const char *opt) { |
| 128 |
cfg->build_opts[cfg->num_build_opts] = strdup(opt); |
| 129 |
cfg->num_build_opts++; |
| 130 |
cfg->build_opts = (char **) realloc(cfg->build_opts, (cfg->num_build_opts + 1) * sizeof(char *)); |
| 131 |
cfg->build_opts[cfg->num_build_opts] = NULL; |
| 132 |
} |
| 133 |
|
| 134 |
void futhark_context_config_set_device(struct futhark_context_config *cfg, const char *s) { |
| 135 |
int x = 0; |
| 136 |
if (*s == '#') { |
| 137 |
s++; |
| 138 |
while (isdigit(*s)) { |
| 139 |
x = x * 10 + (*s++)-'0'; |
| 140 |
} |
| 141 |
// Skip trailing spaces. |
| 142 |
while (isspace(*s)) { |
| 143 |
s++; |
| 144 |
} |
| 145 |
} |
| 146 |
free(cfg->preferred_device); |
| 147 |
cfg->preferred_device = strdup(s); |
| 148 |
cfg->preferred_device_num = x; |
| 149 |
} |
| 150 |
|
| 151 |
|
| 152 |
const char* futhark_context_config_get_program(struct futhark_context_config *cfg) { |
| 153 |
return cfg->program; |
| 154 |
} |
| 155 |
|
| 156 |
void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) { |
| 157 |
free(cfg->program); |
| 158 |
cfg->program = strdup(s); |
| 159 |
} |
| 160 |
|
| 161 |
void futhark_context_config_set_unified_memory(struct futhark_context_config* cfg, int flag) { |
| 162 |
cfg->unified_memory = flag; |
| 163 |
} |
| 164 |
|
| 165 |
struct futhark_context { |
| 166 |
struct futhark_context_config* cfg; |
| 167 |
int detail_memory; |
| 168 |
int debugging; |
| 169 |
int profiling; |
| 170 |
int profiling_paused; |
| 171 |
int logging; |
| 172 |
lock_t lock; |
| 173 |
char *error; |
| 174 |
lock_t error_lock; |
| 175 |
FILE *log; |
| 176 |
struct constants *constants; |
| 177 |
struct free_list free_list; |
| 178 |
struct event_list event_list; |
| 179 |
int64_t peak_mem_usage_default; |
| 180 |
int64_t cur_mem_usage_default; |
| 181 |
bool program_initialised; |
| 182 |
// Uniform fields above. |
| 183 |
|
| 184 |
void* global_failure; |
| 185 |
void* global_failure_args; |
| 186 |
struct tuning_params tuning_params; |
| 187 |
// True if a potentially failing kernel has been enqueued. |
| 188 |
int32_t failure_is_an_option; |
| 189 |
int total_runs; |
| 190 |
long int total_runtime; |
| 191 |
int64_t peak_mem_usage_device; |
| 192 |
int64_t cur_mem_usage_device; |
| 193 |
struct program* program; |
| 194 |
|
| 195 |
hipDevice_t dev; |
| 196 |
int dev_id; |
| 197 |
hipModule_t module; |
| 198 |
hipStream_t stream; |
| 199 |
|
| 200 |
struct free_list gpu_free_list; |
| 201 |
|
| 202 |
size_t max_thread_block_size; |
| 203 |
size_t max_grid_size; |
| 204 |
size_t max_tile_size; |
| 205 |
size_t max_threshold; |
| 206 |
size_t max_shared_memory; |
| 207 |
size_t max_bespoke; |
| 208 |
size_t max_registers; |
| 209 |
size_t max_cache; |
| 210 |
|
| 211 |
size_t lockstep_width; |
| 212 |
|
| 213 |
struct builtin_kernels* kernels; |
| 214 |
}; |
| 215 |
|
| 216 |
static int device_query(int dev_id, hipDeviceAttribute_t attr) { |
| 217 |
int val; |
| 218 |
HIP_SUCCEED_FATAL(hipDeviceGetAttribute(&val, attr, dev_id)); |
| 219 |
return val; |
| 220 |
} |
| 221 |
|
| 222 |
static int function_query(hipFunction_t f, hipFunction_attribute attr) { |
| 223 |
int val; |
| 224 |
HIP_SUCCEED_FATAL(hipFuncGetAttribute(&val, attr, f)); |
| 225 |
return val; |
| 226 |
} |
| 227 |
|
| 228 |
static int hip_device_setup(struct futhark_context *ctx) { |
| 229 |
struct futhark_context_config *cfg = ctx->cfg; |
| 230 |
int count, chosen = -1; |
| 231 |
hipDevice_t dev; |
| 232 |
|
| 233 |
HIP_SUCCEED_FATAL(hipGetDeviceCount(&count)); |
| 234 |
if (count == 0) { return 1; } |
| 235 |
|
| 236 |
int num_device_matches = 0; |
| 237 |
|
| 238 |
for (int i = 0; i < count; i++) { |
| 239 |
hipDeviceProp_t prop; |
| 240 |
hipGetDeviceProperties(&prop, i); |
| 241 |
|
| 242 |
if (cfg->logging) { |
| 243 |
fprintf(ctx->log, "Device #%d: name=\"%s\"\n", i, prop.name); |
| 244 |
} |
| 245 |
|
| 246 |
if (strstr(prop.name, cfg->preferred_device) != NULL && |
| 247 |
num_device_matches++ == cfg->preferred_device_num) { |
| 248 |
chosen = i; |
| 249 |
break; |
| 250 |
} |
| 251 |
} |
| 252 |
|
| 253 |
if (chosen == -1) { return 1; } |
| 254 |
|
| 255 |
if (cfg->logging) { |
| 256 |
fprintf(ctx->log, "Using device #%d\n", chosen); |
| 257 |
} |
| 258 |
|
| 259 |
ctx->dev_id = chosen; |
| 260 |
HIP_SUCCEED_FATAL(hipDeviceGet(&ctx->dev, ctx->dev_id)); |
| 261 |
return 0; |
| 262 |
} |
| 263 |
|
| 264 |
static void hip_load_code_from_cache(struct futhark_context_config *cfg, |
| 265 |
const char *src, |
| 266 |
const char *opts[], size_t n_opts, |
| 267 |
struct cache_hash *h, const char *cache_fname, |
| 268 |
char **code, size_t *code_size) { |
| 269 |
if (cfg->logging) { |
| 270 |
fprintf(stderr, "Restoring cache from from %s...\n", cache_fname); |
| 271 |
} |
| 272 |
cache_hash_init(h); |
| 273 |
for (size_t i = 0; i < n_opts; i++) { |
| 274 |
cache_hash(h, opts[i], strlen(opts[i])); |
| 275 |
} |
| 276 |
cache_hash(h, src, strlen(src)); |
| 277 |
errno = 0; |
| 278 |
if (cache_restore(cache_fname, h, (unsigned char**)code, code_size) != 0) { |
| 279 |
if (cfg->logging) { |
| 280 |
fprintf(stderr, "Failed to restore cache (errno: %s)\n", strerror(errno)); |
| 281 |
} |
| 282 |
} |
| 283 |
} |
| 284 |
|
| 285 |
static void hip_size_setup(struct futhark_context *ctx) { |
| 286 |
struct futhark_context_config *cfg = ctx->cfg; |
| 287 |
if (cfg->gpu.default_block_size > ctx->max_thread_block_size) { |
| 288 |
if (cfg->gpu.default_block_size_changed) { |
| 289 |
fprintf(stderr, |
| 290 |
"Note: Device limits default block size to %zu (down from %zu).\n", |
| 291 |
ctx->max_thread_block_size, cfg->gpu.default_block_size); |
| 292 |
} |
| 293 |
cfg->gpu.default_block_size = ctx->max_thread_block_size; |
| 294 |
} |
| 295 |
if (cfg->gpu.default_grid_size > ctx->max_grid_size) { |
| 296 |
if (cfg->gpu.default_grid_size_changed) { |
| 297 |
fprintf(stderr, |
| 298 |
"Note: Device limits default grid size to %zu (down from %zu).\n", |
| 299 |
ctx->max_grid_size, cfg->gpu.default_grid_size); |
| 300 |
} |
| 301 |
cfg->gpu.default_grid_size = ctx->max_grid_size; |
| 302 |
} |
| 303 |
if (cfg->gpu.default_tile_size > ctx->max_tile_size) { |
| 304 |
if (cfg->gpu.default_tile_size_changed) { |
| 305 |
fprintf(stderr, |
| 306 |
"Note: Device limits default tile size to %zu (down from %zu).\n", |
| 307 |
ctx->max_tile_size, cfg->gpu.default_tile_size); |
| 308 |
} |
| 309 |
cfg->gpu.default_tile_size = ctx->max_tile_size; |
| 310 |
} |
| 311 |
|
| 312 |
if (!cfg->gpu.default_grid_size_changed) { |
| 313 |
cfg->gpu.default_grid_size = |
| 314 |
(device_query(ctx->dev, hipDeviceAttributePhysicalMultiProcessorCount) * |
| 315 |
device_query(ctx->dev, hipDeviceAttributeMaxThreadsPerMultiProcessor)) |
| 316 |
/ cfg->gpu.default_block_size; |
| 317 |
} |
| 318 |
|
| 319 |
for (int i = 0; i < cfg->num_tuning_params; i++) { |
| 320 |
const char *size_class = cfg->tuning_param_classes[i]; |
| 321 |
int64_t *size_value = &cfg->tuning_params[i]; |
| 322 |
const char* size_name = cfg->tuning_param_names[i]; |
| 323 |
int64_t max_value = 0, default_value = 0; |
| 324 |
|
| 325 |
if (strstr(size_class, "thread_block_size") == size_class) { |
| 326 |
max_value = ctx->max_thread_block_size; |
| 327 |
default_value = cfg->gpu.default_block_size; |
| 328 |
} else if (strstr(size_class, "grid_size") == size_class) { |
| 329 |
max_value = ctx->max_grid_size; |
| 330 |
default_value = cfg->gpu.default_grid_size; |
| 331 |
// XXX: as a quick and dirty hack, use twice as many threads for |
| 332 |
// histograms by default. We really should just be smarter |
| 333 |
// about sizes somehow. |
| 334 |
if (strstr(size_name, ".seghist_") != NULL) { |
| 335 |
default_value *= 2; |
| 336 |
} |
| 337 |
} else if (strstr(size_class, "tile_size") == size_class) { |
| 338 |
max_value = ctx->max_tile_size; |
| 339 |
default_value = cfg->gpu.default_tile_size; |
| 340 |
} else if (strstr(size_class, "reg_tile_size") == size_class) { |
| 341 |
max_value = 0; // No limit. |
| 342 |
default_value = cfg->gpu.default_reg_tile_size; |
| 343 |
} else if (strstr(size_class, "shared_memory") == size_class) { |
| 344 |
max_value = ctx->max_shared_memory; |
| 345 |
default_value = ctx->max_shared_memory; |
| 346 |
} else if (strstr(size_class, "cache") == size_class) { |
| 347 |
max_value = ctx->max_cache; |
| 348 |
default_value = ctx->max_cache; |
| 349 |
} else if (strstr(size_class, "threshold") == size_class) { |
| 350 |
// Threshold can be as large as it takes. |
| 351 |
default_value = cfg->gpu.default_threshold; |
| 352 |
} else { |
| 353 |
// Bespoke sizes have no limit or default. |
| 354 |
} |
| 355 |
|
| 356 |
if (*size_value == 0) { |
| 357 |
*size_value = default_value; |
| 358 |
} else if (max_value > 0 && *size_value > max_value) { |
| 359 |
fprintf(stderr, "Note: Device limits %s to %zu (down from %zu)\n", |
| 360 |
size_name, max_value, *size_value); |
| 361 |
*size_value = max_value; |
| 362 |
} |
| 363 |
} |
| 364 |
} |
| 365 |
|
| 366 |
static char* hiprtc_build(const char *src, const char *opts[], size_t n_opts, |
| 367 |
char **code, size_t *code_size) { |
| 368 |
hiprtcProgram prog; |
| 369 |
char *problem = NULL; |
| 370 |
|
| 371 |
problem = HIPRTC_SUCCEED_NONFATAL(hiprtcCreateProgram(&prog, src, "futhark-hip", 0, NULL, NULL)); |
| 372 |
|
| 373 |
if (problem) { |
| 374 |
return problem; |
| 375 |
} |
| 376 |
|
| 377 |
hiprtcResult res = hiprtcCompileProgram(prog, n_opts, opts); |
| 378 |
if (res != HIPRTC_SUCCESS) { |
| 379 |
size_t log_size; |
| 380 |
if (hiprtcGetProgramLogSize(prog, &log_size) == HIPRTC_SUCCESS) { |
| 381 |
char *log = (char*) malloc(log_size+1); |
| 382 |
log[log_size] = 0; // HIPRTC does not zero-terminate. |
| 383 |
if (hiprtcGetProgramLog(prog, log) == HIPRTC_SUCCESS) { |
| 384 |
problem = msgprintf("HIPRTC compilation failed.\n\n%s\n", log); |
| 385 |
} else { |
| 386 |
problem = msgprintf("Could not retrieve compilation log\n"); |
| 387 |
} |
| 388 |
free(log); |
| 389 |
} |
| 390 |
return problem; |
| 391 |
} |
| 392 |
|
| 393 |
HIPRTC_SUCCEED_FATAL(hiprtcGetCodeSize(prog, code_size)); |
| 394 |
*code = (char*) malloc(*code_size); |
| 395 |
HIPRTC_SUCCEED_FATAL(hiprtcGetCode(prog, *code)); |
| 396 |
HIPRTC_SUCCEED_FATAL(hiprtcDestroyProgram(&prog)); |
| 397 |
return NULL; |
| 398 |
} |
| 399 |
|
| 400 |
static void hiprtc_mk_build_options(struct futhark_context *ctx, const char *extra_opts[], |
| 401 |
char*** opts_out, size_t *n_opts) { |
| 402 |
int arch_set = 0, num_extra_opts; |
| 403 |
struct futhark_context_config *cfg = ctx->cfg; |
| 404 |
|
| 405 |
char** macro_names; |
| 406 |
int64_t* macro_vals; |
| 407 |
int num_macros = gpu_macros(ctx, ¯o_names, ¯o_vals); |
| 408 |
|
| 409 |
for (num_extra_opts = 0; extra_opts[num_extra_opts] != NULL; num_extra_opts++) { |
| 410 |
if (strstr(extra_opts[num_extra_opts], "--gpu-architecture") |
| 411 |
== extra_opts[num_extra_opts]) { |
| 412 |
arch_set = 1; |
| 413 |
} |
| 414 |
} |
| 415 |
|
| 416 |
size_t i = 0, n_opts_alloc = 20 + num_macros + num_extra_opts + cfg->num_tuning_params; |
| 417 |
char **opts = (char**) malloc(n_opts_alloc * sizeof(char *)); |
| 418 |
if (!arch_set) { |
| 419 |
hipDeviceProp_t props; |
| 420 |
HIP_SUCCEED_FATAL(hipGetDeviceProperties(&props, ctx->dev_id)); |
| 421 |
opts[i++] = msgprintf("--gpu-architecture=%s", props.gcnArchName); |
| 422 |
} |
| 423 |
if (cfg->debugging) { |
| 424 |
opts[i++] = strdup("-G"); |
| 425 |
opts[i++] = strdup("-lineinfo"); |
| 426 |
} |
| 427 |
opts[i++] = msgprintf("-D%s=%d", |
| 428 |
"max_thread_block_size", |
| 429 |
(int)ctx->max_thread_block_size); |
| 430 |
opts[i++] = msgprintf("-D%s=%d", |
| 431 |
"max_shared_memory", |
| 432 |
(int)ctx->max_shared_memory); |
| 433 |
opts[i++] = msgprintf("-D%s=%d", |
| 434 |
"max_registers", |
| 435 |
(int)ctx->max_registers); |
| 436 |
|
| 437 |
for (int j = 0; j < num_macros; j++) { |
| 438 |
opts[i++] = msgprintf("-D%s=%zu", macro_names[j], macro_vals[j]); |
| 439 |
} |
| 440 |
|
| 441 |
for (int j = 0; j < cfg->num_tuning_params; j++) { |
| 442 |
opts[i++] = msgprintf("-D%s=%zu", cfg->tuning_param_vars[j], |
| 443 |
cfg->tuning_params[j]); |
| 444 |
} |
| 445 |
opts[i++] = msgprintf("-DLOCKSTEP_WIDTH=%zu", ctx->lockstep_width); |
| 446 |
opts[i++] = msgprintf("-DMAX_THREADS_PER_BLOCK=%zu", ctx->max_thread_block_size); |
| 447 |
|
| 448 |
for (int j = 0; extra_opts[j] != NULL; j++) { |
| 449 |
opts[i++] = strdup(extra_opts[j]); |
| 450 |
} |
| 451 |
|
| 452 |
opts[i++] = msgprintf("-DTR_BLOCK_DIM=%d", TR_BLOCK_DIM); |
| 453 |
opts[i++] = msgprintf("-DTR_TILE_DIM=%d", TR_TILE_DIM); |
| 454 |
opts[i++] = msgprintf("-DTR_ELEMS_PER_THREAD=%d", TR_ELEMS_PER_THREAD); |
| 455 |
|
| 456 |
free(macro_names); |
| 457 |
free(macro_vals); |
| 458 |
|
| 459 |
*n_opts = i; |
| 460 |
*opts_out = opts; |
| 461 |
} |
| 462 |
|
| 463 |
static char* hip_module_setup(struct futhark_context *ctx, |
| 464 |
const char *src, |
| 465 |
const char *extra_opts[], |
| 466 |
const char* cache_fname) { |
| 467 |
char *code = NULL; |
| 468 |
size_t code_size = 0; |
| 469 |
struct futhark_context_config *cfg = ctx->cfg; |
| 470 |
|
| 471 |
char **opts; |
| 472 |
size_t n_opts; |
| 473 |
hiprtc_mk_build_options(ctx, extra_opts, &opts, &n_opts); |
| 474 |
|
| 475 |
if (cfg->logging) { |
| 476 |
fprintf(stderr, "HIPRTC build options:\n"); |
| 477 |
for (size_t j = 0; j < n_opts; j++) { |
| 478 |
fprintf(stderr, "\t%s\n", opts[j]); |
| 479 |
} |
| 480 |
fprintf(stderr, "\n"); |
| 481 |
} |
| 482 |
|
| 483 |
struct cache_hash h; |
| 484 |
int loaded_code_from_cache = 0; |
| 485 |
if (cache_fname != NULL) { |
| 486 |
hip_load_code_from_cache(cfg, src, (const char**)opts, n_opts, &h, cache_fname, &code, &code_size); |
| 487 |
|
| 488 |
if (code != NULL) { |
| 489 |
if (cfg->logging) { |
| 490 |
fprintf(stderr, "Restored compiled code from cache; now loading module...\n"); |
| 491 |
} |
| 492 |
if (hipModuleLoadData(&ctx->module, code) == hipSuccess) { |
| 493 |
if (cfg->logging) { |
| 494 |
fprintf(stderr, "Success!\n"); |
| 495 |
} |
| 496 |
loaded_code_from_cache = 1; |
| 497 |
} else { |
| 498 |
if (cfg->logging) { |
| 499 |
fprintf(stderr, "Failed!\n"); |
| 500 |
} |
| 501 |
free(code); |
| 502 |
code = NULL; |
| 503 |
} |
| 504 |
} |
| 505 |
} |
| 506 |
|
| 507 |
if (code == NULL) { |
| 508 |
char* problem = hiprtc_build(src, (const char**)opts, n_opts, &code, &code_size); |
| 509 |
if (problem != NULL) { |
| 510 |
return problem; |
| 511 |
} |
| 512 |
} |
| 513 |
|
| 514 |
if (!loaded_code_from_cache) { |
| 515 |
HIP_SUCCEED_FATAL(hipModuleLoadData(&ctx->module, code)); |
| 516 |
} |
| 517 |
|
| 518 |
if (cache_fname != NULL && !loaded_code_from_cache) { |
| 519 |
if (cfg->logging) { |
| 520 |
fprintf(stderr, "Caching compiled code in %s...\n", cache_fname); |
| 521 |
} |
| 522 |
errno = 0; |
| 523 |
if (cache_store(cache_fname, &h, (const unsigned char*)code, code_size) != 0) { |
| 524 |
fprintf(stderr, "Failed to cache compiled code: %s\n", strerror(errno)); |
| 525 |
} |
| 526 |
} |
| 527 |
|
| 528 |
for (size_t i = 0; i < n_opts; i++) { |
| 529 |
free((char *)opts[i]); |
| 530 |
} |
| 531 |
free(opts); |
| 532 |
free(code); |
| 533 |
|
| 534 |
return NULL; |
| 535 |
} |
| 536 |
|
| 537 |
struct hip_event { |
| 538 |
hipEvent_t start; |
| 539 |
hipEvent_t end; |
| 540 |
}; |
| 541 |
|
| 542 |
static struct hip_event* hip_event_new(struct futhark_context* ctx) { |
| 543 |
if (ctx->profiling && !ctx->profiling_paused) { |
| 544 |
struct hip_event* e = malloc(sizeof(struct hip_event)); |
| 545 |
hipEventCreate(&e->start); |
| 546 |
hipEventCreate(&e->end); |
| 547 |
return e; |
| 548 |
} else { |
| 549 |
return NULL; |
| 550 |
} |
| 551 |
} |
| 552 |
|
| 553 |
static int hip_event_report(struct str_builder* sb, struct hip_event* e) { |
| 554 |
float ms; |
| 555 |
hipError_t err; |
| 556 |
if ((err = hipEventElapsedTime(&ms, e->start, e->end)) != hipSuccess) { |
| 557 |
return err; |
| 558 |
} |
| 559 |
|
| 560 |
// HIP provides milisecond resolution, but we want microseconds. |
| 561 |
str_builder(sb, ",\"duration\":%f", ms*1000); |
| 562 |
|
| 563 |
if ((err = hipEventDestroy(e->start)) != hipSuccess) { |
| 564 |
return 1; |
| 565 |
} |
| 566 |
|
| 567 |
if ((err = hipEventDestroy(e->end)) != hipSuccess) { |
| 568 |
return 1; |
| 569 |
} |
| 570 |
|
| 571 |
free(e); |
| 572 |
|
| 573 |
return 0; |
| 574 |
} |
| 575 |
|
| 576 |
int futhark_context_sync(struct futhark_context* ctx) { |
| 577 |
HIP_SUCCEED_OR_RETURN(hipStreamSynchronize(ctx->stream)); |
| 578 |
if (ctx->failure_is_an_option) { |
| 579 |
// Check for any delayed error. |
| 580 |
int32_t failure_idx; |
| 581 |
HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(&failure_idx, |
| 582 |
ctx->global_failure, |
| 583 |
sizeof(int32_t))); |
| 584 |
ctx->failure_is_an_option = 0; |
| 585 |
|
| 586 |
if (failure_idx >= 0) { |
| 587 |
// We have to clear global_failure so that the next entry point |
| 588 |
// is not considered a failure from the start. |
| 589 |
int32_t no_failure = -1; |
| 590 |
HIP_SUCCEED_OR_RETURN(hipMemcpyHtoD(ctx->global_failure, |
| 591 |
&no_failure, |
| 592 |
sizeof(int32_t))); |
| 593 |
|
| 594 |
int64_t args[max_failure_args+1]; |
| 595 |
HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(&args, |
| 596 |
ctx->global_failure_args, |
| 597 |
sizeof(args))); |
| 598 |
|
| 599 |
ctx->error = get_failure_msg(failure_idx, args); |
| 600 |
|
| 601 |
return FUTHARK_PROGRAM_ERROR; |
| 602 |
} |
| 603 |
} |
| 604 |
return 0; |
| 605 |
} |
| 606 |
|
| 607 |
struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx); |
| 608 |
void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels); |
| 609 |
|
| 610 |
int backend_context_setup(struct futhark_context* ctx) { |
| 611 |
ctx->failure_is_an_option = 0; |
| 612 |
ctx->total_runs = 0; |
| 613 |
ctx->total_runtime = 0; |
| 614 |
ctx->peak_mem_usage_device = 0; |
| 615 |
ctx->cur_mem_usage_device = 0; |
| 616 |
ctx->kernels = NULL; |
| 617 |
|
| 618 |
HIP_SUCCEED_FATAL(hipInit(0)); |
| 619 |
if (hip_device_setup(ctx) != 0) { |
| 620 |
futhark_panic(-1, "No suitable HIP device found.\n"); |
| 621 |
} |
| 622 |
|
| 623 |
free_list_init(&ctx->gpu_free_list); |
| 624 |
|
| 625 |
if (ctx->cfg->unified_memory == 2) { |
| 626 |
ctx->cfg->unified_memory = device_query(ctx->dev, hipDeviceAttributeManagedMemory); |
| 627 |
} |
| 628 |
|
| 629 |
if (ctx->cfg->logging) { |
| 630 |
if (ctx->cfg->unified_memory) { |
| 631 |
fprintf(ctx->log, "Using managed memory\n"); |
| 632 |
} else { |
| 633 |
fprintf(ctx->log, "Using unmanaged memory\n"); |
| 634 |
} |
| 635 |
} |
| 636 |
|
| 637 |
ctx->max_thread_block_size = device_query(ctx->dev, hipDeviceAttributeMaxThreadsPerBlock); |
| 638 |
ctx->max_grid_size = device_query(ctx->dev, hipDeviceAttributeMaxGridDimX); |
| 639 |
ctx->max_tile_size = sqrt(ctx->max_thread_block_size); |
| 640 |
ctx->max_threshold = 1U<<31; // No limit. |
| 641 |
ctx->max_bespoke = 0; |
| 642 |
ctx->max_registers = device_query(ctx->dev, hipDeviceAttributeMaxRegistersPerBlock); |
| 643 |
|
| 644 |
if (ctx->cfg->gpu.default_shared_memory != 0) { |
| 645 |
ctx->max_shared_memory = ctx->cfg->gpu.default_shared_memory; |
| 646 |
} else { |
| 647 |
ctx->max_shared_memory = device_query(ctx->dev, hipDeviceAttributeMaxSharedMemoryPerBlock); |
| 648 |
} |
| 649 |
|
| 650 |
if (ctx->cfg->gpu.default_cache != 0) { |
| 651 |
ctx->max_cache = ctx->cfg->gpu.default_cache; |
| 652 |
} else { |
| 653 |
ctx->max_cache = device_query(ctx->dev, hipDeviceAttributeL2CacheSize); |
| 654 |
} |
| 655 |
|
| 656 |
// FIXME: in principle we should query hipDeviceAttributeWarpSize |
| 657 |
// from the device, which will provide 64 on AMD GPUs. |
| 658 |
// Unfortunately, we currently do nasty implicit intra-warp |
| 659 |
// synchronisation in codegen, which does not work when this is 64. |
| 660 |
// Once our codegen properly synchronises intra-warp operations, we |
| 661 |
// can use the actual hardware lockstep width instead. |
| 662 |
ctx->lockstep_width = 32; |
| 663 |
HIP_SUCCEED_FATAL(hipStreamCreate(&ctx->stream)); |
| 664 |
hip_size_setup(ctx); |
| 665 |
|
| 666 |
gpu_init_log(ctx); |
| 667 |
|
| 668 |
ctx->error = hip_module_setup(ctx, |
| 669 |
ctx->cfg->program, |
| 670 |
(const char**)ctx->cfg->build_opts, |
| 671 |
ctx->cfg->cache_fname); |
| 672 |
|
| 673 |
if (ctx->error != NULL) { |
| 674 |
futhark_panic(1, "During HIP initialisation:\n%s\n", ctx->error); |
| 675 |
} |
| 676 |
|
| 677 |
int32_t no_error = -1; |
| 678 |
HIP_SUCCEED_FATAL(hipMalloc(&ctx->global_failure, sizeof(no_error))); |
| 679 |
HIP_SUCCEED_FATAL(hipMemcpyHtoD(ctx->global_failure, &no_error, sizeof(no_error))); |
| 680 |
// The +1 is to avoid zero-byte allocations. |
| 681 |
HIP_SUCCEED_FATAL(hipMalloc(&ctx->global_failure_args, sizeof(int64_t)*(max_failure_args+1))); |
| 682 |
|
| 683 |
if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) { |
| 684 |
return 1; |
| 685 |
} |
| 686 |
|
| 687 |
return 0; |
| 688 |
} |
| 689 |
|
| 690 |
void backend_context_teardown(struct futhark_context* ctx) { |
| 691 |
if (ctx->kernels != NULL) { |
| 692 |
free_builtin_kernels(ctx, ctx->kernels); |
| 693 |
hipFree(ctx->global_failure); |
| 694 |
hipFree(ctx->global_failure_args); |
| 695 |
HIP_SUCCEED_FATAL(gpu_free_all(ctx)); |
| 696 |
HIP_SUCCEED_FATAL(hipStreamDestroy(ctx->stream)); |
| 697 |
HIP_SUCCEED_FATAL(hipModuleUnload(ctx->module)); |
| 698 |
} |
| 699 |
free_list_destroy(&ctx->gpu_free_list); |
| 700 |
} |
| 701 |
|
| 702 |
// GPU ABSTRACTION LAYER |
| 703 |
|
| 704 |
typedef hipFunction_t gpu_kernel; |
| 705 |
typedef hipDeviceptr_t gpu_mem; |
| 706 |
|
| 707 |
static void gpu_create_kernel(struct futhark_context *ctx, |
| 708 |
gpu_kernel* kernel, |
| 709 |
const char* name) { |
| 710 |
if (ctx->debugging) { |
| 711 |
fprintf(ctx->log, "Creating kernel %s.\n", name); |
| 712 |
} |
| 713 |
HIP_SUCCEED_FATAL(hipModuleGetFunction(kernel, ctx->module, name)); |
| 714 |
} |
| 715 |
|
| 716 |
static void gpu_free_kernel(struct futhark_context *ctx, |
| 717 |
gpu_kernel kernel) { |
| 718 |
(void)ctx; |
| 719 |
(void)kernel; |
| 720 |
} |
| 721 |
|
| 722 |
static int gpu_scalar_to_device(struct futhark_context* ctx, |
| 723 |
const char *provenance, |
| 724 |
gpu_mem dst, size_t offset, size_t size, |
| 725 |
void *src) { |
| 726 |
struct hip_event *event = hip_event_new(ctx); |
| 727 |
if (event != NULL) { |
| 728 |
add_event(ctx, |
| 729 |
"copy_scalar_to_dev", |
| 730 |
provenance, |
| 731 |
NULL, |
| 732 |
event, |
| 733 |
(event_report_fn)hip_event_report); |
| 734 |
HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream)); |
| 735 |
} |
| 736 |
HIP_SUCCEED_OR_RETURN(hipMemcpyHtoD((unsigned char*)dst + offset, src, size)); |
| 737 |
if (event != NULL) { |
| 738 |
HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream)); |
| 739 |
} |
| 740 |
return FUTHARK_SUCCESS; |
| 741 |
} |
| 742 |
|
| 743 |
static int gpu_scalar_from_device(struct futhark_context* ctx, |
| 744 |
const char *provenance, |
| 745 |
void *dst, |
| 746 |
gpu_mem src, size_t offset, size_t size) { |
| 747 |
struct hip_event *event = hip_event_new(ctx); |
| 748 |
if (event != NULL) { |
| 749 |
add_event(ctx, |
| 750 |
"copy_scalar_from_dev", |
| 751 |
provenance, |
| 752 |
NULL, |
| 753 |
event, |
| 754 |
(event_report_fn)hip_event_report); |
| 755 |
HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream)); |
| 756 |
} |
| 757 |
HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(dst, (unsigned char*)src + offset, size)); |
| 758 |
if (event != NULL) { |
| 759 |
HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream)); |
| 760 |
} |
| 761 |
return FUTHARK_SUCCESS; |
| 762 |
} |
| 763 |
|
| 764 |
static int gpu_memcpy(struct futhark_context* ctx, |
| 765 |
const char *provenance, |
| 766 |
gpu_mem dst, int64_t dst_offset, |
| 767 |
gpu_mem src, int64_t src_offset, |
| 768 |
int64_t nbytes) { |
| 769 |
struct hip_event *event = hip_event_new(ctx); |
| 770 |
if (event != NULL) { |
| 771 |
add_event(ctx, |
| 772 |
"copy_dev_to_dev", |
| 773 |
provenance, |
| 774 |
NULL, |
| 775 |
event, |
| 776 |
(event_report_fn)hip_event_report); |
| 777 |
HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream)); |
| 778 |
} |
| 779 |
HIP_SUCCEED_OR_RETURN(hipMemcpyWithStream((unsigned char*)dst+dst_offset, (unsigned char*)src+src_offset, |
| 780 |
nbytes, hipMemcpyDeviceToDevice, ctx->stream)); |
| 781 |
if (event != NULL) { |
| 782 |
HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream)); |
| 783 |
} |
| 784 |
return FUTHARK_SUCCESS; |
| 785 |
} |
| 786 |
|
| 787 |
static int memcpy_host2gpu(struct futhark_context* ctx, |
| 788 |
const char *provenance, |
| 789 |
bool sync, |
| 790 |
gpu_mem dst, int64_t dst_offset, |
| 791 |
const unsigned char* src, int64_t src_offset, |
| 792 |
int64_t nbytes) { |
| 793 |
if (nbytes > 0) { |
| 794 |
struct hip_event *event = hip_event_new(ctx); |
| 795 |
if (event != NULL) { |
| 796 |
add_event(ctx, |
| 797 |
"copy_host_to_dev", |
| 798 |
provenance, |
| 799 |
NULL, |
| 800 |
event, |
| 801 |
(event_report_fn)hip_event_report); |
| 802 |
HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream)); |
| 803 |
} |
| 804 |
if (sync) { |
| 805 |
HIP_SUCCEED_OR_RETURN |
| 806 |
(hipMemcpyHtoD((unsigned char*)dst + dst_offset, |
| 807 |
(unsigned char*)src + src_offset, nbytes)); |
| 808 |
} else { |
| 809 |
HIP_SUCCEED_OR_RETURN |
| 810 |
(hipMemcpyHtoDAsync((unsigned char*)dst + dst_offset, |
| 811 |
(unsigned char*)src + src_offset, |
| 812 |
nbytes, ctx->stream)); |
| 813 |
} |
| 814 |
if (event != NULL) { |
| 815 |
HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream)); |
| 816 |
} |
| 817 |
} |
| 818 |
return FUTHARK_SUCCESS; |
| 819 |
} |
| 820 |
|
| 821 |
static int memcpy_gpu2host(struct futhark_context* ctx, |
| 822 |
const char *provenance, |
| 823 |
bool sync, |
| 824 |
unsigned char* dst, int64_t dst_offset, |
| 825 |
gpu_mem src, int64_t src_offset, |
| 826 |
int64_t nbytes) { |
| 827 |
if (nbytes > 0) { |
| 828 |
struct hip_event *event = hip_event_new(ctx); |
| 829 |
if (event != NULL) { |
| 830 |
add_event(ctx, |
| 831 |
"copy_dev_to_host", |
| 832 |
provenance, |
| 833 |
NULL, |
| 834 |
event, |
| 835 |
(event_report_fn)hip_event_report); |
| 836 |
HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream)); |
| 837 |
} |
| 838 |
if (sync) { |
| 839 |
HIP_SUCCEED_OR_RETURN |
| 840 |
(hipMemcpyDtoH(dst + dst_offset, |
| 841 |
(unsigned char*)src + src_offset, |
| 842 |
nbytes)); |
| 843 |
} else { |
| 844 |
HIP_SUCCEED_OR_RETURN |
| 845 |
(hipMemcpyDtoHAsync(dst + dst_offset, |
| 846 |
(unsigned char*)src + src_offset, |
| 847 |
nbytes, ctx->stream)); |
| 848 |
} |
| 849 |
if (event != NULL) { |
| 850 |
HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream)); |
| 851 |
} |
| 852 |
if (sync && |
| 853 |
ctx->failure_is_an_option && |
| 854 |
futhark_context_sync(ctx) != 0) { |
| 855 |
return 1; |
| 856 |
} |
| 857 |
} |
| 858 |
return FUTHARK_SUCCESS; |
| 859 |
} |
| 860 |
|
| 861 |
static int gpu_launch_kernel(struct futhark_context* ctx, |
| 862 |
gpu_kernel kernel, |
| 863 |
const char *name, const char *provenance, |
| 864 |
const int32_t grid[3], |
| 865 |
const int32_t block[3], |
| 866 |
unsigned int shared_mem_bytes, |
| 867 |
int num_args, |
| 868 |
void* args[num_args], |
| 869 |
size_t args_sizes[num_args]) { |
| 870 |
(void) args_sizes; |
| 871 |
|
| 872 |
if (shared_mem_bytes > ctx->max_shared_memory) { |
| 873 |
set_error(ctx, msgprintf("Kernel %s with %d bytes of memory exceeds device limit of %d\n", |
| 874 |
name, shared_mem_bytes, (int)ctx->max_shared_memory)); |
| 875 |
return 1; |
| 876 |
} |
| 877 |
|
| 878 |
int64_t time_start = 0, time_end = 0; |
| 879 |
if (ctx->debugging) { |
| 880 |
time_start = get_wall_time(); |
| 881 |
} |
| 882 |
|
| 883 |
struct hip_event *event = hip_event_new(ctx); |
| 884 |
|
| 885 |
if (event != NULL) { |
| 886 |
HIP_SUCCEED_FATAL(hipEventRecord(event->start, ctx->stream)); |
| 887 |
|
| 888 |
struct kvs *kvs = kvs_new(); |
| 889 |
kvs_printf(kvs, "kernel", "\"%s\"", name); |
| 890 |
kvs_printf(kvs, "grid", "[%d,%d,%d]", grid[0], grid[1], grid[2]); |
| 891 |
kvs_printf(kvs, "block", "[%d,%d,%d]", block[0], block[1], block[2]); |
| 892 |
kvs_printf(kvs, "shared memory", "%d", shared_mem_bytes); |
| 893 |
|
| 894 |
add_event(ctx, |
| 895 |
name, |
| 896 |
provenance, |
| 897 |
kvs, |
| 898 |
event, |
| 899 |
(event_report_fn)hip_event_report); |
| 900 |
} |
| 901 |
|
| 902 |
HIP_SUCCEED_OR_RETURN |
| 903 |
(hipModuleLaunchKernel(kernel, |
| 904 |
grid[0], grid[1], grid[2], |
| 905 |
block[0], block[1], block[2], |
| 906 |
shared_mem_bytes, ctx->stream, |
| 907 |
args, NULL)); |
| 908 |
|
| 909 |
if (event != NULL) { |
| 910 |
HIP_SUCCEED_FATAL(hipEventRecord(event->end, ctx->stream)); |
| 911 |
} |
| 912 |
|
| 913 |
if (ctx->debugging) { |
| 914 |
HIP_SUCCEED_FATAL(hipStreamSynchronize(ctx->stream)); |
| 915 |
time_end = get_wall_time(); |
| 916 |
long int time_diff = time_end - time_start; |
| 917 |
fprintf(ctx->log, " runtime: %ldus\n", time_diff); |
| 918 |
} |
| 919 |
if (ctx->logging) { |
| 920 |
fprintf(ctx->log, "\n"); |
| 921 |
} |
| 922 |
|
| 923 |
return FUTHARK_SUCCESS; |
| 924 |
} |
| 925 |
|
| 926 |
static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) { |
| 927 |
hipError_t res; |
| 928 |
|
| 929 |
if (ctx->cfg->unified_memory) { |
| 930 |
res = hipMallocManaged(mem_out, size, hipMemAttachGlobal); |
| 931 |
} else { |
| 932 |
res = hipMalloc(mem_out, size); |
| 933 |
} |
| 934 |
|
| 935 |
if (res == hipErrorOutOfMemory) { |
| 936 |
return FUTHARK_OUT_OF_MEMORY; |
| 937 |
} |
| 938 |
HIP_SUCCEED_OR_RETURN(res); |
| 939 |
return FUTHARK_SUCCESS; |
| 940 |
} |
| 941 |
|
| 942 |
static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) { |
| 943 |
(void)ctx; |
| 944 |
HIP_SUCCEED_OR_RETURN(hipFree(mem)); |
| 945 |
return FUTHARK_SUCCESS; |
| 946 |
} |
| 947 |
|
| 948 |
// End of backends/hip.h. |