Debian Codegraph @

src/imagination/pco/uscgen/pco_clc.c

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 * Copyright 2023 Alyssa Rosenzweig
4 * Copyright 2020 Intel Corporation
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "common/pvr_device_info.h"
9 #include "compiler/glsl_types.h"
10 #include "compiler/shader_enums.h"
11 #include "compiler/spirv/nir_spirv.h"
12 #include "nir/nir.h"
13 #include "nir/nir_builder.h"
14 #include "nir/nir_builder_opcodes.h"
15 #include "nir/nir_intrinsics.h"
16 #include "nir/nir_precompiled.h"
17 #include "pco/pco.h"
18 #include "pco/pco_data.h"
19 #include "util/macros.h"
20
21 #include <assert.h>
22 #include <fcntl.h>
23 #include <inttypes.h>
24 #include <libgen.h>
25 #include <stdio.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <sys/mman.h>
29
30 #define CLC_PREFIX "pco_usclib"
31
32 #define VS_PREFIX "vs_"
33 #define FS_PREFIX "fs_"
34 #define CS_PREFIX "cs_"
35 #define COMMON_SUFFIX "_common"
36 #define COMMON_DEVICE "common"
37
38 static const struct pvr_device_info pvr_device_info_common = {
39 .ident =
40 (struct pvr_device_ident){
41 .device_id = 0,
42 .series_name = COMMON_DEVICE,
43 .public_name = COMMON_DEVICE,
44 },
45
46 .features =
47 (struct pvr_device_features){
48 .has_common_store_size_in_dwords = true,
49 .has_compute = true,
50 .has_ipf_creq_pf = true,
51 .has_isp_max_tiles_in_flight = true,
52 .has_isp_samples_per_pixel = true,
53 .has_max_instances_per_pds_task = true,
54 .has_max_multisample = true,
55 .has_max_partitions = true,
56 .has_max_usc_tasks = true,
57 .has_num_clusters = true,
58 .has_num_raster_pipes = true,
59 .has_pbe2_in_xe = true,
60 .has_pbe_filterable_f16 = true,
61 .has_pbe_yuv = true,
62 .has_roguexe = true,
63 .has_screen_size8K = true,
64 .has_simple_internal_parameter_format = true,
65 .has_simple_internal_parameter_format_v2 = true,
66 .has_simple_parameter_format_version = true,
67 .has_slc_cache_line_size_bits = true,
68 .has_tile_size_16x16 = true,
69 .has_tile_size_x = true,
70 .has_tile_size_y = true,
71 .has_tpu_border_colour_enhanced = true,
72 .has_tpu_extended_integer_lookup = true,
73 .has_tpu_image_state_v2 = true,
74 .has_unified_store_depth = true,
75 .has_usc_f16sop_u8 = true,
76 .has_usc_min_output_registers_per_pix = true,
77 .has_usc_pixel_partition_mask = true,
78 .has_usc_slots = true,
79 .has_uvs_banks = true,
80 .has_uvs_pba_entries = true,
81 .has_uvs_vtx_entries = true,
82 .has_vdm_cam_size = true,
83 .has_vdm_degenerate_culling = true,
84
85 .common_store_size_in_dwords = 512U * 4U * 4U,
86 .isp_max_tiles_in_flight = 1U,
87 .isp_samples_per_pixel = 1U,
88 .max_instances_per_pds_task = 32U,
89 .max_multisample = 4U,
90 .max_partitions = 4U,
91 .max_usc_tasks = 24U,
92 .num_clusters = 1U,
93 .num_raster_pipes = 1U,
94 .simple_parameter_format_version = 2U,
95 .slc_cache_line_size_bits = 512U,
96 .tile_size_x = 16U,
97 .tile_size_y = 16U,
98 .unified_store_depth = 64U,
99 .usc_min_output_registers_per_pix = 1U,
100 .usc_slots = 14U,
101 .uvs_banks = 2U,
102 .uvs_pba_entries = 320U,
103 .uvs_vtx_entries = 288U,
104 .vdm_cam_size = 32U,
105
106 .has_s8xe = true,
107 .has_usc_itr_parallel_instances = true,
108
109 .usc_itr_parallel_instances = 4U,
110 },
111
112 .enhancements = (struct pvr_device_enhancements){ 0 },
113 .quirks = (struct pvr_device_quirks){ 0 },
114 };
115
116 /* Standard optimization loop */
117 static void optimize(nir_shader *nir)
118 {
119 bool progress;
120 do {
121 progress = false;
122
123 NIR_PASS(progress, nir, nir_split_var_copies);
124 NIR_PASS(progress, nir, nir_split_struct_vars, nir_var_function_temp);
125 NIR_PASS(progress, nir, nir_lower_var_copies);
126 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
127
128 NIR_PASS(progress, nir, nir_opt_copy_prop);
129 NIR_PASS(progress, nir, nir_opt_remove_phis);
130 NIR_PASS(progress, nir, nir_lower_all_phis_to_scalar);
131 NIR_PASS(progress, nir, nir_opt_dce);
132 NIR_PASS(progress, nir, nir_opt_dead_cf);
133 NIR_PASS(progress, nir, nir_opt_cse);
134 nir_opt_peephole_select_options peep_opts = {
135 .limit = 64,
136 .expensive_alu_ok = true,
137 };
138 NIR_PASS(progress, nir, nir_opt_peephole_select, &peep_opts);
139 NIR_PASS(progress, nir, nir_opt_phi_precision);
140 NIR_PASS(progress, nir, nir_opt_algebraic);
141 NIR_PASS(progress, nir, nir_opt_constant_folding);
142
143 NIR_PASS(progress, nir, nir_opt_deref);
144 NIR_PASS(progress, nir, nir_opt_copy_prop_vars);
145 NIR_PASS(progress, nir, nir_opt_undef);
146 NIR_PASS(progress, nir, nir_lower_undef_to_zero);
147
148 NIR_PASS(progress, nir, nir_opt_shrink_vectors, true);
149 NIR_PASS(progress, nir, nir_opt_loop_unroll);
150
151 } while (progress);
152 }
153
154 static nir_shader *
155 spv_to_nir(void *mem_ctx, uint32_t *spirv_map, unsigned spirv_len)
156 {
157 static const struct spirv_to_nir_options precomp_spirv_options = {
158 .environment = NIR_SPIRV_OPENCL,
159 .shared_addr_format = nir_address_format_62bit_generic,
160 .global_addr_format = nir_address_format_62bit_generic,
161 .temp_addr_format = nir_address_format_62bit_generic,
162 .constant_addr_format = nir_address_format_64bit_global,
163 .create_library = true,
164 };
165
166 nir_shader *nir = spirv_to_nir(spirv_map,
167 spirv_len / 4,
168 NULL,
169 0,
170 MESA_SHADER_KERNEL,
171 "library",
172 &precomp_spirv_options,
173 pco_nir_options());
174
175 nir_validate_shader(nir, "after spirv_to_nir");
176 nir_validate_ssa_dominance(nir, "after spirv_to_nir");
177 ralloc_steal(mem_ctx, nir);
178
179 nir_fixup_is_exported(nir);
180
181 NIR_PASS(_, nir, nir_lower_system_values);
182 NIR_PASS(_, nir, nir_lower_calls_to_builtins);
183
184 nir_lower_compute_system_values_options cs = { .global_id_is_32bit = true };
185 NIR_PASS(_, nir, nir_lower_compute_system_values, &cs);
186
187 /* We have to lower away local constant initializers right before we
188 * inline functions. That way they get properly initialized at the top
189 * of the function and not at the top of its caller.
190 */
191 NIR_PASS(_, nir, nir_lower_variable_initializers, nir_var_function_temp);
192 NIR_PASS(_, nir, nir_lower_returns);
193 NIR_PASS(_, nir, nir_inline_functions);
194 nir_remove_non_exported(nir);
195 NIR_PASS(_, nir, nir_opt_copy_prop);
196 NIR_PASS(_, nir, nir_opt_deref);
197
198 /* We can't deal with constant data, get rid of it */
199 nir_lower_constant_to_temp(nir);
200
201 /* We can go ahead and lower the rest of the constant initializers. We do
202 * this here so that nir_remove_dead_variables and
203 * split_per_member_structs below see the corresponding stores.
204 */
205 NIR_PASS(_, nir, nir_lower_variable_initializers, ~0);
206
207 /* LLVM loves to take advantage of the fact that vec3s in OpenCL are 16B
208 * aligned and so it can just read/write them as vec4s. This results in a
209 * LOT of vec4->vec3 casts on loads and stores. One solution to this
210 * problem is to get rid of all vec3 variables.
211 */
212 NIR_PASS(_,
213 nir,
214 nir_lower_vec3_to_vec4,
215 nir_var_shader_temp | nir_var_function_temp | nir_var_mem_shared |
216 nir_var_mem_global | nir_var_mem_constant);
217
218 /* We assign explicit types early so that the optimizer can take advantage
219 * of that information and hopefully get rid of some of our memcpys.
220 */
221 NIR_PASS(_,
222 nir,
223 nir_lower_vars_to_explicit_types,
224 nir_var_uniform | nir_var_shader_temp | nir_var_function_temp |
225 nir_var_mem_shared | nir_var_mem_global,
226 glsl_get_cl_type_size_align);
227
228 optimize(nir);
229
230 NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_all, NULL);
231
232 /* Lower again, this time after dead-variables to get more compact
233 * variable layouts.
234 */
235 NIR_PASS(_,
236 nir,
237 nir_lower_vars_to_explicit_types,
238 nir_var_shader_temp | nir_var_function_temp | nir_var_mem_shared |
239 nir_var_mem_global | nir_var_mem_constant,
240 glsl_get_cl_type_size_align);
241 assert(nir->constant_data_size == 0);
242
243 NIR_PASS(_, nir, nir_lower_memcpy);
244
245 NIR_PASS(_,
246 nir,
247 nir_lower_explicit_io,
248 nir_var_mem_constant,
249 nir_address_format_64bit_global);
250
251 NIR_PASS(_,
252 nir,
253 nir_lower_explicit_io,
254 nir_var_uniform,
255 nir_address_format_32bit_offset_as_64bit);
256
257 /* Note: we cannot lower explicit I/O here, because we need derefs intact
258 * for function calls into the library to work.
259 */
260
261 NIR_PASS(_, nir, nir_lower_convert_alu_types, NULL);
262 NIR_PASS(_, nir, nir_opt_if, 0);
263 NIR_PASS(_, nir, nir_opt_idiv_const, 16);
264
265 optimize(nir);
266
267 return nir;
268 }
269
270 static nir_def *load_kernel_input(nir_builder *b,
271 unsigned num_components,
272 unsigned bit_size,
273 unsigned base)
274 {
275 return nir_load_preamble(b, num_components, bit_size, .base = base);
276 }
277
278 /**
279 * Common function to build a NIR shader and export the binary.
280 *
281 * \param ctx PCO context.
282 * \param nir NIR shader.
283 * \param data Shader data.
284 * \return The finalized PCO shader.
285 */
286 static pco_shader *build_shader(pco_ctx *ctx, nir_shader *nir, pco_data *data)
287 {
288 pco_preprocess_nir(ctx, nir);
289 pco_lower_nir(ctx, nir, data);
290 pco_postprocess_nir(ctx, nir, data);
291
292 pco_shader *shader = pco_trans_nir(ctx, nir, data, NULL);
293 pco_process_ir(ctx, shader);
294 pco_encode_ir(ctx, shader);
295
296 return shader;
297 }
298
299 static inline mesa_shader_stage get_shader_stage(const char *name)
300 {
301 if (!strncmp(VS_PREFIX, name, strlen(VS_PREFIX)))
302 return MESA_SHADER_VERTEX;
303
304 if (!strncmp(FS_PREFIX, name, strlen(FS_PREFIX)))
305 return MESA_SHADER_FRAGMENT;
306
307 if (!strncmp(CS_PREFIX, name, strlen(CS_PREFIX)))
308 return MESA_SHADER_COMPUTE;
309
310 UNREACHABLE("");
311 }
312
313 static inline bool is_shader_common(const char *name)
314 {
315 unsigned name_len = strlen(name);
316 unsigned suffix_len = strlen(COMMON_SUFFIX);
317
318 if (name_len < suffix_len)
319 return false;
320
321 return !strcmp(&name[name_len - suffix_len], COMMON_SUFFIX);
322 }
323
324 static const char *
325 remap_variant(nir_function *func, UNUSED unsigned variant, const char *target)
326 {
327 return is_shader_common(func->name) ? COMMON_DEVICE : target;
328 }
329
330 int main(int argc, char *argv[argc])
331 {
332 if (argc < 4) {
333 fprintf(
334 stderr,
335 "Usage: %s <input SPIR-V> <output header> <output source> [device(s...)]\n",
336 argv[0]);
337
338 return 1;
339 }
340
341 void *mem_ctx = ralloc_context(NULL);
342
343 char *spv_file = argv[1];
344 char *hdr_file = argv[2];
345 char *src_file = argv[3];
346
347 unsigned num_devices = argc - 4;
348 const char **devices =
349 ralloc_array_size(mem_ctx, sizeof(*devices), num_devices + 1);
350 char **device_names =
351 ralloc_array_size(mem_ctx, sizeof(*device_names), num_devices + 1);
352 uint64_t *device_ids =
353 ralloc_array_size(mem_ctx, sizeof(*device_ids), num_devices + 1);
354 struct pvr_device_info *device_infos =
355 ralloc_array_size(mem_ctx, sizeof(*device_infos), num_devices + 1);
356
357 for (unsigned d = 0; d < num_devices; ++d) {
358 devices[d] = ralloc_strdup(devices, argv[4 + d]);
359 device_names[d] = ralloc_strdup(device_names, devices[d]);
360 for (unsigned u = 0; u < strlen(device_names[d]); ++u)
361 if (device_names[d][u] == '-')
362 device_names[d][u] = '_';
363
364 pvr_device_info_init_public_name(&device_infos[d], devices[d]);
365 device_ids[d] = pvr_get_packed_bvnc(&device_infos[d]);
366 }
367
368 /* Common device. */
369 devices[num_devices] = ralloc_strdup(devices, COMMON_DEVICE);
370 device_names[num_devices] = ralloc_strdup(device_names, COMMON_DEVICE);
371 device_ids[num_devices] = 0U;
372 memcpy(&device_infos[num_devices],
373 &pvr_device_info_common,
374 sizeof(pvr_device_info_common));
375 ++num_devices;
376
377 int fd = open(spv_file, O_RDONLY);
378 if (fd < 0) {
379 fprintf(stderr, "Failed to open %s\n", spv_file);
380 goto err_free_mem_ctx;
381 }
382
383 off_t spirv_len = lseek(fd, 0, SEEK_END);
384 assert(spirv_len % 4 == 0);
385
386 void *spirv_map = mmap(NULL, spirv_len, PROT_READ, MAP_PRIVATE, fd, 0);
387 close(fd);
388
389 if (spirv_map == MAP_FAILED) {
390 fprintf(stderr,
391 "Failed to mmap the file: errno=%d, %s\n",
392 errno,
393 strerror(errno));
394 goto err_free_mem_ctx;
395 }
396
397 FILE *fp_hdr = fopen(hdr_file, "w");
398 if (!fp_hdr) {
399 fprintf(stderr, "Failed to open %s\n", hdr_file);
400 goto err_unmap_spirv;
401 }
402
403 FILE *fp_src = fopen(src_file, "w");
404 if (!fp_src) {
405 fprintf(stderr, "Failed to open %s\n", src_file);
406 goto err_close_hdr;
407 }
408
409 nir_precomp_print_header(fp_src,
410 fp_hdr,
411 "Imagination Technologies Ltd.",
412 basename(hdr_file));
413
414 pco_ctx *ctx = pco_ctx_create(NULL, mem_ctx);
415 nir_shader *nir = spv_to_nir(mem_ctx, spirv_map, spirv_len);
416 struct nir_precomp_opts opts = { 0 };
417
418 nir_precomp_print_target_enum_map(fp_src,
419 fp_hdr,
420 CLC_PREFIX,
421 num_devices,
422 (const char **)device_names,
423 device_ids);
424
425 nir_precomp_print_program_enum(fp_hdr, nir, CLC_PREFIX);
426 nir_precomp_print_dispatch_macros(fp_hdr, &opts, nir);
427
428 nir_foreach_entrypoint (func, nir) {
429 unsigned num_variants = nir_precomp_nr_variants(func);
430
431 nir_precomp_print_layout_struct(fp_hdr, &opts, func);
432
433 mesa_shader_stage stage = get_shader_stage(func->name);
434 bool is_common = is_shader_common(func->name);
435
436 for (unsigned variant = 0; variant < num_variants; ++variant) {
437 nir_shader *s = nir_precompiled_build_variant(func,
438 stage,
439 variant,
440 pco_nir_options(),
441 &opts,
442 load_kernel_input);
443
444 nir_link_shader_functions(s, nir);
445 NIR_PASS(_, s, nir_inline_functions);
446 nir_remove_non_entrypoints(s);
447 NIR_PASS(_, s, nir_opt_deref);
448 NIR_PASS(_, s, nir_lower_vars_to_ssa);
449 NIR_PASS(_, s, nir_remove_dead_derefs);
450 NIR_PASS(_,
451 s,
452 nir_remove_dead_variables,
453 nir_var_function_temp | nir_var_shader_temp,
454 NULL);
455 NIR_PASS(_,
456 s,
457 nir_lower_vars_to_explicit_types,
458 nir_var_shader_temp | nir_var_function_temp,
459 glsl_get_cl_type_size_align);
460
461 NIR_PASS(_,
462 s,
463 nir_lower_vars_to_explicit_types,
464 nir_var_mem_shared,
465 glsl_get_cl_type_size_align);
466
467 NIR_PASS(_,
468 s,
469 nir_lower_explicit_io,
470 nir_var_shader_temp | nir_var_function_temp |
471 nir_var_mem_shared | nir_var_mem_global,
472 nir_address_format_62bit_generic);
473
474 for (unsigned d = 0; d < num_devices; ++d) {
475 if (is_common && d != (num_devices - 1))
476 continue;
477
478 pco_ctx_update_dev_info(ctx, &device_infos[d]);
479
480 nir_shader *clone = nir_shader_clone(NULL, s);
481
482 pco_data data = { 0 };
483 pco_shader *shader = build_shader(ctx, clone, &data);
484 pco_precomp_data precomp_data = pco_get_precomp_data(shader);
485
486 unsigned binary_size =
487 pco_shader_binary_size(shader) + sizeof(precomp_data);
488 assert(!(binary_size % sizeof(uint32_t)));
489
490 uint32_t *binary_data = malloc(binary_size);
491 memcpy(binary_data, &precomp_data, sizeof(precomp_data));
492 memcpy((uint8_t *)binary_data + sizeof(precomp_data),
493 pco_shader_binary_data(shader),
494 pco_shader_binary_size(shader));
495
496 nir_precomp_print_blob(fp_src,
497 func->name,
498 device_names[d],
499 variant,
500 binary_data,
501 binary_size,
502 true);
503
504 free(binary_data);
505 ralloc_free(shader);
506 ralloc_free(clone);
507 }
508
509 ralloc_free(s);
510 }
511 }
512
513 for (unsigned d = 0; d < num_devices; ++d) {
514 nir_precomp_print_extern_binary_map(fp_hdr, CLC_PREFIX, device_names[d]);
515 nir_precomp_print_binary_map(fp_src,
516 nir,
517 CLC_PREFIX,
518 device_names[d],
519 remap_variant);
520 }
521
522 nir_precomp_print_target_binary_map(fp_src,
523 fp_hdr,
524 CLC_PREFIX,
525 num_devices,
526 (const char **)device_names);
527
528 /* Remove common shaders - no need to preserve their NIR. */
529 nir_foreach_entrypoint_safe (func, nir) {
530 if (!is_shader_common(func->name))
531 continue;
532
533 exec_node_remove(&func->node);
534 }
535
536 nir_precomp_print_nir(fp_src, fp_hdr, nir, CLC_PREFIX, "nir");
537
538 ralloc_free(mem_ctx);
539 munmap(spirv_map, spirv_len);
540 fclose(fp_src);
541 fclose(fp_hdr);
542
543 return 0;
544
545 err_close_hdr:
546 fclose(fp_hdr);
547
548 err_unmap_spirv:
549 munmap(spirv_map, spirv_len);
550
551 err_free_mem_ctx:
552 ralloc_free(mem_ctx);
553
554 return 1;
555 }