Debian Codegraph @

src/imagination/pco/pco_nir.c

raw
1 /*
2 * Copyright © 2024 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pco_nir.c
9 *
10 * \brief NIR-specific functions.
11 */
12
13 #include "nir/nir_builder.h"
14 #include "nir/nir_lower_blend.h"
15 #include "pco.h"
16 #include "pco_internal.h"
17 #include "pvr_limits.h"
18
19 #include <stdio.h>
20
21 /** SPIR-V to NIR options. */
22 static const struct spirv_to_nir_options spirv_options = {
23 .environment = NIR_SPIRV_VULKAN,
24
25 .ubo_addr_format = nir_address_format_vec2_index_32bit_offset,
26 .ssbo_addr_format = nir_address_format_vec2_index_32bit_offset,
27 .phys_ssbo_addr_format = nir_address_format_2x32bit_global,
28 .push_const_addr_format = nir_address_format_32bit_offset,
29 .shared_addr_format = nir_address_format_32bit_offset,
30
31 .min_ubo_alignment = PVR_UNIFORM_BUFFER_OFFSET_ALIGNMENT,
32 .min_ssbo_alignment = PVR_STORAGE_BUFFER_OFFSET_ALIGNMENT,
33 };
34
35 /** NIR options. */
36 static const nir_shader_compiler_options nir_options = {
37 .discard_is_demote = true,
38 .fuse_ffma32 = true,
39
40 .has_f2i32_rtne = true,
41 .has_fused_comp_and_csel = true,
42
43 .instance_id_includes_base_index = true,
44
45 .lower_device_index_to_zero = true,
46
47 .lower_fdiv = true,
48 .lower_ffract = true,
49 .lower_find_lsb = true,
50 .lower_fquantize2f16 = true,
51 .lower_flrp32 = true,
52 .lower_fmod = true,
53 .lower_fpow = true,
54 .lower_fsqrt = true,
55 .lower_ftrunc = true,
56 .lower_iadd_sat = true,
57 .lower_ifind_msb = true,
58 .lower_layer_fs_input_to_sysval = true,
59 .lower_mul_2x32_64 = true,
60 .compact_arrays = true,
61 .scalarize_ddx = true,
62
63 .lower_int64_options = ~0U,
64 .lower_pack_64_2x32 = true,
65 .lower_pack_64_2x32_split = true,
66 .lower_unpack_64_2x32_split = true,
67
68 .lower_interpolate_at = true,
69
70 .lower_helper_invocation = true,
71
72 .max_unroll_iterations = 16,
73 .max_samples = 4,
74
75 .io_options = nir_io_vectorizer_ignores_types,
76 };
77
78 /**
79 * \brief Returns the SPIR-V to NIR options.
80 *
81 * \return The SPIR-V to NIR options.
82 */
83 const struct spirv_to_nir_options *pco_spirv_options(void)
84 {
85 return &spirv_options;
86 }
87
88 /**
89 * \brief Returns the NIR options for a PCO compiler
90 * context.
91 *
92 * \return The NIR options.
93 */
94 const nir_shader_compiler_options *pco_nir_options(void)
95 {
96 return &nir_options;
97 }
98
99 /**
100 * \brief Returns the GLSL type size.
101 *
102 * \param[in] type Type.
103 * \param[in] bindless Whether the access is bindless.
104 * \return The size.
105 */
106 static int glsl_type_size(const struct glsl_type *type, UNUSED bool bindless)
107 {
108 return glsl_count_attribute_slots(type, false);
109 }
110
111 /**
112 * \brief Returns the vectorization with for a given instruction.
113 *
114 * \param[in] instr Instruction.
115 * \param[in] data User data.
116 * \return The vectorization width.
117 */
118 static uint8_t vectorize_filter(const nir_instr *instr, UNUSED const void *data)
119 {
120 if (instr->type == nir_instr_type_load_const)
121 return 1;
122
123 if (instr->type != nir_instr_type_alu)
124 return 0;
125
126 /* TODO */
127 nir_alu_instr *alu = nir_instr_as_alu(instr);
128 switch (alu->op) {
129 default:
130 break;
131 }
132
133 /* Basic for now. */
134 return 2;
135 }
136
137 /**
138 * \brief Filter for fragment shader inputs that need to be scalar.
139 *
140 * \param[in] instr Instruction.
141 * \param[in] data User data.
142 * \return True if the instruction was found.
143 */
144 static bool frag_in_scalar_filter(const nir_intrinsic_instr *intr, const void *data)
145 {
146 nir_shader *nir = (nir_shader *)data;
147
148 if (intr->intrinsic != nir_intrinsic_load_input &&
149 intr->intrinsic != nir_intrinsic_load_interpolated_input) {
150 return false;
151 }
152
153 gl_varying_slot location = nir_intrinsic_io_semantics(intr).location;
154 if (location == VARYING_SLOT_POS)
155 return true;
156
157 nir_variable *var =
158 nir_find_variable_with_location(nir, nir_var_shader_in, location);
159 assert(var);
160
161 enum glsl_interp_mode interp_mode = var->data.interpolation;
162
163 if (intr->intrinsic == nir_intrinsic_load_interpolated_input) {
164 nir_intrinsic_instr *bary = nir_src_as_intrinsic(intr->src[0]);
165 assert(bary);
166
167 interp_mode = nir_intrinsic_interp_mode(bary);
168 }
169
170 if (interp_mode == INTERP_MODE_FLAT)
171 return true;
172
173 return false;
174 }
175
176 static void
177 shared_var_info(const struct glsl_type *type, unsigned *size, unsigned *align)
178 {
179 assert(glsl_type_is_vector_or_scalar(type));
180
181 uint32_t comp_size =
182 glsl_type_is_boolean(type) ? 4 : glsl_get_bit_size(type) / 8;
183 unsigned length = glsl_get_vector_elements(type);
184 *size = comp_size * length, *align = comp_size;
185 }
186
187 /**
188 * \brief Checks whether two varying variables are the same.
189 *
190 * \param[in] out_var The first varying being compared.
191 * \param[in] in_var The second varying being compared.
192 * \return True if the varyings match.
193 */
194 static bool varyings_match(nir_variable *out_var, nir_variable *in_var)
195 {
196 return in_var->data.location == out_var->data.location &&
197 in_var->data.location_frac == out_var->data.location_frac &&
198 in_var->type == out_var->type;
199 }
200
201 /**
202 * \brief Gather fragment shader data pass.
203 *
204 * \param[in] b NIR builder.
205 * \param[in] intr NIR intrinsic instruction.
206 * \param[in,out] cb_data Callback data.
207 * \return True if the shader was modified (always return false).
208 */
209 static bool gather_fs_data_pass(struct nir_builder *b,
210 nir_intrinsic_instr *intr,
211 void *cb_data)
212 {
213 pco_data *data = cb_data;
214
215 switch (intr->intrinsic) {
216 /* Check whether the shader accesses z/w. */
217 case nir_intrinsic_load_input:
218 case nir_intrinsic_load_interpolated_input: {
219 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
220 if (io_semantics.location != VARYING_SLOT_POS)
221 return false;
222
223 unsigned component = nir_intrinsic_component(intr);
224 unsigned chans = intr->def.num_components;
225 assert(component == 2 || chans == 1);
226
227 data->fs.uses.z |= (component == 2);
228 data->fs.uses.w |= (component + chans > 3);
229 break;
230 }
231
232 case nir_intrinsic_load_fs_coeffs_pco: {
233 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
234 b->shader->info.inputs_read |= BITFIELD64_BIT(io_semantics.location);
235
236 if (io_semantics.location != VARYING_SLOT_POS)
237 return false;
238
239 unsigned component = nir_intrinsic_component(intr);
240 data->fs.uses.z |= (component == 2);
241 data->fs.uses.w |= (component == 3);
242 break;
243 }
244
245 case nir_intrinsic_load_blend_const_color_rgba:
246 data->fs.blend_consts_needed |= PIPE_MASK_RGBA;
247 break;
248
249 case nir_intrinsic_load_front_face_op_pco:
250 BITSET_SET(b->shader->info.system_values_read, SYSTEM_VALUE_FRONT_FACE);
251 break;
252
253 default:
254 break;
255 }
256
257 return false;
258 }
259
260 /**
261 * \brief Gathers fragment shader data.
262 *
263 * \param[in] nir NIR shader.
264 * \param[in,out] data Shader data.
265 */
266 static void gather_fs_data(nir_shader *nir, pco_data *data)
267 {
268 nir_shader_intrinsics_pass(nir, gather_fs_data_pass, nir_metadata_all, data);
269
270 /* If any inputs use smooth shading, then w is needed. */
271 if (!data->fs.uses.w) {
272 nir_foreach_shader_in_variable (var, nir) {
273 if (var->data.interpolation > INTERP_MODE_SMOOTH)
274 continue;
275
276 data->fs.uses.w = true;
277 break;
278 }
279 }
280
281 data->fs.uses.fbfetch = nir->info.fs.uses_fbfetch_output;
282 data->fs.uses.fbfetch |= data->fs.meta_present.color_write_enable;
283
284 data->fs.uses.early_frag = nir->info.fs.early_fragment_tests;
285 data->fs.uses.sample_shading |= nir->info.fs.uses_sample_shading;
286 }
287
288 /**
289 * \brief Gathers vertex shader data.
290 *
291 * \param[in] nir NIR shader.
292 * \param[in,out] data Shader data.
293 */
294 static void gather_vs_data(nir_shader *nir, pco_data *data)
295 {
296 pco_vs_data *vs_data = &data->vs;
297
298 vs_data->clip_count = nir->info.clip_distance_array_size;
299 vs_data->cull_count = nir->info.cull_distance_array_size;
300 }
301
302 /**
303 * \brief Gathers compute shader data.
304 *
305 * \param[in] nir NIR shader.
306 * \param[in,out] data Shader data.
307 */
308 static void gather_cs_data(nir_shader *nir, pco_data *data)
309 {
310 for (unsigned u = 0; u < ARRAY_SIZE(data->cs.workgroup_size); ++u)
311 data->cs.workgroup_size[u] = nir->info.workgroup_size[u];
312 }
313
314 /**
315 * \brief Checks whether a NIR intrinsic op is atomic.
316 *
317 * \param[in] op The NIR intrinsic op.
318 * \return True if the intrinsic op is atomic, else false.
319 */
320 /* TODO: what about emulated atomic ops? */
321 static inline bool intr_op_is_atomic(nir_intrinsic_op op)
322 {
323 switch (op) {
324 case nir_intrinsic_ssbo_atomic:
325 case nir_intrinsic_shared_atomic:
326 case nir_intrinsic_shared_atomic_swap:
327 return true;
328
329 default:
330 break;
331 }
332 return false;
333 }
334
335 static void gather_common_store_data(nir_intrinsic_instr *intr,
336 pco_common_data *common)
337 {
338 nir_src *offset_src;
339 unsigned num_components;
340 switch (intr->intrinsic) {
341 case nir_intrinsic_load_push_constant:
342 offset_src = &intr->src[0];
343 num_components = intr->def.num_components;
344 break;
345
346 default:
347 return;
348 }
349
350 if (nir_src_is_const(*offset_src) && common->push_consts.used != ~0U) {
351 unsigned offset = nir_src_as_uint(*offset_src);
352 common->push_consts.used =
353 MAX2(common->push_consts.used, offset + num_components);
354 } else {
355 common->push_consts.used = ~0U;
356 }
357 }
358
359 /**
360 * \brief Gather common data pass.
361 *
362 * \param[in] b NIR builder.
363 * \param[in] intr NIR intrinsic instruction.
364 * \param[in,out] cb_data Callback data.
365 * \return True if the shader was modified (always return false).
366 */
367 static bool gather_common_data_pass(UNUSED struct nir_builder *b,
368 nir_intrinsic_instr *intr,
369 void *cb_data)
370 {
371 pco_data *data = cb_data;
372 data->common.uses.atomics |= intr_op_is_atomic(intr->intrinsic);
373 gather_common_store_data(intr, &data->common);
374
375 return false;
376 }
377
378 /**
379 * \brief Gathers data common to all shader stages.
380 *
381 * \param[in] nir NIR shader.
382 * \param[in,out] data Shader data.
383 */
384 static void gather_common_data(nir_shader *nir, pco_data *data)
385 {
386 nir_shader_intrinsics_pass(nir,
387 gather_common_data_pass,
388 nir_metadata_all,
389 data);
390
391 data->common.scratch = nir->scratch_size;
392 }
393
394 /**
395 * \brief Gathers shader data.
396 *
397 * \param[in] nir NIR shader.
398 * \param[in,out] data Shader data.
399 */
400 static void gather_data(nir_shader *nir, pco_data *data)
401 {
402 gather_common_data(nir, data);
403
404 switch (nir->info.stage) {
405 case MESA_SHADER_FRAGMENT:
406 return gather_fs_data(nir, data);
407
408 case MESA_SHADER_VERTEX:
409 return gather_vs_data(nir, data);
410
411 case MESA_SHADER_COMPUTE:
412 return gather_cs_data(nir, data);
413
414 default:
415 break;
416 }
417
418 UNREACHABLE("");
419 }
420
421 static bool should_vectorize_mem_cb(unsigned align_mul,
422 unsigned align_offset,
423 unsigned bit_size,
424 unsigned num_components,
425 int64_t hole_size,
426 nir_intrinsic_instr *low,
427 nir_intrinsic_instr *high,
428 void *data)
429 {
430 if (bit_size > 32 || hole_size > 0)
431 return false;
432
433 if (!nir_num_components_valid(num_components))
434 return false;
435
436 return true;
437 }
438
439 static void pco_nir_opt(pco_ctx *ctx, nir_shader *nir, bool algebraic)
440 {
441 bool progress;
442
443 unsigned count = 0;
444 do {
445 progress = false;
446
447 if (count++ > 1000) {
448 printf("WARNING! Infinite opt loop!\n");
449 break;
450 }
451
452 NIR_PASS(progress, nir, nir_shrink_vec_array_vars, nir_var_function_temp);
453 NIR_PASS(progress, nir, nir_opt_deref);
454
455 bool progress_opt_memcpy = false;
456 NIR_PASS(progress_opt_memcpy, nir, nir_opt_memcpy);
457 progress |= progress_opt_memcpy;
458
459 if (progress_opt_memcpy)
460 NIR_PASS(progress, nir, nir_split_var_copies);
461
462 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
463
464 if (!nir->info.var_copies_lowered)
465 NIR_PASS(progress, nir, nir_opt_find_array_copies);
466 NIR_PASS(progress, nir, nir_opt_copy_prop_vars);
467 NIR_PASS(progress, nir, nir_opt_dead_write_vars);
468
469 NIR_PASS(progress, nir, nir_opt_copy_prop);
470 NIR_PASS(progress, nir, nir_opt_remove_phis);
471 NIR_PASS(progress, nir, nir_opt_dce);
472 NIR_PASS(progress, nir, nir_opt_dead_cf);
473 NIR_PASS(progress, nir, nir_opt_cse);
474 NIR_PASS(progress, nir, nir_opt_if, nir_opt_if_optimize_phi_true_false);
475 nir_opt_peephole_select_options peep_opts = {
476 .limit = 64,
477 .expensive_alu_ok = true,
478 };
479 NIR_PASS(progress, nir, nir_opt_peephole_select, &peep_opts);
480
481 NIR_PASS(progress, nir, nir_lower_int64);
482
483 NIR_PASS(progress, nir, nir_opt_phi_precision);
484 NIR_PASS(progress, nir, nir_lower_alu);
485 NIR_PASS(progress, nir, nir_lower_pack);
486
487 if (algebraic) {
488 NIR_PASS(progress, nir, nir_opt_algebraic);
489 NIR_PASS(progress, nir, pco_nir_lower_algebraic);
490 }
491
492 NIR_PASS(progress, nir, nir_opt_constant_folding);
493
494 nir_load_store_vectorize_options vectorize_opts = {
495 .modes = nir_var_mem_ubo | nir_var_mem_ssbo | nir_var_mem_global,
496 .callback = should_vectorize_mem_cb,
497 };
498 NIR_PASS(progress, nir, nir_opt_load_store_vectorize, &vectorize_opts);
499
500 NIR_PASS(progress, nir, nir_opt_shrink_stores, false);
501 NIR_PASS(progress, nir, nir_opt_shrink_vectors, true);
502
503 NIR_PASS(progress, nir, nir_opt_loop);
504 NIR_PASS(progress, nir, nir_lower_alu_to_scalar, NULL, NULL);
505 NIR_PASS(progress, nir, nir_opt_undef);
506 NIR_PASS(progress, nir, nir_opt_loop_unroll);
507 } while (progress);
508 }
509
510 static bool check_mem_writes(nir_builder *b,
511 nir_intrinsic_instr *intr,
512 UNUSED void *cb_data)
513 {
514 b->shader->info.writes_memory |= nir_intrinsic_writes_external_memory(intr);
515 return false;
516 }
517
518 /**
519 * \brief Runs pre-processing passes on a NIR shader.
520 *
521 * \param[in] ctx PCO compiler context.
522 * \param[in,out] nir NIR shader.
523 */
524 void pco_preprocess_nir(pco_ctx *ctx, nir_shader *nir)
525 {
526 if (nir->info.stage == MESA_SHADER_FRAGMENT)
527 nir_shader_intrinsics_pass(nir, check_mem_writes, nir_metadata_all, NULL);
528
529 if (nir->info.stage == MESA_SHADER_COMPUTE)
530 NIR_PASS(_, nir, pco_nir_compute_instance_check);
531
532 if (nir->info.internal)
533 NIR_PASS(_, nir, nir_lower_returns);
534
535 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
536 const struct nir_lower_sysvals_to_varyings_options sysvals_to_varyings = {
537 .frag_coord = true,
538 .point_coord = true,
539 .primitive_id = nir->info.stage == MESA_SHADER_FRAGMENT,
540 };
541 NIR_PASS(_, nir, nir_lower_sysvals_to_varyings, &sysvals_to_varyings);
542 NIR_PASS(_, nir, nir_lower_helper_writes, true);
543 NIR_PASS(_, nir, nir_lower_is_helper_invocation);
544 NIR_PASS(_, nir, nir_lower_terminate_to_demote);
545 NIR_PASS(_, nir, nir_lower_halt_to_return);
546 NIR_PASS(_, nir, nir_lower_returns);
547 }
548
549 NIR_PASS(_, nir, nir_lower_system_values);
550
551 if (nir->info.stage == MESA_SHADER_COMPUTE) {
552 NIR_PASS(_,
553 nir,
554 nir_lower_compute_system_values,
555 &(nir_lower_compute_system_values_options){
556 .lower_cs_local_id_to_index = true,
557 });
558 }
559
560 NIR_PASS(_, nir, pco_nir_lower_subgroups);
561
562 NIR_PASS(_,
563 nir,
564 nir_lower_io_vars_to_temporaries,
565 nir_shader_get_entrypoint(nir),
566 nir_var_shader_out | nir_var_shader_in);
567
568 NIR_PASS(_, nir, nir_lower_global_vars_to_local);
569 NIR_PASS(_, nir, nir_split_var_copies);
570 NIR_PASS(_, nir, nir_lower_var_copies);
571 NIR_PASS(_, nir, nir_split_per_member_structs);
572 NIR_PASS(_,
573 nir,
574 nir_split_struct_vars,
575 nir_var_function_temp | nir_var_shader_temp);
576 NIR_PASS(_,
577 nir,
578 nir_split_array_vars,
579 nir_var_function_temp | nir_var_shader_temp);
580
581 pco_nir_opt(ctx, nir, true);
582
583 NIR_PASS(_,
584 nir,
585 nir_lower_indirect_derefs_to_if_else_trees,
586 nir_var_shader_in | nir_var_shader_out,
587 UINT32_MAX);
588
589 NIR_PASS(_, nir, nir_lower_vars_to_ssa);
590
591 if (!nir->info.internal) {
592 /* TODO: test with different size_threshold values. */
593 NIR_PASS(_,
594 nir,
595 nir_lower_vars_to_scratch,
596 8,
597 glsl_get_natural_size_align_bytes,
598 glsl_get_word_size_align_bytes);
599 }
600
601 NIR_PASS(_,
602 nir,
603 nir_lower_indirect_derefs_to_if_else_trees,
604 nir_var_function_temp,
605 UINT32_MAX);
606
607 pco_nir_opt(ctx, nir, true);
608 NIR_PASS(_, nir, nir_opt_idiv_const, 32);
609 NIR_PASS(_,
610 nir,
611 nir_lower_idiv,
612 &(nir_lower_idiv_options){
613 .allow_fp16 = false,
614 });
615
616 NIR_PASS(_, nir, nir_scale_fdiv);
617 NIR_PASS(_, nir, nir_lower_frexp);
618 NIR_PASS(_, nir, nir_lower_flrp, 32, true);
619
620 NIR_PASS(_, nir, nir_remove_dead_derefs);
621 NIR_PASS(_, nir, nir_opt_undef);
622 NIR_PASS(_, nir, nir_lower_undef_to_zero);
623 NIR_PASS(_, nir, nir_opt_cse);
624 NIR_PASS(_, nir, nir_opt_dce);
625 NIR_PASS(_,
626 nir,
627 nir_remove_dead_variables,
628 nir_var_function_temp | nir_var_shader_temp,
629 NULL);
630
631 /* Fold constant offset srcs for IO. */
632 NIR_PASS(_,
633 nir,
634 nir_opt_constant_folding);
635
636 NIR_PASS(_,
637 nir,
638 nir_lower_io_array_vars_to_elements_no_indirects,
639 nir->info.stage == MESA_SHADER_VERTEX);
640
641 pco_nir_opt(ctx, nir, true);
642
643 if (pco_should_print_nir(nir)) {
644 puts("after pco_preprocess_nir:");
645 nir_print_shader(nir, stdout);
646 }
647 }
648
649 static bool can_remove_var(nir_variable *var, UNUSED void *data)
650 {
651 return !var->data.always_active_io;
652 }
653
654 /**
655 * \brief Performs linking optimizations on consecutive NIR shader stages.
656 *
657 * \param[in] ctx PCO compiler context.
658 * \param[in,out] producer NIR producer shader.
659 * \param[in,out] consumer NIR consumer shader.
660 * \param[in,out] producer_data Producer shader data.
661 * \param[in,out] consumer_data Consumer shader data.
662 */
663 void pco_link_nir(pco_ctx *ctx,
664 nir_shader *producer,
665 nir_shader *consumer,
666 pco_data *producer_data,
667 pco_data *consumer_data)
668 {
669 pco_nir_link_multiview(producer, consumer, consumer_data);
670 pco_nir_link_clip_cull_vars(producer, consumer);
671
672 nir_lower_io_array_vars_to_elements(producer, consumer);
673 nir_validate_shader(producer, "after nir_lower_io_array_vars_to_elements");
674 nir_validate_shader(consumer, "after nir_lower_io_array_vars_to_elements");
675
676 NIR_PASS(_, producer, nir_lower_io_vars_to_scalar, nir_var_shader_out);
677 NIR_PASS(_, consumer, nir_lower_io_vars_to_scalar, nir_var_shader_in);
678
679 pco_nir_opt(ctx, producer, true);
680 pco_nir_opt(ctx, consumer, true);
681
682 if (nir_link_opt_varyings(producer, consumer))
683 pco_nir_opt(ctx, consumer, true);
684
685 nir_remove_dead_variables_options rdv = {
686 .can_remove_var = can_remove_var,
687 };
688
689 NIR_PASS(_, producer, nir_remove_dead_variables, nir_var_shader_out, &rdv);
690 NIR_PASS(_, consumer, nir_remove_dead_variables, nir_var_shader_in, &rdv);
691
692 bool progress = nir_remove_unused_varyings(producer, consumer);
693 nir_compact_varyings(producer, consumer, true);
694
695 if (progress) {
696 NIR_PASS(_, producer, nir_lower_global_vars_to_local);
697 NIR_PASS(_, consumer, nir_lower_global_vars_to_local);
698
699 NIR_PASS(_,
700 producer,
701 nir_lower_indirect_derefs_to_if_else_trees,
702 nir_var_shader_in | nir_var_shader_out,
703 UINT32_MAX);
704 NIR_PASS(_,
705 consumer,
706 nir_lower_indirect_derefs_to_if_else_trees,
707 nir_var_shader_in | nir_var_shader_out,
708 UINT32_MAX);
709
710 pco_nir_opt(ctx, producer, true);
711 pco_nir_opt(ctx, consumer, true);
712 }
713
714 NIR_PASS(_, producer, nir_opt_vectorize_io_vars, nir_var_shader_out);
715 NIR_PASS(_, producer, nir_opt_combine_stores, nir_var_shader_out);
716 NIR_PASS(_, consumer, nir_opt_vectorize_io_vars, nir_var_shader_in);
717
718 if (pco_should_print_nir(producer)) {
719 puts("producer after pco_link_nir:");
720 nir_print_shader(producer, stdout);
721 }
722
723 if (pco_should_print_nir(consumer)) {
724 puts("consumer after pco_link_nir:");
725 nir_print_shader(consumer, stdout);
726 }
727 }
728
729 /**
730 * \brief Performs reverse linking optimizations on consecutive NIR shader
731 * stages.
732 *
733 * \param[in] ctx PCO compiler context.
734 * \param[in,out] producer NIR producer shader.
735 * \param[in,out] consumer NIR consumer shader.
736 */
737 void pco_rev_link_nir(pco_ctx *ctx, nir_shader *producer, nir_shader *consumer)
738 {
739 /* Propagate back/adjust the interpolation qualifiers. */
740 nir_foreach_shader_in_variable (in_var, consumer) {
741 if (in_var->data.location == VARYING_SLOT_POS ||
742 in_var->data.location == VARYING_SLOT_PNTC) {
743 in_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
744 } else if (in_var->data.interpolation == INTERP_MODE_NONE) {
745 in_var->data.interpolation = INTERP_MODE_SMOOTH;
746 }
747
748 nir_foreach_shader_out_variable (out_var, producer) {
749 if (!varyings_match(out_var, in_var))
750 continue;
751
752 out_var->data.interpolation = in_var->data.interpolation;
753 break;
754 }
755 }
756
757 if (pco_should_print_nir(producer)) {
758 puts("producer after pco_rev_link_nir:");
759 nir_print_shader(producer, stdout);
760 }
761
762 if (pco_should_print_nir(consumer)) {
763 puts("consumer after pco_rev_link_nir:");
764 nir_print_shader(consumer, stdout);
765 }
766 }
767
768 static bool robustness_filter(const nir_intrinsic_instr *intr,
769 UNUSED const void *data)
770 {
771 switch (intr->intrinsic) {
772 case nir_intrinsic_load_ubo:
773 case nir_intrinsic_load_ssbo:
774 case nir_intrinsic_store_ssbo:
775 case nir_intrinsic_ssbo_atomic:
776 case nir_intrinsic_ssbo_atomic_swap:
777 return true;
778
779 default:
780 break;
781 }
782
783 return false;
784 }
785
786 /**
787 * \brief Lowers a NIR shader.
788 *
789 * \param[in] ctx PCO compiler context.
790 * \param[in,out] nir NIR shader.
791 * \param[in,out] data Shader data.
792 */
793 void pco_lower_nir(pco_ctx *ctx, nir_shader *nir, pco_data *data)
794 {
795 NIR_PASS(_,
796 nir,
797 nir_opt_access,
798 &(nir_opt_access_options){ .is_vulkan = true });
799
800 NIR_PASS(_, nir, nir_opt_barrier_modes);
801 NIR_PASS(_, nir, nir_opt_combine_barriers, NULL, NULL);
802 NIR_PASS(_, nir, pco_nir_lower_barriers, data);
803
804 NIR_PASS(_, nir, nir_lower_memory_model);
805
806 NIR_PASS(_, nir, nir_opt_licm);
807
808 NIR_PASS(_, nir, nir_lower_memcpy);
809
810 if (nir->info.stage == MESA_SHADER_FRAGMENT)
811 NIR_PASS(_, nir, nir_opt_vectorize_io_vars, nir_var_shader_out);
812
813 NIR_PASS(_,
814 nir,
815 nir_lower_explicit_io,
816 nir_var_mem_ubo | nir_var_mem_ssbo,
817 nir_address_format_vec2_index_32bit_offset);
818
819 NIR_PASS(_,
820 nir,
821 nir_lower_explicit_io,
822 nir_var_mem_global,
823 nir_address_format_2x32bit_global);
824
825 nir_move_options move_options = nir_move_load_global | nir_move_load_ubo |
826 nir_move_load_ssbo | nir_move_load_input |
827 nir_move_load_frag_coord | nir_move_alu |
828 nir_move_comparisons | nir_move_copies;
829
830 NIR_PASS(_, nir, nir_opt_sink, move_options);
831 NIR_PASS(_, nir, nir_opt_move, move_options);
832
833 if (!nir->info.shared_memory_explicit_layout) {
834 NIR_PASS(_,
835 nir,
836 nir_lower_vars_to_explicit_types,
837 nir_var_mem_shared,
838 shared_var_info);
839 }
840
841 NIR_PASS(_,
842 nir,
843 nir_lower_explicit_io,
844 nir_var_mem_push_const | nir_var_mem_shared,
845 nir_address_format_32bit_offset);
846 NIR_PASS(_,
847 nir,
848 nir_lower_io_to_scalar,
849 nir_var_mem_push_const | nir_var_mem_shared,
850 NULL,
851 NULL);
852
853 if (data->common.robust_buffer_access)
854 NIR_PASS(_, nir, nir_lower_robust_access, robustness_filter, NULL);
855
856 if (data->common.null_descriptor) {
857 NIR_PASS(_,
858 nir,
859 pco_nir_lower_null_descriptors,
860 pco_nir_lower_null_descriptor_all);
861 }
862
863 NIR_PASS(_, nir, pco_nir_lower_vk, data);
864 NIR_PASS(_, nir, pco_nir_lower_io);
865
866 NIR_PASS(_, nir, nir_opt_constant_folding);
867
868 NIR_PASS(_,
869 nir,
870 nir_lower_io,
871 nir_var_shader_in | nir_var_shader_out,
872 glsl_type_size,
873 nir_lower_io_lower_64bit_to_32 |
874 nir_lower_io_use_interpolated_input_intrinsics);
875
876 nir_variable_mode vec_modes = nir->info.stage == MESA_SHADER_FRAGMENT
877 ? nir_var_shader_out
878 : nir_var_shader_in;
879 NIR_PASS(_, nir, nir_lower_io_to_scalar, vec_modes, NULL, NULL);
880 NIR_PASS(_, nir, nir_opt_copy_prop);
881 NIR_PASS(_, nir, nir_opt_dce);
882 NIR_PASS(_, nir, nir_opt_cse);
883 NIR_PASS(_, nir, nir_opt_vectorize_io, vec_modes, false);
884
885 NIR_PASS(_, nir, nir_opt_dce);
886 NIR_PASS(_, nir, nir_opt_constant_folding);
887
888 /* Internal shaders will be using invalid32 types at this stage. */
889 if (!nir->info.internal)
890 NIR_PASS(_, nir, nir_unlower_io_to_vars, true);
891
892 if (nir->info.stage == MESA_SHADER_VERTEX)
893 NIR_PASS(_, nir, pco_nir_lower_clip_cull_vars);
894
895 NIR_PASS(_, nir, pco_nir_lower_images, data, ctx);
896 NIR_PASS(_, nir, pco_nir_lower_atomics, data);
897 NIR_PASS(_,
898 nir,
899 nir_lower_tex,
900 &(nir_lower_tex_options){
901 .lower_txd_cube_map = true,
902 .lower_tg4_offsets = true,
903 });
904 NIR_PASS(_, nir, pco_nir_lower_tex, data, ctx);
905
906 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
907 NIR_PASS(_, nir, pco_nir_lower_alpha_to_coverage);
908
909 NIR_PASS(_, nir, nir_lower_blend, &data->fs.blend_opts);
910
911 nir_opt_peephole_select_options peep_opts = {
912 .limit = 0,
913 .discard_ok = true,
914 };
915 NIR_PASS(_, nir, nir_opt_peephole_select, &peep_opts);
916 NIR_PASS(_, nir, pco_nir_lower_interpolation, &data->fs);
917 NIR_PASS(_, nir, pco_nir_pfo, &data->fs);
918 NIR_PASS(_, nir, pco_nir_lower_fs_intrinsics);
919 } else if (nir->info.stage == MESA_SHADER_VERTEX) {
920 NIR_PASS(_,
921 nir,
922 nir_lower_point_size,
923 PVR_POINT_SIZE_RANGE_MIN,
924 PVR_POINT_SIZE_RANGE_MAX);
925
926 if (!nir->info.internal)
927 NIR_PASS(_, nir, pco_nir_point_size);
928
929 NIR_PASS(_, nir, pco_nir_pvi, &data->vs);
930 NIR_PASS(_, nir, pco_nir_lower_vs_intrinsics);
931 }
932
933 if (data->common.uses.usclib) {
934 assert(ctx->usclib);
935
936 nir_link_shader_functions(nir, ctx->usclib);
937 NIR_PASS(_, nir, nir_inline_functions);
938 nir_remove_non_entrypoints(nir);
939 NIR_PASS(_, nir, nir_opt_deref);
940 NIR_PASS(_, nir, nir_lower_vars_to_ssa);
941 NIR_PASS(_, nir, nir_remove_dead_derefs);
942 NIR_PASS(_,
943 nir,
944 nir_remove_dead_variables,
945 nir_var_function_temp | nir_var_shader_temp,
946 NULL);
947 NIR_PASS(_,
948 nir,
949 nir_lower_vars_to_explicit_types,
950 nir_var_shader_temp | nir_var_function_temp,
951 glsl_get_cl_type_size_align);
952
953 NIR_PASS(_, nir, nir_opt_idiv_const, 32);
954 NIR_PASS(_,
955 nir,
956 nir_lower_idiv,
957 &(nir_lower_idiv_options){
958 .allow_fp16 = false,
959 });
960 }
961
962 NIR_PASS(_,
963 nir,
964 nir_lower_io_to_scalar,
965 nir_var_shader_in | nir_var_shader_out,
966 NULL,
967 NULL);
968
969 NIR_PASS(_, nir, nir_lower_vars_to_ssa);
970 NIR_PASS(_, nir, nir_opt_copy_prop_vars);
971 NIR_PASS(_, nir, nir_opt_dead_write_vars);
972 NIR_PASS(_, nir, nir_opt_combine_stores, nir_var_all);
973
974 pco_nir_opt(ctx, nir, true);
975
976 vec_modes = nir_var_shader_in;
977 /* Fragment shader needs scalar writes after pfo. */
978 if (nir->info.stage != MESA_SHADER_FRAGMENT)
979 vec_modes |= nir_var_shader_out;
980
981 NIR_PASS(_, nir, nir_opt_vectorize_io, vec_modes, false);
982
983 /* Special case for frag coords:
984 * - x,y come from (non-consecutive) special regs - always scalar.
985 * - z,w are iterated and driver will make sure they're consecutive.
986 * - TODO: keep scalar for now, but add pass to vectorize.
987 */
988 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
989 NIR_PASS(_,
990 nir,
991 nir_lower_io_to_scalar,
992 nir_var_shader_in,
993 frag_in_scalar_filter,
994 nir);
995 }
996
997 pco_nir_opt(ctx, nir, true);
998
999 if (pco_should_print_nir(nir)) {
1000 puts("after pco_lower_nir:");
1001 nir_print_shader(nir, stdout);
1002 }
1003 }
1004
1005 static bool is_phi_with_undefs(const nir_instr *instr,
1006 UNUSED const void *cb_data)
1007 {
1008 if (instr->type != nir_instr_type_phi)
1009 return false;
1010
1011 nir_phi_instr *phi = nir_instr_as_phi(instr);
1012
1013 nir_foreach_phi_src (phi_src, phi) {
1014 if (nir_src_is_undef(phi_src->src))
1015 return true;
1016 }
1017
1018 return false;
1019 }
1020
1021 static nir_def *
1022 lower_phi_with_undefs(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
1023 {
1024 nir_phi_instr *phi = nir_instr_as_phi(instr);
1025
1026 nir_foreach_phi_src (phi_src, phi) {
1027 if (nir_src_is_undef(phi_src->src)) {
1028 b->cursor = nir_after_block(phi_src->pred);
1029 nir_src_rewrite(&phi_src->src,
1030 nir_imm_intN_t(b, 0, phi_src->src.ssa->bit_size));
1031 }
1032 }
1033
1034 return NIR_LOWER_INSTR_PROGRESS;
1035 }
1036
1037 static bool
1038 remat_load_const(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
1039 {
1040 if (instr->type != nir_instr_type_load_const)
1041 return false;
1042
1043 nir_load_const_instr *nconst = nir_instr_as_load_const(instr);
1044
1045 if (list_is_singular(&nconst->def.uses))
1046 return false;
1047
1048 nir_foreach_use_safe (src, &nconst->def) {
1049 nir_instr *use_instr = nir_src_parent_instr(src);
1050 b->cursor = nir_before_instr(use_instr);
1051
1052 nir_def *remat_const = nir_build_imm(b,
1053 nconst->def.num_components,
1054 nconst->def.bit_size,
1055 nconst->value);
1056
1057 nir_src_rewrite(src, remat_const);
1058 }
1059
1060 nir_instr_remove(instr);
1061
1062 return true;
1063 }
1064 /**
1065 * \brief Runs post-processing passes on a NIR shader.
1066 *
1067 * \param[in] ctx PCO compiler context.
1068 * \param[in,out] nir NIR shader.
1069 * \param[in,out] data Shader data.
1070 */
1071 void pco_postprocess_nir(pco_ctx *ctx, nir_shader *nir, pco_data *data)
1072 {
1073 nir_move_options move_options = nir_move_const_undef | nir_move_copies |
1074 nir_move_comparisons | nir_move_alu;
1075 NIR_PASS(_, nir, nir_opt_sink, move_options);
1076 NIR_PASS(_, nir, nir_opt_move, move_options);
1077
1078 bool progress;
1079 do {
1080 progress = false;
1081
1082 NIR_PASS(_, nir, nir_opt_algebraic_late);
1083 NIR_PASS(_, nir, pco_nir_lower_algebraic_late);
1084 NIR_PASS(_, nir, nir_opt_constant_folding);
1085 NIR_PASS(_, nir, nir_lower_load_const_to_scalar);
1086 NIR_PASS(_, nir, nir_opt_copy_prop);
1087 NIR_PASS(_, nir, nir_opt_dce);
1088 NIR_PASS(_, nir, nir_opt_cse);
1089 } while (progress);
1090
1091 pco_nir_opt(ctx, nir, false);
1092
1093 NIR_PASS(_, nir, nir_lower_all_phis_to_scalar);
1094
1095 /* Temporary: lower phi undefs to zero because at this stage we don't want to
1096 * lower *all* undefs to zero, but still want to avoid undefined behaviour...
1097 */
1098 nir_shader_lower_instructions(nir,
1099 is_phi_with_undefs,
1100 lower_phi_with_undefs,
1101 NULL);
1102
1103 NIR_PASS(_, nir, nir_convert_from_ssa, true, false);
1104 NIR_PASS(_, nir, nir_opt_copy_prop);
1105 NIR_PASS(_, nir, nir_move_vec_src_uses_to_dest, false);
1106 NIR_PASS(_, nir, nir_opt_dce);
1107
1108 progress = false;
1109 NIR_PASS(progress, nir, nir_opt_rematerialize_compares);
1110 if (progress)
1111 NIR_PASS(_, nir, nir_opt_dce);
1112
1113 NIR_PASS(_, nir, nir_trivialize_registers);
1114
1115 if (!nir->info.internal) {
1116 nir_shader_instructions_pass(nir,
1117 remat_load_const,
1118 nir_metadata_none,
1119 NULL);
1120 }
1121
1122 /* Re-index everything. */
1123 nir_foreach_function_with_impl (_, impl, nir) {
1124 nir_index_blocks(impl);
1125 nir_index_instrs(impl);
1126 nir_index_ssa_defs(impl);
1127 }
1128
1129 nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
1130
1131 gather_data(nir, data);
1132
1133 if (pco_should_print_nir(nir)) {
1134 puts("after pco_postprocess_nir:");
1135 nir_print_shader(nir, stdout);
1136 }
1137 }