Debian Codegraph @

src/imagination/pco/pco_nir_pvfio.c

raw
1 /*
2 * Copyright © 2024 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pco_nir_pvfio.c
9 *
10 * \brief PCO NIR per-vertex/fragment input/output passes.
11 */
12
13 #include "compiler/glsl_types.h"
14 #include "compiler/shader_enums.h"
15 #include "nir.h"
16 #include "nir_builder.h"
17 #include "nir_format_convert.h"
18 #include "pco.h"
19 #include "pco_builder.h"
20 #include "pco_internal.h"
21 #include "util/macros.h"
22 #include "util/u_dynarray.h"
23
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <stdio.h>
27
28 /** Per-fragment output pass state. */
29 struct pfo_state {
30 struct util_dynarray loads; /** List of fragment loads. */
31 struct util_dynarray stores; /** List of fragment stores. */
32
33 /* Src for depth feedback (NULL if unused). */
34 nir_def *depth_feedback_src;
35
36 nir_def *discard_cond_reg;
37 bool has_discards;
38
39 nir_intrinsic_instr *last_discard_store;
40
41 bool has_sample_check;
42
43 /* nir_instr *terminate; */
44
45 pco_fs_data *fs; /** Fragment-specific data. */
46 };
47
48 /** Per-vertex input pass state. */
49 struct pvi_state {
50 nir_def *attribs[MAX_VERTEX_GENERIC_ATTRIBS]; /** Loaded vertex attribs. */
51 pco_vs_data *vs; /** Vertex-specific data. */
52 };
53
54 /**
55 * \brief Returns the GLSL base type equivalent of a pipe format.
56 *
57 * \param[in] format Pipe format.
58 * \return The GLSL base type, or GLSL_TYPE_ERROR if unsupported/invalid.
59 */
60 static inline enum glsl_base_type base_type_from_fmt(enum pipe_format format)
61 {
62 const struct util_format_description *desc = util_format_description(format);
63 int chan = util_format_get_first_non_void_channel(format);
64 if (chan < 0)
65 return GLSL_TYPE_ERROR;
66
67 switch (desc->channel[chan].type) {
68 case UTIL_FORMAT_TYPE_UNSIGNED:
69 return GLSL_TYPE_UINT;
70
71 case UTIL_FORMAT_TYPE_SIGNED:
72 return GLSL_TYPE_INT;
73
74 case UTIL_FORMAT_TYPE_FLOAT:
75 return GLSL_TYPE_FLOAT;
76
77 default:
78 break;
79 }
80
81 return GLSL_TYPE_ERROR;
82 }
83
84 static enum pipe_format
85 to_pbe_format(nir_builder *b, enum pipe_format format, nir_def **input)
86 {
87 switch (format) {
88 case PIPE_FORMAT_B5G6R5_UNORM:
89 return PIPE_FORMAT_R8G8B8_UNORM;
90
91 case PIPE_FORMAT_A4R4G4B4_UNORM:
92 case PIPE_FORMAT_B5G5R5A1_UNORM:
93 case PIPE_FORMAT_B8G8R8A8_UNORM:
94 return PIPE_FORMAT_R8G8B8A8_UNORM;
95
96 case PIPE_FORMAT_R8G8B8A8_SRGB:
97 case PIPE_FORMAT_B8G8R8A8_SRGB:
98 if (input)
99 *input = nir_fsat(b, *input);
100 FALLTHROUGH;
101
102 case PIPE_FORMAT_R10G10B10A2_UNORM:
103 return PIPE_FORMAT_R16G16B16A16_FLOAT;
104
105 case PIPE_FORMAT_R11G11B10_FLOAT:
106 return PIPE_FORMAT_R16G16B16_FLOAT;
107
108 /* For loadops. */
109 case PIPE_FORMAT_Z32_FLOAT:
110 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
111 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
112 case PIPE_FORMAT_Z16_UNORM:
113 case PIPE_FORMAT_Z24X8_UNORM:
114 assert(b->shader->info.internal);
115 return PIPE_FORMAT_R32_FLOAT;
116
117 default:
118 break;
119 }
120
121 return format;
122 }
123
124 static unsigned format_chans_per_dword(enum pipe_format format)
125 {
126 const struct util_format_description *desc = util_format_description(format);
127 unsigned accum_bits = 0;
128
129 for (unsigned u = 0; u < desc->nr_channels; ++u) {
130 /* Exactly one dword, great! */
131 if (accum_bits == 32)
132 return u;
133
134 /* Went over, back off by one. */
135 if (accum_bits > 32) {
136 /* We don't support formats with channels > 1 dword. */
137 assert(u > 1);
138 return u - 1;
139 }
140
141 accum_bits += desc->channel[u].size;
142 }
143
144 /* Loop finished, all channels can fit. */
145 return desc->nr_channels;
146 }
147
148 static nir_def *pack_to_format(nir_builder *b,
149 nir_def *input,
150 nir_alu_type src_type,
151 enum pipe_format format)
152 {
153 const struct util_format_description *desc = util_format_description(format);
154
155 nir_def *zero = nir_imm_int(b, 0);
156 nir_def *one = nir_alu_type_get_base_type(src_type) == nir_type_float
157 ? nir_imm_float(b, 1.0f)
158 : nir_imm_int(b, 1);
159
160 nir_def *input_comps[4];
161 /* Populate any missing components .*/
162 for (unsigned u = 0; u < ARRAY_SIZE(input_comps); ++u) {
163 enum pipe_swizzle s = desc->swizzle[u];
164
165 if (s <= PIPE_SWIZZLE_W)
166 input_comps[u] = nir_channel(b, input, s);
167 else if (s == PIPE_SWIZZLE_0)
168 input_comps[u] = zero;
169 else if (s == PIPE_SWIZZLE_1)
170 input_comps[u] = one;
171 else
172 UNREACHABLE("");
173 }
174
175 unsigned format_bits = util_format_get_blocksizebits(format);
176 unsigned format_dwords = DIV_ROUND_UP(format_bits, 32);
177 nir_def *packed_comps[] = { zero, zero, zero, zero };
178
179 /* Special case: no packing required. */
180 if (util_format_get_max_channel_size(format) == 32)
181 return nir_vec(b, input_comps, format_dwords);
182
183 /* Special case: can't be packed with op, need bit-packing instead. */
184 if (util_format_is_pure_integer(format)) {
185 for (unsigned u = 0; u < desc->nr_channels; ++u) {
186 unsigned dword = desc->channel[u].shift / 32;
187 unsigned offset = desc->channel[u].shift % 32;
188 unsigned size = desc->channel[u].size;
189
190 packed_comps[dword] = nir_bitfield_insert_imm(b,
191 packed_comps[dword],
192 input_comps[u],
193 offset,
194 size);
195 }
196
197 return nir_vec(b, packed_comps, format_dwords);
198 }
199
200 unsigned chans_per_dword = format_chans_per_dword(format);
201 unsigned chans_remaining = desc->nr_channels;
202 input = nir_vec(b, input_comps, desc->nr_channels);
203 for (unsigned u = 0; u < format_dwords; ++u) {
204 unsigned chans_to_pack =
205 chans_remaining > chans_per_dword ? chans_per_dword : chans_remaining;
206 unsigned chans_packed = desc->nr_channels - chans_remaining;
207
208 nir_def *input_chans =
209 nir_channels(b, input, BITFIELD_RANGE(chans_packed, chans_to_pack));
210 packed_comps[u] = nir_pack_pco(b, input_chans, .format = format);
211
212 chans_remaining -= chans_to_pack;
213 }
214
215 assert(!chans_remaining);
216 return nir_vec(b, packed_comps, format_dwords);
217 }
218
219 static nir_def *unpack_from_format(nir_builder *b,
220 nir_def *packed_comps[static 4],
221 nir_alu_type dest_type,
222 enum pipe_format format,
223 unsigned components_needed)
224 {
225 const struct util_format_description *desc = util_format_description(format);
226
227 nir_def *unpacked_comps[4];
228
229 unsigned format_bits = util_format_get_blocksizebits(format);
230 unsigned format_dwords = DIV_ROUND_UP(format_bits, 32);
231
232 /* Special case: no unpacking required. */
233 if (util_format_get_max_channel_size(format) == 32) {
234 for (unsigned u = 0; u < desc->nr_channels; ++u)
235 unpacked_comps[u] = packed_comps[u];
236 }
237
238 /* Special case: can't be unpacked with op, need bit-unpacking instead. */
239 else if (util_format_is_pure_integer(format)) {
240 nir_def *(*nir_bitfield_extract_imm)(nir_builder *,
241 nir_def *,
242 uint32_t,
243 uint32_t) =
244 util_format_is_pure_uint(format) ? nir_ubitfield_extract_imm
245 : nir_ibitfield_extract_imm;
246
247 for (unsigned u = 0; u < desc->nr_channels; ++u) {
248 unsigned dword = desc->channel[u].shift / 32;
249 unsigned offset = desc->channel[u].shift % 32;
250 unsigned size = desc->channel[u].size;
251
252 unpacked_comps[u] =
253 nir_bitfield_extract_imm(b, packed_comps[dword], offset, size);
254 }
255 }
256
257 else {
258 unsigned chans_per_dword = format_chans_per_dword(format);
259 unsigned chans_remaining = desc->nr_channels;
260
261 for (unsigned u = 0; u < format_dwords; ++u) {
262 unsigned chans_to_unpack = chans_remaining > chans_per_dword
263 ? chans_per_dword
264 : chans_remaining;
265
266 nir_def *unpacked = nir_unpack_pco(b,
267 chans_to_unpack,
268 packed_comps[u],
269 .format = format);
270
271 unsigned chans_unpacked = desc->nr_channels - chans_remaining;
272 for (unsigned v = 0; v < chans_to_unpack; ++v)
273 unpacked_comps[chans_unpacked + v] = nir_channel(b, unpacked, v);
274
275 chans_remaining -= chans_to_unpack;
276 }
277
278 assert(!chans_remaining);
279 }
280
281 nir_def *zero = nir_imm_int(b, 0);
282 nir_def *one = nir_alu_type_get_base_type(dest_type) == nir_type_float
283 ? nir_imm_float(b, 1.0f)
284 : nir_imm_int(b, 1);
285
286 nir_def *output_comps[4];
287 /* Populate any missing components .*/
288 for (unsigned u = 0; u < ARRAY_SIZE(output_comps); ++u) {
289 enum pipe_swizzle s = desc->swizzle[u];
290
291 if (s <= PIPE_SWIZZLE_W)
292 output_comps[u] = unpacked_comps[s];
293 else if (s == PIPE_SWIZZLE_0)
294 output_comps[u] = zero;
295 else if (s == PIPE_SWIZZLE_1)
296 output_comps[u] = one;
297 else
298 UNREACHABLE("");
299 }
300
301 return nir_vec(b, output_comps, components_needed);
302 }
303
304 static inline bool is_processed(nir_intrinsic_instr *intr)
305 {
306 nir_alu_type type;
307
308 if (nir_intrinsic_has_src_type(intr))
309 type = nir_intrinsic_src_type(intr);
310 else if (nir_intrinsic_has_dest_type(intr))
311 type = nir_intrinsic_dest_type(intr);
312 else
313 return true;
314
315 return nir_alu_type_get_base_type(type) == nir_type_invalid;
316 }
317
318 static nir_def *lower_pfo_store(nir_builder *b,
319 nir_intrinsic_instr *intr,
320 struct pfo_state *state)
321 {
322 /* Skip stores we've already processed. */
323 if (is_processed(intr)) {
324 util_dynarray_append(&state->stores, intr);
325 return NULL;
326 }
327
328 nir_def *input = intr->src[0].ssa;
329 nir_src *offset = &intr->src[1];
330 assert(nir_src_as_uint(*offset) == 0);
331
332 ASSERTED unsigned bit_size = input->bit_size;
333 assert(bit_size == 32);
334
335 unsigned component = nir_intrinsic_component(intr);
336 assert(!component);
337
338 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
339 gl_frag_result location = io_semantics.location;
340
341 b->cursor = nir_before_instr(&intr->instr);
342
343 enum pipe_format format = state->fs->output_formats[location];
344 if (format == PIPE_FORMAT_NONE)
345 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
346
347 format = to_pbe_format(b, format, &input);
348
349 nir_alu_type src_type = nir_intrinsic_src_type(intr);
350 nir_def *output = pack_to_format(b, input, src_type, format);
351
352 /* Emit and track the new store. */
353 nir_intrinsic_instr *store =
354 nir_store_output(b,
355 output,
356 offset->ssa,
357 .base = nir_intrinsic_base(intr),
358 .write_mask = BITFIELD_MASK(output->num_components),
359 .src_type = nir_type_invalid | 32,
360 .component = component,
361 .io_semantics = io_semantics,
362 .io_xfb = nir_intrinsic_io_xfb(intr));
363
364 util_dynarray_append(&state->stores, store);
365
366 /* Update the type of the stored variable. */
367 nir_variable *var =
368 nir_find_variable_with_location(b->shader, nir_var_shader_out, location);
369 assert(var);
370 var->type = glsl_uvec_type(output->num_components);
371
372 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
373 }
374
375 static nir_def *lower_pfo_load(nir_builder *b,
376 nir_intrinsic_instr *intr,
377 struct pfo_state *state)
378 {
379 /* Skip loads we've already processed. */
380 if (is_processed(intr)) {
381 util_dynarray_append(&state->loads, intr);
382 return NULL;
383 }
384
385 unsigned base = nir_intrinsic_base(intr);
386
387 nir_src *offset = &intr->src[0];
388 assert(nir_src_as_uint(*offset) == 0);
389
390 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
391 gl_frag_result location = io_semantics.location;
392
393 enum pipe_format format;
394 /* Special case for input attachments. */
395 if (location == FRAG_RESULT_COLOR) {
396 format = state->fs->ia_formats[base];
397 } else {
398 assert(location >= FRAG_RESULT_DATA0);
399 assert(!base);
400 format = state->fs->output_formats[location];
401 }
402
403 if (format == PIPE_FORMAT_NONE)
404 return nir_undef(b, intr->def.num_components, intr->def.bit_size);
405
406 format = to_pbe_format(b, format, NULL);
407
408 nir_def *packed_comps[4];
409 for (unsigned c = 0; c < ARRAY_SIZE(packed_comps); ++c) {
410 packed_comps[c] = nir_load_output(b,
411 1,
412 32,
413 offset->ssa,
414 .base = base,
415 .component = c,
416 .dest_type = nir_type_invalid | 32,
417 .io_semantics = io_semantics);
418
419 nir_intrinsic_instr *load =
420 nir_def_as_intrinsic(packed_comps[c]);
421
422 util_dynarray_append(&state->loads, load);
423 }
424
425 nir_alu_type dest_type = nir_intrinsic_dest_type(intr);
426 return unpack_from_format(b,
427 packed_comps,
428 dest_type,
429 format,
430 intr->def.num_components);
431 }
432
433 /**
434 * \brief Filters PFO-related instructions.
435 *
436 * \param[in] instr NIR instruction.
437 * \param[in] cb_data User callback data.
438 * \return True if the instruction matches the filter.
439 */
440 static bool is_pfo(const nir_instr *instr, UNUSED const void *cb_data)
441 {
442 if (instr->type != nir_instr_type_intrinsic)
443 return false;
444
445 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
446 switch (intr->intrinsic) {
447 case nir_intrinsic_store_output:
448 case nir_intrinsic_load_output:
449 case nir_intrinsic_demote:
450 case nir_intrinsic_demote_if:
451 return true;
452
453 default:
454 break;
455 }
456
457 return false;
458 }
459
460 /**
461 * \brief Lowers a PFO-related instruction.
462 *
463 * \param[in] b NIR builder.
464 * \param[in] instr NIR instruction.
465 * \param[in] cb_data User callback data.
466 * \return The replacement/lowered def.
467 */
468 static nir_def *lower_pfo(nir_builder *b, nir_instr *instr, void *cb_data)
469 {
470 struct pfo_state *state = cb_data;
471 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
472
473 switch (intr->intrinsic) {
474 case nir_intrinsic_store_output: {
475 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
476 if (sem.location >= FRAG_RESULT_DATA0)
477 return lower_pfo_store(b, intr, state);
478
479 if (sem.location == FRAG_RESULT_DEPTH) {
480 assert(!state->depth_feedback_src);
481 state->depth_feedback_src = nir_fsat(b, intr->src[0].ssa);
482
483 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
484 }
485
486 if (sem.location == FRAG_RESULT_SAMPLE_MASK) {
487 nir_def *smp_msk =
488 nir_ishl(b, nir_imm_int(b, 1), nir_load_sample_id(b));
489
490 smp_msk = nir_iand(b, smp_msk, nir_load_sample_mask_in(b));
491 smp_msk = nir_iand(b, smp_msk, intr->src[0].ssa);
492 nir_def *cond = nir_ieq_imm(b, smp_msk, 0);
493
494 state->has_discards = true;
495 state->has_sample_check = true;
496 nir_def *val = nir_load_reg(b, state->discard_cond_reg);
497 val = nir_ior(b, val, cond);
498 state->last_discard_store =
499 nir_build_store_reg(b, val, state->discard_cond_reg);
500 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
501 }
502
503 UNREACHABLE("");
504 }
505
506 case nir_intrinsic_load_output:
507 return lower_pfo_load(b, intr, state);
508
509 case nir_intrinsic_demote:
510 state->has_discards = true;
511 state->last_discard_store =
512 nir_build_store_reg(b, nir_imm_true(b), state->discard_cond_reg);
513 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
514
515 case nir_intrinsic_demote_if: {
516 state->has_discards = true;
517 nir_def *val = nir_load_reg(b, state->discard_cond_reg);
518 val = nir_ior(b, val, intr->src[0].ssa);
519 state->last_discard_store =
520 nir_build_store_reg(b, val, state->discard_cond_reg);
521 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
522 }
523
524 default:
525 break;
526 }
527
528 return false;
529 }
530
531 static bool lower_isp_fb(nir_builder *b, struct pfo_state *state)
532 {
533 if (b->shader->info.internal)
534 return false;
535
536 bool has_depth_feedback = !!state->depth_feedback_src;
537 if (b->shader->info.writes_memory && !has_depth_feedback) {
538 nir_variable *var_pos = nir_get_variable_with_location(b->shader,
539 nir_var_shader_in,
540 VARYING_SLOT_POS,
541 glsl_vec4_type());
542 var_pos->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
543
544 b->cursor = nir_before_block(
545 nir_start_block(nir_shader_get_entrypoint(b->shader)));
546
547 state->depth_feedback_src =
548 nir_load_input(b,
549 1,
550 32,
551 nir_imm_int(b, 0),
552 .component = 2,
553 .dest_type = nir_type_float32,
554 .io_semantics = (nir_io_semantics){
555 .location = VARYING_SLOT_POS,
556 .num_slots = 1,
557 });
558
559 has_depth_feedback = true;
560 }
561
562 if (!state->has_sample_check) {
563 b->cursor = nir_after_instr(&state->last_discard_store->instr);
564
565 nir_def *smp_msk = nir_ishl(b, nir_imm_int(b, 1), nir_load_sample_id(b));
566 smp_msk = nir_iand(b, smp_msk, nir_load_sample_mask_in(b));
567 nir_def *cond = nir_ieq_imm(b, smp_msk, 0);
568
569 nir_def *val = nir_load_reg(b, state->discard_cond_reg);
570 val = nir_ior(b, val, cond);
571 state->last_discard_store =
572 nir_build_store_reg(b, val, state->discard_cond_reg);
573
574 state->has_sample_check = true;
575 state->has_discards = true;
576 }
577
578 /* Insert isp feedback instruction before the first store,
579 * or if there are no stores, at the end.
580 */
581 if (state->stores.size > 0)
582 b->cursor = nir_before_instr(
583 &(*(nir_intrinsic_instr **)util_dynarray_begin(&state->stores))->instr);
584 else
585 b->cursor = nir_after_block(
586 nir_impl_last_block(nir_shader_get_entrypoint(b->shader)));
587
588 nir_def *undef = nir_undef(b, 1, 32);
589
590 nir_isp_feedback_pco(
591 b,
592 state->has_discards ? nir_i2b(b, nir_load_reg(b, state->discard_cond_reg))
593 : undef,
594 has_depth_feedback ? state->depth_feedback_src : undef);
595
596 state->fs->uses.discard = state->has_discards;
597 state->fs->uses.depth_feedback = has_depth_feedback;
598
599 return true;
600 }
601
602 static bool sink_outputs(nir_shader *shader, struct pfo_state *state)
603 {
604 bool progress = false;
605
606 nir_instr *after_instr = nir_block_last_instr(
607 nir_impl_last_block(nir_shader_get_entrypoint(shader)));
608
609 util_dynarray_foreach (&state->stores, nir_intrinsic_instr *, store) {
610 nir_instr *instr = &(*store)->instr;
611
612 progress |= nir_instr_move(nir_after_instr(after_instr), instr);
613 after_instr = instr;
614 }
615
616 return progress;
617 }
618
619 static bool z_replicate(nir_shader *shader, struct pfo_state *state)
620 {
621 if (shader->info.internal || state->fs->z_replicate == ~0u)
622 return false;
623
624 assert(!nir_find_variable_with_location(shader,
625 nir_var_shader_out,
626 state->fs->z_replicate));
627
628 nir_create_variable_with_location(shader,
629 nir_var_shader_out,
630 state->fs->z_replicate,
631 glsl_float_type());
632
633 if (!state->depth_feedback_src) {
634 nir_variable *var_pos = nir_get_variable_with_location(shader,
635 nir_var_shader_in,
636 VARYING_SLOT_POS,
637 glsl_vec4_type());
638 var_pos->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
639
640 nir_builder b = nir_builder_at(
641 nir_before_block(nir_start_block(nir_shader_get_entrypoint(shader))));
642
643 state->depth_feedback_src =
644 nir_load_input(&b,
645 1,
646 32,
647 nir_imm_int(&b, 0),
648 .component = 2,
649 .dest_type = nir_type_float32,
650 .io_semantics = (nir_io_semantics){
651 .location = VARYING_SLOT_POS,
652 .num_slots = 1,
653 });
654 }
655
656 nir_builder b = nir_builder_at(
657 nir_after_block(nir_impl_last_block(nir_shader_get_entrypoint(shader))));
658 nir_store_output(&b,
659 state->depth_feedback_src,
660 nir_imm_int(&b, 0),
661 .write_mask = 1,
662 .src_type = nir_type_invalid | 32,
663 .io_semantics = (nir_io_semantics){
664 .location = state->fs->z_replicate,
665 .num_slots = 1,
666 });
667
668 return true;
669 }
670
671 static bool is_frag_color_out(const nir_instr *instr,
672 UNUSED const void *cb_data)
673 {
674 if (instr->type != nir_instr_type_intrinsic)
675 return false;
676
677 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
678 if (intr->intrinsic != nir_intrinsic_store_output)
679 return false;
680
681 gl_frag_result location = nir_intrinsic_io_semantics(intr).location;
682 return location >= FRAG_RESULT_DATA0 && location < FRAG_RESULT_MAX;
683 }
684
685 static bool lower_demote_samples(nir_builder *b,
686 nir_intrinsic_instr *intr,
687 UNUSED void *cb_data)
688 {
689 if (intr->intrinsic != nir_intrinsic_demote_samples)
690 return false;
691
692 b->cursor = nir_before_instr(&intr->instr);
693 nir_def *to_keep = nir_u2u32(b, nir_inot(b, intr->src[0].ssa));
694 nir_def *sample_mask = nir_load_savmsk_vm_pco(b);
695 nir_def *current_mask =
696 nir_ishl(b, nir_imm_int(b, 1), nir_load_sample_id(b));
697 nir_def *cond = nir_iand(b, to_keep, nir_iand(b, sample_mask, current_mask));
698 nir_demote_if(b, nir_ieq_imm(b, cond, 0));
699
700 nir_instr_remove(&intr->instr);
701
702 return true;
703 }
704
705 bool pco_nir_lower_alpha_to_coverage(nir_shader *shader)
706 {
707 if (shader->info.internal)
708 return false;
709
710 nir_builder b = nir_builder_create(nir_shader_get_entrypoint(shader));
711 b.cursor =
712 nir_before_block(nir_start_block(nir_shader_get_entrypoint(shader)));
713 nir_def *a2c_enabled = nir_ine_imm(
714 &b,
715 nir_ubitfield_extract_imm(&b, nir_load_fs_meta_pco(&b), 25, 1),
716 0);
717
718 nir_lower_alpha_to_coverage(shader, true, a2c_enabled);
719
720 nir_shader_intrinsics_pass(shader,
721 lower_demote_samples,
722 nir_metadata_control_flow,
723 NULL);
724
725 return true;
726 }
727
728 static nir_def *
729 lower_alpha_to_one(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
730 {
731 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
732
733 nir_src *input_src = &intr->src[0];
734 nir_def *input = input_src->ssa;
735 nir_src *offset = &intr->src[1];
736 assert(nir_src_as_uint(*offset) == 0);
737
738 /* Skip color write that don't include alpha. */
739 if (input->num_components != 4)
740 return NULL;
741
742 b->cursor = nir_before_instr(&intr->instr);
743
744 /* TODO: define or other way of representing bit 0 of metadata... */
745 nir_def *alpha_to_one_enabled =
746 nir_ine_imm(b,
747 nir_ubitfield_extract_imm(b, nir_load_fs_meta_pco(b), 0, 1),
748 0);
749
750 nir_def *alpha = nir_bcsel(b,
751 alpha_to_one_enabled,
752 nir_imm_float(b, 1.0f),
753 nir_channel(b, input, 3));
754
755 nir_src_rewrite(input_src, nir_vector_insert_imm(b, input, alpha, 3));
756
757 return NIR_LOWER_INSTR_PROGRESS;
758 }
759
760 static bool is_load_sample_mask(const nir_instr *instr,
761 UNUSED const void *cb_data)
762 {
763 if (instr->type != nir_instr_type_intrinsic)
764 return false;
765
766 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
767 return intr->intrinsic == nir_intrinsic_load_sample_mask_in;
768 }
769
770 static nir_def *
771 lower_load_sample_mask(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
772 {
773 b->cursor = nir_before_instr(instr);
774
775 nir_def *smp_msk =
776 nir_ubitfield_extract_imm(b, nir_load_fs_meta_pco(b), 9, 16);
777
778 smp_msk = nir_iand(b, smp_msk, nir_load_savmsk_vm_pco(b));
779
780 return smp_msk;
781 }
782
783 static nir_def *
784 lower_color_write_enable(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
785 {
786 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
787
788 nir_src *input_src = &intr->src[0];
789 nir_def *input = input_src->ssa;
790 nir_def *offset = intr->src[1].ssa;
791
792 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
793 unsigned color_write_index = io_semantics.location - FRAG_RESULT_DATA0;
794 io_semantics.fb_fetch_output = true;
795
796 b->cursor = nir_before_instr(&intr->instr);
797
798 /* TODO: nir op that returns bool based on whether a bit is set. */
799 /* TODO: define for 1 */
800 nir_def *color_write_enabled =
801 nir_ine_imm(b,
802 nir_ubitfield_extract_imm(b,
803 nir_load_fs_meta_pco(b),
804 1 + color_write_index,
805 1),
806 0);
807
808 nir_def *prev_input =
809 nir_load_output(b,
810 input->num_components,
811 input->bit_size,
812 offset,
813 .base = nir_intrinsic_base(intr),
814 .range = nir_intrinsic_range(intr),
815 .component = nir_intrinsic_component(intr),
816 .dest_type = nir_intrinsic_src_type(intr),
817 .io_semantics = io_semantics);
818
819 nir_src_rewrite(input_src,
820 nir_bcsel(b, color_write_enabled, input, prev_input));
821
822 return NIR_LOWER_INSTR_PROGRESS;
823 }
824
825 /**
826 * \brief Per-fragment output pass.
827 *
828 * \param[in,out] shader NIR shader.
829 * \param[in,out] fs Fragment shader-specific data.
830 * \return True if the pass made progress.
831 */
832 bool pco_nir_pfo(nir_shader *shader, pco_fs_data *fs)
833 {
834 assert(shader->info.stage == MESA_SHADER_FRAGMENT);
835
836 nir_builder b = nir_builder_create(nir_shader_get_entrypoint(shader));
837 b.cursor =
838 nir_before_block(nir_start_block(nir_shader_get_entrypoint(shader)));
839
840 struct pfo_state state = {
841 .fs = fs,
842 .discard_cond_reg = nir_decl_reg(&b, 1, 1, 0),
843 };
844 state.last_discard_store =
845 nir_build_store_reg(&b, nir_imm_false(&b), state.discard_cond_reg);
846
847 state.loads = UTIL_DYNARRAY_INIT;
848 state.stores = UTIL_DYNARRAY_INIT;
849
850 bool progress = false;
851
852 /* TODO: instead of doing multiple passes, probably better to just cache all
853 * the stores
854 */
855 if (!shader->info.internal) {
856 progress |= nir_shader_lower_instructions(shader,
857 is_frag_color_out,
858 lower_alpha_to_one,
859 &state);
860 }
861
862 if (fs->meta_present.color_write_enable)
863 progress |= nir_shader_lower_instructions(shader,
864 is_frag_color_out,
865 lower_color_write_enable,
866 NULL);
867
868 progress |= nir_shader_lower_instructions(shader, is_pfo, lower_pfo, &state);
869 progress |= lower_isp_fb(&b, &state);
870
871 progress |= sink_outputs(shader, &state);
872 progress |= z_replicate(shader, &state);
873
874 progress |= nir_shader_lower_instructions(shader,
875 is_load_sample_mask,
876 lower_load_sample_mask,
877 NULL);
878
879 util_dynarray_fini(&state.stores);
880 util_dynarray_fini(&state.loads);
881
882 return progress;
883 }
884
885 static nir_def *lower_pvi(nir_builder *b, nir_instr *instr, void *cb_data)
886 {
887 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
888 struct pvi_state *state = cb_data;
889
890 unsigned start_comp = nir_intrinsic_component(intr);
891 unsigned num_comps = intr->def.num_components;
892
893 ASSERTED nir_src *offset = &intr->src[0];
894 assert(nir_src_as_uint(*offset) == 0);
895
896 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
897 gl_vert_attrib location = io_semantics.location;
898 nir_def *attrib = state->attribs[location - VERT_ATTRIB_GENERIC0];
899 assert(attrib);
900
901 b->cursor = nir_before_instr(&intr->instr);
902 return nir_channels(b, attrib, BITFIELD_RANGE(start_comp, num_comps));
903 }
904
905 static bool is_pvi(const nir_instr *instr, const void *cb_data)
906 {
907 if (instr->type != nir_instr_type_intrinsic)
908 return false;
909
910 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
911 if (intr->intrinsic != nir_intrinsic_load_input)
912 return false;
913
914 if (is_processed(intr))
915 return false;
916
917 ASSERTED gl_vert_attrib location = nir_intrinsic_io_semantics(intr).location;
918 assert(location >= VERT_ATTRIB_GENERIC0 &&
919 location <= VERT_ATTRIB_GENERIC15);
920
921 return true;
922 }
923
924 /**
925 * \brief Per-vertex input pass.
926 *
927 * \param[in,out] shader NIR shader.
928 * \param[in,out] vs Vertex shader-specific data.
929 * \return True if the pass made progress.
930 */
931 bool pco_nir_pvi(nir_shader *shader, pco_vs_data *vs)
932 {
933 assert(shader->info.stage == MESA_SHADER_VERTEX);
934
935 struct pvi_state state = { .vs = vs };
936
937 nir_builder b = nir_builder_at(
938 nir_before_block(nir_start_block(nir_shader_get_entrypoint(shader))));
939 for (unsigned u = 0; u < ARRAY_SIZE(state.attribs); ++u) {
940 gl_vert_attrib location = u + VERT_ATTRIB_GENERIC0;
941 enum pipe_format format = vs->attrib_formats[location];
942 if (format == PIPE_FORMAT_NONE)
943 continue;
944
945 /* Update the type of the stored variable, remove any fractional vars. */
946 nir_variable *var = NULL;
947 nir_alu_type base_type = 0;
948 nir_foreach_variable_with_modes_safe (iter_var,
949 shader,
950 nir_var_shader_in) {
951 if (iter_var->data.location != location)
952 continue;
953
954 if (!base_type)
955 base_type = nir_get_nir_type_for_glsl_type(iter_var->type);
956 #ifndef NDEBUG
957 else
958 assert(base_type == nir_get_nir_type_for_glsl_type(iter_var->type));
959 #endif /* NDEBUG */
960
961 if (!iter_var->data.location_frac) {
962 assert(!var);
963 var = iter_var;
964 continue;
965 }
966
967 exec_node_remove(&iter_var->node);
968 }
969
970 if (!var) {
971 if (!base_type)
972 continue;
973
974 /* An attrib var was found but was fractional so we dropped it. */
975 var = nir_variable_create(shader, nir_var_shader_in, NULL, NULL);
976 var->data.location = location;
977 }
978
979 unsigned format_dwords =
980 DIV_ROUND_UP(util_format_get_blocksize(format), sizeof(uint32_t));
981 var->type = glsl_uvec_type(format_dwords);
982
983 nir_def *packed_comps[4];
984 for (unsigned c = 0; c < ARRAY_SIZE(packed_comps); ++c) {
985 packed_comps[c] = nir_load_input(&b,
986 1,
987 32,
988 nir_imm_int(&b, 0),
989 .range = 1,
990 .component = c,
991 .dest_type = nir_type_invalid | 32,
992 .io_semantics = (nir_io_semantics){
993 .location = location,
994 .num_slots = 1,
995 });
996 }
997
998 state.attribs[u] =
999 unpack_from_format(&b, packed_comps, base_type, format, 4);
1000 }
1001
1002 nir_shader_lower_instructions(shader, is_pvi, lower_pvi, &state);
1003
1004 return true;
1005 }
1006
1007 /**
1008 * \brief Checks if the point size is written.
1009 *
1010 * \param[in] b NIR builder.
1011 * \param[in] intr NIR intrinsic instruction.
1012 * \param[in] cb_data User callback data.
1013 * \return True if the instruction was lowered.
1014 */
1015 static bool
1016 check_psiz_write(nir_builder *b, nir_intrinsic_instr *intr, void *cb_data)
1017 {
1018 if (intr->intrinsic != nir_intrinsic_store_output)
1019 return false;
1020
1021 bool *writes_psiz = cb_data;
1022
1023 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
1024 *writes_psiz |= (io_semantics.location == VARYING_SLOT_PSIZ);
1025
1026 return false;
1027 }
1028
1029 /**
1030 * \brief Vertex shader point size pass.
1031 *
1032 * \param[in,out] shader NIR shader.
1033 * \return True if the pass made progress.
1034 */
1035 bool pco_nir_point_size(nir_shader *shader)
1036 {
1037 assert(shader->info.stage == MESA_SHADER_VERTEX);
1038 if (shader->info.internal)
1039 return false;
1040
1041 bool writes_psiz = false;
1042 nir_shader_intrinsics_pass(shader,
1043 check_psiz_write,
1044 nir_metadata_all,
1045 &writes_psiz);
1046
1047 /* Nothing to do if the shader already writes the point size. */
1048 if (writes_psiz)
1049 return false;
1050
1051 /* Create a point size variable if there isn't one. */
1052 nir_get_variable_with_location(shader,
1053 nir_var_shader_out,
1054 VARYING_SLOT_PSIZ,
1055 glsl_float_type());
1056
1057 /* Add a point size write. */
1058 nir_builder b = nir_builder_at(
1059 nir_after_block(nir_impl_last_block(nir_shader_get_entrypoint(shader))));
1060
1061 nir_store_output(&b,
1062 nir_imm_float(&b, PVR_POINT_SIZE_RANGE_MIN),
1063 nir_imm_int(&b, 0),
1064 .base = 0,
1065 .range = 1,
1066 .write_mask = 1,
1067 .component = 0,
1068 .src_type = nir_type_float32,
1069 .io_semantics = (nir_io_semantics){
1070 .location = VARYING_SLOT_PSIZ,
1071 .num_slots = 1,
1072 });
1073
1074 return true;
1075 }
1076
1077 static bool is_fs_intr(const nir_instr *instr, UNUSED const void *cb_data)
1078 {
1079 if (instr->type != nir_instr_type_intrinsic)
1080 return false;
1081
1082 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1083 switch (intr->intrinsic) {
1084 case nir_intrinsic_load_front_face:
1085 return true;
1086
1087 default:
1088 break;
1089 }
1090
1091 return false;
1092 }
1093
1094 static nir_def *lower_front_face(nir_builder *b, nir_intrinsic_instr *intr)
1095 {
1096 nir_def *face_ccw = nir_load_face_ccw_pco(b);
1097 nir_def *front_face = nir_ieq_imm(b, face_ccw, 0);
1098
1099 nir_def *ff_op = nir_load_front_face_op_pco(b);
1100 nir_def *ff_elems[] = {
1101 [PCO_FRONT_FACE_OP_NOP] = front_face,
1102 [PCO_FRONT_FACE_OP_SWAP] = nir_inot(b, front_face),
1103 [PCO_FRONT_FACE_OP_TRUE] = nir_imm_true(b),
1104 };
1105
1106 return nir_select_from_ssa_def_array(b,
1107 ff_elems,
1108 ARRAY_SIZE(ff_elems),
1109 ff_op);
1110 }
1111
1112 static nir_def *
1113 lower_fs_intr(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
1114 {
1115 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1116 switch (intr->intrinsic) {
1117 case nir_intrinsic_load_front_face:
1118 return lower_front_face(b, intr);
1119
1120 default:
1121 break;
1122 }
1123
1124 UNREACHABLE("");
1125 }
1126
1127 bool pco_nir_lower_fs_intrinsics(nir_shader *shader)
1128 {
1129 assert(shader->info.stage == MESA_SHADER_FRAGMENT);
1130
1131 return nir_shader_lower_instructions(shader, is_fs_intr, lower_fs_intr, NULL);
1132 }
1133
1134 static bool
1135 lower_vs_intr(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *cb_data)
1136 {
1137 b->cursor = nir_before_instr(&intr->instr);
1138
1139 switch (intr->intrinsic) {
1140 /* First vs base vertex is handled in the PDS, so they're equivalent. */
1141 case nir_intrinsic_load_first_vertex:
1142 nir_def_replace(&intr->def, nir_load_base_vertex(b));
1143 nir_instr_free(&intr->instr);
1144 return true;
1145
1146 default:
1147 break;
1148 }
1149
1150 return false;
1151 }
1152
1153 bool pco_nir_lower_vs_intrinsics(nir_shader *shader)
1154 {
1155 assert(shader->info.stage == MESA_SHADER_VERTEX);
1156
1157 return nir_shader_intrinsics_pass(shader,
1158 lower_vs_intr,
1159 nir_metadata_control_flow,
1160 NULL);
1161 }
1162
1163 bool pco_nir_lower_clip_cull_vars(nir_shader *shader)
1164 {
1165 if (shader->info.internal)
1166 return false;
1167
1168 unsigned clip_cull_comps = shader->info.clip_distance_array_size +
1169 shader->info.cull_distance_array_size;
1170 if (!clip_cull_comps)
1171 return false;
1172
1173 /* Remove the old variables. */
1174 const gl_varying_slot clip_cull_locations[] = {
1175 VARYING_SLOT_CLIP_DIST0,
1176 VARYING_SLOT_CLIP_DIST1,
1177 };
1178
1179 nir_variable *var;
1180 for (unsigned u = 0; u < ARRAY_SIZE(clip_cull_locations); ++u) {
1181 gl_varying_slot location = clip_cull_locations[u];
1182 while ((var = nir_find_variable_with_location(shader,
1183 nir_var_shader_out,
1184 location))) {
1185 exec_node_remove(&var->node);
1186 }
1187 }
1188
1189 /* Create new variables. */
1190 nir_create_variable_with_location(shader,
1191 nir_var_shader_out,
1192 VARYING_SLOT_CLIP_DIST0,
1193 glsl_vec_type(MIN2(clip_cull_comps, 4)));
1194
1195 if (clip_cull_comps > 4) {
1196 nir_create_variable_with_location(shader,
1197 nir_var_shader_out,
1198 VARYING_SLOT_CLIP_DIST1,
1199 glsl_vec_type(clip_cull_comps - 4));
1200 }
1201
1202 nir_metadata_invalidate(shader);
1203
1204 return true;
1205 }
1206
1207 static bool
1208 clone_clip_cull_stores(nir_builder *b, nir_intrinsic_instr *intr, void *data)
1209 {
1210 if (intr->intrinsic != nir_intrinsic_store_deref)
1211 return false;
1212
1213 nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
1214 if (deref->deref_type != nir_deref_type_array)
1215 return false;
1216
1217 nir_variable *var = nir_deref_instr_get_variable(deref);
1218 if (var->data.location != VARYING_SLOT_CLIP_DIST0 &&
1219 var->data.location != VARYING_SLOT_CLIP_DIST1)
1220 return false;
1221
1222 b->cursor = nir_after_instr(&intr->instr);
1223
1224 unsigned var_index = var->data.location - VARYING_SLOT_CLIP_DIST0;
1225 nir_def *index =
1226 nir_iadd_imm(b, deref->arr.index.ssa, var->data.location_frac);
1227 index = nir_iadd_imm(b, index, var_index * 4);
1228
1229 nir_variable *clone_var = data;
1230 nir_store_array_var(b, clone_var, index, intr->src[1].ssa, 1);
1231
1232 return true;
1233 }
1234
1235 static bool is_clip_cull_load(const nir_instr *instr,
1236 UNUSED const void *cb_data)
1237 {
1238 if (instr->type != nir_instr_type_intrinsic)
1239 return false;
1240
1241 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1242 if (intr->intrinsic != nir_intrinsic_load_deref)
1243 return false;
1244
1245 nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
1246 if (deref->deref_type != nir_deref_type_array)
1247 return false;
1248
1249 nir_variable *var = nir_deref_instr_get_variable(deref);
1250
1251 return var->data.location == VARYING_SLOT_CLIP_DIST0 ||
1252 var->data.location == VARYING_SLOT_CLIP_DIST1;
1253 }
1254
1255 static nir_def *
1256 swap_clip_cull_load(nir_builder *b, nir_instr *instr, void *cb_data)
1257 {
1258 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1259 nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
1260 nir_variable *var = nir_deref_instr_get_variable(deref);
1261
1262 unsigned var_index = var->data.location - VARYING_SLOT_CLIP_DIST0;
1263 nir_def *index =
1264 nir_iadd_imm(b, deref->arr.index.ssa, var->data.location_frac);
1265 index = nir_iadd_imm(b, index, var_index * 4);
1266
1267 nir_variable *clone_var = cb_data;
1268 return nir_load_array_var(b, clone_var, index);
1269 }
1270
1271 bool pco_nir_link_clip_cull_vars(nir_shader *producer, nir_shader *consumer)
1272 {
1273 if (producer->info.stage != MESA_SHADER_VERTEX ||
1274 consumer->info.stage != MESA_SHADER_FRAGMENT) {
1275 return false;
1276 }
1277
1278 unsigned clip_cull_comps = consumer->info.clip_distance_array_size +
1279 consumer->info.cull_distance_array_size;
1280 /* Skip if clip/cull comps aren't actually consumed. */
1281 if (!clip_cull_comps)
1282 return false;
1283
1284 const glsl_type *clone_var_type =
1285 glsl_array_type(glsl_float_type(), clip_cull_comps, 0);
1286
1287 /* Find unused varying slot to use and create the variables. */
1288 gl_varying_slot clone_slot = VARYING_SLOT_VAR0;
1289 nir_foreach_shader_out_variable (var, producer) {
1290 clone_slot = MAX2(clone_slot, var->data.location + 1);
1291 }
1292 assert(clone_slot < VARYING_SLOT_MAX);
1293
1294 nir_variable *clone_var =
1295 nir_variable_create(producer, nir_var_shader_out, clone_var_type, NULL);
1296 clone_var->data.location = clone_slot;
1297
1298 nir_shader_intrinsics_pass(producer,
1299 clone_clip_cull_stores,
1300 nir_metadata_block_index | nir_metadata_dominance,
1301 clone_var);
1302
1303 clone_var =
1304 nir_variable_create(consumer, nir_var_shader_in, clone_var_type, NULL);
1305 clone_var->data.location = clone_slot;
1306
1307 nir_shader_lower_instructions(consumer,
1308 is_clip_cull_load,
1309 swap_clip_cull_load,
1310 clone_var);
1311
1312 return true;
1313 }
1314
1315 static nir_def *alu_iter(nir_builder *b,
1316 nir_def *coords,
1317 unsigned component,
1318 struct nir_io_semantics io_semantics)
1319 {
1320 nir_def *coeffs = nir_load_fs_coeffs_pco(b,
1321 .component = component,
1322 .io_semantics = io_semantics);
1323
1324 nir_def *result = nir_ffma(b,
1325 nir_channel(b, coeffs, 1),
1326 nir_channel(b, coords, 1),
1327 nir_channel(b, coeffs, 2));
1328 result =
1329 nir_ffma(b, nir_channel(b, coeffs, 0), nir_channel(b, coords, 0), result);
1330
1331 return result;
1332 }
1333
1334 static bool
1335 lower_sample_pos(nir_builder *b, nir_intrinsic_instr *intr, pco_fs_data *fs)
1336 {
1337 b->cursor = nir_before_instr(&intr->instr);
1338
1339 nir_def *msaa_samples = nir_bit_count(
1340 b,
1341 nir_u2u32(b, nir_alpha_to_coverage(b, nir_imm_float(b, 1.0f))));
1342
1343 nir_def *sample_id = intr->intrinsic == nir_intrinsic_load_sample_pos
1344 ? nir_load_sample_id(b)
1345 : intr->src[0].ssa;
1346
1347 nir_def *dword_index =
1348 nir_ishr_imm(b, nir_iadd(b, msaa_samples, sample_id), 2);
1349
1350 nir_def *packed_sample_location =
1351 nir_load_packed_sample_location_pco(b, dword_index);
1352 fs->uses.sample_locations = true;
1353
1354 nir_def *byte_index = nir_iand_imm(b, sample_id, 0b11);
1355
1356 packed_sample_location =
1357 nir_extract_u8(b, packed_sample_location, byte_index);
1358
1359 nir_def *sample_location =
1360 nir_vec2(b,
1361 nir_ubitfield_extract_imm(b, packed_sample_location, 0, 4),
1362 nir_ubitfield_extract_imm(b, packed_sample_location, 4, 4));
1363
1364 sample_location = nir_u2f32(b, sample_location);
1365 sample_location = nir_fdiv_imm(b, sample_location, 16.0f);
1366 sample_location = nir_bcsel(b,
1367 nir_ieq_imm(b, msaa_samples, 1),
1368 nir_imm_vec2(b, 0.5f, 0.5f),
1369 sample_location);
1370
1371 nir_def_replace(&intr->def, sample_location);
1372 nir_instr_free(&intr->instr);
1373
1374 return true;
1375 }
1376
1377 static bool
1378 lower_interp(nir_builder *b, nir_intrinsic_instr *intr, void *cb_data)
1379 {
1380 pco_fs_data *fs = cb_data;
1381 b->cursor = nir_before_instr(&intr->instr);
1382
1383 switch (intr->intrinsic) {
1384 case nir_intrinsic_load_sample_pos:
1385 case nir_intrinsic_load_sample_pos_from_id:
1386 return lower_sample_pos(b, intr, fs);
1387
1388 case nir_intrinsic_load_interpolated_input:
1389 break;
1390
1391 default:
1392 return false;
1393 }
1394
1395 nir_intrinsic_instr *bary = nir_src_as_intrinsic(intr->src[0]);
1396 assert(bary);
1397
1398 /* Skip cases that don't need handling. */
1399 if (bary->intrinsic != nir_intrinsic_load_barycentric_at_offset)
1400 return false;
1401
1402 assert(nir_src_as_uint(intr->src[1]) == 0);
1403
1404 nir_def *coords = nir_load_tile_coord_pco(b, 2);
1405 coords = nir_fadd(b, coords, bary->src[0].ssa);
1406
1407 enum glsl_interp_mode interp_mode = nir_intrinsic_interp_mode(bary);
1408 nir_def *rhw = alu_iter(b,
1409 coords,
1410 3,
1411 (struct nir_io_semantics){
1412 .location = VARYING_SLOT_POS,
1413 .num_slots = 1,
1414 });
1415
1416 nir_def *comps[4];
1417 for (unsigned u = 0; u < intr->def.num_components; ++u) {
1418 comps[u] = alu_iter(b, coords, u, nir_intrinsic_io_semantics(intr));
1419 if (interp_mode != INTERP_MODE_NOPERSPECTIVE)
1420 comps[u] = nir_fdiv(b, comps[u], rhw);
1421 }
1422
1423 nir_def *repl = nir_vec(b, comps, intr->def.num_components);
1424 nir_def_replace(&intr->def, repl);
1425 nir_instr_free(&intr->instr);
1426
1427 return true;
1428 }
1429
1430 bool pco_nir_lower_interpolation(nir_shader *shader, pco_fs_data *fs)
1431 {
1432 bool progress = false;
1433
1434 progress |= nir_opt_barycentric(shader, true);
1435 progress |= nir_shader_intrinsics_pass(shader,
1436 lower_interp,
1437 nir_metadata_control_flow,
1438 fs);
1439
1440 return progress;
1441 }
1442
1443 static bool lower_load_view_index_fs(struct nir_builder *b,
1444 nir_intrinsic_instr *intr,
1445 void *cb_data)
1446 {
1447 if (intr->intrinsic != nir_intrinsic_load_view_index)
1448 return false;
1449
1450 nir_variable *view_index_var = cb_data;
1451 b->cursor = nir_before_instr(&intr->instr);
1452 nir_def_replace(&intr->def, nir_load_var(b, view_index_var));
1453 nir_instr_free(&intr->instr);
1454
1455 return true;
1456 }
1457
1458 bool pco_nir_link_multiview(nir_shader *producer,
1459 nir_shader *consumer,
1460 pco_data *consumer_data)
1461 {
1462 if (producer->info.stage != MESA_SHADER_VERTEX ||
1463 consumer->info.stage != MESA_SHADER_FRAGMENT ||
1464 !consumer_data->common.multiview) {
1465 return false;
1466 }
1467
1468 /* Find unused varying slot for the view index. */
1469 gl_varying_slot view_index_slot = VARYING_SLOT_VAR0;
1470 nir_foreach_shader_out_variable (var, producer) {
1471 view_index_slot = MAX2(view_index_slot, var->data.location + 1);
1472 }
1473 assert(view_index_slot < VARYING_SLOT_MAX);
1474 consumer_data->fs.view_index_slot = view_index_slot;
1475
1476 /* Create output variable in the producer. */
1477 nir_variable *view_index_var = nir_variable_create(producer,
1478 nir_var_shader_out,
1479 glsl_uint_type(),
1480 "view_index");
1481 view_index_var->data.location = view_index_slot;
1482 view_index_var->data.interpolation = INTERP_MODE_FLAT;
1483 view_index_var->data.always_active_io = true;
1484
1485 /* Store view index in the producer. */
1486 nir_builder b = nir_builder_at(nir_after_block(
1487 nir_impl_last_block(nir_shader_get_entrypoint(producer))));
1488 nir_store_var(&b, view_index_var, nir_load_view_index(&b), 1);
1489
1490 /* Create input variable in the consumer. */
1491 view_index_var = nir_variable_create(consumer,
1492 nir_var_shader_in,
1493 glsl_uint_type(),
1494 "view_index");
1495 view_index_var->data.location = view_index_slot;
1496 view_index_var->data.interpolation = INTERP_MODE_FLAT;
1497 view_index_var->data.always_active_io = true;
1498
1499 /* Lower view index loads in the consumer. */
1500 nir_shader_intrinsics_pass(consumer,
1501 lower_load_view_index_fs,
1502 nir_metadata_all,
1503 view_index_var);
1504
1505 return true;
1506 }