Debian Codegraph @

src/imagination/vulkan/pvr_usc.c

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pvr_usc.c
9 *
10 * \brief USC internal shader generation.
11 */
12
13 #include "hwdef/rogue_hw_utils.h"
14 #include "nir/nir.h"
15 #include "nir/nir_builder.h"
16 #include "nir/nir_format_convert.h"
17 #include "nir/nir_conversion_builder.h"
18 #include "pco/pco.h"
19 #include "pco/pco_common.h"
20 #include "pco/pco_data.h"
21 #include "pco_uscgen_programs.h"
22 #include "pco/usclib/pco_usclib.h"
23 #include "pvr_common.h"
24 #include "pvr_formats.h"
25 #include "pvr_hw_pass.h"
26 #include "pvr_pass.h"
27 #include "pvr_sampler.h"
28 #include "pvr_usc.h"
29 #include "util/macros.h"
30
31 #define PVR_MAX_SAMPLE_COUNT 8
32
33 /**
34 * Common function to build a NIR shader and export the binary.
35 *
36 * \param ctx PCO context.
37 * \param nir NIR shader.
38 * \param data Shader data.
39 * \return The finalized PCO shader.
40 */
41 static pco_shader *build_shader(pco_ctx *ctx, nir_shader *nir, pco_data *data)
42 {
43 pco_preprocess_nir(ctx, nir);
44 pco_lower_nir(ctx, nir, data);
45 pco_postprocess_nir(ctx, nir, data);
46
47 pco_shader *shader = pco_trans_nir(ctx, nir, data, NULL);
48 ralloc_steal(shader, nir);
49 pco_process_ir(ctx, shader);
50 pco_encode_ir(ctx, shader);
51
52 return shader;
53 }
54
55 /**
56 * Generate a fragment passthrough shader that does a dummy store.
57 *
58 * \param ctx PCO context.
59 * \return The fragment passthrough shader.
60 */
61 nir_shader *pvr_usc_fs_pfo_passthrough_nir(pco_ctx *ctx)
62 {
63 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
64 pco_nir_options(),
65 "fs_pfo_passthrough");
66
67 /* Need to mark the shader as not internal for the compiler to
68 * emit the pfo code for the sample mask.
69 */
70 b.shader->info.internal = false;
71 nir_frag_store_pco(&b, nir_imm_int(&b, 0), .base = 0);
72 nir_jump(&b, nir_jump_return);
73
74 pco_preprocess_nir(ctx, b.shader);
75
76 return b.shader;
77 }
78
79 /**
80 * Generate an end-of-tile shader.
81 *
82 * \param ctx PCO context.
83 * \param props End of tile shader properties.
84 * \return The end-of-tile shader.
85 */
86 pco_shader *pvr_usc_eot(pco_ctx *ctx,
87 struct pvr_eot_props *props,
88 const struct pvr_device_info *dev_info)
89 {
90 nir_builder b =
91 nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
92 pco_nir_options(),
93 "eot%u.%s",
94 props->emit_count,
95 props->shared_words ? "sh" : "imm");
96
97 /* TODO: tile buffer support. */
98
99 nir_intrinsic_instr *last_emit = NULL;
100 for (unsigned u = 0; u < props->emit_count; ++u) {
101 if (u > 0)
102 nir_wop_pco(&b);
103
104 if (props->tile_buffer_addrs[u]) {
105 uint64_t tile_buffer_addr = props->tile_buffer_addrs[u];
106
107 unsigned data_size =
108 (PVR_GET_FEATURE_VALUE(dev_info, tile_size_x, 0U) *
109 PVR_GET_FEATURE_VALUE(dev_info, tile_size_y, 0U) *
110 props->num_output_regs) /
111 rogue_num_uscs_per_tile(dev_info);
112 assert(data_size);
113
114 assert(props->msaa_samples);
115 if (props->msaa_samples > 1) {
116 if (PVR_HAS_FEATURE(dev_info, pbe2_in_xe)) {
117 if (PVR_GET_FEATURE_VALUE(dev_info, isp_samples_per_pixel, 0U) ==
118 4) {
119 /* Scale the size according to the number of samples. */
120 data_size *= props->msaa_samples;
121 }
122
123 /* Else, the size is reduced proportionally so the
124 * total amount of data remains the same as for non-MSAA, i.e.
125 * data_size is unchanged.
126 */
127 } else {
128 /* The size is multiplied by the number of samples, but, for the
129 * 4x and 8x modes the size is reduced proportionally so the
130 * total amount of data remains the same as for 2x.
131 */
132 data_size *= 2;
133 }
134 }
135
136 /* We can burst up to 1024 dwords at a time. */
137 unsigned num_loads = DIV_ROUND_UP(data_size, 1024);
138 unsigned scale = rogue_usc_indexed_pixel_output_index_scale(dev_info);
139 for (unsigned l = 0; l < num_loads; ++l) {
140 unsigned offset = l * 1024;
141 unsigned idx_offset = offset / scale;
142 bool last_load = l == (num_loads - 1);
143 unsigned range = last_load ? data_size - offset : 1024;
144
145 nir_flush_tile_buffer_pco(
146 &b,
147 nir_imm_int(&b, tile_buffer_addr & 0xffffffff),
148 nir_imm_int(&b, tile_buffer_addr >> 32),
149 .base = idx_offset,
150 .range = range);
151
152 tile_buffer_addr += 1024 * sizeof(uint32_t);
153 }
154 }
155
156 nir_def *state0;
157 nir_def *state1;
158 if (props->shared_words) {
159 state0 = nir_load_preamble(&b, 1, 32, .base = props->state_regs[u]);
160 state1 =
161 nir_load_preamble(&b, 1, 32, .base = props->state_regs[u] + 1);
162 } else {
163 unsigned state_off = u * ROGUE_NUM_PBESTATE_STATE_WORDS;
164 state0 = nir_imm_int(&b, props->state_words[state_off]);
165 state1 = nir_imm_int(&b, props->state_words[state_off + 1]);
166 }
167
168 last_emit = nir_emitpix_pco(&b, state0, state1);
169 }
170
171 assert(last_emit);
172 nir_intrinsic_set_freep(last_emit, true);
173
174 /* Just return. */
175 nir_jump(&b, nir_jump_return);
176
177 pco_data data = {
178 .fs.uses.olchk_skip = true,
179 };
180 return build_shader(ctx, b.shader, &data);
181 }
182
183 /**
184 * Generate a transfer queue shader.
185 *
186 * \param ctx PCO context.
187 * \param props Transfer queue shader properties.
188 * \return The transfer queue shader.
189 */
190 pco_shader *pvr_usc_tq(pco_ctx *ctx, struct pvr_tq_props *props)
191 {
192 UNREACHABLE("finishme: pvr_usc_tq");
193 }
194
195 static bool needs_packing(enum pvr_transfer_pbe_pixel_src format)
196 {
197 switch (format) {
198 case PVR_TRANSFER_PBE_PIXEL_SRC_RAW64:
199 case PVR_TRANSFER_PBE_PIXEL_SRC_F32X2:
200 case PVR_TRANSFER_PBE_PIXEL_SRC_MOV_BY45:
201 case PVR_TRANSFER_PBE_PIXEL_SRC_D32S8:
202 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D24_D32:
203 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D32U_D32F:
204 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D32_D24S8:
205 case PVR_TRANSFER_PBE_PIXEL_SRC_RAW32:
206 case PVR_TRANSFER_PBE_PIXEL_SRC_F32:
207 case PVR_TRANSFER_PBE_PIXEL_SRC_SWAP_LMSB:
208 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_S8D24_D24S8:
209 case PVR_TRANSFER_PBE_PIXEL_SRC_D24S8:
210 case PVR_TRANSFER_PBE_PIXEL_SRC_S8D24:
211 case PVR_TRANSFER_PBE_PIXEL_SRC_RAW128:
212 case PVR_TRANSFER_PBE_PIXEL_SRC_F32X4:
213 return false;
214 default:
215 break;
216 }
217 return true;
218 }
219
220 static bool needs_conversion(enum pvr_transfer_pbe_pixel_src format)
221 {
222 switch (format) {
223 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D24_D32:
224 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D32U_D32F:
225 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D32_D24S8:
226 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32_D24S8:
227 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32U_D24S8:
228 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D32S8:
229 case PVR_TRANSFER_PBE_PIXEL_SRC_SWAP_LMSB:
230 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_S8D24_D24S8:
231 case PVR_TRANSFER_PBE_PIXEL_SRC_MOV_BY45:
232 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D32S8_D32S8:
233 case PVR_TRANSFER_PBE_PIXEL_SRC_Y_UV_INTERLEAVED:
234 case PVR_TRANSFER_PBE_PIXEL_SRC_YVU_PACKED:
235 case PVR_TRANSFER_PBE_PIXEL_SRC_Y_U_V:
236 case PVR_TRANSFER_PBE_PIXEL_SRC_YUV_PACKED:
237 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_S8_D32S8:
238 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_S8_D24S8:
239 return true;
240 default:
241 break;
242 }
243 return false;
244 }
245
246 static void
247 int_format_signs(enum pvr_transfer_pbe_pixel_src format, bool *src, bool *dst)
248 {
249 switch (format) {
250 case PVR_TRANSFER_PBE_PIXEL_SRC_UU8888:
251 case PVR_TRANSFER_PBE_PIXEL_SRC_US8888:
252 case PVR_TRANSFER_PBE_PIXEL_SRC_UU16U16:
253 case PVR_TRANSFER_PBE_PIXEL_SRC_US16S16:
254 case PVR_TRANSFER_PBE_PIXEL_SRC_US32S32:
255 case PVR_TRANSFER_PBE_PIXEL_SRC_U4XS32:
256 case PVR_TRANSFER_PBE_PIXEL_SRC_UU1010102:
257 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_UU1010102:
258 *src = false;
259 break;
260 case PVR_TRANSFER_PBE_PIXEL_SRC_SU8888:
261 case PVR_TRANSFER_PBE_PIXEL_SRC_SS8888:
262 case PVR_TRANSFER_PBE_PIXEL_SRC_SU16U16:
263 case PVR_TRANSFER_PBE_PIXEL_SRC_SS16S16:
264 case PVR_TRANSFER_PBE_PIXEL_SRC_SU32U32:
265 case PVR_TRANSFER_PBE_PIXEL_SRC_S4XU32:
266 case PVR_TRANSFER_PBE_PIXEL_SRC_SU1010102:
267 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_SU1010102:
268 *src = true;
269 break;
270 default:
271 UNREACHABLE("Invalid format");
272 }
273
274 switch (format) {
275 case PVR_TRANSFER_PBE_PIXEL_SRC_UU8888:
276 case PVR_TRANSFER_PBE_PIXEL_SRC_SU8888:
277 case PVR_TRANSFER_PBE_PIXEL_SRC_UU16U16:
278 case PVR_TRANSFER_PBE_PIXEL_SRC_SU16U16:
279 case PVR_TRANSFER_PBE_PIXEL_SRC_SU32U32:
280 case PVR_TRANSFER_PBE_PIXEL_SRC_S4XU32:
281 case PVR_TRANSFER_PBE_PIXEL_SRC_UU1010102:
282 case PVR_TRANSFER_PBE_PIXEL_SRC_SU1010102:
283 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_UU1010102:
284 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_SU1010102:
285 *dst = false;
286 break;
287 case PVR_TRANSFER_PBE_PIXEL_SRC_US8888:
288 case PVR_TRANSFER_PBE_PIXEL_SRC_SS8888:
289 case PVR_TRANSFER_PBE_PIXEL_SRC_US16S16:
290 case PVR_TRANSFER_PBE_PIXEL_SRC_SS16S16:
291 case PVR_TRANSFER_PBE_PIXEL_SRC_US32S32:
292 case PVR_TRANSFER_PBE_PIXEL_SRC_U4XS32:
293 *dst = true;
294 break;
295 default:
296 UNREACHABLE("Invalid format");
297 }
298 }
299
300 static nir_def *
301 picked_component(nir_builder *b,
302 nir_def *src,
303 unsigned *next_sh,
304 struct pvr_tq_frag_sh_reg_layout *sh_reg_layout)
305 {
306 unsigned base_sh = sh_reg_layout->dynamic_consts.offset;
307 nir_variable *pos = nir_get_variable_with_location(b->shader,
308 nir_var_shader_in,
309 VARYING_SLOT_POS,
310 glsl_vec4_type());
311 nir_def *coord_x = nir_f2i32(b, nir_channel(b, nir_load_var(b, pos), 0));
312 nir_def *mask = nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh);
313 nir_def *offset =
314 nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh + 1);
315 nir_def *comp_idx = nir_iand(b, nir_isub(b, coord_x, offset), mask);
316 nir_def *shift_val = nir_imul_imm(b, comp_idx, 8);
317
318 *next_sh += 2;
319 return nir_ushr(b, src, shift_val);
320 }
321
322 static nir_def *pack_int_value(nir_builder *b,
323 unsigned *next_sh,
324 struct pvr_tq_frag_sh_reg_layout *sh_reg_layout,
325 bool pick_component,
326 nir_def *src,
327 enum pvr_transfer_pbe_pixel_src format)
328 {
329 unsigned src_num_components = 4;
330 const unsigned bits_8[] = { 8, 8, 8, 8 };
331 const unsigned bits_10[] = { 10, 10, 10, 2 };
332 const unsigned bits_16[] = { 16, 16, 16, 16 };
333 const unsigned bits_32[] = { 32, 32, 32, 32 };
334 const unsigned *bits;
335 bool src_signed, dst_signed;
336 int_format_signs(format, &src_signed, &dst_signed);
337
338 switch (format) {
339 case PVR_TRANSFER_PBE_PIXEL_SRC_UU8888:
340 case PVR_TRANSFER_PBE_PIXEL_SRC_US8888:
341 case PVR_TRANSFER_PBE_PIXEL_SRC_SU8888:
342 case PVR_TRANSFER_PBE_PIXEL_SRC_SS8888:
343 bits = bits_8;
344 break;
345 case PVR_TRANSFER_PBE_PIXEL_SRC_UU16U16:
346 case PVR_TRANSFER_PBE_PIXEL_SRC_US16S16:
347 case PVR_TRANSFER_PBE_PIXEL_SRC_SU16U16:
348 case PVR_TRANSFER_PBE_PIXEL_SRC_SS16S16:
349 bits = bits_16;
350 break;
351 case PVR_TRANSFER_PBE_PIXEL_SRC_SU32U32:
352 case PVR_TRANSFER_PBE_PIXEL_SRC_S4XU32:
353 case PVR_TRANSFER_PBE_PIXEL_SRC_US32S32:
354 case PVR_TRANSFER_PBE_PIXEL_SRC_U4XS32:
355 bits = bits_32;
356 break;
357 case PVR_TRANSFER_PBE_PIXEL_SRC_UU1010102:
358 case PVR_TRANSFER_PBE_PIXEL_SRC_SU1010102:
359 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_UU1010102:
360 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_SU1010102:
361 bits = bits_10;
362 break;
363 default:
364 UNREACHABLE("Invalid format");
365 }
366
367 if (format == PVR_TRANSFER_PBE_PIXEL_SRC_SU32U32 ||
368 format == PVR_TRANSFER_PBE_PIXEL_SRC_US32S32) {
369 src_num_components = 2;
370 }
371
372 if (format == PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_UU1010102 ||
373 format == PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_SU1010102) {
374 unsigned swiz[] = { 2, 1, 0, 3 };
375 src = nir_swizzle(b, src, swiz, 4);
376 }
377
378 if (src_signed != dst_signed) {
379 src = nir_convert_with_rounding(b,
380 src,
381 src_signed ? nir_type_int : nir_type_uint,
382 dst_signed ? nir_type_int32
383 : nir_type_uint32,
384 nir_rounding_mode_undef,
385 true);
386 }
387
388 if (dst_signed)
389 src = nir_format_clamp_sint(b, src, bits);
390 else
391 src = nir_format_clamp_uint(b, src, bits);
392 if ((bits[0] < 32) && dst_signed)
393 src = nir_format_mask_uvec(b, src, bits);
394
395 if (bits != bits_16) {
396 src = nir_format_pack_uint(b, src, bits, src_num_components);
397 } else {
398 src =
399 nir_vec2(b,
400 nir_format_pack_uint(b, nir_channels(b, src, 0x3), bits, 2),
401 nir_format_pack_uint(b, nir_channels(b, src, 0xc), bits, 2));
402 }
403
404 if (!pick_component)
405 return src;
406
407 return picked_component(b, src, next_sh, sh_reg_layout);
408 }
409
410 static nir_def *merge_depth_stencil(nir_builder *b,
411 nir_def *src,
412 enum pipe_format format,
413 bool merge_depth,
414 unsigned load_idx)
415 {
416 nir_def *dst;
417 unsigned mask;
418
419 assert(format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ||
420 format == PIPE_FORMAT_Z24_UNORM_S8_UINT);
421
422 dst = nir_load_output(b,
423 format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ? 2 : 1,
424 32,
425 nir_imm_int(b, 0),
426 .base = 0,
427 .dest_type = nir_type_invalid | 32,
428 .io_semantics.location = FRAG_RESULT_DATA0 + load_idx,
429 .io_semantics.num_slots = 1,
430 .io_semantics.fb_fetch_output = true);
431
432 b->shader->info.outputs_read |= BITFIELD64_BIT(FRAG_RESULT_DATA0 + load_idx);
433 b->shader->info.fs.uses_fbfetch_output = true;
434
435 if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
436 if (merge_depth)
437 return nir_vec2(b, nir_channel(b, src, 0), nir_channel(b, dst, 1));
438 else
439 return nir_vec2(b, nir_channel(b, dst, 0), nir_channel(b, src, 1));
440 }
441
442 if (merge_depth)
443 mask = BITFIELD_MASK(24);
444 else
445 mask = BITFIELD_RANGE(24, 8);
446
447 return nir_ior(b, nir_iand_imm(b, src, mask), nir_iand_imm(b, dst, ~mask));
448 }
449
450 static nir_def *
451 pvr_uscgen_tq_frag_pack(nir_builder *b,
452 unsigned *next_sh,
453 struct pvr_tq_frag_sh_reg_layout *sh_reg_layout,
454 bool pick_component,
455 nir_def *src,
456 enum pvr_transfer_pbe_pixel_src format,
457 unsigned load_idx)
458 {
459 if (!needs_packing(format))
460 return src;
461
462 /* Integer packing */
463 switch (format) {
464 case PVR_TRANSFER_PBE_PIXEL_SRC_UU8888:
465 case PVR_TRANSFER_PBE_PIXEL_SRC_US8888:
466 case PVR_TRANSFER_PBE_PIXEL_SRC_UU16U16:
467 case PVR_TRANSFER_PBE_PIXEL_SRC_US16S16:
468 case PVR_TRANSFER_PBE_PIXEL_SRC_SU8888:
469 case PVR_TRANSFER_PBE_PIXEL_SRC_SS8888:
470 case PVR_TRANSFER_PBE_PIXEL_SRC_SU16U16:
471 case PVR_TRANSFER_PBE_PIXEL_SRC_SS16S16:
472 case PVR_TRANSFER_PBE_PIXEL_SRC_SU32U32:
473 case PVR_TRANSFER_PBE_PIXEL_SRC_S4XU32:
474 case PVR_TRANSFER_PBE_PIXEL_SRC_US32S32:
475 case PVR_TRANSFER_PBE_PIXEL_SRC_U4XS32:
476 case PVR_TRANSFER_PBE_PIXEL_SRC_UU1010102:
477 case PVR_TRANSFER_PBE_PIXEL_SRC_SU1010102:
478 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_UU1010102:
479 case PVR_TRANSFER_PBE_PIXEL_SRC_RBSWAP_SU1010102:
480 return pack_int_value(b,
481 next_sh,
482 sh_reg_layout,
483 pick_component,
484 src,
485 format);
486
487 case PVR_TRANSFER_PBE_PIXEL_SRC_F16F16:
488 return nir_vec2(b,
489 nir_pack_half_2x16(b, nir_channels(b, src, 0x3)),
490 nir_pack_half_2x16(b, nir_channels(b, src, 0xc)));
491 case PVR_TRANSFER_PBE_PIXEL_SRC_U16NORM:
492 return nir_vec2(b,
493 nir_pack_unorm_2x16(b, nir_channels(b, src, 0x3)),
494 nir_pack_unorm_2x16(b, nir_channels(b, src, 0xc)));
495 case PVR_TRANSFER_PBE_PIXEL_SRC_S16NORM:
496 return nir_vec2(b,
497 nir_pack_snorm_2x16(b, nir_channels(b, src, 0x3)),
498 nir_pack_snorm_2x16(b, nir_channels(b, src, 0xc)));
499 case PVR_TRANSFER_PBE_PIXEL_SRC_F16_U8:
500 return nir_pack_unorm_4x8(b, src);
501
502 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_S8_D32S8:
503 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D32S8:
504 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D32S8_D32S8:
505 return merge_depth_stencil(b,
506 src,
507 PIPE_FORMAT_Z32_FLOAT_S8X24_UINT,
508 false,
509 load_idx);
510
511 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32S8_D32S8:
512 return merge_depth_stencil(b,
513 src,
514 PIPE_FORMAT_Z32_FLOAT_S8X24_UINT,
515 true,
516 load_idx);
517
518 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_S8_D24S8:
519 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D24S8:
520 return merge_depth_stencil(b,
521 src,
522 PIPE_FORMAT_Z24_UNORM_S8_UINT,
523 false,
524 load_idx);
525
526 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D24S8_D24S8:
527 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32_D24S8:
528 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32U_D24S8:
529 return merge_depth_stencil(b,
530 src,
531 PIPE_FORMAT_Z24_UNORM_S8_UINT,
532 true,
533 load_idx);
534 default:
535 UNREACHABLE("Unimplemented pvr_transfer_pbe_pixel_src");
536 }
537 }
538
539 static bool uses_int_resolve(enum pvr_transfer_pbe_pixel_src format)
540 {
541 switch (format) {
542 case PVR_TRANSFER_PBE_PIXEL_SRC_F32:
543 case PVR_TRANSFER_PBE_PIXEL_SRC_F16F16:
544 case PVR_TRANSFER_PBE_PIXEL_SRC_F16_U8:
545 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32S8_D32S8:
546 return false;
547 case PVR_TRANSFER_PBE_PIXEL_SRC_RAW32:
548 case PVR_TRANSFER_PBE_PIXEL_SRC_RAW64:
549 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D24S8:
550 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D24S8_D24S8:
551 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32U_D24S8:
552 case PVR_TRANSFER_PBE_PIXEL_SRC_SWAP_LMSB:
553 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D32S8_D32S8:
554 return true;
555 default:
556 UNREACHABLE("Unsupported pvr_transfer_pbe_pixel_src");
557 }
558 return false;
559 }
560
561 static void prepare_samples_for_resolve(nir_builder *b,
562 nir_def **samples,
563 unsigned num_samples,
564 enum pvr_transfer_pbe_pixel_src format,
565 enum pvr_resolve_op resolve_op)
566 {
567 unsigned num_components;
568
569 if (resolve_op == PVR_RESOLVE_MIN || resolve_op == PVR_RESOLVE_MAX) {
570 if (format != PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D24S8_D24S8)
571 return;
572
573 /* Mask out the stencil component since it is in the significant bits */
574 for (unsigned i = 0; i < num_samples; i++)
575 samples[i] = nir_iand_imm(b, samples[i], BITFIELD_MASK(24));
576
577 return;
578 }
579
580 assert(resolve_op == PVR_RESOLVE_BLEND);
581
582 switch (format) {
583 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D24S8:
584 case PVR_TRANSFER_PBE_PIXEL_SRC_SWAP_LMSB:
585 /* Mask out depth and convert to f32 */
586 for (unsigned i = 0; i < num_samples; i++) {
587 samples[i] = nir_ushr_imm(b, samples[i], 24);
588 samples[i] = nir_u2f32(b, nir_channel(b, samples[i], 0));
589 }
590 return;
591
592 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D24S8_D24S8:
593 /* Mask out stencil and convert to f32 */
594 for (unsigned i = 0; i < num_samples; i++) {
595 samples[i] = nir_iand_imm(b, samples[i], ~BITFIELD_RANGE(24, 8));
596 samples[i] = nir_u2f32(b, nir_channel(b, samples[i], 0));
597 }
598 return;
599
600 case PVR_TRANSFER_PBE_PIXEL_SRC_F32:
601 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32S8_D32S8:
602 num_components = 1;
603 break;
604 case PVR_TRANSFER_PBE_PIXEL_SRC_F32X2:
605 num_components = 2;
606 break;
607 default:
608 assert(pvr_pbe_pixel_is_norm(format));
609 num_components = 4;
610 break;
611 }
612
613 for (unsigned i = 0; i < num_samples; i++)
614 samples[i] = nir_trim_vector(b, samples[i], num_components);
615 }
616
617 static nir_def *post_process_resolve(nir_builder *b,
618 nir_def *src,
619 enum pvr_transfer_pbe_pixel_src format,
620 enum pvr_resolve_op resolve_op)
621 {
622 unsigned bits;
623
624 if (resolve_op != PVR_RESOLVE_BLEND)
625 return src;
626
627 switch (format) {
628 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D24S8:
629 case PVR_TRANSFER_PBE_PIXEL_SRC_SWAP_LMSB:
630 /* Convert back to unorm and shift back to correct place */
631 bits = 8;
632 assert(src->num_components == 1);
633 src = nir_format_float_to_unorm(b, src, &bits);
634 return nir_ishl_imm(b, src, 24);
635
636 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D24S8_D24S8:
637 /* Convert back to unorm */
638 bits = 24;
639 assert(src->num_components == 1);
640 return nir_format_float_to_unorm(b, src, &bits);
641
642 default:
643 break;
644 }
645
646 return src;
647 }
648
649 static nir_def *resolve_samples(nir_builder *b,
650 nir_def **samples,
651 unsigned num_samples,
652 enum pvr_transfer_pbe_pixel_src format,
653 enum pvr_resolve_op resolve_op)
654 {
655 nir_def *accum = NULL;
656 nir_def *coeff = NULL;
657 nir_op op;
658
659 switch (resolve_op) {
660 case PVR_RESOLVE_BLEND:
661 op = nir_op_ffma;
662 coeff = nir_imm_float(b, 1.0 / num_samples);
663 break;
664
665 case PVR_RESOLVE_MIN:
666 op = uses_int_resolve(format) ? nir_op_imin : nir_op_fmin;
667 break;
668
669 case PVR_RESOLVE_MAX:
670 op = uses_int_resolve(format) ? nir_op_imax : nir_op_fmax;
671 break;
672
673 default:
674 UNREACHABLE("Unsupported pvr_transfer_pbe_pixel_src");
675 }
676
677 prepare_samples_for_resolve(b, samples, num_samples, format, resolve_op);
678
679 if (resolve_op == PVR_RESOLVE_BLEND)
680 accum = nir_fmul(b, samples[0], coeff);
681 else
682 accum = samples[0];
683
684 for (unsigned i = 1; i < num_samples; i++) {
685 if (resolve_op == PVR_RESOLVE_BLEND)
686 accum = nir_ffma(b, samples[i], coeff, accum);
687 else
688 accum = nir_build_alu2(b, op, samples[i], accum);
689 }
690
691 return post_process_resolve(b, accum, format, resolve_op);
692 }
693
694 static nir_def *pvr_uscgen_tq_frag_conv(nir_builder *b,
695 nir_def *src,
696 enum pvr_transfer_pbe_pixel_src format)
697 {
698 unsigned bits;
699 switch (format) {
700 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D24_D32:
701 bits = 32;
702 return nir_format_unorm_to_float(
703 b,
704 nir_iand_imm(b, nir_channel(b, src, 0), BITFIELD_MASK(24)),
705 &bits);
706
707 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D32U_D32F:
708 bits = 32;
709 return nir_format_unorm_to_float(b, nir_channel(b, src, 0), &bits);
710
711 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_D32_D24S8:
712 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32_D24S8:
713 bits = 24;
714 return nir_format_float_to_unorm(b, nir_channel(b, src, 0), &bits);
715
716 case PVR_TRANSFER_PBE_PIXEL_SRC_DMRG_D32U_D24S8:
717 return nir_ushr_imm(b, nir_channel(b, src, 0), 8);
718
719 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D24S8_D32S8:
720 return nir_vec2(b,
721 nir_undef(b, 1, 32),
722 nir_ushr_imm(b, nir_channel(b, src, 0), 24));
723
724 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_D32S8_D32S8:
725 return nir_vec2(b, nir_undef(b, 1, 32), nir_channel(b, src, 1));
726
727 case PVR_TRANSFER_PBE_PIXEL_SRC_SWAP_LMSB:
728 return nir_ushr_imm(b, nir_channel(b, src, 0), 24);
729
730 case PVR_TRANSFER_PBE_PIXEL_SRC_CONV_S8D24_D24S8:
731 src = nir_channel(b, src, 0);
732 return nir_mask_shift_or(b,
733 nir_ushr_imm(b, src, 24),
734 src,
735 BITFIELD_MASK(24),
736 8);
737
738 case PVR_TRANSFER_PBE_PIXEL_SRC_MOV_BY45:
739 return nir_vec2(b,
740 nir_undef(b, 1, 32),
741 nir_ushr_imm(b, nir_channel(b, src, 0), 24));
742
743 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_S8_D32S8:
744 return nir_vec2(b, nir_undef(b, 1, 32), nir_channel(b, src, 0));
745
746 case PVR_TRANSFER_PBE_PIXEL_SRC_SMRG_S8_D24S8:
747 return nir_ishl_imm(b, nir_channel(b, src, 0), 24);
748
749 default:
750 assert(!needs_conversion(format));
751 }
752
753 return src;
754 }
755
756 static nir_def *
757 pvr_uscgen_tq_frag_load(nir_builder *b,
758 uint32_t load_idx,
759 nir_def *coords,
760 const struct pvr_tq_shader_properties *shader_props,
761 struct pvr_tq_frag_sh_reg_layout *sh_reg_layout)
762 {
763 const struct pvr_tq_layer_properties *layer_props =
764 &shader_props->layer_props;
765
766 const unsigned num_samples = (shader_props->full_rate || !layer_props->msaa)
767 ? 1
768 : layer_props->sample_count;
769
770 nir_def *samples[PVR_MAX_SAMPLE_COUNT];
771
772 for (unsigned sample_idx = 0; sample_idx < num_samples; sample_idx++) {
773 assert(load_idx < sh_reg_layout->combined_image_samplers.count);
774
775 nir_def *tex_state = nir_load_preamble(
776 b,
777 4,
778 32,
779 .base =
780 sh_reg_layout->combined_image_samplers.offsets[load_idx].image);
781
782 nir_def *smp_state = nir_load_preamble(
783 b,
784 4,
785 32,
786 .base =
787 sh_reg_layout->combined_image_samplers.offsets[load_idx].sampler);
788
789 pco_smp_params params = {
790 .tex_state = tex_state,
791 .smp_state = smp_state,
792
793 .dest_type = pvr_pbe_pixel_is_norm(layer_props->pbe_format)
794 ? nir_type_float32
795 : nir_type_uint32,
796
797 .nncoords = shader_props->layer_props.linear ||
798 !shader_props->iterated,
799 .coords = coords,
800 };
801
802 if (layer_props->msaa) {
803 if (shader_props->full_rate) {
804 params.ms_index = nir_load_sample_id(b);
805 b->shader->info.fs.uses_sample_shading = true;
806 } else if (layer_props->resolve_op >= PVR_RESOLVE_SAMPLE0) {
807 params.ms_index =
808 nir_imm_int(b, layer_props->resolve_op - PVR_RESOLVE_SAMPLE0);
809 } else {
810 params.ms_index = nir_imm_int(b, sample_idx);
811 }
812 }
813
814 params.sampler_dim = GLSL_SAMPLER_DIM_2D;
815 if (layer_props->msaa)
816 params.sampler_dim = GLSL_SAMPLER_DIM_MS;
817 else if (layer_props->sample)
818 params.sampler_dim = GLSL_SAMPLER_DIM_3D;
819
820 nir_intrinsic_instr *smp = pco_emit_nir_smp(b, &params);
821 samples[sample_idx] = &smp->def;
822 }
823
824 if (num_samples == 1)
825 return samples[0];
826
827 return resolve_samples(b,
828 samples,
829 num_samples,
830 layer_props->pbe_format,
831 layer_props->resolve_op);
832 }
833
834 static nir_def *
835 pvr_uscgen_tq_frag_coords(nir_builder *b,
836 unsigned *next_sh,
837 const struct pvr_tq_shader_properties *shader_props,
838 struct pvr_tq_frag_sh_reg_layout *sh_reg_layout)
839 {
840 const struct pvr_tq_layer_properties *layer_props =
841 &shader_props->layer_props;
842 unsigned base_sh = sh_reg_layout->dynamic_consts.offset;
843 bool varying = shader_props->iterated;
844 unsigned location = varying ? VARYING_SLOT_VAR0 : VARYING_SLOT_POS;
845 unsigned pos_chans = varying ? (layer_props->sample ? 3 : 2) : 4;
846
847 const struct glsl_type *var_type = glsl_vec_type(pos_chans);
848 nir_variable *pos = nir_get_variable_with_location(b->shader,
849 nir_var_shader_in,
850 location,
851 var_type);
852 nir_def *coords_var = nir_load_var(b, pos);
853 nir_def *coords = nir_channels(b, coords_var, nir_component_mask(2));
854
855 assert(layer_props->layer_floats != PVR_INT_COORD_SET_FLOATS_6);
856 if (!varying && layer_props->layer_floats == PVR_INT_COORD_SET_FLOATS_4) {
857 /* coords.xy = coords.xy * (sh[0], sh[2]) + (sh[1], s[3]) */
858 nir_def *mult =
859 nir_vec2(b,
860 nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh),
861 nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh + 2));
862 nir_def *add =
863 nir_vec2(b,
864 nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh + 1),
865 nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh + 3));
866 coords = nir_fmad(b, coords, mult, add);
867 *next_sh += 4;
868 }
869
870 /* 3D texture, the depth comes from shared regs, or is iterated */
871 if (layer_props->sample) {
872 nir_def *depth =
873 varying ? nir_channel(b, coords_var, 2)
874 : nir_load_preamble(b, 1, 32, .base = *next_sh + base_sh);
875
876 coords = nir_pad_vector(b, coords, 3);
877 coords = nir_vector_insert_imm(b, coords, depth, 2);
878 (*next_sh)++;
879 }
880
881 return coords;
882 }
883
884 pco_shader *pvr_uscgen_tq(pco_ctx *ctx,
885 const struct pvr_tq_shader_properties *shader_props,
886 struct pvr_tq_frag_sh_reg_layout *sh_reg_layout)
887 {
888 const struct pvr_tq_layer_properties *layer_props =
889 &shader_props->layer_props;
890 unsigned next_sh = 0;
891
892 unsigned pixel_size = pvr_pbe_pixel_size(layer_props->pbe_format);
893
894 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
895 pco_nir_options(),
896 "TQ");
897
898 assert(layer_props->layer_floats != PVR_INT_COORD_SET_FLOATS_6);
899 assert(layer_props->byte_unwind == 0);
900 assert(layer_props->linear == false);
901
902 assert(pvr_pbe_pixel_num_loads(layer_props->pbe_format) == 1);
903
904 pco_data data = { 0 };
905
906 switch (pixel_size) {
907 case 1:
908 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32_UINT;
909 break;
910
911 case 2:
912 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32G32_UINT;
913 break;
914
915 case 3:
916 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32G32B32_UINT;
917 break;
918
919 case 4:
920 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32G32B32A32_UINT;
921 break;
922
923 default:
924 UNREACHABLE("");
925 }
926
927 data.fs.outputs[FRAG_RESULT_DATA0] = (pco_range){
928 .start = 0,
929 .count = pixel_size,
930 };
931
932 nir_def *loaded_data;
933 nir_def *coords =
934 pvr_uscgen_tq_frag_coords(&b, &next_sh, shader_props, sh_reg_layout);
935
936 assert(!layer_props->linear);
937
938 loaded_data =
939 pvr_uscgen_tq_frag_load(&b, 0, coords, shader_props, sh_reg_layout);
940
941 loaded_data =
942 pvr_uscgen_tq_frag_conv(&b, loaded_data, layer_props->pbe_format);
943
944 loaded_data = pvr_uscgen_tq_frag_pack(&b,
945 &next_sh,
946 sh_reg_layout,
947 shader_props->pick_component,
948 loaded_data,
949 layer_props->pbe_format,
950 0);
951
952 nir_store_output(&b,
953 nir_resize_vector(&b, loaded_data, pixel_size),
954 nir_imm_int(&b, 0),
955 .base = 0,
956 .src_type = nir_type_invalid | 32,
957 .write_mask = BITFIELD_MASK(pixel_size),
958 .io_semantics.location = FRAG_RESULT_DATA0,
959 .io_semantics.num_slots = 1);
960
961 b.shader->info.outputs_written |= BITFIELD64_BIT(FRAG_RESULT_DATA0);
962
963 nir_variable *pos = nir_find_variable_with_location(b.shader,
964 nir_var_shader_in,
965 VARYING_SLOT_POS);
966 if (pos)
967 pos->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
968
969 nir_variable *var0 = nir_find_variable_with_location(b.shader,
970 nir_var_shader_in,
971 VARYING_SLOT_VAR0);
972 if (var0) {
973 var0->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
974 /* TODO: port and use allocate_var from pvr_pipeline.c */
975 data.fs.varyings[VARYING_SLOT_VAR0] = (pco_range){
976 .start = 0,
977 .count = glsl_count_dword_slots(var0->type, false)
978
979 * ROGUE_USC_COEFFICIENT_SET_SIZE,
980 };
981 }
982
983 nir_create_variable_with_location(b.shader,
984 nir_var_shader_out,
985 FRAG_RESULT_DATA0,
986 glsl_uvec_type(pixel_size));
987
988 sh_reg_layout->dynamic_consts.count = next_sh;
989 sh_reg_layout->driver_total += sh_reg_layout->dynamic_consts.count;
990 sh_reg_layout->compiler_out_total = 0;
991 sh_reg_layout->compiler_out.usc_constants.count = 0;
992
993 nir_jump(&b, nir_jump_return);
994
995 return build_shader(ctx, b.shader, &data);
996 }
997
998 static inline VkFormat pvr_uscgen_format_for_accum(VkFormat vk_format)
999 {
1000 if (!vk_format_has_depth(vk_format))
1001 return vk_format;
1002
1003 switch (vk_format) {
1004 case VK_FORMAT_D16_UNORM:
1005 case VK_FORMAT_X8_D24_UNORM_PACK32:
1006 case VK_FORMAT_D32_SFLOAT:
1007 case VK_FORMAT_D24_UNORM_S8_UINT:
1008 return VK_FORMAT_R32_SFLOAT;
1009
1010 case VK_FORMAT_D32_SFLOAT_S8_UINT:
1011 return VK_FORMAT_R32G32_SFLOAT;
1012
1013 default:
1014 break;
1015 }
1016
1017 UNREACHABLE("");
1018 }
1019
1020 pco_shader *pvr_uscgen_loadop(pco_ctx *ctx, struct pvr_load_op *load_op)
1021 {
1022 unsigned rt_mask = load_op->clears_loads_state.rt_clear_mask |
1023 load_op->clears_loads_state.rt_load_mask;
1024 bool depth_to_reg = load_op->clears_loads_state.depth_clear_to_reg !=
1025 PVR_NO_DEPTH_CLEAR_TO_REG;
1026 const struct usc_mrt_setup *mrt_setup =
1027 load_op->clears_loads_state.mrt_setup;
1028
1029 pco_data data = { 0 };
1030 bool has_non_tile_buffer_stores = false;
1031
1032 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
1033 pco_nir_options(),
1034 "loadop");
1035
1036 u_foreach_bit (rt_idx, rt_mask) {
1037 bool is_clear = BITFIELD_BIT(rt_idx) &
1038 load_op->clears_loads_state.rt_clear_mask;
1039
1040 VkFormat vk_format = pvr_uscgen_format_for_accum(
1041 load_op->clears_loads_state.dest_vk_format[rt_idx]);
1042 unsigned accum_size_dwords =
1043 DIV_ROUND_UP(pvr_get_pbe_accum_format_size_in_bytes(vk_format),
1044 sizeof(uint32_t));
1045
1046 const glsl_type *type;
1047 if (is_clear) {
1048 type = glsl_vec_type(accum_size_dwords);
1049
1050 if (vk_format_is_int(vk_format))
1051 type = glsl_ivec_type(accum_size_dwords);
1052 else if (vk_format_is_uint(vk_format))
1053 type = glsl_uvec_type(accum_size_dwords);
1054
1055 switch (accum_size_dwords) {
1056 case 1:
1057 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx] =
1058 PIPE_FORMAT_R32_UINT;
1059 break;
1060
1061 case 2:
1062 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx] =
1063 PIPE_FORMAT_R32G32_UINT;
1064 break;
1065
1066 case 3:
1067 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx] =
1068 PIPE_FORMAT_R32G32B32_UINT;
1069 break;
1070
1071 case 4:
1072 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx] =
1073 PIPE_FORMAT_R32G32B32A32_UINT;
1074 break;
1075
1076 default:
1077 UNREACHABLE("");
1078 }
1079 } else {
1080 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx] =
1081 vk_format_to_pipe_format(
1082 load_op->clears_loads_state.dest_vk_format[rt_idx]);
1083
1084 type = glsl_vec4_type();
1085 if (util_format_is_pure_sint(
1086 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx])) {
1087 type = glsl_ivec4_type();
1088 } else if (util_format_is_pure_uint(
1089 data.fs.output_formats[FRAG_RESULT_DATA0 + rt_idx])) {
1090 type = glsl_uvec4_type();
1091 }
1092 }
1093
1094 struct usc_mrt_resource *mrt_resource = &mrt_setup->mrt_resources[rt_idx];
1095 bool tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
1096 has_non_tile_buffer_stores |= !tile_buffer;
1097
1098 data.fs.outputs[FRAG_RESULT_DATA0 + rt_idx] = (pco_range){
1099 .start = tile_buffer ? mrt_resource->mem.tile_buffer
1100 : mrt_resource->reg.output_reg,
1101 .count = accum_size_dwords,
1102 };
1103
1104 if (tile_buffer) {
1105 data.fs.num_tile_buffers =
1106 MAX2(data.fs.num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
1107 data.fs.output_tile_buffers |= BITFIELD_BIT(rt_idx);
1108 data.fs.outputs[FRAG_RESULT_DATA0 + rt_idx].offset =
1109 mrt_resource->mem.offset_dw;
1110 }
1111
1112 nir_create_variable_with_location(b.shader,
1113 nir_var_shader_out,
1114 FRAG_RESULT_DATA0 + rt_idx,
1115 type);
1116 }
1117
1118 unsigned shared_regs = 0;
1119 u_foreach_bit (rt_idx, load_op->clears_loads_state.rt_clear_mask) {
1120 for (unsigned u = 0;
1121 u < data.fs.outputs[FRAG_RESULT_DATA0 + rt_idx].count;
1122 ++u) {
1123 nir_def *chan = nir_load_preamble(&b, 1, 32, .base = shared_regs++);
1124
1125 nir_store_output(&b,
1126 chan,
1127 nir_imm_int(&b, 0),
1128 .base = 0,
1129 .component = u,
1130 .src_type = nir_type_invalid | 32,
1131 .write_mask = 1,
1132 .io_semantics.location = FRAG_RESULT_DATA0 + rt_idx,
1133 .io_semantics.num_slots = 1);
1134 }
1135 }
1136
1137 if (depth_to_reg) {
1138 int32_t depth_idx = load_op->clears_loads_state.depth_clear_to_reg;
1139
1140 struct usc_mrt_resource *mrt_resource =
1141 &mrt_setup->mrt_resources[depth_idx];
1142 bool tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
1143 has_non_tile_buffer_stores |= !tile_buffer;
1144
1145 unsigned accum_size_dwords =
1146 DIV_ROUND_UP(mrt_resource->intermediate_size, sizeof(uint32_t));
1147 assert(accum_size_dwords == 1);
1148
1149 data.fs.output_formats[FRAG_RESULT_DATA0 + depth_idx] =
1150 PIPE_FORMAT_R32_FLOAT;
1151
1152 const glsl_type *type = glsl_float_type();
1153
1154 data.fs.outputs[FRAG_RESULT_DATA0 + depth_idx] = (pco_range){
1155 .start = tile_buffer ? mrt_resource->mem.tile_buffer
1156 : mrt_resource->reg.output_reg,
1157 .count = accum_size_dwords,
1158 };
1159
1160 if (tile_buffer) {
1161 data.fs.num_tile_buffers =
1162 MAX2(data.fs.num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
1163 data.fs.output_tile_buffers |= BITFIELD_BIT(depth_idx);
1164 data.fs.outputs[FRAG_RESULT_DATA0 + depth_idx].offset =
1165 mrt_resource->mem.offset_dw;
1166 }
1167
1168 nir_create_variable_with_location(b.shader,
1169 nir_var_shader_out,
1170 FRAG_RESULT_DATA0 + depth_idx,
1171 type);
1172
1173 nir_def *chan = nir_load_preamble(&b, 1, 32, .base = shared_regs++);
1174
1175 nir_store_output(&b,
1176 chan,
1177 nir_imm_int(&b, 0),
1178 .base = 0,
1179 .component = 0,
1180 .src_type = nir_type_invalid | 32,
1181 .write_mask = 1,
1182 .io_semantics.location = FRAG_RESULT_DATA0 + depth_idx,
1183 .io_semantics.num_slots = 1);
1184 }
1185
1186 if (load_op->clears_loads_state.rt_load_mask) {
1187 nir_variable *pos = nir_get_variable_with_location(b.shader,
1188 nir_var_shader_in,
1189 VARYING_SLOT_POS,
1190 glsl_vec4_type());
1191 pos->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1192 nir_def *coords = nir_channels(&b, nir_load_var(&b, pos), 0b11);
1193
1194 const bool msaa = load_op->clears_loads_state.unresolved_msaa_mask &
1195 load_op->clears_loads_state.rt_load_mask;
1196
1197 b.shader->info.fs.uses_sample_shading = msaa;
1198
1199 shared_regs = ALIGN_POT(shared_regs, 4);
1200
1201 u_foreach_bit (rt_idx, load_op->clears_loads_state.rt_load_mask) {
1202 nir_def *tex_state = nir_load_preamble(&b, 4, 32, .base = shared_regs);
1203 shared_regs += sizeof(struct pvr_image_descriptor) / sizeof(uint32_t);
1204
1205 nir_def *smp_state = nir_load_preamble(&b, 4, 32, .base = shared_regs);
1206 shared_regs +=
1207 sizeof(struct pvr_sampler_descriptor) / sizeof(uint32_t);
1208
1209 nir_variable *var =
1210 nir_find_variable_with_location(b.shader,
1211 nir_var_shader_out,
1212 FRAG_RESULT_DATA0 + rt_idx);
1213 assert(var);
1214 unsigned chans = glsl_get_vector_elements(var->type);
1215
1216 pco_smp_params params = {
1217 .tex_state = tex_state,
1218 .smp_state = smp_state,
1219
1220 .dest_type = nir_get_nir_type_for_glsl_type(var->type),
1221
1222 .nncoords = true,
1223 .coords = coords,
1224
1225 .sampler_dim = msaa ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D,
1226
1227 .ms_index = msaa ? nir_load_sample_id(&b) : NULL,
1228 };
1229
1230 nir_intrinsic_instr *smp = pco_emit_nir_smp(&b, &params);
1231 nir_def *smp_data =
1232 nir_channels(&b, &smp->def, nir_component_mask(chans));
1233
1234 nir_store_output(&b,
1235 smp_data,
1236 nir_imm_int(&b, 0),
1237 .base = 0,
1238 .component = 0,
1239 .src_type = params.dest_type,
1240 .write_mask = BITFIELD_MASK(chans),
1241 .io_semantics.location = FRAG_RESULT_DATA0 + rt_idx,
1242 .io_semantics.num_slots = 1);
1243 }
1244 }
1245
1246 if (data.fs.num_tile_buffers > 0) {
1247 unsigned tile_buffer_addr_dwords =
1248 data.fs.num_tile_buffers * (sizeof(uint64_t) / sizeof(uint32_t));
1249
1250 data.fs.tile_buffers = (pco_range){
1251 .start = shared_regs,
1252 .count = tile_buffer_addr_dwords,
1253 .stride = sizeof(uint64_t) / sizeof(uint32_t),
1254 };
1255
1256 shared_regs += tile_buffer_addr_dwords;
1257
1258 load_op->num_tile_buffers = data.fs.num_tile_buffers;
1259 }
1260
1261 if (!has_non_tile_buffer_stores)
1262 nir_dummy_load_store_pco(&b);
1263
1264 nir_jump(&b, nir_jump_return);
1265
1266 load_op->const_shareds_count = shared_regs;
1267 load_op->shareds_count = shared_regs;
1268
1269 return build_shader(ctx, b.shader, &data);
1270 }
1271
1272 pco_shader *pvr_uscgen_clear_attach(pco_ctx *ctx,
1273 struct pvr_clear_attach_props *props)
1274 {
1275 pco_data data = { 0 };
1276
1277 nir_builder b = nir_builder_init_simple_shader(
1278 MESA_SHADER_FRAGMENT,
1279 pco_nir_options(),
1280 "clear_attach(%s, %u dwords, %u offset)",
1281 props->uses_tile_buffer ? "tiled" : "register",
1282 props->dword_count,
1283 props->offset);
1284
1285 assert(props->dword_count + props->offset <= 4);
1286
1287 if (props->uses_tile_buffer) {
1288 nir_def *valid_mask = nir_load_savmsk_vm_pco(&b);
1289
1290 nir_def *tile_addr_lo =
1291 nir_load_preamble(&b,
1292 1,
1293 32,
1294 .base = PVR_CLEAR_ATTACH_DATA_TILE_ADDR_LO);
1295 nir_def *tile_addr_hi =
1296 nir_load_preamble(&b,
1297 1,
1298 32,
1299 .base = PVR_CLEAR_ATTACH_DATA_TILE_ADDR_HI);
1300
1301 for (unsigned u = 0; u < props->dword_count; ++u) {
1302 nir_def *tiled_offset =
1303 nir_load_tiled_offset_pco(&b, .component = u + props->offset);
1304
1305 nir_def *addr =
1306 nir_uadd64_32(&b, tile_addr_lo, tile_addr_hi, tiled_offset);
1307
1308 nir_def *data =
1309 nir_load_preamble(&b,
1310 1,
1311 32,
1312 .base = PVR_CLEAR_ATTACH_DATA_DWORD0 + u);
1313
1314 nir_def *addr_data = nir_vec3(&b,
1315 nir_channel(&b, addr, 0),
1316 nir_channel(&b, addr, 1),
1317 data);
1318
1319 nir_dma_st_tiled_pco(&b, addr_data, valid_mask);
1320 }
1321
1322 nir_dummy_load_store_pco(&b);
1323 } else {
1324 for (unsigned u = 0; u < props->dword_count; ++u) {
1325 nir_def *data =
1326 nir_load_preamble(&b,
1327 1,
1328 32,
1329 .base = PVR_CLEAR_ATTACH_DATA_DWORD0 + u);
1330
1331 nir_frag_store_pco(&b, data, u + props->offset);
1332 }
1333 }
1334
1335 nir_jump(&b, nir_jump_return);
1336
1337 return build_shader(ctx, b.shader, &data);
1338 }
1339
1340 pco_shader *
1341 pvr_usc_zero_init_wg_mem(pco_ctx *ctx, unsigned start, unsigned count)
1342 {
1343 pco_data data = {
1344 .cs.shmem.start = start,
1345 .cs.shmem.count = count,
1346 .common.uses.usclib = true,
1347 };
1348
1349 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,
1350 pco_nir_options(),
1351 "zero_init_wg_mem(%u, %u)",
1352 start,
1353 count);
1354
1355 usclib_zero_init_wg_mem(&b, nir_imm_int(&b, count));
1356
1357 nir_jump(&b, nir_jump_return);
1358
1359 return build_shader(ctx, b.shader, &data);
1360 }
1361
1362 pco_shader *pvr_uscgen_spm_load(pco_ctx *ctx, struct pvr_spm_load_props *props)
1363 {
1364 pco_data data = { 0 };
1365
1366 nir_builder b = nir_builder_init_simple_shader(
1367 MESA_SHADER_FRAGMENT,
1368 pco_nir_options(),
1369 "spm_load(%u output regs, %u tile buffers, %s)",
1370 props->output_reg_count,
1371 props->tile_buffer_count,
1372 props->is_multisampled ? "ms" : "non-ms");
1373
1374 b.shader->info.fs.uses_sample_shading = props->is_multisampled;
1375
1376 nir_variable *pos = nir_get_variable_with_location(b.shader,
1377 nir_var_shader_in,
1378 VARYING_SLOT_POS,
1379 glsl_vec4_type());
1380 pos->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1381
1382 nir_def *coords = nir_channels(&b, nir_load_var(&b, pos), 0b11);
1383 nir_def *ms_index = props->is_multisampled ? nir_load_sample_id(&b) : NULL;
1384
1385 nir_def *smp_state = nir_load_preamble(&b,
1386 ROGUE_NUM_TEXSTATE_DWORDS,
1387 32,
1388 .base = PVR_SPM_LOAD_DATA_SMP);
1389
1390 /* Initialize common params. */
1391 pco_smp_params params = {
1392 .smp_state = smp_state,
1393 .dest_type = nir_type_uint32,
1394 .sampler_dim = GLSL_SAMPLER_DIM_2D,
1395 .coords = coords,
1396 .lod_replace = nir_imm_int(&b, 0),
1397 .ms_index = ms_index,
1398 };
1399
1400 nir_def *valid_mask = nir_load_savmsk_vm_pco(&b);
1401 nir_intrinsic_instr *smp;
1402
1403 /* Emit tile buffer sample + writes. */
1404 /* TODO: emit nir_store_outputs instead, needs backend to handle
1405 * discontiguous tile buffer locations.
1406 */
1407 for (unsigned buffer = 0; buffer < props->tile_buffer_count; ++buffer) {
1408 unsigned tex_base = pvr_uscgen_spm_buffer_data(buffer, false);
1409 params.tex_state =
1410 nir_load_preamble(&b, ROGUE_NUM_TEXSTATE_DWORDS, 32, .base = tex_base);
1411 params.sample_components = 4;
1412
1413 smp = pco_emit_nir_smp(&b, &params);
1414
1415 unsigned tile_addr_base = pvr_uscgen_spm_buffer_data(buffer, true);
1416 nir_def *tile_addr_lo =
1417 nir_load_preamble(&b, 1, 32, .base = tile_addr_base);
1418 nir_def *tile_addr_hi =
1419 nir_load_preamble(&b, 1, 32, .base = tile_addr_base + 1);
1420
1421 for (unsigned u = 0; u < params.sample_components; ++u) {
1422 nir_def *tiled_offset = nir_load_tiled_offset_pco(&b, .component = u);
1423
1424 nir_def *addr =
1425 nir_uadd64_32(&b, tile_addr_lo, tile_addr_hi, tiled_offset);
1426
1427 nir_def *data = nir_channel(&b, &smp->def, u);
1428
1429 nir_def *addr_data = nir_vec3(&b,
1430 nir_channel(&b, addr, 0),
1431 nir_channel(&b, addr, 1),
1432 data);
1433
1434 nir_dma_st_tiled_pco(&b, addr_data, valid_mask);
1435 }
1436 }
1437
1438 /* Emit output reg sample + write. */
1439 switch (props->output_reg_count) {
1440 case 1:
1441 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32_UINT;
1442 break;
1443
1444 case 2:
1445 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32G32_UINT;
1446 break;
1447
1448 case 4:
1449 data.fs.output_formats[FRAG_RESULT_DATA0] = PIPE_FORMAT_R32G32B32A32_UINT;
1450 break;
1451
1452 default:
1453 UNREACHABLE("");
1454 }
1455
1456 data.fs.outputs[FRAG_RESULT_DATA0] = (pco_range){
1457 .start = 0,
1458 .count = props->output_reg_count,
1459 };
1460
1461 nir_create_variable_with_location(b.shader,
1462 nir_var_shader_out,
1463 FRAG_RESULT_DATA0,
1464 glsl_uvec_type(props->output_reg_count));
1465
1466 params.tex_state = nir_load_preamble(&b,
1467 ROGUE_NUM_TEXSTATE_DWORDS,
1468 32,
1469 .base = PVR_SPM_LOAD_DATA_REG_TEX);
1470 params.sample_components = props->output_reg_count;
1471
1472 smp = pco_emit_nir_smp(&b, &params);
1473
1474 for (unsigned u = 0; u < props->output_reg_count; ++u) {
1475 nir_store_output(&b,
1476 nir_channel(&b, &smp->def, u),
1477 nir_imm_int(&b, 0),
1478 .base = 0,
1479 .component = u,
1480 .src_type = nir_type_invalid | 32,
1481 .write_mask = 1,
1482 .io_semantics.location = FRAG_RESULT_DATA0,
1483 .io_semantics.num_slots = 1);
1484 }
1485
1486 nir_jump(&b, nir_jump_return);
1487
1488 return build_shader(ctx, b.shader, &data);
1489 }