Debian Codegraph @

src/imagination/pco/pco_nir_tex.c

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pco_nir_tex.c
9 *
10 * \brief PCO NIR texture/image/sampler lowering passes.
11 */
12
13 #include "hwdef/rogue_hw_defs.h"
14 #include "nir.h"
15 #include "nir_builder.h"
16 #include "nir_builtin_builder.h"
17 #include "pco.h"
18 #include "pco_builder.h"
19 #include "pco_common.h"
20 #include "pco_internal.h"
21 #include "pco_usclib.h"
22 #include "pvr_iface.h"
23 #include "util/macros.h"
24
25 #include <assert.h>
26 #include <stdbool.h>
27 #include <stdio.h>
28
29 struct state {
30 pco_data *data;
31 pco_ctx *ctx;
32 };
33
34 static inline nir_def *get_src_def(nir_tex_instr *tex,
35 nir_tex_src_type src_type)
36 {
37 int src_idx = nir_tex_instr_src_index(tex, src_type);
38 return src_idx >= 0 ? tex->src[src_idx].src.ssa : NULL;
39 }
40
41 /**
42 * \brief Lowers a basic texture query (no sampling required).
43 *
44 * \param[in] b NIR builder.
45 * \param[in] tex NIR texture instruction.
46 * \param[in] tex_state Texture state words.
47 * \return The replacement/lowered def.
48 */
49 static nir_def *lower_tex_query_basic(nir_builder *b,
50 nir_tex_instr *tex,
51 nir_def *tex_state,
52 nir_def *tex_meta,
53 pco_data *data)
54 {
55 switch (tex->op) {
56 case nir_texop_query_levels:
57 data->common.uses.usclib = true;
58 return usclib_tex_state_levels(b, tex_state);
59
60 case nir_texop_texture_samples:
61 data->common.uses.usclib = true;
62 return usclib_tex_state_samples(b, tex_state);
63
64 case nir_texop_txs: {
65 if (tex->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
66 assert(tex->def.num_components == 1);
67 assert(!tex->is_array);
68
69 return nir_channel(b, tex_meta, PCO_IMAGE_META_BUFFER_ELEMS);
70 }
71
72 nir_def *num_comps = nir_imm_int(b, tex->def.num_components);
73 nir_def *dim = nir_imm_int(b, tex->sampler_dim);
74 nir_def *is_array = nir_imm_bool(b, tex->is_array);
75 nir_def *is_image = nir_imm_bool(b, false);
76 nir_def *lod = get_src_def(tex, nir_tex_src_lod);
77 nir_def *size_comps = usclib_tex_state_size(b,
78 tex_state,
79 num_comps,
80 dim,
81 is_array,
82 is_image,
83 lod);
84
85 data->common.uses.usclib = true;
86
87 return nir_trim_vector(b, size_comps, tex->def.num_components);
88 }
89
90 default:
91 break;
92 }
93
94 UNREACHABLE("");
95 }
96
97 static inline enum pco_dim to_pco_dim(enum glsl_sampler_dim dim)
98 {
99 switch (dim) {
100 case GLSL_SAMPLER_DIM_1D:
101 case GLSL_SAMPLER_DIM_BUF:
102 return PCO_DIM_1D;
103
104 case GLSL_SAMPLER_DIM_2D:
105 case GLSL_SAMPLER_DIM_MS:
106 case GLSL_SAMPLER_DIM_SUBPASS:
107 case GLSL_SAMPLER_DIM_SUBPASS_MS:
108 return PCO_DIM_2D;
109
110 case GLSL_SAMPLER_DIM_3D:
111 case GLSL_SAMPLER_DIM_CUBE:
112 return PCO_DIM_3D;
113
114 /* case GLSL_SAMPLER_DIM_RECT: */
115 /* case GLSL_SAMPLER_DIM_EXTERNAL: */
116 default:
117 break;
118 }
119
120 UNREACHABLE("");
121 }
122
123 static nir_def *lower_tex_query_lod(nir_builder *b,
124 nir_def *coords,
125 nir_def *smp_coeffs,
126 pco_smp_params *params,
127 pco_data *data,
128 pco_ctx *ctx)
129 {
130 nir_def *lod_dval_post_clamp =
131 nir_channel(b, smp_coeffs, ROGUE_SMP_COEFF_LOD_DVAL_POST_CLAMP);
132 nir_def *lod_dval_pre_clamp =
133 nir_channel(b, smp_coeffs, ROGUE_SMP_COEFF_LOD_DVAL_PRE_CLAMP);
134 nir_def *tfrac_post_clamp =
135 nir_channel(b, smp_coeffs, ROGUE_SMP_COEFF_TFRAC_POST_CLAMP);
136 nir_def *tfrac_pre_clamp =
137 nir_channel(b, smp_coeffs, ROGUE_SMP_COEFF_TFRAC_PRE_CLAMP);
138
139 /* Unpack. */
140 lod_dval_post_clamp = nir_fmul_imm(b, lod_dval_post_clamp, 255.0f);
141 lod_dval_pre_clamp = nir_fmul_imm(b, lod_dval_pre_clamp, 255.0f);
142
143 tfrac_post_clamp = nir_fmul_imm(b, tfrac_post_clamp, 255.0f);
144 tfrac_pre_clamp = nir_fmul_imm(b, tfrac_pre_clamp, 255.0f);
145
146 /* Scale. */
147 tfrac_post_clamp = nir_fdiv_imm(b, tfrac_post_clamp, 256.0f);
148 tfrac_pre_clamp = nir_fdiv_imm(b, tfrac_pre_clamp, 256.0f);
149
150 if (PVR_HAS_QUIRK(ctx->dev_info, 74056)) {
151 lod_dval_post_clamp =
152 usclib_tex_lod_dval_post_clamp_resource_to_view_space(
153 b,
154 params->tex_state,
155 params->smp_state,
156 lod_dval_post_clamp);
157
158 data->common.uses.usclib = true;
159 }
160
161 /* Calculate coord deltas. */
162 nir_def *coord_deltas = nir_imm_int(b, 0);
163 for (unsigned c = 0; c < coords->num_components; ++c) {
164 nir_def *coord = nir_channel(b, coords, c);
165 coord_deltas = nir_fadd(b,
166 coord_deltas,
167 nir_fadd(b,
168 nir_fabs(b, nir_ddx(b, coord)),
169 nir_fabs(b, nir_ddy(b, coord))));
170 }
171
172 nir_def *lod_comps[2] = {
173 [0] = nir_fadd(b, lod_dval_post_clamp, tfrac_post_clamp),
174 [1] = nir_fadd(
175 b,
176 nir_fadd_imm(b, tfrac_pre_clamp, -128.0f),
177 nir_fcsel(b, coord_deltas, lod_dval_pre_clamp, nir_imm_float(b, 0.0f))),
178 };
179
180 return nir_vec(b, lod_comps, ARRAY_SIZE(lod_comps));
181 }
182
183 static inline unsigned process_coords(nir_builder *b,
184 bool is_array,
185 bool coords_are_float,
186 nir_def *coords,
187 nir_def **float_coords,
188 nir_def **int_coords,
189 nir_def **float_array_index,
190 nir_def **int_array_index)
191 {
192 unsigned num_comps = coords->num_components;
193
194 *float_coords = coords_are_float ? coords : nir_i2f32(b, coords);
195 *int_coords = !coords_are_float ? coords : nir_f2i32(b, coords);
196 *float_array_index = NULL;
197 *int_array_index = NULL;
198
199 if (!is_array)
200 return num_comps;
201
202 *float_array_index = nir_channel(b, *float_coords, num_comps - 1);
203 *int_array_index = coords_are_float
204 ? nir_f2i32_rtne(b, *float_array_index)
205 : nir_channel(b, *int_coords, num_comps - 1);
206
207 *float_coords = nir_trim_vector(b, *float_coords, num_comps - 1);
208 *int_coords = nir_trim_vector(b, *int_coords, num_comps - 1);
209
210 return num_comps - 1;
211 }
212
213 static inline bool tex_src_is_float(nir_tex_instr *tex,
214 nir_tex_src_type src_type)
215 {
216 int src_idx = nir_tex_instr_src_index(tex, src_type);
217 assert(src_idx >= 0);
218 return nir_tex_instr_src_type(tex, src_idx) == nir_type_float;
219 }
220
221 nir_intrinsic_instr *pco_emit_nir_smp(nir_builder *b, pco_smp_params *params)
222 {
223 nir_def *comps[NIR_MAX_VEC_COMPONENTS];
224 unsigned count = 0;
225 pco_smp_flags smp_flags = {
226 .dim = to_pco_dim(params->sampler_dim),
227 .fcnorm = nir_alu_type_get_base_type(params->dest_type) == nir_type_float,
228 .nncoords = params->nncoords,
229 .lod_mode = PCO_LOD_MODE_NORMAL,
230 .integer = params->int_mode,
231 };
232
233 /* Emit coords (excluding array component if present). */
234 for (unsigned c = 0; c < params->coords->num_components; ++c)
235 comps[count++] = nir_channel(b, params->coords, c);
236
237 /* Emit projector (if present). */
238 if (params->proj) {
239 comps[count++] = params->proj;
240 smp_flags.proj = true;
241 }
242
243 /* Emit hardware array component (if present). */
244 if (params->array_index) {
245 comps[count++] = params->array_index;
246 smp_flags.array = true;
247 }
248
249 /* Emit LOD (if present). */
250 bool lod_present = false;
251 assert(!!params->lod_ddx == !!params->lod_ddy);
252 assert((!!params->lod_bias + !!params->lod_replace + !!params->lod_ddx) < 2);
253 if (params->lod_bias) {
254 lod_present = true;
255 comps[count++] = params->lod_bias;
256
257 smp_flags.pplod = true;
258 smp_flags.lod_mode = PCO_LOD_MODE_BIAS;
259 } else if (params->lod_replace) {
260 lod_present = true;
261 comps[count++] = params->lod_replace;
262
263 smp_flags.pplod = true;
264 smp_flags.lod_mode = PCO_LOD_MODE_REPLACE;
265 } else if (params->lod_ddx) {
266 lod_present = true;
267
268 for (unsigned c = 0; c < params->lod_ddx->num_components; ++c) {
269 comps[count++] = nir_channel(b, params->lod_ddx, c);
270 comps[count++] = nir_channel(b, params->lod_ddy, c);
271 }
272
273 smp_flags.lod_mode = PCO_LOD_MODE_GRADIENTS;
274 }
275
276 /* Emit address override (if present). */
277 assert(!!params->addr_lo == !!params->addr_hi);
278 if (params->addr_lo) {
279 /* Set a per-pixel lod bias of 0 if none has been set yet. */
280 if (!lod_present) {
281 comps[count++] = nir_imm_int(b, 0);
282 smp_flags.pplod = true;
283 smp_flags.lod_mode = PCO_LOD_MODE_BIAS;
284 lod_present = true;
285 }
286
287 comps[count++] = params->addr_lo;
288 comps[count++] = params->addr_hi;
289
290 smp_flags.tao = true;
291 }
292
293 /* Emit lookup options (if present). */
294 if (params->offset || params->ms_index) {
295 nir_def *lookup = nir_imm_int(b, 0);
296
297 if (params->offset) {
298 const unsigned packed_offset_start[] = { ROGUE_SMP_LOOKUP_U_OFF_START,
299 ROGUE_SMP_LOOKUP_V_OFF_START,
300 ROGUE_SMP_LOOKUP_W_OFF_START };
301 const unsigned packed_offset_size[] = { ROGUE_SMP_LOOKUP_U_OFF_BITS,
302 ROGUE_SMP_LOOKUP_V_OFF_BITS,
303 ROGUE_SMP_LOOKUP_W_OFF_BITS };
304
305 for (unsigned c = 0; c < params->offset->num_components; ++c) {
306 lookup = nir_bitfield_insert(b,
307 lookup,
308 nir_channel(b, params->offset, c),
309 nir_imm_int(b, packed_offset_start[c]),
310 nir_imm_int(b, packed_offset_size[c]));
311 }
312
313 smp_flags.soo = true;
314 }
315
316 if (params->ms_index) {
317 lookup =
318 nir_bitfield_insert(b,
319 lookup,
320 params->ms_index,
321 nir_imm_int(b, ROGUE_SMP_LOOKUP_MS_NUM_START),
322 nir_imm_int(b, ROGUE_SMP_LOOKUP_MS_NUM_BITS));
323
324 smp_flags.sno = true;
325 }
326
327 comps[count++] = lookup;
328 }
329
330 /* Emit write data (if present). */
331 if (params->write_data) {
332 for (unsigned c = 0; c < params->write_data->num_components; ++c)
333 comps[count++] = nir_channel(b, params->write_data, c);
334
335 smp_flags.wrt = true;
336 }
337
338 /* Pad out the rest of the data words. */
339 assert(count <= NIR_MAX_VEC_COMPONENTS);
340
341 nir_def *undef = nir_undef(b, 1, 32);
342 for (unsigned c = count; c < ARRAY_SIZE(comps); ++c)
343 comps[c] = undef;
344
345 nir_def *smp_data = nir_vec(b, comps, ARRAY_SIZE(comps));
346
347 if (params->sample_coeffs) {
348 assert(!params->sample_raw);
349 assert(!params->sample_components);
350 assert(!params->write_data);
351
352 nir_def *def = nir_smp_coeffs_pco(b,
353 smp_data,
354 params->tex_state,
355 params->smp_state,
356 .smp_flags_pco = smp_flags._,
357 .range = count);
358
359 return nir_def_as_intrinsic(def);
360 }
361
362 if (params->sample_raw) {
363 assert(!params->sample_coeffs);
364 assert(!params->sample_components);
365 assert(!params->write_data);
366
367 nir_def *def = nir_smp_raw_pco(b,
368 smp_data,
369 params->tex_state,
370 params->smp_state,
371 .smp_flags_pco = smp_flags._,
372 .range = count);
373
374 return nir_def_as_intrinsic(def);
375 }
376
377 if (params->write_data) {
378 assert(!params->sample_coeffs);
379 assert(!params->sample_raw);
380 assert(!params->sample_components);
381
382 return nir_smp_write_pco(b,
383 smp_data,
384 params->tex_state,
385 params->smp_state,
386 .smp_flags_pco = smp_flags._,
387 .range = count);
388 }
389
390 assert(!params->sample_coeffs);
391 assert(!params->sample_raw);
392 assert(!params->write_data);
393
394 if (!params->sample_components)
395 params->sample_components = 4;
396
397 nir_def *def = nir_smp_pco(b,
398 params->sample_components,
399 smp_data,
400 params->tex_state,
401 params->smp_state,
402 .smp_flags_pco = smp_flags._,
403 .range = count);
404
405 return nir_def_as_intrinsic(def);
406 }
407
408 static nir_def *
409 lower_tex_gather(nir_builder *b, nir_tex_instr *tex, nir_def *raw_data)
410 {
411 assert(!nir_tex_instr_has_explicit_tg4_offsets(tex));
412
413 #define TG4_SEL(sample) (((sample) * 4) + tex->component)
414 unsigned swiz[] = { TG4_SEL(2), TG4_SEL(3), TG4_SEL(1), TG4_SEL(0) };
415 #undef TG4_SEL
416
417 return nir_swizzle(b, raw_data, swiz, ARRAY_SIZE(swiz));
418 }
419
420 static nir_def *lower_tex_shadow(nir_builder *b,
421 nir_def *data,
422 nir_def *comparator,
423 nir_def *compare_op)
424 {
425 nir_def *result_comps[NIR_MAX_VEC_COMPONENTS];
426
427 for (unsigned u = 0; u < data->num_components; ++u) {
428 result_comps[u] =
429 nir_alphatst_pco(b, nir_channel(b, data, u), comparator, compare_op);
430 }
431
432 return nir_vec(b, result_comps, data->num_components);
433 }
434
435 /**
436 * \brief Lowers a texture instruction.
437 *
438 * \param[in] b NIR builder.
439 * \param[in] instr NIR instruction.
440 * \param[in] cb_data User callback data.
441 * \return The replacement/lowered def.
442 */
443 static nir_def *lower_tex(nir_builder *b, nir_instr *instr, void *cb_data)
444 {
445 nir_tex_instr *tex = nir_instr_as_tex(instr);
446 struct state *state = cb_data;
447 pco_data *data = state->data;
448 pco_ctx *ctx = state->ctx;
449 const struct pvr_device_info *dev_info = ctx->dev_info;
450
451 unsigned tex_desc_set;
452 unsigned tex_binding;
453 pco_unpack_desc(tex->texture_index, &tex_desc_set, &tex_binding);
454
455 unsigned smp_desc_set;
456 unsigned smp_binding;
457 pco_unpack_desc(tex->sampler_index, &smp_desc_set, &smp_binding);
458
459 bool hw_array_support = false;
460 bool hw_int_support =
461 PCO_DEBUG(INT_SMP)
462 ? PVR_HAS_FEATURE(dev_info, tpu_extended_integer_lookup)
463 : false;
464
465 b->cursor = nir_before_instr(instr);
466
467 /* Process tex sources, build up the smp flags and data words. */
468 BITSET_DECLARE(tex_src_set, nir_num_tex_src_types) = { 0 };
469 nir_def *tex_srcs[nir_num_tex_src_types];
470 pco_smp_params params = {
471 .dest_type = tex->dest_type,
472 .sampler_dim = tex->sampler_dim,
473 };
474
475 for (unsigned s = 0; s < nir_num_tex_src_types; ++s)
476 if ((tex_srcs[s] = get_src_def(tex, s)) != NULL)
477 BITSET_SET(tex_src_set, s);
478
479 nir_def *tex_elem = nir_imm_int(b, 0);
480 if (BITSET_TEST(tex_src_set, nir_tex_src_backend1)) {
481 tex_elem = tex_srcs[nir_tex_src_backend1];
482 BITSET_CLEAR(tex_src_set, nir_tex_src_backend1);
483 }
484
485 nir_def *smp_elem = nir_imm_int(b, 0);
486 if (BITSET_TEST(tex_src_set, nir_tex_src_backend2)) {
487 smp_elem = tex_srcs[nir_tex_src_backend2];
488 BITSET_CLEAR(tex_src_set, nir_tex_src_backend2);
489 }
490
491 nir_def *tex_state = nir_load_tex_state_pco(b,
492 ROGUE_NUM_TEXSTATE_DWORDS,
493 tex_elem,
494 .desc_set = tex_desc_set,
495 .binding = tex_binding);
496
497 nir_def *tex_meta = nir_load_tex_meta_pco(b,
498 PCO_IMAGE_META_COUNT,
499 tex_elem,
500 .desc_set = tex_desc_set,
501 .binding = tex_binding);
502
503 if (nir_tex_instr_is_query(tex) && tex->op != nir_texop_lod)
504 return lower_tex_query_basic(b, tex, tex_state, tex_meta, data);
505
506 nir_def *smp_state =
507 nir_load_smp_state_pco(b,
508 ROGUE_NUM_TEXSTATE_DWORDS,
509 smp_elem,
510 .desc_set = smp_desc_set,
511 .binding = smp_binding,
512 .flags = tex->op == nir_texop_tg4);
513
514 params.tex_state = tex_state;
515 params.smp_state = smp_state;
516
517 bool is_cube_array = tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
518 tex->is_array;
519 bool is_2d_view_of_3d = false;
520
521 /* Special case, override buffers to be 2D. */
522 if ((tex->op == nir_texop_txf || tex->op == nir_texop_txf_ms) &&
523 tex->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
524 assert(!tex_src_is_float(tex, nir_tex_src_coord));
525
526 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
527 params.sampler_dim = tex->sampler_dim;
528 tex_srcs[nir_tex_src_coord] =
529 nir_vec2(b,
530 nir_umod_imm(b, tex_srcs[nir_tex_src_coord], 8192),
531 nir_udiv_imm(b, tex_srcs[nir_tex_src_coord], 8192));
532 } else if (data->common.image_2d_view_of_3d && tex->op != nir_texop_lod &&
533 tex->sampler_dim == GLSL_SAMPLER_DIM_2D && !tex->is_array) {
534 tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
535 params.sampler_dim = tex->sampler_dim;
536
537 nir_def *tex_meta = nir_load_tex_meta_pco(b,
538 PCO_IMAGE_META_COUNT,
539 tex_elem,
540 .desc_set = tex_desc_set,
541 .binding = tex_binding);
542
543 nir_def *z_slice = nir_channel(b, tex_meta, PCO_IMAGE_META_Z_SLICE);
544
545 if (tex_src_is_float(tex, nir_tex_src_coord))
546 z_slice = nir_i2f32(b, z_slice);
547
548 tex_srcs[nir_tex_src_coord] =
549 nir_pad_vector(b, tex_srcs[nir_tex_src_coord], 3);
550 tex_srcs[nir_tex_src_coord] =
551 nir_vector_insert_imm(b, tex_srcs[nir_tex_src_coord], z_slice, 2);
552
553 is_2d_view_of_3d = true;
554 }
555
556 assert((BITSET_TEST(tex_src_set, nir_tex_src_bias) +
557 BITSET_TEST(tex_src_set, nir_tex_src_lod) +
558 BITSET_TEST(tex_src_set, nir_tex_src_ddx)) < 2);
559
560 bool lod_set = false;
561 if (BITSET_TEST(tex_src_set, nir_tex_src_bias)) {
562 params.lod_bias = tex_src_is_float(tex, nir_tex_src_bias)
563 ? tex_srcs[nir_tex_src_bias]
564 : nir_i2f32(b, tex_srcs[nir_tex_src_bias]);
565
566 lod_set = true;
567 BITSET_CLEAR(tex_src_set, nir_tex_src_bias);
568 } else if (BITSET_TEST(tex_src_set, nir_tex_src_lod)) {
569 params.lod_replace = tex_src_is_float(tex, nir_tex_src_lod)
570 ? tex_srcs[nir_tex_src_lod]
571 : nir_i2f32(b, tex_srcs[nir_tex_src_lod]);
572
573 lod_set = true;
574 BITSET_CLEAR(tex_src_set, nir_tex_src_lod);
575 } else if (BITSET_TEST(tex_src_set, nir_tex_src_ddx)) {
576 assert(BITSET_TEST(tex_src_set, nir_tex_src_ddy));
577 assert(tex_src_is_float(tex, nir_tex_src_ddx) &&
578 tex_src_is_float(tex, nir_tex_src_ddy));
579
580 params.lod_ddx = tex_srcs[nir_tex_src_ddx];
581 params.lod_ddy = tex_srcs[nir_tex_src_ddy];
582
583 if (is_2d_view_of_3d) {
584 params.lod_ddx = nir_pad_vector(b, params.lod_ddx, 3);
585 params.lod_ddx =
586 nir_vector_insert_imm(b, params.lod_ddx, nir_imm_int(b, 0), 2);
587
588 params.lod_ddy = nir_pad_vector(b, params.lod_ddy, 3);
589 params.lod_ddy =
590 nir_vector_insert_imm(b, params.lod_ddy, nir_imm_int(b, 0), 2);
591 }
592
593 lod_set = true;
594 BITSET_CLEAR(tex_src_set, nir_tex_src_ddx);
595 BITSET_CLEAR(tex_src_set, nir_tex_src_ddy);
596 }
597
598 /* Disable integer coordinate support if LOD is present due to lack of
599 * clamping in this scenario.
600 */
601 if (lod_set)
602 hw_int_support = false;
603
604 nir_def *float_coords;
605 nir_def *int_coords;
606 nir_def *float_array_index;
607 nir_def *int_array_index;
608 process_coords(b,
609 tex->is_array && tex->op != nir_texop_lod,
610 tex_src_is_float(tex, nir_tex_src_coord),
611 tex_srcs[nir_tex_src_coord],
612 &float_coords,
613 &int_coords,
614 &float_array_index,
615 &int_array_index);
616
617 bool use_int_coords = !tex_src_is_float(tex, nir_tex_src_coord) &&
618 hw_int_support;
619
620 params.int_mode = use_int_coords,
621
622 assert(BITSET_TEST(tex_src_set, nir_tex_src_coord));
623 if (BITSET_TEST(tex_src_set, nir_tex_src_coord)) {
624 params.coords = use_int_coords ? int_coords : float_coords;
625 BITSET_CLEAR(tex_src_set, nir_tex_src_coord);
626 }
627
628 nir_def *proj = NULL;
629 if (BITSET_TEST(tex_src_set, nir_tex_src_projector)) {
630 assert(tex_src_is_float(tex, nir_tex_src_projector));
631 proj = tex_srcs[nir_tex_src_projector];
632 params.proj = use_int_coords ? nir_f2i32(b, proj) : proj;
633 BITSET_CLEAR(tex_src_set, nir_tex_src_projector);
634 }
635
636 if (tex->op == nir_texop_tg4) {
637 assert(!lod_set);
638 params.lod_replace = nir_imm_int(b, 0);
639 lod_set = true;
640 }
641
642 if (!lod_set && is_2d_view_of_3d) {
643 params.lod_bias = nir_imm_int(b, 0);
644 lod_set = true;
645 }
646
647 if (tex->is_array && tex->op != nir_texop_lod) {
648 if (hw_array_support) {
649 params.array_index = int_array_index;
650 UNREACHABLE("Hardware support for array indexing not implemented");
651 } else {
652 nir_def *array_index = int_array_index;
653 assert(array_index);
654
655 nir_def *array_max = usclib_tex_state_array_max(b, tex_state);
656 array_index = nir_iclamp(b, array_index, nir_imm_int(b, 0), array_max);
657 if (is_cube_array)
658 array_index = nir_imul_imm(b, array_index, 6);
659
660 nir_def *tex_meta = nir_load_tex_meta_pco(b,
661 PCO_IMAGE_META_COUNT,
662 tex_elem,
663 .desc_set = tex_desc_set,
664 .binding = tex_binding);
665
666 nir_def *array_stride =
667 nir_channel(b, tex_meta, PCO_IMAGE_META_LAYER_SIZE);
668
669 nir_def *array_offset = nir_imul(b, array_index, array_stride);
670
671 nir_def *base_addr = usclib_tex_state_address(b, tex_state);
672 nir_def *base_addr_lo = nir_channel(b, base_addr, 0);
673 nir_def *base_addr_hi = nir_channel(b, base_addr, 1);
674
675 nir_def *addr =
676 nir_uadd64_32(b, base_addr_lo, base_addr_hi, array_offset);
677
678 params.addr_lo = nir_channel(b, addr, 0);
679 params.addr_hi = nir_channel(b, addr, 1);
680
681 data->common.uses.usclib = true;
682 }
683 }
684
685 if (BITSET_TEST(tex_src_set, nir_tex_src_offset)) {
686 params.offset = tex_srcs[nir_tex_src_offset];
687 BITSET_CLEAR(tex_src_set, nir_tex_src_offset);
688 }
689
690 if (BITSET_TEST(tex_src_set, nir_tex_src_ms_index)) {
691 params.ms_index = tex_srcs[nir_tex_src_ms_index];
692 BITSET_CLEAR(tex_src_set, nir_tex_src_ms_index);
693 }
694
695 /* Shadow comparator. */
696 nir_def *comparator = NULL;
697 if (BITSET_TEST(tex_src_set, nir_tex_src_comparator)) {
698 comparator = tex_srcs[nir_tex_src_comparator];
699
700 if (proj)
701 comparator = nir_fdiv(b, comparator, proj);
702
703 BITSET_CLEAR(tex_src_set, nir_tex_src_comparator);
704 }
705
706 assert(BITSET_IS_EMPTY(tex_src_set));
707
708 nir_def *result;
709 nir_intrinsic_instr *smp;
710 switch (tex->op) {
711 case nir_texop_lod:
712 params.sample_coeffs = true;
713 smp = pco_emit_nir_smp(b, &params);
714 result =
715 lower_tex_query_lod(b, float_coords, &smp->def, &params, data, ctx);
716 break;
717
718 case nir_texop_txf:
719 case nir_texop_txf_ms:
720 params.nncoords = !use_int_coords;
721 FALLTHROUGH;
722
723 case nir_texop_tex:
724 case nir_texop_txb:
725 case nir_texop_txd:
726 case nir_texop_txl:
727 params.sample_components = tex->def.num_components;
728 smp = pco_emit_nir_smp(b, &params);
729 result = &smp->def;
730 break;
731
732 case nir_texop_tg4:
733 params.sample_raw = true;
734 smp = pco_emit_nir_smp(b, &params);
735 result = lower_tex_gather(b, tex, &smp->def);
736 break;
737
738 default:
739 UNREACHABLE("");
740 }
741
742 if (tex->is_shadow) {
743 nir_def *compare_op =
744 nir_load_smp_meta_pco(b,
745 1,
746 smp_elem,
747 .desc_set = smp_desc_set,
748 .binding = smp_binding,
749 .component = PCO_SAMPLER_META_COMPARE_OP);
750
751 /*
752 * From the Vulkan 1.4.339 spec, 17.2.4. Depth Compare Operation:
753 *
754 * "If the image being sampled has an unsigned normalized fixed-point
755 * format, then Dref is clamped to [0,1] before the compare operation."
756 *
757 * -> Clamp the comparator to the range 0.0 - 1.0 for unorm formats.
758 */
759
760 nir_def *pck_info = nir_channel(b, tex_meta, PCO_IMAGE_META_PCK_INFO);
761 nir_def *is_unorm = nir_ubitfield_extract_imm(b, pck_info, 8, 1);
762 is_unorm = nir_ine_imm(b, is_unorm, 0);
763
764 comparator = nir_bcsel(b, is_unorm, nir_fsat(b, comparator), comparator);
765
766 result = lower_tex_shadow(b, result, comparator, compare_op);
767 }
768
769 return result;
770 }
771
772 /**
773 * \brief Filters texture instructions.
774 *
775 * \param[in] instr NIR instruction.
776 * \param[in] cb_data User callback data.
777 * \return True if the instruction matches the filter.
778 */
779 static bool is_tex(const nir_instr *instr, UNUSED const void *cb_data)
780 {
781 return instr->type == nir_instr_type_tex;
782 }
783
784 /**
785 * \brief Texture lowering pass.
786 *
787 * \param[in,out] shader NIR shader.
788 * \param[in,out] data Shader data.
789 * \param[in] ctx PCO compiler context.
790 * \return True if the pass made progress.
791 */
792 bool pco_nir_lower_tex(nir_shader *shader, pco_data *data, pco_ctx *ctx)
793 {
794 struct state state = {
795 .data = data,
796 .ctx = ctx,
797 };
798
799 return nir_shader_lower_instructions(shader, is_tex, lower_tex, &state);
800 }
801
802 static enum util_format_type nir_type_to_util_type(nir_alu_type nir_type)
803 {
804 switch (nir_alu_type_get_base_type(nir_type)) {
805 case nir_type_int:
806 return UTIL_FORMAT_TYPE_SIGNED;
807
808 case nir_type_uint:
809 return UTIL_FORMAT_TYPE_UNSIGNED;
810
811 case nir_type_float:
812 return UTIL_FORMAT_TYPE_FLOAT;
813
814 default:
815 break;
816 }
817
818 UNREACHABLE("Unsupported nir_alu_type.");
819 }
820
821 static enum pipe_format nir_type_to_pipe_format(nir_alu_type nir_type,
822 unsigned num_components)
823 {
824 enum util_format_type format_type = nir_type_to_util_type(nir_type);
825 unsigned bits = nir_alu_type_get_type_size(nir_type);
826 bool pure_integer = format_type != UTIL_FORMAT_TYPE_FLOAT;
827
828 return util_format_get_array(format_type,
829 bits,
830 num_components,
831 false,
832 pure_integer);
833 }
834
835 static nir_def *lower_image(nir_builder *b, nir_instr *instr, void *cb_data)
836 {
837 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
838 struct state *state = cb_data;
839 pco_data *data = state->data;
840 pco_ctx *ctx = state->ctx;
841 const struct pvr_device_info *dev_info = ctx->dev_info;
842 enum glsl_sampler_dim image_dim = nir_intrinsic_image_dim(intr);
843 bool is_array = nir_intrinsic_image_array(intr);
844 enum pipe_format format = nir_intrinsic_format(intr);
845 unsigned desc_set = nir_src_comp_as_uint(intr->src[0], 0);
846 unsigned binding = nir_src_comp_as_uint(intr->src[0], 1);
847 nir_def *elem = nir_channel(b, intr->src[0].ssa, 2);
848
849 nir_def *lod = NULL;
850 switch (intr->intrinsic) {
851 case nir_intrinsic_image_deref_load:
852 lod = intr->src[3].ssa;
853 break;
854
855 case nir_intrinsic_image_deref_store:
856 lod = intr->src[4].ssa;
857 break;
858
859 case nir_intrinsic_image_deref_size:
860 lod = intr->src[1].ssa;
861 break;
862
863 case nir_intrinsic_image_deref_atomic:
864 case nir_intrinsic_image_deref_atomic_swap:
865 lod = nir_imm_int(b, 0);
866 break;
867
868 default:
869 UNREACHABLE("");
870 }
871
872 if (intr->intrinsic == nir_intrinsic_image_deref_size) {
873 if (image_dim == GLSL_SAMPLER_DIM_BUF) {
874 assert(intr->def.num_components == 1);
875 nir_def *tex_meta = nir_load_tex_meta_pco(b,
876 PCO_IMAGE_META_COUNT,
877 elem,
878 .desc_set = desc_set,
879 .binding = binding);
880
881 return nir_channel(b, tex_meta, PCO_IMAGE_META_BUFFER_ELEMS);
882 }
883
884 nir_def *tex_state = nir_load_tex_state_pco(b,
885 ROGUE_NUM_TEXSTATE_DWORDS,
886 elem,
887 .desc_set = desc_set,
888 .binding = binding);
889
890 nir_def *num_comps = nir_imm_int(b, intr->def.num_components);
891 nir_def *dim = nir_imm_int(b, image_dim);
892 nir_def *is_array_ = nir_imm_bool(b, is_array);
893 nir_def *is_image = nir_imm_bool(b, true);
894 nir_def *size_comps = usclib_tex_state_size(b,
895 tex_state,
896 num_comps,
897 dim,
898 is_array_,
899 is_image,
900 lod);
901
902 data->common.uses.usclib = true;
903
904 return nir_trim_vector(b, size_comps, intr->def.num_components);
905 }
906
907 nir_alu_type type = nir_type_invalid;
908 if (intr->intrinsic == nir_intrinsic_image_deref_load)
909 type = nir_intrinsic_dest_type(intr);
910 else if (intr->intrinsic == nir_intrinsic_image_deref_store)
911 type = nir_intrinsic_src_type(intr);
912
913 bool msaa = image_dim == GLSL_SAMPLER_DIM_MS ||
914 image_dim == GLSL_SAMPLER_DIM_SUBPASS_MS;
915
916 nir_def *coords = intr->src[1].ssa;
917 nir_def *sample_index = msaa ? intr->src[2].ssa : NULL;
918
919 nir_def *write_data = intr->intrinsic == nir_intrinsic_image_deref_store
920 ? intr->src[3].ssa
921 : NULL;
922
923 bool hw_array_support = false;
924 bool hw_int_support =
925 PCO_DEBUG(INT_SMP)
926 ? PVR_HAS_FEATURE(dev_info, tpu_extended_integer_lookup)
927 : false;
928
929 if (write_data) {
930 assert(intr->num_components == 4);
931 assert(write_data->num_components == 4);
932
933 if (format != PIPE_FORMAT_NONE) {
934 const struct util_format_description *desc =
935 util_format_description(format);
936
937 enum pipe_format data_format =
938 nir_type_to_pipe_format(type, desc->nr_channels);
939
940 if (format != data_format) {
941 bool scale = false;
942 bool roundzero = false;
943 bool split = false;
944 enum pco_pck_format pck_format =
945 pco_pipe_to_pck_format(format, &scale, &roundzero, &split);
946
947 if (pck_format != ~0) {
948 if (split) {
949 nir_def *lower =
950 nir_pck_prog_pco(b,
951 nir_channels(b, write_data, 0b0011),
952 nir_imm_int(b, pck_format),
953 .scale = scale,
954 .roundzero = roundzero);
955 nir_def *upper =
956 nir_pck_prog_pco(b,
957 nir_channels(b, write_data, 0b1100),
958 nir_imm_int(b, pck_format),
959 .scale = scale,
960 .roundzero = roundzero);
961
962 write_data = nir_vec4(b,
963 nir_channel(b, lower, 0),
964 nir_channel(b, lower, 1),
965 nir_channel(b, upper, 0),
966 nir_channel(b, upper, 1));
967 } else {
968 write_data = nir_pck_prog_pco(b,
969 write_data,
970 nir_imm_int(b, pck_format),
971 .scale = scale,
972 .roundzero = roundzero);
973 }
974 }
975 }
976 } else {
977 /* Formatless write. */
978 nir_def *tex_meta = nir_load_tex_meta_pco(b,
979 PCO_IMAGE_META_COUNT,
980 elem,
981 .desc_set = desc_set,
982 .binding = binding);
983
984 nir_def *pck_info = nir_channel(b, tex_meta, PCO_IMAGE_META_PCK_INFO);
985 nir_def *pck_format =
986 nir_ubitfield_extract_imm(b,
987 pck_info,
988 PVR_PCK_INFO_FORMAT_OFFSET,
989 PVR_PCK_INFO_FORMAT_LENGTH);
990 nir_def *pck_skip = nir_ieq_imm(b, pck_format, PVR_PCK_FORMAT_INVALID);
991 nir_def *pck_split =
992 nir_ubitfield_extract_imm(b, pck_info, PVR_PCK_INFO_SPLIT_OFFSET, 1);
993 pck_split = nir_ine_imm(b, pck_split, 0);
994 nir_def *pck_scale =
995 nir_ubitfield_extract_imm(b, pck_info, PVR_PCK_INFO_SCALE_OFFSET, 1);
996 pck_scale = nir_ine_imm(b, pck_scale, 0);
997 /* nir_def *pck_roundzero = nir_ubitfield_extract_imm(b, pck_info,
998 * PVR_PCK_INFO_ROUNDZERO_OFFSET,
999 * 1); */
1000 /* pck_roundzero = nir_ine_imm(b, pck_roundzero, 0); */
1001
1002 /* TODO: ideally would like for only 4 regs to be used, since only one
1003 * of these code paths is going to be taken... look into conditional
1004 * execution instead
1005 */
1006 /* TODO: probably nicest to do this in uscgen? */
1007
1008 nir_def *write_data_scale =
1009 nir_pck_prog_pco(b, write_data, pck_format, .scale = true);
1010 nir_def *write_data_noscale =
1011 nir_pck_prog_pco(b, write_data, pck_format, .scale = false);
1012
1013 nir_def *split_lower_scale =
1014 nir_pck_prog_pco(b,
1015 nir_channels(b, write_data, 0b0011),
1016 pck_format,
1017 .scale = true);
1018 nir_def *split_upper_scale =
1019 nir_pck_prog_pco(b,
1020 nir_channels(b, write_data, 0b1100),
1021 pck_format,
1022 .scale = true);
1023 nir_def *write_data_split_scale =
1024 nir_vec4(b,
1025 nir_channel(b, split_lower_scale, 0),
1026 nir_channel(b, split_lower_scale, 1),
1027 nir_channel(b, split_upper_scale, 0),
1028 nir_channel(b, split_upper_scale, 1));
1029
1030 nir_def *split_lower_noscale =
1031 nir_pck_prog_pco(b,
1032 nir_channels(b, write_data, 0b0011),
1033 pck_format,
1034 .scale = false);
1035 nir_def *split_upper_noscale =
1036 nir_pck_prog_pco(b,
1037 nir_channels(b, write_data, 0b1100),
1038 pck_format,
1039 .scale = false);
1040 nir_def *write_data_split_noscale =
1041 nir_vec4(b,
1042 nir_channel(b, split_lower_noscale, 0),
1043 nir_channel(b, split_lower_noscale, 1),
1044 nir_channel(b, split_upper_noscale, 0),
1045 nir_channel(b, split_upper_noscale, 1));
1046
1047 nir_def *write_data_split = nir_bcsel(b,
1048 pck_scale,
1049 write_data_split_scale,
1050 write_data_split_noscale);
1051
1052 nir_def *write_data_unsplit =
1053 nir_bcsel(b, pck_scale, write_data_scale, write_data_noscale);
1054
1055 write_data = nir_bcsel(
1056 b,
1057 pck_skip,
1058 write_data,
1059 nir_bcsel(b, pck_split, write_data_split, write_data_unsplit));
1060 }
1061 }
1062
1063 bool ia = image_dim == GLSL_SAMPLER_DIM_SUBPASS ||
1064 image_dim == GLSL_SAMPLER_DIM_SUBPASS_MS;
1065
1066 if (ia) {
1067 assert(!is_array);
1068 nir_load_const_instr *load =
1069 nir_def_as_load_const(intr->src[0].ssa);
1070 bool onchip = load->def.num_components == 4;
1071
1072 if (onchip) {
1073 unsigned ia_idx = nir_src_comp_as_uint(intr->src[0], 3);
1074 return nir_load_output(b,
1075 intr->def.num_components,
1076 intr->def.bit_size,
1077 nir_imm_int(b, 0),
1078 .base = ia_idx,
1079 .component = 0,
1080 .dest_type = nir_intrinsic_dest_type(intr),
1081 .io_semantics.location = FRAG_RESULT_COLOR,
1082 .io_semantics.num_slots = 1/*,
1083 .io_semantics.fb_fetch_output = true*/);
1084 }
1085 }
1086
1087 nir_def *tex_state = nir_load_tex_state_pco(b,
1088 ROGUE_NUM_TEXSTATE_DWORDS,
1089 elem,
1090 .desc_set = desc_set,
1091 .binding = binding);
1092
1093 unsigned num_coord_comps = nir_image_intrinsic_coord_components(intr);
1094 if (coords)
1095 coords = nir_trim_vector(b, coords, num_coord_comps);
1096
1097 if (intr->intrinsic == nir_intrinsic_image_deref_atomic ||
1098 intr->intrinsic == nir_intrinsic_image_deref_atomic_swap) {
1099 assert(util_format_is_plain(format));
1100 assert(util_format_is_pure_integer(format));
1101
1102 assert(util_format_get_nr_components(format) == 1);
1103 assert(util_format_get_blockwidth(format) == 1);
1104 assert(util_format_get_blockheight(format) == 1);
1105 assert(util_format_get_blockdepth(format) == 1);
1106 assert(util_format_get_blocksize(format) == sizeof(uint32_t));
1107
1108 if (image_dim == GLSL_SAMPLER_DIM_CUBE) {
1109 image_dim = GLSL_SAMPLER_DIM_2D;
1110 is_array = true;
1111 } else if (image_dim == GLSL_SAMPLER_DIM_BUF) {
1112 image_dim = GLSL_SAMPLER_DIM_1D;
1113 }
1114
1115 nir_def *twiddled_offset = NULL;
1116 nir_def *array_index = NULL;
1117 switch (image_dim) {
1118 case GLSL_SAMPLER_DIM_1D: {
1119 twiddled_offset = nir_channel(b, coords, 0);
1120 twiddled_offset =
1121 nir_imul_imm(b, twiddled_offset, util_format_get_blocksize(format));
1122 if (is_array)
1123 array_index = nir_channel(b, coords, 1);
1124 break;
1125 }
1126
1127 case GLSL_SAMPLER_DIM_2D: {
1128 /* Calculate untwiddled offset. */
1129 nir_def *x = nir_i2i16(b, nir_channel(b, coords, 0));
1130 nir_def *y = nir_i2i16(b, nir_channel(b, coords, 1));
1131 twiddled_offset = nir_interleave(b, y, x);
1132 twiddled_offset =
1133 nir_imul_imm(b, twiddled_offset, util_format_get_blocksize(format));
1134
1135 if (is_array)
1136 array_index = nir_channel(b, coords, 2);
1137
1138 break;
1139 }
1140
1141 case GLSL_SAMPLER_DIM_3D: {
1142 assert(!is_array);
1143
1144 /* Calculate untwiddled offset. */
1145 nir_def *num_comps = nir_imm_int(b, 3);
1146 nir_def *dim = nir_imm_int(b, image_dim);
1147 nir_def *_is_array = nir_imm_bool(b, is_array);
1148 nir_def *is_image = nir_imm_bool(b, true);
1149 nir_def *size_comps = usclib_tex_state_size(b,
1150 tex_state,
1151 num_comps,
1152 dim,
1153 _is_array,
1154 is_image,
1155 lod);
1156
1157 twiddled_offset = usclib_twiddle3d(b, coords, size_comps);
1158 data->common.uses.usclib = true;
1159
1160 twiddled_offset =
1161 nir_imul_imm(b, twiddled_offset, util_format_get_blocksize(format));
1162
1163 break;
1164 }
1165
1166 default:
1167 UNREACHABLE("");
1168 }
1169
1170 assert(twiddled_offset);
1171
1172 if (is_array) {
1173 assert(array_index);
1174 nir_def *array_max = usclib_tex_state_array_max(b, tex_state);
1175 array_index = nir_iclamp(b, array_index, nir_imm_int(b, 0), array_max);
1176
1177 nir_def *tex_meta = nir_load_tex_meta_pco(b,
1178 PCO_IMAGE_META_COUNT,
1179 elem,
1180 .desc_set = desc_set,
1181 .binding = binding);
1182
1183 nir_def *array_stride =
1184 nir_channel(b, tex_meta, PCO_IMAGE_META_LAYER_SIZE);
1185
1186 nir_def *array_offset = nir_imul(b, array_index, array_stride);
1187 twiddled_offset = nir_iadd(b, twiddled_offset, array_offset);
1188 }
1189
1190 /* Offset the address by the co-ordinates. */
1191 nir_def *base_addr = usclib_tex_state_address(b, tex_state);
1192 nir_def *base_addr_lo = nir_channel(b, base_addr, 0);
1193 nir_def *base_addr_hi = nir_channel(b, base_addr, 1);
1194
1195 nir_def *addr =
1196 nir_uadd64_32(b, base_addr_lo, base_addr_hi, twiddled_offset);
1197
1198 nir_def *addr_lo = nir_channel(b, addr, 0);
1199 nir_def *addr_hi = nir_channel(b, addr, 1);
1200
1201 if (intr->intrinsic == nir_intrinsic_image_deref_atomic_swap) {
1202 nir_def *compare = intr->src[3].ssa;
1203 nir_def *dma_data = intr->src[4].ssa;
1204
1205 nir_def *addr_data = nir_vec4(b, addr_lo, addr_hi, compare, dma_data);
1206
1207 return nir_global_atomic_swap_pco(b,
1208 addr_data,
1209 .atomic_op =
1210 nir_intrinsic_atomic_op(intr));
1211 }
1212
1213 nir_def *dma_data = intr->src[3].ssa;
1214
1215 nir_def *addr_data = nir_vec3(b, addr_lo, addr_hi, dma_data);
1216
1217 data->common.uses.usclib = true;
1218
1219 return nir_global_atomic_pco(b,
1220 addr_data,
1221 .atomic_op = nir_intrinsic_atomic_op(intr));
1222 }
1223
1224 unsigned smp_desc = ia ? PCO_IA_SAMPLER : PCO_POINT_SAMPLER;
1225 nir_def *smp_state = nir_load_smp_state_pco(b,
1226 ROGUE_NUM_TEXSTATE_DWORDS,
1227 nir_imm_int(b, 0),
1228 .desc_set = smp_desc,
1229 .binding = smp_desc);
1230
1231 /* Special case, override buffers to be 2D. */
1232 if (image_dim == GLSL_SAMPLER_DIM_BUF) {
1233 image_dim = GLSL_SAMPLER_DIM_2D;
1234 coords = nir_vec2(b,
1235 nir_umod_imm(b, coords, 8192),
1236 nir_udiv_imm(b, coords, 8192));
1237 }
1238 /* Special case; lower image cube to arrayed 2d textures. */
1239 else if (image_dim == GLSL_SAMPLER_DIM_CUBE) {
1240 image_dim = GLSL_SAMPLER_DIM_2D;
1241 is_array = true;
1242 } else if (ia) {
1243 nir_variable *pos = nir_get_variable_with_location(b->shader,
1244 nir_var_shader_in,
1245 VARYING_SLOT_POS,
1246 glsl_vec4_type());
1247 pos->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
1248
1249 nir_def *frag_coords = nir_load_input(b,
1250 2,
1251 32,
1252 nir_imm_int(b, 0),
1253 .dest_type = nir_type_float32,
1254 .io_semantics = (nir_io_semantics){
1255 .location = VARYING_SLOT_POS,
1256 .num_slots = 1,
1257 });
1258
1259 frag_coords = nir_f2i32(b, frag_coords);
1260 coords = nir_iadd(b, frag_coords, coords);
1261
1262 nir_def *layer = nir_load_layer_id(b);
1263
1264 /* Use the view index instead if we're in multiview. */
1265 if (data->common.multiview) {
1266 assert(data->fs.view_index_slot >= VARYING_SLOT_VAR0 &&
1267 data->fs.view_index_slot < VARYING_SLOT_MAX);
1268 layer = nir_load_input(b,
1269 1,
1270 32,
1271 nir_imm_int(b, 0),
1272 .dest_type = nir_type_uint32,
1273 .io_semantics = (nir_io_semantics){
1274 .location = data->fs.view_index_slot,
1275 .num_slots = 1,
1276 });
1277
1278 nir_variable *view_index_var =
1279 nir_get_variable_with_location(b->shader,
1280 nir_var_shader_in,
1281 data->fs.view_index_slot,
1282 glsl_uint_type());
1283
1284 view_index_var->data.interpolation = INTERP_MODE_FLAT;
1285 }
1286
1287 coords = nir_pad_vector(b, coords, 3);
1288 coords = nir_vector_insert_imm(b, coords, layer, 2);
1289 is_array = true;
1290 } else if (data->common.image_2d_view_of_3d &&
1291 image_dim == GLSL_SAMPLER_DIM_2D && !is_array) {
1292 image_dim = GLSL_SAMPLER_DIM_3D;
1293
1294 nir_def *tex_meta = nir_load_tex_meta_pco(b,
1295 PCO_IMAGE_META_COUNT,
1296 elem,
1297 .desc_set = desc_set,
1298 .binding = binding);
1299
1300 nir_def *z_slice = nir_channel(b, tex_meta, PCO_IMAGE_META_Z_SLICE);
1301
1302 coords = nir_pad_vector(b, coords, 3);
1303 coords = nir_vector_insert_imm(b, coords, z_slice, 2);
1304 }
1305
1306 nir_def *float_coords;
1307 nir_def *int_coords;
1308 nir_def *float_array_index;
1309 nir_def *int_array_index;
1310 num_coord_comps = process_coords(b,
1311 is_array,
1312 false,
1313 coords,
1314 &float_coords,
1315 &int_coords,
1316 &float_array_index,
1317 &int_array_index);
1318
1319 pco_smp_params params = {
1320 .tex_state = tex_state,
1321 .smp_state = smp_state,
1322
1323 .dest_type = type,
1324
1325 .sampler_dim = image_dim,
1326
1327 .nncoords = !hw_int_support,
1328 .coords = hw_int_support ? int_coords : float_coords,
1329
1330 .ms_index = sample_index,
1331
1332 .write_data = write_data,
1333
1334 .lod_replace = lod,
1335
1336 .sample_components = intr->intrinsic == nir_intrinsic_image_deref_load
1337 ? intr->def.num_components
1338 : 0,
1339 .int_mode = hw_int_support,
1340 };
1341
1342 if (is_array) {
1343 if (hw_array_support) {
1344 params.array_index = int_array_index;
1345 UNREACHABLE("Hardware support for array indexing not implemented");
1346 } else {
1347 nir_def *array_index = int_array_index;
1348 assert(array_index);
1349
1350 nir_def *array_max = usclib_tex_state_array_max(b, tex_state);
1351 array_index = nir_iclamp(b, array_index, nir_imm_int(b, 0), array_max);
1352
1353 nir_def *tex_meta = nir_load_tex_meta_pco(b,
1354 PCO_IMAGE_META_COUNT,
1355 elem,
1356 .desc_set = desc_set,
1357 .binding = binding);
1358
1359 nir_def *array_stride =
1360 nir_channel(b, tex_meta, PCO_IMAGE_META_LAYER_SIZE);
1361
1362 nir_def *array_offset = nir_imul(b, array_index, array_stride);
1363
1364 nir_def *base_addr = usclib_tex_state_address(b, tex_state);
1365 nir_def *base_addr_lo = nir_channel(b, base_addr, 0);
1366 nir_def *base_addr_hi = nir_channel(b, base_addr, 1);
1367
1368 nir_def *addr =
1369 nir_uadd64_32(b, base_addr_lo, base_addr_hi, array_offset);
1370
1371 params.addr_lo = nir_channel(b, addr, 0);
1372 params.addr_hi = nir_channel(b, addr, 1);
1373
1374 data->common.uses.usclib = true;
1375 }
1376 }
1377
1378 nir_intrinsic_instr *smp = pco_emit_nir_smp(b, &params);
1379
1380 if (intr->intrinsic == nir_intrinsic_image_deref_load)
1381 return &smp->def;
1382
1383 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
1384 }
1385
1386 static bool is_image(const nir_instr *instr, UNUSED const void *cb_data)
1387 {
1388 if (instr->type != nir_instr_type_intrinsic)
1389 return false;
1390
1391 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1392 switch (intr->intrinsic) {
1393 case nir_intrinsic_image_deref_load:
1394 case nir_intrinsic_image_deref_store:
1395 case nir_intrinsic_image_deref_atomic:
1396 case nir_intrinsic_image_deref_atomic_swap:
1397 case nir_intrinsic_image_deref_size:
1398 return true;
1399
1400 default:
1401 break;
1402 }
1403
1404 return false;
1405 }
1406
1407 bool pco_nir_lower_images(nir_shader *shader, pco_data *data, pco_ctx *ctx)
1408 {
1409 struct state state = {
1410 .data = data,
1411 .ctx = ctx,
1412 };
1413 return nir_shader_lower_instructions(shader, is_image, lower_image, &state);
1414 }