Debian Codegraph @

src/imagination/pco/pco_trans_nir.c

raw
1 /*
2 * Copyright © 2024 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pco_trans_nir.c
9 *
10 * \brief NIR translation functions.
11 */
12
13 #include "compiler/shader_enums.h"
14 #include "hwdef/rogue_hw_defs.h"
15 #include "pco.h"
16 #include "pco_builder.h"
17 #include "pco_internal.h"
18 #include "util/bitset.h"
19 #include "util/list.h"
20 #include "util/macros.h"
21 #include "util/ralloc.h"
22
23 #include <assert.h>
24 #include <stdio.h>
25
26 /** Translation context. */
27 typedef struct _trans_ctx {
28 pco_ctx *pco_ctx; /** PCO compiler context. */
29 pco_shader *shader; /** Current shader. */
30 pco_func *func; /** Current function. */
31 pco_builder b; /** Builder. */
32 mesa_shader_stage stage; /** Shader stage. */
33 enum pco_cf_node_flag flag; /** Implementation-defined control-flow flag. */
34 bool ftz32b;
35 bool olchk;
36
37 BITSET_WORD *float_types; /** NIR SSA float vars. */
38 BITSET_WORD *int_types; /** NIR SSA int vars. */
39 } trans_ctx;
40
41 /* Forward declarations. */
42 static pco_block *trans_cf_nodes(trans_ctx *tctx,
43 pco_cf_node *parent_cf_node,
44 struct list_head *cf_node_list,
45 struct exec_list *nir_cf_node_list);
46
47 /**
48 * \brief Splits a vector destination into scalar components.
49 *
50 * \param[in,out] tctx Translation context.
51 * \param[in] instr Instruction producing the vector destination.
52 * \param[in] dest Instruction destination.
53 */
54 static void split_dest_comps(trans_ctx *tctx, pco_instr *instr, pco_ref dest)
55 {
56 unsigned chans = pco_ref_get_chans(dest);
57 assert(chans > 1);
58
59 pco_func *func = tctx->func;
60
61 pco_vec_info *vec_info = rzalloc_size(func->vec_infos, sizeof(*vec_info));
62 vec_info->instr = instr;
63 vec_info->comps =
64 rzalloc_array_size(vec_info, sizeof(*vec_info->comps), chans);
65
66 for (unsigned u = 0; u < chans; ++u) {
67 pco_ref comp = pco_ref_new_ssa(func, pco_ref_get_bits(dest), 1);
68 vec_info->comps[u] = pco_comp(&tctx->b, comp, dest, pco_ref_val16(u));
69 }
70
71 _mesa_hash_table_u64_insert(func->vec_infos, dest.val, vec_info);
72 }
73
74 /**
75 * \brief Translates a NIR def into a PCO reference.
76 *
77 * \param[in] def The nir def.
78 * \return The PCO reference.
79 */
80 static inline pco_ref pco_ref_nir_def(const nir_def *def)
81 {
82 return pco_ref_ssa(def->index, def->bit_size, def->num_components);
83 }
84
85 /**
86 * \brief Translates a NIR src into a PCO reference.
87 *
88 * \param[in] src The nir src.
89 * \return The PCO reference.
90 */
91 static inline pco_ref pco_ref_nir_src(const nir_src *src)
92 {
93 return pco_ref_nir_def(src->ssa);
94 }
95
96 /**
97 * \brief Translates a NIR def into a PCO reference with type information.
98 *
99 * \param[in] def The nir def.
100 * \param[in] tctx Translation context.
101 * \return The PCO reference.
102 */
103 static inline pco_ref pco_ref_nir_def_t(const nir_def *def, trans_ctx *tctx)
104 {
105 pco_ref ref = pco_ref_nir_def(def);
106
107 bool is_float = BITSET_TEST(tctx->float_types, def->index);
108 bool is_int = BITSET_TEST(tctx->int_types, def->index);
109
110 if (is_float)
111 ref.dtype = PCO_DTYPE_FLOAT;
112 else if (is_int)
113 ref.dtype = PCO_DTYPE_UNSIGNED;
114
115 return ref;
116 }
117
118 /**
119 * \brief Translates a NIR src into a PCO reference with type information.
120 *
121 * \param[in] src The nir src.
122 * \param[in] tctx Translation context.
123 * \return The PCO reference.
124 */
125 static inline pco_ref pco_ref_nir_src_t(const nir_src *src, trans_ctx *tctx)
126 {
127 return pco_ref_nir_def_t(src->ssa, tctx);
128 }
129
130 /**
131 * \brief Translates a NIR alu src into a PCO reference with type information,
132 * extracting from and/or building new vectors as needed.
133 *
134 * \param[in] src The nir src.
135 * \param[in,out] tctx Translation context.
136 * \return The PCO reference.
137 */
138 static inline pco_ref
139 pco_ref_nir_alu_src_t(const nir_alu_instr *alu, unsigned src, trans_ctx *tctx)
140 {
141 const nir_alu_src *alu_src = &alu->src[src];
142 /* unsigned chans = nir_src_num_components(alu_src->src); */
143 unsigned chans = nir_ssa_alu_instr_src_components(alu, src);
144
145 bool seq_comps =
146 nir_is_sequential_comp_swizzle((uint8_t *)alu_src->swizzle, chans);
147 pco_ref ref = pco_ref_nir_src_t(&alu_src->src, tctx);
148 uint8_t swizzle0 = alu_src->swizzle[0];
149
150 /* Multiple channels, but referencing the entire vector; return as-is. */
151 if (!swizzle0 && seq_comps && chans == nir_src_num_components(alu_src->src))
152 return ref;
153
154 pco_vec_info *vec_info =
155 _mesa_hash_table_u64_search(tctx->func->vec_infos, ref.val);
156 ASSERTED bool replicated_scalar =
157 nir_is_same_comp_swizzle((uint8_t *)alu_src->swizzle, chans) &&
158 !swizzle0 && pco_ref_get_chans(ref) == 1;
159 assert(!!vec_info != replicated_scalar);
160
161 /* One channel; just return its component. */
162 if (chans == 1)
163 return vec_info ? vec_info->comps[swizzle0]->dest[0] : ref;
164
165 /* Multiple channels, either a partial vec and/or swizzling; we need to build
166 * a new vec for this.
167 */
168 pco_ref comps[NIR_MAX_VEC_COMPONENTS] = { 0 };
169 for (unsigned u = 0; u < chans; ++u)
170 comps[u] = vec_info ? vec_info->comps[alu_src->swizzle[u]]->dest[0] : ref;
171
172 pco_ref vec = pco_ref_new_ssa(tctx->func, pco_ref_get_bits(ref), chans);
173 pco_instr *instr = pco_vec(&tctx->b, vec, chans, comps);
174
175 split_dest_comps(tctx, instr, vec);
176
177 return vec;
178 }
179
180 /**
181 * \brief Translates a NIR vs load_input intrinsic into PCO.
182 *
183 * \param[in,out] tctx Translation context.
184 * \param[in] intr load_input intrinsic.
185 * \param[in] dest Instruction destination.
186 * \return The translated PCO instruction.
187 */
188 static pco_instr *
189 trans_load_input_vs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest)
190 {
191 UNUSED unsigned base = nir_intrinsic_base(intr);
192
193 /* TODO: f16 support. */
194
195 ASSERTED const nir_src offset = intr->src[0];
196 assert(nir_src_as_uint(offset) == 0);
197
198 gl_vert_attrib location = nir_intrinsic_io_semantics(intr).location;
199 unsigned component = nir_intrinsic_component(intr);
200 unsigned chans = pco_ref_get_chans(dest);
201
202 const pco_range *range = &tctx->shader->data.vs.attribs[location];
203 assert(component + chans <= range->count);
204
205 pco_ref src =
206 pco_ref_hwreg_vec(range->start + component, PCO_REG_CLASS_VTXIN, chans);
207 return pco_mov(&tctx->b, dest, src, .rpt = chans);
208 }
209
210 /**
211 * \brief Translates a NIR vs store_output intrinsic into PCO.
212 *
213 * \param[in,out] tctx Translation context.
214 * \param[in] intr store_output intrinsic.
215 * \param[in] src Instruction source.
216 * \return The translated PCO instruction.
217 */
218 static pco_instr *
219 trans_store_output_vs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref src)
220 {
221 UNUSED unsigned base = nir_intrinsic_base(intr);
222 ASSERTED nir_alu_type type = nir_intrinsic_src_type(intr);
223 assert(type == nir_type_float32 || type == nir_type_uint32 ||
224 type == nir_type_int32);
225 /* TODO: f16 support. */
226
227 ASSERTED const nir_src offset = intr->src[1];
228 assert(nir_src_as_uint(offset) == 0);
229
230 gl_varying_slot location = nir_intrinsic_io_semantics(intr).location;
231 unsigned component = nir_intrinsic_component(intr);
232 unsigned chans = pco_ref_get_chans(src);
233
234 /* Only contiguous write masks supported. */
235 ASSERTED unsigned write_mask = nir_intrinsic_write_mask(intr);
236 assert(write_mask == BITFIELD_MASK(chans));
237
238 const pco_range *range = &tctx->shader->data.vs.varyings[location];
239 assert(component + chans <= range->count);
240
241 pco_ref vtxout_addr = pco_ref_val8(range->start + component);
242 return pco_uvsw_write(&tctx->b, src, vtxout_addr, .rpt = chans);
243 }
244
245 static pco_instr *trans_uvsw_write(trans_ctx *tctx,
246 nir_intrinsic_instr *intr,
247 UNUSED pco_ref offset_src,
248 pco_ref data_src)
249 {
250 unsigned chans = pco_ref_get_chans(data_src);
251
252 nir_src *noffset_src = &intr->src[0];
253 /* TODO: support non-immediate uvsw variant. */
254 assert(nir_src_is_const(*noffset_src));
255 pco_ref vtxout_addr = pco_ref_val8(nir_src_as_uint(*noffset_src));
256
257 return pco_uvsw_write(&tctx->b, data_src, vtxout_addr, .rpt = chans);
258 }
259
260 static pco_instr *trans_load_reg(trans_ctx *tctx,
261 nir_intrinsic_instr *intr,
262 pco_ref dest,
263 UNUSED pco_ref offset_src,
264 enum pco_reg_class class)
265 {
266 unsigned chans = pco_ref_get_chans(dest);
267
268 nir_src *noffset_src = &intr->src[0];
269 /* TODO: support indexed source offset. */
270 assert(nir_src_is_const(*noffset_src));
271 unsigned offset = nir_src_as_uint(*noffset_src);
272 pco_ref src = pco_ref_hwreg_vec(offset, class, chans);
273
274 return pco_mov(&tctx->b, dest, src, .rpt = chans);
275 }
276
277 static pco_instr *trans_load_tiled_offset(trans_ctx *tctx,
278 nir_intrinsic_instr *intr,
279 pco_ref dest)
280 {
281 unsigned component = nir_intrinsic_component(intr);
282 bool store = !!nir_intrinsic_flags(intr);
283
284 unsigned base0 = store ? PCO_SR_TILED_ST_COMP0 : PCO_SR_TILED_LD_COMP0;
285 unsigned base4 = store ? PCO_SR_TILED_ST_COMP4 : PCO_SR_TILED_LD_COMP4;
286
287 unsigned sr_index = component < 4 ? component + base0
288 : component + base4 - 4;
289
290 pco_ref tiled_offset = pco_ref_hwreg(sr_index, PCO_REG_CLASS_SPEC);
291
292 return pco_mov(&tctx->b, dest, tiled_offset, .olchk = tctx->olchk);
293 }
294
295 static inline pco_instr *build_itr(pco_builder *b,
296 pco_ref dest,
297 enum pco_drc drc,
298 pco_ref coeffs,
299 pco_ref wcoeffs,
300 unsigned _itr_count,
301 enum pco_itr_mode itr_mode,
302 bool d)
303 {
304 pco_func *func = pco_cursor_func(b->cursor);
305 pco_ref itr_count = pco_ref_val16(_itr_count);
306 bool p = !pco_ref_is_null(wcoeffs);
307 pco_instr *instr = pco_instr_create(func, 1, 3 + p);
308
309 instr->op = d ? (p ? PCO_OP_DITRP : PCO_OP_DITR)
310 : (p ? PCO_OP_FITRP : PCO_OP_FITR);
311
312 instr->dest[0] = dest;
313 instr->src[0] = pco_ref_drc(drc);
314 instr->src[1] = coeffs;
315 instr->src[2] = p ? wcoeffs : itr_count;
316 if (p)
317 instr->src[3] = itr_count;
318
319 pco_instr_set_itr_mode(instr, itr_mode);
320
321 pco_builder_insert_instr(b, instr);
322
323 return instr;
324 }
325
326 static pco_ref fs_is_single_sampled(trans_ctx *tctx)
327 {
328 assert(tctx->stage == MESA_SHADER_FRAGMENT);
329
330 /* n samples = ...
331 * 1 = 0b00000001
332 * 2 = 0b00000011
333 * 4 = 0b00001111
334 * 8 = 0b11111111
335 */
336 pco_ref smp_rate_mask = pco_ref_new_ssa32(tctx->func);
337 pco_pck(&tctx->b, smp_rate_mask, pco_fone, .pck_fmt = PCO_PCK_FMT_COV);
338
339 /* n samples = ...
340 * 1 = 0b00000000
341 * 2 = 0b00000001
342 * 4 = 0b00000111
343 * 8 = 0b01111111
344 */
345 pco_ref smp_rate_mask_shr1 = pco_ref_new_ssa32(tctx->func);
346 pco_shift(&tctx->b,
347 smp_rate_mask_shr1,
348 smp_rate_mask,
349 pco_one,
350 pco_ref_null(),
351 .shiftop = PCO_SHIFTOP_SHR);
352
353 pco_ref is_single_sampled = pco_ref_new_ssa32(tctx->func);
354 pco_tstz(&tctx->b,
355 is_single_sampled,
356 pco_ref_null(),
357 smp_rate_mask_shr1,
358 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
359
360 return is_single_sampled;
361 }
362
363 /**
364 * \brief Translates a NIR fs load_input intrinsic into PCO.
365 *
366 * \param[in,out] tctx Translation context.
367 * \param[in] intr load_input intrinsic.
368 * \param[in] dest Instruction destination.
369 * \return The translated PCO instruction.
370 */
371 static pco_instr *trans_load_input_fs(trans_ctx *tctx,
372 nir_intrinsic_instr *intr,
373 pco_ref dest,
374 bool interp)
375 {
376 pco_fs_data *fs_data = &tctx->shader->data.fs;
377 UNUSED unsigned base = nir_intrinsic_base(intr);
378
379 unsigned component = nir_intrinsic_component(intr);
380 unsigned chans = pco_ref_get_chans(dest);
381
382 const nir_src offset = interp ? intr->src[1] : intr->src[0];
383 assert(nir_src_as_uint(offset) == 0);
384
385 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
386 gl_varying_slot location = io_semantics.location;
387
388 nir_variable *var = nir_find_variable_with_location(tctx->shader->nir,
389 nir_var_shader_in,
390 location);
391
392 enum glsl_interp_mode interp_mode = var->data.interpolation;
393
394 /* Setup iteration mode. */
395 enum pco_itr_mode itr_mode = PCO_ITR_MODE_PIXEL;
396 assert(!(var->data.sample && var->data.centroid));
397 if (var->data.sample)
398 itr_mode = PCO_ITR_MODE_SAMPLE;
399 else if (var->data.centroid)
400 itr_mode = PCO_ITR_MODE_CENTROID;
401
402 /* Override if load_input_interpolated. */
403 if (interp) {
404 nir_intrinsic_instr *bary = nir_src_as_intrinsic(intr->src[0]);
405 assert(bary);
406
407 interp_mode = nir_intrinsic_interp_mode(bary);
408
409 switch (bary->intrinsic) {
410 case nir_intrinsic_load_barycentric_pixel:
411 itr_mode = PCO_ITR_MODE_PIXEL;
412 break;
413
414 case nir_intrinsic_load_barycentric_sample:
415 itr_mode = PCO_ITR_MODE_SAMPLE;
416 break;
417
418 case nir_intrinsic_load_barycentric_centroid:
419 itr_mode = PCO_ITR_MODE_CENTROID;
420 break;
421
422 default:
423 UNREACHABLE("");
424 }
425 }
426
427 if (location == VARYING_SLOT_POS) {
428 /* Only scalar supported for now. */
429 /* TODO: support vector for zw. */
430 assert(chans == 1);
431
432 /* TODO: support packing/partial vars. */
433 assert(!var->data.location_frac);
434
435 assert(interp_mode == INTERP_MODE_NOPERSPECTIVE);
436
437 /* Special case: x and y are loaded from special registers. */
438 switch (component) {
439 case 0: /* x */
440 case 1: /* y */ {
441 pco_ref xy_s[] = { pco_ref_hwreg(PCO_SR_X_S, PCO_REG_CLASS_SPEC),
442 pco_ref_hwreg(PCO_SR_Y_S, PCO_REG_CLASS_SPEC) };
443 pco_ref xy_p[] = { pco_ref_hwreg(PCO_SR_X_P, PCO_REG_CLASS_SPEC),
444 pco_ref_hwreg(PCO_SR_Y_P, PCO_REG_CLASS_SPEC) };
445
446 return pco_csel(&tctx->b,
447 dest,
448 fs_is_single_sampled(tctx),
449 xy_p[component],
450 xy_s[component],
451 .tst_op_main = PCO_TST_OP_MAIN_GZERO,
452 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
453 }
454
455 case 2:
456 assert(fs_data->uses.z);
457 component = 0;
458 break;
459
460 case 3:
461 assert(fs_data->uses.w);
462 component = fs_data->uses.z ? 1 : 0;
463 break;
464
465 default:
466 UNREACHABLE("");
467 }
468 }
469
470 const pco_range *range = &fs_data->varyings[location];
471 assert(component + (ROGUE_USC_COEFFICIENT_SET_SIZE * chans) <= range->count);
472
473 unsigned coeffs_index =
474 range->start + (ROGUE_USC_COEFFICIENT_SET_SIZE * component);
475
476 pco_ref coeffs = pco_ref_hwreg_vec(coeffs_index,
477 PCO_REG_CLASS_COEFF,
478 ROGUE_USC_COEFFICIENT_SET_SIZE * chans);
479
480 bool usc_itrsmp_enhanced =
481 PVR_HAS_FEATURE(tctx->pco_ctx->dev_info, usc_itrsmp_enhanced);
482
483 switch (interp_mode) {
484 case INTERP_MODE_SMOOTH: {
485 assert(fs_data->uses.w);
486
487 unsigned wcoeffs_index = fs_data->uses.z ? ROGUE_USC_COEFFICIENT_SET_SIZE
488 : 0;
489
490 pco_ref wcoeffs = pco_ref_hwreg_vec(wcoeffs_index,
491 PCO_REG_CLASS_COEFF,
492 ROGUE_USC_COEFFICIENT_SET_SIZE);
493
494 return build_itr(&tctx->b,
495 dest,
496 PCO_DRC_0,
497 coeffs,
498 wcoeffs,
499 chans,
500 itr_mode,
501 usc_itrsmp_enhanced);
502 }
503
504 case INTERP_MODE_FLAT: {
505 pco_ref coeff_c =
506 pco_ref_hwreg(coeffs_index + ROGUE_USC_COEFFICIENT_SET_C,
507 PCO_REG_CLASS_COEFF);
508
509 assert(chans == 1);
510 return pco_mov(&tctx->b, dest, coeff_c);
511 }
512
513 case INTERP_MODE_NOPERSPECTIVE:
514 return build_itr(&tctx->b,
515 dest,
516 PCO_DRC_0,
517 coeffs,
518 pco_ref_null(),
519 chans,
520 itr_mode,
521 usc_itrsmp_enhanced);
522
523 default:
524 /* Should have been previously lowered. */
525 UNREACHABLE("");
526 }
527 }
528
529 static pco_instr *
530 trans_load_fs_coeffs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest)
531 {
532 pco_fs_data *fs_data = &tctx->shader->data.fs;
533
534 unsigned component = nir_intrinsic_component(intr);
535 unsigned chans = pco_ref_get_chans(dest);
536
537 struct nir_io_semantics io_semantics = nir_intrinsic_io_semantics(intr);
538 gl_varying_slot location = io_semantics.location;
539
540 /* Special case, get z/w coeffs. */
541 if (location == VARYING_SLOT_POS) {
542 if (component == 2) {
543 assert(fs_data->uses.z);
544
545 pco_ref zcoeffs = pco_ref_hwreg_vec(0, PCO_REG_CLASS_COEFF, chans);
546
547 return pco_mov(&tctx->b, dest, zcoeffs, .rpt = chans);
548 } else if (component == 3) {
549 assert(fs_data->uses.w);
550
551 unsigned wcoeffs_index =
552 fs_data->uses.z ? ROGUE_USC_COEFFICIENT_SET_SIZE : 0;
553
554 pco_ref wcoeffs =
555 pco_ref_hwreg_vec(wcoeffs_index, PCO_REG_CLASS_COEFF, chans);
556
557 return pco_mov(&tctx->b, dest, wcoeffs, .rpt = chans);
558 }
559
560 UNREACHABLE("");
561 }
562
563 const pco_range *range = &fs_data->varyings[location];
564 assert(component + chans <= range->count);
565
566 unsigned coeffs_index =
567 range->start + (ROGUE_USC_COEFFICIENT_SET_SIZE * component);
568
569 pco_ref coeffs = pco_ref_hwreg_vec(coeffs_index, PCO_REG_CLASS_COEFF, chans);
570
571 return pco_mov(&tctx->b, dest, coeffs, .rpt = chans);
572 }
573
574 /**
575 * \brief Translates a NIR fs store_output intrinsic into PCO.
576 *
577 * \param[in,out] tctx Translation context.
578 * \param[in] intr store_output intrinsic.
579 * \param[in] src Instruction source.
580 * \return The translated PCO instruction.
581 */
582 static pco_instr *
583 trans_store_output_fs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref src)
584 {
585 pco_fs_data *fs_data = &tctx->shader->data.fs;
586 ASSERTED unsigned base = nir_intrinsic_base(intr);
587 assert(!base);
588
589 assert(pco_ref_is_scalar(src));
590 unsigned component = nir_intrinsic_component(intr);
591
592 ASSERTED const nir_src offset = intr->src[1];
593 assert(nir_src_as_uint(offset) == 0);
594
595 gl_frag_result location = nir_intrinsic_io_semantics(intr).location;
596
597 const pco_range *range = &fs_data->outputs[location];
598 assert(component < range->count);
599
600 unsigned idx = location - FRAG_RESULT_DATA0;
601 bool tile_buffer = fs_data->output_tile_buffers & BITFIELD_BIT(idx);
602 if (!tile_buffer) {
603 pco_ref dest =
604 pco_ref_hwreg(range->start + component, PCO_REG_CLASS_PIXOUT);
605 return pco_mov(&tctx->b, dest, src, .olchk = tctx->olchk);
606 }
607
608 unsigned tile_buffer_id = range->start;
609 pco_range *tile_buffers = &fs_data->tile_buffers;
610 assert(tile_buffer_id < (tile_buffers->count / tile_buffers->stride));
611 unsigned sh_index =
612 tile_buffers->start + tile_buffer_id * tile_buffers->stride;
613
614 pco_ref base_addr[2];
615 pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr);
616
617 pco_ref addr_data_comps[3] = {
618 [2] = src,
619 };
620 pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps);
621
622 component += range->offset;
623 assert(component < 8);
624
625 unsigned sr_index = component < 4 ? component + PCO_SR_TILED_ST_COMP0
626 : component + PCO_SR_TILED_ST_COMP4 - 4;
627 pco_ref tiled_offset = pco_ref_hwreg(sr_index, PCO_REG_CLASS_SPEC);
628
629 pco_add64_32(&tctx->b,
630 addr_data_comps[0],
631 addr_data_comps[1],
632 base_addr[0],
633 base_addr[1],
634 tiled_offset,
635 pco_ref_null(),
636 .olchk = tctx->olchk,
637 .s = true);
638
639 unsigned chans = pco_ref_get_chans(src);
640 pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans);
641 pco_vec(&tctx->b, addr_data, ARRAY_SIZE(addr_data_comps), addr_data_comps);
642
643 pco_ref data_comp =
644 pco_ref_new_ssa(tctx->func, pco_ref_get_bits(src), chans);
645 pco_comp(&tctx->b, data_comp, addr_data, pco_ref_val16(2));
646
647 pco_ref cov_mask_ss = pco_ref_new_ssa32(tctx->func);
648 pco_savmsk(&tctx->b,
649 cov_mask_ss,
650 pco_ref_null(),
651 .savmsk_mode = PCO_SAVMSK_MODE_VM);
652
653 pco_ref cov_mask_ms = pco_ref_new_ssa32(tctx->func);
654 pco_savmsk(&tctx->b,
655 cov_mask_ms,
656 pco_ref_null(),
657 .savmsk_mode = PCO_SAVMSK_MODE_ICM);
658
659 pco_ref cov_mask = pco_ref_new_ssa32(tctx->func);
660 pco_csel(&tctx->b,
661 cov_mask,
662 fs_is_single_sampled(tctx),
663 cov_mask_ss,
664 cov_mask_ms,
665 .tst_op_main = PCO_TST_OP_MAIN_GZERO,
666 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
667
668 return pco_st_tiled(&tctx->b,
669 data_comp,
670 pco_ref_imm8(PCO_DSIZE_32BIT),
671 pco_ref_drc(PCO_DRC_0),
672 pco_ref_imm8(chans),
673 addr_data,
674 cov_mask,
675 .olchk = tctx->olchk);
676 }
677
678 static pco_instr *trans_flush_tile_buffer(trans_ctx *tctx,
679 nir_intrinsic_instr *intr,
680 pco_ref src_addr_lo,
681 pco_ref src_addr_hi)
682 {
683 pco_ref addr_comps[2];
684 pco_ref_new_ssa_addr_comps(tctx->func, addr_comps);
685
686 pco_ref tiled_offset =
687 pco_ref_hwreg(PCO_SR_TILED_LD_COMP0, PCO_REG_CLASS_SPEC);
688
689 pco_add64_32(&tctx->b,
690 addr_comps[0],
691 addr_comps[1],
692 src_addr_lo,
693 src_addr_hi,
694 tiled_offset,
695 pco_ref_null(),
696 .olchk = tctx->olchk,
697 .s = true);
698
699 pco_ref addr = pco_ref_new_ssa_addr(tctx->func);
700 pco_vec(&tctx->b, addr, ARRAY_SIZE(addr_comps), addr_comps);
701
702 unsigned idx_reg_num = 0;
703 pco_ref idx_reg =
704 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
705
706 unsigned base = nir_intrinsic_base(intr);
707 pco_movi32(&tctx->b, idx_reg, pco_ref_imm32(base));
708
709 pco_ref burst_len = pco_ref_new_ssa32(tctx->func);
710 unsigned range = nir_intrinsic_range(intr);
711 assert(range <= 1024);
712 if (range == 1024)
713 range = 0;
714
715 pco_movi32(&tctx->b, burst_len, pco_ref_imm32(range));
716
717 pco_ref dest = pco_ref_hwreg(0, PCO_REG_CLASS_PIXOUT);
718 dest = pco_ref_hwreg_idx_from(idx_reg_num, dest);
719
720 return pco_ld_regbl(&tctx->b, dest, pco_ref_drc(PCO_DRC_0), burst_len, addr);
721 }
722
723 static unsigned fetch_resource_base_reg(const pco_common_data *common,
724 unsigned desc_set,
725 unsigned binding,
726 unsigned elem,
727 unsigned *stride,
728 bool *is_img_smp)
729 {
730 const pco_range *range;
731 if (desc_set == PCO_POINT_SAMPLER && binding == PCO_POINT_SAMPLER) {
732 assert(common->uses.point_sampler);
733 range = &common->point_sampler;
734
735 if (is_img_smp)
736 *is_img_smp = false;
737 } else if (desc_set == PCO_IA_SAMPLER && binding == PCO_IA_SAMPLER) {
738 assert(common->uses.ia_sampler);
739 range = &common->ia_sampler;
740
741 if (is_img_smp)
742 *is_img_smp = false;
743 } else {
744 assert(desc_set < ARRAY_SIZE(common->desc_sets));
745 const pco_descriptor_set_data *desc_set_data =
746 &common->desc_sets[desc_set];
747 assert(desc_set_data->used);
748 assert(desc_set_data->bindings && binding < desc_set_data->binding_count);
749
750 const pco_binding_data *binding_data = &desc_set_data->bindings[binding];
751 assert(binding_data->used);
752
753 range = &binding_data->range;
754
755 if (is_img_smp)
756 *is_img_smp = binding_data->is_img_smp;
757 }
758
759 if (stride)
760 *stride = range->stride;
761
762 unsigned reg_offset = elem * range->stride;
763 assert(reg_offset < range->count);
764
765 unsigned reg_index = range->start + reg_offset;
766 return reg_index;
767 }
768
769 static unsigned fetch_resource_base_reg_packed(const pco_common_data *common,
770 uint32_t packed_desc,
771 unsigned elem,
772 unsigned *stride,
773 bool *is_img_smp)
774 {
775 unsigned desc_set;
776 unsigned binding;
777 pco_unpack_desc(packed_desc, &desc_set, &binding);
778
779 return fetch_resource_base_reg(common,
780 desc_set,
781 binding,
782 elem,
783 stride,
784 is_img_smp);
785 }
786
787 /**
788 * \brief Translates a NIR fs load_output intrinsic into PCO.
789 *
790 * \param[in,out] tctx Translation context.
791 * \param[in] intr load_output intrinsic.
792 * \return The translated PCO instruction.
793 */
794 static pco_instr *
795 trans_load_output_fs(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest)
796 {
797 pco_fs_data *fs_data = &tctx->shader->data.fs;
798 unsigned base = nir_intrinsic_base(intr);
799
800 assert(pco_ref_is_scalar(dest));
801 unsigned component = nir_intrinsic_component(intr);
802
803 ASSERTED const nir_src offset = intr->src[0];
804 assert(nir_src_as_uint(offset) == 0);
805
806 gl_frag_result location = nir_intrinsic_io_semantics(intr).location;
807
808 const pco_range *range;
809 bool tile_buffer;
810 if (location >= FRAG_RESULT_DATA0) {
811 assert(!base);
812 range = &tctx->shader->data.fs.outputs[location];
813 unsigned idx = location - FRAG_RESULT_DATA0;
814 tile_buffer = fs_data->output_tile_buffers & BITFIELD_BIT(idx);
815 } else if (location == FRAG_RESULT_COLOR) {
816 /* Special case for on-chip input attachments. */
817 assert(base < ARRAY_SIZE(tctx->shader->data.fs.ias_onchip));
818 range = &tctx->shader->data.fs.ias_onchip[base];
819 tile_buffer = fs_data->ia_tile_buffers & BITFIELD_BIT(base);
820 } else {
821 UNREACHABLE("");
822 }
823
824 assert(component < range->count);
825
826 if (!tile_buffer) {
827 pco_ref src =
828 pco_ref_hwreg(range->start + component, PCO_REG_CLASS_PIXOUT);
829 return pco_mov(&tctx->b, dest, src, .olchk = tctx->olchk);
830 }
831
832 unsigned tile_buffer_id = range->start;
833 pco_range *tile_buffers = &fs_data->tile_buffers;
834 assert(tile_buffer_id < (tile_buffers->count / tile_buffers->stride));
835 unsigned sh_index =
836 tile_buffers->start + tile_buffer_id * tile_buffers->stride;
837
838 pco_ref base_addr[2];
839 pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr);
840
841 pco_ref addr_comps[2];
842 pco_ref_new_ssa_addr_comps(tctx->func, addr_comps);
843
844 component += range->offset;
845 assert(component < 8);
846
847 unsigned sr_index = component < 4 ? component + PCO_SR_TILED_LD_COMP0
848 : component + PCO_SR_TILED_LD_COMP4 - 4;
849 pco_ref tiled_offset = pco_ref_hwreg(sr_index, PCO_REG_CLASS_SPEC);
850
851 pco_add64_32(&tctx->b,
852 addr_comps[0],
853 addr_comps[1],
854 base_addr[0],
855 base_addr[1],
856 tiled_offset,
857 pco_ref_null(),
858 .olchk = tctx->olchk,
859 .s = true);
860
861 pco_ref addr = pco_ref_new_ssa_addr(tctx->func);
862 pco_vec(&tctx->b, addr, ARRAY_SIZE(addr_comps), addr_comps);
863
864 unsigned chans = pco_ref_get_chans(dest);
865 return pco_ld(&tctx->b,
866 dest,
867 pco_ref_drc(PCO_DRC_0),
868 pco_ref_imm8(chans),
869 addr);
870 }
871
872 static pco_instr *trans_load_common_store(trans_ctx *tctx,
873 nir_intrinsic_instr *intr,
874 pco_ref dest,
875 pco_ref offset_src,
876 bool coeffs,
877 pco_range *range)
878 {
879 nir_src *noffset_src = &intr->src[0];
880 enum pco_reg_class reg_class = coeffs ? PCO_REG_CLASS_COEFF
881 : PCO_REG_CLASS_SHARED;
882
883 unsigned chans = pco_ref_get_chans(dest);
884 ASSERTED unsigned bits = pco_ref_get_bits(dest);
885 assert(bits == 32);
886
887 assert(range->count > 0);
888
889 if (pco_ref_is_null(offset_src) || nir_src_is_const(*noffset_src)) {
890 unsigned offset =
891 pco_ref_is_null(offset_src) ? 0 : nir_src_as_uint(*noffset_src);
892 assert(offset < range->count);
893
894 pco_ref src = pco_ref_hwreg_vec(range->start + offset, reg_class, chans);
895 return pco_mbyp(&tctx->b, dest, src, .rpt = chans);
896 }
897
898 pco_ref src_base = pco_ref_hwreg_vec(range->start, reg_class, chans);
899 return pco_mov_offset(&tctx->b,
900 dest,
901 src_base,
902 offset_src,
903 .offset_sd = PCO_OFFSET_SD_SRC,
904 .rpt = chans);
905 }
906
907 static pco_instr *trans_store_common_store(trans_ctx *tctx,
908 nir_intrinsic_instr *intr,
909 pco_ref data,
910 pco_ref offset_src,
911 bool coeffs,
912 pco_range *range)
913 {
914 nir_src *noffset_src = &intr->src[1];
915 enum pco_reg_class reg_class = coeffs ? PCO_REG_CLASS_COEFF
916 : PCO_REG_CLASS_SHARED;
917
918 unsigned chans = pco_ref_get_chans(data);
919 ASSERTED unsigned bits = pco_ref_get_bits(data);
920 assert(bits == 32);
921
922 assert(range->count > 0);
923
924 if (nir_src_is_const(*noffset_src)) {
925 unsigned offset = nir_src_as_uint(*noffset_src);
926 assert(offset < range->count);
927
928 pco_ref dest = pco_ref_hwreg_vec(range->start + offset, reg_class, chans);
929 return pco_mbyp(&tctx->b, dest, data, .rpt = chans);
930 }
931
932 pco_ref dest_base = pco_ref_hwreg_vec(range->start, reg_class, chans);
933 return pco_mov_offset(&tctx->b,
934 dest_base,
935 data,
936 offset_src,
937 .offset_sd = PCO_OFFSET_SD_DEST,
938 .rpt = chans);
939 }
940
941 static inline enum pco_atom_op to_atom_op(nir_atomic_op op)
942 {
943 switch (op) {
944 case nir_atomic_op_iadd:
945 return PCO_ATOM_OP_ADD;
946
947 case nir_atomic_op_xchg:
948 return PCO_ATOM_OP_XCHG;
949
950 case nir_atomic_op_cmpxchg:
951 return PCO_ATOM_OP_CMPXCHG;
952
953 case nir_atomic_op_umin:
954 return PCO_ATOM_OP_UMIN;
955
956 case nir_atomic_op_imin:
957 return PCO_ATOM_OP_IMIN;
958
959 case nir_atomic_op_umax:
960 return PCO_ATOM_OP_UMAX;
961
962 case nir_atomic_op_imax:
963 return PCO_ATOM_OP_IMAX;
964
965 case nir_atomic_op_iand:
966 return PCO_ATOM_OP_AND;
967
968 case nir_atomic_op_ior:
969 return PCO_ATOM_OP_OR;
970
971 case nir_atomic_op_ixor:
972 return PCO_ATOM_OP_XOR;
973
974 default:
975 break;
976 }
977
978 UNREACHABLE("");
979 }
980
981 static pco_instr *trans_atomic_shared(trans_ctx *tctx,
982 nir_intrinsic_instr *intr,
983 pco_ref dest,
984 pco_ref offset_src,
985 pco_ref value,
986 pco_ref value_swap)
987 {
988 nir_src *noffset_src = &intr->src[0];
989
990 unsigned chans = pco_ref_get_chans(dest);
991 ASSERTED unsigned bits = pco_ref_get_bits(dest);
992
993 assert(bits == 32);
994 assert(chans == 1);
995
996 assert(tctx->shader->data.cs.shmem.count > 0);
997
998 pco_ref shmem_ref = pco_ref_hwreg_vec(tctx->shader->data.cs.shmem.start,
999 PCO_REG_CLASS_COEFF,
1000 chans);
1001
1002 bool const_offset = nir_src_is_const(*noffset_src);
1003 if (const_offset) {
1004 unsigned offset = nir_src_as_uint(*noffset_src);
1005 assert(offset < tctx->shader->data.cs.shmem.count);
1006 shmem_ref = pco_ref_offset(shmem_ref, offset);
1007 } else {
1008 enum pco_atom_op atom_op = to_atom_op(nir_intrinsic_atomic_op(intr));
1009 return pco_op_atomic_offset(&tctx->b,
1010 dest,
1011 shmem_ref,
1012 shmem_ref,
1013 value,
1014 value_swap,
1015 offset_src,
1016 .atom_op = atom_op,
1017 .rpt = chans);
1018 }
1019
1020 pco_instr *instr;
1021 switch (nir_intrinsic_atomic_op(intr)) {
1022 case nir_atomic_op_iadd:
1023 instr = pco_iadd32_atomic(&tctx->b,
1024 dest,
1025 shmem_ref,
1026 shmem_ref,
1027 value,
1028 pco_ref_null(),
1029 .s = true);
1030 break;
1031
1032 case nir_atomic_op_xchg:
1033 instr = pco_xchg_atomic(&tctx->b, dest, shmem_ref, shmem_ref, value);
1034 break;
1035
1036 case nir_atomic_op_umin:
1037 instr = pco_min_atomic(&tctx->b,
1038 dest,
1039 shmem_ref,
1040 shmem_ref,
1041 value,
1042 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
1043 break;
1044
1045 case nir_atomic_op_imin:
1046 instr = pco_min_atomic(&tctx->b,
1047 dest,
1048 shmem_ref,
1049 shmem_ref,
1050 value,
1051 .tst_type_main = PCO_TST_TYPE_MAIN_S32);
1052 break;
1053
1054 case nir_atomic_op_umax:
1055 instr = pco_max_atomic(&tctx->b,
1056 dest,
1057 shmem_ref,
1058 shmem_ref,
1059 value,
1060 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
1061 break;
1062
1063 case nir_atomic_op_imax:
1064 instr = pco_max_atomic(&tctx->b,
1065 dest,
1066 shmem_ref,
1067 shmem_ref,
1068 value,
1069 .tst_type_main = PCO_TST_TYPE_MAIN_S32);
1070 break;
1071
1072 case nir_atomic_op_iand:
1073 instr = pco_logical_atomic(&tctx->b,
1074 dest,
1075 shmem_ref,
1076 shmem_ref,
1077 value,
1078 .logiop = PCO_LOGIOP_AND);
1079 break;
1080
1081 case nir_atomic_op_ior:
1082 instr = pco_logical_atomic(&tctx->b,
1083 dest,
1084 shmem_ref,
1085 shmem_ref,
1086 value,
1087 .logiop = PCO_LOGIOP_OR);
1088 break;
1089
1090 case nir_atomic_op_ixor:
1091 instr = pco_logical_atomic(&tctx->b,
1092 dest,
1093 shmem_ref,
1094 shmem_ref,
1095 value,
1096 .logiop = PCO_LOGIOP_XOR);
1097 break;
1098
1099 case nir_atomic_op_cmpxchg:
1100 instr = pco_cmpxchg_atomic(&tctx->b,
1101 dest,
1102 shmem_ref,
1103 shmem_ref,
1104 value,
1105 value_swap,
1106 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
1107 break;
1108
1109 default:
1110 UNREACHABLE("");
1111 }
1112
1113 pco_instr_set_rpt(instr, chans);
1114
1115 return instr;
1116 }
1117
1118 static pco_instr *trans_load_buffer(trans_ctx *tctx,
1119 nir_intrinsic_instr *intr,
1120 pco_ref dest,
1121 pco_ref offset_src)
1122 {
1123 const pco_common_data *common = &tctx->shader->data.common;
1124
1125 unsigned chans = pco_ref_get_chans(dest);
1126 ASSERTED unsigned bits = pco_ref_get_bits(dest);
1127 assert(bits == 32);
1128
1129 uint32_t packed_desc = nir_src_comp_as_uint(intr->src[0], 0);
1130 unsigned elem = nir_src_comp_as_uint(intr->src[0], 1);
1131 unsigned sh_index =
1132 fetch_resource_base_reg_packed(common, packed_desc, elem, NULL, NULL);
1133
1134 pco_ref base_addr[2];
1135 pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr);
1136
1137 pco_ref addr_comps[2];
1138 pco_ref_new_ssa_addr_comps(tctx->func, addr_comps);
1139
1140 pco_add64_32(&tctx->b,
1141 addr_comps[0],
1142 addr_comps[1],
1143 base_addr[0],
1144 base_addr[1],
1145 offset_src,
1146 pco_ref_null(),
1147 .s = true);
1148
1149 pco_ref addr_comps_dyn_off[2];
1150 pco_ref_new_ssa_addr_comps(tctx->func, addr_comps_dyn_off);
1151
1152 pco_ref dyn_off_reg = pco_ref_hwreg(sh_index, PCO_REG_CLASS_SHARED);
1153 dyn_off_reg = pco_ref_offset(dyn_off_reg, 3);
1154
1155 pco_add64_32(&tctx->b,
1156 addr_comps_dyn_off[0],
1157 addr_comps_dyn_off[1],
1158 addr_comps[0],
1159 addr_comps[1],
1160 dyn_off_reg,
1161 pco_ref_null(),
1162 .s = true);
1163
1164 pco_ref addr = pco_ref_new_ssa_addr(tctx->func);
1165 pco_vec(&tctx->b, addr, ARRAY_SIZE(addr_comps_dyn_off), addr_comps_dyn_off);
1166
1167 return pco_ld(&tctx->b,
1168 dest,
1169 pco_ref_drc(PCO_DRC_0),
1170 pco_ref_imm8(chans),
1171 addr);
1172 }
1173
1174 static pco_instr *trans_load_global(trans_ctx *tctx,
1175 nir_intrinsic_instr *intr,
1176 pco_ref dest,
1177 pco_ref addr)
1178 {
1179 unsigned chans = pco_ref_get_chans(dest);
1180 ASSERTED unsigned bits = pco_ref_get_bits(dest);
1181 assert(bits == 32);
1182
1183 return pco_ld(&tctx->b,
1184 dest,
1185 pco_ref_drc(PCO_DRC_0),
1186 pco_ref_imm8(chans),
1187 addr);
1188 }
1189
1190 static pco_instr *
1191 trans_store_global(trans_ctx *tctx, pco_ref data_src, pco_ref addr_src)
1192 {
1193 unsigned chans = pco_ref_get_chans(data_src);
1194 unsigned bits = pco_ref_get_bits(data_src);
1195
1196 pco_ref addr_data_comps[3] = {
1197 [2] = data_src,
1198 };
1199
1200 pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps);
1201 pco_comp(&tctx->b, addr_data_comps[0], addr_src, pco_ref_val16(0));
1202 pco_comp(&tctx->b, addr_data_comps[1], addr_src, pco_ref_val16(1));
1203
1204 pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans);
1205 pco_vec(&tctx->b, addr_data, ARRAY_SIZE(addr_data_comps), addr_data_comps);
1206
1207 pco_ref data_comp = pco_ref_new_ssa(tctx->func, bits, chans);
1208 pco_comp(&tctx->b, data_comp, addr_data, pco_ref_val16(2));
1209
1210 switch (bits) {
1211 case 32:
1212 return pco_st32(&tctx->b,
1213 data_comp,
1214 pco_ref_drc(PCO_DRC_0),
1215 pco_ref_imm8(chans),
1216 addr_data,
1217 pco_ref_null());
1218
1219 default:
1220 break;
1221 }
1222
1223 UNREACHABLE("");
1224 }
1225
1226 static pco_instr *
1227 trans_get_buffer_size(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest)
1228 {
1229 const pco_common_data *common = &tctx->shader->data.common;
1230
1231 ASSERTED unsigned chans = pco_ref_get_chans(dest);
1232 ASSERTED unsigned bits = pco_ref_get_bits(dest);
1233
1234 assert(chans == 1);
1235 assert(bits == 32);
1236
1237 uint32_t packed_desc = nir_src_comp_as_uint(intr->src[0], 0);
1238 unsigned elem = nir_src_comp_as_uint(intr->src[0], 1);
1239 unsigned sh_index =
1240 fetch_resource_base_reg_packed(common, packed_desc, elem, NULL, NULL);
1241
1242 pco_ref size_reg = pco_ref_hwreg(sh_index, PCO_REG_CLASS_SHARED);
1243 size_reg = pco_ref_offset(size_reg, 2);
1244
1245 return pco_mov(&tctx->b, dest, size_reg);
1246 }
1247
1248 static pco_instr *trans_store_buffer(trans_ctx *tctx,
1249 nir_intrinsic_instr *intr,
1250 pco_ref data_src,
1251 pco_ref offset_src)
1252 {
1253 const pco_common_data *common = &tctx->shader->data.common;
1254
1255 unsigned chans = pco_ref_get_chans(data_src);
1256 unsigned bits = pco_ref_get_bits(data_src);
1257
1258 uint32_t packed_desc = nir_src_comp_as_uint(intr->src[1], 0);
1259 unsigned elem = nir_src_comp_as_uint(intr->src[1], 1);
1260 unsigned sh_index =
1261 fetch_resource_base_reg_packed(common, packed_desc, elem, NULL, NULL);
1262
1263 pco_ref base_addr[2];
1264 pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr);
1265
1266 pco_ref addr_comps[2];
1267 pco_ref_new_ssa_addr_comps(tctx->func, addr_comps);
1268
1269 pco_add64_32(&tctx->b,
1270 addr_comps[0],
1271 addr_comps[1],
1272 base_addr[0],
1273 base_addr[1],
1274 offset_src,
1275 pco_ref_null(),
1276 .s = true);
1277
1278 pco_ref addr_data_comps_dyn_off[3] = {
1279 [2] = data_src,
1280 };
1281 pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps_dyn_off);
1282
1283 pco_ref dyn_off_reg = pco_ref_hwreg(sh_index, PCO_REG_CLASS_SHARED);
1284 dyn_off_reg = pco_ref_offset(dyn_off_reg, 3);
1285
1286 pco_add64_32(&tctx->b,
1287 addr_data_comps_dyn_off[0],
1288 addr_data_comps_dyn_off[1],
1289 addr_comps[0],
1290 addr_comps[1],
1291 dyn_off_reg,
1292 pco_ref_null(),
1293 .s = true);
1294
1295 pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans);
1296 pco_vec(&tctx->b,
1297 addr_data,
1298 ARRAY_SIZE(addr_data_comps_dyn_off),
1299 addr_data_comps_dyn_off);
1300
1301 pco_ref data_comp = pco_ref_new_ssa(tctx->func,
1302 pco_ref_get_bits(data_src),
1303 pco_ref_get_chans(data_src));
1304 pco_comp(&tctx->b, data_comp, addr_data, pco_ref_val16(2));
1305
1306 switch (bits) {
1307 case 32:
1308 return pco_st32(&tctx->b,
1309 data_comp,
1310 pco_ref_drc(PCO_DRC_0),
1311 pco_ref_imm8(chans),
1312 addr_data,
1313 pco_ref_null());
1314
1315 default:
1316 break;
1317 }
1318
1319 UNREACHABLE("");
1320 }
1321
1322 static pco_instr *trans_atomic_buffer(trans_ctx *tctx,
1323 nir_intrinsic_instr *intr,
1324 pco_ref dest,
1325 pco_ref offset_src,
1326 pco_ref data_src)
1327 {
1328 const pco_common_data *common = &tctx->shader->data.common;
1329
1330 enum pco_atom_op atom_op = to_atom_op(nir_intrinsic_atomic_op(intr));
1331 /* Should have been lowered. */
1332 assert(atom_op != PCO_ATOM_OP_CMPXCHG);
1333
1334 unsigned chans = pco_ref_get_chans(dest);
1335 unsigned bits = pco_ref_get_bits(dest);
1336
1337 uint32_t packed_desc = nir_src_comp_as_uint(intr->src[0], 0);
1338 unsigned elem = nir_src_comp_as_uint(intr->src[0], 1);
1339 unsigned sh_index =
1340 fetch_resource_base_reg_packed(common, packed_desc, elem, NULL, NULL);
1341
1342 pco_ref base_addr[2];
1343 pco_ref_hwreg_addr_comps(sh_index, PCO_REG_CLASS_SHARED, base_addr);
1344
1345 pco_ref addr_comps[2];
1346 pco_ref_new_ssa_addr_comps(tctx->func, addr_comps);
1347
1348 pco_add64_32(&tctx->b,
1349 addr_comps[0],
1350 addr_comps[1],
1351 base_addr[0],
1352 base_addr[1],
1353 offset_src,
1354 pco_ref_null(),
1355 .s = true);
1356
1357 pco_ref addr_data_comps_dyn_off[3] = {
1358 [2] = data_src,
1359 };
1360 pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps_dyn_off);
1361
1362 pco_ref dyn_off_reg = pco_ref_hwreg(sh_index, PCO_REG_CLASS_SHARED);
1363 dyn_off_reg = pco_ref_offset(dyn_off_reg, 3);
1364
1365 pco_add64_32(&tctx->b,
1366 addr_data_comps_dyn_off[0],
1367 addr_data_comps_dyn_off[1],
1368 addr_comps[0],
1369 addr_comps[1],
1370 dyn_off_reg,
1371 pco_ref_null(),
1372 .s = true);
1373
1374 pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans);
1375 pco_vec(&tctx->b,
1376 addr_data,
1377 ARRAY_SIZE(addr_data_comps_dyn_off),
1378 addr_data_comps_dyn_off);
1379
1380 switch (bits) {
1381 case 32:
1382 return pco_atomic(&tctx->b,
1383 dest,
1384 pco_ref_drc(PCO_DRC_0),
1385 addr_data,
1386 .atom_op = atom_op);
1387
1388 default:
1389 break;
1390 }
1391
1392 UNREACHABLE("");
1393 }
1394
1395 static pco_instr *trans_global_atomic_buffer(trans_ctx *tctx,
1396 nir_intrinsic_instr *intr,
1397 pco_ref dest,
1398 pco_ref addr_data)
1399 {
1400 enum pco_atom_op atom_op = to_atom_op(nir_intrinsic_atomic_op(intr));
1401 /* Should have been lowered. */
1402 assert(atom_op != PCO_ATOM_OP_CMPXCHG);
1403
1404 ASSERTED unsigned chans = pco_ref_get_chans(dest);
1405 unsigned bits = pco_ref_get_bits(dest);
1406
1407 assert(chans == 1);
1408
1409 switch (bits) {
1410 case 32:
1411 return pco_atomic(&tctx->b,
1412 dest,
1413 pco_ref_drc(PCO_DRC_0),
1414 addr_data,
1415 .atom_op = atom_op);
1416
1417 default:
1418 break;
1419 }
1420
1421 UNREACHABLE("");
1422 }
1423
1424 static pco_instr *trans_scratch(trans_ctx *tctx,
1425 pco_ref dest,
1426 pco_ref offset_src,
1427 pco_ref data_src)
1428 {
1429 const pco_common_data *common = &tctx->shader->data.common;
1430
1431 assert(common->scratch_info.count > 0);
1432 unsigned base_addr_idx = common->scratch_info.start;
1433 unsigned block_size_idx = common->scratch_info.start + 2;
1434
1435 pco_ref base_addr[2];
1436 pco_ref_hwreg_addr_comps(base_addr_idx, PCO_REG_CLASS_SHARED, base_addr);
1437
1438 pco_ref block_size = pco_ref_hwreg(block_size_idx, PCO_REG_CLASS_SHARED);
1439 pco_ref local_addr_inst_num =
1440 pco_ref_hwreg(PCO_SR_LOCAL_ADDR_INST_NUM, PCO_REG_CLASS_SPEC);
1441
1442 pco_ref inst_base_addr[2];
1443 pco_ref_new_ssa_addr_comps(tctx->func, inst_base_addr);
1444
1445 pco_imadd64(&tctx->b,
1446 inst_base_addr[0],
1447 inst_base_addr[1],
1448 block_size,
1449 local_addr_inst_num,
1450 base_addr[0],
1451 base_addr[1],
1452 pco_ref_null());
1453
1454 pco_ref addr_data_comps[3];
1455 pco_ref_new_ssa_addr_comps(tctx->func, addr_data_comps);
1456
1457 pco_add64_32(&tctx->b,
1458 addr_data_comps[0],
1459 addr_data_comps[1],
1460 inst_base_addr[0],
1461 inst_base_addr[1],
1462 offset_src,
1463 pco_ref_null(),
1464 .s = true);
1465
1466 bool is_load = pco_ref_is_null(data_src);
1467 if (is_load) {
1468 ASSERTED unsigned bits = pco_ref_get_bits(dest);
1469
1470 /* TODO: 8/16-bit support via masking. */
1471 assert(bits == 32);
1472
1473 pco_ref addr = pco_ref_new_ssa_addr(tctx->func);
1474 pco_vec(&tctx->b, addr, 2, addr_data_comps);
1475
1476 unsigned chans = pco_ref_get_chans(dest);
1477 return pco_ld(&tctx->b,
1478 dest,
1479 pco_ref_drc(PCO_DRC_0),
1480 pco_ref_imm8(chans),
1481 addr);
1482 }
1483
1484 unsigned chans = pco_ref_get_chans(data_src);
1485 pco_ref addr_data = pco_ref_new_ssa_addr_data(tctx->func, chans);
1486
1487 addr_data_comps[2] = data_src;
1488 pco_vec(&tctx->b, addr_data, ARRAY_SIZE(addr_data_comps), addr_data_comps);
1489
1490 pco_ref data_comp = pco_ref_new_ssa(tctx->func,
1491 pco_ref_get_bits(data_src),
1492 pco_ref_get_chans(data_src));
1493 pco_comp(&tctx->b, data_comp, addr_data, pco_ref_val16(2));
1494
1495 return pco_st32(&tctx->b,
1496 data_comp,
1497 pco_ref_drc(PCO_DRC_0),
1498 pco_ref_imm8(chans),
1499 addr_data,
1500 pco_ref_null());
1501 }
1502
1503 static inline enum pco_reg_class sys_val_to_reg_class(gl_system_value sys_val,
1504 mesa_shader_stage stage)
1505 {
1506 switch (stage) {
1507 case MESA_SHADER_FRAGMENT:
1508 return PCO_REG_CLASS_SHARED;
1509
1510 case MESA_SHADER_VERTEX:
1511 return PCO_REG_CLASS_VTXIN;
1512
1513 case MESA_SHADER_COMPUTE:
1514 switch (sys_val) {
1515 case SYSTEM_VALUE_LOCAL_INVOCATION_INDEX:
1516 return PCO_REG_CLASS_VTXIN;
1517
1518 case SYSTEM_VALUE_WORKGROUP_ID:
1519 case SYSTEM_VALUE_NUM_WORKGROUPS:
1520 return PCO_REG_CLASS_COEFF;
1521
1522 default:
1523 break;
1524 }
1525 break;
1526
1527 default:
1528 break;
1529 }
1530
1531 UNREACHABLE("");
1532 }
1533
1534 /**
1535 * \brief Translates a NIR load system value intrinsic into PCO.
1536 *
1537 * \param[in,out] tctx Translation context.
1538 * \param[in] intr System value intrinsic.
1539 * \param[in] dest Instruction destination.
1540 * \return The translated PCO instruction.
1541 */
1542 static pco_instr *
1543 trans_load_sysval(trans_ctx *tctx, nir_intrinsic_instr *intr, pco_ref dest)
1544 {
1545 gl_system_value sys_val;
1546 /* Overrides. */
1547 switch (intr->intrinsic) {
1548 case nir_intrinsic_load_front_face_op_pco:
1549 sys_val = SYSTEM_VALUE_FRONT_FACE;
1550 break;
1551
1552 default:
1553 sys_val = nir_system_value_from_intrinsic(intr->intrinsic);
1554 break;
1555 }
1556
1557 const pco_range *range = &tctx->shader->data.common.sys_vals[sys_val];
1558
1559 unsigned chans = pco_ref_get_chans(dest);
1560 assert(chans == range->count);
1561
1562 pco_ref src = pco_ref_hwreg_vec(range->start,
1563 sys_val_to_reg_class(sys_val, tctx->stage),
1564 chans);
1565 return pco_mov(&tctx->b, dest, src, .rpt = chans);
1566 }
1567
1568 static pco_instr *trans_load_packed_sample_location(trans_ctx *tctx,
1569 nir_intrinsic_instr *intr,
1570 pco_ref dest,
1571 pco_ref offset_src)
1572 {
1573 unsigned idx_reg_num = 0;
1574 pco_ref idx_reg =
1575 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
1576
1577 pco_mov(&tctx->b, idx_reg, offset_src);
1578
1579 const pco_range *range = &tctx->shader->data.fs.sample_locations;
1580 assert(range->count > 0);
1581
1582 /* TODO: add the start onto the offset src instead? */
1583 assert(range->start < 256);
1584
1585 pco_ref src = pco_ref_hwreg(range->start, PCO_REG_CLASS_SHARED);
1586 src = pco_ref_hwreg_idx_from(idx_reg_num, src);
1587
1588 return pco_mov(&tctx->b, dest, src);
1589 }
1590
1591 static bool desc_set_binding_is_comb_img_smp(unsigned desc_set,
1592 unsigned binding,
1593 const pco_common_data *common)
1594 {
1595 const pco_descriptor_set_data *desc_set_data = &common->desc_sets[desc_set];
1596 assert(desc_set_data->used);
1597 assert(desc_set_data->bindings && binding < desc_set_data->binding_count);
1598
1599 const pco_binding_data *binding_data = &desc_set_data->bindings[binding];
1600 assert(binding_data->used);
1601
1602 return binding_data->is_img_smp;
1603 }
1604
1605 static pco_instr *lower_load_tex_smp_state(trans_ctx *tctx,
1606 nir_intrinsic_instr *intr,
1607 pco_ref dest,
1608 bool smp)
1609 {
1610 unsigned desc_set = nir_intrinsic_desc_set(intr);
1611 unsigned binding = nir_intrinsic_binding(intr);
1612 unsigned start_comp = nir_intrinsic_component(intr);
1613 unsigned chans = pco_ref_get_chans(dest);
1614 assert(start_comp + chans <= ROGUE_NUM_TEXSTATE_DWORDS);
1615 unsigned elem = nir_src_as_uint(intr->src[0]);
1616
1617 const pco_common_data *common = &tctx->shader->data.common;
1618 bool is_img_smp;
1619 unsigned sh_index = fetch_resource_base_reg(common,
1620 desc_set,
1621 binding,
1622 elem,
1623 NULL,
1624 &is_img_smp);
1625 pco_ref state_words =
1626 pco_ref_hwreg_vec(sh_index, PCO_REG_CLASS_SHARED, chans);
1627
1628 /* Sampler state comes after image state and metadata in combined image
1629 * samplers.
1630 */
1631 if (smp && is_img_smp) {
1632 state_words = pco_ref_offset(state_words, ROGUE_NUM_TEXSTATE_DWORDS);
1633 state_words = pco_ref_offset(state_words, PCO_IMAGE_META_COUNT);
1634 }
1635
1636 /* Gather sampler words come after standard sampler state and metadata. */
1637 if (smp && nir_intrinsic_flags(intr)) {
1638 state_words = pco_ref_offset(state_words, ROGUE_NUM_TEXSTATE_DWORDS);
1639 state_words = pco_ref_offset(state_words, PCO_SAMPLER_META_COUNT);
1640 }
1641
1642 state_words = pco_ref_offset(state_words, start_comp);
1643
1644 return pco_mov(&tctx->b, dest, state_words, .rpt = chans);
1645 }
1646
1647 static pco_instr *lower_load_tex_smp_meta(trans_ctx *tctx,
1648 nir_intrinsic_instr *intr,
1649 pco_ref dest,
1650 bool smp)
1651 {
1652 unsigned desc_set = nir_intrinsic_desc_set(intr);
1653 unsigned binding = nir_intrinsic_binding(intr);
1654 unsigned start_comp = nir_intrinsic_component(intr);
1655 unsigned chans = pco_ref_get_chans(dest);
1656 unsigned elem = nir_src_as_uint(intr->src[0]);
1657
1658 const pco_common_data *common = &tctx->shader->data.common;
1659 bool is_img_smp;
1660 unsigned sh_index = fetch_resource_base_reg(common,
1661 desc_set,
1662 binding,
1663 elem,
1664 NULL,
1665 &is_img_smp);
1666 pco_ref state_words =
1667 pco_ref_hwreg_vec(sh_index, PCO_REG_CLASS_SHARED, chans);
1668
1669 assert(start_comp + chans <=
1670 (smp ? PCO_SAMPLER_META_COUNT : PCO_IMAGE_META_COUNT));
1671
1672 state_words = pco_ref_offset(state_words, ROGUE_NUM_TEXSTATE_DWORDS);
1673
1674 if (smp && is_img_smp) {
1675 state_words = pco_ref_offset(state_words, PCO_IMAGE_META_COUNT);
1676 state_words = pco_ref_offset(state_words, ROGUE_NUM_TEXSTATE_DWORDS);
1677 }
1678
1679 state_words = pco_ref_offset(state_words, start_comp);
1680
1681 return pco_mov(&tctx->b, dest, state_words, .rpt = chans);
1682 }
1683
1684 static pco_instr *lower_smp(trans_ctx *tctx,
1685 nir_intrinsic_instr *intr,
1686 pco_ref *dest,
1687 pco_ref data,
1688 pco_ref tex_state,
1689 pco_ref smp_state)
1690 {
1691 pco_smp_flags smp_flags = { ._ = nir_intrinsic_smp_flags_pco(intr) };
1692 unsigned data_comps = nir_intrinsic_range(intr);
1693
1694 data = pco_ref_chans(data, data_comps);
1695
1696 unsigned chans = pco_ref_get_chans(*dest);
1697 enum pco_sb_mode sb_mode = PCO_SB_MODE_NONE;
1698 switch (intr->intrinsic) {
1699 case nir_intrinsic_smp_coeffs_pco:
1700 /* Shrink the destination to its actual size.
1701 * Trilinear filtering will produce two sets of coeffs;
1702 * reserve both just in case so that we don't clobber output regs.
1703 */
1704 *dest = pco_ref_chans(*dest, ROGUE_SMP_COEFF_COUNT * 2u);
1705 chans = 1; /* Chans must be 1 for coeff mode. */
1706
1707 sb_mode = PCO_SB_MODE_COEFFS;
1708 break;
1709
1710 case nir_intrinsic_smp_raw_pco:
1711 chans = 4;
1712 sb_mode = PCO_SB_MODE_RAWDATA;
1713 break;
1714
1715 case nir_intrinsic_smp_pco:
1716 /* Destination and chans should be correct. */
1717 break;
1718
1719 case nir_intrinsic_smp_write_pco:
1720 chans = 4;
1721 break;
1722
1723 default:
1724 UNREACHABLE("");
1725 }
1726
1727 pco_ref shared_lod = pco_ref_null();
1728
1729 pco_instr *smp = pco_smp(&tctx->b,
1730 *dest,
1731 pco_ref_drc(PCO_DRC_0),
1732 tex_state,
1733 data,
1734 smp_state,
1735 shared_lod,
1736 pco_ref_imm8(chans),
1737 .dim = smp_flags.dim,
1738 .proj = smp_flags.proj,
1739 .fcnorm = smp_flags.fcnorm,
1740 .nncoords = smp_flags.nncoords,
1741 .lod_mode = smp_flags.lod_mode,
1742 .pplod = smp_flags.pplod,
1743 .tao = smp_flags.tao,
1744 .soo = smp_flags.soo,
1745 .sno = smp_flags.sno,
1746 .sb_mode = sb_mode,
1747 .array = smp_flags.array,
1748 .integer = smp_flags.integer,
1749 .wrt = smp_flags.wrt);
1750
1751 return smp;
1752 }
1753
1754 static pco_instr *lower_alphatst(trans_ctx *tctx,
1755 pco_ref dest,
1756 pco_ref src0,
1757 pco_ref src1,
1758 pco_ref src2)
1759 {
1760 pco_alphatst(&tctx->b,
1761 pco_ref_pred(PCO_PRED_P0),
1762 pco_ref_drc(PCO_DRC_0),
1763 src0,
1764 src1,
1765 src2);
1766
1767 return pco_psel(&tctx->b,
1768 dest,
1769 pco_ref_pred(PCO_PRED_P0),
1770 pco_fone,
1771 pco_zero);
1772 }
1773
1774 static inline unsigned lookup_reg_bits(nir_intrinsic_instr *intr)
1775 {
1776 nir_def *reg;
1777 switch (intr->intrinsic) {
1778 case nir_intrinsic_load_reg:
1779 reg = intr->src[0].ssa;
1780 break;
1781
1782 case nir_intrinsic_store_reg:
1783 reg = intr->src[1].ssa;
1784 break;
1785
1786 default:
1787 UNREACHABLE("");
1788 }
1789
1790 nir_intrinsic_instr *reg_decl = nir_reg_get_decl(reg);
1791 return nir_intrinsic_bit_size(reg_decl);
1792 }
1793
1794 static pco_instr *trans_reg_intr(trans_ctx *tctx,
1795 nir_intrinsic_instr *intr,
1796 pco_ref dest,
1797 pco_ref src0,
1798 pco_ref src1)
1799 {
1800 pco_func *func = tctx->func;
1801
1802 /* Special case; consume and reserve. */
1803 if (intr->intrinsic == nir_intrinsic_decl_reg) {
1804 assert(nir_intrinsic_num_components(intr) == 1);
1805 assert(!nir_intrinsic_num_array_elems(intr));
1806
1807 /* nir_trivialize_registers puts decl_regs into the start block. */
1808 assert(func->next_vreg == dest.val);
1809 ++func->next_vreg;
1810
1811 return NULL;
1812 }
1813
1814 unsigned reg_bits = lookup_reg_bits(intr);
1815 switch (intr->intrinsic) {
1816 case nir_intrinsic_load_reg:
1817 assert(!nir_intrinsic_base(intr));
1818 assert(!nir_intrinsic_legacy_fabs(intr));
1819 assert(!nir_intrinsic_legacy_fneg(intr));
1820
1821 return pco_mbyp(&tctx->b, dest, pco_ref_ssa_vreg(func, src0, reg_bits));
1822
1823 case nir_intrinsic_store_reg:
1824 assert(!nir_intrinsic_base(intr));
1825 assert(nir_intrinsic_write_mask(intr) == 1);
1826 assert(!nir_intrinsic_legacy_fsat(intr));
1827
1828 return pco_mbyp(&tctx->b, pco_ref_ssa_vreg(func, src1, reg_bits), src0);
1829
1830 default:
1831 break;
1832 }
1833
1834 UNREACHABLE("");
1835 }
1836
1837 static enum pco_pck_fmt pco_pck_format_from_pipe_format(enum pipe_format fmt)
1838 {
1839 const struct util_format_description *desc = util_format_description(fmt);
1840 int c = util_format_get_largest_non_void_channel(fmt);
1841 assert(c >= 0);
1842 const struct util_format_channel_description *chan = &desc->channel[c];
1843
1844 switch (chan->size) {
1845 case 8:
1846 if (chan->type == UTIL_FORMAT_TYPE_UNSIGNED)
1847 return PCO_PCK_FMT_U8888;
1848 else if (chan->type == UTIL_FORMAT_TYPE_SIGNED)
1849 return PCO_PCK_FMT_S8888;
1850 break;
1851
1852 case 10:
1853 if (chan->type == UTIL_FORMAT_TYPE_UNSIGNED)
1854 return PCO_PCK_FMT_U1010102;
1855 else if (chan->type == UTIL_FORMAT_TYPE_SIGNED)
1856 return PCO_PCK_FMT_S1010102;
1857 break;
1858
1859 case 11:
1860 if (chan->type == UTIL_FORMAT_TYPE_FLOAT)
1861 return PCO_PCK_FMT_F111110;
1862 break;
1863
1864 case 16:
1865 if (chan->type == UTIL_FORMAT_TYPE_UNSIGNED)
1866 return PCO_PCK_FMT_U1616;
1867 else if (chan->type == UTIL_FORMAT_TYPE_SIGNED)
1868 return PCO_PCK_FMT_S1616;
1869 else if (chan->type == UTIL_FORMAT_TYPE_FLOAT)
1870 return PCO_PCK_FMT_F16F16;
1871 break;
1872
1873 default:
1874 break;
1875 }
1876
1877 UNREACHABLE("Unsupported format.");
1878 }
1879
1880 /**
1881 * \brief Translates a NIR intrinsic instruction into PCO.
1882 *
1883 * \param[in,out] tctx Translation context.
1884 * \param[in] intr The nir intrinsic instruction.
1885 * \return The PCO instruction.
1886 */
1887 static pco_instr *trans_intr(trans_ctx *tctx, nir_intrinsic_instr *intr)
1888 {
1889 const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic];
1890
1891 pco_ref dest = info->has_dest ? pco_ref_nir_def_t(&intr->def, tctx)
1892 : pco_ref_null();
1893
1894 pco_ref src[NIR_MAX_VEC_COMPONENTS] = { 0 };
1895 for (unsigned s = 0; s < info->num_srcs; ++s)
1896 src[s] = pco_ref_nir_src_t(&intr->src[s], tctx);
1897
1898 pco_instr *instr;
1899 switch (intr->intrinsic) {
1900 case nir_intrinsic_load_input:
1901 if (tctx->stage == MESA_SHADER_VERTEX)
1902 instr = trans_load_input_vs(tctx, intr, dest);
1903 else if (tctx->stage == MESA_SHADER_FRAGMENT)
1904 instr = trans_load_input_fs(tctx, intr, dest, false);
1905 else
1906 UNREACHABLE("Unsupported stage for \"nir_intrinsic_load_input\".");
1907 break;
1908
1909 case nir_intrinsic_load_interpolated_input:
1910 assert(tctx->stage == MESA_SHADER_FRAGMENT);
1911 instr = trans_load_input_fs(tctx, intr, dest, true);
1912 break;
1913
1914 /* Just consume. */
1915 case nir_intrinsic_load_barycentric_pixel:
1916 case nir_intrinsic_load_barycentric_sample:
1917 case nir_intrinsic_load_barycentric_centroid:
1918 return NULL;
1919
1920 case nir_intrinsic_store_output:
1921 if (tctx->stage == MESA_SHADER_VERTEX)
1922 instr = trans_store_output_vs(tctx, intr, src[0]);
1923 else if (tctx->stage == MESA_SHADER_FRAGMENT)
1924 instr = trans_store_output_fs(tctx, intr, src[0]);
1925 else
1926 UNREACHABLE("Unsupported stage for \"nir_intrinsic_store_output\".");
1927 break;
1928
1929 case nir_intrinsic_load_fs_coeffs_pco:
1930 instr = trans_load_fs_coeffs(tctx, intr, dest);
1931 break;
1932
1933 case nir_intrinsic_uvsw_write_pco:
1934 instr = trans_uvsw_write(tctx, intr, src[0], src[1]);
1935 break;
1936
1937 case nir_intrinsic_load_vtxin_pco:
1938 instr = trans_load_reg(tctx, intr, dest, src[0], PCO_REG_CLASS_VTXIN);
1939 break;
1940
1941 case nir_intrinsic_load_coeff_pco:
1942 instr = trans_load_reg(tctx, intr, dest, src[0], PCO_REG_CLASS_COEFF);
1943 break;
1944
1945 case nir_intrinsic_load_tiled_offset_pco:
1946 instr = trans_load_tiled_offset(tctx, intr, dest);
1947 break;
1948
1949 case nir_intrinsic_frag_store_pco: {
1950 unsigned base = nir_intrinsic_base(intr);
1951
1952 pco_ref dest = pco_ref_hwreg(base, PCO_REG_CLASS_PIXOUT);
1953 instr = pco_mov(&tctx->b, dest, src[0], .olchk = tctx->olchk);
1954 break;
1955 }
1956
1957 case nir_intrinsic_load_output:
1958 assert(tctx->stage == MESA_SHADER_FRAGMENT);
1959 instr = trans_load_output_fs(tctx, intr, dest);
1960 break;
1961
1962 case nir_intrinsic_dummy_load_store_pco: {
1963 assert(tctx->stage == MESA_SHADER_FRAGMENT);
1964 pco_ref pixout =
1965 pco_ref_hwreg(nir_intrinsic_base(intr), PCO_REG_CLASS_PIXOUT);
1966 return pco_mov(&tctx->b, pixout, pixout, .olchk = tctx->olchk);
1967 break;
1968 }
1969
1970 case nir_intrinsic_flush_tile_buffer_pco:
1971 assert(tctx->stage == MESA_SHADER_FRAGMENT);
1972 instr = trans_flush_tile_buffer(tctx, intr, src[0], src[1]);
1973 break;
1974
1975 case nir_intrinsic_load_preamble:
1976 instr = pco_mov(&tctx->b,
1977 dest,
1978 pco_ref_hwreg_vec(nir_intrinsic_base(intr),
1979 PCO_REG_CLASS_SHARED,
1980 pco_ref_get_chans(dest)),
1981 .rpt = pco_ref_get_chans(dest));
1982 break;
1983
1984 case nir_intrinsic_store_preamble:
1985 instr = pco_mov(&tctx->b,
1986 pco_ref_hwreg_vec(nir_intrinsic_base(intr),
1987 PCO_REG_CLASS_SHARED,
1988 pco_ref_get_chans(src[0])),
1989 src[0],
1990 .rpt = pco_ref_get_chans(src[0]));
1991 break;
1992
1993 case nir_intrinsic_store_preamble_dynamic: {
1994 unsigned idx_reg_num = 0;
1995 pco_ref idx_reg =
1996 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
1997
1998 pco_mov(&tctx->b, idx_reg, src[1]);
1999
2000 pco_ref dest = pco_ref_hwreg(0, PCO_REG_CLASS_SHARED);
2001 dest = pco_ref_hwreg_idx_from(idx_reg_num, dest);
2002
2003 instr = pco_mov(&tctx->b, dest, src[0], .rpt = pco_ref_get_chans(src[0]));
2004 break;
2005 }
2006
2007 case nir_intrinsic_load_push_constant:
2008 instr =
2009 trans_load_common_store(tctx,
2010 intr,
2011 dest,
2012 src[0],
2013 false,
2014 &tctx->shader->data.common.push_consts.range);
2015 break;
2016
2017 case nir_intrinsic_load_blend_const_color_rgba:
2018 assert(tctx->stage == MESA_SHADER_FRAGMENT);
2019 instr = trans_load_common_store(tctx,
2020 intr,
2021 dest,
2022 pco_ref_null(),
2023 false,
2024 &tctx->shader->data.fs.blend_consts);
2025 break;
2026
2027 case nir_intrinsic_load_packed_sample_location_pco:
2028 assert(tctx->stage == MESA_SHADER_FRAGMENT);
2029 instr = trans_load_packed_sample_location(tctx, intr, dest, src[0]);
2030 break;
2031
2032 case nir_intrinsic_load_shared:
2033 assert(tctx->stage == MESA_SHADER_COMPUTE);
2034 instr = trans_load_common_store(tctx,
2035 intr,
2036 dest,
2037 src[0],
2038 true,
2039 &tctx->shader->data.cs.shmem);
2040 break;
2041
2042 case nir_intrinsic_store_shared:
2043 assert(tctx->stage == MESA_SHADER_COMPUTE);
2044 instr = trans_store_common_store(tctx,
2045 intr,
2046 src[0],
2047 src[1],
2048 true,
2049 &tctx->shader->data.cs.shmem);
2050 break;
2051
2052 case nir_intrinsic_shared_atomic:
2053 case nir_intrinsic_shared_atomic_swap:
2054 instr = trans_atomic_shared(tctx, intr, dest, src[0], src[1], src[2]);
2055 break;
2056
2057 case nir_intrinsic_load_ubo:
2058 case nir_intrinsic_load_ssbo:
2059 instr = trans_load_buffer(tctx, intr, dest, src[1]);
2060 break;
2061
2062 case nir_intrinsic_load_global_2x32:
2063 instr = trans_load_global(tctx, intr, dest, src[0]);
2064 break;
2065
2066 case nir_intrinsic_store_global_2x32:
2067 instr = trans_store_global(tctx, src[0], src[1]);
2068 break;
2069
2070 case nir_intrinsic_get_ubo_size:
2071 case nir_intrinsic_get_ssbo_size:
2072 instr = trans_get_buffer_size(tctx, intr, dest);
2073 break;
2074
2075 case nir_intrinsic_store_ssbo:
2076 instr = trans_store_buffer(tctx, intr, src[0], src[2]);
2077 break;
2078
2079 case nir_intrinsic_ssbo_atomic:
2080 instr = trans_atomic_buffer(tctx, intr, dest, src[1], src[2]);
2081 break;
2082
2083 case nir_intrinsic_global_atomic_pco:
2084 instr = trans_global_atomic_buffer(tctx, intr, dest, src[0]);
2085 break;
2086
2087 case nir_intrinsic_is_null_descriptor: {
2088 const pco_common_data *common = &tctx->shader->data.common;
2089 uint32_t packed_desc = nir_src_comp_as_uint(intr->src[0], 0);
2090 unsigned elem = nir_src_comp_as_uint(intr->src[0], 1);
2091 unsigned stride;
2092 bool is_img_smp;
2093 unsigned sh_index = fetch_resource_base_reg_packed(common,
2094 packed_desc,
2095 elem,
2096 &stride,
2097 &is_img_smp);
2098
2099 if (is_img_smp)
2100 stride = ROGUE_NUM_TEXSTATE_DWORDS;
2101
2102 pco_ref all_words_zero;
2103 for (unsigned u = 0; u < stride; ++u) {
2104 pco_ref word = pco_ref_hwreg(sh_index + u, PCO_REG_CLASS_SHARED);
2105 pco_ref word_is_zero =
2106 pco_ref_new_ssa(tctx->func, pco_ref_get_bits(dest), 1);
2107
2108 pco_tstz(&tctx->b,
2109 word_is_zero,
2110 pco_ref_null(),
2111 word,
2112 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
2113
2114 if (!u) {
2115 all_words_zero = word_is_zero;
2116 continue;
2117 }
2118
2119 pco_ref _all_words_zero =
2120 pco_ref_new_ssa(tctx->func, pco_ref_get_bits(dest), 1);
2121 pco_logical(&tctx->b,
2122 _all_words_zero,
2123 pco_ref_null(),
2124 all_words_zero,
2125 pco_ref_null(),
2126 word_is_zero,
2127 .logiop = PCO_LOGIOP_AND);
2128
2129 all_words_zero = _all_words_zero;
2130 }
2131
2132 instr = pco_mov(&tctx->b, dest, all_words_zero);
2133
2134 break;
2135 }
2136
2137 case nir_intrinsic_load_scratch:
2138 instr = trans_scratch(tctx, dest, src[0], src[1]);
2139 break;
2140
2141 case nir_intrinsic_store_scratch:
2142 instr = trans_scratch(tctx, dest, src[1], src[0]);
2143 break;
2144
2145 case nir_intrinsic_dma_ld_pco: {
2146 unsigned chans = pco_ref_get_chans(dest);
2147
2148 instr = pco_ld(&tctx->b,
2149 dest,
2150 pco_ref_drc(PCO_DRC_0),
2151 pco_ref_imm8(chans),
2152 src[0]);
2153
2154 break;
2155 }
2156
2157 case nir_intrinsic_dma_st_pco: {
2158 unsigned chans = pco_ref_get_chans(src[0]) - 2;
2159 unsigned flags = nir_intrinsic_flags(intr);
2160
2161 pco_ref data_comp =
2162 pco_ref_new_ssa(tctx->func, pco_ref_get_bits(src[0]), chans);
2163 pco_comp(&tctx->b, data_comp, src[0], pco_ref_val16(2));
2164
2165 instr = pco_st32(&tctx->b,
2166 data_comp,
2167 pco_ref_drc(PCO_DRC_0),
2168 pco_ref_imm8(chans),
2169 src[0],
2170 pco_ref_null(),
2171 .idf = !!flags);
2172
2173 break;
2174 }
2175
2176 case nir_intrinsic_dma_ld_shregs_pco: {
2177 pco_ref addr = src[0];
2178 pco_ref burst_len = src[1];
2179 pco_ref dynsrc_offset = src[2];
2180
2181 unsigned idx_reg_num = 0;
2182 pco_ref idx_reg =
2183 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
2184
2185 pco_mov(&tctx->b, idx_reg, dynsrc_offset);
2186
2187 pco_ref dest = pco_ref_hwreg(0, PCO_REG_CLASS_SHARED);
2188 dest = pco_ref_hwreg_idx_from(idx_reg_num, dest);
2189
2190 instr =
2191 pco_ld_regbl(&tctx->b, dest, pco_ref_drc(PCO_DRC_0), burst_len, addr);
2192
2193 break;
2194 }
2195
2196 case nir_intrinsic_dma_st_shregs_pco: {
2197 pco_ref addr = src[0];
2198 pco_ref burst_len = src[1];
2199 pco_ref dynsrc_offset = src[2];
2200 unsigned flags = nir_intrinsic_flags(intr);
2201
2202 unsigned idx_reg_num = 0;
2203 pco_ref idx_reg =
2204 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
2205
2206 pco_mov(&tctx->b, idx_reg, dynsrc_offset);
2207
2208 pco_ref data = pco_ref_hwreg(0, PCO_REG_CLASS_SHARED);
2209 data = pco_ref_hwreg_idx_from(idx_reg_num, data);
2210
2211 instr = pco_st32_regbl(&tctx->b,
2212 data,
2213 pco_ref_drc(PCO_DRC_0),
2214 burst_len,
2215 addr,
2216 pco_ref_null(),
2217 .idf = !!flags);
2218
2219 break;
2220 }
2221
2222 case nir_intrinsic_dma_st_tiled_pco: {
2223 unsigned chans = pco_ref_get_chans(src[0]) - 2;
2224
2225 pco_ref data_comp =
2226 pco_ref_new_ssa(tctx->func, pco_ref_get_bits(src[0]), chans);
2227 pco_comp(&tctx->b, data_comp, src[0], pco_ref_val16(2));
2228
2229 instr = pco_st_tiled(&tctx->b,
2230 data_comp,
2231 pco_ref_imm8(PCO_DSIZE_32BIT),
2232 pco_ref_drc(PCO_DRC_0),
2233 pco_ref_imm8(chans),
2234 src[0],
2235 src[1],
2236 .olchk = tctx->olchk);
2237
2238 break;
2239 }
2240
2241 /* Vertex sysvals. */
2242 case nir_intrinsic_load_vertex_id:
2243 case nir_intrinsic_load_instance_id:
2244 case nir_intrinsic_load_base_instance:
2245 case nir_intrinsic_load_base_vertex:
2246 case nir_intrinsic_load_draw_id:
2247 case nir_intrinsic_load_view_index:
2248
2249 /* Compute sysvals. */
2250 case nir_intrinsic_load_local_invocation_index:
2251 case nir_intrinsic_load_workgroup_id:
2252 case nir_intrinsic_load_num_workgroups:
2253
2254 /* Fragment sysvals. */
2255 case nir_intrinsic_load_front_face_op_pco:
2256 instr = trans_load_sysval(tctx, intr, dest);
2257 break;
2258
2259 case nir_intrinsic_load_fs_meta_pco:
2260 assert(tctx->stage == MESA_SHADER_FRAGMENT);
2261 assert(tctx->shader->data.fs.meta.count > 0);
2262
2263 return pco_mov(&tctx->b,
2264 dest,
2265 pco_ref_hwreg(tctx->shader->data.fs.meta.start,
2266 PCO_REG_CLASS_SHARED));
2267 break;
2268
2269 case nir_intrinsic_ddx:
2270 case nir_intrinsic_ddx_fine:
2271 case nir_intrinsic_ddx_coarse:
2272 instr = intr->intrinsic == nir_intrinsic_ddx_fine
2273 ? pco_fdsxf(&tctx->b, dest, src[0])
2274 : pco_fdsx(&tctx->b, dest, src[0]);
2275 break;
2276
2277 case nir_intrinsic_ddy:
2278 case nir_intrinsic_ddy_fine:
2279 case nir_intrinsic_ddy_coarse:
2280 instr = intr->intrinsic == nir_intrinsic_ddy_fine
2281 ? pco_fdsyf(&tctx->b, dest, src[0])
2282 : pco_fdsy(&tctx->b, dest, src[0]);
2283 break;
2284
2285 /* Texture-related intrinsics. */
2286 case nir_intrinsic_load_tex_state_pco:
2287 instr = lower_load_tex_smp_state(tctx, intr, dest, false);
2288 break;
2289
2290 case nir_intrinsic_load_smp_state_pco:
2291 instr = lower_load_tex_smp_state(tctx, intr, dest, true);
2292 break;
2293
2294 case nir_intrinsic_load_tex_meta_pco:
2295 instr = lower_load_tex_smp_meta(tctx, intr, dest, false);
2296 break;
2297
2298 case nir_intrinsic_load_smp_meta_pco:
2299 instr = lower_load_tex_smp_meta(tctx, intr, dest, true);
2300 break;
2301
2302 case nir_intrinsic_smp_coeffs_pco:
2303 case nir_intrinsic_smp_raw_pco:
2304 case nir_intrinsic_smp_pco:
2305 case nir_intrinsic_smp_write_pco:
2306 instr = lower_smp(tctx, intr, &dest, src[0], src[1], src[2]);
2307 break;
2308
2309 case nir_intrinsic_alphatst_pco:
2310 instr = lower_alphatst(tctx, dest, src[0], src[1], src[2]);
2311 break;
2312
2313 case nir_intrinsic_isp_feedback_pco: {
2314 assert(tctx->stage == MESA_SHADER_FRAGMENT);
2315 bool does_discard = !nir_src_is_undef(intr->src[0]);
2316 bool does_depthf = !nir_src_is_undef(intr->src[1]);
2317
2318 does_depthf &= (tctx->shader->data.fs.uses.depth_feedback &&
2319 !tctx->shader->data.fs.uses.early_frag);
2320
2321 if (does_discard) {
2322 pco_tstz(&tctx->b,
2323 pco_ref_null(),
2324 pco_ref_pred(PCO_PRED_P0),
2325 src[0],
2326 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
2327 }
2328
2329 instr = does_depthf ? pco_depthf(&tctx->b,
2330 pco_ref_drc(PCO_DRC_0),
2331 src[1],
2332 .olchk = tctx->olchk)
2333 : pco_alphaf(&tctx->b,
2334 pco_ref_null(),
2335 pco_ref_drc(PCO_DRC_0),
2336 pco_zero,
2337 pco_zero,
2338 pco_7,
2339 .olchk = tctx->olchk);
2340
2341 if (does_discard)
2342 pco_instr_set_exec_cnd(instr, PCO_EXEC_CND_E1_Z1);
2343
2344 break;
2345 }
2346
2347 case nir_intrinsic_alpha_to_coverage:
2348 assert(tctx->stage == MESA_SHADER_FRAGMENT);
2349 instr = pco_pck(&tctx->b,
2350 pco_ref_bits(dest, 32),
2351 src[0],
2352 .pck_fmt = PCO_PCK_FMT_COV);
2353 break;
2354
2355 case nir_intrinsic_mutex_pco:
2356 instr = pco_mutex(&tctx->b,
2357 pco_ref_imm8(nir_intrinsic_mutex_id_pco(intr)),
2358 .mutex_op = nir_intrinsic_mutex_op_pco(intr));
2359 break;
2360
2361 case nir_intrinsic_load_instance_num_pco:
2362 instr = pco_mov(&tctx->b,
2363 dest,
2364 pco_ref_hwreg(PCO_SR_INST_NUM, PCO_REG_CLASS_SPEC));
2365 break;
2366
2367 case nir_intrinsic_load_shared_reg_alloc_size_pco:
2368 instr = pco_mov(&tctx->b,
2369 dest,
2370 pco_ref_hwreg(PCO_SR_SH_ALLOC_SIZE, PCO_REG_CLASS_SPEC));
2371 break;
2372
2373 case nir_intrinsic_load_sample_id:
2374 instr = pco_mov(&tctx->b,
2375 dest,
2376 pco_ref_hwreg(PCO_SR_SAMP_NUM, PCO_REG_CLASS_SPEC));
2377 break;
2378
2379 case nir_intrinsic_load_layer_id:
2380 instr = pco_mov(&tctx->b,
2381 dest,
2382 pco_ref_hwreg(PCO_SR_RENDER_TGT_ID, PCO_REG_CLASS_SPEC));
2383 break;
2384
2385 case nir_intrinsic_load_face_ccw_pco:
2386 instr = pco_mov(&tctx->b,
2387 dest,
2388 pco_ref_hwreg(PCO_SR_FACE_ORIENT, PCO_REG_CLASS_SPEC));
2389 break;
2390
2391 case nir_intrinsic_load_tile_coord_pco: {
2392 assert(tctx->stage == MESA_SHADER_FRAGMENT);
2393
2394 pco_ref xy[] = {
2395 pco_ref_hwreg(PCO_SR_TILE_X_P, PCO_REG_CLASS_SPEC),
2396 pco_ref_hwreg(PCO_SR_TILE_Y_P, PCO_REG_CLASS_SPEC),
2397 };
2398
2399 unsigned component = nir_intrinsic_component(intr);
2400 unsigned chans = pco_ref_get_chans(dest);
2401 assert(component + chans <= ARRAY_SIZE(xy));
2402
2403 instr = pco_vec(&tctx->b, dest, chans, &xy[component]);
2404 break;
2405 }
2406
2407 case nir_intrinsic_load_savmsk_vm_pco:
2408 instr = pco_savmsk(&tctx->b,
2409 dest,
2410 pco_ref_null(),
2411 .savmsk_mode = PCO_SAVMSK_MODE_VM);
2412 break;
2413
2414 case nir_intrinsic_decl_reg:
2415 case nir_intrinsic_load_reg:
2416 case nir_intrinsic_store_reg:
2417 instr = trans_reg_intr(tctx, intr, dest, src[0], src[1]);
2418 break;
2419
2420 case nir_intrinsic_emitpix_pco:
2421 instr = pco_emitpix(&tctx->b,
2422 src[0],
2423 src[1],
2424 .freep = nir_intrinsic_freep(intr));
2425 break;
2426
2427 case nir_intrinsic_wop_pco:
2428 instr = pco_wop(&tctx->b);
2429 break;
2430
2431 case nir_intrinsic_pck_prog_pco:
2432 instr = pco_pck_prog(&tctx->b,
2433 dest,
2434 src[0],
2435 src[1],
2436 .scale = nir_intrinsic_scale(intr),
2437 .roundzero = nir_intrinsic_roundzero(intr),
2438 .rpt = pco_ref_get_chans(dest));
2439 break;
2440
2441 case nir_intrinsic_pack_pco: {
2442 enum pipe_format fmt = nir_intrinsic_format(intr);
2443 enum pco_pck_fmt pck_fmt = pco_pck_format_from_pipe_format(fmt);
2444 bool scale = util_format_is_unorm(fmt) || util_format_is_snorm(fmt);
2445 unsigned chans = pco_ref_get_chans(src[0]);
2446 instr = pco_pck(&tctx->b,
2447 dest,
2448 src[0],
2449 .rpt = chans,
2450 .pck_fmt = pck_fmt,
2451 .scale = scale);
2452 break;
2453 }
2454
2455 case nir_intrinsic_unpack_pco: {
2456 enum pipe_format fmt = nir_intrinsic_format(intr);
2457 enum pco_pck_fmt pck_fmt = pco_pck_format_from_pipe_format(fmt);
2458 unsigned chans = pco_ref_get_chans(dest);
2459 bool scale = util_format_is_unorm(fmt) || util_format_is_snorm(fmt);
2460 instr = pco_unpck(&tctx->b,
2461 dest,
2462 pco_ref_elem(src[0], 0),
2463 .rpt = chans,
2464 .pck_fmt = pck_fmt,
2465 .scale = scale);
2466 break;
2467 }
2468
2469 default:
2470 printf("Unsupported intrinsic: \"");
2471 nir_print_instr(&intr->instr, stdout);
2472 printf("\"\n");
2473 UNREACHABLE("");
2474 break;
2475 }
2476
2477 if (!pco_ref_is_scalar(dest))
2478 split_dest_comps(tctx, instr, dest);
2479
2480 return instr;
2481 }
2482
2483 /**
2484 * \brief Attempts to collate a vector within a vector.
2485 *
2486 * If a vector references another vector in its entirety in order/without
2487 * swizzling, we try to store a reference to said vector rather than its
2488 * individual components.
2489 *
2490 * \param[in] src The source/vector channel to start checking from.
2491 * \param[in] from The instruction the vector components are from.
2492 * \param[in] vec The potential vector reference from the parent instruction.
2493 * \param[in] vec_chans The number of sources/vector channels.
2494 * \return The number of collated sources, or 0 if collation failed.
2495 */
2496 static pco_ref
2497 try_collate_vec(pco_ref *src, pco_instr *from, pco_ref vec, unsigned vec_chans)
2498 {
2499 /* Skip the first one since it's our reference (and we already know its
2500 * component is 0.
2501 */
2502 for (unsigned s = 1; s < vec_chans; ++s) {
2503 pco_instr *parent_instr = find_parent_instr_from(src[s], from);
2504 assert(parent_instr);
2505
2506 if (parent_instr->op != PCO_OP_COMP)
2507 return pco_ref_null();
2508
2509 pco_ref comp_src = parent_instr->src[0];
2510 unsigned comp_idx = pco_ref_get_imm(parent_instr->src[1]);
2511 ASSERTED unsigned chans = pco_ref_get_chans(comp_src);
2512
2513 /* TODO: can this be true? */
2514 if (!pco_refs_are_equal(comp_src, vec, false))
2515 return pco_ref_null();
2516
2517 assert(chans == vec_chans);
2518
2519 if (comp_idx != s)
2520 return pco_ref_null();
2521 }
2522
2523 return vec;
2524 }
2525
2526 /**
2527 * \brief Attempts to collate vector sources.
2528 *
2529 * \param[in] tctx Translation context.
2530 * \param[in] dest Instruction destination.
2531 * \param[in] num_srcs The number of sources/vector channels.
2532 * \param[in] src The sources/vector components.
2533 * \return The number of collated sources, or 0 if collation failed.
2534 */
2535 static unsigned try_collate_vec_srcs(trans_ctx *tctx,
2536 unsigned num_srcs,
2537 pco_ref *src,
2538 pco_ref *collated_src)
2539 {
2540 bool collated_vector = false;
2541 unsigned num_srcs_collated = 0;
2542 pco_instr *from = pco_cursor_instr(tctx->b.cursor);
2543 if (!from) {
2544 from = pco_last_instr(
2545 pco_prev_block_nonempty(pco_cursor_block(tctx->b.cursor)));
2546 }
2547
2548 assert(from);
2549
2550 for (unsigned s = 0; s < num_srcs; ++s) {
2551 pco_instr *parent_instr = find_parent_instr_from(src[s], from);
2552 assert(parent_instr);
2553
2554 /* This is a purely scalar source; append it and continue. */
2555 if (parent_instr->op != PCO_OP_COMP) {
2556 collated_src[num_srcs_collated++] = src[s];
2557 continue;
2558 }
2559
2560 pco_ref comp_src = parent_instr->src[0];
2561 unsigned comp_idx = pco_ref_get_imm(parent_instr->src[1]);
2562 unsigned chans = pco_ref_get_chans(comp_src);
2563
2564 /* We have a vector source, but it either:
2565 * - doesn't start from the first element
2566 * - is bigger than the remaining channels of *this* vec
2567 * so it's impossible for it to be contained in its entirety;
2568 * append the component and continue.
2569 */
2570 if (comp_idx != 0 || chans > (num_srcs - s)) {
2571 collated_src[num_srcs_collated++] = src[s];
2572 continue;
2573 }
2574
2575 /* We have a candidate for an entire vector to be inserted. */
2576 pco_ref collated_ref = try_collate_vec(&src[s], from, comp_src, chans);
2577 if (pco_ref_is_null(collated_ref)) {
2578 collated_src[num_srcs_collated++] = src[s];
2579 continue;
2580 }
2581
2582 /* We were successful, record this and increment accordingly. */
2583 collated_src[num_srcs_collated++] = collated_ref;
2584
2585 s += chans - 1;
2586 collated_vector = true;
2587 }
2588
2589 return collated_vector ? num_srcs_collated : 0;
2590 }
2591
2592 /**
2593 * \brief Translates a NIR vec instruction into PCO, attempting collation.
2594 *
2595 * \param[in] tctx Translation context.
2596 * \param[in] dest Instruction destination.
2597 * \param[in] num_srcs The number of sources/vector components.
2598 * \param[in] src The sources/vector components.
2599 * \return The PCO instruction.
2600 */
2601 static pco_instr *pco_trans_nir_vec(trans_ctx *tctx,
2602 pco_ref dest,
2603 unsigned num_srcs,
2604 pco_ref *src)
2605
2606 {
2607 /* If a vec contains entire other vecs, try to reference them directly. */
2608 pco_ref collated_src[NIR_MAX_VEC_COMPONENTS] = { 0 };
2609 unsigned num_srcs_collated =
2610 try_collate_vec_srcs(tctx, num_srcs, src, collated_src);
2611 if (!num_srcs_collated)
2612 return pco_vec(&tctx->b, dest, num_srcs, src);
2613
2614 pco_instr *instr = pco_vec(&tctx->b, dest, num_srcs_collated, collated_src);
2615
2616 /* Record the collated vectors. */
2617 for (unsigned s = 0; s < num_srcs_collated; ++s) {
2618 if (pco_ref_is_scalar(collated_src[s]))
2619 continue;
2620
2621 pco_vec_info *vec_info =
2622 _mesa_hash_table_u64_search(tctx->func->vec_infos,
2623 collated_src[s].val);
2624 assert(vec_info);
2625
2626 /* Skip if there are multiple users. */
2627 vec_info->vec_user = vec_info->vec_user ? VEC_USER_MULTI : instr;
2628 }
2629
2630 return instr;
2631 }
2632
2633 static inline enum pco_tst_op_main to_tst_op_main(nir_op op)
2634 {
2635 switch (op) {
2636 case nir_op_fcsel:
2637 case nir_op_icsel_eqz:
2638 return PCO_TST_OP_MAIN_ZERO;
2639
2640 case nir_op_fcsel_gt:
2641 case nir_op_i32csel_gt:
2642 return PCO_TST_OP_MAIN_GZERO;
2643
2644 case nir_op_fcsel_ge:
2645 case nir_op_i32csel_ge:
2646 return PCO_TST_OP_MAIN_GEZERO;
2647
2648 case nir_op_slt:
2649 case nir_op_flt:
2650 case nir_op_ilt:
2651 case nir_op_ult:
2652 return PCO_TST_OP_MAIN_LESS;
2653
2654 case nir_op_sge:
2655 case nir_op_fge:
2656 case nir_op_ige:
2657 case nir_op_uge:
2658 return PCO_TST_OP_MAIN_GEQUAL;
2659
2660 case nir_op_seq:
2661 case nir_op_feq:
2662 case nir_op_ieq:
2663 return PCO_TST_OP_MAIN_EQUAL;
2664
2665 case nir_op_sne:
2666 case nir_op_fneu:
2667 case nir_op_ine:
2668 return PCO_TST_OP_MAIN_NOTEQUAL;
2669
2670 default:
2671 break;
2672 }
2673
2674 UNREACHABLE("");
2675 }
2676
2677 static inline enum pco_tst_type_main to_tst_type_main(nir_op op, pco_ref src)
2678 {
2679 switch (op) {
2680 case nir_op_fcsel:
2681 case nir_op_fcsel_gt:
2682 case nir_op_fcsel_ge:
2683
2684 case nir_op_slt:
2685 case nir_op_sge:
2686 case nir_op_seq:
2687 case nir_op_sne:
2688
2689 case nir_op_flt:
2690 case nir_op_fge:
2691 case nir_op_feq:
2692 case nir_op_fneu:
2693
2694 case nir_op_fmin:
2695 case nir_op_fmax:
2696 return PCO_TST_TYPE_MAIN_F32;
2697
2698 case nir_op_icsel_eqz:
2699 case nir_op_i32csel_gt:
2700 case nir_op_i32csel_ge:
2701
2702 case nir_op_ilt:
2703 case nir_op_ige:
2704 case nir_op_ieq:
2705 case nir_op_ine:
2706
2707 case nir_op_imin:
2708 case nir_op_imax:
2709 return PCO_TST_TYPE_MAIN_S32;
2710
2711 case nir_op_ult:
2712 case nir_op_uge:
2713
2714 case nir_op_umin:
2715 case nir_op_umax:
2716 return PCO_TST_TYPE_MAIN_U32;
2717
2718 default:
2719 break;
2720 }
2721
2722 UNREACHABLE("");
2723 }
2724
2725 static inline bool is_scmp(nir_op op)
2726 {
2727 switch (op) {
2728 case nir_op_slt:
2729 case nir_op_sge:
2730 case nir_op_seq:
2731 case nir_op_sne:
2732 return true;
2733
2734 default:
2735 break;
2736 }
2737
2738 return false;
2739 }
2740
2741 /**
2742 * \brief Translates a NIR comparison op into PCO.
2743 *
2744 * \param[in,out] tctx Translation context.
2745 * \param[in] op The NIR op.
2746 * \param[in] dest Instruction destination.
2747 * \param[in] src0 First comparison source.
2748 * \param[in] src1 Second comparison source.
2749 * \return The translated PCO instruction.
2750 */
2751 static pco_instr *
2752 trans_cmp(trans_ctx *tctx, nir_op op, pco_ref dest, pco_ref src0, pco_ref src1)
2753 {
2754 enum pco_tst_op_main tst_op_main = to_tst_op_main(op);
2755 enum pco_tst_type_main tst_type_main = to_tst_type_main(op, src0);
2756
2757 return is_scmp(op)
2758 ? pco_scmp(&tctx->b, dest, src0, src1, .tst_op_main = tst_op_main)
2759 : pco_bcmp(&tctx->b,
2760 dest,
2761 src0,
2762 src1,
2763 .tst_op_main = tst_op_main,
2764 .tst_type_main = tst_type_main);
2765 }
2766
2767 /**
2768 * \brief Translates a NIR {i,f}csel op into PCO.
2769 *
2770 * \param[in,out] tctx Translation context.
2771 * \param[in] op The NIR op.
2772 * \param[in] src Instruction source.
2773 * \return The translated PCO instruction.
2774 */
2775 static pco_instr *trans_csel(trans_ctx *tctx,
2776 nir_op op,
2777 pco_ref dest,
2778 pco_ref src0,
2779 pco_ref src1,
2780 pco_ref src2)
2781 {
2782 enum pco_tst_op_main tst_op_main = to_tst_op_main(op);
2783 enum pco_tst_type_main tst_type_main = to_tst_type_main(op, src0);
2784
2785 if (op == nir_op_fcsel) {
2786 pco_ref tmp = src2;
2787 src2 = src1;
2788 src1 = tmp;
2789 }
2790
2791 return pco_csel(&tctx->b,
2792 dest,
2793 src0,
2794 src1,
2795 src2,
2796 .tst_op_main = tst_op_main,
2797 .tst_type_main = tst_type_main);
2798 }
2799
2800 /**
2801 * \brief Translates a NIR bitwise logical op into PCO.
2802 *
2803 * \param[in,out] tctx Translation context.
2804 * \param[in] op The NIR op.
2805 * \param[in] dest Instruction destination.
2806 * \param[in] src0 First logical operand.
2807 * \param[in] src1 Second logical operand.
2808 * \return The translated PCO instruction.
2809 */
2810 static pco_instr *trans_logical(trans_ctx *tctx,
2811 nir_op op,
2812 pco_ref dest,
2813 pco_ref src0,
2814 pco_ref src1)
2815 {
2816 ASSERTED unsigned bits = pco_ref_get_bits(dest);
2817
2818 /* TODO: 8/16-bit support via masking. */
2819 assert(bits == 1 || bits == 32);
2820
2821 enum pco_logiop logiop;
2822 switch (op) {
2823 case nir_op_iand:
2824 logiop = PCO_LOGIOP_AND;
2825 break;
2826
2827 case nir_op_ior:
2828 logiop = PCO_LOGIOP_OR;
2829 break;
2830
2831 case nir_op_ixor:
2832 logiop = PCO_LOGIOP_XOR;
2833 break;
2834
2835 case nir_op_inot:
2836 logiop = PCO_LOGIOP_XNOR;
2837 src1 = pco_zero;
2838 break;
2839
2840 default:
2841 UNREACHABLE("");
2842 }
2843
2844 return pco_logical(&tctx->b,
2845 dest,
2846 pco_ref_null(),
2847 src0,
2848 pco_ref_null(),
2849 src1,
2850 .logiop = logiop);
2851 }
2852
2853 /**
2854 * \brief Translates a NIR shift op into PCO.
2855 *
2856 * \param[in,out] tctx Translation context.
2857 * \param[in] op The NIR op.
2858 * \param[in] src Instruction source.
2859 * \return The translated PCO instruction.
2860 */
2861 static pco_instr *trans_shift(trans_ctx *tctx,
2862 nir_op op,
2863 pco_ref dest,
2864 pco_ref src0,
2865 pco_ref src1)
2866 {
2867 ASSERTED unsigned bits = pco_ref_get_bits(dest);
2868 assert(bits == 32);
2869
2870 enum pco_shiftop shiftop;
2871 switch (op) {
2872 case nir_op_ishl:
2873 shiftop = PCO_SHIFTOP_LSL;
2874 break;
2875
2876 case nir_op_ishr:
2877 shiftop = PCO_SHIFTOP_ASR_TWB;
2878 break;
2879
2880 case nir_op_ushr:
2881 shiftop = PCO_SHIFTOP_SHR;
2882 break;
2883
2884 default:
2885 UNREACHABLE("");
2886 }
2887
2888 return pco_shift(&tctx->b,
2889 dest,
2890 src0,
2891 src1,
2892 pco_ref_null(),
2893 .shiftop = shiftop);
2894 }
2895
2896 static inline bool is_min(nir_op op)
2897 {
2898 switch (op) {
2899 case nir_op_fmin:
2900 case nir_op_imin:
2901 case nir_op_umin:
2902 return true;
2903
2904 case nir_op_fmax:
2905 case nir_op_imax:
2906 case nir_op_umax:
2907 return false;
2908
2909 default:
2910 break;
2911 }
2912
2913 UNREACHABLE("");
2914 }
2915
2916 /**
2917 * \brief Translates a NIR min/max op into PCO.
2918 *
2919 * \param[in,out] tctx Translation context.
2920 * \param[in] op The NIR op.
2921 * \param[in] src Instruction source.
2922 * \return The translated PCO instruction.
2923 */
2924 static pco_instr *trans_min_max(trans_ctx *tctx,
2925 nir_op op,
2926 pco_ref dest,
2927 pco_ref src0,
2928 pco_ref src1)
2929 {
2930 enum pco_tst_type_main tst_type_main = to_tst_type_main(op, src0);
2931
2932 return is_min(op) ? pco_min(&tctx->b,
2933 dest,
2934 src0,
2935 src1,
2936 .tst_type_main = tst_type_main)
2937 : pco_max(&tctx->b,
2938 dest,
2939 src0,
2940 src1,
2941 .tst_type_main = tst_type_main);
2942 }
2943
2944 /**
2945 * \brief Translates a NIR trigonometric op into PCO.
2946 *
2947 * \param[in,out] tctx Translation context.
2948 * \param[in] op The NIR op.
2949 * \param[in] dest Instruction destination.
2950 * \param[in] src Instruction source.
2951 * \return The translated PCO instruction.
2952 */
2953 static pco_instr *
2954 trans_trig(trans_ctx *tctx, nir_op op, pco_ref dest, pco_ref src)
2955 {
2956 assert(pco_ref_get_chans(dest) == 1);
2957 assert(pco_ref_get_bits(dest) == 32);
2958
2959 enum pco_fred_type fred_type;
2960 switch (op) {
2961 case nir_op_fsin:
2962 fred_type = PCO_FRED_TYPE_SIN;
2963 break;
2964
2965 case nir_op_fcos:
2966 fred_type = PCO_FRED_TYPE_COS;
2967 break;
2968
2969 /* TODO: arctan, arctanc, sinc, cosc. */
2970 default:
2971 UNREACHABLE("");
2972 }
2973
2974 pco_ref fred_dest_a = pco_ref_new_ssa32(tctx->func);
2975 pco_fred(&tctx->b,
2976 pco_ref_null(),
2977 fred_dest_a,
2978 pco_ref_null(),
2979 src,
2980 pco_ref_null(),
2981 pco_ref_imm8(0),
2982 .fred_type = fred_type,
2983 .fred_part = PCO_FRED_PART_A);
2984
2985 pco_ref fred_dest_b = pco_ref_new_ssa32(tctx->func);
2986 pco_fred(&tctx->b,
2987 fred_dest_b,
2988 pco_ref_null(),
2989 pco_ref_null(),
2990 src,
2991 fred_dest_a,
2992 pco_ref_imm8(0),
2993 .fred_type = fred_type,
2994 .fred_part = PCO_FRED_PART_B);
2995
2996 pco_ref trig_dest = pco_ref_new_ssa32(tctx->func);
2997 switch (op) {
2998 case nir_op_fsin:
2999 case nir_op_fcos:
3000 pco_fsinc(&tctx->b, trig_dest, pco_ref_pred(PCO_PRED_P0), fred_dest_b);
3001 break;
3002
3003 default:
3004 UNREACHABLE("");
3005 }
3006
3007 return pco_psel_trig(&tctx->b,
3008 dest,
3009 pco_ref_pred(PCO_PRED_P0),
3010 trig_dest,
3011 fred_dest_b);
3012 }
3013
3014 static pco_instr *
3015 trans_conv(trans_ctx *tctx, nir_op op, pco_ref dest, pco_ref src)
3016 {
3017 bool roundzero;
3018 enum pco_pck_fmt pck_fmt;
3019 bool unpck;
3020
3021 switch (op) {
3022 /* b2* ops are just sels. */
3023 case nir_op_b2f32:
3024 case nir_op_b2i32:
3025 return pco_bcsel(&tctx->b,
3026 dest,
3027 src,
3028 op == nir_op_b2f32 ? pco_fone : pco_one,
3029 pco_zero);
3030
3031 case nir_op_f2f16:
3032 case nir_op_f2f16_rtne:
3033 case nir_op_f2f16_rtz:
3034 assert(pco_ref_get_bits(src) == 32);
3035
3036 roundzero = (op != nir_op_f2f16_rtne);
3037 pck_fmt = PCO_PCK_FMT_F16F16;
3038 unpck = false;
3039
3040 /* TODO: support non-32 bit vars. */
3041 dest = pco_ref_bits(dest, 32);
3042 break;
3043
3044 case nir_op_f2f32:
3045 assert(pco_ref_get_bits(src) == 16);
3046
3047 roundzero = false;
3048 pck_fmt = PCO_PCK_FMT_F16F16;
3049 unpck = true;
3050
3051 /* TODO: support non-32 bit vars. */
3052 src = pco_ref_bits(src, 32);
3053 break;
3054
3055 case nir_op_f2i32:
3056 case nir_op_f2i32_rtne:
3057 case nir_op_f2u32:
3058 roundzero = (op != nir_op_f2i32_rtne);
3059 pck_fmt = (op == nir_op_f2u32) ? PCO_PCK_FMT_U32 : PCO_PCK_FMT_S32;
3060 unpck = false;
3061 break;
3062
3063 /* Just consume/treat as 32-bit for now. */
3064 /* TODO: support non-32 bit vars. */
3065 case nir_op_i2i16:
3066 case nir_op_i2i32:
3067 case nir_op_u2u16:
3068 case nir_op_u2u32:
3069 dest = pco_ref_bits(dest, 32);
3070 src = pco_ref_bits(src, 32);
3071 return pco_mov(&tctx->b, dest, src);
3072
3073 case nir_op_i2f32:
3074 case nir_op_u2f32:
3075 roundzero = tctx->ftz32b;
3076 pck_fmt = (op == nir_op_u2f32) ? PCO_PCK_FMT_U32 : PCO_PCK_FMT_S32;
3077 unpck = true;
3078 break;
3079
3080 default:
3081 UNREACHABLE("Unsupported conversion op.");
3082 }
3083
3084 return unpck ? pco_unpck(&tctx->b,
3085 dest,
3086 pco_ref_elem(src, 0),
3087 .pck_fmt = pck_fmt,
3088 .roundzero = roundzero)
3089 : pco_pck(&tctx->b,
3090 dest,
3091 src,
3092 .pck_fmt = pck_fmt,
3093 .roundzero = roundzero);
3094 }
3095
3096 /**
3097 * \brief Translates a NIR alu instruction into PCO.
3098 *
3099 * \param[in] tctx Translation context.
3100 * \param[in] alu The nir alu instruction.
3101 * \return The PCO instruction.
3102 */
3103 static pco_instr *trans_alu(trans_ctx *tctx, nir_alu_instr *alu)
3104 {
3105 const nir_op_info *info = &nir_op_infos[alu->op];
3106 unsigned num_srcs = info->num_inputs;
3107
3108 pco_ref dest = pco_ref_nir_def_t(&alu->def, tctx);
3109
3110 pco_ref src[NIR_MAX_VEC_COMPONENTS] = { 0 };
3111 for (unsigned s = 0; s < num_srcs; ++s)
3112 src[s] = pco_ref_nir_alu_src_t(alu, s, tctx);
3113
3114 pco_instr *instr;
3115 switch (alu->op) {
3116 case nir_op_fneg:
3117 instr = pco_fneg(&tctx->b, dest, src[0]);
3118 break;
3119
3120 case nir_op_fabs:
3121 instr = pco_fabs(&tctx->b, dest, src[0]);
3122 break;
3123
3124 case nir_op_ffloor:
3125 instr = pco_fflr(&tctx->b, dest, src[0]);
3126 break;
3127
3128 case nir_op_fceil:
3129 instr = pco_fceil(&tctx->b, dest, src[0]);
3130 break;
3131
3132 case nir_op_fadd:
3133 instr = pco_fadd(&tctx->b, dest, src[0], src[1]);
3134 break;
3135
3136 case nir_op_fsub:
3137 instr = pco_fadd(&tctx->b, dest, pco_ref_neg(src[1]), src[0]);
3138 break;
3139
3140 case nir_op_fmul:
3141 instr = pco_fmul(&tctx->b, dest, src[0], src[1]);
3142 break;
3143
3144 case nir_op_ffma:
3145 instr = pco_fmad(&tctx->b, dest, src[0], src[1], src[2]);
3146 break;
3147
3148 case nir_op_frcp:
3149 instr = pco_frcp(&tctx->b, dest, src[0]);
3150 break;
3151
3152 case nir_op_frsq:
3153 instr = pco_frsq(&tctx->b, dest, src[0]);
3154 break;
3155
3156 case nir_op_fexp2:
3157 instr = pco_fexp(&tctx->b, dest, src[0]);
3158 break;
3159
3160 case nir_op_flog2:
3161 instr = pco_flog(&tctx->b, dest, src[0]);
3162 break;
3163
3164 case nir_op_fsign:
3165 instr = pco_fsign(&tctx->b, dest, src[0]);
3166 break;
3167
3168 case nir_op_fsat:
3169 instr = pco_fadd(&tctx->b, dest, src[0], pco_zero, .sat = true);
3170 break;
3171
3172 case nir_op_fsin:
3173 case nir_op_fcos:
3174 instr = trans_trig(tctx, alu->op, dest, src[0]);
3175 break;
3176
3177 case nir_op_isign:
3178 instr = pco_isign(&tctx->b, dest, src[0]);
3179 break;
3180
3181 case nir_op_fcopysign_pco:
3182 instr = pco_copysign(&tctx->b, dest, src[0], src[1]);
3183 break;
3184
3185 case nir_op_ineg:
3186 instr = pco_iadd32(&tctx->b,
3187 dest,
3188 pco_ref_neg(src[0]),
3189 pco_zero,
3190 pco_ref_null(),
3191 .s = true);
3192 break;
3193
3194 case nir_op_iabs:
3195 instr = pco_iadd32(&tctx->b,
3196 dest,
3197 pco_ref_abs(src[0]),
3198 pco_zero,
3199 pco_ref_null(),
3200 .s = true);
3201 break;
3202
3203 case nir_op_iadd:
3204 instr = pco_iadd32(&tctx->b, dest, src[0], src[1], pco_ref_null());
3205 break;
3206
3207 /* TODO: PCO pass to combine u{add,sub}{carry,borrow}s with the same srcs. */
3208 case nir_op_uadd_carry:
3209 instr = pco_uadd_carry(&tctx->b, pco_ref_null(), dest, src[0], src[1]);
3210 break;
3211
3212 case nir_op_usub_borrow:
3213 instr = pco_uadd_carry(&tctx->b,
3214 pco_ref_null(),
3215 dest,
3216 pco_ref_neg(src[1]),
3217 src[0]);
3218 break;
3219
3220 case nir_op_uadd_sat:
3221 instr = pco_uadd_sat(&tctx->b, dest, src[0], src[1], pco_u32max);
3222 break;
3223
3224 case nir_op_usub_sat:
3225 instr =
3226 pco_uadd_sat(&tctx->b, dest, pco_ref_neg(src[1]), src[0], pco_zero);
3227 break;
3228
3229 case nir_op_uadd64_32: {
3230 pco_ref dest_comps[2] = {
3231 [0] = pco_ref_new_ssa32(tctx->func),
3232 [1] = pco_ref_new_ssa32(tctx->func),
3233 };
3234
3235 pco_add64_32(&tctx->b,
3236 dest_comps[0],
3237 dest_comps[1],
3238 src[0],
3239 src[1],
3240 src[2],
3241 pco_ref_null());
3242
3243 /* TODO: mark this vec as being non-contiguous,
3244 * add pass for expanding.
3245 */
3246 instr = pco_trans_nir_vec(tctx, dest, 2, dest_comps);
3247 break;
3248 }
3249
3250 case nir_op_umad64_32: {
3251 pco_ref dest_comps[2] = {
3252 [0] = pco_ref_new_ssa32(tctx->func),
3253 [1] = pco_ref_new_ssa32(tctx->func),
3254 };
3255
3256 pco_imadd64(&tctx->b,
3257 dest_comps[0],
3258 dest_comps[1],
3259 src[0],
3260 src[1],
3261 src[2],
3262 src[3],
3263 pco_ref_null());
3264
3265 /* TODO: mark this vec as being non-contiguous,
3266 * add pass for expanding.
3267 */
3268 instr = pco_trans_nir_vec(tctx, dest, 2, dest_comps);
3269 break;
3270 }
3271
3272 case nir_op_imul:
3273 instr = pco_imul32(&tctx->b, dest, src[0], src[1], pco_ref_null());
3274 break;
3275
3276 case nir_op_imul_high:
3277 instr = pco_imadd64(&tctx->b,
3278 pco_ref_null(),
3279 dest,
3280 src[0],
3281 src[1],
3282 pco_zero,
3283 pco_zero,
3284 pco_ref_null(),
3285 .s = true);
3286 break;
3287
3288 case nir_op_umul_high:
3289 instr = pco_imadd64(&tctx->b,
3290 pco_ref_null(),
3291 dest,
3292 src[0],
3293 src[1],
3294 pco_zero,
3295 pco_zero,
3296 pco_ref_null());
3297 break;
3298
3299 case nir_op_umul_16x16:
3300 instr = pco_imadd64(&tctx->b,
3301 dest,
3302 pco_ref_null(),
3303 src[0],
3304 src[1],
3305 pco_zero,
3306 pco_zero,
3307 pco_ref_null());
3308 break;
3309
3310 case nir_op_imad:
3311 instr = pco_imadd32(&tctx->b,
3312 dest,
3313 src[0],
3314 src[1],
3315 src[2],
3316 pco_ref_null(),
3317 .s = true);
3318 break;
3319
3320 /* Set-on (float) comparisons. */
3321 case nir_op_slt:
3322 case nir_op_sge:
3323 case nir_op_seq:
3324 case nir_op_sne:
3325
3326 /* Float comparisons. */
3327 case nir_op_flt:
3328 case nir_op_fge:
3329 case nir_op_feq:
3330 case nir_op_fneu:
3331
3332 /* Signed int comparisons. */
3333 case nir_op_ilt:
3334 case nir_op_ige:
3335 case nir_op_ieq:
3336 case nir_op_ine:
3337
3338 /* Unsigned int comparisons. */
3339 case nir_op_ult:
3340 case nir_op_uge:
3341 instr = trans_cmp(tctx, alu->op, dest, src[0], src[1]);
3342 break;
3343
3344 case nir_op_bcsel:
3345 instr = pco_bcsel(&tctx->b, dest, src[0], src[1], src[2]);
3346 break;
3347
3348 case nir_op_fcsel:
3349 case nir_op_fcsel_gt:
3350 case nir_op_fcsel_ge:
3351
3352 case nir_op_icsel_eqz:
3353 case nir_op_i32csel_gt:
3354 case nir_op_i32csel_ge:
3355 instr = trans_csel(tctx, alu->op, dest, src[0], src[1], src[2]);
3356 break;
3357
3358 case nir_op_iand:
3359 case nir_op_ior:
3360 case nir_op_ixor:
3361 case nir_op_inot:
3362 instr = trans_logical(tctx, alu->op, dest, src[0], src[1]);
3363 break;
3364
3365 case nir_op_ishl:
3366 case nir_op_ishr:
3367 case nir_op_ushr:
3368 instr = trans_shift(tctx, alu->op, dest, src[0], src[1]);
3369 break;
3370
3371 case nir_op_bit_count:
3372 instr = pco_cbs(&tctx->b, dest, src[0]);
3373 break;
3374
3375 case nir_op_ufind_msb:
3376 instr = pco_ftb(&tctx->b, dest, src[0]);
3377 break;
3378
3379 case nir_op_ibitfield_extract: {
3380 pco_ref bfe = pco_ref_new_ssa32(tctx->func);
3381 pco_ibfe(&tctx->b, bfe, src[0], src[1], src[2]);
3382 instr = pco_bcsel(&tctx->b, dest, src[2], bfe, pco_zero);
3383 break;
3384 }
3385
3386 case nir_op_ubitfield_extract: {
3387 pco_ref bfe = pco_ref_new_ssa32(tctx->func);
3388 pco_ubfe(&tctx->b, bfe, src[0], src[1], src[2]);
3389 instr = pco_bcsel(&tctx->b, dest, src[2], bfe, pco_zero);
3390 break;
3391 }
3392
3393 case nir_op_bitfield_insert: {
3394 pco_ref bfi = pco_ref_new_ssa32(tctx->func);
3395 pco_bfi(&tctx->b, bfi, src[0], src[1], src[2], src[3]);
3396 instr = pco_bcsel(&tctx->b, dest, src[3], bfi, src[0]);
3397 break;
3398 }
3399
3400 case nir_op_bitfield_reverse:
3401 instr = pco_rev(&tctx->b, dest, src[0]);
3402 break;
3403
3404 case nir_op_interleave:
3405 instr = pco_shuffle(&tctx->b, dest, src[0], src[1]);
3406 break;
3407
3408 case nir_op_b2f32:
3409 case nir_op_b2i32:
3410 case nir_op_f2f16:
3411 case nir_op_f2f16_rtne:
3412 case nir_op_f2f16_rtz:
3413 case nir_op_f2f32:
3414 case nir_op_f2i32:
3415 case nir_op_f2i32_rtne:
3416 case nir_op_f2u32:
3417 case nir_op_i2f32:
3418 case nir_op_i2i16:
3419 case nir_op_i2i32:
3420 case nir_op_u2f32:
3421 case nir_op_u2u16:
3422 case nir_op_u2u32:
3423 instr = trans_conv(tctx, alu->op, dest, src[0]);
3424 break;
3425
3426 case nir_op_fmin:
3427 case nir_op_fmax:
3428
3429 case nir_op_imin:
3430 case nir_op_imax:
3431
3432 case nir_op_umin:
3433 case nir_op_umax:
3434 instr = trans_min_max(tctx, alu->op, dest, src[0], src[1]);
3435 break;
3436
3437 case nir_op_pack_half_2x16:
3438 instr = pco_pck(&tctx->b,
3439 dest,
3440 src[0],
3441 .rpt = 2,
3442 .pck_fmt = PCO_PCK_FMT_F16F16);
3443 break;
3444
3445 case nir_op_unpack_half_2x16:
3446 instr = pco_unpck(&tctx->b,
3447 dest,
3448 pco_ref_elem(src[0], 0),
3449 .rpt = 2,
3450 .pck_fmt = PCO_PCK_FMT_F16F16);
3451 break;
3452
3453 case nir_op_pack_snorm_4x8:
3454 instr = pco_pck(&tctx->b,
3455 dest,
3456 src[0],
3457 .rpt = 4,
3458 .pck_fmt = PCO_PCK_FMT_S8888,
3459 .scale = true);
3460 break;
3461
3462 case nir_op_unpack_snorm_4x8:
3463 instr = pco_unpck(&tctx->b,
3464 dest,
3465 pco_ref_elem(src[0], 0),
3466 .rpt = 4,
3467 .pck_fmt = PCO_PCK_FMT_S8888,
3468 .scale = true);
3469 break;
3470
3471 case nir_op_pack_unorm_4x8:
3472 instr = pco_pck(&tctx->b,
3473 dest,
3474 src[0],
3475 .rpt = 4,
3476 .pck_fmt = PCO_PCK_FMT_U8888,
3477 .scale = true);
3478 break;
3479
3480 case nir_op_unpack_unorm_4x8:
3481 instr = pco_unpck(&tctx->b,
3482 dest,
3483 pco_ref_elem(src[0], 0),
3484 .rpt = 4,
3485 .pck_fmt = PCO_PCK_FMT_U8888,
3486 .scale = true);
3487 break;
3488
3489 case nir_op_pack_snorm_2x16:
3490 instr = pco_pck(&tctx->b,
3491 dest,
3492 src[0],
3493 .rpt = 2,
3494 .pck_fmt = PCO_PCK_FMT_S1616,
3495 .scale = true);
3496 break;
3497
3498 case nir_op_unpack_snorm_2x16:
3499 instr = pco_unpck(&tctx->b,
3500 dest,
3501 pco_ref_elem(src[0], 0),
3502 .rpt = 2,
3503 .pck_fmt = PCO_PCK_FMT_S1616,
3504 .scale = true);
3505 break;
3506
3507 case nir_op_pack_unorm_2x16:
3508 instr = pco_pck(&tctx->b,
3509 dest,
3510 src[0],
3511 .rpt = 2,
3512 .pck_fmt = PCO_PCK_FMT_U1616,
3513 .scale = true);
3514 break;
3515
3516 case nir_op_unpack_unorm_2x16:
3517 instr = pco_unpck(&tctx->b,
3518 dest,
3519 pco_ref_elem(src[0], 0),
3520 .rpt = 2,
3521 .pck_fmt = PCO_PCK_FMT_U1616,
3522 .scale = true);
3523 break;
3524
3525 case nir_op_vec2:
3526 case nir_op_vec3:
3527 case nir_op_vec4:
3528 case nir_op_vec5:
3529 case nir_op_vec8:
3530 case nir_op_vec16:
3531 instr = pco_trans_nir_vec(tctx, dest, num_srcs, src);
3532 break;
3533
3534 case nir_op_mov:
3535 instr = pco_mov(&tctx->b, dest, src[0]);
3536 break;
3537
3538 default:
3539 printf("Unsupported alu instruction: \"");
3540 nir_print_instr(&alu->instr, stdout);
3541 printf("\"\n");
3542 UNREACHABLE("");
3543 }
3544
3545 if (!pco_ref_is_scalar(dest))
3546 split_dest_comps(tctx, instr, dest);
3547
3548 return instr;
3549 }
3550
3551 /**
3552 * \brief Translates a NIR load constant instruction into PCO.
3553 *
3554 * \param[in] tctx Translation context.
3555 * \param[in] nconst The nir load constant instruction.
3556 * \return The PCO instruction.
3557 */
3558 static pco_instr *trans_const(trans_ctx *tctx, nir_load_const_instr *nconst)
3559 {
3560 unsigned num_bits = nconst->def.bit_size;
3561 unsigned chans = nconst->def.num_components;
3562
3563 pco_ref dest = pco_ref_nir_def_t(&nconst->def, tctx);
3564
3565 if (num_bits == 1) {
3566 assert(chans == 1);
3567
3568 bool val = nir_const_value_as_bool(nconst->value[0], 1);
3569 pco_ref imm_reg = pco_ref_bits(val ? pco_true : pco_zero, 1);
3570
3571 return pco_mov(&tctx->b, dest, imm_reg);
3572 }
3573
3574 /* No 64-bit support yet. */
3575 assert(num_bits <= 32);
3576
3577 if (pco_ref_is_scalar(dest)) {
3578 assert(chans == 1);
3579
3580 uint64_t val = nir_const_value_as_uint(nconst->value[0], num_bits);
3581 pco_ref imm =
3582 pco_ref_imm(val, pco_bits(num_bits), pco_ref_get_dtype(dest));
3583
3584 /* Override to 32-bit. */
3585 dest = pco_ref_bits(dest, 32);
3586 imm = pco_ref_bits(imm, 32);
3587
3588 return pco_movi32(&tctx->b, dest, imm);
3589 }
3590
3591 pco_ref comps[NIR_MAX_VEC_COMPONENTS] = { 0 };
3592 for (unsigned c = 0; c < chans; ++c) {
3593 comps[c] = pco_ref_new_ssa(tctx->func, pco_ref_get_bits(dest), 1);
3594
3595 uint64_t val = nir_const_value_as_uint(nconst->value[c], num_bits);
3596 pco_ref imm =
3597 pco_ref_imm(val, pco_bits(num_bits), pco_ref_get_dtype(dest));
3598
3599 pco_movi32(&tctx->b, comps[c], imm);
3600 }
3601
3602 pco_instr *instr = pco_vec(&tctx->b, dest, chans, comps);
3603
3604 split_dest_comps(tctx, instr, dest);
3605
3606 return instr;
3607 }
3608
3609 static pco_instr *trans_undef(trans_ctx *tctx, nir_undef_instr *undef)
3610 {
3611 unsigned num_bits = undef->def.bit_size;
3612 unsigned chans = undef->def.num_components;
3613
3614 pco_ref dest = pco_ref_nir_def_t(&undef->def, tctx);
3615
3616 if (num_bits == 1) {
3617 assert(chans == 1);
3618 return pco_mov(&tctx->b, dest, pco_ref_bits(pco_zero, 1));
3619 }
3620
3621 assert(num_bits == 32);
3622
3623 if (pco_ref_is_scalar(dest)) {
3624 assert(chans == 1);
3625
3626 pco_ref imm = pco_ref_imm(0, pco_bits(num_bits), pco_ref_get_dtype(dest));
3627
3628 return pco_movi32(&tctx->b, dest, imm);
3629 }
3630
3631 pco_ref comps[NIR_MAX_VEC_COMPONENTS];
3632 for (unsigned c = 0; c < chans; ++c)
3633 comps[c] = pco_zero;
3634
3635 pco_instr *instr = pco_vec(&tctx->b, dest, chans, comps);
3636
3637 split_dest_comps(tctx, instr, dest);
3638
3639 return instr;
3640 }
3641
3642 /**
3643 * \brief Translates a NIR jump instruction into PCO.
3644 *
3645 * \param[in] tctx Translation context.
3646 * \param[in] jump The NIR jump instruction.
3647 * \return The PCO instruction.
3648 */
3649 static pco_instr *trans_jump(trans_ctx *tctx, nir_jump_instr *jump)
3650 {
3651 switch (jump->type) {
3652 case nir_jump_break:
3653 return pco_break(&tctx->b);
3654
3655 case nir_jump_continue:
3656 return pco_continue(&tctx->b);
3657
3658 default:
3659 break;
3660 }
3661
3662 UNREACHABLE("");
3663 }
3664
3665 /**
3666 * \brief Translates a NIR instruction into PCO.
3667 *
3668 * \param[in] tctx Translation context.
3669 * \param[in] ninstr The nir instruction.
3670 * \return The PCO instruction.
3671 */
3672 static pco_instr *trans_instr(trans_ctx *tctx, nir_instr *ninstr)
3673 {
3674 switch (ninstr->type) {
3675 case nir_instr_type_intrinsic:
3676 return trans_intr(tctx, nir_instr_as_intrinsic(ninstr));
3677
3678 case nir_instr_type_load_const:
3679 return trans_const(tctx, nir_instr_as_load_const(ninstr));
3680
3681 case nir_instr_type_alu:
3682 return trans_alu(tctx, nir_instr_as_alu(ninstr));
3683
3684 case nir_instr_type_jump:
3685 return trans_jump(tctx, nir_instr_as_jump(ninstr));
3686
3687 case nir_instr_type_undef:
3688 return trans_undef(tctx, nir_instr_as_undef(ninstr));
3689
3690 default:
3691 break;
3692 }
3693
3694 UNREACHABLE("");
3695 }
3696
3697 /**
3698 * \brief Translates a NIR block into PCO.
3699 *
3700 * \param[in] tctx Translation context.
3701 * \param[in] parent_cf_node The parent cf node.
3702 * \param[in] cf_node_list The PCO cf node list.
3703 * \param[in] nblock The nir block.
3704 * \return The PCO block.
3705 */
3706 static pco_block *trans_block(trans_ctx *tctx,
3707 pco_cf_node *parent_cf_node,
3708 struct list_head *cf_node_list,
3709 nir_block *nblock)
3710 {
3711 pco_block *block = pco_block_create(tctx->func);
3712
3713 block->cf_node.flag = tctx->flag;
3714 block->cf_node.parent = parent_cf_node;
3715 list_addtail(&block->cf_node.link, cf_node_list);
3716
3717 tctx->b = pco_builder_create(tctx->func, pco_cursor_after_block(block));
3718
3719 nir_foreach_instr (ninstr, nblock) {
3720 trans_instr(tctx, ninstr);
3721 }
3722
3723 return block;
3724 }
3725
3726 /**
3727 * \brief Translates a NIR if into PCO.
3728 *
3729 * \param[in] tctx Translation context.
3730 * \param[in] parent_cf_node The parent cf node.
3731 * \param[in] cf_node_list The PCO cf node list.
3732 * \param[in] nif The nir if.
3733 */
3734 static void trans_if(trans_ctx *tctx,
3735 pco_cf_node *parent_cf_node,
3736 struct list_head *cf_node_list,
3737 nir_if *nif)
3738 {
3739 pco_if *pif = pco_if_create(tctx->func);
3740
3741 pif->cf_node.flag = tctx->flag;
3742 pif->cf_node.parent = parent_cf_node;
3743 list_addtail(&pif->cf_node.link, cf_node_list);
3744
3745 pif->cond = pco_ref_nir_src_t(&nif->condition, tctx);
3746 assert(pco_ref_is_scalar(pif->cond));
3747
3748 bool has_then = !nir_cf_list_is_empty_block(&nif->then_list);
3749 bool has_else = !nir_cf_list_is_empty_block(&nif->else_list);
3750 assert(has_then || has_else);
3751
3752 enum pco_cf_node_flag flag = tctx->flag;
3753 if (has_then) {
3754 tctx->flag = PCO_CF_NODE_FLAG_IF_THEN;
3755 trans_cf_nodes(tctx, &pif->cf_node, &pif->then_body, &nif->then_list);
3756 }
3757
3758 if (has_else) {
3759 tctx->flag = PCO_CF_NODE_FLAG_IF_ELSE;
3760 trans_cf_nodes(tctx, &pif->cf_node, &pif->else_body, &nif->else_list);
3761 }
3762
3763 tctx->flag = flag;
3764 }
3765
3766 /**
3767 * \brief Translates a NIR loop into PCO.
3768 *
3769 * \param[in] tctx Translation context.
3770 * \param[in] parent_cf_node The parent cf node.
3771 * \param[in] cf_node_list The PCO cf node list.
3772 * \param[in] nloop The nir loop.
3773 */
3774 static void trans_loop(trans_ctx *tctx,
3775 pco_cf_node *parent_cf_node,
3776 struct list_head *cf_node_list,
3777 nir_loop *nloop)
3778 {
3779 pco_loop *loop = pco_loop_create(tctx->func);
3780
3781 loop->cf_node.flag = tctx->flag;
3782 loop->cf_node.parent = parent_cf_node;
3783 list_addtail(&loop->cf_node.link, cf_node_list);
3784
3785 assert(!nir_cf_list_is_empty_block(&nloop->body));
3786 assert(!nir_loop_has_continue_construct(nloop));
3787
3788 enum pco_cf_node_flag flag = tctx->flag;
3789 tctx->flag = PCO_CF_NODE_FLAG_BODY;
3790
3791 trans_cf_nodes(tctx, &loop->cf_node, &loop->body, &nloop->body);
3792
3793 tctx->flag = flag;
3794 }
3795
3796 /**
3797 * \brief Translates a NIR function into PCO.
3798 *
3799 * \param[in] tctx Translation context.
3800 * \param[in] impl The nir function impl.
3801 * \return The PCO function.
3802 */
3803 static pco_func *trans_func(trans_ctx *tctx, nir_function_impl *impl)
3804 {
3805 nir_function *nfunc = impl->function;
3806 enum pco_func_type func_type = PCO_FUNC_TYPE_CALLABLE;
3807
3808 if (nfunc->is_preamble)
3809 func_type = PCO_FUNC_TYPE_PREAMBLE;
3810 else if (nfunc->is_entrypoint)
3811 func_type = PCO_FUNC_TYPE_ENTRYPOINT;
3812
3813 pco_func *func = pco_func_create(tctx->shader, func_type, nfunc->num_params);
3814 tctx->func = func;
3815
3816 func->name = ralloc_strdup(func, nfunc->name);
3817 func->next_ssa = impl->ssa_alloc;
3818
3819 /* TODO: Function parameter support. */
3820 assert(func->num_params == 0 && func->params == NULL);
3821
3822 /* Gather types. */
3823 tctx->float_types = BITSET_RZALLOC(NULL, impl->ssa_alloc);
3824 tctx->int_types = BITSET_RZALLOC(NULL, impl->ssa_alloc);
3825 nir_gather_types(impl, tctx->float_types, tctx->int_types);
3826
3827 tctx->flag = PCO_CF_NODE_FLAG_BODY;
3828 trans_cf_nodes(tctx, &func->cf_node, &func->body, &impl->body);
3829
3830 ralloc_free(tctx->float_types);
3831 ralloc_free(tctx->int_types);
3832
3833 return func;
3834 }
3835
3836 /**
3837 * \brief Translates NIR control flow nodes into PCO.
3838 *
3839 * \param[in] tctx Translation context.
3840 * \param[in] parent_cf_node The parent cf node.
3841 * \param[in] cf_node_list The PCO cf node list.
3842 * \param[in,out] nir_cf_node_list The NIR cf node list.
3843 * \return The first block from the cf nodes.
3844 */
3845 static pco_block *trans_cf_nodes(trans_ctx *tctx,
3846 pco_cf_node *parent_cf_node,
3847 struct list_head *cf_node_list,
3848 struct exec_list *nir_cf_node_list)
3849 {
3850 pco_block *start_block = NULL;
3851
3852 foreach_list_typed (nir_cf_node, ncf_node, node, nir_cf_node_list) {
3853 switch (ncf_node->type) {
3854 case nir_cf_node_block: {
3855 pco_block *block = trans_block(tctx,
3856 parent_cf_node,
3857 cf_node_list,
3858 nir_cf_node_as_block(ncf_node));
3859 if (!start_block)
3860 start_block = block;
3861 break;
3862 }
3863
3864 case nir_cf_node_if: {
3865 trans_if(tctx,
3866 parent_cf_node,
3867 cf_node_list,
3868 nir_cf_node_as_if(ncf_node));
3869 break;
3870 }
3871
3872 case nir_cf_node_loop: {
3873 trans_loop(tctx,
3874 parent_cf_node,
3875 cf_node_list,
3876 nir_cf_node_as_loop(ncf_node));
3877 break;
3878 }
3879
3880 default:
3881 UNREACHABLE("");
3882 }
3883 }
3884
3885 return start_block;
3886 }
3887
3888 /**
3889 * \brief Translates a NIR shader into a PCO shader.
3890 *
3891 * \param[in] ctx PCO compiler context.
3892 * \param[in] nir NIR shader.
3893 * \param[in] data Shader-specific data.
3894 * \param[in] mem_ctx Ralloc memory allocation context.
3895 * \return The PCO shader.
3896 */
3897 pco_shader *
3898 pco_trans_nir(pco_ctx *ctx, nir_shader *nir, pco_data *data, void *mem_ctx)
3899 {
3900 pco_shader *shader = pco_shader_create(ctx, nir, mem_ctx);
3901
3902 if (data)
3903 memcpy(&shader->data, data, sizeof(*data));
3904
3905 trans_ctx tctx = {
3906 .pco_ctx = ctx,
3907 .shader = shader,
3908 .stage = shader->stage,
3909 .ftz32b =
3910 nir_is_rounding_mode_rtz(nir->info.float_controls_execution_mode, 32),
3911 };
3912
3913 if (shader->stage == MESA_SHADER_FRAGMENT)
3914 tctx.olchk = !data->fs.uses.olchk_skip;
3915
3916 nir_foreach_function_with_impl (func, impl, nir) {
3917 trans_func(&tctx, impl);
3918 }
3919
3920 if (pco_should_print_shader(shader))
3921 pco_print_shader(shader, stdout, "before passes");
3922
3923 return shader;
3924 }