Debian Codegraph @

src/imagination/pco/pco_legalize.c

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pco_legalize.c
9 *
10 * \brief PCO legalizing pass.
11 */
12
13 #include "pco.h"
14 #include "pco_builder.h"
15 #include "pco_internal.h"
16 #include "util/macros.h"
17
18 #include <stdbool.h>
19
20 static inline bool ref_needs_olchk(pco_ref ref)
21 {
22 if (!pco_ref_is_reg(ref))
23 return false;
24
25 switch (pco_ref_get_reg_class(ref)) {
26 case PCO_REG_CLASS_PIXOUT:
27 return true;
28
29 case PCO_REG_CLASS_SPEC:
30 return ref.val == PCO_SR_OUTPUT_PART ||
31 (ref.val >= PCO_SR_TILED_LD_COMP0 &&
32 ref.val <= PCO_SR_TILED_ST_COMP3) ||
33 (ref.val >= PCO_SR_TILED_LD_COMP4 &&
34 ref.val <= PCO_SR_TILED_ST_COMP7);
35
36 default:
37 break;
38 }
39
40 return false;
41 }
42
43 /**
44 * \brief Insert a mov to legalize how a hardware register is referenced.
45 *
46 * \param[in,out] instr PCO instr.
47 * \param[in,out] ref Reference to be legalized.
48 * \param[in] needs_s124 Whether the mapping needs to use S{1,2,4}
49 * rather than S{0,2,3}.
50 * \return True if progress was made.
51 */
52 static void insert_mov_ref(pco_instr *instr, pco_ref *ref, bool needs_s124)
53 {
54 if (instr->op == PCO_OP_MBYP && needs_s124 && !pco_ref_has_mods_set(*ref)) {
55 instr->op = PCO_OP_MOVS1;
56 return;
57 } else if (instr->op == PCO_OP_MOVS1 && !needs_s124) {
58 instr->op = PCO_OP_MBYP;
59 return;
60 }
61
62 assert(pco_ref_is_scalar(*ref));
63 pco_ref new_ref = pco_ref_new_ssa(instr->parent_func,
64 pco_ref_get_bits(*ref),
65 pco_ref_get_chans(*ref));
66
67 pco_ref_xfer_mods(&new_ref, ref, true);
68
69 pco_builder b =
70 pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr));
71
72 enum pco_exec_cnd exec_cnd = pco_instr_get_exec_cnd(instr);
73 pco_instr *mov_instr;
74 if (needs_s124)
75 mov_instr = pco_movs1(&b, new_ref, *ref, .exec_cnd = exec_cnd);
76 else
77 mov_instr = pco_mbyp(&b, new_ref, *ref, .exec_cnd = exec_cnd);
78
79 if (pco_instr_has_olchk(instr) && pco_instr_get_olchk(instr) &&
80 ref_needs_olchk(*ref)) {
81 assert(pco_instr_has_olchk(mov_instr));
82 pco_instr_set_olchk(mov_instr, true);
83 pco_instr_set_olchk(instr, false);
84 }
85
86 *ref = new_ref;
87 }
88
89 /**
90 * \brief Try to legalize an instruction's hardware source mappings.
91 *
92 * \param[in,out] instr PCO instr.
93 * \param[in] info PCO op info.
94 * \return True if progress was made.
95 */
96 static bool try_legalize_src_mappings(pco_instr *instr,
97 const struct pco_op_info *info)
98 {
99 bool progress = false;
100 bool needs_s124;
101
102 /* Check dests. */
103 pco_foreach_instr_dest (pdest, instr) {
104 unsigned dest_index = pdest - instr->dest;
105 if (!info->dest_intrn_map[dest_index])
106 continue;
107
108 enum pco_io mapped_src = PCO_IO_S0 + info->dest_intrn_map[dest_index] - 1;
109
110 if (ref_src_map_valid(*pdest, mapped_src, &needs_s124))
111 continue;
112
113 insert_mov_ref(instr, pdest, needs_s124);
114 progress = true;
115 }
116
117 /* Check srcs. */
118 pco_foreach_instr_src (psrc, instr) {
119 unsigned src_index = psrc - instr->src;
120 if (!info->src_intrn_map[src_index])
121 continue;
122
123 enum pco_io mapped_src = PCO_IO_S0 + info->src_intrn_map[src_index] - 1;
124
125 if (ref_src_map_valid(*psrc, mapped_src, &needs_s124))
126 continue;
127
128 insert_mov_ref(instr, psrc, needs_s124);
129 progress = true;
130 }
131
132 return progress;
133 }
134
135 static inline bool xfer_op_mods(pco_instr *dest, pco_instr *src)
136 {
137 bool all_xfered = true;
138
139 for (enum pco_op_mod mod = PCO_OP_MOD_NONE + 1; mod < _PCO_OP_MOD_COUNT;
140 ++mod) {
141 bool dest_has_mod = pco_instr_has_mod(dest, mod);
142 bool src_has_mod = pco_instr_has_mod(src, mod);
143
144 if (!dest_has_mod && !src_has_mod)
145 continue;
146
147 if (dest_has_mod != src_has_mod) {
148 all_xfered = false;
149 continue;
150 }
151
152 pco_instr_set_mod(dest, mod, pco_instr_get_mod(src, mod));
153 }
154
155 return all_xfered;
156 }
157
158 /**
159 * \brief Legalize fence pseudo instruction.
160 *
161 * \param[in,out] instr PCO instr.
162 * \return True if progress was made.
163 */
164 static bool legalize_fence(pco_instr *instr)
165 {
166 pco_builder b =
167 pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr));
168
169 pco_flush_p0(&b);
170 pco_br_next(&b, .exec_cnd = PCO_EXEC_CND_E1_Z1);
171 pco_br_next(&b, .exec_cnd = PCO_EXEC_CND_E1_Z0);
172
173 pco_instr_delete(instr);
174
175 return true;
176 }
177
178 static bool legalize_pseudo(pco_instr *instr)
179 {
180 switch (instr->op) {
181 case PCO_OP_FENCE:
182 return legalize_fence(instr);
183
184 case PCO_OP_MOV:
185 if (pco_ref_is_reg(instr->src[0]) &&
186 pco_ref_get_reg_class(instr->src[0]) == PCO_REG_CLASS_SPEC)
187 instr->op = PCO_OP_MOVS1;
188 else
189 instr->op = PCO_OP_MBYP;
190
191 return true;
192
193 case PCO_OP_MOV_OFFSET: {
194 pco_builder b =
195 pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr));
196
197 pco_ref dest = instr->dest[0];
198 pco_ref src = instr->src[0];
199 pco_ref offset = instr->src[1];
200
201 unsigned idx_reg_num = 0;
202 pco_ref idx_reg =
203 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
204
205 pco_mbyp(&b, idx_reg, offset, .exec_cnd = pco_instr_get_exec_cnd(instr));
206
207 if (pco_instr_get_offset_sd(instr) == PCO_OFFSET_SD_SRC)
208 src = pco_ref_hwreg_idx_from(idx_reg_num, src);
209 else
210 dest = pco_ref_hwreg_idx_from(idx_reg_num, dest);
211
212 pco_instr *mbyp = pco_mbyp(&b, dest, src);
213 xfer_op_mods(mbyp, instr);
214
215 pco_instr_delete(instr);
216
217 return true;
218 }
219
220 case PCO_OP_OP_ATOMIC_OFFSET: {
221 pco_builder b =
222 pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr));
223
224 pco_ref dest = instr->dest[0];
225 pco_ref shmem_dest = instr->dest[1];
226
227 pco_ref shmem_src = instr->src[0];
228 pco_ref value = instr->src[1];
229 pco_ref value_swap = instr->src[2];
230 pco_ref offset = instr->src[3];
231
232 unsigned idx_reg_num = 0;
233 pco_ref idx_reg =
234 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
235
236 pco_mbyp(&b, idx_reg, offset, .exec_cnd = pco_instr_get_exec_cnd(instr));
237
238 shmem_dest = pco_ref_hwreg_idx_from(idx_reg_num, shmem_dest);
239 shmem_src = pco_ref_hwreg_idx_from(idx_reg_num, shmem_src);
240
241 pco_instr *repl;
242 enum pco_atom_op atom_op = pco_instr_get_atom_op(instr);
243 switch (atom_op) {
244 case PCO_ATOM_OP_ADD:
245 assert(pco_ref_is_null(value_swap));
246 repl = pco_iadd32_atomic(&b,
247 dest,
248 shmem_dest,
249 shmem_src,
250 value,
251 pco_ref_null(),
252 .s = true);
253 break;
254
255 case PCO_ATOM_OP_XCHG:
256 assert(pco_ref_is_null(value_swap));
257 repl = pco_xchg_atomic(&b, dest, shmem_dest, shmem_src, value);
258 break;
259
260 case PCO_ATOM_OP_CMPXCHG:
261 assert(!pco_ref_is_null(value_swap));
262 repl = pco_cmpxchg_atomic(&b,
263 dest,
264 shmem_dest,
265 shmem_src,
266 value,
267 value_swap,
268 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
269 break;
270
271 case PCO_ATOM_OP_UMIN:
272 assert(pco_ref_is_null(value_swap));
273 repl = pco_min_atomic(&b,
274 dest,
275 shmem_dest,
276 shmem_src,
277 value,
278 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
279 break;
280
281 case PCO_ATOM_OP_IMIN:
282 assert(pco_ref_is_null(value_swap));
283 repl = pco_min_atomic(&b,
284 dest,
285 shmem_dest,
286 shmem_src,
287 value,
288 .tst_type_main = PCO_TST_TYPE_MAIN_S32);
289 break;
290
291 case PCO_ATOM_OP_UMAX:
292 assert(pco_ref_is_null(value_swap));
293 repl = pco_max_atomic(&b,
294 dest,
295 shmem_dest,
296 shmem_src,
297 value,
298 .tst_type_main = PCO_TST_TYPE_MAIN_U32);
299 break;
300
301 case PCO_ATOM_OP_IMAX:
302 assert(pco_ref_is_null(value_swap));
303 repl = pco_max_atomic(&b,
304 dest,
305 shmem_dest,
306 shmem_src,
307 value,
308 .tst_type_main = PCO_TST_TYPE_MAIN_S32);
309 break;
310
311 case PCO_ATOM_OP_AND:
312 assert(pco_ref_is_null(value_swap));
313 repl = pco_logical_atomic(&b,
314 dest,
315 shmem_dest,
316 shmem_src,
317 value,
318 .logiop = PCO_LOGIOP_AND);
319 break;
320
321 case PCO_ATOM_OP_OR:
322 assert(pco_ref_is_null(value_swap));
323 repl = pco_logical_atomic(&b,
324 dest,
325 shmem_dest,
326 shmem_src,
327 value,
328 .logiop = PCO_LOGIOP_OR);
329 break;
330
331 case PCO_ATOM_OP_XOR:
332 assert(pco_ref_is_null(value_swap));
333 repl = pco_logical_atomic(&b,
334 dest,
335 shmem_dest,
336 shmem_src,
337 value,
338 .logiop = PCO_LOGIOP_XOR);
339 break;
340
341 default:
342 UNREACHABLE("");
343 }
344
345 xfer_op_mods(repl, instr);
346
347 pco_instr_delete(instr);
348
349 return true;
350 }
351
352 default:
353 break;
354 }
355
356 return false;
357 }
358
359 static bool try_legalize_large_hwreg_offsets(pco_instr *instr,
360 const struct pco_op_info *info)
361 {
362 unsigned large_hwreg_count = 0;
363 pco_ref *large_hwregs[_PCO_OP_MAX_SRCS + _PCO_OP_MAX_DESTS] = { 0 };
364
365 enum pco_reg_class hwreg_class = ~0;
366 unsigned min_large_offset = ~0;
367 ASSERTED unsigned max_large_offset = 0;
368
369 /* Check dests. */
370 pco_foreach_instr_dest_hwreg (pdest, instr) {
371 if (pco_ref_get_reg_index(*pdest) < 256)
372 continue;
373
374 large_hwregs[large_hwreg_count++] = pdest;
375 assert(hwreg_class == ~0 || hwreg_class == pco_ref_get_reg_class(*pdest));
376 hwreg_class = pco_ref_get_reg_class(*pdest);
377
378 min_large_offset = MIN2(min_large_offset, pco_ref_get_reg_index(*pdest));
379 max_large_offset = MAX2(max_large_offset, pco_ref_get_reg_index(*pdest));
380 }
381
382 /* Check srcs. */
383 pco_foreach_instr_src_hwreg (psrc, instr) {
384 if (pco_ref_get_reg_index(*psrc) < 256)
385 continue;
386
387 large_hwregs[large_hwreg_count++] = psrc;
388 assert(hwreg_class == ~0 || hwreg_class == pco_ref_get_reg_class(*psrc));
389 hwreg_class = pco_ref_get_reg_class(*psrc);
390
391 min_large_offset = MIN2(min_large_offset, pco_ref_get_reg_index(*psrc));
392 max_large_offset = MAX2(max_large_offset, pco_ref_get_reg_index(*psrc));
393 }
394
395 if (!large_hwreg_count)
396 return false;
397
398 /* We'd need more than one indexed register to support this. */
399 assert((max_large_offset - min_large_offset) < 256);
400
401 pco_builder b =
402 pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr));
403
404 unsigned idx_reg_num = 0;
405 pco_ref idx_reg =
406 pco_ref_hwreg_idx(idx_reg_num, idx_reg_num, PCO_REG_CLASS_INDEX);
407
408 pco_ref imm = pco_ref_imm32(min_large_offset);
409 pco_movi32(&b, idx_reg, imm, .exec_cnd = pco_instr_get_exec_cnd(instr));
410
411 /* Remap the offset for each large hwreg and replace it with the indexed
412 * register.
413 */
414 for (unsigned u = 0; u < large_hwreg_count; ++u) {
415 *large_hwregs[u] = pco_ref_offset(*large_hwregs[u], -min_large_offset);
416 *large_hwregs[u] = pco_ref_hwreg_idx_from(idx_reg_num, *large_hwregs[u]);
417 }
418
419 return true;
420 }
421
422 /**
423 * \brief Insert pseudo fences around DITR and DITRP instructions.
424 *
425 * \param[in,out] instr PCO instr.
426 * \return True if progress was made.
427 */
428 static bool try_legalize_ditr_fence(pco_instr *instr)
429 {
430 if (instr->op != PCO_OP_DITR && instr->op != PCO_OP_DITRP)
431 return false;
432
433 pco_builder b =
434 pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr));
435 pco_fence(&b);
436
437 b.cursor = pco_cursor_after_instr(instr);
438 pco_fence(&b);
439
440 return true;
441 }
442
443 /**
444 * \brief Try to legalizes an instruction.
445 *
446 * \param[in,out] instr PCO instr.
447 * \return True if progress was made.
448 */
449 static bool try_legalize(pco_instr *instr)
450 {
451 const struct pco_op_info *info = &pco_op_info[instr->op];
452 bool progress = false;
453
454 progress |= try_legalize_large_hwreg_offsets(instr, info);
455 progress |= try_legalize_ditr_fence(instr);
456
457 return progress;
458 }
459
460 /**
461 * \brief Legalizes instructions where additional restrictions apply.
462 * This should be run after register allocation.
463 *
464 * \param[in,out] shader PCO shader.
465 * \return True if the pass made progress.
466 */
467 bool pco_pre_ra_legalize(pco_shader *shader)
468 {
469 bool progress = false;
470
471 assert(!shader->is_grouped);
472 assert(!shader->is_legalized);
473
474 const struct pco_op_info *info;
475
476 pco_foreach_func_in_shader (func, shader) {
477 pco_foreach_instr_in_func_safe (instr, func) {
478 info = &pco_op_info[instr->op];
479 if (info->type != PCO_OP_TYPE_PSEUDO)
480 progress |= try_legalize_src_mappings(instr, info);
481 }
482 }
483
484 return progress;
485 }
486
487 /**
488 * \brief Post-RA legalization pass.
489 *
490 * \param[in,out] shader PCO shader.
491 * \return True if the pass made progress.
492 */
493 bool pco_post_ra_legalize(pco_shader *shader)
494 {
495 assert(!shader->is_grouped);
496
497 bool progress = false;
498
499 pco_foreach_func_in_shader (func, shader) {
500 pco_foreach_instr_in_func_safe (instr, func) {
501 progress |= try_legalize(instr);
502 }
503 }
504
505 const struct pco_op_info *info;
506
507 pco_foreach_func_in_shader (func, shader) {
508 pco_foreach_instr_in_func_safe (instr, func) {
509 info = &pco_op_info[instr->op];
510 if (info->type == PCO_OP_TYPE_PSEUDO)
511 progress |= legalize_pseudo(instr);
512 }
513 }
514
515 shader->is_legalized = true;
516 return progress;
517 }