Debian Codegraph @

src/imagination/pco/pco_internal.h

raw
1 /*
2 * Copyright © 2024 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #ifndef PCO_INTERNAL_H
8 #define PCO_INTERNAL_H
9
10 /**
11 * \file pco_internal.h
12 *
13 * \brief PCO internal header.
14 */
15
16 #include "compiler/spirv/nir_spirv.h"
17 #include "hwdef/rogue_hw_utils.h"
18 #include "pco.h"
19 #include "pco_common.h"
20 #include "pco_data.h"
21 #include "pco_ops.h"
22 #include "spirv/nir_spirv.h"
23 #include "util/compiler.h"
24 #include "util/hash_table.h"
25 #include "util/macros.h"
26 #include "util/list.h"
27 #include "util/u_debug.h"
28 #include "util/u_dynarray.h"
29 #include "util/u_math.h"
30
31 #include <assert.h>
32 #include <stdbool.h>
33 #include <stdint.h>
34
35 /** PCO compiler context. */
36 typedef struct _pco_ctx {
37 /** Device information. */
38 const struct pvr_device_info *dev_info;
39
40 /** Device-specific NIR options. */
41 nir_shader_compiler_options nir_options;
42
43 /** Device-specific SPIR-V to NIR options. */
44 struct spirv_to_nir_options spirv_options;
45
46 /** USC library. */
47 const nir_shader *usclib;
48 } pco_ctx;
49
50 void pco_setup_spirv_options(const struct pvr_device_info *dev_info,
51 struct spirv_to_nir_options *spirv_options);
52 void pco_setup_nir_options(const struct pvr_device_info *dev_info,
53 nir_shader_compiler_options *nir_options);
54
55 /* Debug. */
56 enum pco_debug {
57 PCO_DEBUG_VAL_SKIP = BITFIELD64_BIT(0),
58 PCO_DEBUG_REINDEX = BITFIELD64_BIT(1),
59 PCO_DEBUG_NO_PRED_CF = BITFIELD64_BIT(2),
60 PCO_DEBUG_ALLOC_EXTRA_VTXINS = BITFIELD64_BIT(3),
61 PCO_DEBUG_INT_SMP = BITFIELD64_BIT(4),
62 };
63
64 extern uint64_t pco_debug;
65
66 #define PCO_DEBUG(flag) unlikely(pco_debug &(PCO_DEBUG_##flag))
67
68 enum pco_debug_print {
69 PCO_DEBUG_PRINT_VS = BITFIELD64_BIT(0),
70 PCO_DEBUG_PRINT_FS = BITFIELD64_BIT(1),
71 PCO_DEBUG_PRINT_CS = BITFIELD64_BIT(2),
72 PCO_DEBUG_PRINT_ALL = PCO_DEBUG_PRINT_VS | PCO_DEBUG_PRINT_FS |
73 PCO_DEBUG_PRINT_CS,
74 PCO_DEBUG_PRINT_INTERNAL = BITFIELD64_BIT(3),
75 PCO_DEBUG_PRINT_PASSES = BITFIELD64_BIT(4),
76 PCO_DEBUG_PRINT_NIR = BITFIELD64_BIT(5),
77 PCO_DEBUG_PRINT_BINARY = BITFIELD64_BIT(6),
78 PCO_DEBUG_PRINT_VERBOSE = BITFIELD64_BIT(7),
79 PCO_DEBUG_PRINT_RA = BITFIELD64_BIT(8),
80 PCO_DEBUG_PRINT_STATS = BITFIELD64_BIT(9),
81 };
82
83 extern uint64_t pco_debug_print;
84
85 extern const char *pco_skip_passes;
86
87 #define PCO_DEBUG_PRINT(flag) \
88 unlikely(pco_debug_print &(PCO_DEBUG_PRINT_##flag))
89
90 extern bool pco_color;
91
92 void pco_debug_init(void);
93
94 typedef struct _pco_cf_node pco_cf_node;
95 typedef struct _pco_func pco_func;
96 typedef struct _pco_block pco_block;
97 typedef struct _pco_instr pco_instr;
98
99 #define PCO_REF_VAL_BITS (32U)
100
101 #define PCO_REF_IDX_NUM_BITS (2U)
102 #define PCO_REF_IDX_OFFSET_BITS (8U)
103 #define PCO_REF_IDX_PAD_BITS \
104 (PCO_REF_VAL_BITS - (PCO_REF_IDX_NUM_BITS + PCO_REF_IDX_OFFSET_BITS))
105
106 /** PCO reference index. */
107 typedef struct PACKED _pco_ref {
108 /** Reference value. */
109 union PACKED {
110 unsigned val : PCO_REF_VAL_BITS;
111
112 struct PACKED {
113 unsigned num : PCO_REF_IDX_NUM_BITS; /** Index register number. */
114 unsigned offset : PCO_REF_IDX_OFFSET_BITS; /** Offset. */
115 unsigned _pad : PCO_REF_IDX_PAD_BITS;
116 } idx_reg;
117 };
118
119 /** Source/destination modifiers. */
120 bool oneminus : 1;
121 bool clamp : 1;
122 bool flr : 1;
123 bool abs : 1;
124 bool neg : 1;
125 enum pco_elem elem : 4; /** .e0.e1.e2.e3 */
126
127 enum pco_dtype dtype : 2; /** Reference data-type. */
128 unsigned chans : 10; /** Number of channels (1-1024). */
129 enum pco_bits bits : 3; /** Bit width. */
130 enum pco_ref_type type : 3; /** Reference type. */
131 enum pco_reg_class reg_class : 4; /** Register class. */
132
133 unsigned _pad : 1;
134 } pco_ref;
135 static_assert(sizeof(pco_ref) == 8, "sizeof(pco_ref) != 8");
136
137 /** PCO phi source. */
138 typedef struct _pco_phi_src {
139 struct list_head link; /** Link in pco_instr::phi_srcs. */
140
141 pco_block *pred; /** Predecessor block. */
142 pco_ref ref; /** Source reference. */
143 } pco_phi_src;
144
145 /** PCO instruction group. */
146 typedef struct _pco_igrp {
147 struct list_head link; /** Node in pco_block::instrs. */
148 pco_block *parent_block; /** Basic block containing the igrp. */
149 pco_func *parent_func; /** Parent function. */
150
151 pco_instr *instrs[_PCO_OP_PHASE_COUNT]; /** Instruction/group list. */
152
153 /** Instruction group header. */
154 struct {
155 unsigned da;
156 unsigned length;
157 union {
158 enum pco_oporg oporg;
159 enum pco_opcnt opcnt;
160 };
161 bool olchk;
162 bool w1p;
163 bool w0p;
164 enum pco_cc cc;
165 enum pco_alutype alutype;
166 union {
167 struct {
168 bool end;
169 bool atom;
170 unsigned rpt;
171 };
172 struct {
173 unsigned miscctl;
174 enum pco_ctrlop ctrlop;
175 };
176 };
177 } hdr;
178
179 struct {
180 pco_ref s[ROGUE_MAX_ALU_INPUTS];
181 } srcs;
182
183 struct {
184 pco_ref is[ROGUE_MAX_ALU_INTERNAL_SOURCES];
185 } iss;
186
187 struct {
188 pco_ref w[ROGUE_MAX_ALU_OUTPUTS];
189 } dests;
190
191 struct {
192 enum pco_igrp_hdr_variant hdr;
193 union {
194 enum pco_main_variant main;
195 enum pco_backend_variant backend;
196 enum pco_bitwise_variant bitwise;
197 enum pco_ctrl_variant control;
198 } instr[_PCO_OP_PHASE_COUNT];
199 enum pco_src_variant lower_src;
200 enum pco_src_variant upper_src;
201 enum pco_iss_variant iss;
202 enum pco_dst_variant dest;
203 } variant;
204
205 struct {
206 struct {
207 unsigned hdr;
208 unsigned lower_srcs;
209 unsigned upper_srcs;
210 unsigned iss;
211 unsigned dests;
212 unsigned instrs[_PCO_OP_PHASE_COUNT];
213 unsigned word_padding;
214 unsigned align_padding;
215 unsigned total;
216 } len;
217
218 unsigned offset;
219 } enc;
220
221 unsigned index; /** Igrp index. */
222 char *comment; /** Comment string. */
223
224 } pco_igrp;
225
226 /** PCO instruction. */
227 typedef struct _pco_instr {
228 union {
229 struct {
230 struct list_head link; /** Node in pco_block::instrs. */
231 pco_block *parent_block; /** Basic block containing the instruction. */
232 };
233
234 struct {
235 enum pco_op_phase phase; /** Igrp phase the instruction is in. */
236 pco_igrp *parent_igrp; /** Igrp containing the instruction. */
237 };
238 };
239
240 pco_func *parent_func; /** Parent function. */
241
242 enum pco_op op;
243
244 unsigned num_dests;
245 pco_ref *dest;
246 unsigned num_srcs;
247 pco_ref *src;
248
249 union {
250 struct list_head phi_srcs;
251 pco_cf_node *target_cf_node;
252 };
253
254 /** Instruction flags/modifiers. */
255 uint32_t mod[_PCO_OP_MAX_MODS];
256
257 unsigned index; /** Instruction index. */
258 char *comment; /** Comment string. */
259 } pco_instr;
260
261 /** PCO control-flow node type. */
262 enum pco_cf_node_type {
263 PCO_CF_NODE_TYPE_BLOCK,
264 PCO_CF_NODE_TYPE_IF,
265 PCO_CF_NODE_TYPE_LOOP,
266 PCO_CF_NODE_TYPE_FUNC,
267 };
268
269 /** PCO control-flow flag. */
270 enum pco_cf_node_flag {
271 PCO_CF_NODE_FLAG_BODY = 0,
272
273 PCO_CF_NODE_FLAG_IF_THEN,
274 PCO_CF_NODE_FLAG_IF_ELSE,
275
276 PCO_CF_NODE_FLAG_PROLOGUE,
277 PCO_CF_NODE_FLAG_INTERLOGUE,
278 PCO_CF_NODE_FLAG_EPILOGUE,
279 };
280
281 /** PCO control-flow node. */
282 typedef struct _pco_cf_node {
283 struct list_head link; /** Node in lists of pco_cf_nodes. */
284 enum pco_cf_node_type type; /** CF node type. */
285 struct _pco_cf_node *parent; /** Parent cf node. */
286 enum pco_cf_node_flag flag; /** Implementation-defined flag. */
287 } pco_cf_node;
288
289 /** PCO basic block. */
290 typedef struct _pco_block {
291 pco_cf_node cf_node; /** Control flow node. */
292 pco_func *parent_func; /** Parent function. */
293 struct list_head instrs; /** Instruction/group list. */
294 unsigned index; /** Block index. */
295 } pco_block;
296
297 /** PCO if cf construct. */
298 typedef struct _pco_if {
299 pco_cf_node cf_node; /** CF node. */
300 pco_func *parent_func; /** Parent function. */
301 pco_ref cond; /** If condition. */
302 bool pred_exec; /** Whether this if construct uses predicated execution. */
303 struct list_head prologue; /** List of pco_cf_nodes for if prologue. */
304 struct list_head then_body; /** List of pco_cf_nodes for if body. */
305 struct list_head interlogue; /** List of pco_cf_nodes for if interlogue. */
306 struct list_head else_body; /** List of pco_cf_nodes for else body. */
307 struct list_head epilogue; /** List of pco_cf_nodes for if epilogue. */
308 unsigned index; /** If index. */
309 } pco_if;
310
311 /** PCO loop cf construct. */
312 typedef struct _pco_loop {
313 pco_cf_node cf_node; /** CF node. */
314 pco_func *parent_func; /** Parent function. */
315 struct list_head prologue; /** List of pco_cf_nodes for loop prologue. */
316 struct list_head body; /** List of pco_cf_nodes for loop body. */
317 struct list_head interlogue; /** List of pco_cf_nodes for loop interlogue. */
318 struct list_head epilogue; /** List of pco_cf_nodes for loop epilogue. */
319 unsigned index; /** Loop index. */
320 } pco_loop;
321
322 #define VEC_USER_MULTI ((void *)(~0ULL))
323
324 /** PCO vector information. */
325 typedef struct _pco_vec_info {
326 pco_instr *instr; /** Vector producer. */
327 pco_instr **comps; /** Array of vector components. */
328 pco_instr *vec_user; /** Vector user, or none, or multi. */
329 } pco_vec_info;
330
331 /** PCO function. */
332 typedef struct _pco_func {
333 struct list_head link; /** Node in pco_shader::funcs. */
334 pco_cf_node cf_node; /** Control flow node. */
335
336 pco_shader *parent_shader; /** Shader containing the function. */
337
338 enum pco_func_type type; /** Function type. */
339 unsigned index; /** Function index. */
340 const char *name; /** Function name. */
341
342 struct list_head body; /** List of pco_cf_nodes for function body. */
343
344 unsigned num_params;
345 pco_ref *params;
346
347 struct hash_table_u64 *vec_infos;
348
349 unsigned next_ssa; /** Next SSA node index. */
350 unsigned next_vreg; /** Next virtual register index. */
351 unsigned next_instr; /** Next instruction index. */
352 unsigned next_igrp; /** Next igrp index. */
353 unsigned next_block; /** Next block index. */
354 unsigned next_if; /** Next if index. */
355 unsigned next_loop; /** Next loop index. */
356
357 unsigned temps; /** Number of temps allocated. */
358 unsigned vtxins; /** Number of vertex input registers used. */
359
360 pco_ref emc; /** Execution mask counter register. */
361
362 unsigned enc_offset; /** Encoding offset. */
363 } pco_func;
364
365 /** PCO shader. */
366 typedef struct _pco_shader {
367 pco_ctx *ctx; /** Compiler context. */
368 nir_shader *nir; /** Source NIR shader. */
369
370 mesa_shader_stage stage; /** Shader stage. */
371 const char *name; /** Shader name. */
372 bool is_internal; /** Whether this is an internal shader. */
373 bool is_grouped; /** Whether the shader uses igrps. */
374 bool is_legalized; /** Whether the shader has been legalized. */
375
376 struct list_head funcs; /** List of functions. */
377 unsigned next_func; /** Next function index. */
378
379 pco_data data; /** Shader data. */
380 struct util_dynarray binary; /** Shader binary. */
381 } pco_shader;
382
383 /** Op info. */
384 struct pco_op_info {
385 const char *str; /** Op name string. */
386 unsigned num_dests; /** Number of dests. */
387 unsigned num_srcs; /** Number of sources. */
388 uint64_t mods; /** Supported mods. */
389 uint8_t mod_map[_PCO_OP_MOD_COUNT]; /** Index into pco_instr::mod. */
390 uint64_t dest_mods[_PCO_OP_MAX_DESTS]; /** Supported dest mods. */
391 uint64_t src_mods[_PCO_OP_MAX_SRCS]; /** Supported source mods. */
392 enum pco_op_type type; /** Op type. */
393 bool has_target_cf_node; /** Set if op has a cf-node as a target. */
394 uint8_t dest_intrn_map[_PCO_OP_MAX_DESTS];
395 uint8_t src_intrn_map[_PCO_OP_MAX_SRCS];
396 #ifndef NDEBUG
397 uint32_t grp_dest_maps[_PCO_OP_PHASE_COUNT][_PCO_OP_MAX_DESTS];
398 uint32_t grp_src_maps[_PCO_OP_PHASE_COUNT][_PCO_OP_MAX_SRCS];
399 #endif /* NDEBUG */
400 };
401 extern const struct pco_op_info pco_op_info[_PCO_OP_COUNT];
402 static_assert(_PCO_REF_MAP_COUNT <= 32,
403 "enum pco_ref_map must fit into an uint32_t");
404
405 /** Op mod info. */
406 struct pco_op_mod_info {
407 bool print_early : 1; /** Set if printed before the op. */
408 bool is_bitset : 1; /** Set if type is an enum bitset. */
409 enum pco_mod_type type; /** Datatype. */
410 union {
411 const char *str; /** Mod name. */
412 const char **strs; /** Mod names (enums). */
413 };
414 uint32_t nzdefault; /** Default value if non-zero. */
415 };
416 extern const struct pco_op_mod_info pco_op_mod_info[_PCO_OP_MOD_COUNT];
417
418 /** Reference mod info. */
419 struct pco_ref_mod_info {
420 bool is_bitset : 1; /** Set if type is an enum bitset. */
421 enum pco_mod_type type; /** Datatype. */
422 union {
423 const char *str; /** Mod name. */
424 const char **strs; /** Mod names (enums). */
425 };
426 };
427 extern const struct pco_ref_mod_info pco_ref_mod_info[_PCO_REF_MOD_COUNT];
428
429 pco_shader *pco_shader_create(pco_ctx *ctx, nir_shader *nir, void *mem_ctx);
430 pco_func *pco_func_create(pco_shader *shader,
431 enum pco_func_type type,
432 unsigned num_params);
433 pco_block *pco_block_create(pco_func *func);
434 pco_if *pco_if_create(pco_func *func);
435 pco_loop *pco_loop_create(pco_func *func);
436 pco_instr *
437 pco_instr_create(pco_func *func, unsigned num_dests, unsigned num_srcs);
438 pco_igrp *pco_igrp_create(pco_func *func);
439
440 void pco_instr_delete(pco_instr *instr);
441
442 /* Cast helpers. */
443
444 /* CF nodes. */
445 #define PCO_DEFINE_CAST(name, in_type, out_type, field, type_field, type_value) \
446 static inline out_type *name(const in_type *parent) \
447 { \
448 assert(parent && parent->type_field == type_value); \
449 return list_entry(parent, out_type, field); \
450 }
451
452 PCO_DEFINE_CAST(pco_cf_node_as_block,
453 pco_cf_node,
454 pco_block,
455 cf_node,
456 type,
457 PCO_CF_NODE_TYPE_BLOCK)
458 PCO_DEFINE_CAST(pco_cf_node_as_if,
459 pco_cf_node,
460 pco_if,
461 cf_node,
462 type,
463 PCO_CF_NODE_TYPE_IF)
464 PCO_DEFINE_CAST(pco_cf_node_as_loop,
465 pco_cf_node,
466 pco_loop,
467 cf_node,
468 type,
469 PCO_CF_NODE_TYPE_LOOP)
470 PCO_DEFINE_CAST(pco_cf_node_as_func,
471 pco_cf_node,
472 pco_func,
473 cf_node,
474 type,
475 PCO_CF_NODE_TYPE_FUNC)
476
477 #undef PCO_DEFINE_CAST
478
479 /* Iterators. */
480 #define pco_foreach_func_in_shader(func, shader) \
481 list_for_each_entry (pco_func, func, &(shader)->funcs, link)
482
483 #define pco_foreach_func_in_shader_rev(func, shader) \
484 list_for_each_entry_rev (pco_func, func, &(shader)->funcs, link)
485
486 #define pco_foreach_cf_node_in_if_prologue(cf_node, _if) \
487 list_for_each_entry (pco_cf_node, cf_node, &(_if)->prologue, link)
488
489 #define pco_foreach_cf_node_in_if_then(cf_node, _if) \
490 list_for_each_entry (pco_cf_node, cf_node, &(_if)->then_body, link)
491
492 #define pco_foreach_cf_node_in_if_interlogue(cf_node, _if) \
493 list_for_each_entry (pco_cf_node, cf_node, &(_if)->interlogue, link)
494
495 #define pco_foreach_cf_node_in_if_else(cf_node, _if) \
496 list_for_each_entry (pco_cf_node, cf_node, &(_if)->else_body, link)
497
498 #define pco_foreach_cf_node_in_if_epilogue(cf_node, _if) \
499 list_for_each_entry (pco_cf_node, cf_node, &(_if)->epilogue, link)
500
501 #define pco_foreach_cf_node_in_loop_prologue(cf_node, loop) \
502 list_for_each_entry (pco_cf_node, cf_node, &(loop)->prologue, link)
503
504 #define pco_foreach_cf_node_in_loop(cf_node, loop) \
505 list_for_each_entry (pco_cf_node, cf_node, &(loop)->body, link)
506
507 #define pco_foreach_cf_node_in_loop_interlogue(cf_node, loop) \
508 list_for_each_entry (pco_cf_node, cf_node, &(loop)->interlogue, link)
509
510 #define pco_foreach_cf_node_in_loop_epilogue(cf_node, loop) \
511 list_for_each_entry (pco_cf_node, cf_node, &(loop)->epilogue, link)
512
513 #define pco_foreach_cf_node_in_func(cf_node, func) \
514 list_for_each_entry (pco_cf_node, cf_node, &(func)->body, link)
515
516 #define pco_foreach_cf_node_in_func_structured(cf_node, \
517 cf_node_completed, \
518 func) \
519 for (pco_cf_node *cf_node = pco_cf_node_head(&(func)->body), \
520 *cf_node_completed = NULL; \
521 cf_node != NULL; \
522 cf_node = pco_next_cf_node(cf_node, &cf_node_completed))
523
524 #define pco_foreach_block_in_func(block, func) \
525 for (pco_block *block = pco_func_first_block(func); block != NULL; \
526 block = pco_next_block(block))
527
528 #define pco_foreach_block_in_func_from(block, from) \
529 for (pco_block *block = pco_next_block(from); block != NULL; \
530 block = pco_next_block(block))
531
532 #define pco_foreach_block_in_func_from_rev(block, from) \
533 for (pco_block *block = pco_prev_block(from); block != NULL; \
534 block = pco_prev_block(block))
535
536 #define pco_foreach_block_in_func_rev(block, func) \
537 for (pco_block *block = pco_func_last_block(func); block != NULL; \
538 block = pco_prev_block(block))
539
540 #define pco_foreach_if_in_func(pif, func) \
541 for (pco_if *pif = pco_func_first_if(func); pif != NULL; \
542 pif = pco_next_if(pif))
543
544 #define pco_foreach_ssa_if_in_func(pif, func) \
545 for (pco_if *pif = pco_func_first_if(func); pif != NULL; \
546 pif = pco_next_if(pif)) \
547 if (pco_ref_is_ssa(pif->cond))
548
549 #define pco_foreach_ssa_bool_if_in_func(pif, func) \
550 for (pco_if *pif = pco_func_first_if(func); pif != NULL; \
551 pif = pco_next_if(pif)) \
552 if (pco_ref_is_ssa(pif->cond) && pco_ref_get_bits(pif->cond) == 1)
553
554 #define pco_foreach_vreg_if_in_func(pif, func) \
555 for (pco_if *pif = pco_func_first_if(func); pif != NULL; \
556 pif = pco_next_if(pif)) \
557 if (pco_ref_is_vreg(pif->cond))
558
559 #define pco_foreach_vreg_ssa_if_in_func(pif, func) \
560 for (pco_if *pif = pco_func_first_if(func); pif != NULL; \
561 pif = pco_next_if(pif)) \
562 if (pco_ref_is_vreg(pif->cond) || pco_ref_is_ssa(pif->cond))
563
564 #define pco_foreach_if_in_func_from(pif, from) \
565 for (pco_if *pif = pco_next_if(from); pif != NULL; pif = pco_next_if(pif))
566
567 #define pco_foreach_if_in_func_from_rev(pif, from) \
568 for (pco_if *pif = pco_prev_if(from); pif != NULL; pif = pco_prev_if(pif))
569
570 #define pco_foreach_if_in_func_rev(pif, func) \
571 for (pco_if *pif = pco_func_last_if(func); pif != NULL; \
572 pif = pco_prev_if(pif))
573
574 #define pco_foreach_loop_in_func(loop, func) \
575 for (pco_loop *loop = pco_func_first_loop(func); loop != NULL; \
576 loop = pco_next_loop(loop))
577
578 #define pco_foreach_loop_in_func_from(loop, from) \
579 for (pco_loop *loop = pco_next_loop(from); loop != NULL; \
580 loop = pco_next_loop(loop))
581
582 #define pco_foreach_loop_in_func_from_rev(loop, from) \
583 for (pco_loop *loop = pco_prev_loop(from); loop != NULL; \
584 loop = pco_prev_loop(loop))
585
586 #define pco_foreach_loop_in_func_rev(loop, func) \
587 for (pco_loop *loop = pco_func_last_loop(func); loop != NULL; \
588 loop = pco_prev_loop(loop))
589
590 #define pco_foreach_instr_in_block(instr, block) \
591 list_for_each_entry (pco_instr, instr, &(block)->instrs, link)
592
593 #define pco_foreach_instr_in_block_from(instr, block, from) \
594 foreach_list_typed_from (pco_instr, instr, node, _, &from->node)
595
596 #define pco_foreach_instr_in_block_from_rev(instr, block, from) \
597 foreach_list_typed_from_reverse (pco_instr, instr, node, _, &from->node)
598
599 #define pco_foreach_instr_in_block_safe(instr, block) \
600 list_for_each_entry_safe (pco_instr, instr, &(block)->instrs, link)
601
602 #define pco_foreach_instr_in_block_rev(instr, block) \
603 list_for_each_entry_rev (pco_instr, instr, &(block)->instrs, link)
604
605 #define pco_foreach_instr_in_block_safe_rev(instr, block) \
606 list_for_each_entry_safe_rev (pco_instr, instr, &(block)->instrs, link)
607
608 #define pco_foreach_igrp_in_block(igrp, block) \
609 list_for_each_entry (pco_igrp, igrp, &(block)->instrs, link)
610
611 #define pco_foreach_phi_src_in_instr(phi_src, instr) \
612 list_for_each_entry (pco_phi_src, phi_src, &(instr)->phi_srcs, link)
613
614 #define pco_foreach_instr_in_func(instr, func) \
615 pco_foreach_block_in_func (block, func) \
616 pco_foreach_instr_in_block (instr, block)
617
618 #define pco_foreach_instr_in_func_from(instr, from) \
619 for (pco_instr *instr = pco_next_instr(from); instr != NULL; \
620 instr = pco_next_instr(instr))
621
622 #define pco_foreach_instr_in_func_from_rev(instr, from) \
623 for (pco_instr *instr = pco_prev_instr(from); instr != NULL; \
624 instr = pco_prev_instr(instr))
625
626 #define pco_foreach_instr_in_func_safe(instr, func) \
627 pco_foreach_block_in_func (block, func) \
628 pco_foreach_instr_in_block_safe (instr, block)
629
630 #define pco_foreach_instr_in_func_rev(instr, func) \
631 pco_foreach_block_in_func_rev (block, func) \
632 pco_foreach_instr_in_block_rev (instr, block)
633
634 #define pco_foreach_instr_in_func_safe_rev(instr, func) \
635 pco_foreach_block_in_func_rev (block, func) \
636 pco_foreach_instr_in_block_safe_rev (instr, block)
637
638 #define pco_foreach_igrp_in_func(igrp, func) \
639 pco_foreach_block_in_func (block, func) \
640 pco_foreach_igrp_in_block (igrp, block)
641
642 #define pco_foreach_phase(phase) \
643 for (enum pco_op_phase phase = 0; phase < _PCO_OP_PHASE_COUNT; ++phase)
644
645 #define pco_foreach_phase_rev(phase) \
646 for (enum pco_op_phase phase = _PCO_OP_PHASE_COUNT; phase-- > 0;)
647
648 #define pco_foreach_phase_from(phase, from) \
649 for (enum pco_op_phase phase = from + 1; phase < _PCO_OP_PHASE_COUNT; \
650 ++phase)
651
652 #define pco_foreach_phase_from_rev(phase, from) \
653 for (enum pco_op_phase phase = from; phase-- > 0;)
654
655 #define pco_foreach_phase_in_igrp(igrp, phase) \
656 pco_foreach_phase (phase) \
657 if (igrp->instrs[phase])
658
659 #define pco_foreach_phase_in_igrp_rev(igrp, phase) \
660 pco_foreach_phase_rev (phase) \
661 if (igrp->instrs[phase])
662
663 #define pco_foreach_phase_in_igrp_from(igrp, phase, from) \
664 pco_foreach_phase_from (phase, from) \
665 if (igrp->instrs[phase])
666
667 #define pco_foreach_phase_in_igrp_from_rev(igrp, phase, from) \
668 pco_foreach_phase_from_rev (phase, from) \
669 if (igrp->instrs[phase])
670
671 #define pco_foreach_instr_in_igrp(instr, igrp) \
672 for (pco_instr *instr = pco_igrp_first_instr(igrp); instr != NULL; \
673 instr = pco_igrp_next_instr(instr))
674
675 #define pco_foreach_instr_in_igrp_rev(instr, igrp) \
676 for (pco_instr *instr = pco_igrp_last_instr(igrp); instr != NULL; \
677 instr = pco_igrp_prev_instr(instr))
678
679 #define pco_foreach_instr_dest(pdest, instr) \
680 for (pco_ref *pdest = &instr->dest[0]; \
681 pdest < &instr->dest[instr->num_dests]; \
682 ++pdest)
683
684 #define pco_foreach_instr_dest_from(pdest, instr, pdest_from) \
685 for (pco_ref *pdest = pdest_from + 1; \
686 pdest < &instr->dest[instr->num_dests]; \
687 ++pdest)
688
689 #define pco_foreach_instr_dest_ssa(pdest, instr) \
690 pco_foreach_instr_dest (pdest, instr) \
691 if (pco_ref_is_ssa(*pdest))
692
693 #define pco_foreach_instr_dest_ssa_from(pdest, instr, pdest_from) \
694 pco_foreach_instr_dest_from (pdest, instr, pdest_from) \
695 if (pco_ref_is_ssa(*pdest))
696
697 #define pco_foreach_instr_dest_vreg(pdest, instr) \
698 pco_foreach_instr_dest (pdest, instr) \
699 if (pco_ref_is_vreg(*pdest))
700
701 #define pco_foreach_instr_dest_vreg_ssa(pdest, instr) \
702 pco_foreach_instr_dest (pdest, instr) \
703 if (pco_ref_is_vreg(*pdest) || pco_ref_is_ssa(*pdest))
704
705 #define pco_foreach_instr_dest_hwreg(pdest, instr) \
706 pco_foreach_instr_dest (pdest, instr) \
707 if (pco_ref_is_hwreg(*pdest))
708
709 #define pco_foreach_instr_src(psrc, instr) \
710 for (pco_ref *psrc = &instr->src[0]; psrc < &instr->src[instr->num_srcs]; \
711 ++psrc)
712
713 #define pco_foreach_instr_src_from(psrc, instr, psrc_from) \
714 for (pco_ref *psrc = psrc_from + 1; psrc < &instr->src[instr->num_srcs]; \
715 ++psrc)
716
717 #define pco_foreach_instr_src_ssa(psrc, instr) \
718 pco_foreach_instr_src (psrc, instr) \
719 if (pco_ref_is_ssa(*psrc))
720
721 #define pco_foreach_instr_src_ssa_from(psrc, instr, psrc_from) \
722 pco_foreach_instr_src_from (psrc, instr, psrc_from) \
723 if (pco_ref_is_ssa(*psrc))
724
725 #define pco_foreach_instr_src_vreg(psrc, instr) \
726 pco_foreach_instr_src (psrc, instr) \
727 if (pco_ref_is_vreg(*psrc))
728
729 #define pco_foreach_instr_src_vreg_ssa(psrc, instr) \
730 pco_foreach_instr_src (psrc, instr) \
731 if (pco_ref_is_vreg(*psrc) || pco_ref_is_ssa(*psrc))
732
733 #define pco_foreach_instr_src_hwreg(psrc, instr) \
734 pco_foreach_instr_src (psrc, instr) \
735 if (pco_ref_is_hwreg(*psrc))
736
737 #define pco_foreach_instr_src_vtxin_reg(psrc, instr) \
738 pco_foreach_instr_src (psrc, instr) \
739 if (pco_ref_is_vtxin(*psrc))
740
741 #define pco_cf_node_head(list) list_first_entry(list, pco_cf_node, link)
742 #define pco_cf_node_tail(list) list_last_entry(list, pco_cf_node, link)
743
744 /**
745 * \brief Returns the preamble function of a PCO shader.
746 *
747 * \param[in] shader The PCO shader.
748 * \return The preamble function, or NULL if the shader has no preamble.
749 */
750 static inline pco_func *pco_preamble(pco_shader *shader)
751 {
752 if (list_is_empty(&shader->funcs))
753 return NULL;
754
755 pco_func *func = list_first_entry(&shader->funcs, pco_func, link);
756 if (func->type == PCO_FUNC_TYPE_PREAMBLE)
757 return func;
758
759 return NULL;
760 }
761
762 /**
763 * \brief Returns the entrypoint function of a PCO shader.
764 *
765 * \param[in] shader The PCO shader.
766 * \return The entrypoint function, or NULL if the shader has no entrypoint.
767 */
768 static inline pco_func *pco_entrypoint(pco_shader *shader)
769 {
770 if (list_is_empty(&shader->funcs))
771 return NULL;
772
773 /* Entrypoint will either be the first or second function in the shader,
774 * depending on whether or not there is a preamble.
775 */
776 pco_func *preamble = pco_preamble(shader);
777 pco_func *func = !preamble ? list_first_entry(&shader->funcs, pco_func, link)
778 : list_entry(preamble->link.next, pco_func, link);
779
780 if (func->type == PCO_FUNC_TYPE_ENTRYPOINT)
781 return func;
782
783 return NULL;
784 }
785
786 /**
787 * \brief Returns the variant of an instruction in an instruction group.
788 *
789 * \param[in] igrp The instruction group.
790 * \param[in] phase The instruction phase.
791 * \return The instruction variant.
792 */
793 static inline unsigned pco_igrp_variant(const pco_igrp *igrp,
794 enum pco_op_phase phase)
795 {
796 switch (igrp->hdr.alutype) {
797 case PCO_ALUTYPE_MAIN:
798 return phase == PCO_OP_PHASE_BACKEND ? igrp->variant.instr[phase].backend
799 : igrp->variant.instr[phase].main;
800
801 case PCO_ALUTYPE_BITWISE:
802 return igrp->variant.instr[phase].bitwise;
803
804 case PCO_ALUTYPE_CONTROL:
805 return igrp->variant.instr[phase].control;
806
807 default:
808 break;
809 }
810
811 UNREACHABLE("");
812 }
813
814 /* Motions. */
815
816 /**
817 * \brief Returns the first CF node in a PCO function.
818 *
819 * \param[in] func PCO function.
820 * \return The first CF node.
821 */
822 static inline pco_cf_node *pco_first_func_cf_node(pco_func *func)
823 {
824 if (!list_is_empty(&func->body))
825 return pco_cf_node_head(&func->body);
826
827 UNREACHABLE("Empty function.");
828 }
829
830 /**
831 * \brief Returns the last CF node in a PCO function.
832 *
833 * \param[in] func PCO function.
834 * \return The last CF node.
835 */
836 static inline pco_cf_node *pco_last_func_cf_node(pco_func *func)
837 {
838 if (!list_is_empty(&func->body))
839 return pco_cf_node_tail(&func->body);
840
841 UNREACHABLE("Empty function.");
842 }
843
844 /**
845 * \brief Returns the first CF node in a PCO if.
846 *
847 * \param[in] pif PCO if.
848 * \return The first CF node.
849 */
850 static inline pco_cf_node *pco_first_if_cf_node(pco_if *pif)
851 {
852 if (!list_is_empty(&pif->prologue))
853 return pco_cf_node_head(&pif->prologue);
854
855 if (!list_is_empty(&pif->then_body))
856 return pco_cf_node_head(&pif->then_body);
857
858 if (!list_is_empty(&pif->interlogue))
859 return pco_cf_node_head(&pif->interlogue);
860
861 if (!list_is_empty(&pif->else_body))
862 return pco_cf_node_head(&pif->else_body);
863
864 if (!list_is_empty(&pif->epilogue))
865 return pco_cf_node_head(&pif->epilogue);
866
867 UNREACHABLE("Empty if.");
868 }
869
870 /**
871 * \brief Returns the last CF node in a PCO if.
872 *
873 * \param[in] pif PCO if.
874 * \return The last CF node.
875 */
876 static inline pco_cf_node *pco_last_if_cf_node(pco_if *pif)
877 {
878 if (!list_is_empty(&pif->epilogue))
879 return pco_cf_node_tail(&pif->epilogue);
880
881 if (!list_is_empty(&pif->else_body))
882 return pco_cf_node_tail(&pif->else_body);
883
884 if (!list_is_empty(&pif->interlogue))
885 return pco_cf_node_tail(&pif->interlogue);
886
887 if (!list_is_empty(&pif->then_body))
888 return pco_cf_node_tail(&pif->then_body);
889
890 if (!list_is_empty(&pif->prologue))
891 return pco_cf_node_tail(&pif->prologue);
892
893 UNREACHABLE("Empty if.");
894 }
895
896 /**
897 * \brief Returns the next CF node in a PCO if.
898 *
899 * \param[in] cf_node The current CF node within the PCO if.
900 * \return The next CF node, or NULL if we've reached the end.
901 */
902 static inline pco_cf_node *pco_next_if_cf_node(pco_cf_node *cf_node)
903 {
904 pco_if *pif = pco_cf_node_as_if(cf_node->parent);
905
906 switch (cf_node->flag) {
907 case PCO_CF_NODE_FLAG_PROLOGUE:
908 if (!list_is_empty(&pif->then_body))
909 return pco_cf_node_head(&pif->then_body);
910
911 FALLTHROUGH;
912
913 case PCO_CF_NODE_FLAG_IF_THEN:
914 if (!list_is_empty(&pif->interlogue))
915 return pco_cf_node_head(&pif->interlogue);
916
917 FALLTHROUGH;
918
919 case PCO_CF_NODE_FLAG_INTERLOGUE:
920 if (!list_is_empty(&pif->else_body))
921 return pco_cf_node_head(&pif->else_body);
922
923 FALLTHROUGH;
924
925 case PCO_CF_NODE_FLAG_IF_ELSE:
926 if (!list_is_empty(&pif->epilogue))
927 return pco_cf_node_head(&pif->epilogue);
928
929 FALLTHROUGH;
930
931 case PCO_CF_NODE_FLAG_EPILOGUE:
932 return NULL;
933
934 default:
935 break;
936 }
937
938 UNREACHABLE("");
939 }
940
941 /**
942 * \brief Returns the previous CF node in a PCO if.
943 *
944 * \param[in] cf_node The current CF node within the PCO if.
945 * \return The previous CF node, or NULL if we've reached the end.
946 */
947 static inline pco_cf_node *pco_prev_if_cf_node(pco_cf_node *cf_node)
948 {
949 pco_if *pif = pco_cf_node_as_if(cf_node->parent);
950
951 switch (cf_node->flag) {
952 case PCO_CF_NODE_FLAG_EPILOGUE:
953 if (!list_is_empty(&pif->else_body))
954 return pco_cf_node_tail(&pif->else_body);
955
956 FALLTHROUGH;
957
958 case PCO_CF_NODE_FLAG_IF_ELSE:
959 if (!list_is_empty(&pif->interlogue))
960 return pco_cf_node_tail(&pif->interlogue);
961
962 FALLTHROUGH;
963
964 case PCO_CF_NODE_FLAG_INTERLOGUE:
965 if (!list_is_empty(&pif->then_body))
966 return pco_cf_node_tail(&pif->then_body);
967
968 FALLTHROUGH;
969
970 case PCO_CF_NODE_FLAG_IF_THEN:
971 if (!list_is_empty(&pif->prologue))
972 return pco_cf_node_tail(&pif->prologue);
973
974 FALLTHROUGH;
975
976 case PCO_CF_NODE_FLAG_PROLOGUE:
977 return NULL;
978
979 default:
980 break;
981 }
982
983 UNREACHABLE("");
984 }
985
986 /**
987 * \brief Returns the first CF node in a PCO loop.
988 *
989 * \param[in] loop PCO loop.
990 * \return The first CF node.
991 */
992 static inline pco_cf_node *pco_first_loop_cf_node(pco_loop *loop)
993 {
994 if (!list_is_empty(&loop->prologue))
995 return pco_cf_node_head(&loop->prologue);
996
997 if (!list_is_empty(&loop->body))
998 return pco_cf_node_head(&loop->body);
999
1000 if (!list_is_empty(&loop->interlogue))
1001 return pco_cf_node_head(&loop->interlogue);
1002
1003 if (!list_is_empty(&loop->epilogue))
1004 return pco_cf_node_head(&loop->epilogue);
1005
1006 UNREACHABLE("Empty loop.");
1007 }
1008
1009 /**
1010 * \brief Returns the last CF node in a PCO loop.
1011 *
1012 * \param[in] loop PCO loop.
1013 * \return The last CF node.
1014 */
1015 static inline pco_cf_node *pco_last_loop_cf_node(pco_loop *loop)
1016 {
1017 if (!list_is_empty(&loop->epilogue))
1018 return pco_cf_node_tail(&loop->epilogue);
1019
1020 if (!list_is_empty(&loop->interlogue))
1021 return pco_cf_node_tail(&loop->interlogue);
1022
1023 if (!list_is_empty(&loop->body))
1024 return pco_cf_node_tail(&loop->body);
1025
1026 if (!list_is_empty(&loop->prologue))
1027 return pco_cf_node_tail(&loop->prologue);
1028
1029 UNREACHABLE("Empty loop.");
1030 }
1031
1032 /**
1033 * \brief Returns the next CF node in a PCO loop.
1034 *
1035 * \param[in] cf_node The current CF node within the PCO loop.
1036 * \return The next CF node, or NULL if we've reached the end.
1037 */
1038 static inline pco_cf_node *pco_next_loop_cf_node(pco_cf_node *cf_node)
1039 {
1040 pco_loop *loop = pco_cf_node_as_loop(cf_node->parent);
1041
1042 switch (cf_node->flag) {
1043 case PCO_CF_NODE_FLAG_PROLOGUE:
1044 if (!list_is_empty(&loop->body))
1045 return pco_cf_node_head(&loop->body);
1046
1047 FALLTHROUGH;
1048
1049 case PCO_CF_NODE_FLAG_BODY:
1050 if (!list_is_empty(&loop->interlogue))
1051 return pco_cf_node_head(&loop->interlogue);
1052
1053 FALLTHROUGH;
1054
1055 case PCO_CF_NODE_FLAG_INTERLOGUE:
1056 if (!list_is_empty(&loop->epilogue))
1057 return pco_cf_node_head(&loop->epilogue);
1058
1059 FALLTHROUGH;
1060
1061 case PCO_CF_NODE_FLAG_EPILOGUE:
1062 return NULL;
1063
1064 default:
1065 break;
1066 }
1067
1068 UNREACHABLE("");
1069 }
1070
1071 /**
1072 * \brief Returns the previous CF node in a PCO loop.
1073 *
1074 * \param[in] cf_node The current CF node within the PCO loop.
1075 * \return The previous CF node, or NULL if we've reached the end.
1076 */
1077 static inline pco_cf_node *pco_prev_loop_cf_node(pco_cf_node *cf_node)
1078 {
1079 pco_loop *loop = pco_cf_node_as_loop(cf_node->parent);
1080
1081 switch (cf_node->flag) {
1082 case PCO_CF_NODE_FLAG_EPILOGUE:
1083 if (!list_is_empty(&loop->interlogue))
1084 return pco_cf_node_tail(&loop->interlogue);
1085
1086 FALLTHROUGH;
1087
1088 case PCO_CF_NODE_FLAG_INTERLOGUE:
1089 if (!list_is_empty(&loop->body))
1090 return pco_cf_node_tail(&loop->body);
1091
1092 FALLTHROUGH;
1093
1094 case PCO_CF_NODE_FLAG_BODY:
1095 if (!list_is_empty(&loop->prologue))
1096 return pco_cf_node_tail(&loop->prologue);
1097
1098 FALLTHROUGH;
1099
1100 case PCO_CF_NODE_FLAG_PROLOGUE:
1101 return NULL;
1102
1103 default:
1104 break;
1105 }
1106
1107 UNREACHABLE("");
1108 }
1109
1110 static inline struct list_head *pco_parent_if_head(pco_cf_node *cf_node)
1111 {
1112 pco_if *parent = pco_cf_node_as_if(cf_node->parent);
1113
1114 switch (cf_node->flag) {
1115 case PCO_CF_NODE_FLAG_PROLOGUE:
1116 return &parent->prologue;
1117 case PCO_CF_NODE_FLAG_IF_THEN:
1118 return &parent->then_body;
1119 case PCO_CF_NODE_FLAG_INTERLOGUE:
1120 return &parent->interlogue;
1121 case PCO_CF_NODE_FLAG_IF_ELSE:
1122 return &parent->else_body;
1123 case PCO_CF_NODE_FLAG_EPILOGUE:
1124 return &parent->epilogue;
1125 default:
1126 UNREACHABLE("");
1127 }
1128 }
1129
1130 static inline struct list_head *pco_parent_loop_head(pco_cf_node *cf_node)
1131 {
1132 pco_loop *parent = pco_cf_node_as_loop(cf_node->parent);
1133 switch (cf_node->flag) {
1134 case PCO_CF_NODE_FLAG_PROLOGUE:
1135 return &parent->prologue;
1136 case PCO_CF_NODE_FLAG_BODY:
1137 return &parent->body;
1138 case PCO_CF_NODE_FLAG_INTERLOGUE:
1139 return &parent->interlogue;
1140 case PCO_CF_NODE_FLAG_EPILOGUE:
1141 return &parent->epilogue;
1142 default:
1143 UNREACHABLE("");
1144 }
1145 }
1146
1147 static inline struct list_head *pco_parent_func_head(pco_cf_node *cf_node)
1148 {
1149 pco_func *parent = pco_cf_node_as_func(cf_node->parent);
1150 switch (cf_node->flag) {
1151 case PCO_CF_NODE_FLAG_BODY:
1152 return &parent->body;
1153 default:
1154 UNREACHABLE("");
1155 }
1156 }
1157
1158 static inline struct list_head *pco_parent_cf_node_head(pco_cf_node *cf_node)
1159 {
1160 pco_cf_node *parent_cf_node = cf_node->parent;
1161 switch (parent_cf_node->type) {
1162 case PCO_CF_NODE_TYPE_IF:
1163 return pco_parent_if_head(cf_node);
1164 case PCO_CF_NODE_TYPE_LOOP:
1165 return pco_parent_loop_head(cf_node);
1166 case PCO_CF_NODE_TYPE_FUNC:
1167 return pco_parent_func_head(cf_node);
1168 default:
1169 UNREACHABLE("");
1170 }
1171 }
1172
1173 /**
1174 * \brief Returns the next CF node.
1175 *
1176 * \param[in] cf_node The current CF node.
1177 * \param[out] cf_node_completed The last parent CF node (function, if, loop)
1178 * completed, or NULL if none/not a parent.
1179 * \return The next CF node.
1180 */
1181 static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node,
1182 pco_cf_node **cf_node_completed)
1183 {
1184 #define pco_cf_node_next(cf_node) \
1185 list_entry((cf_node)->link.next, pco_cf_node, link)
1186
1187 if (cf_node_completed)
1188 *cf_node_completed = NULL;
1189
1190 if (!cf_node)
1191 return NULL;
1192
1193 switch (cf_node->type) {
1194 case PCO_CF_NODE_TYPE_BLOCK: {
1195 struct list_head *parent_list = pco_parent_cf_node_head(cf_node);
1196
1197 /* Not yet reached the end of the list, return the next cf node. */
1198 if (cf_node != pco_cf_node_tail(parent_list))
1199 return pco_cf_node_next(cf_node);
1200
1201 break;
1202 }
1203
1204 case PCO_CF_NODE_TYPE_IF:
1205 return pco_first_if_cf_node(pco_cf_node_as_if(cf_node));
1206
1207 case PCO_CF_NODE_TYPE_LOOP:
1208 return pco_first_loop_cf_node(pco_cf_node_as_loop(cf_node));
1209
1210 default:
1211 UNREACHABLE("");
1212 }
1213
1214 /* Reached the end; go to the next cf node from the parent cf node. */
1215 pco_cf_node *parent_cf_node = cf_node->parent;
1216 switch (parent_cf_node->type) {
1217 case PCO_CF_NODE_TYPE_IF: {
1218 pco_cf_node *next_cf_node = pco_next_if_cf_node(cf_node);
1219 if (next_cf_node)
1220 return next_cf_node;
1221
1222 break;
1223 }
1224
1225 case PCO_CF_NODE_TYPE_LOOP: {
1226 pco_cf_node *next_cf_node = pco_next_loop_cf_node(cf_node);
1227 if (next_cf_node)
1228 return next_cf_node;
1229
1230 break;
1231 }
1232
1233 /* End of the function; return NULL. */
1234 case PCO_CF_NODE_TYPE_FUNC:
1235 if (cf_node_completed)
1236 *cf_node_completed = parent_cf_node;
1237
1238 return NULL;
1239
1240 default:
1241 UNREACHABLE("");
1242 }
1243
1244 if (cf_node_completed)
1245 *cf_node_completed = parent_cf_node;
1246
1247 return pco_cf_node_next(parent_cf_node);
1248 #undef pco_cf_node_next
1249 }
1250
1251 /**
1252 * \brief Returns the previous CF node.
1253 *
1254 * \param[in] cf_node The current CF node.
1255 * \param[out] cf_node_completed The last parent CF node (function, if, loop)
1256 * completed, or NULL if none/not a parent.
1257 * \return The previous CF node.
1258 */
1259 static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node,
1260 pco_cf_node **cf_node_completed)
1261 {
1262 #define pco_cf_node_prev(cf_node) \
1263 list_entry((cf_node)->link.prev, pco_cf_node, link)
1264
1265 if (cf_node_completed)
1266 *cf_node_completed = NULL;
1267
1268 if (!cf_node)
1269 return NULL;
1270
1271 switch (cf_node->type) {
1272 case PCO_CF_NODE_TYPE_BLOCK: {
1273 struct list_head *parent_list = pco_parent_cf_node_head(cf_node);
1274
1275 /* Not yet reached the start of the list, return the previous cf node. */
1276 if (cf_node != pco_cf_node_head(parent_list))
1277 return pco_cf_node_prev(cf_node);
1278
1279 break;
1280 }
1281
1282 case PCO_CF_NODE_TYPE_IF:
1283 return pco_last_if_cf_node(pco_cf_node_as_if(cf_node));
1284
1285 case PCO_CF_NODE_TYPE_LOOP:
1286 return pco_last_loop_cf_node(pco_cf_node_as_loop(cf_node));
1287
1288 default:
1289 UNREACHABLE("");
1290 }
1291
1292 /* Reached the start; go to the previous cf node from the parent cf node. */
1293 pco_cf_node *parent_cf_node = cf_node->parent;
1294 switch (parent_cf_node->type) {
1295 case PCO_CF_NODE_TYPE_IF: {
1296 pco_cf_node *prev_cf_node = pco_prev_if_cf_node(cf_node);
1297 if (prev_cf_node)
1298 return prev_cf_node;
1299
1300 break;
1301 }
1302
1303 case PCO_CF_NODE_TYPE_LOOP: {
1304 pco_cf_node *prev_cf_node = pco_prev_loop_cf_node(cf_node);
1305 if (prev_cf_node)
1306 return prev_cf_node;
1307
1308 break;
1309 }
1310
1311 /* Start of the function; return NULL. */
1312 case PCO_CF_NODE_TYPE_FUNC:
1313 if (cf_node_completed)
1314 *cf_node_completed = parent_cf_node;
1315
1316 return NULL;
1317
1318 default:
1319 UNREACHABLE("");
1320 }
1321
1322 if (cf_node_completed)
1323 *cf_node_completed = parent_cf_node;
1324
1325 return pco_cf_node_prev(parent_cf_node);
1326 #undef pco_cf_node_prev
1327 }
1328
1329 /**
1330 * \brief Returns the next CF node of the given type, if one exists.
1331 *
1332 * \param[in] cf_node The current CF node.
1333 * \param[in] type The type of the next CF node.
1334 * \return The next CF node if one exists, else NULL.
1335 */
1336 static inline pco_cf_node *pco_next_cf_node_type(pco_cf_node *cf_node,
1337 enum pco_cf_node_type type)
1338 {
1339 do {
1340 cf_node = pco_next_cf_node(cf_node, NULL);
1341 } while (cf_node != NULL && cf_node->type != type);
1342
1343 return cf_node;
1344 }
1345
1346 /**
1347 * \brief Returns the first CF node of the given type, if one exists.
1348 *
1349 * \param[in] func PCO function.
1350 * \param[in] type The type of the first CF node.
1351 * \return The first CF node if one exists, else NULL.
1352 */
1353 static inline pco_cf_node *
1354 pco_func_first_cf_node_type(pco_func *func, enum pco_cf_node_type type)
1355 {
1356 assert(!list_is_empty(&func->body));
1357
1358 pco_cf_node *cf_node = pco_cf_node_head(&func->body);
1359 if (cf_node->type == type)
1360 return cf_node;
1361
1362 return pco_next_cf_node_type(cf_node, type);
1363 }
1364
1365 /**
1366 * \brief Returns the previous CF node of the given type, if one exists.
1367 *
1368 * \param[in] cf_node The current CF node.
1369 * \param[in] type The type of the previous CF node.
1370 * \return The previous CF node if one exists, else NULL.
1371 */
1372 static inline pco_cf_node *pco_prev_cf_node_type(pco_cf_node *cf_node,
1373 enum pco_cf_node_type type)
1374 {
1375 do {
1376 cf_node = pco_prev_cf_node(cf_node, NULL);
1377 } while (cf_node != NULL && cf_node->type != type);
1378
1379 return cf_node;
1380 }
1381
1382 /**
1383 * \brief Returns the last CF node of the given type, if one exists.
1384 *
1385 * \param[in] func PCO function.
1386 * \param[in] type The type of the last CF node.
1387 * \return The last CF node if one exists, else NULL.
1388 */
1389 static inline pco_cf_node *
1390 pco_func_last_cf_node_type(pco_func *func, enum pco_cf_node_type type)
1391 {
1392 assert(!list_is_empty(&func->body));
1393
1394 pco_cf_node *cf_node = pco_cf_node_tail(&func->body);
1395 if (cf_node->type == type)
1396 return cf_node;
1397
1398 return pco_prev_cf_node_type(cf_node, type);
1399 }
1400
1401 /* CF helper iterators. */
1402 #define PCO_DEFINE_CF_ITER(type, cf_node_type) \
1403 static inline pco_##type *pco_next_##type(pco_##type *current_##type) \
1404 { \
1405 pco_cf_node *cf_node = \
1406 pco_next_cf_node_type(&current_##type->cf_node, cf_node_type); \
1407 return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \
1408 } \
1409 \
1410 static inline pco_##type *pco_func_first_##type(pco_func *func) \
1411 { \
1412 pco_cf_node *cf_node = pco_func_first_cf_node_type(func, cf_node_type); \
1413 return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \
1414 } \
1415 \
1416 static inline pco_##type *pco_prev_##type(pco_##type *current_##type) \
1417 { \
1418 pco_cf_node *cf_node = \
1419 pco_prev_cf_node_type(&current_##type->cf_node, cf_node_type); \
1420 return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \
1421 } \
1422 \
1423 static inline pco_##type *pco_func_last_##type(pco_func *func) \
1424 { \
1425 pco_cf_node *cf_node = pco_func_last_cf_node_type(func, cf_node_type); \
1426 return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \
1427 }
1428
1429 PCO_DEFINE_CF_ITER(block, PCO_CF_NODE_TYPE_BLOCK)
1430 PCO_DEFINE_CF_ITER(if, PCO_CF_NODE_TYPE_IF)
1431 PCO_DEFINE_CF_ITER(loop, PCO_CF_NODE_TYPE_LOOP)
1432
1433 #undef PCO_DEFINE_CF_ITER
1434
1435 /**
1436 * \brief Returns the next non-empty block.
1437 *
1438 * \param[in] block The current block.
1439 * \return The next non-empty block, or NULL if the end of the function has been
1440 * reached.
1441 */
1442 static inline pco_block *pco_next_block_nonempty(pco_block *block)
1443 {
1444 block = pco_next_block(block);
1445
1446 /* Skip over empty blocks. */
1447 while (block && list_is_empty(&block->instrs))
1448 block = pco_next_block(block);
1449
1450 return block;
1451 }
1452
1453 /**
1454 * \brief Returns the previous non-empty block.
1455 *
1456 * \param[in] block The current block.
1457 * \return The previous non-empty block, or NULL if the end of the function has
1458 * been reached.
1459 */
1460 static inline pco_block *pco_prev_block_nonempty(pco_block *block)
1461 {
1462 block = pco_prev_block(block);
1463
1464 /* Skip over empty blocks. */
1465 while (block && list_is_empty(&block->instrs))
1466 block = pco_prev_block(block);
1467
1468 return block;
1469 }
1470
1471 /**
1472 * \brief Returns the first instruction in a block.
1473 *
1474 * \param[in] block The block.
1475 * \return The first instruction, or NULL if the block is empty.
1476 */
1477 static inline pco_instr *pco_first_instr(pco_block *block)
1478 {
1479 if (list_is_empty(&block->instrs))
1480 return NULL;
1481
1482 return list_first_entry(&block->instrs, pco_instr, link);
1483 }
1484
1485 /**
1486 * \brief Returns the last instruction in a block.
1487 *
1488 * \param[in] block The block.
1489 * \return The last instruction, or NULL if the block is empty.
1490 */
1491 static inline pco_instr *pco_last_instr(pco_block *block)
1492 {
1493 if (list_is_empty(&block->instrs))
1494 return NULL;
1495
1496 return list_last_entry(&block->instrs, pco_instr, link);
1497 }
1498
1499 /**
1500 * \brief Returns the next instruction.
1501 *
1502 * \param[in] instr The current instruction.
1503 * \return The next instruction, or NULL if the end of the function has been
1504 * reached.
1505 */
1506 static inline pco_instr *pco_next_instr(pco_instr *instr)
1507 {
1508 if (!instr)
1509 return NULL;
1510
1511 if (instr != pco_last_instr(instr->parent_block))
1512 return list_entry(instr->link.next, pco_instr, link);
1513
1514 pco_block *block = pco_next_block_nonempty(instr->parent_block);
1515 return !block ? NULL : pco_first_instr(block);
1516 }
1517
1518 /**
1519 * \brief Returns the previous instruction.
1520 *
1521 * \param[in] instr The current instruction.
1522 * \return The previous instruction, or NULL if the start of the function has
1523 * been reached.
1524 */
1525 static inline pco_instr *pco_prev_instr(pco_instr *instr)
1526 {
1527 if (!instr)
1528 return NULL;
1529
1530 if (instr != pco_first_instr(instr->parent_block))
1531 return list_entry(instr->link.prev, pco_instr, link);
1532
1533 pco_block *block = pco_prev_block_nonempty(instr->parent_block);
1534 return !block ? NULL : pco_last_instr(block);
1535 }
1536
1537 /**
1538 * \brief Returns the first instruction group in a block.
1539 *
1540 * \param[in] block The block.
1541 * \return The first instruction group, or NULL if the block is empty.
1542 */
1543 static inline pco_igrp *pco_first_igrp(pco_block *block)
1544 {
1545 if (list_is_empty(&block->instrs))
1546 return NULL;
1547
1548 return list_first_entry(&block->instrs, pco_igrp, link);
1549 }
1550
1551 /**
1552 * \brief Returns the last instruction group in a block.
1553 *
1554 * \param[in] block The block.
1555 * \return The last instruction group, or NULL if the block is empty.
1556 */
1557 static inline pco_igrp *pco_last_igrp(pco_block *block)
1558 {
1559 if (list_is_empty(&block->instrs))
1560 return NULL;
1561
1562 return list_last_entry(&block->instrs, pco_igrp, link);
1563 }
1564
1565 /**
1566 * \brief Returns the next instruction group.
1567 *
1568 * \param[in] igrp The current instruction group.
1569 * \return The next instruction group, or NULL if the end of the function has
1570 * been reached.
1571 */
1572 static inline pco_igrp *pco_next_igrp(pco_igrp *igrp)
1573 {
1574 if (!igrp)
1575 return NULL;
1576
1577 if (igrp != pco_last_igrp(igrp->parent_block))
1578 return list_entry(igrp->link.next, pco_igrp, link);
1579
1580 pco_block *block = pco_next_block(igrp->parent_block);
1581 return !block ? NULL : pco_first_igrp(block);
1582 }
1583
1584 /**
1585 * \brief Returns the previous instruction group.
1586 *
1587 * \param[in] igrp The current instruction group.
1588 * \return The previous instruction group, or NULL if the start of the function
1589 * has been reached.
1590 */
1591 static inline pco_igrp *pco_prev_igrp(pco_igrp *igrp)
1592 {
1593 if (!igrp)
1594 return NULL;
1595
1596 if (igrp != pco_first_igrp(igrp->parent_block))
1597 return list_entry(igrp->link.prev, pco_igrp, link);
1598
1599 pco_block *block = pco_prev_block(igrp->parent_block);
1600 return !block ? NULL : pco_last_igrp(block);
1601 }
1602
1603 /**
1604 * \brief Returns the first instruction in an igrp.
1605 *
1606 * \param[in] igrp The igrp.
1607 * \return The first instruction, or NULL if the igrp is empty.
1608 */
1609 static inline pco_instr *pco_igrp_first_instr(pco_igrp *igrp)
1610 {
1611 pco_foreach_phase_in_igrp (igrp, p)
1612 return igrp->instrs[p];
1613
1614 return NULL;
1615 }
1616
1617 /**
1618 * \brief Returns the last instruction in an igrp.
1619 *
1620 * \param[in] igrp The igrp.
1621 * \return The last instruction, or NULL if the igrp is empty.
1622 */
1623 static inline pco_instr *pco_igrp_last_instr(pco_igrp *igrp)
1624 {
1625 pco_foreach_phase_in_igrp_rev (igrp, p)
1626 return igrp->instrs[p];
1627
1628 return NULL;
1629 }
1630
1631 /**
1632 * \brief Returns the next instruction in an igrp.
1633 *
1634 * \param[in] instr The current instruction.
1635 * \return The next instruction, or NULL if we've reached the end of the igrp.
1636 */
1637 static inline pco_instr *pco_igrp_next_instr(pco_instr *instr)
1638 {
1639 if (!instr)
1640 return NULL;
1641
1642 pco_igrp *igrp = instr->parent_igrp;
1643 pco_foreach_phase_in_igrp_from (igrp, p, instr->phase)
1644 return igrp->instrs[p];
1645
1646 return NULL;
1647 }
1648
1649 /**
1650 * \brief Returns the previous instruction in an igrp.
1651 *
1652 * \param[in] instr The current instruction.
1653 * \return The previous instruction, or NULL if we've reached the end of the
1654 * igrp.
1655 */
1656 static inline pco_instr *pco_igrp_prev_instr(pco_instr *instr)
1657 {
1658 if (!instr)
1659 return NULL;
1660
1661 pco_igrp *igrp = instr->parent_igrp;
1662 pco_foreach_phase_in_igrp_from_rev (igrp, p, instr->phase)
1663 return igrp->instrs[p];
1664
1665 return NULL;
1666 }
1667
1668 /* Debug printing helpers. */
1669 static inline bool pco_should_print_nir(nir_shader *nir)
1670 {
1671 if (!PCO_DEBUG_PRINT(NIR))
1672 return false;
1673
1674 if (nir->info.internal && !PCO_DEBUG_PRINT(INTERNAL))
1675 return false;
1676
1677 if (nir->info.stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS))
1678 return false;
1679 else if (nir->info.stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS))
1680 return false;
1681 else if (nir->info.stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS))
1682 return false;
1683
1684 return true;
1685 }
1686
1687 static inline bool pco_should_print_shader(pco_shader *shader)
1688 {
1689 if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL))
1690 return false;
1691
1692 if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS))
1693 return false;
1694 else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS))
1695 return false;
1696 else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS))
1697 return false;
1698
1699 return true;
1700 }
1701
1702 static inline bool pco_should_print_shader_pass(pco_shader *shader)
1703 {
1704 if (!PCO_DEBUG_PRINT(PASSES))
1705 return false;
1706
1707 if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL))
1708 return false;
1709
1710 if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS))
1711 return false;
1712 else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS))
1713 return false;
1714 else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS))
1715 return false;
1716
1717 return true;
1718 }
1719
1720 static inline bool pco_should_print_binary(pco_shader *shader)
1721 {
1722 if (!PCO_DEBUG_PRINT(BINARY))
1723 return false;
1724
1725 if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL))
1726 return false;
1727
1728 if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS))
1729 return false;
1730 else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS))
1731 return false;
1732 else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS))
1733 return false;
1734
1735 return true;
1736 }
1737
1738 static inline bool pco_should_print_stats(pco_shader *shader)
1739 {
1740 if (!PCO_DEBUG_PRINT(STATS))
1741 return false;
1742
1743 if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL))
1744 return false;
1745
1746 if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS))
1747 return false;
1748 else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS))
1749 return false;
1750 else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS))
1751 return false;
1752
1753 return true;
1754 }
1755
1756 /* Interface with NIR. */
1757 typedef union PACKED _pco_smp_flags {
1758 struct PACKED {
1759 unsigned dim : 2;
1760 bool proj : 1;
1761 bool fcnorm : 1;
1762 bool nncoords : 1;
1763 enum pco_lod_mode lod_mode : 2;
1764 bool pplod : 1;
1765 bool tao : 1;
1766 bool soo : 1;
1767 bool sno : 1;
1768 bool array : 1;
1769 bool integer : 1;
1770 bool wrt : 1;
1771 unsigned pad : 2;
1772 };
1773
1774 uint16_t _;
1775 } pco_smp_flags;
1776 static_assert(sizeof(pco_smp_flags) == sizeof(uint16_t),
1777 "sizeof(pco_smp_flags) != sizeof(uint16_t)");
1778
1779 /* PCO IR passes. */
1780 bool pco_const_imms(pco_shader *shader);
1781 bool pco_bool(pco_shader *shader);
1782 bool pco_cf(pco_shader *shader);
1783 bool pco_dce(pco_shader *shader);
1784 bool pco_post_ra_legalize(pco_shader *shader);
1785 bool pco_pre_ra_legalize(pco_shader *shader);
1786 bool pco_end(pco_shader *shader);
1787 bool pco_group_instrs(pco_shader *shader);
1788 bool pco_index(pco_shader *shader, bool skip_ssa);
1789 bool pco_opt_comp_only_vecs(pco_shader *shader);
1790 bool pco_nir_compute_instance_check(nir_shader *shader);
1791 bool pco_nir_link_clip_cull_vars(nir_shader *producer, nir_shader *consumer);
1792 bool pco_nir_link_multiview(nir_shader *producer,
1793 nir_shader *consumer,
1794 pco_data *consumer_data);
1795 bool pco_nir_lower_algebraic(nir_shader *shader);
1796 bool pco_nir_lower_algebraic_late(nir_shader *shader);
1797 bool pco_nir_lower_alpha_to_coverage(nir_shader *shader);
1798 bool pco_nir_lower_atomics(nir_shader *shader, pco_data *data);
1799 bool pco_nir_lower_barriers(nir_shader *shader, pco_data *data);
1800 bool pco_nir_lower_clip_cull_vars(nir_shader *shader);
1801 bool pco_nir_lower_fs_intrinsics(nir_shader *shader);
1802 bool pco_nir_lower_vs_intrinsics(nir_shader *shader);
1803 bool pco_nir_lower_images(nir_shader *shader, pco_data *data, pco_ctx *ctx);
1804 bool pco_nir_lower_interpolation(nir_shader *shader, pco_fs_data *fs);
1805 bool pco_nir_lower_io(nir_shader *shader);
1806 bool pco_nir_lower_subgroups(nir_shader *shader);
1807 bool pco_nir_lower_tex(nir_shader *shader, pco_data *data, pco_ctx *ctx);
1808 bool pco_nir_lower_variables(nir_shader *shader, bool inputs, bool outputs);
1809 bool pco_nir_lower_vk(nir_shader *shader, pco_data *data);
1810 bool pco_nir_pfo(nir_shader *shader, pco_fs_data *fs);
1811 bool pco_nir_point_size(nir_shader *shader);
1812 bool pco_nir_pvi(nir_shader *shader, pco_vs_data *vs);
1813 bool pco_opt(pco_shader *shader);
1814 bool pco_ra(pco_shader *shader);
1815 bool pco_schedule(pco_shader *shader);
1816 bool pco_shrink_vecs(pco_shader *shader);
1817
1818 typedef enum {
1819 pco_nir_lower_null_descriptor_ubo = (1 << 0),
1820 pco_nir_lower_null_descriptor_ssbo = (1 << 1),
1821 pco_nir_lower_null_descriptor_global = (1 << 2),
1822 pco_nir_lower_null_descriptor_texture = (1 << 3),
1823 pco_nir_lower_null_descriptor_image = (1 << 4),
1824
1825 pco_nir_lower_null_descriptor_all = BITFIELD_MASK(5),
1826 } pco_nir_lower_null_descriptor_options;
1827
1828 bool pco_nir_lower_null_descriptors(
1829 nir_shader *shader,
1830 pco_nir_lower_null_descriptor_options options);
1831
1832 /**
1833 * \brief Returns the PCO bits for a bit size.
1834 *
1835 * \param[in] bits Reference.
1836 * \return PCO bits.
1837 */
1838 static inline enum pco_bits pco_bits(unsigned bits)
1839 {
1840 switch (bits) {
1841 case 1:
1842 return PCO_BITS_1;
1843
1844 case 8:
1845 return PCO_BITS_8;
1846
1847 case 16:
1848 return PCO_BITS_16;
1849
1850 case 32:
1851 return PCO_BITS_32;
1852
1853 case 64:
1854 return PCO_BITS_64;
1855
1856 default:
1857 break;
1858 }
1859
1860 UNREACHABLE("");
1861 }
1862
1863 /* PCO ref checkers. */
1864 /**
1865 * \brief Returns whether a reference is null.
1866 *
1867 * \param[in] ref PCO reference.
1868 * \return True if the reference is null.
1869 */
1870 static inline bool pco_ref_is_null(pco_ref ref)
1871 {
1872 return ref.type == PCO_REF_TYPE_NULL;
1873 }
1874
1875 /**
1876 * \brief Returns whether a reference is an SSA variable.
1877 *
1878 * \param[in] ref PCO reference.
1879 * \return True if the reference is an SSA variable.
1880 */
1881 static inline bool pco_ref_is_ssa(pco_ref ref)
1882 {
1883 return ref.type == PCO_REF_TYPE_SSA;
1884 }
1885
1886 /**
1887 * \brief Returns whether a reference is a virtual register.
1888 *
1889 * \param[in] ref PCO reference.
1890 * \return True if the reference is a virtual register.
1891 */
1892 static inline bool pco_ref_is_vreg(pco_ref ref)
1893 {
1894 return ref.type == PCO_REF_TYPE_REG && ref.reg_class == PCO_REG_CLASS_VIRT;
1895 }
1896
1897 /**
1898 * \brief Returns whether a reference is a register.
1899 *
1900 * \param[in] ref PCO reference.
1901 * \return True if the reference is a register.
1902 */
1903 static inline bool pco_ref_is_reg(pco_ref ref)
1904 {
1905 return ref.type == PCO_REF_TYPE_REG;
1906 }
1907
1908 /**
1909 * \brief Returns whether a reference is a hardware register.
1910 *
1911 * \param[in] ref PCO reference.
1912 * \return True if the reference is a hardware register.
1913 */
1914 static inline bool pco_ref_is_hwreg(pco_ref ref)
1915 {
1916 return ref.type == PCO_REF_TYPE_REG && ref.reg_class != PCO_REG_CLASS_VIRT;
1917 }
1918
1919 /**
1920 * \brief Returns whether a reference is an index register.
1921 *
1922 * \param[in] ref PCO reference.
1923 * \return True if the reference is an index register.
1924 */
1925 static inline bool pco_ref_is_idx_reg(pco_ref ref)
1926 {
1927 return ref.type == PCO_REF_TYPE_IDX_REG;
1928 }
1929
1930 /**
1931 * \brief Returns whether a reference is an index register pointing to itself.
1932 *
1933 * \param[in] ref PCO reference.
1934 * \return True if the reference is an index register pointing to itself.
1935 */
1936 static inline bool pco_ref_is_self_idx_reg(pco_ref ref)
1937 {
1938 return pco_ref_is_idx_reg(ref) && ref.reg_class == PCO_REG_CLASS_INDEX;
1939 }
1940
1941 /**
1942 * \brief Returns whether a reference is an immediate.
1943 *
1944 * \param[in] ref PCO reference.
1945 * \return True if the reference is an immediate.
1946 */
1947 static inline bool pco_ref_is_imm(pco_ref ref)
1948 {
1949 return ref.type == PCO_REF_TYPE_IMM;
1950 }
1951
1952 /**
1953 * \brief Returns whether a reference is I/O.
1954 *
1955 * \param[in] ref PCO reference.
1956 * \return True if the reference is I/O.
1957 */
1958 static inline bool pco_ref_is_io(pco_ref ref)
1959 {
1960 return ref.type == PCO_REF_TYPE_IO;
1961 }
1962
1963 /**
1964 * \brief Returns whether a reference is a predicate.
1965 *
1966 * \param[in] ref PCO reference.
1967 * \return True if the reference is a predicate.
1968 */
1969 static inline bool pco_ref_is_pred(pco_ref ref)
1970 {
1971 return ref.type == PCO_REF_TYPE_PRED;
1972 }
1973
1974 /**
1975 * \brief Returns whether a reference is a dependant read counter.
1976 *
1977 * \param[in] ref PCO reference.
1978 * \return True if the reference is a dependant read counter.
1979 */
1980 static inline bool pco_ref_is_drc(pco_ref ref)
1981 {
1982 return ref.type == PCO_REF_TYPE_DRC;
1983 }
1984
1985 /**
1986 * \brief Returns whether a reference is scalar.
1987 *
1988 * \param[in] ref PCO reference.
1989 * \return True if the reference is scalar.
1990 */
1991 static inline bool pco_ref_is_scalar(pco_ref ref)
1992 {
1993 return !ref.chans;
1994 }
1995
1996 /**
1997 * \brief Return whether a reference is a vertex input register.
1998 *
1999 * \param[in] ref PCO reference.
2000 * \return True if the reference is a vertex input register.
2001 */
2002 static inline bool pco_ref_is_vtxin(pco_ref ref)
2003 {
2004 return ref.type == PCO_REF_TYPE_REG && ref.reg_class == PCO_REG_CLASS_VTXIN;
2005 }
2006
2007 /* PCO ref getters. */
2008 /**
2009 * \brief Returns the pointee component of an indexed register reference.
2010 *
2011 * \param[in] ref Indexed register reference.
2012 * \return Pointee component of the indexed register reference.
2013 */
2014 static inline pco_ref pco_ref_get_idx_pointee(pco_ref ref)
2015 {
2016 assert(pco_ref_is_idx_reg(ref));
2017
2018 pco_ref pointee = ref;
2019 pointee.val = ref.idx_reg.offset;
2020 pointee.type = PCO_REF_TYPE_REG;
2021
2022 return pointee;
2023 }
2024
2025 /**
2026 * \brief Returns the data type of a reference.
2027 *
2028 * \param[in] ref Reference.
2029 * \return Datatype.
2030 */
2031 static inline enum pco_dtype pco_ref_get_dtype(pco_ref ref)
2032 {
2033 return ref.dtype;
2034 }
2035
2036 /**
2037 * \brief Returns the number of channels for a reference type.
2038 *
2039 * \param[in] ref Reference.
2040 * \return Number of channels.
2041 */
2042 static inline unsigned pco_ref_get_chans(pco_ref ref)
2043 {
2044 return ref.chans + 1;
2045 }
2046
2047 /**
2048 * \brief Returns the number of bits for a reference type.
2049 *
2050 * \param[in] ref Reference.
2051 * \return Number of bits.
2052 */
2053 static inline unsigned pco_ref_get_bits(pco_ref ref)
2054 {
2055 switch (ref.bits) {
2056 case PCO_BITS_1:
2057 return 1;
2058
2059 case PCO_BITS_8:
2060 return 8;
2061
2062 case PCO_BITS_16:
2063 return 16;
2064
2065 case PCO_BITS_32:
2066 return 32;
2067
2068 case PCO_BITS_64:
2069 return 64;
2070
2071 default:
2072 break;
2073 }
2074
2075 UNREACHABLE("");
2076 }
2077
2078 /**
2079 * \brief Returns the bit-sized value in an immediate reference.
2080 *
2081 * \param[in] ref Reference.
2082 * \return Immediate value.
2083 */
2084 static inline uint64_t pco_ref_get_imm(pco_ref ref)
2085 {
2086 assert(pco_ref_is_imm(ref));
2087
2088 unsigned num_bits = pco_ref_get_bits(ref);
2089
2090 switch (ref.dtype) {
2091 case PCO_DTYPE_FLOAT:
2092 assert(num_bits == 32);
2093 FALLTHROUGH;
2094 case PCO_DTYPE_ANY:
2095 FALLTHROUGH;
2096 case PCO_DTYPE_UNSIGNED:
2097 return ref.val & BITFIELD_MASK(num_bits);
2098
2099 case PCO_DTYPE_SIGNED:
2100 return util_sign_extend(ref.val, num_bits);
2101
2102 default:
2103 break;
2104 }
2105
2106 UNREACHABLE("");
2107 }
2108
2109 /**
2110 * \brief Returns the register class of a reference type.
2111 *
2112 * \param[in] ref Reference.
2113 * \return Register class.
2114 */
2115 static inline enum pco_reg_class pco_ref_get_reg_class(pco_ref ref)
2116 {
2117 assert(pco_ref_is_reg(ref) || pco_ref_is_idx_reg(ref));
2118 return ref.reg_class;
2119 }
2120
2121 /**
2122 * \brief Returns the register index of a reference type.
2123 *
2124 * \param[in] ref Reference.
2125 * \return Register index.
2126 */
2127 static inline unsigned pco_ref_get_reg_index(pco_ref ref)
2128 {
2129 assert(pco_ref_is_reg(ref) || pco_ref_is_idx_reg(ref));
2130
2131 unsigned index = pco_ref_is_idx_reg(ref) ? ref.idx_reg.offset : ref.val;
2132
2133 return index;
2134 }
2135
2136 /**
2137 * \brief Returns the register index control of a reference type.
2138 *
2139 * \param[in] ref Reference.
2140 * \return Register index control.
2141 */
2142 static inline enum pco_idx_ctrl pco_ref_get_reg_idx_ctrl(pco_ref ref)
2143 {
2144 assert(pco_ref_is_reg(ref) || pco_ref_is_idx_reg(ref));
2145
2146 if (pco_ref_is_reg(ref))
2147 return PCO_IDX_CTRL_NONE;
2148
2149 return PCO_IDX_CTRL_IDX0 + ref.idx_reg.num;
2150 }
2151
2152 /**
2153 * \brief Returns the temp register index from its reference type.
2154 *
2155 * \param[in] ref Reference.
2156 * \return Temp index.
2157 */
2158 static inline unsigned pco_ref_get_temp(pco_ref ref)
2159 {
2160 assert(pco_ref_is_reg(ref));
2161 assert(pco_ref_get_reg_class(ref) == PCO_REG_CLASS_TEMP);
2162
2163 return pco_ref_get_reg_index(ref);
2164 }
2165
2166 /**
2167 * \brief Returns the coefficient register index from its reference type.
2168 *
2169 * \param[in] ref Reference.
2170 * \return Coefficient index.
2171 */
2172 static inline unsigned pco_ref_get_coeff(pco_ref ref)
2173 {
2174 assert(pco_ref_is_reg(ref));
2175 assert(pco_ref_get_reg_class(ref) == PCO_REG_CLASS_COEFF);
2176
2177 return pco_ref_get_reg_index(ref);
2178 }
2179
2180 /**
2181 * \brief Returns the I/O from its reference type.
2182 *
2183 * \param[in] ref Reference.
2184 * \return I/O.
2185 */
2186 static inline enum pco_io pco_ref_get_io(pco_ref ref)
2187 {
2188 assert(pco_ref_is_io(ref));
2189 assert(ref.val < _PCO_IO_COUNT);
2190 return ref.val;
2191 }
2192
2193 /**
2194 * \brief Returns the movw01 value of an I/O reference type.
2195 *
2196 * \param[in] ref Reference.
2197 * \return movw01 value.
2198 */
2199 static inline enum pco_movw01 pco_ref_get_movw01(pco_ref ref)
2200 {
2201 /* Default/unused case. */
2202 if (pco_ref_is_null(ref))
2203 return PCO_MOVW01_FT0;
2204
2205 switch (pco_ref_get_io(ref)) {
2206 case PCO_IO_FT0:
2207 return PCO_MOVW01_FT0;
2208
2209 case PCO_IO_FT1:
2210 return PCO_MOVW01_FT1;
2211
2212 case PCO_IO_FT2:
2213 return PCO_MOVW01_FT2;
2214
2215 case PCO_IO_FTE:
2216 return PCO_MOVW01_FTE;
2217
2218 default:
2219 break;
2220 }
2221
2222 UNREACHABLE("");
2223 }
2224
2225 /**
2226 * \brief Returns the predicate from its reference type.
2227 *
2228 * \param[in] ref Reference.
2229 * \return Predicate.
2230 */
2231 static inline enum pco_pred pco_ref_get_pred(pco_ref ref)
2232 {
2233 assert(pco_ref_is_pred(ref));
2234 assert(ref.val < _PCO_PRED_COUNT);
2235 return ref.val;
2236 }
2237
2238 /**
2239 * \brief Returns the dependent read counter from its reference type.
2240 *
2241 * \param[in] ref Reference.
2242 * \return Dependent read counter.
2243 */
2244 static inline enum pco_drc pco_ref_get_drc(pco_ref ref)
2245 {
2246 assert(pco_ref_is_drc(ref));
2247 assert(ref.val < _PCO_DRC_COUNT);
2248 return ref.val;
2249 }
2250
2251 /**
2252 * \brief Returns whether the reference has any mods set.
2253 *
2254 * \param[in] ref Reference.
2255 * \return True if any mods are set.
2256 */
2257 static inline bool pco_ref_has_mods_set(pco_ref ref)
2258 {
2259 return ref.oneminus || ref.clamp || ref.abs || ref.neg || ref.flr ||
2260 (ref.elem != 0);
2261 }
2262
2263 /**
2264 * \brief Returns whether a reference is a temporary register.
2265 *
2266 * \param[in] ref PCO reference.
2267 * \return True if the reference is a temporary register.
2268 */
2269 static inline bool pco_ref_is_temp(pco_ref ref)
2270 {
2271 return pco_ref_is_reg(ref) &&
2272 pco_ref_get_reg_class(ref) == PCO_REG_CLASS_TEMP;
2273 }
2274
2275 /* PCO ref builders. */
2276 /**
2277 * \brief Builds and returns a null reference.
2278 *
2279 * \return NULL reference.
2280 */
2281 static inline pco_ref pco_ref_null(void)
2282 {
2283 return (pco_ref){
2284 .type = PCO_REF_TYPE_NULL,
2285 };
2286 }
2287
2288 /**
2289 * \brief Builds and returns an SSA reference.
2290 *
2291 * \return SSA reference.
2292 */
2293 static inline pco_ref pco_ref_ssa(unsigned index, unsigned bits, unsigned chans)
2294 {
2295 return (pco_ref){
2296 .val = index,
2297 .chans = chans - 1,
2298 .bits = pco_bits(bits),
2299 .type = PCO_REF_TYPE_SSA,
2300 };
2301 }
2302
2303 /**
2304 * \brief Builds and returns a new SSA reference.
2305 *
2306 * \param[in,out] func The function.
2307 * \param[in] bits Number of bits.
2308 * \param[in] chans Number of channels.
2309 * \return SSA reference.
2310 */
2311 static inline pco_ref
2312 pco_ref_new_ssa(pco_func *func, unsigned bits, unsigned chans)
2313 {
2314 return pco_ref_ssa(func->next_ssa++, bits, chans);
2315 }
2316
2317 /**
2318 * \brief Builds and returns a new SSA reference cloning the properties of
2319 * another.
2320 *
2321 * \param[in,out] func The function.
2322 * \param [in] ref The base reference.
2323 * \return SSA reference.
2324 */
2325 static inline pco_ref pco_ref_new_ssa_clone(pco_func *func, pco_ref ref)
2326 {
2327 return pco_ref_ssa(func->next_ssa++,
2328 pco_ref_get_bits(ref),
2329 pco_ref_get_chans(ref));
2330 }
2331
2332 /**
2333 * \brief Builds and returns a new 32x1 SSA reference.
2334 *
2335 * \param[in,out] func The function.
2336 * \return SSA reference.
2337 */
2338 static inline pco_ref pco_ref_new_ssa32(pco_func *func)
2339 {
2340 return pco_ref_new_ssa(func, 32, 1);
2341 }
2342
2343 /**
2344 * \brief Builds and returns a new 32x2 SSA address reference.
2345 *
2346 * \param[in,out] func The function.
2347 * \return SSA address reference.
2348 */
2349 static inline pco_ref pco_ref_new_ssa_addr(pco_func *func)
2350 {
2351 return pco_ref_new_ssa(func, 32, 2);
2352 }
2353
2354 /**
2355 * \brief Builds new 32x1[2] SSA address component references.
2356 *
2357 * \param[in,out] func The function.
2358 * \param[out] addr_comps The new address component references.
2359 */
2360 static inline void pco_ref_new_ssa_addr_comps(pco_func *func,
2361 pco_ref addr_comps[static 2])
2362 {
2363 addr_comps[0] = pco_ref_new_ssa32(func);
2364 addr_comps[1] = pco_ref_new_ssa32(func);
2365 }
2366
2367 /**
2368 * \brief Builds and returns a new 32x(2+n) SSA address and data reference.
2369 *
2370 * \param[in,out] func The function.
2371 * \param[in] data_size The size of the data.
2372 * \return SSA address and data reference.
2373 */
2374 static inline pco_ref pco_ref_new_ssa_addr_data(pco_func *func,
2375 unsigned data_size)
2376 {
2377 return pco_ref_new_ssa(func, 32, 2 + data_size);
2378 }
2379
2380 /**
2381 * \brief Builds and returns a virtual register reference.
2382 *
2383 * \param[in] index Virtual register index.
2384 * \return Virtual register reference.
2385 */
2386 static inline pco_ref pco_ref_vreg(unsigned index)
2387 {
2388 return (pco_ref){
2389 .val = index,
2390 .bits = PCO_BITS_32,
2391 .type = PCO_REF_TYPE_REG,
2392 .reg_class = PCO_REG_CLASS_VIRT,
2393 };
2394 }
2395
2396 /**
2397 * \brief Builds and returns a new virtual register.
2398 *
2399 * \param[in,out] func The function.
2400 * \return Virtual register reference.
2401 */
2402 static inline pco_ref pco_ref_new_vreg(pco_func *func)
2403 {
2404 return pco_ref_vreg(func->next_vreg++);
2405 }
2406
2407 /**
2408 * \brief Remaps an SSA register to an existing virtual register.
2409 *
2410 * \param[in,out] func The function.
2411 * \param[in] ref Base SSA reference.
2412 * \param[in] bits New bit width, or 0 if unchanged.
2413 * \return Virtual register reference.
2414 */
2415 static inline pco_ref
2416 pco_ref_ssa_vreg(ASSERTED pco_func *func, pco_ref ref, unsigned bits)
2417 {
2418 assert(pco_ref_is_ssa(ref));
2419 assert(ref.val < func->next_vreg);
2420
2421 ref.type = PCO_REF_TYPE_REG;
2422 ref.reg_class = PCO_REG_CLASS_VIRT;
2423
2424 if (bits)
2425 ref.bits = pco_bits(bits);
2426
2427 return ref;
2428 }
2429
2430 /**
2431 * \brief Builds and returns a vector hardware register reference.
2432 *
2433 * \param[in] index Register index.
2434 * \param[in] reg_class Register class.
2435 * \param[in] chans Number of channels.
2436 * \return Hardware register reference.
2437 */
2438 static inline pco_ref
2439 pco_ref_hwreg_vec(unsigned index, enum pco_reg_class reg_class, unsigned chans)
2440 {
2441 assert(reg_class != PCO_REG_CLASS_VIRT);
2442
2443 return (pco_ref){
2444 .val = index,
2445 .chans = chans - 1,
2446 .bits = PCO_BITS_32,
2447 .type = PCO_REF_TYPE_REG,
2448 .reg_class = reg_class,
2449 };
2450 }
2451
2452 /**
2453 * \brief Builds and returns a scalar hardware register reference.
2454 *
2455 * \param[in] index Register index.
2456 * \param[in] reg_class Register class.
2457 * \return Hardware register reference.
2458 */
2459 static inline pco_ref pco_ref_hwreg(unsigned index,
2460 enum pco_reg_class reg_class)
2461 {
2462 return pco_ref_hwreg_vec(index, reg_class, 1);
2463 }
2464
2465 /**
2466 * \brief Builds 32x1[2] hardware register address component references.
2467 *
2468 * \param[in] index Register index.
2469 * \param[in] reg_class Register class.
2470 * \param[out] addr_comps The new address component references.
2471 */
2472 static inline void pco_ref_hwreg_addr_comps(unsigned index,
2473 enum pco_reg_class reg_class,
2474 pco_ref addr_comps[static 2])
2475 {
2476 addr_comps[0] = pco_ref_hwreg(index, reg_class);
2477 addr_comps[1] = pco_ref_hwreg(index + 1, reg_class);
2478 }
2479
2480 static inline bool idx_reg_pointee_is_valid(pco_ref ref)
2481 {
2482 assert(pco_ref_is_idx_reg(ref));
2483
2484 switch (pco_ref_get_reg_class(ref)) {
2485 case PCO_REG_CLASS_TEMP:
2486 case PCO_REG_CLASS_VTXIN:
2487 case PCO_REG_CLASS_COEFF:
2488 case PCO_REG_CLASS_SHARED:
2489 case PCO_REG_CLASS_INDEX:
2490 case PCO_REG_CLASS_PIXOUT:
2491 return true;
2492
2493 default:
2494 break;
2495 }
2496
2497 return false;
2498 }
2499
2500 /**
2501 * \brief Builds and returns an indexed vector hardware register reference.
2502 *
2503 * \param[in] num Index register number.
2504 * \param[in] offset Pointee offset.
2505 * \param[in] reg_class Register class.
2506 * \param[in] chans Number of channels.
2507 * \return Hardware register reference.
2508 */
2509 static inline pco_ref pco_ref_hwreg_idx_vec(unsigned num,
2510 unsigned offset,
2511 enum pco_reg_class reg_class,
2512 unsigned chans)
2513 {
2514 assert(reg_class != PCO_REG_CLASS_VIRT);
2515
2516 return (pco_ref){
2517 .idx_reg = {
2518 .num = num,
2519 .offset = offset,
2520 },
2521 .chans = chans - 1,
2522 .bits = PCO_BITS_32,
2523 .type = PCO_REF_TYPE_IDX_REG,
2524 .reg_class = reg_class,
2525 };
2526 }
2527
2528 /**
2529 * \brief Builds and returns an indexed scalar hardware register reference.
2530 *
2531 * \param[in] num Index register number.
2532 * \param[in] offset Pointee offset.
2533 * \param[in] reg_class Register class.
2534 * \return Hardware register reference.
2535 */
2536 static inline pco_ref
2537 pco_ref_hwreg_idx(unsigned num, unsigned offset, enum pco_reg_class reg_class)
2538 {
2539 return pco_ref_hwreg_idx_vec(num, offset, reg_class, 1);
2540 }
2541
2542 /**
2543 * \brief Builds and returns an indexed hardware register reference using an
2544 * existing hardware register reference.
2545 *
2546 * \param[in] ref Base reference
2547 * \param[in] reg_class Register class.
2548 * \param[in] chans Number of channels.
2549 * \return Hardware register reference.
2550 */
2551 static inline pco_ref pco_ref_hwreg_idx_from(unsigned num, pco_ref ref)
2552 {
2553 assert(pco_ref_is_reg(ref));
2554
2555 pco_ref idx_ref = {
2556 .idx_reg = {
2557 .num = num,
2558 .offset = ref.val,
2559 },
2560 .chans = ref.chans,
2561 .bits = ref.bits,
2562 .type = PCO_REF_TYPE_IDX_REG,
2563 .reg_class = ref.reg_class,
2564 };
2565
2566 assert(idx_reg_pointee_is_valid(idx_ref));
2567 return idx_ref;
2568 }
2569
2570 /**
2571 * \brief Builds and returns an immediate reference.
2572 *
2573 * \param[in] val Immediate value.
2574 * \param[in] bits Immediate bit size.
2575 * \param[in] dtype Immediate datatype.
2576 * \return Immediate reference.
2577 */
2578 static inline pco_ref
2579 pco_ref_imm(uint32_t val, enum pco_bits bits, enum pco_dtype dtype)
2580 {
2581 return (pco_ref){
2582 .val = val,
2583 .dtype = dtype,
2584 .bits = bits,
2585 .type = PCO_REF_TYPE_IMM,
2586 };
2587 }
2588
2589 /**
2590 * \brief Builds and returns an 8-bit immediate reference.
2591 *
2592 * \param[in] val 8-bit immediate.
2593 * \return Immediate reference.
2594 */
2595 static inline pco_ref pco_ref_imm8(uint8_t val)
2596 {
2597 return pco_ref_imm(val, PCO_BITS_8, PCO_DTYPE_UNSIGNED);
2598 }
2599
2600 /**
2601 * \brief Builds and returns a 16-bit immediate reference.
2602 *
2603 * \param[in] val 16-bit immediate.
2604 * \return Immediate reference.
2605 */
2606 static inline pco_ref pco_ref_imm16(uint16_t val)
2607 {
2608 return pco_ref_imm(val, PCO_BITS_16, PCO_DTYPE_UNSIGNED);
2609 }
2610
2611 /**
2612 * \brief Builds and returns a 32-bit immediate reference.
2613 *
2614 * \param[in] val 32-bit immediate.
2615 * \return Immediate reference.
2616 */
2617 static inline pco_ref pco_ref_imm32(uint32_t val)
2618 {
2619 return pco_ref_imm(val, PCO_BITS_32, PCO_DTYPE_UNSIGNED);
2620 }
2621
2622 /**
2623 * \brief Builds and returns an untyped 8-bit immediate reference.
2624 *
2625 * \param[in] val 8-bit immediate.
2626 * \return Immediate reference.
2627 */
2628 static inline pco_ref pco_ref_val8(uint8_t val)
2629 {
2630 return pco_ref_imm(val, PCO_BITS_8, PCO_DTYPE_ANY);
2631 }
2632
2633 /**
2634 * \brief Builds and returns an untyped 16-bit immediate reference.
2635 *
2636 * \param[in] val 16-bit immediate.
2637 * \return Immediate reference.
2638 */
2639 static inline pco_ref pco_ref_val16(uint16_t val)
2640 {
2641 return pco_ref_imm(val, PCO_BITS_16, PCO_DTYPE_ANY);
2642 }
2643
2644 /**
2645 * \brief Builds and returns an untyped 32-bit immediate reference.
2646 *
2647 * \param[in] val 32-bit immediate.
2648 * \return Immediate reference.
2649 */
2650 static inline pco_ref pco_ref_val32(uint32_t val)
2651 {
2652 return pco_ref_imm(val, PCO_BITS_32, PCO_DTYPE_UNSIGNED);
2653 }
2654
2655 /**
2656 * \brief Builds and returns an I/O reference.
2657 *
2658 * \param[in] io I/O.
2659 * \return I/O reference.
2660 */
2661 static inline pco_ref pco_ref_io(enum pco_io io)
2662 {
2663 return (pco_ref){
2664 .val = io,
2665 .type = PCO_REF_TYPE_IO,
2666 };
2667 }
2668
2669 /**
2670 * \brief Builds and returns a predicate reference.
2671 *
2672 * \param[in] pred predicate.
2673 * \return Predicate reference.
2674 */
2675 static inline pco_ref pco_ref_pred(enum pco_pred pred)
2676 {
2677 return (pco_ref){
2678 .val = pred,
2679 .type = PCO_REF_TYPE_PRED,
2680 };
2681 }
2682
2683 /**
2684 * \brief Builds and returns a dependent read counter reference.
2685 *
2686 * \param[in] drc Dependent read counter.
2687 * \return Dependent read counter reference.
2688 */
2689 static inline pco_ref pco_ref_drc(enum pco_drc drc)
2690 {
2691 return (pco_ref){
2692 .val = drc,
2693 .type = PCO_REF_TYPE_DRC,
2694 };
2695 }
2696
2697 /* PCO ref utils. */
2698
2699 /**
2700 * \brief Updates a reference to reset its mods.
2701 *
2702 * \param[in] ref Base reference.
2703 * \return Updated reference.
2704 */
2705 static inline pco_ref pco_ref_reset_mods(pco_ref ref)
2706 {
2707 ref.oneminus = false;
2708 ref.clamp = false;
2709 ref.flr = false;
2710 ref.abs = false;
2711 ref.neg = false;
2712 ref.elem = 0;
2713
2714 return ref;
2715 }
2716
2717 /**
2718 * \brief Transfers reference mods, optionally resetting them.
2719 *
2720 * \param[in,out] dest Reference to transfer mods to.
2721 * \param[in,out] source Reference to transfer mods from.
2722 * \param[in] reset Whether to reset the source mods.
2723 */
2724 static inline void pco_ref_xfer_mods(pco_ref *dest, pco_ref *source, bool reset)
2725 {
2726 dest->oneminus = source->oneminus;
2727 dest->clamp = source->clamp;
2728 dest->flr = source->flr;
2729 dest->abs = source->abs;
2730 dest->neg = source->neg;
2731 dest->elem = source->elem;
2732
2733 if (reset)
2734 *source = pco_ref_reset_mods(*source);
2735 }
2736
2737 /**
2738 * \brief Updates a reference to set the oneminus modifier.
2739 *
2740 * \param[in] ref Base reference.
2741 * \return Updated reference.
2742 */
2743 static inline pco_ref pco_ref_oneminus(pco_ref ref)
2744 {
2745 ref.oneminus = true;
2746 return ref;
2747 }
2748
2749 /**
2750 * \brief Updates a reference to set the clamp modifier.
2751 *
2752 * \param[in] ref Base reference.
2753 * \return Updated reference.
2754 */
2755 static inline pco_ref pco_ref_clamp(pco_ref ref)
2756 {
2757 ref.clamp = true;
2758 return ref;
2759 }
2760
2761 /**
2762 * \brief Updates a reference to set the floor modifier.
2763 *
2764 * \param[in] ref Base reference.
2765 * \return Updated reference.
2766 */
2767 static inline pco_ref pco_ref_flr(pco_ref ref)
2768 {
2769 ref.flr = true;
2770 ref.abs = false;
2771 ref.neg = false;
2772 return ref;
2773 }
2774
2775 /**
2776 * \brief Updates a reference to set the abs modifier.
2777 *
2778 * \param[in] ref Base reference.
2779 * \return Updated reference.
2780 */
2781 static inline pco_ref pco_ref_abs(pco_ref ref)
2782 {
2783 ref.abs = true;
2784 ref.neg = false;
2785 return ref;
2786 }
2787
2788 /**
2789 * \brief Updates a reference to set the negate modifier.
2790 *
2791 * \param[in] ref Base reference.
2792 * \return Updated reference.
2793 */
2794 static inline pco_ref pco_ref_neg(pco_ref ref)
2795 {
2796 ref.neg = !ref.neg;
2797 return ref;
2798 }
2799
2800 /**
2801 * \brief Updates a reference to set the element modifier.
2802 *
2803 * \param[in] ref Base reference.
2804 * \param[in] elem New element modifier.
2805 * \return Updated reference.
2806 */
2807 static inline pco_ref pco_ref_elem(pco_ref ref, enum pco_elem elem)
2808 {
2809 ref.elem = elem;
2810 return ref;
2811 }
2812
2813 /**
2814 * \brief Updates a reference to set the number of channels.
2815 *
2816 * \param[in] ref Base reference.
2817 * \param[in] chans New number of channels.
2818 * \return Updated reference.
2819 */
2820 static inline pco_ref pco_ref_chans(pco_ref ref, unsigned chans)
2821 {
2822 ref.chans = chans - 1;
2823 return ref;
2824 }
2825
2826 /**
2827 * \brief Updates a reference to set the bit width.
2828 *
2829 * \param[in] ref Base reference.
2830 * \param[in] bits New bit width.
2831 * \return Updated reference.
2832 */
2833 static inline pco_ref pco_ref_bits(pco_ref ref, unsigned bits)
2834 {
2835 ref.bits = pco_bits(bits);
2836 return ref;
2837 }
2838
2839 /**
2840 * \brief Updates a reference value with the provided offset.
2841 *
2842 * \param[in] ref Base reference.
2843 * \param[in] offset Offset to apply.
2844 * \return Updated reference.
2845 */
2846 static inline pco_ref pco_ref_offset(pco_ref ref, signed offset)
2847 {
2848 int64_t val = pco_ref_is_idx_reg(ref) ? ref.idx_reg.offset : ref.val;
2849 val += offset;
2850
2851 if (pco_ref_is_idx_reg(ref)) {
2852 assert(util_last_bit64(val) <= PCO_REF_IDX_OFFSET_BITS);
2853 ref.idx_reg.offset = val;
2854 } else {
2855 assert(util_last_bit64(val) <= PCO_REF_VAL_BITS);
2856 ref.val = val;
2857 }
2858
2859 return ref;
2860 }
2861
2862 /**
2863 * \brief Checks whether two reference modifiers are the same.
2864 *
2865 * \param[in] ref0 First reference.
2866 * \param[in] ref1 Second reference.
2867 * \return True if both reference modifiers are the same.
2868 */
2869 static inline bool pco_ref_mods_are_equal(pco_ref ref0, pco_ref ref1)
2870 {
2871 return (ref0.oneminus == ref1.oneminus) && (ref0.clamp == ref1.clamp) &&
2872 (ref0.flr == ref1.flr) && (ref0.abs == ref1.abs) &&
2873 (ref0.neg == ref1.neg) && (ref0.elem == ref1.elem);
2874 }
2875
2876 /**
2877 * \brief Checks whether two references are the same.
2878 *
2879 * \param[in] ref0 First reference.
2880 * \param[in] ref1 Second reference.
2881 * \param[in] ignore_dtype Whether to ignore the datatype.
2882 * \return True if both references are the same.
2883 */
2884 /* TODO: can this be simplified? */
2885 static inline bool
2886 pco_refs_are_equal(pco_ref ref0, pco_ref ref1, bool ignore_dtype)
2887 {
2888 if (ref0.type != ref1.type)
2889 return false;
2890
2891 if (pco_ref_is_idx_reg(ref0)) {
2892 if ((ref0.idx_reg.num != ref1.idx_reg.num) ||
2893 (ref0.idx_reg.offset != ref1.idx_reg.offset)) {
2894 return false;
2895 }
2896 } else if (ref0.val != ref1.val) {
2897 return false;
2898 }
2899
2900 if (pco_ref_is_idx_reg(ref0) || pco_ref_is_reg(ref0))
2901 if (ref0.reg_class != ref1.reg_class)
2902 return false;
2903
2904 if (!pco_ref_mods_are_equal(ref0, ref1))
2905 return false;
2906
2907 if (ref0.chans != ref1.chans)
2908 return false;
2909
2910 if (!ignore_dtype && pco_ref_get_dtype(ref0) != pco_ref_get_dtype(ref1))
2911 return false;
2912
2913 if (pco_ref_get_bits(ref0) != pco_ref_get_bits(ref1))
2914 return false;
2915
2916 return true;
2917 }
2918
2919 /**
2920 * \brief Checks a reference has a valid hardware source mapping.
2921 *
2922 * \param[in] ref Reference.
2923 * \param[in] mapped_src Hardware source mapping.
2924 * \param[out] needs_s124 Whether the mapping needs to use S{1,2,4}
2925 * rather than S{0,2,3}.
2926 * \return True if the mapping is valid.
2927 */
2928 static inline bool
2929 ref_src_map_valid(pco_ref ref, enum pco_io mapped_src, bool *needs_s124)
2930 {
2931 if (needs_s124)
2932 *needs_s124 = false;
2933
2934 /* Restrictions only apply to hardware registers. */
2935 if (!pco_ref_is_idx_reg(ref) && !pco_ref_is_reg(ref))
2936 return true;
2937
2938 if (pco_ref_is_idx_reg(ref)) {
2939 return (mapped_src == PCO_IO_S0) || (mapped_src == PCO_IO_S2) ||
2940 (mapped_src == PCO_IO_S3);
2941 }
2942
2943 switch (pco_ref_get_reg_class(ref)) {
2944 case PCO_REG_CLASS_COEFF:
2945 case PCO_REG_CLASS_SHARED:
2946 case PCO_REG_CLASS_INDEX:
2947 case PCO_REG_CLASS_PIXOUT:
2948 return (mapped_src == PCO_IO_S0) || (mapped_src == PCO_IO_S2) ||
2949 (mapped_src == PCO_IO_S3);
2950
2951 case PCO_REG_CLASS_SPEC:
2952 if (needs_s124)
2953 *needs_s124 = true;
2954
2955 return (mapped_src == PCO_IO_S1) || (mapped_src == PCO_IO_S2) ||
2956 (mapped_src == PCO_IO_S4);
2957
2958 default:
2959 return true;
2960 }
2961
2962 return false;
2963 }
2964
2965 static inline enum pco_srcsel pco_ref_srcsel(pco_ref ref)
2966 {
2967 enum pco_io io = pco_ref_get_io(ref);
2968
2969 switch (io) {
2970 case PCO_IO_S0:
2971 return PCO_SRCSEL_S0;
2972
2973 case PCO_IO_S1:
2974 return PCO_SRCSEL_S1;
2975
2976 case PCO_IO_S2:
2977 return PCO_SRCSEL_S2;
2978
2979 case PCO_IO_S3:
2980 return PCO_SRCSEL_S3;
2981
2982 case PCO_IO_S4:
2983 return PCO_SRCSEL_S4;
2984
2985 case PCO_IO_S5:
2986 return PCO_SRCSEL_S5;
2987
2988 default:
2989 break;
2990 }
2991
2992 UNREACHABLE("");
2993 }
2994
2995 static inline enum pco_count_src pco_ref_count_src(pco_ref ref)
2996 {
2997 enum pco_io io = pco_ref_get_io(ref);
2998
2999 switch (io) {
3000 case PCO_IO_S2:
3001 return PCO_COUNT_SRC_S2;
3002
3003 case PCO_IO_FT2:
3004 return PCO_COUNT_SRC_FT2;
3005
3006 default:
3007 break;
3008 }
3009
3010 UNREACHABLE("");
3011 }
3012
3013 /**
3014 * \brief Returns whether none of the lower/upper sources in an instruction
3015 * group are set.
3016 *
3017 * \param[in] igrp The instruction group.
3018 * \param[in] upper True if checking the upper sources.
3019 * \return True if none of the lower/upper sources are set.
3020 */
3021 static inline bool pco_igrp_srcs_unset(pco_igrp *igrp, bool upper)
3022 {
3023 unsigned offset = upper ? ROGUE_ALU_INPUT_GROUP_SIZE : 0;
3024
3025 for (unsigned u = 0; u < ROGUE_ALU_INPUT_GROUP_SIZE; ++u)
3026 if (!pco_ref_is_null(igrp->srcs.s[u + offset]))
3027 return false;
3028
3029 return true;
3030 }
3031
3032 /**
3033 * \brief Returns whether none of the internal source selectors in an
3034 * instruction group are set.
3035 *
3036 * \param[in] igrp The instruction group.
3037 * \return True if none of the internal source selectors are set.
3038 */
3039 static inline bool pco_igrp_iss_unset(pco_igrp *igrp)
3040 {
3041 for (unsigned u = 0; u < ARRAY_SIZE(igrp->iss.is); ++u)
3042 if (!pco_ref_is_null(igrp->iss.is[u]))
3043 return false;
3044
3045 return true;
3046 }
3047
3048 /**
3049 * \brief Returns whether none of the destinations in an instruction group are
3050 * set.
3051 *
3052 * \param[in] igrp The instruction group.
3053 * \return True if none of the destinations are set.
3054 */
3055 static inline bool pco_igrp_dests_unset(pco_igrp *igrp)
3056 {
3057 for (unsigned u = 0; u < ARRAY_SIZE(igrp->dests.w); ++u)
3058 if (!pco_ref_is_null(igrp->dests.w[u]))
3059 return false;
3060
3061 return true;
3062 }
3063
3064 /**
3065 * \brief Iterates backwards to find the parent instruction of a source.
3066 *
3067 * \param[in] src The source whose parent is to be found.
3068 * \param[in] from The instruction to start iterating back from (inclusive).
3069 * \return The parent instruction if found, else NULL.
3070 */
3071 static inline pco_instr *find_parent_instr_from(pco_ref src, pco_instr *from)
3072 {
3073 pco_foreach_instr_dest_ssa (pdest, from) {
3074 if (pco_refs_are_equal(*pdest, src, false))
3075 return from;
3076 }
3077
3078 pco_foreach_instr_in_func_from_rev (instr, from) {
3079 pco_foreach_instr_dest_ssa (pdest, instr) {
3080 if (pco_refs_are_equal(*pdest, src, false))
3081 return instr;
3082 }
3083 }
3084
3085 return NULL;
3086 }
3087
3088 static inline unsigned pco_igrp_offset(pco_igrp *igrp)
3089 {
3090 return igrp->enc.offset;
3091 }
3092
3093 static inline unsigned pco_cf_node_offset(pco_cf_node *cf_node)
3094 {
3095 pco_block *block = pco_cf_node_as_block(cf_node);
3096 pco_igrp *igrp = pco_first_igrp(block);
3097 if (!igrp) {
3098 block = pco_next_block_nonempty(block);
3099 igrp = pco_first_igrp(block);
3100 }
3101
3102 return pco_igrp_offset(igrp);
3103 }
3104
3105 static inline unsigned pco_branch_rel_offset(pco_igrp *br, pco_cf_node *cf_node)
3106 {
3107 return pco_cf_node_offset(cf_node) - pco_igrp_offset(br);
3108 }
3109
3110 static inline unsigned pco_branch_rel_offset_next_igrp(pco_igrp *br)
3111 {
3112 pco_igrp *next_igrp = pco_next_igrp(br);
3113 assert(next_igrp);
3114
3115 return pco_igrp_offset(next_igrp) - pco_igrp_offset(br);
3116 }
3117
3118 static inline bool pco_should_skip_pass(const char *pass)
3119 {
3120 return comma_separated_list_contains(pco_skip_passes, pass);
3121 }
3122
3123 #define PCO_PASS(progress, shader, pass, ...) \
3124 do { \
3125 if (pco_should_skip_pass(#pass)) { \
3126 fprintf(stdout, "Skipping pass '%s'\n", #pass); \
3127 break; \
3128 } \
3129 \
3130 if (pass(shader, ##__VA_ARGS__)) { \
3131 UNUSED bool _; \
3132 progress = true; \
3133 \
3134 if (PCO_DEBUG(REINDEX)) \
3135 pco_index(shader, false); \
3136 \
3137 pco_validate_shader(shader, "after " #pass); \
3138 \
3139 if (pco_should_print_shader_pass(shader)) \
3140 pco_print_shader(shader, stdout, "after " #pass); \
3141 } \
3142 } while (0)
3143
3144 /* Common hw constants/references. */
3145
3146 /** Integer/float zero. */
3147 #define pco_zero pco_ref_hwreg(0, PCO_REG_CLASS_CONST)
3148
3149 /** Integer one. */
3150 #define pco_one pco_ref_hwreg(1, PCO_REG_CLASS_CONST)
3151
3152 /** Integer 2. */
3153 #define pco_2 pco_ref_hwreg(2, PCO_REG_CLASS_CONST)
3154
3155 /** Integer 4. */
3156 #define pco_4 pco_ref_hwreg(4, PCO_REG_CLASS_CONST)
3157
3158 /** Integer 5. */
3159 #define pco_5 pco_ref_hwreg(5, PCO_REG_CLASS_CONST)
3160
3161 /** Integer 7. */
3162 #define pco_7 pco_ref_hwreg(7, PCO_REG_CLASS_CONST)
3163
3164 /** Integer 31. */
3165 #define pco_31 pco_ref_hwreg(31, PCO_REG_CLASS_CONST)
3166
3167 /** Integer -1/true/0xffffffff/u32max. */
3168 #define pco_true pco_ref_hwreg(143, PCO_REG_CLASS_CONST)
3169 #define pco_u32max pco_true
3170
3171 /** Float 1. */
3172 #define pco_fone pco_ref_hwreg(64, PCO_REG_CLASS_CONST)
3173
3174 /** Float -1. */
3175 #define pco_fnegone pco_ref_neg(pco_ref_hwreg(64, PCO_REG_CLASS_CONST))
3176
3177 /** Float infinity. */
3178 #define pco_finf pco_ref_hwreg(142, PCO_REG_CLASS_CONST)
3179
3180 #define pco_p0 pco_ref_pred(PCO_PRED_P0)
3181
3182 /* Printing. */
3183 void pco_print_ref(pco_shader *shader, pco_ref ref);
3184 void pco_print_instr(pco_shader *shader, pco_instr *instr);
3185 void pco_print_igrp(pco_shader *shader, pco_igrp *igrp);
3186 void pco_print_cf_node_name(pco_shader *shader, pco_cf_node *cf_node);
3187 void pco_print_shader_info(pco_shader *shader);
3188 void pco_print_phase(pco_shader *shader,
3189 enum pco_alutype alutype,
3190 enum pco_op_phase phase);
3191
3192 /**
3193 * \brief Packs a descriptor set and binding into a uint32_t.
3194 *
3195 * \param[in] desc_set The descriptor set.
3196 * \param[in] binding The binding.
3197 * \return The packed descriptor set and binding.
3198 */
3199 static inline uint32_t pco_pack_desc(unsigned desc_set, unsigned binding)
3200 {
3201 assert(desc_set <= UINT16_MAX);
3202 assert(binding <= UINT16_MAX);
3203
3204 return desc_set | binding << 16;
3205 }
3206
3207 /**
3208 * \brief Unpacks a descriptor set and binding from a uint32_t.
3209 *
3210 * \param[in] packed The packed descriptor set and binding.
3211 * \param[out] desc_set The descriptor set.
3212 * \param[out] binding The binding.
3213 */
3214 static inline void
3215 pco_unpack_desc(uint32_t packed, unsigned *desc_set, unsigned *binding)
3216 {
3217 assert(desc_set);
3218 assert(binding);
3219
3220 *desc_set = packed & 0xffff;
3221 *binding = packed >> 16;
3222 }
3223
3224 #endif /* PCO_INTERNAL_H */