/* * Copyright © 2024 Imagination Technologies Ltd. * * SPDX-License-Identifier: MIT */ #ifndef PCO_INTERNAL_H #define PCO_INTERNAL_H /** * \file pco_internal.h * * \brief PCO internal header. */ #include "compiler/spirv/nir_spirv.h" #include "hwdef/rogue_hw_utils.h" #include "pco.h" #include "pco_common.h" #include "pco_data.h" #include "pco_ops.h" #include "spirv/nir_spirv.h" #include "util/compiler.h" #include "util/hash_table.h" #include "util/macros.h" #include "util/list.h" #include "util/u_debug.h" #include "util/u_dynarray.h" #include "util/u_math.h" #include #include #include /** PCO compiler context. */ typedef struct _pco_ctx { /** Device information. */ const struct pvr_device_info *dev_info; /** Device-specific NIR options. */ nir_shader_compiler_options nir_options; /** Device-specific SPIR-V to NIR options. */ struct spirv_to_nir_options spirv_options; /** USC library. */ const nir_shader *usclib; } pco_ctx; void pco_setup_spirv_options(const struct pvr_device_info *dev_info, struct spirv_to_nir_options *spirv_options); void pco_setup_nir_options(const struct pvr_device_info *dev_info, nir_shader_compiler_options *nir_options); /* Debug. */ enum pco_debug { PCO_DEBUG_VAL_SKIP = BITFIELD64_BIT(0), PCO_DEBUG_REINDEX = BITFIELD64_BIT(1), PCO_DEBUG_NO_PRED_CF = BITFIELD64_BIT(2), PCO_DEBUG_ALLOC_EXTRA_VTXINS = BITFIELD64_BIT(3), PCO_DEBUG_INT_SMP = BITFIELD64_BIT(4), }; extern uint64_t pco_debug; #define PCO_DEBUG(flag) unlikely(pco_debug &(PCO_DEBUG_##flag)) enum pco_debug_print { PCO_DEBUG_PRINT_VS = BITFIELD64_BIT(0), PCO_DEBUG_PRINT_FS = BITFIELD64_BIT(1), PCO_DEBUG_PRINT_CS = BITFIELD64_BIT(2), PCO_DEBUG_PRINT_ALL = PCO_DEBUG_PRINT_VS | PCO_DEBUG_PRINT_FS | PCO_DEBUG_PRINT_CS, PCO_DEBUG_PRINT_INTERNAL = BITFIELD64_BIT(3), PCO_DEBUG_PRINT_PASSES = BITFIELD64_BIT(4), PCO_DEBUG_PRINT_NIR = BITFIELD64_BIT(5), PCO_DEBUG_PRINT_BINARY = BITFIELD64_BIT(6), PCO_DEBUG_PRINT_VERBOSE = BITFIELD64_BIT(7), PCO_DEBUG_PRINT_RA = BITFIELD64_BIT(8), PCO_DEBUG_PRINT_STATS = BITFIELD64_BIT(9), }; extern uint64_t pco_debug_print; extern const char *pco_skip_passes; #define PCO_DEBUG_PRINT(flag) \ unlikely(pco_debug_print &(PCO_DEBUG_PRINT_##flag)) extern bool pco_color; void pco_debug_init(void); typedef struct _pco_cf_node pco_cf_node; typedef struct _pco_func pco_func; typedef struct _pco_block pco_block; typedef struct _pco_instr pco_instr; #define PCO_REF_VAL_BITS (32U) #define PCO_REF_IDX_NUM_BITS (2U) #define PCO_REF_IDX_OFFSET_BITS (8U) #define PCO_REF_IDX_PAD_BITS \ (PCO_REF_VAL_BITS - (PCO_REF_IDX_NUM_BITS + PCO_REF_IDX_OFFSET_BITS)) /** PCO reference index. */ typedef struct PACKED _pco_ref { /** Reference value. */ union PACKED { unsigned val : PCO_REF_VAL_BITS; struct PACKED { unsigned num : PCO_REF_IDX_NUM_BITS; /** Index register number. */ unsigned offset : PCO_REF_IDX_OFFSET_BITS; /** Offset. */ unsigned _pad : PCO_REF_IDX_PAD_BITS; } idx_reg; }; /** Source/destination modifiers. */ bool oneminus : 1; bool clamp : 1; bool flr : 1; bool abs : 1; bool neg : 1; enum pco_elem elem : 4; /** .e0.e1.e2.e3 */ enum pco_dtype dtype : 2; /** Reference data-type. */ unsigned chans : 10; /** Number of channels (1-1024). */ enum pco_bits bits : 3; /** Bit width. */ enum pco_ref_type type : 3; /** Reference type. */ enum pco_reg_class reg_class : 4; /** Register class. */ unsigned _pad : 1; } pco_ref; static_assert(sizeof(pco_ref) == 8, "sizeof(pco_ref) != 8"); /** PCO phi source. */ typedef struct _pco_phi_src { struct list_head link; /** Link in pco_instr::phi_srcs. */ pco_block *pred; /** Predecessor block. */ pco_ref ref; /** Source reference. */ } pco_phi_src; /** PCO instruction group. */ typedef struct _pco_igrp { struct list_head link; /** Node in pco_block::instrs. */ pco_block *parent_block; /** Basic block containing the igrp. */ pco_func *parent_func; /** Parent function. */ pco_instr *instrs[_PCO_OP_PHASE_COUNT]; /** Instruction/group list. */ /** Instruction group header. */ struct { unsigned da; unsigned length; union { enum pco_oporg oporg; enum pco_opcnt opcnt; }; bool olchk; bool w1p; bool w0p; enum pco_cc cc; enum pco_alutype alutype; union { struct { bool end; bool atom; unsigned rpt; }; struct { unsigned miscctl; enum pco_ctrlop ctrlop; }; }; } hdr; struct { pco_ref s[ROGUE_MAX_ALU_INPUTS]; } srcs; struct { pco_ref is[ROGUE_MAX_ALU_INTERNAL_SOURCES]; } iss; struct { pco_ref w[ROGUE_MAX_ALU_OUTPUTS]; } dests; struct { enum pco_igrp_hdr_variant hdr; union { enum pco_main_variant main; enum pco_backend_variant backend; enum pco_bitwise_variant bitwise; enum pco_ctrl_variant control; } instr[_PCO_OP_PHASE_COUNT]; enum pco_src_variant lower_src; enum pco_src_variant upper_src; enum pco_iss_variant iss; enum pco_dst_variant dest; } variant; struct { struct { unsigned hdr; unsigned lower_srcs; unsigned upper_srcs; unsigned iss; unsigned dests; unsigned instrs[_PCO_OP_PHASE_COUNT]; unsigned word_padding; unsigned align_padding; unsigned total; } len; unsigned offset; } enc; unsigned index; /** Igrp index. */ char *comment; /** Comment string. */ } pco_igrp; /** PCO instruction. */ typedef struct _pco_instr { union { struct { struct list_head link; /** Node in pco_block::instrs. */ pco_block *parent_block; /** Basic block containing the instruction. */ }; struct { enum pco_op_phase phase; /** Igrp phase the instruction is in. */ pco_igrp *parent_igrp; /** Igrp containing the instruction. */ }; }; pco_func *parent_func; /** Parent function. */ enum pco_op op; unsigned num_dests; pco_ref *dest; unsigned num_srcs; pco_ref *src; union { struct list_head phi_srcs; pco_cf_node *target_cf_node; }; /** Instruction flags/modifiers. */ uint32_t mod[_PCO_OP_MAX_MODS]; unsigned index; /** Instruction index. */ char *comment; /** Comment string. */ } pco_instr; /** PCO control-flow node type. */ enum pco_cf_node_type { PCO_CF_NODE_TYPE_BLOCK, PCO_CF_NODE_TYPE_IF, PCO_CF_NODE_TYPE_LOOP, PCO_CF_NODE_TYPE_FUNC, }; /** PCO control-flow flag. */ enum pco_cf_node_flag { PCO_CF_NODE_FLAG_BODY = 0, PCO_CF_NODE_FLAG_IF_THEN, PCO_CF_NODE_FLAG_IF_ELSE, PCO_CF_NODE_FLAG_PROLOGUE, PCO_CF_NODE_FLAG_INTERLOGUE, PCO_CF_NODE_FLAG_EPILOGUE, }; /** PCO control-flow node. */ typedef struct _pco_cf_node { struct list_head link; /** Node in lists of pco_cf_nodes. */ enum pco_cf_node_type type; /** CF node type. */ struct _pco_cf_node *parent; /** Parent cf node. */ enum pco_cf_node_flag flag; /** Implementation-defined flag. */ } pco_cf_node; /** PCO basic block. */ typedef struct _pco_block { pco_cf_node cf_node; /** Control flow node. */ pco_func *parent_func; /** Parent function. */ struct list_head instrs; /** Instruction/group list. */ unsigned index; /** Block index. */ } pco_block; /** PCO if cf construct. */ typedef struct _pco_if { pco_cf_node cf_node; /** CF node. */ pco_func *parent_func; /** Parent function. */ pco_ref cond; /** If condition. */ bool pred_exec; /** Whether this if construct uses predicated execution. */ struct list_head prologue; /** List of pco_cf_nodes for if prologue. */ struct list_head then_body; /** List of pco_cf_nodes for if body. */ struct list_head interlogue; /** List of pco_cf_nodes for if interlogue. */ struct list_head else_body; /** List of pco_cf_nodes for else body. */ struct list_head epilogue; /** List of pco_cf_nodes for if epilogue. */ unsigned index; /** If index. */ } pco_if; /** PCO loop cf construct. */ typedef struct _pco_loop { pco_cf_node cf_node; /** CF node. */ pco_func *parent_func; /** Parent function. */ struct list_head prologue; /** List of pco_cf_nodes for loop prologue. */ struct list_head body; /** List of pco_cf_nodes for loop body. */ struct list_head interlogue; /** List of pco_cf_nodes for loop interlogue. */ struct list_head epilogue; /** List of pco_cf_nodes for loop epilogue. */ unsigned index; /** Loop index. */ } pco_loop; #define VEC_USER_MULTI ((void *)(~0ULL)) /** PCO vector information. */ typedef struct _pco_vec_info { pco_instr *instr; /** Vector producer. */ pco_instr **comps; /** Array of vector components. */ pco_instr *vec_user; /** Vector user, or none, or multi. */ } pco_vec_info; /** PCO function. */ typedef struct _pco_func { struct list_head link; /** Node in pco_shader::funcs. */ pco_cf_node cf_node; /** Control flow node. */ pco_shader *parent_shader; /** Shader containing the function. */ enum pco_func_type type; /** Function type. */ unsigned index; /** Function index. */ const char *name; /** Function name. */ struct list_head body; /** List of pco_cf_nodes for function body. */ unsigned num_params; pco_ref *params; struct hash_table_u64 *vec_infos; unsigned next_ssa; /** Next SSA node index. */ unsigned next_vreg; /** Next virtual register index. */ unsigned next_instr; /** Next instruction index. */ unsigned next_igrp; /** Next igrp index. */ unsigned next_block; /** Next block index. */ unsigned next_if; /** Next if index. */ unsigned next_loop; /** Next loop index. */ unsigned temps; /** Number of temps allocated. */ unsigned vtxins; /** Number of vertex input registers used. */ pco_ref emc; /** Execution mask counter register. */ unsigned enc_offset; /** Encoding offset. */ } pco_func; /** PCO shader. */ typedef struct _pco_shader { pco_ctx *ctx; /** Compiler context. */ nir_shader *nir; /** Source NIR shader. */ mesa_shader_stage stage; /** Shader stage. */ const char *name; /** Shader name. */ bool is_internal; /** Whether this is an internal shader. */ bool is_grouped; /** Whether the shader uses igrps. */ bool is_legalized; /** Whether the shader has been legalized. */ struct list_head funcs; /** List of functions. */ unsigned next_func; /** Next function index. */ pco_data data; /** Shader data. */ struct util_dynarray binary; /** Shader binary. */ } pco_shader; /** Op info. */ struct pco_op_info { const char *str; /** Op name string. */ unsigned num_dests; /** Number of dests. */ unsigned num_srcs; /** Number of sources. */ uint64_t mods; /** Supported mods. */ uint8_t mod_map[_PCO_OP_MOD_COUNT]; /** Index into pco_instr::mod. */ uint64_t dest_mods[_PCO_OP_MAX_DESTS]; /** Supported dest mods. */ uint64_t src_mods[_PCO_OP_MAX_SRCS]; /** Supported source mods. */ enum pco_op_type type; /** Op type. */ bool has_target_cf_node; /** Set if op has a cf-node as a target. */ uint8_t dest_intrn_map[_PCO_OP_MAX_DESTS]; uint8_t src_intrn_map[_PCO_OP_MAX_SRCS]; #ifndef NDEBUG uint32_t grp_dest_maps[_PCO_OP_PHASE_COUNT][_PCO_OP_MAX_DESTS]; uint32_t grp_src_maps[_PCO_OP_PHASE_COUNT][_PCO_OP_MAX_SRCS]; #endif /* NDEBUG */ }; extern const struct pco_op_info pco_op_info[_PCO_OP_COUNT]; static_assert(_PCO_REF_MAP_COUNT <= 32, "enum pco_ref_map must fit into an uint32_t"); /** Op mod info. */ struct pco_op_mod_info { bool print_early : 1; /** Set if printed before the op. */ bool is_bitset : 1; /** Set if type is an enum bitset. */ enum pco_mod_type type; /** Datatype. */ union { const char *str; /** Mod name. */ const char **strs; /** Mod names (enums). */ }; uint32_t nzdefault; /** Default value if non-zero. */ }; extern const struct pco_op_mod_info pco_op_mod_info[_PCO_OP_MOD_COUNT]; /** Reference mod info. */ struct pco_ref_mod_info { bool is_bitset : 1; /** Set if type is an enum bitset. */ enum pco_mod_type type; /** Datatype. */ union { const char *str; /** Mod name. */ const char **strs; /** Mod names (enums). */ }; }; extern const struct pco_ref_mod_info pco_ref_mod_info[_PCO_REF_MOD_COUNT]; pco_shader *pco_shader_create(pco_ctx *ctx, nir_shader *nir, void *mem_ctx); pco_func *pco_func_create(pco_shader *shader, enum pco_func_type type, unsigned num_params); pco_block *pco_block_create(pco_func *func); pco_if *pco_if_create(pco_func *func); pco_loop *pco_loop_create(pco_func *func); pco_instr * pco_instr_create(pco_func *func, unsigned num_dests, unsigned num_srcs); pco_igrp *pco_igrp_create(pco_func *func); void pco_instr_delete(pco_instr *instr); /* Cast helpers. */ /* CF nodes. */ #define PCO_DEFINE_CAST(name, in_type, out_type, field, type_field, type_value) \ static inline out_type *name(const in_type *parent) \ { \ assert(parent && parent->type_field == type_value); \ return list_entry(parent, out_type, field); \ } PCO_DEFINE_CAST(pco_cf_node_as_block, pco_cf_node, pco_block, cf_node, type, PCO_CF_NODE_TYPE_BLOCK) PCO_DEFINE_CAST(pco_cf_node_as_if, pco_cf_node, pco_if, cf_node, type, PCO_CF_NODE_TYPE_IF) PCO_DEFINE_CAST(pco_cf_node_as_loop, pco_cf_node, pco_loop, cf_node, type, PCO_CF_NODE_TYPE_LOOP) PCO_DEFINE_CAST(pco_cf_node_as_func, pco_cf_node, pco_func, cf_node, type, PCO_CF_NODE_TYPE_FUNC) #undef PCO_DEFINE_CAST /* Iterators. */ #define pco_foreach_func_in_shader(func, shader) \ list_for_each_entry (pco_func, func, &(shader)->funcs, link) #define pco_foreach_func_in_shader_rev(func, shader) \ list_for_each_entry_rev (pco_func, func, &(shader)->funcs, link) #define pco_foreach_cf_node_in_if_prologue(cf_node, _if) \ list_for_each_entry (pco_cf_node, cf_node, &(_if)->prologue, link) #define pco_foreach_cf_node_in_if_then(cf_node, _if) \ list_for_each_entry (pco_cf_node, cf_node, &(_if)->then_body, link) #define pco_foreach_cf_node_in_if_interlogue(cf_node, _if) \ list_for_each_entry (pco_cf_node, cf_node, &(_if)->interlogue, link) #define pco_foreach_cf_node_in_if_else(cf_node, _if) \ list_for_each_entry (pco_cf_node, cf_node, &(_if)->else_body, link) #define pco_foreach_cf_node_in_if_epilogue(cf_node, _if) \ list_for_each_entry (pco_cf_node, cf_node, &(_if)->epilogue, link) #define pco_foreach_cf_node_in_loop_prologue(cf_node, loop) \ list_for_each_entry (pco_cf_node, cf_node, &(loop)->prologue, link) #define pco_foreach_cf_node_in_loop(cf_node, loop) \ list_for_each_entry (pco_cf_node, cf_node, &(loop)->body, link) #define pco_foreach_cf_node_in_loop_interlogue(cf_node, loop) \ list_for_each_entry (pco_cf_node, cf_node, &(loop)->interlogue, link) #define pco_foreach_cf_node_in_loop_epilogue(cf_node, loop) \ list_for_each_entry (pco_cf_node, cf_node, &(loop)->epilogue, link) #define pco_foreach_cf_node_in_func(cf_node, func) \ list_for_each_entry (pco_cf_node, cf_node, &(func)->body, link) #define pco_foreach_cf_node_in_func_structured(cf_node, \ cf_node_completed, \ func) \ for (pco_cf_node *cf_node = pco_cf_node_head(&(func)->body), \ *cf_node_completed = NULL; \ cf_node != NULL; \ cf_node = pco_next_cf_node(cf_node, &cf_node_completed)) #define pco_foreach_block_in_func(block, func) \ for (pco_block *block = pco_func_first_block(func); block != NULL; \ block = pco_next_block(block)) #define pco_foreach_block_in_func_from(block, from) \ for (pco_block *block = pco_next_block(from); block != NULL; \ block = pco_next_block(block)) #define pco_foreach_block_in_func_from_rev(block, from) \ for (pco_block *block = pco_prev_block(from); block != NULL; \ block = pco_prev_block(block)) #define pco_foreach_block_in_func_rev(block, func) \ for (pco_block *block = pco_func_last_block(func); block != NULL; \ block = pco_prev_block(block)) #define pco_foreach_if_in_func(pif, func) \ for (pco_if *pif = pco_func_first_if(func); pif != NULL; \ pif = pco_next_if(pif)) #define pco_foreach_ssa_if_in_func(pif, func) \ for (pco_if *pif = pco_func_first_if(func); pif != NULL; \ pif = pco_next_if(pif)) \ if (pco_ref_is_ssa(pif->cond)) #define pco_foreach_ssa_bool_if_in_func(pif, func) \ for (pco_if *pif = pco_func_first_if(func); pif != NULL; \ pif = pco_next_if(pif)) \ if (pco_ref_is_ssa(pif->cond) && pco_ref_get_bits(pif->cond) == 1) #define pco_foreach_vreg_if_in_func(pif, func) \ for (pco_if *pif = pco_func_first_if(func); pif != NULL; \ pif = pco_next_if(pif)) \ if (pco_ref_is_vreg(pif->cond)) #define pco_foreach_vreg_ssa_if_in_func(pif, func) \ for (pco_if *pif = pco_func_first_if(func); pif != NULL; \ pif = pco_next_if(pif)) \ if (pco_ref_is_vreg(pif->cond) || pco_ref_is_ssa(pif->cond)) #define pco_foreach_if_in_func_from(pif, from) \ for (pco_if *pif = pco_next_if(from); pif != NULL; pif = pco_next_if(pif)) #define pco_foreach_if_in_func_from_rev(pif, from) \ for (pco_if *pif = pco_prev_if(from); pif != NULL; pif = pco_prev_if(pif)) #define pco_foreach_if_in_func_rev(pif, func) \ for (pco_if *pif = pco_func_last_if(func); pif != NULL; \ pif = pco_prev_if(pif)) #define pco_foreach_loop_in_func(loop, func) \ for (pco_loop *loop = pco_func_first_loop(func); loop != NULL; \ loop = pco_next_loop(loop)) #define pco_foreach_loop_in_func_from(loop, from) \ for (pco_loop *loop = pco_next_loop(from); loop != NULL; \ loop = pco_next_loop(loop)) #define pco_foreach_loop_in_func_from_rev(loop, from) \ for (pco_loop *loop = pco_prev_loop(from); loop != NULL; \ loop = pco_prev_loop(loop)) #define pco_foreach_loop_in_func_rev(loop, func) \ for (pco_loop *loop = pco_func_last_loop(func); loop != NULL; \ loop = pco_prev_loop(loop)) #define pco_foreach_instr_in_block(instr, block) \ list_for_each_entry (pco_instr, instr, &(block)->instrs, link) #define pco_foreach_instr_in_block_from(instr, block, from) \ foreach_list_typed_from (pco_instr, instr, node, _, &from->node) #define pco_foreach_instr_in_block_from_rev(instr, block, from) \ foreach_list_typed_from_reverse (pco_instr, instr, node, _, &from->node) #define pco_foreach_instr_in_block_safe(instr, block) \ list_for_each_entry_safe (pco_instr, instr, &(block)->instrs, link) #define pco_foreach_instr_in_block_rev(instr, block) \ list_for_each_entry_rev (pco_instr, instr, &(block)->instrs, link) #define pco_foreach_instr_in_block_safe_rev(instr, block) \ list_for_each_entry_safe_rev (pco_instr, instr, &(block)->instrs, link) #define pco_foreach_igrp_in_block(igrp, block) \ list_for_each_entry (pco_igrp, igrp, &(block)->instrs, link) #define pco_foreach_phi_src_in_instr(phi_src, instr) \ list_for_each_entry (pco_phi_src, phi_src, &(instr)->phi_srcs, link) #define pco_foreach_instr_in_func(instr, func) \ pco_foreach_block_in_func (block, func) \ pco_foreach_instr_in_block (instr, block) #define pco_foreach_instr_in_func_from(instr, from) \ for (pco_instr *instr = pco_next_instr(from); instr != NULL; \ instr = pco_next_instr(instr)) #define pco_foreach_instr_in_func_from_rev(instr, from) \ for (pco_instr *instr = pco_prev_instr(from); instr != NULL; \ instr = pco_prev_instr(instr)) #define pco_foreach_instr_in_func_safe(instr, func) \ pco_foreach_block_in_func (block, func) \ pco_foreach_instr_in_block_safe (instr, block) #define pco_foreach_instr_in_func_rev(instr, func) \ pco_foreach_block_in_func_rev (block, func) \ pco_foreach_instr_in_block_rev (instr, block) #define pco_foreach_instr_in_func_safe_rev(instr, func) \ pco_foreach_block_in_func_rev (block, func) \ pco_foreach_instr_in_block_safe_rev (instr, block) #define pco_foreach_igrp_in_func(igrp, func) \ pco_foreach_block_in_func (block, func) \ pco_foreach_igrp_in_block (igrp, block) #define pco_foreach_phase(phase) \ for (enum pco_op_phase phase = 0; phase < _PCO_OP_PHASE_COUNT; ++phase) #define pco_foreach_phase_rev(phase) \ for (enum pco_op_phase phase = _PCO_OP_PHASE_COUNT; phase-- > 0;) #define pco_foreach_phase_from(phase, from) \ for (enum pco_op_phase phase = from + 1; phase < _PCO_OP_PHASE_COUNT; \ ++phase) #define pco_foreach_phase_from_rev(phase, from) \ for (enum pco_op_phase phase = from; phase-- > 0;) #define pco_foreach_phase_in_igrp(igrp, phase) \ pco_foreach_phase (phase) \ if (igrp->instrs[phase]) #define pco_foreach_phase_in_igrp_rev(igrp, phase) \ pco_foreach_phase_rev (phase) \ if (igrp->instrs[phase]) #define pco_foreach_phase_in_igrp_from(igrp, phase, from) \ pco_foreach_phase_from (phase, from) \ if (igrp->instrs[phase]) #define pco_foreach_phase_in_igrp_from_rev(igrp, phase, from) \ pco_foreach_phase_from_rev (phase, from) \ if (igrp->instrs[phase]) #define pco_foreach_instr_in_igrp(instr, igrp) \ for (pco_instr *instr = pco_igrp_first_instr(igrp); instr != NULL; \ instr = pco_igrp_next_instr(instr)) #define pco_foreach_instr_in_igrp_rev(instr, igrp) \ for (pco_instr *instr = pco_igrp_last_instr(igrp); instr != NULL; \ instr = pco_igrp_prev_instr(instr)) #define pco_foreach_instr_dest(pdest, instr) \ for (pco_ref *pdest = &instr->dest[0]; \ pdest < &instr->dest[instr->num_dests]; \ ++pdest) #define pco_foreach_instr_dest_from(pdest, instr, pdest_from) \ for (pco_ref *pdest = pdest_from + 1; \ pdest < &instr->dest[instr->num_dests]; \ ++pdest) #define pco_foreach_instr_dest_ssa(pdest, instr) \ pco_foreach_instr_dest (pdest, instr) \ if (pco_ref_is_ssa(*pdest)) #define pco_foreach_instr_dest_ssa_from(pdest, instr, pdest_from) \ pco_foreach_instr_dest_from (pdest, instr, pdest_from) \ if (pco_ref_is_ssa(*pdest)) #define pco_foreach_instr_dest_vreg(pdest, instr) \ pco_foreach_instr_dest (pdest, instr) \ if (pco_ref_is_vreg(*pdest)) #define pco_foreach_instr_dest_vreg_ssa(pdest, instr) \ pco_foreach_instr_dest (pdest, instr) \ if (pco_ref_is_vreg(*pdest) || pco_ref_is_ssa(*pdest)) #define pco_foreach_instr_dest_hwreg(pdest, instr) \ pco_foreach_instr_dest (pdest, instr) \ if (pco_ref_is_hwreg(*pdest)) #define pco_foreach_instr_src(psrc, instr) \ for (pco_ref *psrc = &instr->src[0]; psrc < &instr->src[instr->num_srcs]; \ ++psrc) #define pco_foreach_instr_src_from(psrc, instr, psrc_from) \ for (pco_ref *psrc = psrc_from + 1; psrc < &instr->src[instr->num_srcs]; \ ++psrc) #define pco_foreach_instr_src_ssa(psrc, instr) \ pco_foreach_instr_src (psrc, instr) \ if (pco_ref_is_ssa(*psrc)) #define pco_foreach_instr_src_ssa_from(psrc, instr, psrc_from) \ pco_foreach_instr_src_from (psrc, instr, psrc_from) \ if (pco_ref_is_ssa(*psrc)) #define pco_foreach_instr_src_vreg(psrc, instr) \ pco_foreach_instr_src (psrc, instr) \ if (pco_ref_is_vreg(*psrc)) #define pco_foreach_instr_src_vreg_ssa(psrc, instr) \ pco_foreach_instr_src (psrc, instr) \ if (pco_ref_is_vreg(*psrc) || pco_ref_is_ssa(*psrc)) #define pco_foreach_instr_src_hwreg(psrc, instr) \ pco_foreach_instr_src (psrc, instr) \ if (pco_ref_is_hwreg(*psrc)) #define pco_foreach_instr_src_vtxin_reg(psrc, instr) \ pco_foreach_instr_src (psrc, instr) \ if (pco_ref_is_vtxin(*psrc)) #define pco_cf_node_head(list) list_first_entry(list, pco_cf_node, link) #define pco_cf_node_tail(list) list_last_entry(list, pco_cf_node, link) /** * \brief Returns the preamble function of a PCO shader. * * \param[in] shader The PCO shader. * \return The preamble function, or NULL if the shader has no preamble. */ static inline pco_func *pco_preamble(pco_shader *shader) { if (list_is_empty(&shader->funcs)) return NULL; pco_func *func = list_first_entry(&shader->funcs, pco_func, link); if (func->type == PCO_FUNC_TYPE_PREAMBLE) return func; return NULL; } /** * \brief Returns the entrypoint function of a PCO shader. * * \param[in] shader The PCO shader. * \return The entrypoint function, or NULL if the shader has no entrypoint. */ static inline pco_func *pco_entrypoint(pco_shader *shader) { if (list_is_empty(&shader->funcs)) return NULL; /* Entrypoint will either be the first or second function in the shader, * depending on whether or not there is a preamble. */ pco_func *preamble = pco_preamble(shader); pco_func *func = !preamble ? list_first_entry(&shader->funcs, pco_func, link) : list_entry(preamble->link.next, pco_func, link); if (func->type == PCO_FUNC_TYPE_ENTRYPOINT) return func; return NULL; } /** * \brief Returns the variant of an instruction in an instruction group. * * \param[in] igrp The instruction group. * \param[in] phase The instruction phase. * \return The instruction variant. */ static inline unsigned pco_igrp_variant(const pco_igrp *igrp, enum pco_op_phase phase) { switch (igrp->hdr.alutype) { case PCO_ALUTYPE_MAIN: return phase == PCO_OP_PHASE_BACKEND ? igrp->variant.instr[phase].backend : igrp->variant.instr[phase].main; case PCO_ALUTYPE_BITWISE: return igrp->variant.instr[phase].bitwise; case PCO_ALUTYPE_CONTROL: return igrp->variant.instr[phase].control; default: break; } UNREACHABLE(""); } /* Motions. */ /** * \brief Returns the first CF node in a PCO function. * * \param[in] func PCO function. * \return The first CF node. */ static inline pco_cf_node *pco_first_func_cf_node(pco_func *func) { if (!list_is_empty(&func->body)) return pco_cf_node_head(&func->body); UNREACHABLE("Empty function."); } /** * \brief Returns the last CF node in a PCO function. * * \param[in] func PCO function. * \return The last CF node. */ static inline pco_cf_node *pco_last_func_cf_node(pco_func *func) { if (!list_is_empty(&func->body)) return pco_cf_node_tail(&func->body); UNREACHABLE("Empty function."); } /** * \brief Returns the first CF node in a PCO if. * * \param[in] pif PCO if. * \return The first CF node. */ static inline pco_cf_node *pco_first_if_cf_node(pco_if *pif) { if (!list_is_empty(&pif->prologue)) return pco_cf_node_head(&pif->prologue); if (!list_is_empty(&pif->then_body)) return pco_cf_node_head(&pif->then_body); if (!list_is_empty(&pif->interlogue)) return pco_cf_node_head(&pif->interlogue); if (!list_is_empty(&pif->else_body)) return pco_cf_node_head(&pif->else_body); if (!list_is_empty(&pif->epilogue)) return pco_cf_node_head(&pif->epilogue); UNREACHABLE("Empty if."); } /** * \brief Returns the last CF node in a PCO if. * * \param[in] pif PCO if. * \return The last CF node. */ static inline pco_cf_node *pco_last_if_cf_node(pco_if *pif) { if (!list_is_empty(&pif->epilogue)) return pco_cf_node_tail(&pif->epilogue); if (!list_is_empty(&pif->else_body)) return pco_cf_node_tail(&pif->else_body); if (!list_is_empty(&pif->interlogue)) return pco_cf_node_tail(&pif->interlogue); if (!list_is_empty(&pif->then_body)) return pco_cf_node_tail(&pif->then_body); if (!list_is_empty(&pif->prologue)) return pco_cf_node_tail(&pif->prologue); UNREACHABLE("Empty if."); } /** * \brief Returns the next CF node in a PCO if. * * \param[in] cf_node The current CF node within the PCO if. * \return The next CF node, or NULL if we've reached the end. */ static inline pco_cf_node *pco_next_if_cf_node(pco_cf_node *cf_node) { pco_if *pif = pco_cf_node_as_if(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_PROLOGUE: if (!list_is_empty(&pif->then_body)) return pco_cf_node_head(&pif->then_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_THEN: if (!list_is_empty(&pif->interlogue)) return pco_cf_node_head(&pif->interlogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_INTERLOGUE: if (!list_is_empty(&pif->else_body)) return pco_cf_node_head(&pif->else_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_ELSE: if (!list_is_empty(&pif->epilogue)) return pco_cf_node_head(&pif->epilogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_EPILOGUE: return NULL; default: break; } UNREACHABLE(""); } /** * \brief Returns the previous CF node in a PCO if. * * \param[in] cf_node The current CF node within the PCO if. * \return The previous CF node, or NULL if we've reached the end. */ static inline pco_cf_node *pco_prev_if_cf_node(pco_cf_node *cf_node) { pco_if *pif = pco_cf_node_as_if(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_EPILOGUE: if (!list_is_empty(&pif->else_body)) return pco_cf_node_tail(&pif->else_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_ELSE: if (!list_is_empty(&pif->interlogue)) return pco_cf_node_tail(&pif->interlogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_INTERLOGUE: if (!list_is_empty(&pif->then_body)) return pco_cf_node_tail(&pif->then_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_THEN: if (!list_is_empty(&pif->prologue)) return pco_cf_node_tail(&pif->prologue); FALLTHROUGH; case PCO_CF_NODE_FLAG_PROLOGUE: return NULL; default: break; } UNREACHABLE(""); } /** * \brief Returns the first CF node in a PCO loop. * * \param[in] loop PCO loop. * \return The first CF node. */ static inline pco_cf_node *pco_first_loop_cf_node(pco_loop *loop) { if (!list_is_empty(&loop->prologue)) return pco_cf_node_head(&loop->prologue); if (!list_is_empty(&loop->body)) return pco_cf_node_head(&loop->body); if (!list_is_empty(&loop->interlogue)) return pco_cf_node_head(&loop->interlogue); if (!list_is_empty(&loop->epilogue)) return pco_cf_node_head(&loop->epilogue); UNREACHABLE("Empty loop."); } /** * \brief Returns the last CF node in a PCO loop. * * \param[in] loop PCO loop. * \return The last CF node. */ static inline pco_cf_node *pco_last_loop_cf_node(pco_loop *loop) { if (!list_is_empty(&loop->epilogue)) return pco_cf_node_tail(&loop->epilogue); if (!list_is_empty(&loop->interlogue)) return pco_cf_node_tail(&loop->interlogue); if (!list_is_empty(&loop->body)) return pco_cf_node_tail(&loop->body); if (!list_is_empty(&loop->prologue)) return pco_cf_node_tail(&loop->prologue); UNREACHABLE("Empty loop."); } /** * \brief Returns the next CF node in a PCO loop. * * \param[in] cf_node The current CF node within the PCO loop. * \return The next CF node, or NULL if we've reached the end. */ static inline pco_cf_node *pco_next_loop_cf_node(pco_cf_node *cf_node) { pco_loop *loop = pco_cf_node_as_loop(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_PROLOGUE: if (!list_is_empty(&loop->body)) return pco_cf_node_head(&loop->body); FALLTHROUGH; case PCO_CF_NODE_FLAG_BODY: if (!list_is_empty(&loop->interlogue)) return pco_cf_node_head(&loop->interlogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_INTERLOGUE: if (!list_is_empty(&loop->epilogue)) return pco_cf_node_head(&loop->epilogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_EPILOGUE: return NULL; default: break; } UNREACHABLE(""); } /** * \brief Returns the previous CF node in a PCO loop. * * \param[in] cf_node The current CF node within the PCO loop. * \return The previous CF node, or NULL if we've reached the end. */ static inline pco_cf_node *pco_prev_loop_cf_node(pco_cf_node *cf_node) { pco_loop *loop = pco_cf_node_as_loop(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_EPILOGUE: if (!list_is_empty(&loop->interlogue)) return pco_cf_node_tail(&loop->interlogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_INTERLOGUE: if (!list_is_empty(&loop->body)) return pco_cf_node_tail(&loop->body); FALLTHROUGH; case PCO_CF_NODE_FLAG_BODY: if (!list_is_empty(&loop->prologue)) return pco_cf_node_tail(&loop->prologue); FALLTHROUGH; case PCO_CF_NODE_FLAG_PROLOGUE: return NULL; default: break; } UNREACHABLE(""); } static inline struct list_head *pco_parent_if_head(pco_cf_node *cf_node) { pco_if *parent = pco_cf_node_as_if(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_PROLOGUE: return &parent->prologue; case PCO_CF_NODE_FLAG_IF_THEN: return &parent->then_body; case PCO_CF_NODE_FLAG_INTERLOGUE: return &parent->interlogue; case PCO_CF_NODE_FLAG_IF_ELSE: return &parent->else_body; case PCO_CF_NODE_FLAG_EPILOGUE: return &parent->epilogue; default: UNREACHABLE(""); } } static inline struct list_head *pco_parent_loop_head(pco_cf_node *cf_node) { pco_loop *parent = pco_cf_node_as_loop(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_PROLOGUE: return &parent->prologue; case PCO_CF_NODE_FLAG_BODY: return &parent->body; case PCO_CF_NODE_FLAG_INTERLOGUE: return &parent->interlogue; case PCO_CF_NODE_FLAG_EPILOGUE: return &parent->epilogue; default: UNREACHABLE(""); } } static inline struct list_head *pco_parent_func_head(pco_cf_node *cf_node) { pco_func *parent = pco_cf_node_as_func(cf_node->parent); switch (cf_node->flag) { case PCO_CF_NODE_FLAG_BODY: return &parent->body; default: UNREACHABLE(""); } } static inline struct list_head *pco_parent_cf_node_head(pco_cf_node *cf_node) { pco_cf_node *parent_cf_node = cf_node->parent; switch (parent_cf_node->type) { case PCO_CF_NODE_TYPE_IF: return pco_parent_if_head(cf_node); case PCO_CF_NODE_TYPE_LOOP: return pco_parent_loop_head(cf_node); case PCO_CF_NODE_TYPE_FUNC: return pco_parent_func_head(cf_node); default: UNREACHABLE(""); } } /** * \brief Returns the next CF node. * * \param[in] cf_node The current CF node. * \param[out] cf_node_completed The last parent CF node (function, if, loop) * completed, or NULL if none/not a parent. * \return The next CF node. */ static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node, pco_cf_node **cf_node_completed) { #define pco_cf_node_next(cf_node) \ list_entry((cf_node)->link.next, pco_cf_node, link) if (cf_node_completed) *cf_node_completed = NULL; if (!cf_node) return NULL; switch (cf_node->type) { case PCO_CF_NODE_TYPE_BLOCK: { struct list_head *parent_list = pco_parent_cf_node_head(cf_node); /* Not yet reached the end of the list, return the next cf node. */ if (cf_node != pco_cf_node_tail(parent_list)) return pco_cf_node_next(cf_node); break; } case PCO_CF_NODE_TYPE_IF: return pco_first_if_cf_node(pco_cf_node_as_if(cf_node)); case PCO_CF_NODE_TYPE_LOOP: return pco_first_loop_cf_node(pco_cf_node_as_loop(cf_node)); default: UNREACHABLE(""); } /* Reached the end; go to the next cf node from the parent cf node. */ pco_cf_node *parent_cf_node = cf_node->parent; switch (parent_cf_node->type) { case PCO_CF_NODE_TYPE_IF: { pco_cf_node *next_cf_node = pco_next_if_cf_node(cf_node); if (next_cf_node) return next_cf_node; break; } case PCO_CF_NODE_TYPE_LOOP: { pco_cf_node *next_cf_node = pco_next_loop_cf_node(cf_node); if (next_cf_node) return next_cf_node; break; } /* End of the function; return NULL. */ case PCO_CF_NODE_TYPE_FUNC: if (cf_node_completed) *cf_node_completed = parent_cf_node; return NULL; default: UNREACHABLE(""); } if (cf_node_completed) *cf_node_completed = parent_cf_node; return pco_cf_node_next(parent_cf_node); #undef pco_cf_node_next } /** * \brief Returns the previous CF node. * * \param[in] cf_node The current CF node. * \param[out] cf_node_completed The last parent CF node (function, if, loop) * completed, or NULL if none/not a parent. * \return The previous CF node. */ static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node, pco_cf_node **cf_node_completed) { #define pco_cf_node_prev(cf_node) \ list_entry((cf_node)->link.prev, pco_cf_node, link) if (cf_node_completed) *cf_node_completed = NULL; if (!cf_node) return NULL; switch (cf_node->type) { case PCO_CF_NODE_TYPE_BLOCK: { struct list_head *parent_list = pco_parent_cf_node_head(cf_node); /* Not yet reached the start of the list, return the previous cf node. */ if (cf_node != pco_cf_node_head(parent_list)) return pco_cf_node_prev(cf_node); break; } case PCO_CF_NODE_TYPE_IF: return pco_last_if_cf_node(pco_cf_node_as_if(cf_node)); case PCO_CF_NODE_TYPE_LOOP: return pco_last_loop_cf_node(pco_cf_node_as_loop(cf_node)); default: UNREACHABLE(""); } /* Reached the start; go to the previous cf node from the parent cf node. */ pco_cf_node *parent_cf_node = cf_node->parent; switch (parent_cf_node->type) { case PCO_CF_NODE_TYPE_IF: { pco_cf_node *prev_cf_node = pco_prev_if_cf_node(cf_node); if (prev_cf_node) return prev_cf_node; break; } case PCO_CF_NODE_TYPE_LOOP: { pco_cf_node *prev_cf_node = pco_prev_loop_cf_node(cf_node); if (prev_cf_node) return prev_cf_node; break; } /* Start of the function; return NULL. */ case PCO_CF_NODE_TYPE_FUNC: if (cf_node_completed) *cf_node_completed = parent_cf_node; return NULL; default: UNREACHABLE(""); } if (cf_node_completed) *cf_node_completed = parent_cf_node; return pco_cf_node_prev(parent_cf_node); #undef pco_cf_node_prev } /** * \brief Returns the next CF node of the given type, if one exists. * * \param[in] cf_node The current CF node. * \param[in] type The type of the next CF node. * \return The next CF node if one exists, else NULL. */ static inline pco_cf_node *pco_next_cf_node_type(pco_cf_node *cf_node, enum pco_cf_node_type type) { do { cf_node = pco_next_cf_node(cf_node, NULL); } while (cf_node != NULL && cf_node->type != type); return cf_node; } /** * \brief Returns the first CF node of the given type, if one exists. * * \param[in] func PCO function. * \param[in] type The type of the first CF node. * \return The first CF node if one exists, else NULL. */ static inline pco_cf_node * pco_func_first_cf_node_type(pco_func *func, enum pco_cf_node_type type) { assert(!list_is_empty(&func->body)); pco_cf_node *cf_node = pco_cf_node_head(&func->body); if (cf_node->type == type) return cf_node; return pco_next_cf_node_type(cf_node, type); } /** * \brief Returns the previous CF node of the given type, if one exists. * * \param[in] cf_node The current CF node. * \param[in] type The type of the previous CF node. * \return The previous CF node if one exists, else NULL. */ static inline pco_cf_node *pco_prev_cf_node_type(pco_cf_node *cf_node, enum pco_cf_node_type type) { do { cf_node = pco_prev_cf_node(cf_node, NULL); } while (cf_node != NULL && cf_node->type != type); return cf_node; } /** * \brief Returns the last CF node of the given type, if one exists. * * \param[in] func PCO function. * \param[in] type The type of the last CF node. * \return The last CF node if one exists, else NULL. */ static inline pco_cf_node * pco_func_last_cf_node_type(pco_func *func, enum pco_cf_node_type type) { assert(!list_is_empty(&func->body)); pco_cf_node *cf_node = pco_cf_node_tail(&func->body); if (cf_node->type == type) return cf_node; return pco_prev_cf_node_type(cf_node, type); } /* CF helper iterators. */ #define PCO_DEFINE_CF_ITER(type, cf_node_type) \ static inline pco_##type *pco_next_##type(pco_##type *current_##type) \ { \ pco_cf_node *cf_node = \ pco_next_cf_node_type(¤t_##type->cf_node, cf_node_type); \ return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } \ \ static inline pco_##type *pco_func_first_##type(pco_func *func) \ { \ pco_cf_node *cf_node = pco_func_first_cf_node_type(func, cf_node_type); \ return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } \ \ static inline pco_##type *pco_prev_##type(pco_##type *current_##type) \ { \ pco_cf_node *cf_node = \ pco_prev_cf_node_type(¤t_##type->cf_node, cf_node_type); \ return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } \ \ static inline pco_##type *pco_func_last_##type(pco_func *func) \ { \ pco_cf_node *cf_node = pco_func_last_cf_node_type(func, cf_node_type); \ return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } PCO_DEFINE_CF_ITER(block, PCO_CF_NODE_TYPE_BLOCK) PCO_DEFINE_CF_ITER(if, PCO_CF_NODE_TYPE_IF) PCO_DEFINE_CF_ITER(loop, PCO_CF_NODE_TYPE_LOOP) #undef PCO_DEFINE_CF_ITER /** * \brief Returns the next non-empty block. * * \param[in] block The current block. * \return The next non-empty block, or NULL if the end of the function has been * reached. */ static inline pco_block *pco_next_block_nonempty(pco_block *block) { block = pco_next_block(block); /* Skip over empty blocks. */ while (block && list_is_empty(&block->instrs)) block = pco_next_block(block); return block; } /** * \brief Returns the previous non-empty block. * * \param[in] block The current block. * \return The previous non-empty block, or NULL if the end of the function has * been reached. */ static inline pco_block *pco_prev_block_nonempty(pco_block *block) { block = pco_prev_block(block); /* Skip over empty blocks. */ while (block && list_is_empty(&block->instrs)) block = pco_prev_block(block); return block; } /** * \brief Returns the first instruction in a block. * * \param[in] block The block. * \return The first instruction, or NULL if the block is empty. */ static inline pco_instr *pco_first_instr(pco_block *block) { if (list_is_empty(&block->instrs)) return NULL; return list_first_entry(&block->instrs, pco_instr, link); } /** * \brief Returns the last instruction in a block. * * \param[in] block The block. * \return The last instruction, or NULL if the block is empty. */ static inline pco_instr *pco_last_instr(pco_block *block) { if (list_is_empty(&block->instrs)) return NULL; return list_last_entry(&block->instrs, pco_instr, link); } /** * \brief Returns the next instruction. * * \param[in] instr The current instruction. * \return The next instruction, or NULL if the end of the function has been * reached. */ static inline pco_instr *pco_next_instr(pco_instr *instr) { if (!instr) return NULL; if (instr != pco_last_instr(instr->parent_block)) return list_entry(instr->link.next, pco_instr, link); pco_block *block = pco_next_block_nonempty(instr->parent_block); return !block ? NULL : pco_first_instr(block); } /** * \brief Returns the previous instruction. * * \param[in] instr The current instruction. * \return The previous instruction, or NULL if the start of the function has * been reached. */ static inline pco_instr *pco_prev_instr(pco_instr *instr) { if (!instr) return NULL; if (instr != pco_first_instr(instr->parent_block)) return list_entry(instr->link.prev, pco_instr, link); pco_block *block = pco_prev_block_nonempty(instr->parent_block); return !block ? NULL : pco_last_instr(block); } /** * \brief Returns the first instruction group in a block. * * \param[in] block The block. * \return The first instruction group, or NULL if the block is empty. */ static inline pco_igrp *pco_first_igrp(pco_block *block) { if (list_is_empty(&block->instrs)) return NULL; return list_first_entry(&block->instrs, pco_igrp, link); } /** * \brief Returns the last instruction group in a block. * * \param[in] block The block. * \return The last instruction group, or NULL if the block is empty. */ static inline pco_igrp *pco_last_igrp(pco_block *block) { if (list_is_empty(&block->instrs)) return NULL; return list_last_entry(&block->instrs, pco_igrp, link); } /** * \brief Returns the next instruction group. * * \param[in] igrp The current instruction group. * \return The next instruction group, or NULL if the end of the function has * been reached. */ static inline pco_igrp *pco_next_igrp(pco_igrp *igrp) { if (!igrp) return NULL; if (igrp != pco_last_igrp(igrp->parent_block)) return list_entry(igrp->link.next, pco_igrp, link); pco_block *block = pco_next_block(igrp->parent_block); return !block ? NULL : pco_first_igrp(block); } /** * \brief Returns the previous instruction group. * * \param[in] igrp The current instruction group. * \return The previous instruction group, or NULL if the start of the function * has been reached. */ static inline pco_igrp *pco_prev_igrp(pco_igrp *igrp) { if (!igrp) return NULL; if (igrp != pco_first_igrp(igrp->parent_block)) return list_entry(igrp->link.prev, pco_igrp, link); pco_block *block = pco_prev_block(igrp->parent_block); return !block ? NULL : pco_last_igrp(block); } /** * \brief Returns the first instruction in an igrp. * * \param[in] igrp The igrp. * \return The first instruction, or NULL if the igrp is empty. */ static inline pco_instr *pco_igrp_first_instr(pco_igrp *igrp) { pco_foreach_phase_in_igrp (igrp, p) return igrp->instrs[p]; return NULL; } /** * \brief Returns the last instruction in an igrp. * * \param[in] igrp The igrp. * \return The last instruction, or NULL if the igrp is empty. */ static inline pco_instr *pco_igrp_last_instr(pco_igrp *igrp) { pco_foreach_phase_in_igrp_rev (igrp, p) return igrp->instrs[p]; return NULL; } /** * \brief Returns the next instruction in an igrp. * * \param[in] instr The current instruction. * \return The next instruction, or NULL if we've reached the end of the igrp. */ static inline pco_instr *pco_igrp_next_instr(pco_instr *instr) { if (!instr) return NULL; pco_igrp *igrp = instr->parent_igrp; pco_foreach_phase_in_igrp_from (igrp, p, instr->phase) return igrp->instrs[p]; return NULL; } /** * \brief Returns the previous instruction in an igrp. * * \param[in] instr The current instruction. * \return The previous instruction, or NULL if we've reached the end of the * igrp. */ static inline pco_instr *pco_igrp_prev_instr(pco_instr *instr) { if (!instr) return NULL; pco_igrp *igrp = instr->parent_igrp; pco_foreach_phase_in_igrp_from_rev (igrp, p, instr->phase) return igrp->instrs[p]; return NULL; } /* Debug printing helpers. */ static inline bool pco_should_print_nir(nir_shader *nir) { if (!PCO_DEBUG_PRINT(NIR)) return false; if (nir->info.internal && !PCO_DEBUG_PRINT(INTERNAL)) return false; if (nir->info.stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS)) return false; else if (nir->info.stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS)) return false; else if (nir->info.stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS)) return false; return true; } static inline bool pco_should_print_shader(pco_shader *shader) { if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL)) return false; if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS)) return false; else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS)) return false; else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS)) return false; return true; } static inline bool pco_should_print_shader_pass(pco_shader *shader) { if (!PCO_DEBUG_PRINT(PASSES)) return false; if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL)) return false; if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS)) return false; else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS)) return false; else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS)) return false; return true; } static inline bool pco_should_print_binary(pco_shader *shader) { if (!PCO_DEBUG_PRINT(BINARY)) return false; if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL)) return false; if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS)) return false; else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS)) return false; else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS)) return false; return true; } static inline bool pco_should_print_stats(pco_shader *shader) { if (!PCO_DEBUG_PRINT(STATS)) return false; if (shader->is_internal && !PCO_DEBUG_PRINT(INTERNAL)) return false; if (shader->stage == MESA_SHADER_VERTEX && !PCO_DEBUG_PRINT(VS)) return false; else if (shader->stage == MESA_SHADER_FRAGMENT && !PCO_DEBUG_PRINT(FS)) return false; else if (shader->stage == MESA_SHADER_COMPUTE && !PCO_DEBUG_PRINT(CS)) return false; return true; } /* Interface with NIR. */ typedef union PACKED _pco_smp_flags { struct PACKED { unsigned dim : 2; bool proj : 1; bool fcnorm : 1; bool nncoords : 1; enum pco_lod_mode lod_mode : 2; bool pplod : 1; bool tao : 1; bool soo : 1; bool sno : 1; bool array : 1; bool integer : 1; bool wrt : 1; unsigned pad : 2; }; uint16_t _; } pco_smp_flags; static_assert(sizeof(pco_smp_flags) == sizeof(uint16_t), "sizeof(pco_smp_flags) != sizeof(uint16_t)"); /* PCO IR passes. */ bool pco_const_imms(pco_shader *shader); bool pco_bool(pco_shader *shader); bool pco_cf(pco_shader *shader); bool pco_dce(pco_shader *shader); bool pco_post_ra_legalize(pco_shader *shader); bool pco_pre_ra_legalize(pco_shader *shader); bool pco_end(pco_shader *shader); bool pco_group_instrs(pco_shader *shader); bool pco_index(pco_shader *shader, bool skip_ssa); bool pco_opt_comp_only_vecs(pco_shader *shader); bool pco_nir_compute_instance_check(nir_shader *shader); bool pco_nir_link_clip_cull_vars(nir_shader *producer, nir_shader *consumer); bool pco_nir_link_multiview(nir_shader *producer, nir_shader *consumer, pco_data *consumer_data); bool pco_nir_lower_algebraic(nir_shader *shader); bool pco_nir_lower_algebraic_late(nir_shader *shader); bool pco_nir_lower_alpha_to_coverage(nir_shader *shader); bool pco_nir_lower_atomics(nir_shader *shader, pco_data *data); bool pco_nir_lower_barriers(nir_shader *shader, pco_data *data); bool pco_nir_lower_clip_cull_vars(nir_shader *shader); bool pco_nir_lower_fs_intrinsics(nir_shader *shader); bool pco_nir_lower_vs_intrinsics(nir_shader *shader); bool pco_nir_lower_images(nir_shader *shader, pco_data *data, pco_ctx *ctx); bool pco_nir_lower_interpolation(nir_shader *shader, pco_fs_data *fs); bool pco_nir_lower_io(nir_shader *shader); bool pco_nir_lower_subgroups(nir_shader *shader); bool pco_nir_lower_tex(nir_shader *shader, pco_data *data, pco_ctx *ctx); bool pco_nir_lower_variables(nir_shader *shader, bool inputs, bool outputs); bool pco_nir_lower_vk(nir_shader *shader, pco_data *data); bool pco_nir_pfo(nir_shader *shader, pco_fs_data *fs); bool pco_nir_point_size(nir_shader *shader); bool pco_nir_pvi(nir_shader *shader, pco_vs_data *vs); bool pco_opt(pco_shader *shader); bool pco_ra(pco_shader *shader); bool pco_schedule(pco_shader *shader); bool pco_shrink_vecs(pco_shader *shader); typedef enum { pco_nir_lower_null_descriptor_ubo = (1 << 0), pco_nir_lower_null_descriptor_ssbo = (1 << 1), pco_nir_lower_null_descriptor_global = (1 << 2), pco_nir_lower_null_descriptor_texture = (1 << 3), pco_nir_lower_null_descriptor_image = (1 << 4), pco_nir_lower_null_descriptor_all = BITFIELD_MASK(5), } pco_nir_lower_null_descriptor_options; bool pco_nir_lower_null_descriptors( nir_shader *shader, pco_nir_lower_null_descriptor_options options); /** * \brief Returns the PCO bits for a bit size. * * \param[in] bits Reference. * \return PCO bits. */ static inline enum pco_bits pco_bits(unsigned bits) { switch (bits) { case 1: return PCO_BITS_1; case 8: return PCO_BITS_8; case 16: return PCO_BITS_16; case 32: return PCO_BITS_32; case 64: return PCO_BITS_64; default: break; } UNREACHABLE(""); } /* PCO ref checkers. */ /** * \brief Returns whether a reference is null. * * \param[in] ref PCO reference. * \return True if the reference is null. */ static inline bool pco_ref_is_null(pco_ref ref) { return ref.type == PCO_REF_TYPE_NULL; } /** * \brief Returns whether a reference is an SSA variable. * * \param[in] ref PCO reference. * \return True if the reference is an SSA variable. */ static inline bool pco_ref_is_ssa(pco_ref ref) { return ref.type == PCO_REF_TYPE_SSA; } /** * \brief Returns whether a reference is a virtual register. * * \param[in] ref PCO reference. * \return True if the reference is a virtual register. */ static inline bool pco_ref_is_vreg(pco_ref ref) { return ref.type == PCO_REF_TYPE_REG && ref.reg_class == PCO_REG_CLASS_VIRT; } /** * \brief Returns whether a reference is a register. * * \param[in] ref PCO reference. * \return True if the reference is a register. */ static inline bool pco_ref_is_reg(pco_ref ref) { return ref.type == PCO_REF_TYPE_REG; } /** * \brief Returns whether a reference is a hardware register. * * \param[in] ref PCO reference. * \return True if the reference is a hardware register. */ static inline bool pco_ref_is_hwreg(pco_ref ref) { return ref.type == PCO_REF_TYPE_REG && ref.reg_class != PCO_REG_CLASS_VIRT; } /** * \brief Returns whether a reference is an index register. * * \param[in] ref PCO reference. * \return True if the reference is an index register. */ static inline bool pco_ref_is_idx_reg(pco_ref ref) { return ref.type == PCO_REF_TYPE_IDX_REG; } /** * \brief Returns whether a reference is an index register pointing to itself. * * \param[in] ref PCO reference. * \return True if the reference is an index register pointing to itself. */ static inline bool pco_ref_is_self_idx_reg(pco_ref ref) { return pco_ref_is_idx_reg(ref) && ref.reg_class == PCO_REG_CLASS_INDEX; } /** * \brief Returns whether a reference is an immediate. * * \param[in] ref PCO reference. * \return True if the reference is an immediate. */ static inline bool pco_ref_is_imm(pco_ref ref) { return ref.type == PCO_REF_TYPE_IMM; } /** * \brief Returns whether a reference is I/O. * * \param[in] ref PCO reference. * \return True if the reference is I/O. */ static inline bool pco_ref_is_io(pco_ref ref) { return ref.type == PCO_REF_TYPE_IO; } /** * \brief Returns whether a reference is a predicate. * * \param[in] ref PCO reference. * \return True if the reference is a predicate. */ static inline bool pco_ref_is_pred(pco_ref ref) { return ref.type == PCO_REF_TYPE_PRED; } /** * \brief Returns whether a reference is a dependant read counter. * * \param[in] ref PCO reference. * \return True if the reference is a dependant read counter. */ static inline bool pco_ref_is_drc(pco_ref ref) { return ref.type == PCO_REF_TYPE_DRC; } /** * \brief Returns whether a reference is scalar. * * \param[in] ref PCO reference. * \return True if the reference is scalar. */ static inline bool pco_ref_is_scalar(pco_ref ref) { return !ref.chans; } /** * \brief Return whether a reference is a vertex input register. * * \param[in] ref PCO reference. * \return True if the reference is a vertex input register. */ static inline bool pco_ref_is_vtxin(pco_ref ref) { return ref.type == PCO_REF_TYPE_REG && ref.reg_class == PCO_REG_CLASS_VTXIN; } /* PCO ref getters. */ /** * \brief Returns the pointee component of an indexed register reference. * * \param[in] ref Indexed register reference. * \return Pointee component of the indexed register reference. */ static inline pco_ref pco_ref_get_idx_pointee(pco_ref ref) { assert(pco_ref_is_idx_reg(ref)); pco_ref pointee = ref; pointee.val = ref.idx_reg.offset; pointee.type = PCO_REF_TYPE_REG; return pointee; } /** * \brief Returns the data type of a reference. * * \param[in] ref Reference. * \return Datatype. */ static inline enum pco_dtype pco_ref_get_dtype(pco_ref ref) { return ref.dtype; } /** * \brief Returns the number of channels for a reference type. * * \param[in] ref Reference. * \return Number of channels. */ static inline unsigned pco_ref_get_chans(pco_ref ref) { return ref.chans + 1; } /** * \brief Returns the number of bits for a reference type. * * \param[in] ref Reference. * \return Number of bits. */ static inline unsigned pco_ref_get_bits(pco_ref ref) { switch (ref.bits) { case PCO_BITS_1: return 1; case PCO_BITS_8: return 8; case PCO_BITS_16: return 16; case PCO_BITS_32: return 32; case PCO_BITS_64: return 64; default: break; } UNREACHABLE(""); } /** * \brief Returns the bit-sized value in an immediate reference. * * \param[in] ref Reference. * \return Immediate value. */ static inline uint64_t pco_ref_get_imm(pco_ref ref) { assert(pco_ref_is_imm(ref)); unsigned num_bits = pco_ref_get_bits(ref); switch (ref.dtype) { case PCO_DTYPE_FLOAT: assert(num_bits == 32); FALLTHROUGH; case PCO_DTYPE_ANY: FALLTHROUGH; case PCO_DTYPE_UNSIGNED: return ref.val & BITFIELD_MASK(num_bits); case PCO_DTYPE_SIGNED: return util_sign_extend(ref.val, num_bits); default: break; } UNREACHABLE(""); } /** * \brief Returns the register class of a reference type. * * \param[in] ref Reference. * \return Register class. */ static inline enum pco_reg_class pco_ref_get_reg_class(pco_ref ref) { assert(pco_ref_is_reg(ref) || pco_ref_is_idx_reg(ref)); return ref.reg_class; } /** * \brief Returns the register index of a reference type. * * \param[in] ref Reference. * \return Register index. */ static inline unsigned pco_ref_get_reg_index(pco_ref ref) { assert(pco_ref_is_reg(ref) || pco_ref_is_idx_reg(ref)); unsigned index = pco_ref_is_idx_reg(ref) ? ref.idx_reg.offset : ref.val; return index; } /** * \brief Returns the register index control of a reference type. * * \param[in] ref Reference. * \return Register index control. */ static inline enum pco_idx_ctrl pco_ref_get_reg_idx_ctrl(pco_ref ref) { assert(pco_ref_is_reg(ref) || pco_ref_is_idx_reg(ref)); if (pco_ref_is_reg(ref)) return PCO_IDX_CTRL_NONE; return PCO_IDX_CTRL_IDX0 + ref.idx_reg.num; } /** * \brief Returns the temp register index from its reference type. * * \param[in] ref Reference. * \return Temp index. */ static inline unsigned pco_ref_get_temp(pco_ref ref) { assert(pco_ref_is_reg(ref)); assert(pco_ref_get_reg_class(ref) == PCO_REG_CLASS_TEMP); return pco_ref_get_reg_index(ref); } /** * \brief Returns the coefficient register index from its reference type. * * \param[in] ref Reference. * \return Coefficient index. */ static inline unsigned pco_ref_get_coeff(pco_ref ref) { assert(pco_ref_is_reg(ref)); assert(pco_ref_get_reg_class(ref) == PCO_REG_CLASS_COEFF); return pco_ref_get_reg_index(ref); } /** * \brief Returns the I/O from its reference type. * * \param[in] ref Reference. * \return I/O. */ static inline enum pco_io pco_ref_get_io(pco_ref ref) { assert(pco_ref_is_io(ref)); assert(ref.val < _PCO_IO_COUNT); return ref.val; } /** * \brief Returns the movw01 value of an I/O reference type. * * \param[in] ref Reference. * \return movw01 value. */ static inline enum pco_movw01 pco_ref_get_movw01(pco_ref ref) { /* Default/unused case. */ if (pco_ref_is_null(ref)) return PCO_MOVW01_FT0; switch (pco_ref_get_io(ref)) { case PCO_IO_FT0: return PCO_MOVW01_FT0; case PCO_IO_FT1: return PCO_MOVW01_FT1; case PCO_IO_FT2: return PCO_MOVW01_FT2; case PCO_IO_FTE: return PCO_MOVW01_FTE; default: break; } UNREACHABLE(""); } /** * \brief Returns the predicate from its reference type. * * \param[in] ref Reference. * \return Predicate. */ static inline enum pco_pred pco_ref_get_pred(pco_ref ref) { assert(pco_ref_is_pred(ref)); assert(ref.val < _PCO_PRED_COUNT); return ref.val; } /** * \brief Returns the dependent read counter from its reference type. * * \param[in] ref Reference. * \return Dependent read counter. */ static inline enum pco_drc pco_ref_get_drc(pco_ref ref) { assert(pco_ref_is_drc(ref)); assert(ref.val < _PCO_DRC_COUNT); return ref.val; } /** * \brief Returns whether the reference has any mods set. * * \param[in] ref Reference. * \return True if any mods are set. */ static inline bool pco_ref_has_mods_set(pco_ref ref) { return ref.oneminus || ref.clamp || ref.abs || ref.neg || ref.flr || (ref.elem != 0); } /** * \brief Returns whether a reference is a temporary register. * * \param[in] ref PCO reference. * \return True if the reference is a temporary register. */ static inline bool pco_ref_is_temp(pco_ref ref) { return pco_ref_is_reg(ref) && pco_ref_get_reg_class(ref) == PCO_REG_CLASS_TEMP; } /* PCO ref builders. */ /** * \brief Builds and returns a null reference. * * \return NULL reference. */ static inline pco_ref pco_ref_null(void) { return (pco_ref){ .type = PCO_REF_TYPE_NULL, }; } /** * \brief Builds and returns an SSA reference. * * \return SSA reference. */ static inline pco_ref pco_ref_ssa(unsigned index, unsigned bits, unsigned chans) { return (pco_ref){ .val = index, .chans = chans - 1, .bits = pco_bits(bits), .type = PCO_REF_TYPE_SSA, }; } /** * \brief Builds and returns a new SSA reference. * * \param[in,out] func The function. * \param[in] bits Number of bits. * \param[in] chans Number of channels. * \return SSA reference. */ static inline pco_ref pco_ref_new_ssa(pco_func *func, unsigned bits, unsigned chans) { return pco_ref_ssa(func->next_ssa++, bits, chans); } /** * \brief Builds and returns a new SSA reference cloning the properties of * another. * * \param[in,out] func The function. * \param [in] ref The base reference. * \return SSA reference. */ static inline pco_ref pco_ref_new_ssa_clone(pco_func *func, pco_ref ref) { return pco_ref_ssa(func->next_ssa++, pco_ref_get_bits(ref), pco_ref_get_chans(ref)); } /** * \brief Builds and returns a new 32x1 SSA reference. * * \param[in,out] func The function. * \return SSA reference. */ static inline pco_ref pco_ref_new_ssa32(pco_func *func) { return pco_ref_new_ssa(func, 32, 1); } /** * \brief Builds and returns a new 32x2 SSA address reference. * * \param[in,out] func The function. * \return SSA address reference. */ static inline pco_ref pco_ref_new_ssa_addr(pco_func *func) { return pco_ref_new_ssa(func, 32, 2); } /** * \brief Builds new 32x1[2] SSA address component references. * * \param[in,out] func The function. * \param[out] addr_comps The new address component references. */ static inline void pco_ref_new_ssa_addr_comps(pco_func *func, pco_ref addr_comps[static 2]) { addr_comps[0] = pco_ref_new_ssa32(func); addr_comps[1] = pco_ref_new_ssa32(func); } /** * \brief Builds and returns a new 32x(2+n) SSA address and data reference. * * \param[in,out] func The function. * \param[in] data_size The size of the data. * \return SSA address and data reference. */ static inline pco_ref pco_ref_new_ssa_addr_data(pco_func *func, unsigned data_size) { return pco_ref_new_ssa(func, 32, 2 + data_size); } /** * \brief Builds and returns a virtual register reference. * * \param[in] index Virtual register index. * \return Virtual register reference. */ static inline pco_ref pco_ref_vreg(unsigned index) { return (pco_ref){ .val = index, .bits = PCO_BITS_32, .type = PCO_REF_TYPE_REG, .reg_class = PCO_REG_CLASS_VIRT, }; } /** * \brief Builds and returns a new virtual register. * * \param[in,out] func The function. * \return Virtual register reference. */ static inline pco_ref pco_ref_new_vreg(pco_func *func) { return pco_ref_vreg(func->next_vreg++); } /** * \brief Remaps an SSA register to an existing virtual register. * * \param[in,out] func The function. * \param[in] ref Base SSA reference. * \param[in] bits New bit width, or 0 if unchanged. * \return Virtual register reference. */ static inline pco_ref pco_ref_ssa_vreg(ASSERTED pco_func *func, pco_ref ref, unsigned bits) { assert(pco_ref_is_ssa(ref)); assert(ref.val < func->next_vreg); ref.type = PCO_REF_TYPE_REG; ref.reg_class = PCO_REG_CLASS_VIRT; if (bits) ref.bits = pco_bits(bits); return ref; } /** * \brief Builds and returns a vector hardware register reference. * * \param[in] index Register index. * \param[in] reg_class Register class. * \param[in] chans Number of channels. * \return Hardware register reference. */ static inline pco_ref pco_ref_hwreg_vec(unsigned index, enum pco_reg_class reg_class, unsigned chans) { assert(reg_class != PCO_REG_CLASS_VIRT); return (pco_ref){ .val = index, .chans = chans - 1, .bits = PCO_BITS_32, .type = PCO_REF_TYPE_REG, .reg_class = reg_class, }; } /** * \brief Builds and returns a scalar hardware register reference. * * \param[in] index Register index. * \param[in] reg_class Register class. * \return Hardware register reference. */ static inline pco_ref pco_ref_hwreg(unsigned index, enum pco_reg_class reg_class) { return pco_ref_hwreg_vec(index, reg_class, 1); } /** * \brief Builds 32x1[2] hardware register address component references. * * \param[in] index Register index. * \param[in] reg_class Register class. * \param[out] addr_comps The new address component references. */ static inline void pco_ref_hwreg_addr_comps(unsigned index, enum pco_reg_class reg_class, pco_ref addr_comps[static 2]) { addr_comps[0] = pco_ref_hwreg(index, reg_class); addr_comps[1] = pco_ref_hwreg(index + 1, reg_class); } static inline bool idx_reg_pointee_is_valid(pco_ref ref) { assert(pco_ref_is_idx_reg(ref)); switch (pco_ref_get_reg_class(ref)) { case PCO_REG_CLASS_TEMP: case PCO_REG_CLASS_VTXIN: case PCO_REG_CLASS_COEFF: case PCO_REG_CLASS_SHARED: case PCO_REG_CLASS_INDEX: case PCO_REG_CLASS_PIXOUT: return true; default: break; } return false; } /** * \brief Builds and returns an indexed vector hardware register reference. * * \param[in] num Index register number. * \param[in] offset Pointee offset. * \param[in] reg_class Register class. * \param[in] chans Number of channels. * \return Hardware register reference. */ static inline pco_ref pco_ref_hwreg_idx_vec(unsigned num, unsigned offset, enum pco_reg_class reg_class, unsigned chans) { assert(reg_class != PCO_REG_CLASS_VIRT); return (pco_ref){ .idx_reg = { .num = num, .offset = offset, }, .chans = chans - 1, .bits = PCO_BITS_32, .type = PCO_REF_TYPE_IDX_REG, .reg_class = reg_class, }; } /** * \brief Builds and returns an indexed scalar hardware register reference. * * \param[in] num Index register number. * \param[in] offset Pointee offset. * \param[in] reg_class Register class. * \return Hardware register reference. */ static inline pco_ref pco_ref_hwreg_idx(unsigned num, unsigned offset, enum pco_reg_class reg_class) { return pco_ref_hwreg_idx_vec(num, offset, reg_class, 1); } /** * \brief Builds and returns an indexed hardware register reference using an * existing hardware register reference. * * \param[in] ref Base reference * \param[in] reg_class Register class. * \param[in] chans Number of channels. * \return Hardware register reference. */ static inline pco_ref pco_ref_hwreg_idx_from(unsigned num, pco_ref ref) { assert(pco_ref_is_reg(ref)); pco_ref idx_ref = { .idx_reg = { .num = num, .offset = ref.val, }, .chans = ref.chans, .bits = ref.bits, .type = PCO_REF_TYPE_IDX_REG, .reg_class = ref.reg_class, }; assert(idx_reg_pointee_is_valid(idx_ref)); return idx_ref; } /** * \brief Builds and returns an immediate reference. * * \param[in] val Immediate value. * \param[in] bits Immediate bit size. * \param[in] dtype Immediate datatype. * \return Immediate reference. */ static inline pco_ref pco_ref_imm(uint32_t val, enum pco_bits bits, enum pco_dtype dtype) { return (pco_ref){ .val = val, .dtype = dtype, .bits = bits, .type = PCO_REF_TYPE_IMM, }; } /** * \brief Builds and returns an 8-bit immediate reference. * * \param[in] val 8-bit immediate. * \return Immediate reference. */ static inline pco_ref pco_ref_imm8(uint8_t val) { return pco_ref_imm(val, PCO_BITS_8, PCO_DTYPE_UNSIGNED); } /** * \brief Builds and returns a 16-bit immediate reference. * * \param[in] val 16-bit immediate. * \return Immediate reference. */ static inline pco_ref pco_ref_imm16(uint16_t val) { return pco_ref_imm(val, PCO_BITS_16, PCO_DTYPE_UNSIGNED); } /** * \brief Builds and returns a 32-bit immediate reference. * * \param[in] val 32-bit immediate. * \return Immediate reference. */ static inline pco_ref pco_ref_imm32(uint32_t val) { return pco_ref_imm(val, PCO_BITS_32, PCO_DTYPE_UNSIGNED); } /** * \brief Builds and returns an untyped 8-bit immediate reference. * * \param[in] val 8-bit immediate. * \return Immediate reference. */ static inline pco_ref pco_ref_val8(uint8_t val) { return pco_ref_imm(val, PCO_BITS_8, PCO_DTYPE_ANY); } /** * \brief Builds and returns an untyped 16-bit immediate reference. * * \param[in] val 16-bit immediate. * \return Immediate reference. */ static inline pco_ref pco_ref_val16(uint16_t val) { return pco_ref_imm(val, PCO_BITS_16, PCO_DTYPE_ANY); } /** * \brief Builds and returns an untyped 32-bit immediate reference. * * \param[in] val 32-bit immediate. * \return Immediate reference. */ static inline pco_ref pco_ref_val32(uint32_t val) { return pco_ref_imm(val, PCO_BITS_32, PCO_DTYPE_UNSIGNED); } /** * \brief Builds and returns an I/O reference. * * \param[in] io I/O. * \return I/O reference. */ static inline pco_ref pco_ref_io(enum pco_io io) { return (pco_ref){ .val = io, .type = PCO_REF_TYPE_IO, }; } /** * \brief Builds and returns a predicate reference. * * \param[in] pred predicate. * \return Predicate reference. */ static inline pco_ref pco_ref_pred(enum pco_pred pred) { return (pco_ref){ .val = pred, .type = PCO_REF_TYPE_PRED, }; } /** * \brief Builds and returns a dependent read counter reference. * * \param[in] drc Dependent read counter. * \return Dependent read counter reference. */ static inline pco_ref pco_ref_drc(enum pco_drc drc) { return (pco_ref){ .val = drc, .type = PCO_REF_TYPE_DRC, }; } /* PCO ref utils. */ /** * \brief Updates a reference to reset its mods. * * \param[in] ref Base reference. * \return Updated reference. */ static inline pco_ref pco_ref_reset_mods(pco_ref ref) { ref.oneminus = false; ref.clamp = false; ref.flr = false; ref.abs = false; ref.neg = false; ref.elem = 0; return ref; } /** * \brief Transfers reference mods, optionally resetting them. * * \param[in,out] dest Reference to transfer mods to. * \param[in,out] source Reference to transfer mods from. * \param[in] reset Whether to reset the source mods. */ static inline void pco_ref_xfer_mods(pco_ref *dest, pco_ref *source, bool reset) { dest->oneminus = source->oneminus; dest->clamp = source->clamp; dest->flr = source->flr; dest->abs = source->abs; dest->neg = source->neg; dest->elem = source->elem; if (reset) *source = pco_ref_reset_mods(*source); } /** * \brief Updates a reference to set the oneminus modifier. * * \param[in] ref Base reference. * \return Updated reference. */ static inline pco_ref pco_ref_oneminus(pco_ref ref) { ref.oneminus = true; return ref; } /** * \brief Updates a reference to set the clamp modifier. * * \param[in] ref Base reference. * \return Updated reference. */ static inline pco_ref pco_ref_clamp(pco_ref ref) { ref.clamp = true; return ref; } /** * \brief Updates a reference to set the floor modifier. * * \param[in] ref Base reference. * \return Updated reference. */ static inline pco_ref pco_ref_flr(pco_ref ref) { ref.flr = true; ref.abs = false; ref.neg = false; return ref; } /** * \brief Updates a reference to set the abs modifier. * * \param[in] ref Base reference. * \return Updated reference. */ static inline pco_ref pco_ref_abs(pco_ref ref) { ref.abs = true; ref.neg = false; return ref; } /** * \brief Updates a reference to set the negate modifier. * * \param[in] ref Base reference. * \return Updated reference. */ static inline pco_ref pco_ref_neg(pco_ref ref) { ref.neg = !ref.neg; return ref; } /** * \brief Updates a reference to set the element modifier. * * \param[in] ref Base reference. * \param[in] elem New element modifier. * \return Updated reference. */ static inline pco_ref pco_ref_elem(pco_ref ref, enum pco_elem elem) { ref.elem = elem; return ref; } /** * \brief Updates a reference to set the number of channels. * * \param[in] ref Base reference. * \param[in] chans New number of channels. * \return Updated reference. */ static inline pco_ref pco_ref_chans(pco_ref ref, unsigned chans) { ref.chans = chans - 1; return ref; } /** * \brief Updates a reference to set the bit width. * * \param[in] ref Base reference. * \param[in] bits New bit width. * \return Updated reference. */ static inline pco_ref pco_ref_bits(pco_ref ref, unsigned bits) { ref.bits = pco_bits(bits); return ref; } /** * \brief Updates a reference value with the provided offset. * * \param[in] ref Base reference. * \param[in] offset Offset to apply. * \return Updated reference. */ static inline pco_ref pco_ref_offset(pco_ref ref, signed offset) { int64_t val = pco_ref_is_idx_reg(ref) ? ref.idx_reg.offset : ref.val; val += offset; if (pco_ref_is_idx_reg(ref)) { assert(util_last_bit64(val) <= PCO_REF_IDX_OFFSET_BITS); ref.idx_reg.offset = val; } else { assert(util_last_bit64(val) <= PCO_REF_VAL_BITS); ref.val = val; } return ref; } /** * \brief Checks whether two reference modifiers are the same. * * \param[in] ref0 First reference. * \param[in] ref1 Second reference. * \return True if both reference modifiers are the same. */ static inline bool pco_ref_mods_are_equal(pco_ref ref0, pco_ref ref1) { return (ref0.oneminus == ref1.oneminus) && (ref0.clamp == ref1.clamp) && (ref0.flr == ref1.flr) && (ref0.abs == ref1.abs) && (ref0.neg == ref1.neg) && (ref0.elem == ref1.elem); } /** * \brief Checks whether two references are the same. * * \param[in] ref0 First reference. * \param[in] ref1 Second reference. * \param[in] ignore_dtype Whether to ignore the datatype. * \return True if both references are the same. */ /* TODO: can this be simplified? */ static inline bool pco_refs_are_equal(pco_ref ref0, pco_ref ref1, bool ignore_dtype) { if (ref0.type != ref1.type) return false; if (pco_ref_is_idx_reg(ref0)) { if ((ref0.idx_reg.num != ref1.idx_reg.num) || (ref0.idx_reg.offset != ref1.idx_reg.offset)) { return false; } } else if (ref0.val != ref1.val) { return false; } if (pco_ref_is_idx_reg(ref0) || pco_ref_is_reg(ref0)) if (ref0.reg_class != ref1.reg_class) return false; if (!pco_ref_mods_are_equal(ref0, ref1)) return false; if (ref0.chans != ref1.chans) return false; if (!ignore_dtype && pco_ref_get_dtype(ref0) != pco_ref_get_dtype(ref1)) return false; if (pco_ref_get_bits(ref0) != pco_ref_get_bits(ref1)) return false; return true; } /** * \brief Checks a reference has a valid hardware source mapping. * * \param[in] ref Reference. * \param[in] mapped_src Hardware source mapping. * \param[out] needs_s124 Whether the mapping needs to use S{1,2,4} * rather than S{0,2,3}. * \return True if the mapping is valid. */ static inline bool ref_src_map_valid(pco_ref ref, enum pco_io mapped_src, bool *needs_s124) { if (needs_s124) *needs_s124 = false; /* Restrictions only apply to hardware registers. */ if (!pco_ref_is_idx_reg(ref) && !pco_ref_is_reg(ref)) return true; if (pco_ref_is_idx_reg(ref)) { return (mapped_src == PCO_IO_S0) || (mapped_src == PCO_IO_S2) || (mapped_src == PCO_IO_S3); } switch (pco_ref_get_reg_class(ref)) { case PCO_REG_CLASS_COEFF: case PCO_REG_CLASS_SHARED: case PCO_REG_CLASS_INDEX: case PCO_REG_CLASS_PIXOUT: return (mapped_src == PCO_IO_S0) || (mapped_src == PCO_IO_S2) || (mapped_src == PCO_IO_S3); case PCO_REG_CLASS_SPEC: if (needs_s124) *needs_s124 = true; return (mapped_src == PCO_IO_S1) || (mapped_src == PCO_IO_S2) || (mapped_src == PCO_IO_S4); default: return true; } return false; } static inline enum pco_srcsel pco_ref_srcsel(pco_ref ref) { enum pco_io io = pco_ref_get_io(ref); switch (io) { case PCO_IO_S0: return PCO_SRCSEL_S0; case PCO_IO_S1: return PCO_SRCSEL_S1; case PCO_IO_S2: return PCO_SRCSEL_S2; case PCO_IO_S3: return PCO_SRCSEL_S3; case PCO_IO_S4: return PCO_SRCSEL_S4; case PCO_IO_S5: return PCO_SRCSEL_S5; default: break; } UNREACHABLE(""); } static inline enum pco_count_src pco_ref_count_src(pco_ref ref) { enum pco_io io = pco_ref_get_io(ref); switch (io) { case PCO_IO_S2: return PCO_COUNT_SRC_S2; case PCO_IO_FT2: return PCO_COUNT_SRC_FT2; default: break; } UNREACHABLE(""); } /** * \brief Returns whether none of the lower/upper sources in an instruction * group are set. * * \param[in] igrp The instruction group. * \param[in] upper True if checking the upper sources. * \return True if none of the lower/upper sources are set. */ static inline bool pco_igrp_srcs_unset(pco_igrp *igrp, bool upper) { unsigned offset = upper ? ROGUE_ALU_INPUT_GROUP_SIZE : 0; for (unsigned u = 0; u < ROGUE_ALU_INPUT_GROUP_SIZE; ++u) if (!pco_ref_is_null(igrp->srcs.s[u + offset])) return false; return true; } /** * \brief Returns whether none of the internal source selectors in an * instruction group are set. * * \param[in] igrp The instruction group. * \return True if none of the internal source selectors are set. */ static inline bool pco_igrp_iss_unset(pco_igrp *igrp) { for (unsigned u = 0; u < ARRAY_SIZE(igrp->iss.is); ++u) if (!pco_ref_is_null(igrp->iss.is[u])) return false; return true; } /** * \brief Returns whether none of the destinations in an instruction group are * set. * * \param[in] igrp The instruction group. * \return True if none of the destinations are set. */ static inline bool pco_igrp_dests_unset(pco_igrp *igrp) { for (unsigned u = 0; u < ARRAY_SIZE(igrp->dests.w); ++u) if (!pco_ref_is_null(igrp->dests.w[u])) return false; return true; } /** * \brief Iterates backwards to find the parent instruction of a source. * * \param[in] src The source whose parent is to be found. * \param[in] from The instruction to start iterating back from (inclusive). * \return The parent instruction if found, else NULL. */ static inline pco_instr *find_parent_instr_from(pco_ref src, pco_instr *from) { pco_foreach_instr_dest_ssa (pdest, from) { if (pco_refs_are_equal(*pdest, src, false)) return from; } pco_foreach_instr_in_func_from_rev (instr, from) { pco_foreach_instr_dest_ssa (pdest, instr) { if (pco_refs_are_equal(*pdest, src, false)) return instr; } } return NULL; } static inline unsigned pco_igrp_offset(pco_igrp *igrp) { return igrp->enc.offset; } static inline unsigned pco_cf_node_offset(pco_cf_node *cf_node) { pco_block *block = pco_cf_node_as_block(cf_node); pco_igrp *igrp = pco_first_igrp(block); if (!igrp) { block = pco_next_block_nonempty(block); igrp = pco_first_igrp(block); } return pco_igrp_offset(igrp); } static inline unsigned pco_branch_rel_offset(pco_igrp *br, pco_cf_node *cf_node) { return pco_cf_node_offset(cf_node) - pco_igrp_offset(br); } static inline unsigned pco_branch_rel_offset_next_igrp(pco_igrp *br) { pco_igrp *next_igrp = pco_next_igrp(br); assert(next_igrp); return pco_igrp_offset(next_igrp) - pco_igrp_offset(br); } static inline bool pco_should_skip_pass(const char *pass) { return comma_separated_list_contains(pco_skip_passes, pass); } #define PCO_PASS(progress, shader, pass, ...) \ do { \ if (pco_should_skip_pass(#pass)) { \ fprintf(stdout, "Skipping pass '%s'\n", #pass); \ break; \ } \ \ if (pass(shader, ##__VA_ARGS__)) { \ UNUSED bool _; \ progress = true; \ \ if (PCO_DEBUG(REINDEX)) \ pco_index(shader, false); \ \ pco_validate_shader(shader, "after " #pass); \ \ if (pco_should_print_shader_pass(shader)) \ pco_print_shader(shader, stdout, "after " #pass); \ } \ } while (0) /* Common hw constants/references. */ /** Integer/float zero. */ #define pco_zero pco_ref_hwreg(0, PCO_REG_CLASS_CONST) /** Integer one. */ #define pco_one pco_ref_hwreg(1, PCO_REG_CLASS_CONST) /** Integer 2. */ #define pco_2 pco_ref_hwreg(2, PCO_REG_CLASS_CONST) /** Integer 4. */ #define pco_4 pco_ref_hwreg(4, PCO_REG_CLASS_CONST) /** Integer 5. */ #define pco_5 pco_ref_hwreg(5, PCO_REG_CLASS_CONST) /** Integer 7. */ #define pco_7 pco_ref_hwreg(7, PCO_REG_CLASS_CONST) /** Integer 31. */ #define pco_31 pco_ref_hwreg(31, PCO_REG_CLASS_CONST) /** Integer -1/true/0xffffffff/u32max. */ #define pco_true pco_ref_hwreg(143, PCO_REG_CLASS_CONST) #define pco_u32max pco_true /** Float 1. */ #define pco_fone pco_ref_hwreg(64, PCO_REG_CLASS_CONST) /** Float -1. */ #define pco_fnegone pco_ref_neg(pco_ref_hwreg(64, PCO_REG_CLASS_CONST)) /** Float infinity. */ #define pco_finf pco_ref_hwreg(142, PCO_REG_CLASS_CONST) #define pco_p0 pco_ref_pred(PCO_PRED_P0) /* Printing. */ void pco_print_ref(pco_shader *shader, pco_ref ref); void pco_print_instr(pco_shader *shader, pco_instr *instr); void pco_print_igrp(pco_shader *shader, pco_igrp *igrp); void pco_print_cf_node_name(pco_shader *shader, pco_cf_node *cf_node); void pco_print_shader_info(pco_shader *shader); void pco_print_phase(pco_shader *shader, enum pco_alutype alutype, enum pco_op_phase phase); /** * \brief Packs a descriptor set and binding into a uint32_t. * * \param[in] desc_set The descriptor set. * \param[in] binding The binding. * \return The packed descriptor set and binding. */ static inline uint32_t pco_pack_desc(unsigned desc_set, unsigned binding) { assert(desc_set <= UINT16_MAX); assert(binding <= UINT16_MAX); return desc_set | binding << 16; } /** * \brief Unpacks a descriptor set and binding from a uint32_t. * * \param[in] packed The packed descriptor set and binding. * \param[out] desc_set The descriptor set. * \param[out] binding The binding. */ static inline void pco_unpack_desc(uint32_t packed, unsigned *desc_set, unsigned *binding) { assert(desc_set); assert(binding); *desc_set = packed & 0xffff; *binding = packed >> 16; } #endif /* PCO_INTERNAL_H */