/* * Copyright © 2018-2019 Igalia S.L. * SPDX-License-Identifier: MIT */ #include "compiler/nir/nir_builder.h" #include "ir3_nir.h" /** * This pass moves to NIR certain offset computations for different I/O * ops that are currently implemented on the IR3 backend compiler, to * give NIR a chance to optimize them: * * - Dword-offset for SSBO load, store and atomics: A new, similar intrinsic * is emitted that replaces the original one, adding a new source that * holds the result of the original byte-offset source divided by 4. */ /* Returns the ir3-specific intrinsic opcode corresponding to an SSBO * instruction that is handled by this pass. It also conveniently returns * the offset source index in @offset_src_idx. * * If @intrinsic is not SSBO, or it is not handled by the pass, -1 is * returned. */ static int get_ir3_intrinsic_for_ssbo_intrinsic(unsigned intrinsic, uint8_t *offset_src_idx) { assert(offset_src_idx); *offset_src_idx = 1; switch (intrinsic) { case nir_intrinsic_store_ssbo: *offset_src_idx = 2; return nir_intrinsic_store_ssbo_ir3; case nir_intrinsic_load_ssbo: return nir_intrinsic_load_ssbo_ir3; case nir_intrinsic_ssbo_atomic: return nir_intrinsic_ssbo_atomic_ir3; case nir_intrinsic_ssbo_atomic_swap: return nir_intrinsic_ssbo_atomic_swap_ir3; default: break; } return -1; } static nir_def * check_and_propagate_bit_shift32(nir_builder *b, nir_alu_instr *alu_instr, int32_t direction, int32_t shift) { nir_def *shift_ssa = alu_instr->src[1].src.ssa; /* Only propagate if the shift is a const value so we can check value range * statically. */ nir_const_value *const_val = nir_src_as_const_value(alu_instr->src[1].src); if (!const_val) return NULL; int32_t current_shift = const_val[0].i32 * direction; int32_t new_shift = current_shift + shift; /* If the merge would reverse the direction, bail out. * e.g, 'x << 2' then 'x >> 4' is not 'x >> 2'. */ if (current_shift * new_shift < 0) return NULL; /* If the propagation would overflow an int32_t, bail out too to be on the * safe side. */ if (new_shift < -31 || new_shift > 31) return NULL; /* Add or substract shift depending on the final direction (SHR vs. SHL). */ if (shift * direction < 0) shift_ssa = nir_iadd_imm(b, shift_ssa, -abs(shift)); else shift_ssa = nir_iadd_imm(b, shift_ssa, abs(shift)); return shift_ssa; } nir_def * ir3_nir_try_propagate_bit_shift(nir_builder *b, nir_def *offset, int32_t shift) { nir_instr *offset_instr = nir_def_instr(offset); if (offset_instr->type != nir_instr_type_alu) return NULL; nir_alu_instr *alu = nir_instr_as_alu(offset_instr); nir_def *shift_ssa; nir_def *new_offset = NULL; /* the first src could be something like ssa_18.x, but we only want * the single component. Otherwise the ishl/ishr/ushr could turn * into a vec4 operation: */ nir_def *src0 = nir_mov_alu(b, alu->src[0], 1); switch (alu->op) { case nir_op_ishl: shift_ssa = check_and_propagate_bit_shift32(b, alu, 1, shift); if (shift_ssa) new_offset = nir_ishl(b, src0, shift_ssa); break; case nir_op_ishr: shift_ssa = check_and_propagate_bit_shift32(b, alu, -1, shift); if (shift_ssa) new_offset = nir_ishr(b, src0, shift_ssa); break; case nir_op_ushr: shift_ssa = check_and_propagate_bit_shift32(b, alu, -1, shift); if (shift_ssa) new_offset = nir_ushr(b, src0, shift_ssa); break; default: return NULL; } return new_offset; } /* isam doesn't have an "untyped" field, so it can only load 1 component at a * time because our storage buffer descriptors use a 1-component format. * Therefore we need to scalarize any loads that would use isam. */ static void scalarize_load(nir_intrinsic_instr *intrinsic, nir_builder *b) { struct nir_def *results[NIR_MAX_VEC_COMPONENTS]; nir_def *descriptor = intrinsic->src[0].ssa; nir_def *offset = intrinsic->src[1].ssa; nir_def *record = nir_channel(b, offset, 0); nir_def *record_offset = nir_channel(b, offset, 1); for (unsigned i = 0; i < intrinsic->def.num_components; i++) { results[i] = nir_load_uav_ir3(b, 1, intrinsic->def.bit_size, descriptor, nir_vec2(b, record, nir_iadd_imm(b, record_offset, i)), .access = nir_intrinsic_access(intrinsic), .align_mul = nir_intrinsic_align_mul(intrinsic), .align_offset = nir_intrinsic_align_offset(intrinsic)); } nir_def *result = nir_vec(b, results, intrinsic->def.num_components); nir_def_rewrite_uses(&intrinsic->def, result); nir_instr_remove(&intrinsic->instr); } static bool lower_offset_for_ssbo(nir_intrinsic_instr *intrinsic, nir_builder *b, unsigned ir3_ssbo_opcode, uint8_t offset_src_idx, struct ir3_compiler *c) { unsigned num_srcs = nir_intrinsic_infos[intrinsic->intrinsic].num_srcs; int shift = 2; bool has_dest = nir_intrinsic_infos[intrinsic->intrinsic].has_dest; nir_def *new_dest = NULL; /* for 16-bit ssbo access, offset is in 16-bit words instead of dwords */ if ((has_dest && intrinsic->def.bit_size == 16) || (!has_dest && intrinsic->src[0].ssa->bit_size == 16)) shift = 1; /* for 8-bit ssbo access, offset is in 8-bit words instead of dwords */ if ((has_dest && intrinsic->def.bit_size == 8) || (!has_dest && intrinsic->src[0].ssa->bit_size == 8)) shift = 0; if ((has_dest && intrinsic->def.bit_size == 64) || (!has_dest && intrinsic->src[0].ssa->bit_size == 64)) { /* a7xx quirk, 64b atomics against a 16b raw buffer have offset * in units of 16b instead of dword: */ if (c->gen == 7) shift = 1; } /* Here we create a new intrinsic and copy over all contents from the old * one. */ nir_intrinsic_instr *new_intrinsic; nir_src *target_src; b->cursor = nir_before_instr(&intrinsic->instr); /* 'offset_src_idx' holds the index of the source that represent the offset. */ new_intrinsic = nir_intrinsic_instr_create(b->shader, ir3_ssbo_opcode); nir_def *offset = intrinsic->src[offset_src_idx].ssa; /* The new source that will hold the dword-offset is always the last * one for every intrinsic. */ target_src = &new_intrinsic->src[num_srcs]; *target_src = nir_src_for_ssa(offset); if (has_dest) { nir_def *dest = &intrinsic->def; nir_def_init(&new_intrinsic->instr, &new_intrinsic->def, dest->num_components, dest->bit_size); new_dest = &new_intrinsic->def; } for (unsigned i = 0; i < num_srcs; i++) new_intrinsic->src[i] = nir_src_for_ssa(intrinsic->src[i].ssa); nir_intrinsic_copy_const_indices(new_intrinsic, intrinsic); new_intrinsic->num_components = intrinsic->num_components; int cur_shift = nir_intrinsic_offset_shift(intrinsic); int extra_shift = shift - cur_shift; /* TODO if the intrinsic has a BASE, we have to be careful when inserting a * right shift as the offset may be negative. So we'd have to add the BASE to * the offset before shifting. For now, as our input intrinsics don't support * BASE, we don't have to implement this yet. */ assert(!nir_intrinsic_has_base(intrinsic)); if (extra_shift > 0) { offset = nir_ushr_imm(b, offset, extra_shift); } else { offset = nir_ishl_imm(b, offset, -extra_shift); } /* Insert the new intrinsic right before the old one. */ nir_builder_instr_insert(b, &new_intrinsic->instr); /* Replace the last source of the new intrinsic by the result of * the offset shifted to the correct unit. */ nir_src_rewrite(target_src, offset); nir_intrinsic_set_offset_shift(new_intrinsic, shift); if (has_dest) { /* Replace the uses of the original destination by that * of the new intrinsic. */ nir_def_rewrite_uses(&intrinsic->def, new_dest); } /* Finally remove the original intrinsic. */ nir_instr_remove(&intrinsic->instr); return true; } static bool lower_io_offsets_block(nir_block *block, nir_builder *b, void *mem_ctx, struct ir3_compiler *c) { bool progress = false; nir_foreach_instr_safe (instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); /* SSBO */ int ir3_intrinsic; uint8_t offset_src_idx; ir3_intrinsic = get_ir3_intrinsic_for_ssbo_intrinsic(intr->intrinsic, &offset_src_idx); if (ir3_intrinsic != -1) { progress |= lower_offset_for_ssbo(intr, b, (unsigned)ir3_intrinsic, offset_src_idx, c); } if (intr->intrinsic == nir_intrinsic_load_uav_ir3 && (nir_intrinsic_access(intr) & ACCESS_CAN_REORDER) && ir3_bindless_resource(intr->src[0]) && intr->num_components > 1) { b->cursor = nir_before_instr(instr); scalarize_load(intr, b); progress = true; } } return progress; } static bool lower_io_offsets_func(nir_function_impl *impl, struct ir3_compiler *c) { void *mem_ctx = ralloc_parent(impl); nir_builder b = nir_builder_create(impl); bool progress = false; nir_foreach_block_safe (block, impl) { progress |= lower_io_offsets_block(block, &b, mem_ctx, c); } return nir_progress(progress, impl, nir_metadata_control_flow); } bool ir3_nir_lower_io_offsets(nir_shader *shader, struct ir3_compiler *c) { bool progress = false; nir_foreach_function (function, shader) { if (function->impl) progress |= lower_io_offsets_func(function->impl, c); } return progress; } uint32_t ir3_nir_max_imm_offset(nir_intrinsic_instr *intrin, const void *data) { const struct ir3_compiler *compiler = data; if (!compiler->info->props.has_ssbo_imm_offsets) return 0; switch (intrin->intrinsic) { case nir_intrinsic_load_ssbo_ir3: if ((nir_intrinsic_access(intrin) & ACCESS_CAN_REORDER) && !(compiler->options.storage_8bit && intrin->def.bit_size == 8)) return 255; /* isam.v */ return 127; /* ldib.b */ case nir_intrinsic_store_ssbo_ir3: return 127; /* stib.b */ default: return 0; } } bool ir3_nir_allow_base_offset_wrap(nir_intrinsic_instr *intrin, const void *data) { return true; } unsigned ir3_nir_max_offset_shift(nir_intrinsic_instr *intr, const void *data) { nir_deref_instr *deref = nir_src_as_deref(intr->src[0]); assert(util_bitcount(deref->modes) == 1); switch (deref->modes) { case nir_var_mem_ssbo: /* SSBO accesses can be up to dword shifted for 32-bit accesses. Request * that we always try to align up to that, so that vectorization can try * to build accesses across bit sizes. We'll legalize the shift for the * actual access size at the end. */ return 2; default: return 0; } }