| 1 |
/* |
| 2 |
* Copyright © 2025 Imagination Technologies Ltd. |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: MIT |
| 5 |
*/ |
| 6 |
|
| 7 |
/** |
| 8 |
* \file pco_nir_vk.c |
| 9 |
* |
| 10 |
* \brief PCO NIR Vulkan lowering pass. |
| 11 |
*/ |
| 12 |
|
| 13 |
#include "nir.h" |
| 14 |
#include "nir_builder.h" |
| 15 |
#include "pco.h" |
| 16 |
#include "pco_builder.h" |
| 17 |
#include "pco_internal.h" |
| 18 |
#include "util/macros.h" |
| 19 |
|
| 20 |
#include <assert.h> |
| 21 |
#include <stdbool.h> |
| 22 |
#include <stdio.h> |
| 23 |
|
| 24 |
static void |
| 25 |
set_resource_used(pco_common_data *common, unsigned desc_set, unsigned binding) |
| 26 |
{ |
| 27 |
assert(desc_set < ARRAY_SIZE(common->desc_sets)); |
| 28 |
pco_descriptor_set_data *desc_set_data = &common->desc_sets[desc_set]; |
| 29 |
desc_set_data->used = true; |
| 30 |
|
| 31 |
assert(desc_set_data->bindings && binding < desc_set_data->binding_count); |
| 32 |
pco_binding_data *binding_data = &desc_set_data->bindings[binding]; |
| 33 |
binding_data->used = true; |
| 34 |
} |
| 35 |
|
| 36 |
/** |
| 37 |
* \brief Lowers load_vulkan_descriptor. |
| 38 |
* |
| 39 |
* Packs the descriptor set, binding, and array element. |
| 40 |
* |
| 41 |
* \param[in] b NIR builder. |
| 42 |
* \param[in] intr NIR intrinsic instruction. |
| 43 |
* \param[in] common Shader common data. |
| 44 |
* \return The replacement/lowered def. |
| 45 |
*/ |
| 46 |
static nir_def *lower_load_vulkan_descriptor(nir_builder *b, |
| 47 |
nir_intrinsic_instr *intr, |
| 48 |
pco_common_data *common) |
| 49 |
{ |
| 50 |
nir_intrinsic_instr *vk_res_idx = nir_src_as_intrinsic(intr->src[0]); |
| 51 |
assert(vk_res_idx->intrinsic == nir_intrinsic_vulkan_resource_index); |
| 52 |
|
| 53 |
assert(nir_intrinsic_desc_type(intr) == nir_intrinsic_desc_type(vk_res_idx)); |
| 54 |
|
| 55 |
unsigned desc_set = nir_intrinsic_desc_set(vk_res_idx); |
| 56 |
unsigned binding = nir_intrinsic_binding(vk_res_idx); |
| 57 |
unsigned elem = nir_src_as_uint(vk_res_idx->src[0]); |
| 58 |
|
| 59 |
set_resource_used(common, desc_set, binding); |
| 60 |
|
| 61 |
uint32_t desc_set_binding = pco_pack_desc(desc_set, binding); |
| 62 |
return nir_imm_ivec3(b, desc_set_binding, elem, 0); |
| 63 |
} |
| 64 |
|
| 65 |
static nir_def *array_elem_from_deref(nir_builder *b, nir_deref_instr *deref) |
| 66 |
{ |
| 67 |
unsigned array_elem = 0; |
| 68 |
if (deref->deref_type != nir_deref_type_var) { |
| 69 |
assert(deref->deref_type == nir_deref_type_array); |
| 70 |
|
| 71 |
array_elem = nir_src_as_uint(deref->arr.index); |
| 72 |
|
| 73 |
deref = nir_deref_instr_parent(deref); |
| 74 |
} |
| 75 |
|
| 76 |
assert(deref->deref_type == nir_deref_type_var); |
| 77 |
return nir_imm_int(b, array_elem); |
| 78 |
} |
| 79 |
|
| 80 |
static inline bool is_comb_img_smp(unsigned desc_set, |
| 81 |
unsigned binding, |
| 82 |
const pco_common_data *common) |
| 83 |
{ |
| 84 |
const pco_descriptor_set_data *desc_set_data = &common->desc_sets[desc_set]; |
| 85 |
assert(desc_set_data->bindings && binding < desc_set_data->binding_count); |
| 86 |
|
| 87 |
const pco_binding_data *binding_data = &desc_set_data->bindings[binding]; |
| 88 |
return binding_data->is_img_smp; |
| 89 |
} |
| 90 |
|
| 91 |
static void lower_tex_deref_to_binding(nir_builder *b, |
| 92 |
nir_tex_instr *tex, |
| 93 |
unsigned deref_index, |
| 94 |
pco_common_data *common) |
| 95 |
{ |
| 96 |
nir_tex_src *deref_src = &tex->src[deref_index]; |
| 97 |
nir_deref_instr *deref = |
| 98 |
nir_def_as_deref(deref_src->src.ssa); |
| 99 |
|
| 100 |
b->cursor = nir_before_instr(&tex->instr); |
| 101 |
|
| 102 |
nir_variable *var = nir_deref_instr_get_variable(deref); |
| 103 |
assert(var); |
| 104 |
unsigned desc_set = var->data.descriptor_set; |
| 105 |
unsigned binding = var->data.binding; |
| 106 |
nir_def *elem = array_elem_from_deref(b, deref); |
| 107 |
|
| 108 |
set_resource_used(common, desc_set, binding); |
| 109 |
|
| 110 |
uint32_t desc_set_binding = pco_pack_desc(desc_set, binding); |
| 111 |
if (deref_src->src_type == nir_tex_src_texture_deref) { |
| 112 |
tex->texture_index = desc_set_binding; |
| 113 |
deref_src->src_type = nir_tex_src_backend1; |
| 114 |
} else { |
| 115 |
tex->sampler_index = desc_set_binding; |
| 116 |
deref_src->src_type = nir_tex_src_backend2; |
| 117 |
} |
| 118 |
|
| 119 |
nir_src_rewrite(&deref_src->src, elem); |
| 120 |
} |
| 121 |
|
| 122 |
static void |
| 123 |
add_txf_sampler(nir_builder *b, nir_tex_instr *tex, pco_common_data *common) |
| 124 |
{ |
| 125 |
int deref_index = nir_tex_instr_src_index(tex, nir_tex_src_backend1); |
| 126 |
assert(deref_index >= 0); |
| 127 |
nir_tex_src *deref_src = &tex->src[deref_index]; |
| 128 |
|
| 129 |
unsigned desc_set; |
| 130 |
unsigned binding; |
| 131 |
pco_unpack_desc(tex->texture_index, &desc_set, &binding); |
| 132 |
nir_def *elem = deref_src->src.ssa; |
| 133 |
|
| 134 |
/* If it's not a combined image/sampler, use the point sampler. */ |
| 135 |
if (!is_comb_img_smp(desc_set, binding, common)) { |
| 136 |
desc_set = PCO_POINT_SAMPLER; |
| 137 |
binding = PCO_POINT_SAMPLER; |
| 138 |
elem = nir_imm_int(b, 0); |
| 139 |
|
| 140 |
common->uses.point_sampler = true; |
| 141 |
} |
| 142 |
|
| 143 |
tex->sampler_index = pco_pack_desc(desc_set, binding); |
| 144 |
nir_tex_instr_add_src(tex, nir_tex_src_backend2, elem); |
| 145 |
} |
| 146 |
|
| 147 |
static inline void |
| 148 |
lower_tex_derefs(nir_builder *b, nir_tex_instr *tex, pco_common_data *common) |
| 149 |
{ |
| 150 |
int deref_index; |
| 151 |
|
| 152 |
deref_index = nir_tex_instr_src_index(tex, nir_tex_src_texture_deref); |
| 153 |
if (deref_index >= 0) |
| 154 |
lower_tex_deref_to_binding(b, tex, deref_index, common); |
| 155 |
|
| 156 |
deref_index = nir_tex_instr_src_index(tex, nir_tex_src_sampler_deref); |
| 157 |
if (deref_index >= 0) |
| 158 |
lower_tex_deref_to_binding(b, tex, deref_index, common); |
| 159 |
else if (tex->op == nir_texop_txf || tex->op == nir_texop_txf_ms) |
| 160 |
add_txf_sampler(b, tex, common); |
| 161 |
} |
| 162 |
|
| 163 |
static nir_def * |
| 164 |
lower_image_derefs(nir_builder *b, nir_intrinsic_instr *intr, pco_data *data) |
| 165 |
{ |
| 166 |
nir_src *deref_src = &intr->src[0]; |
| 167 |
nir_deref_instr *deref = nir_src_as_deref(*deref_src); |
| 168 |
b->cursor = nir_before_instr(&intr->instr); |
| 169 |
|
| 170 |
nir_variable *var = nir_deref_instr_get_variable(deref); |
| 171 |
assert(var); |
| 172 |
|
| 173 |
unsigned desc_set = var->data.descriptor_set; |
| 174 |
unsigned binding = var->data.binding; |
| 175 |
set_resource_used(&data->common, desc_set, binding); |
| 176 |
|
| 177 |
if (nir_intrinsic_format(intr) == PIPE_FORMAT_NONE) |
| 178 |
nir_intrinsic_set_format(intr, var->data.image.format); |
| 179 |
|
| 180 |
enum glsl_sampler_dim image_dim = nir_intrinsic_image_dim(intr); |
| 181 |
bool ia = image_dim == GLSL_SAMPLER_DIM_SUBPASS || |
| 182 |
image_dim == GLSL_SAMPLER_DIM_SUBPASS_MS; |
| 183 |
|
| 184 |
if (ia) { |
| 185 |
unsigned ia_idx = var->data.index; |
| 186 |
bool onchip = data->fs.ia_formats[ia_idx] != PIPE_FORMAT_NONE; |
| 187 |
|
| 188 |
bool is_stencil = data->fs.ia_has_stencil & BITFIELD_BIT(ia_idx); |
| 189 |
is_stencil &= glsl_get_sampler_result_type(glsl_without_array_or_matrix( |
| 190 |
var->type)) != GLSL_TYPE_FLOAT; |
| 191 |
onchip &= !is_stencil; |
| 192 |
|
| 193 |
if (onchip) { |
| 194 |
nir_def *elem = array_elem_from_deref(b, deref); |
| 195 |
nir_def *index = nir_vec4(b, |
| 196 |
nir_imm_int(b, desc_set), |
| 197 |
nir_imm_int(b, binding), |
| 198 |
elem, |
| 199 |
nir_imm_int(b, ia_idx)); |
| 200 |
|
| 201 |
nir_src_rewrite(deref_src, index); |
| 202 |
|
| 203 |
return NIR_LOWER_INSTR_PROGRESS; |
| 204 |
} |
| 205 |
|
| 206 |
/* Sampler not needed for on-chip input attachments. */ |
| 207 |
data->common.uses.ia_sampler = true; |
| 208 |
} else if (intr->intrinsic == nir_intrinsic_image_deref_load || |
| 209 |
intr->intrinsic == nir_intrinsic_image_deref_store) { |
| 210 |
/* Sampler not needed for other types of image accesses. */ |
| 211 |
data->common.uses.point_sampler = true; |
| 212 |
} |
| 213 |
|
| 214 |
nir_def *elem = array_elem_from_deref(b, deref); |
| 215 |
nir_def *index = |
| 216 |
nir_vec3(b, nir_imm_int(b, desc_set), nir_imm_int(b, binding), elem); |
| 217 |
|
| 218 |
nir_src_rewrite(deref_src, index); |
| 219 |
|
| 220 |
return NIR_LOWER_INSTR_PROGRESS; |
| 221 |
} |
| 222 |
|
| 223 |
static nir_def *lower_is_null_descriptor(nir_builder *b, |
| 224 |
nir_intrinsic_instr *intr) |
| 225 |
{ |
| 226 |
nir_src *deref_src = &intr->src[0]; |
| 227 |
nir_deref_instr *deref = nir_src_as_deref(*deref_src); |
| 228 |
|
| 229 |
/* Will be taken care of by lower_load_vulkan_descriptor. */ |
| 230 |
if (!deref) |
| 231 |
return NULL; |
| 232 |
|
| 233 |
b->cursor = nir_before_instr(&intr->instr); |
| 234 |
|
| 235 |
nir_variable *var = nir_deref_instr_get_variable(deref); |
| 236 |
assert(var); |
| 237 |
unsigned desc_set = var->data.descriptor_set; |
| 238 |
unsigned binding = var->data.binding; |
| 239 |
nir_def *elem = array_elem_from_deref(b, deref); |
| 240 |
|
| 241 |
uint32_t desc_set_binding = pco_pack_desc(desc_set, binding); |
| 242 |
nir_def *index = nir_vec2(b, nir_imm_int(b, desc_set_binding), elem); |
| 243 |
|
| 244 |
nir_src_rewrite(deref_src, index); |
| 245 |
return NIR_LOWER_INSTR_PROGRESS; |
| 246 |
} |
| 247 |
|
| 248 |
/** |
| 249 |
* \brief Lowers a Vulkan-related instruction. |
| 250 |
* |
| 251 |
* \param[in] b NIR builder. |
| 252 |
* \param[in] instr NIR instruction. |
| 253 |
* \param[in] cb_data User callback data. |
| 254 |
* \return The replacement/lowered def. |
| 255 |
*/ |
| 256 |
static nir_def *lower_vk(nir_builder *b, nir_instr *instr, void *cb_data) |
| 257 |
{ |
| 258 |
pco_data *data = cb_data; |
| 259 |
pco_common_data *common = &data->common; |
| 260 |
|
| 261 |
switch (instr->type) { |
| 262 |
case nir_instr_type_intrinsic: { |
| 263 |
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 264 |
switch (intr->intrinsic) { |
| 265 |
case nir_intrinsic_load_vulkan_descriptor: |
| 266 |
return lower_load_vulkan_descriptor(b, intr, common); |
| 267 |
|
| 268 |
case nir_intrinsic_image_deref_load: |
| 269 |
case nir_intrinsic_image_deref_store: |
| 270 |
case nir_intrinsic_image_deref_atomic: |
| 271 |
case nir_intrinsic_image_deref_atomic_swap: |
| 272 |
case nir_intrinsic_image_deref_size: |
| 273 |
return lower_image_derefs(b, intr, data); |
| 274 |
|
| 275 |
case nir_intrinsic_is_null_descriptor: |
| 276 |
return lower_is_null_descriptor(b, intr); |
| 277 |
|
| 278 |
default: |
| 279 |
break; |
| 280 |
} |
| 281 |
|
| 282 |
break; |
| 283 |
} |
| 284 |
|
| 285 |
case nir_instr_type_tex: { |
| 286 |
nir_tex_instr *tex = nir_instr_as_tex(instr); |
| 287 |
lower_tex_derefs(b, tex, common); |
| 288 |
return NIR_LOWER_INSTR_PROGRESS; |
| 289 |
} |
| 290 |
|
| 291 |
default: |
| 292 |
break; |
| 293 |
} |
| 294 |
|
| 295 |
UNREACHABLE(""); |
| 296 |
} |
| 297 |
|
| 298 |
/** |
| 299 |
* \brief Filters Vulkan-related instructions. |
| 300 |
* |
| 301 |
* \param[in] instr NIR instruction. |
| 302 |
* \param[in] cb_data User callback data. |
| 303 |
* \return True if the instruction matches the filter. |
| 304 |
*/ |
| 305 |
static bool is_vk(const nir_instr *instr, UNUSED const void *cb_data) |
| 306 |
{ |
| 307 |
switch (instr->type) { |
| 308 |
case nir_instr_type_intrinsic: { |
| 309 |
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); |
| 310 |
switch (intr->intrinsic) { |
| 311 |
case nir_intrinsic_load_vulkan_descriptor: |
| 312 |
case nir_intrinsic_is_null_descriptor: |
| 313 |
case nir_intrinsic_image_deref_load: |
| 314 |
case nir_intrinsic_image_deref_store: |
| 315 |
case nir_intrinsic_image_deref_atomic: |
| 316 |
case nir_intrinsic_image_deref_atomic_swap: |
| 317 |
case nir_intrinsic_image_deref_size: |
| 318 |
return true; |
| 319 |
|
| 320 |
default: |
| 321 |
break; |
| 322 |
} |
| 323 |
|
| 324 |
break; |
| 325 |
} |
| 326 |
|
| 327 |
case nir_instr_type_tex: { |
| 328 |
nir_tex_instr *tex = nir_instr_as_tex(instr); |
| 329 |
if (nir_tex_instr_src_index(tex, nir_tex_src_texture_deref) >= 0 || |
| 330 |
nir_tex_instr_src_index(tex, nir_tex_src_sampler_deref) >= 0) { |
| 331 |
return true; |
| 332 |
} |
| 333 |
|
| 334 |
FALLTHROUGH; |
| 335 |
} |
| 336 |
|
| 337 |
default: |
| 338 |
break; |
| 339 |
} |
| 340 |
|
| 341 |
return false; |
| 342 |
} |
| 343 |
|
| 344 |
/** |
| 345 |
* \brief Vulkan lowering pass. |
| 346 |
* |
| 347 |
* \param[in,out] shader NIR shader. |
| 348 |
* \param[in,out] data Shader data. |
| 349 |
* \return True if the pass made progress. |
| 350 |
*/ |
| 351 |
bool pco_nir_lower_vk(nir_shader *shader, pco_data *data) |
| 352 |
{ |
| 353 |
bool progress = false; |
| 354 |
|
| 355 |
progress |= nir_shader_lower_instructions(shader, is_vk, lower_vk, data); |
| 356 |
|
| 357 |
return progress; |
| 358 |
} |