Debian Codegraph @

src/imagination/pco/pco_nir_sync.c

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * \file pco_nir_sync.c
9 *
10 * \brief PCO NIR sync-related passes.
11 */
12
13 #include "hwdef/rogue_hw_defs.h"
14 #include "nir.h"
15 #include "nir_builder.h"
16 #include "nir_builtin_builder.h"
17 #include "pco.h"
18 #include "pco_builder.h"
19 #include "pco_internal.h"
20 #include "pco_usclib.h"
21 #include "util/macros.h"
22
23 #include <assert.h>
24 #include <stdbool.h>
25 #include <stdio.h>
26
27 /**
28 * \brief Lowers a barrier instruction.
29 *
30 * \param[in] b NIR builder.
31 * \param[in] instr NIR instruction.
32 * \param[in] cb_data User callback data.
33 * \return The replacement/lowered def.
34 */
35 static nir_def *lower_barrier(nir_builder *b, nir_instr *instr, void *cb_data)
36 {
37 struct shader_info *info = &b->shader->info;
38 bool *uses_usclib = cb_data;
39
40 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
41 mesa_scope exec_scope = nir_intrinsic_execution_scope(intr);
42
43 unsigned wg_size = info->workgroup_size[0] * info->workgroup_size[1] *
44 info->workgroup_size[2];
45
46 if (wg_size <= ROGUE_MAX_INSTANCES_PER_TASK || exec_scope == SCOPE_NONE ||
47 exec_scope == SCOPE_SUBGROUP)
48 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
49
50 /* TODO: We might be able to re-use barrier counters. */
51 unsigned counter_offset = info->shared_size;
52 info->shared_size += sizeof(uint32_t);
53 info->zero_initialize_shared_memory = true;
54
55 *uses_usclib = true;
56
57 unsigned num_slots = DIV_ROUND_UP(wg_size, ROGUE_MAX_INSTANCES_PER_TASK);
58
59 b->cursor = nir_before_instr(instr);
60 usclib_barrier(b, nir_imm_int(b, num_slots), nir_imm_int(b, counter_offset));
61
62 return NIR_LOWER_INSTR_PROGRESS_REPLACE;
63 }
64
65 /**
66 * \brief Filters barrier instructions.
67 *
68 * \param[in] instr NIR instruction.
69 * \param[in] cb_data User callback data.
70 * \return True if the instruction matches the filter.
71 */
72 static bool is_barrier(const nir_instr *instr, UNUSED const void *cb_data)
73 {
74 if (instr->type != nir_instr_type_intrinsic)
75 return false;
76
77 return nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_barrier;
78 }
79
80 /**
81 * \brief Barrier lowering pass.
82 *
83 * \param[in,out] shader NIR shader.
84 * \return True if the pass made progress.
85 */
86 bool pco_nir_lower_barriers(nir_shader *shader, pco_data *data)
87 {
88 bool progress = nir_shader_lower_instructions(shader,
89 is_barrier,
90 lower_barrier,
91 &data->common.uses.usclib);
92
93 data->common.uses.barriers |= progress;
94
95 return progress;
96 }
97
98 static nir_def *
99 lower_usclib_atomic(nir_builder *b, nir_instr *instr, void *cb_data)
100 {
101 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
102 bool *uses_usclib = cb_data;
103
104 b->cursor = nir_before_instr(instr);
105
106 if (intr->intrinsic == nir_intrinsic_ssbo_atomic_swap) {
107 nir_def *buffer = intr->src[0].ssa;
108 nir_def *offset = intr->src[1].ssa;
109 nir_def *value = intr->src[2].ssa;
110 nir_def *value_swap = intr->src[3].ssa;
111
112 ASSERTED enum gl_access_qualifier access = nir_intrinsic_access(intr);
113 ASSERTED unsigned num_components = intr->def.num_components;
114 ASSERTED unsigned bit_size = intr->def.bit_size;
115 assert(access == ACCESS_COHERENT);
116 assert(num_components == 1 && bit_size == 32);
117
118 *uses_usclib = true;
119 return usclib_emu_ssbo_atomic_comp_swap(b,
120 buffer,
121 offset,
122 value,
123 value_swap);
124 }
125
126 nir_def *addr_data = intr->src[0].ssa;
127 nir_def *addr = nir_channels(b, addr_data, BITFIELD_RANGE(0, 2));
128 nir_def *value = nir_channel(b, addr_data, 2);
129 nir_def *value_swap = nir_channel(b, addr_data, 3);
130
131 ASSERTED unsigned num_components = intr->def.num_components;
132 ASSERTED unsigned bit_size = intr->def.bit_size;
133 assert(num_components == 1 && bit_size == 32);
134
135 *uses_usclib = true;
136 return usclib_emu_global_atomic_comp_swap(b, addr, value, value_swap);
137 }
138
139 static bool lower_global_atomic_intrinsic(nir_builder *b,
140 nir_intrinsic_instr *intrin,
141 UNUSED void *data)
142 {
143 nir_intrinsic_op op;
144 bool is_swap = false;
145 switch (intrin->intrinsic) {
146 case nir_intrinsic_global_atomic_2x32:
147 op = nir_intrinsic_global_atomic_pco;
148 break;
149 case nir_intrinsic_global_atomic_swap_2x32:
150 op = nir_intrinsic_global_atomic_swap_pco;
151 is_swap = true;
152 break;
153 default:
154 return false;
155 }
156
157 nir_src *addr_src = &intrin->src[0];
158 nir_src *data_src = &intrin->src[1];
159 nir_src *data2_src;
160 if (is_swap)
161 data2_src = &intrin->src[2];
162
163 b->cursor = nir_before_instr(&intrin->instr);
164
165 nir_def *new_src = nir_pad_vector(b, addr_src->ssa, is_swap ? 4 : 3);
166 new_src = nir_vector_insert_imm(b, new_src, data_src->ssa, 2);
167 if (is_swap)
168 new_src = nir_vector_insert_imm(b, new_src, data2_src->ssa, 3);
169
170 nir_intrinsic_instr *new_intrin = nir_intrinsic_instr_create(b->shader, op);
171 new_intrin->num_components = intrin->num_components;
172 nir_def_init(&new_intrin->instr,
173 &new_intrin->def,
174 intrin->def.num_components,
175 intrin->def.bit_size);
176 new_intrin->src[0] = nir_src_for_ssa(new_src);
177 assert(nir_intrinsic_has_atomic_op(intrin));
178 nir_intrinsic_set_atomic_op(new_intrin, nir_intrinsic_atomic_op(intrin));
179
180 nir_builder_instr_insert(b, &new_intrin->instr);
181 nir_def_rewrite_uses(&intrin->def, &new_intrin->def);
182 nir_instr_remove(&intrin->instr);
183
184 return true;
185 }
186
187 /**
188 * \brief Filters atomic instructions emulated with usclib.
189 *
190 * \param[in] instr NIR instruction.
191 * \param[in] cb_data User callback data.
192 * \return True if the instruction matches the filter.
193 */
194 static bool atomic_uses_usclib(const nir_instr *instr,
195 UNUSED const void *cb_data)
196 {
197 if (instr->type != nir_instr_type_intrinsic)
198 return false;
199
200 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
201
202 return intr->intrinsic == nir_intrinsic_ssbo_atomic_swap ||
203 intr->intrinsic == nir_intrinsic_global_atomic_swap_pco;
204 }
205
206 /**
207 * \brief Atomics lowering pass.
208 *
209 * \param[in,out] shader NIR shader.
210 * \return True if the pass made progress.
211 */
212 bool pco_nir_lower_atomics(nir_shader *shader, pco_data *data)
213 {
214 bool progress = false;
215
216 progress |= nir_shader_intrinsics_pass(shader,
217 lower_global_atomic_intrinsic,
218 nir_metadata_none,
219 NULL);
220 progress |= nir_shader_lower_instructions(shader,
221 atomic_uses_usclib,
222 lower_usclib_atomic,
223 &data->common.uses.usclib);
224
225 return progress;
226 }
227
228 static nir_def *
229 lower_subgroup_intrinsic(nir_builder *b, nir_instr *instr, void *cb_data)
230 {
231 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
232 assert(intr->def.num_components == 1);
233
234 switch (intr->intrinsic) {
235 case nir_intrinsic_load_subgroup_size:
236 return nir_imm_int(b, 1);
237
238 case nir_intrinsic_load_subgroup_invocation:
239 return nir_imm_int(b, 0);
240
241 case nir_intrinsic_load_num_subgroups:
242 return nir_imm_int(b,
243 b->shader->info.workgroup_size[0] *
244 b->shader->info.workgroup_size[1] *
245 b->shader->info.workgroup_size[2]);
246
247 case nir_intrinsic_load_subgroup_id:
248 return nir_load_local_invocation_index(b);
249
250 case nir_intrinsic_first_invocation:
251 return nir_imm_int(b, 0);
252
253 case nir_intrinsic_elect:
254 return nir_imm_true(b);
255
256 default:
257 break;
258 }
259
260 UNREACHABLE("");
261 }
262
263 static bool is_subgroup_intrinsic(const nir_instr *instr,
264 UNUSED const void *cb_data)
265 {
266 if (instr->type != nir_instr_type_intrinsic)
267 return false;
268
269 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
270 switch (intr->intrinsic) {
271 case nir_intrinsic_load_subgroup_size:
272 case nir_intrinsic_load_subgroup_invocation:
273 case nir_intrinsic_load_num_subgroups:
274 case nir_intrinsic_load_subgroup_id:
275 case nir_intrinsic_first_invocation:
276 case nir_intrinsic_elect:
277 return true;
278
279 default:
280 break;
281 }
282
283 return false;
284 }
285
286 bool pco_nir_lower_subgroups(nir_shader *shader)
287 {
288 shader->info.api_subgroup_size = 1;
289 shader->info.min_subgroup_size = 1;
290 shader->info.max_subgroup_size = 1;
291
292 return nir_shader_lower_instructions(shader,
293 is_subgroup_intrinsic,
294 lower_subgroup_intrinsic,
295 NULL);
296 }