Debian Codegraph @

src/imagination/vulkan/pvr_arch_pipeline.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on v3dv driver which is:
5 * Copyright © 2019 Raspberry Pi
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * SOFTWARE.
25 */
26
27 #include "pvr_pipeline.h"
28
29 #include <assert.h>
30 #include <stdbool.h>
31 #include <stdint.h>
32 #include <string.h>
33 #include <vulkan/vulkan.h>
34
35 #include "compiler/shader_enums.h"
36 #include "hwdef/rogue_hw_utils.h"
37 #include "nir/nir.h"
38 #include "nir/nir_lower_blend.h"
39 #include "pco/pco.h"
40 #include "pco/pco_data.h"
41
42 #include "pvr_bo.h"
43 #include "pvr_csb.h"
44 #include "pvr_csb_enum_helpers.h"
45 #include "pvr_descriptor_set.h"
46 #include "pvr_device.h"
47 #include "pvr_entrypoints.h"
48 #include "pvr_hw_pass.h"
49 #include "pvr_macros.h"
50 #include "pvr_nir_lower_ycbcr.h"
51 #include "pvr_pass.h"
52 #include "pvr_pds.h"
53 #include "pvr_physical_device.h"
54 #include "pvr_robustness.h"
55 #include "pvr_sampler.h"
56 #include "pvr_types.h"
57 #include "pvr_usc.h"
58
59 #include "util/log.h"
60 #include "util/macros.h"
61 #include "util/ralloc.h"
62 #include "util/shader_stats.h"
63 #include "util/u_dynarray.h"
64 #include "util/u_math.h"
65 #include "vk_alloc.h"
66 #include "vk_blend.h"
67 #include "vk_format.h"
68 #include "vk_graphics_state.h"
69 #include "vk_log.h"
70 #include "vk_object.h"
71 #include "vk_pipeline_cache.h"
72 #include "vk_pipeline_layout.h"
73 #include "vk_render_pass.h"
74 #include "vk_util.h"
75 #include "vulkan/runtime/vk_pipeline.h"
76 #include "vulkan/vulkan_core.h"
77
78 /*****************************************************************************
79 PDS functions
80 *****************************************************************************/
81
82 /* If allocator == NULL, the internal one will be used. */
83 static VkResult pvr_pds_coeff_program_create_and_upload(
84 struct pvr_device *device,
85 const VkAllocationCallbacks *allocator,
86 struct pvr_fragment_shader_state *fragment_state)
87 {
88 struct pvr_pds_coeff_loading_program *program =
89 &fragment_state->pds_coeff_program;
90 uint32_t staging_buffer_size;
91 uint32_t *staging_buffer;
92
93 assert(program->num_fpu_iterators < PVR_MAXIMUM_ITERATIONS);
94
95 /* Get the size of the program and then allocate that much memory. */
96 pvr_pds_coefficient_loading(program, NULL, PDS_GENERATE_SIZES);
97
98 if (!program->code_size) {
99 fragment_state->pds_coeff_program_buffer = NULL;
100 fragment_state->stage_state.pds_temps_count = 0;
101
102 return VK_SUCCESS;
103 }
104
105 staging_buffer_size =
106 PVR_DW_TO_BYTES(program->code_size + program->data_size);
107
108 staging_buffer = vk_alloc2(&device->vk.alloc,
109 allocator,
110 staging_buffer_size,
111 8,
112 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
113 if (!staging_buffer)
114 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
115
116 /* Generate the program into is the staging_buffer. */
117 pvr_pds_coefficient_loading(program,
118 staging_buffer,
119 PDS_GENERATE_CODEDATA_SEGMENTS);
120
121 fragment_state->pds_coeff_program_buffer = staging_buffer;
122 fragment_state->stage_state.pds_temps_count = program->temps_used;
123
124 return VK_SUCCESS;
125 }
126
127 /* If allocator == NULL, the internal one will be used. */
128 static VkResult pvr_pds_fragment_program_create(
129 struct pvr_device *device,
130 const VkAllocationCallbacks *allocator,
131 pco_shader *fs,
132 struct pvr_fragment_shader_state *fragment_state)
133 {
134 struct pvr_pds_kickusc_program *program =
135 &fragment_state->pds_fragment_program;
136 pco_data *fs_data = pco_shader_data(fs);
137 uint32_t staging_buffer_size;
138 uint32_t *staging_buffer;
139
140 const pvr_dev_addr_t exec_addr =
141 PVR_DEV_ADDR_OFFSET(fragment_state->shader_bo->dev_addr,
142 fs_data->common.entry_offset);
143
144 /* Note this is not strictly required to be done before calculating the
145 * staging_buffer_size in this particular case. It can also be done after
146 * allocating the buffer. The size from pvr_pds_kick_usc() is constant.
147 */
148 pvr_pds_setup_doutu(&program->usc_task_control,
149 exec_addr.addr,
150 fs_data->common.temps,
151 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
152 fs_data->fs.uses.phase_change);
153
154 pvr_pds_kick_usc(program, NULL, 0, false, PDS_GENERATE_SIZES);
155
156 staging_buffer_size =
157 PVR_DW_TO_BYTES(program->code_size + program->data_size);
158
159 staging_buffer = vk_alloc2(&device->vk.alloc,
160 allocator,
161 staging_buffer_size,
162 8,
163 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
164 if (!staging_buffer)
165 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
166
167 pvr_pds_kick_usc(program,
168 staging_buffer,
169 0,
170 false,
171 PDS_GENERATE_CODEDATA_SEGMENTS);
172
173 fragment_state->pds_fragment_program_buffer = staging_buffer;
174
175 return VK_SUCCESS;
176 }
177
178 static inline size_t pvr_pds_get_max_vertex_program_const_map_size_in_bytes(
179 const struct pvr_device_info *dev_info,
180 bool robust_buffer_access)
181 {
182 /* FIXME: Use more local variable to improve formatting. */
183
184 /* Maximum memory allocation needed for const map entries in
185 * pvr_pds_generate_vertex_primary_program().
186 * When robustBufferAccess is disabled, it must be >= 410.
187 * When robustBufferAccess is enabled, it must be >= 570.
188 *
189 * 1. Size of entry for base instance
190 * (pvr_const_map_entry_base_instance)
191 *
192 * 2. Max. number of vertex inputs (PVR_MAX_VERTEX_INPUT_BINDINGS) * (
193 * if (!robustBufferAccess)
194 * size of vertex attribute entry
195 * (pvr_const_map_entry_vertex_attribute_address) +
196 * else
197 * size of robust vertex attribute entry
198 * (pvr_const_map_entry_robust_vertex_attribute_address) +
199 * size of entry for max attribute index
200 * (pvr_const_map_entry_vertex_attribute_max_index) +
201 * fi
202 * size of Unified Store burst entry
203 * (pvr_const_map_entry_literal32) +
204 * size of entry for vertex stride
205 * (pvr_const_map_entry_literal32) +
206 * size of entries for DDMAD control word
207 * (num_ddmad_literals * pvr_const_map_entry_literal32))
208 *
209 * 3. Size of entry for DOUTW vertex/instance control word
210 * (pvr_const_map_entry_literal32)
211 *
212 * 4. Size of DOUTU entry (pvr_const_map_entry_doutu_address)
213 */
214
215 const size_t attribute_size =
216 (!robust_buffer_access)
217 ? sizeof(struct pvr_const_map_entry_vertex_attribute_address)
218 : sizeof(struct pvr_const_map_entry_robust_vertex_attribute_address) +
219 sizeof(struct pvr_const_map_entry_vertex_attribute_max_index);
220
221 /* If has_pds_ddmadt the DDMAD control word is now a DDMADT control word
222 * and is increased by one DWORD to contain the data for the DDMADT's
223 * out-of-bounds check.
224 */
225 const size_t pvr_pds_const_map_vertex_entry_num_ddmad_literals =
226 1U + (size_t)PVR_HAS_FEATURE(dev_info, pds_ddmadt);
227
228 return (sizeof(struct pvr_const_map_entry_base_instance) +
229 PVR_MAX_VERTEX_INPUT_BINDINGS *
230 (attribute_size +
231 (2 + pvr_pds_const_map_vertex_entry_num_ddmad_literals) *
232 sizeof(struct pvr_const_map_entry_literal32)) +
233 sizeof(struct pvr_const_map_entry_literal32) +
234 sizeof(struct pvr_const_map_entry_doutu_address));
235 }
236
237 static inline void
238 pvr_pds_free_pds_info_map_entries(struct pvr_device *const device,
239 const VkAllocationCallbacks *const allocator,
240 struct pvr_const_map_entry **const entries)
241 {
242 if (entries && *entries) {
243 vk_free2(&device->vk.alloc, allocator, *entries);
244 *entries = NULL;
245 }
246 }
247
248 static VkResult pvr_pds_vertex_attrib_program_create_and_upload(
249 struct pvr_device *const device,
250 const VkAllocationCallbacks *const allocator,
251 struct pvr_pds_vertex_primary_program_input *const input,
252 struct pvr_pds_attrib_program *const program_out)
253 {
254 const size_t const_entries_size_in_bytes =
255 pvr_pds_get_max_vertex_program_const_map_size_in_bytes(
256 &device->pdevice->dev_info,
257 device->vk.enabled_features.robustBufferAccess);
258 struct pvr_pds_upload *const program = &program_out->program;
259 struct pvr_pds_info *const info = &program_out->info;
260 struct pvr_const_map_entry *new_entries;
261 ASSERTED uint32_t code_size_in_dwords;
262 size_t staging_buffer_size;
263 uint32_t *staging_buffer;
264 VkResult result;
265
266 memset(info, 0, sizeof(*info));
267
268 info->entries = vk_alloc2(&device->vk.alloc,
269 allocator,
270 const_entries_size_in_bytes,
271 8,
272 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
273 if (!info->entries) {
274 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
275 goto err_out;
276 }
277
278 info->entries_size_in_bytes = const_entries_size_in_bytes;
279
280 pvr_pds_generate_vertex_primary_program(
281 input,
282 NULL,
283 info,
284 device->vk.enabled_features.robustBufferAccess,
285 &device->pdevice->dev_info);
286
287 code_size_in_dwords = info->code_size_in_dwords;
288 staging_buffer_size = PVR_DW_TO_BYTES(info->code_size_in_dwords);
289
290 staging_buffer = vk_alloc2(&device->vk.alloc,
291 allocator,
292 staging_buffer_size,
293 8,
294 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
295 if (!staging_buffer) {
296 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
297 goto err_free_entries;
298 }
299
300 /* This also fills in info->entries. */
301 pvr_pds_generate_vertex_primary_program(
302 input,
303 staging_buffer,
304 info,
305 device->vk.enabled_features.robustBufferAccess,
306 &device->pdevice->dev_info);
307
308 assert(info->code_size_in_dwords <= code_size_in_dwords);
309
310 new_entries = vk_realloc2(&device->vk.alloc,
311 allocator,
312 info->entries,
313 info->entries_written_size_in_bytes,
314 8,
315 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
316 if (!new_entries) {
317 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
318 goto err_free_staging_buffer;
319 }
320
321 info->entries = new_entries;
322 info->entries_size_in_bytes = info->entries_written_size_in_bytes;
323
324 /* FIXME: Figure out the define for alignment of 16. */
325 result = pvr_gpu_upload_pds(device,
326 NULL,
327 0,
328 0,
329 staging_buffer,
330 info->code_size_in_dwords,
331 16,
332 16,
333 program);
334 if (result != VK_SUCCESS)
335 goto err_free_staging_buffer;
336
337 vk_free2(&device->vk.alloc, allocator, staging_buffer);
338
339 return VK_SUCCESS;
340
341 err_free_staging_buffer:
342 vk_free2(&device->vk.alloc, allocator, staging_buffer);
343
344 err_free_entries:
345 pvr_pds_free_pds_info_map_entries(device, allocator, &info->entries);
346
347 err_out:
348 return result;
349 }
350
351 static inline void pvr_pds_vertex_attrib_program_destroy(
352 struct pvr_device *const device,
353 const struct VkAllocationCallbacks *const allocator,
354 struct pvr_pds_attrib_program *const program)
355 {
356 pvr_bo_suballoc_free(program->program.pvr_bo);
357 pvr_pds_free_pds_info_map_entries(device, allocator, &program->info.entries);
358 }
359
360 /* This is a const pointer to an array of pvr_pds_attrib_program structs.
361 * The array being pointed to is of PVR_PDS_VERTEX_ATTRIB_PROGRAM_COUNT size.
362 */
363 typedef struct pvr_pds_attrib_program (*const pvr_pds_attrib_programs_array_ptr)
364 [PVR_PDS_VERTEX_ATTRIB_PROGRAM_COUNT];
365
366 /* Generate and uploads a PDS program for DMAing vertex attribs into USC vertex
367 * inputs. This will bake the code segment and create a template of the data
368 * segment for the command buffer to fill in.
369 */
370 /* If allocator == NULL, the internal one will be used.
371 *
372 * programs_out_ptr is a pointer to the array where the outputs will be placed.
373 */
374 static VkResult pvr_pds_vertex_attrib_programs_create_and_upload(
375 struct pvr_device *device,
376 const VkAllocationCallbacks *const allocator,
377 pco_data *shader_data,
378 const struct pvr_pds_vertex_dma
379 dma_descriptions[static const PVR_MAX_VERTEX_ATTRIB_DMAS],
380 uint32_t dma_count,
381 pvr_pds_attrib_programs_array_ptr programs_out_ptr)
382 {
383 struct pvr_pds_vertex_primary_program_input input = {
384 .dma_list = dma_descriptions,
385 .dma_count = dma_count,
386 };
387 uint32_t usc_temp_count = shader_data->common.temps;
388 struct pvr_pds_attrib_program *const programs_out = *programs_out_ptr;
389 VkResult result;
390
391 pco_range *sys_vals = shader_data->common.sys_vals;
392 if (sys_vals[SYSTEM_VALUE_VERTEX_ID].count > 0) {
393 input.flags |= PVR_PDS_VERTEX_FLAGS_VERTEX_ID_REQUIRED;
394 input.vertex_id_register = sys_vals[SYSTEM_VALUE_VERTEX_ID].start;
395 }
396
397 if (sys_vals[SYSTEM_VALUE_INSTANCE_ID].count > 0) {
398 input.flags |= PVR_PDS_VERTEX_FLAGS_INSTANCE_ID_REQUIRED;
399 input.instance_id_register = sys_vals[SYSTEM_VALUE_INSTANCE_ID].start;
400 }
401
402 if (sys_vals[SYSTEM_VALUE_BASE_INSTANCE].count > 0) {
403 input.flags |= PVR_PDS_VERTEX_FLAGS_BASE_INSTANCE_REQUIRED;
404 input.base_instance_register = sys_vals[SYSTEM_VALUE_BASE_INSTANCE].start;
405 }
406
407 if (sys_vals[SYSTEM_VALUE_BASE_VERTEX].count > 0) {
408 input.flags |= PVR_PDS_VERTEX_FLAGS_BASE_VERTEX_REQUIRED;
409 input.base_vertex_register = sys_vals[SYSTEM_VALUE_BASE_VERTEX].start;
410 }
411
412 if (sys_vals[SYSTEM_VALUE_DRAW_ID].count > 0) {
413 input.flags |= PVR_PDS_VERTEX_FLAGS_DRAW_INDEX_REQUIRED;
414 input.draw_index_register = sys_vals[SYSTEM_VALUE_DRAW_ID].start;
415 }
416
417 if (sys_vals[SYSTEM_VALUE_VIEW_INDEX].count > 0) {
418 input.flags |= PVR_PDS_VERTEX_FLAGS_VIEW_INDEX_REQUIRED;
419 input.view_index_register = sys_vals[SYSTEM_VALUE_VIEW_INDEX].start;
420 }
421
422 pvr_pds_setup_doutu(&input.usc_task_control,
423 0,
424 usc_temp_count,
425 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
426 false);
427
428 /* Note: programs_out_ptr is a pointer to an array so this is fine. See the
429 * typedef.
430 */
431 for (uint32_t i = 0; i < ARRAY_SIZE(*programs_out_ptr); i++) {
432 uint32_t extra_flags;
433
434 switch (i) {
435 case PVR_PDS_VERTEX_ATTRIB_PROGRAM_BASIC:
436 extra_flags = 0;
437 break;
438
439 case PVR_PDS_VERTEX_ATTRIB_PROGRAM_BASE_INSTANCE:
440 extra_flags = PVR_PDS_VERTEX_FLAGS_BASE_INSTANCE_VARIANT;
441 break;
442
443 case PVR_PDS_VERTEX_ATTRIB_PROGRAM_DRAW_INDIRECT:
444 extra_flags = PVR_PDS_VERTEX_FLAGS_DRAW_INDIRECT_VARIANT;
445 break;
446
447 default:
448 UNREACHABLE("Invalid vertex attrib program type.");
449 }
450
451 input.flags |= extra_flags;
452
453 result =
454 pvr_pds_vertex_attrib_program_create_and_upload(device,
455 allocator,
456 &input,
457 &programs_out[i]);
458 if (result != VK_SUCCESS) {
459 for (uint32_t j = 0; j < i; j++) {
460 pvr_pds_vertex_attrib_program_destroy(device,
461 allocator,
462 &programs_out[j]);
463 }
464
465 return result;
466 }
467
468 input.flags &= ~extra_flags;
469 }
470
471 return VK_SUCCESS;
472 }
473
474 static VkResult pvr_pds_descriptor_program_create_and_upload(
475 struct pvr_device *const device,
476 const VkAllocationCallbacks *const allocator,
477 const struct vk_pipeline_layout *const layout,
478 mesa_shader_stage stage,
479 pco_data *data,
480 struct pvr_stage_allocation_descriptor_state *const descriptor_state)
481 {
482 const size_t const_entries_size_in_bytes = PVR_PDS_MAX_DESC_UPLOAD_BYTES;
483 struct pvr_pds_info *const pds_info = &descriptor_state->pds_info;
484 struct pvr_pds_descriptor_program_input program = { 0 };
485 struct pvr_const_map_entry *new_entries;
486 ASSERTED uint32_t code_size_in_dwords;
487 uint32_t staging_buffer_size;
488 uint32_t *staging_buffer;
489 VkResult result;
490
491 *pds_info = (struct pvr_pds_info){ 0 };
492
493 for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) {
494 const struct pvr_descriptor_set_layout *set_layout =
495 vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]);
496
497 const pco_descriptor_set_data *desc_set_data =
498 &data->common.desc_sets[desc_set];
499 const pco_range *desc_set_range = &desc_set_data->range;
500 const pco_range *desc_set_dynamic_range = &desc_set_data->dynamic_range;
501
502 /* If the descriptor set isn't for this stage or is unused, skip it. */
503 if (!(BITFIELD_BIT(stage) & set_layout->stage_flags)) {
504 assert(!desc_set_data->used);
505 continue;
506 }
507
508 if (!desc_set_data->used)
509 continue;
510
511 if (desc_set_range->count > 0) {
512 program.descriptor_sets[program.descriptor_set_count++] =
513 (struct pvr_pds_descriptor_set){
514 .descriptor_set = desc_set,
515 .size_in_dwords = desc_set_range->count,
516 .destination = desc_set_range->start,
517 };
518 }
519
520 if (desc_set_dynamic_range->count > 0) {
521 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
522 .type = PVR_BUFFER_TYPE_DYNAMIC,
523 .size_in_dwords = desc_set_dynamic_range->count,
524 .destination = desc_set_dynamic_range->start,
525 .desc_set = desc_set,
526 };
527 }
528 }
529
530 pds_info->entries = vk_alloc2(&device->vk.alloc,
531 allocator,
532 const_entries_size_in_bytes,
533 8,
534 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
535 if (!pds_info->entries) {
536 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
537 goto err_free_static_consts;
538 }
539
540 if (data->common.push_consts.range.count > 0) {
541 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
542 .type = PVR_BUFFER_TYPE_PUSH_CONSTS,
543 .size_in_dwords = data->common.push_consts.range.count,
544 .destination = data->common.push_consts.range.start,
545 };
546 }
547
548 if (stage == MESA_SHADER_FRAGMENT && data->fs.blend_consts.count > 0) {
549 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
550 .type = PVR_BUFFER_TYPE_BLEND_CONSTS,
551 .size_in_dwords = data->fs.blend_consts.count,
552 .destination = data->fs.blend_consts.start,
553 };
554 }
555
556 if (data->common.point_sampler.count > 0) {
557 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
558 .type = PVR_BUFFER_TYPE_POINT_SAMPLER,
559 .size_in_dwords = data->common.point_sampler.count,
560 .destination = data->common.point_sampler.start,
561 };
562 }
563
564 if (data->common.ia_sampler.count > 0) {
565 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
566 .type = PVR_BUFFER_TYPE_IA_SAMPLER,
567 .size_in_dwords = data->common.ia_sampler.count,
568 .destination = data->common.ia_sampler.start,
569 };
570 }
571
572 if (data->common.spill_info.count > 0) {
573 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
574 .type = PVR_BUFFER_TYPE_SPILL_INFO,
575 .size_in_dwords = data->common.spill_info.count,
576 .destination = data->common.spill_info.start,
577 };
578 }
579
580 if (data->common.scratch_info.count > 0) {
581 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
582 .type = PVR_BUFFER_TYPE_SCRATCH_INFO,
583 .size_in_dwords = data->common.scratch_info.count,
584 .destination = data->common.scratch_info.start,
585 };
586 }
587
588 if (stage == MESA_SHADER_FRAGMENT &&
589 data->common.sys_vals[SYSTEM_VALUE_FRONT_FACE].count > 0) {
590 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
591 .type = PVR_BUFFER_TYPE_FRONT_FACE_OP,
592 .size_in_dwords = data->common.sys_vals[SYSTEM_VALUE_FRONT_FACE].count,
593 .destination = data->common.sys_vals[SYSTEM_VALUE_FRONT_FACE].start,
594 };
595 }
596
597 if (stage == MESA_SHADER_FRAGMENT && data->fs.meta.count > 0) {
598 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
599 .type = PVR_BUFFER_TYPE_FS_META,
600 .size_in_dwords = data->fs.meta.count,
601 .destination = data->fs.meta.start,
602 };
603 }
604
605 if (stage == MESA_SHADER_FRAGMENT && data->fs.tile_buffers.count > 0) {
606 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
607 .type = PVR_BUFFER_TYPE_TILE_BUFFERS,
608 .size_in_dwords = data->fs.tile_buffers.count,
609 .destination = data->fs.tile_buffers.start,
610 };
611 }
612
613 if (stage == MESA_SHADER_FRAGMENT && data->fs.sample_locations.count > 0) {
614 program.buffers[program.buffer_count++] = (struct pvr_pds_buffer){
615 .type = PVR_BUFFER_TYPE_SAMPLE_LOCATIONS,
616 .size_in_dwords = data->fs.sample_locations.count,
617 .destination = data->fs.sample_locations.start,
618 };
619 }
620
621 pds_info->entries_size_in_bytes = const_entries_size_in_bytes;
622
623 pvr_pds_generate_descriptor_upload_program(&program, NULL, pds_info);
624
625 code_size_in_dwords = pds_info->code_size_in_dwords;
626 staging_buffer_size = PVR_DW_TO_BYTES(pds_info->code_size_in_dwords);
627
628 if (!staging_buffer_size) {
629 vk_free2(&device->vk.alloc, allocator, pds_info->entries);
630
631 *descriptor_state = (struct pvr_stage_allocation_descriptor_state){ 0 };
632
633 return VK_SUCCESS;
634 }
635
636 staging_buffer = vk_alloc2(&device->vk.alloc,
637 allocator,
638 staging_buffer_size,
639 8,
640 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
641 if (!staging_buffer) {
642 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
643 goto err_free_entries;
644 }
645
646 pvr_pds_generate_descriptor_upload_program(&program,
647 staging_buffer,
648 pds_info);
649
650 assert(pds_info->code_size_in_dwords <= code_size_in_dwords);
651
652 new_entries = vk_realloc2(&device->vk.alloc,
653 allocator,
654 pds_info->entries,
655 pds_info->entries_written_size_in_bytes,
656 8,
657 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
658 if (!new_entries) {
659 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
660 goto err_free_staging_buffer;
661 }
662
663 pds_info->entries = new_entries;
664 pds_info->entries_size_in_bytes = pds_info->entries_written_size_in_bytes;
665
666 /* FIXME: Figure out the define for alignment of 16. */
667 result = pvr_gpu_upload_pds(device,
668 NULL,
669 0,
670 0,
671 staging_buffer,
672 pds_info->code_size_in_dwords,
673 16,
674 16,
675 &descriptor_state->pds_code);
676 if (result != VK_SUCCESS)
677 goto err_free_staging_buffer;
678
679 vk_free2(&device->vk.alloc, allocator, staging_buffer);
680
681 return VK_SUCCESS;
682
683 err_free_staging_buffer:
684 vk_free2(&device->vk.alloc, allocator, staging_buffer);
685
686 err_free_entries:
687 pvr_pds_free_pds_info_map_entries(device, allocator, &pds_info->entries);
688
689 err_free_static_consts:
690 pvr_bo_suballoc_free(descriptor_state->static_consts);
691
692 return result;
693 }
694
695 static void pvr_pds_descriptor_program_destroy(
696 struct pvr_device *const device,
697 const struct VkAllocationCallbacks *const allocator,
698 struct pvr_stage_allocation_descriptor_state *const descriptor_state)
699 {
700 if (!descriptor_state)
701 return;
702
703 pvr_bo_suballoc_free(descriptor_state->pds_code.pvr_bo);
704 pvr_pds_free_pds_info_map_entries(device,
705 allocator,
706 &descriptor_state->pds_info.entries);
707 pvr_bo_suballoc_free(descriptor_state->static_consts);
708 }
709
710 static void pvr_pds_compute_program_setup(
711 const struct pvr_device_info *dev_info,
712 pco_data *cs_data,
713 struct pvr_compute_shader_state *compute_state,
714 struct pvr_pds_compute_shader_program *const program)
715 {
716 pco_range *sys_vals = cs_data->common.sys_vals;
717
718 pvr_pds_compute_shader_program_init(program);
719
720 if (sys_vals[SYSTEM_VALUE_LOCAL_INVOCATION_INDEX].count > 0) {
721 program->local_input_regs[0] =
722 sys_vals[SYSTEM_VALUE_LOCAL_INVOCATION_INDEX].start;
723 }
724
725 for (unsigned u = 0; u < ARRAY_SIZE(program->work_group_input_regs); ++u) {
726 if (sys_vals[SYSTEM_VALUE_WORKGROUP_ID].count > u) {
727 program->work_group_input_regs[u] =
728 sys_vals[SYSTEM_VALUE_WORKGROUP_ID].start + u;
729 }
730 }
731
732 for (unsigned u = 0; u < ARRAY_SIZE(program->num_work_groups_regs); ++u) {
733 if (sys_vals[SYSTEM_VALUE_NUM_WORKGROUPS].count > u) {
734 program->num_work_groups_regs[u] =
735 sys_vals[SYSTEM_VALUE_NUM_WORKGROUPS].start + u;
736 }
737 }
738
739 program->flattened_work_groups = true;
740 program->kick_usc = true;
741
742 pvr_pds_setup_doutu(&program->usc_task_control,
743 compute_state->shader_bo->dev_addr.addr,
744 cs_data->common.temps,
745 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
746 false);
747
748 if (compute_state->coeff_update_shader_bo) {
749 program->has_coefficient_update_task = true;
750 pvr_pds_setup_doutu(&program->usc_task_control_coeff_update,
751 compute_state->coeff_update_shader_bo->dev_addr.addr,
752 compute_state->coeff_update_shader_temps,
753 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
754 false);
755 }
756
757 pvr_pds_compute_shader(program, NULL, PDS_GENERATE_SIZES, dev_info);
758 }
759
760 /* This uploads the code segment and base data segment variant.
761 * This can be patched at dispatch time.
762 */
763 static VkResult pvr_pds_compute_program_create_and_upload(
764 struct pvr_device *const device,
765 const VkAllocationCallbacks *const allocator,
766 struct pvr_compute_shader_state *compute_state,
767 struct pvr_compute_pipeline *compute_pipeline)
768 {
769 pco_range *sys_vals = compute_pipeline->cs_data.common.sys_vals;
770 struct pvr_device_info *dev_info = &device->pdevice->dev_info;
771 struct pvr_pds_compute_shader_program program;
772 uint32_t *code_buffer;
773 uint32_t *data_buffer;
774 VkResult result;
775
776 bool uses_wg_id = sys_vals[SYSTEM_VALUE_WORKGROUP_ID].count > 0;
777 bool uses_num_wgs = sys_vals[SYSTEM_VALUE_NUM_WORKGROUPS].count > 0;
778
779 pvr_pds_compute_program_setup(dev_info,
780 &compute_pipeline->cs_data,
781 compute_state,
782 &program);
783
784 code_buffer = vk_alloc2(&device->vk.alloc,
785 allocator,
786 PVR_DW_TO_BYTES(program.code_size),
787 8,
788 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
789 if (!code_buffer)
790 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
791
792 data_buffer = vk_alloc2(&device->vk.alloc,
793 allocator,
794 PVR_DW_TO_BYTES(program.code_size),
795 8,
796 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
797 if (!data_buffer) {
798 vk_free2(&device->vk.alloc, allocator, code_buffer);
799 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
800 }
801
802 pvr_pds_compute_shader(&program,
803 &code_buffer[0],
804 PDS_GENERATE_CODE_SEGMENT,
805 dev_info);
806
807 pvr_pds_compute_shader(&program,
808 &data_buffer[0],
809 PDS_GENERATE_DATA_SEGMENT,
810 dev_info);
811
812 /* Initialize. */
813 if (uses_wg_id) {
814 unsigned offset = program.base_workgroup_constant_offset_in_dwords[0];
815 for (unsigned u = 0; u < PVR_WORKGROUP_DIMENSIONS; ++u) {
816 data_buffer[offset + u] = 0;
817 }
818 }
819
820 if (uses_num_wgs) {
821 unsigned offset = program.num_workgroups_constant_offset_in_dwords[0];
822 for (unsigned u = 0; u < PVR_WORKGROUP_DIMENSIONS; ++u) {
823 data_buffer[offset + u] = 0;
824 }
825
826 offset = program.num_workgroups_indirect_src;
827 data_buffer[offset] = 0;
828 data_buffer[offset + 1] = 0;
829 }
830
831 /* FIXME: Figure out the define for alignment of 16. */
832 result = pvr_gpu_upload_pds(device,
833 data_buffer,
834 program.data_size,
835 16,
836 code_buffer,
837 program.code_size,
838 16,
839 16,
840 &compute_pipeline->pds_cs_program);
841 if (result != VK_SUCCESS) {
842 vk_free2(&device->vk.alloc, allocator, code_buffer);
843 vk_free2(&device->vk.alloc, allocator, data_buffer);
844 return result;
845 }
846
847 compute_pipeline->pds_cs_data_section = data_buffer;
848
849 /* The base workgroup and num workgroups can be patched in the
850 * PDS data section before dispatch so we save their offsets.
851 */
852 compute_pipeline->base_workgroup_data_patching_offset = ~0u;
853 if (uses_wg_id) {
854 compute_pipeline->base_workgroup_data_patching_offset =
855 program.base_workgroup_constant_offset_in_dwords[0];
856 }
857
858 compute_pipeline->num_workgroups_data_patching_offset = ~0u;
859 compute_pipeline->num_workgroups_indirect_src_patching_offset = ~0u;
860 compute_pipeline->num_workgroups_indirect_src_dma_patching_offset = ~0u;
861 if (uses_num_wgs) {
862 compute_pipeline->num_workgroups_data_patching_offset =
863 program.num_workgroups_constant_offset_in_dwords[0];
864
865 compute_pipeline->num_workgroups_indirect_src_patching_offset =
866 program.num_workgroups_indirect_src;
867
868 compute_pipeline->num_workgroups_indirect_src_dma_patching_offset =
869 program.num_workgroups_indirect_src_dma;
870 }
871
872 compute_pipeline->pds_cs_program_info = (struct pvr_pds_info){
873 .temps_required = program.highest_temp,
874 .code_size_in_dwords = program.code_size,
875 .data_size_in_dwords = program.data_size,
876 };
877
878 vk_free2(&device->vk.alloc, allocator, code_buffer);
879
880 return VK_SUCCESS;
881 }
882
883 static void
884 pvr_pds_compute_program_destroy(struct pvr_device *device,
885 const VkAllocationCallbacks *const allocator,
886 struct pvr_pds_upload *const pds_cs_program,
887 uint32_t *pds_cs_data_section)
888 {
889 pvr_bo_suballoc_free(pds_cs_program->pvr_bo);
890 vk_free2(&device->vk.alloc, allocator, pds_cs_data_section);
891 }
892
893 /******************************************************************************
894 Generic pipeline functions
895 ******************************************************************************/
896
897 static void pvr_pipeline_init(struct pvr_device *device,
898 enum pvr_pipeline_type type,
899 const VkPipelineLayout layout,
900 struct pvr_pipeline *const pipeline)
901 {
902 vk_object_base_init(&device->vk, &pipeline->base, VK_OBJECT_TYPE_PIPELINE);
903
904 pipeline->type = type;
905
906 assert(!pipeline->layout);
907 pipeline->layout = vk_pipeline_layout_from_handle(layout);
908 vk_pipeline_layout_ref(pipeline->layout);
909 }
910
911 static void pvr_pipeline_finish(struct pvr_device *device,
912 struct pvr_pipeline *pipeline)
913 {
914 vk_pipeline_layout_unref(&device->vk, pipeline->layout);
915 vk_object_base_finish(&pipeline->base);
916 }
917
918 static void pvr_early_init_shader_data(pco_data *data,
919 nir_shader *nir,
920 const void *pCreateInfo);
921
922 static void
923 pvr_preprocess_shader_data(pco_data *data,
924 nir_shader *nir,
925 const void *pCreateInfo,
926 struct vk_pipeline_layout *layout,
927 const struct vk_graphics_pipeline_state *state,
928 struct usc_mrt_setup *mrt_setup);
929
930 static void pvr_postprocess_shader_data(pco_data *data,
931 nir_shader *nir,
932 const void *pCreateInfo,
933 struct vk_pipeline_layout *layout,
934 struct usc_mrt_setup *setup);
935
936 struct lower_ycbcr_state {
937 uint32_t set_layout_count;
938 struct vk_descriptor_set_layout *const *set_layouts;
939 };
940
941 static const struct vk_ycbcr_conversion_state *
942 lookup_ycbcr_conversion(const void *_state,
943 uint32_t set,
944 uint32_t binding,
945 uint32_t array_index)
946 {
947 const struct lower_ycbcr_state *state = _state;
948 assert(set < state->set_layout_count);
949 assert(state->set_layouts[set] != NULL);
950 const struct pvr_descriptor_set_layout *set_layout =
951 vk_to_pvr_descriptor_set_layout(state->set_layouts[set]);
952 assert(binding < set_layout->binding_count);
953
954 const struct pvr_descriptor_set_layout_binding *bind_layout =
955 &set_layout->bindings[binding];
956
957 if (bind_layout->immutable_samplers == NULL)
958 return NULL;
959
960 array_index = MIN2(array_index, bind_layout->immutable_sampler_count - 1);
961
962 const struct pvr_sampler *sampler =
963 bind_layout->immutable_samplers[array_index];
964
965 return sampler && sampler->vk.ycbcr_conversion
966 ? &sampler->vk.ycbcr_conversion->state
967 : NULL;
968 }
969
970 /******************************************************************************
971 Compute pipeline functions
972 ******************************************************************************/
973
974 static void
975 pvr_compute_state_save(struct pvr_compute_pipeline *compute_pipeline,
976 pco_shader *cs)
977 {
978 const pco_data *shader_data = pco_shader_data(cs);
979 memcpy(&compute_pipeline->cs_data, shader_data, sizeof(*shader_data));
980 }
981
982 /* Compiles and uploads shaders and PDS programs. */
983 static VkResult pvr_compute_pipeline_compile(
984 struct pvr_device *const device,
985 struct vk_pipeline_cache *cache,
986 const VkComputePipelineCreateInfo *pCreateInfo,
987 const VkAllocationCallbacks *const allocator,
988 struct pvr_compute_pipeline *const compute_pipeline)
989 {
990 struct vk_pipeline_layout *layout = compute_pipeline->base.layout;
991 const uint32_t cache_line_size =
992 pvr_get_slc_cache_line_size(&device->pdevice->dev_info);
993 pco_ctx *pco_ctx = device->pdevice->pco_ctx;
994 void *shader_mem_ctx = ralloc_context(NULL);
995 pco_data shader_data = { 0 };
996 nir_shader *nir;
997 pco_shader *cs;
998
999 struct pvr_compute_shader_state *compute_state =
1000 &compute_pipeline->shader_state;
1001
1002 VkResult result;
1003
1004 result =
1005 vk_pipeline_shader_stage_to_nir(&device->vk,
1006 compute_pipeline->base.pipeline_flags,
1007 &pCreateInfo->stage,
1008 pco_spirv_options(),
1009 pco_nir_options(),
1010 shader_mem_ctx,
1011 &nir);
1012 if (result != VK_SUCCESS)
1013 goto err_free_build_context;
1014
1015 pvr_early_init_shader_data(&shader_data, nir, pCreateInfo);
1016 pco_preprocess_nir(pco_ctx, nir);
1017 pvr_preprocess_shader_data(&shader_data,
1018 nir,
1019 pCreateInfo,
1020 layout,
1021 NULL,
1022 NULL);
1023
1024 const struct lower_ycbcr_state ycbcr_state = {
1025 .set_layout_count = layout->set_count,
1026 .set_layouts = layout->set_layouts,
1027 };
1028 NIR_PASS(_,
1029 nir,
1030 nir_pvr_lower_ycbcr_tex,
1031 lookup_ycbcr_conversion,
1032 &ycbcr_state);
1033 pco_lower_nir(pco_ctx, nir, &shader_data);
1034 pco_postprocess_nir(pco_ctx, nir, &shader_data);
1035 pvr_postprocess_shader_data(&shader_data, nir, pCreateInfo, layout, NULL);
1036
1037 cs = pco_trans_nir(pco_ctx, nir, &shader_data, shader_mem_ctx);
1038 if (!cs) {
1039 result = VK_ERROR_INITIALIZATION_FAILED;
1040 goto err_free_build_context;
1041 }
1042
1043 pco_process_ir(pco_ctx, cs);
1044 pco_encode_ir(pco_ctx, cs);
1045
1046 pvr_compute_state_save(compute_pipeline, cs);
1047
1048 if (pCreateInfo->flags & VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR) {
1049 struct pvr_stats stats = pco_get_pvr_stats(cs);
1050 compute_pipeline->cs_stats =
1051 vk_zalloc2(&device->vk.alloc,
1052 allocator,
1053 sizeof(stats),
1054 8,
1055 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1056
1057 if (!compute_pipeline->cs_stats)
1058 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1059
1060 memcpy(compute_pipeline->cs_stats, &stats, sizeof(stats));
1061 }
1062
1063 if (pCreateInfo->flags &
1064 VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR) {
1065 compute_pipeline->cs_nir_str = nir_shader_as_str(nir, NULL);
1066 }
1067
1068 result = pvr_gpu_upload_usc(device,
1069 pco_shader_binary_data(cs),
1070 pco_shader_binary_size(cs),
1071 cache_line_size,
1072 &compute_pipeline->shader_state.shader_bo);
1073 if (result != VK_SUCCESS)
1074 goto err_free_build_context;
1075
1076 if (compute_pipeline->cs_data.cs.zero_shmem) {
1077 uint32_t start = compute_pipeline->cs_data.cs.shmem.start;
1078 uint32_t count = compute_pipeline->cs_data.cs.shmem.count;
1079 pco_shader *zero_init_shader =
1080 pvr_usc_zero_init_wg_mem(pco_ctx, start, count);
1081
1082 compute_state->coeff_update_shader_temps =
1083 pco_shader_data(zero_init_shader)->common.temps;
1084
1085 result = pvr_gpu_upload_usc(device,
1086 pco_shader_binary_data(zero_init_shader),
1087 pco_shader_binary_size(zero_init_shader),
1088 cache_line_size,
1089 &compute_state->coeff_update_shader_bo);
1090 ralloc_free(zero_init_shader);
1091
1092 if (result != VK_SUCCESS)
1093 goto err_free_shader;
1094 }
1095
1096 result = pvr_pds_descriptor_program_create_and_upload(
1097 device,
1098 allocator,
1099 layout,
1100 MESA_SHADER_COMPUTE,
1101 &compute_pipeline->cs_data,
1102 &compute_pipeline->descriptor_state);
1103 if (result != VK_SUCCESS)
1104 goto err_free_coeff_update_shader;
1105
1106 result = pvr_pds_compute_program_create_and_upload(device,
1107 allocator,
1108 compute_state,
1109 compute_pipeline);
1110 if (result != VK_SUCCESS)
1111 goto err_free_descriptor_program;
1112
1113 ralloc_free(shader_mem_ctx);
1114
1115 return VK_SUCCESS;
1116
1117 err_free_coeff_update_shader:
1118 pvr_bo_suballoc_free(compute_pipeline->shader_state.coeff_update_shader_bo);
1119
1120 err_free_descriptor_program:
1121 pvr_pds_descriptor_program_destroy(device,
1122 allocator,
1123 &compute_pipeline->descriptor_state);
1124
1125 err_free_shader:
1126 pvr_bo_suballoc_free(compute_pipeline->shader_state.shader_bo);
1127
1128 err_free_build_context:
1129 ralloc_free(shader_mem_ctx);
1130 return result;
1131 }
1132
1133 static VkResult
1134 pvr_compute_pipeline_init(struct pvr_device *device,
1135 struct vk_pipeline_cache *cache,
1136 const VkComputePipelineCreateInfo *pCreateInfo,
1137 const VkAllocationCallbacks *allocator,
1138 struct pvr_compute_pipeline *compute_pipeline)
1139 {
1140 VkResult result;
1141
1142 pvr_pipeline_init(device,
1143 PVR_PIPELINE_TYPE_COMPUTE,
1144 pCreateInfo->layout,
1145 &compute_pipeline->base);
1146
1147 result = pvr_compute_pipeline_compile(device,
1148 cache,
1149 pCreateInfo,
1150 allocator,
1151 compute_pipeline);
1152 if (result != VK_SUCCESS) {
1153 pvr_pipeline_finish(device, &compute_pipeline->base);
1154 return result;
1155 }
1156
1157 return VK_SUCCESS;
1158 }
1159
1160 static VkResult
1161 pvr_compute_pipeline_create(struct pvr_device *device,
1162 struct vk_pipeline_cache *cache,
1163 const VkComputePipelineCreateInfo *pCreateInfo,
1164 const VkAllocationCallbacks *allocator,
1165 VkPipeline *const pipeline_out)
1166 {
1167 struct pvr_compute_pipeline *compute_pipeline;
1168 VkResult result;
1169
1170 compute_pipeline = vk_zalloc2(&device->vk.alloc,
1171 allocator,
1172 sizeof(*compute_pipeline),
1173 8,
1174 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
1175 if (!compute_pipeline)
1176 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1177
1178 /* Compiles and uploads shaders and PDS programs. */
1179 result = pvr_compute_pipeline_init(device,
1180 cache,
1181 pCreateInfo,
1182 allocator,
1183 compute_pipeline);
1184 if (result != VK_SUCCESS) {
1185 vk_free2(&device->vk.alloc, allocator, compute_pipeline);
1186 return result;
1187 }
1188
1189 *pipeline_out = pvr_pipeline_to_handle(&compute_pipeline->base);
1190
1191 return VK_SUCCESS;
1192 }
1193
1194 static void pvr_pipeline_destroy_shader_data(pco_data *data);
1195
1196 static void pvr_compute_pipeline_destroy(
1197 struct pvr_device *const device,
1198 const VkAllocationCallbacks *const allocator,
1199 struct pvr_compute_pipeline *const compute_pipeline)
1200 {
1201 pvr_pds_compute_program_destroy(device,
1202 allocator,
1203 &compute_pipeline->pds_cs_program,
1204 compute_pipeline->pds_cs_data_section);
1205 pvr_pds_descriptor_program_destroy(device,
1206 allocator,
1207 &compute_pipeline->descriptor_state);
1208 pvr_bo_suballoc_free(compute_pipeline->shader_state.coeff_update_shader_bo);
1209 pvr_bo_suballoc_free(compute_pipeline->shader_state.shader_bo);
1210
1211 pvr_pipeline_destroy_shader_data(&compute_pipeline->cs_data);
1212
1213 pvr_pipeline_finish(device, &compute_pipeline->base);
1214
1215 vk_free2(&device->vk.alloc, allocator, compute_pipeline->cs_stats);
1216 ralloc_free((void *)compute_pipeline->cs_nir_str);
1217
1218 vk_free2(&device->vk.alloc, allocator, compute_pipeline);
1219 }
1220
1221 VkResult PVR_PER_ARCH(CreateComputePipelines)(
1222 VkDevice _device,
1223 VkPipelineCache pipelineCache,
1224 uint32_t createInfoCount,
1225 const VkComputePipelineCreateInfo *pCreateInfos,
1226 const VkAllocationCallbacks *pAllocator,
1227 VkPipeline *pPipelines)
1228 {
1229 VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
1230 VK_FROM_HANDLE(pvr_device, device, _device);
1231 VkResult result = VK_SUCCESS;
1232
1233 for (uint32_t i = 0; i < createInfoCount; i++) {
1234 const VkResult local_result =
1235 pvr_compute_pipeline_create(device,
1236 cache,
1237 &pCreateInfos[i],
1238 pAllocator,
1239 &pPipelines[i]);
1240 if (local_result != VK_SUCCESS) {
1241 result = local_result;
1242 pPipelines[i] = VK_NULL_HANDLE;
1243 }
1244 }
1245
1246 return result;
1247 }
1248
1249 /******************************************************************************
1250 Graphics pipeline functions
1251 ******************************************************************************/
1252
1253 static void pvr_pipeline_destroy_shader_data(pco_data *data)
1254 {
1255 for (unsigned u = 0; u < ARRAY_SIZE(data->common.desc_sets); ++u) {
1256 ralloc_free(data->common.desc_sets[u].bindings);
1257 }
1258 }
1259
1260 static void
1261 pvr_graphics_pipeline_destroy(struct pvr_device *const device,
1262 const VkAllocationCallbacks *const allocator,
1263 struct pvr_graphics_pipeline *const gfx_pipeline)
1264 {
1265 const uint32_t num_vertex_attrib_programs =
1266 ARRAY_SIZE(gfx_pipeline->shader_state.vertex.pds_attrib_programs);
1267 struct pvr_fragment_shader_state *fragment_state =
1268 &gfx_pipeline->shader_state.fragment;
1269
1270 pvr_pds_descriptor_program_destroy(
1271 device,
1272 allocator,
1273 &gfx_pipeline->shader_state.fragment.descriptor_state);
1274
1275 pvr_pds_descriptor_program_destroy(
1276 device,
1277 allocator,
1278 &gfx_pipeline->shader_state.vertex.descriptor_state);
1279
1280 for (uint32_t i = 0; i < num_vertex_attrib_programs; i++) {
1281 struct pvr_pds_attrib_program *const attrib_program =
1282 &gfx_pipeline->shader_state.vertex.pds_attrib_programs[i];
1283
1284 pvr_pds_vertex_attrib_program_destroy(device, allocator, attrib_program);
1285 }
1286
1287 vk_free2(&device->vk.alloc,
1288 allocator,
1289 fragment_state->pds_fragment_program_buffer);
1290 vk_free2(&device->vk.alloc,
1291 allocator,
1292 fragment_state->pds_coeff_program_buffer);
1293
1294 pvr_bo_suballoc_free(gfx_pipeline->shader_state.fragment.shader_bo);
1295 pvr_bo_suballoc_free(gfx_pipeline->shader_state.vertex.shader_bo);
1296
1297 pvr_pipeline_finish(device, &gfx_pipeline->base);
1298
1299 pvr_pipeline_destroy_shader_data(&gfx_pipeline->vs_data);
1300 pvr_pipeline_destroy_shader_data(&gfx_pipeline->fs_data);
1301
1302 vk_free2(&device->vk.alloc, allocator, gfx_pipeline->vs_stats);
1303 vk_free2(&device->vk.alloc, allocator, gfx_pipeline->fs_stats);
1304
1305 ralloc_free((void *)gfx_pipeline->vs_nir_str);
1306 ralloc_free((void *)gfx_pipeline->fs_nir_str);
1307
1308 vk_free2(&device->vk.alloc, allocator, gfx_pipeline);
1309 }
1310
1311 static void pvr_vertex_state_save(struct pvr_graphics_pipeline *gfx_pipeline,
1312 pco_shader *vs)
1313 {
1314 struct pvr_vertex_shader_state *vertex_state =
1315 &gfx_pipeline->shader_state.vertex;
1316
1317 const pco_data *shader_data = pco_shader_data(vs);
1318 memcpy(&gfx_pipeline->vs_data, shader_data, sizeof(*shader_data));
1319
1320 /* This ends up unused since we'll use the temp_usage for the PDS program we
1321 * end up selecting, and the descriptor PDS program doesn't use any temps.
1322 * Let's set it to ~0 in case it ever gets used.
1323 */
1324 vertex_state->stage_state.pds_temps_count = ~0;
1325 }
1326
1327 static void pvr_fragment_state_save(struct pvr_graphics_pipeline *gfx_pipeline,
1328 pco_shader *fs)
1329 {
1330 struct pvr_fragment_shader_state *fragment_state =
1331 &gfx_pipeline->shader_state.fragment;
1332
1333 const pco_data *shader_data = pco_shader_data(fs);
1334 memcpy(&gfx_pipeline->fs_data, shader_data, sizeof(*shader_data));
1335
1336 /* TODO: add selection for other values of pass type and sample rate. */
1337
1338 /* TODO: do this dynamically as well */
1339 if (shader_data->fs.uses.depth_feedback && !shader_data->fs.uses.early_frag)
1340 fragment_state->pass_type = ROGUE_TA_PASSTYPE_DEPTH_FEEDBACK;
1341 else if (shader_data->fs.uses.discard)
1342 fragment_state->pass_type = ROGUE_TA_PASSTYPE_PUNCH_THROUGH;
1343 else if (shader_data->fs.uses.fbfetch)
1344 fragment_state->pass_type = ROGUE_TA_PASSTYPE_TRANSLUCENT;
1345 else
1346 fragment_state->pass_type = ROGUE_TA_PASSTYPE_OPAQUE;
1347
1348 /* We can't initialize it yet since we still need to generate the PDS
1349 * programs so set it to `~0` to make sure that we set this up later on.
1350 */
1351 fragment_state->stage_state.pds_temps_count = ~0;
1352 }
1353
1354 static void pvr_graphics_pipeline_setup_vertex_dma(
1355 struct pvr_graphics_pipeline *gfx_pipeline,
1356 const VkPipelineVertexInputStateCreateInfo *const vertex_input_state,
1357 const struct vk_vertex_input_state *vi,
1358 struct pvr_pds_vertex_dma *const dma_descriptions,
1359 uint32_t *const dma_count)
1360 {
1361 pco_vs_data *vs_data = &gfx_pipeline->vs_data.vs;
1362
1363 const VkVertexInputBindingDescription
1364 *sorted_bindings[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 };
1365
1366 /* Vertex attributes map to the `layout(location = x)` annotation in the
1367 * shader where `x` is the attribute's location.
1368 * Vertex bindings have NO relation to the shader. They have nothing to do
1369 * with the `layout(set = x, binding = y)` notation. They instead indicate
1370 * where the data for a collection of vertex attributes comes from. The
1371 * application binds a VkBuffer with vkCmdBindVertexBuffers() to a specific
1372 * binding number and based on that we'll know which buffer to DMA the data
1373 * from, to fill in the collection of vertex attributes.
1374 */
1375
1376 for (uint32_t i = 0; i < vertex_input_state->vertexBindingDescriptionCount;
1377 i++) {
1378 const VkVertexInputBindingDescription *binding_desc =
1379 &vertex_input_state->pVertexBindingDescriptions[i];
1380
1381 sorted_bindings[binding_desc->binding] = binding_desc;
1382 }
1383
1384 for (uint32_t i = 0; i < vertex_input_state->vertexAttributeDescriptionCount;
1385 i++) {
1386 const VkVertexInputAttributeDescription *attribute =
1387 &vertex_input_state->pVertexAttributeDescriptions[i];
1388
1389 gl_vert_attrib location = attribute->location + VERT_ATTRIB_GENERIC0;
1390 const VkVertexInputBindingDescription *binding =
1391 sorted_bindings[attribute->binding];
1392 struct pvr_pds_vertex_dma *dma_desc = &dma_descriptions[*dma_count];
1393 const struct util_format_description *fmt_description =
1394 vk_format_description(attribute->format);
1395
1396 const pco_range *attrib_range = &vs_data->attribs[location];
1397
1398 /* Skip unused attributes. */
1399 if (!attrib_range->count)
1400 continue;
1401
1402 /* DMA setup. */
1403
1404 /* The PDS program sets up DDMADs to DMA attributes into vtxin regs.
1405 *
1406 * DDMAD -> Multiply, add, and DOUTD (i.e. DMA from that address).
1407 * DMA source addr = src0 * src1 + src2
1408 * DMA params = src3
1409 *
1410 * In the PDS program we setup src0 with the binding's stride and src1
1411 * with either the instance id or vertex id (both of which get filled by
1412 * the hardware). We setup src2 later on once we know which VkBuffer to
1413 * DMA the data from so it's saved for later when we patch the data
1414 * section.
1415 */
1416
1417 /* TODO: Right now we're setting up a DMA per attribute. In a case where
1418 * there are multiple attributes packed into a single binding with
1419 * adjacent locations we'd still be DMAing them separately. This is not
1420 * great so the DMA setup should be smarter and could do with some
1421 * optimization.
1422 */
1423
1424 *dma_desc = (struct pvr_pds_vertex_dma){ 0 };
1425
1426 /* In relation to the Vulkan spec. 22.4. Vertex Input Address Calculation
1427 * this corresponds to `attribDesc.offset`.
1428 * The PDS program doesn't do anything with it but just save it in the
1429 * PDS program entry.
1430 */
1431 dma_desc->offset = attribute->offset;
1432
1433 /* In relation to the Vulkan spec. 22.4. Vertex Input Address Calculation
1434 * this corresponds to `bindingDesc.stride`.
1435 * The PDS program will calculate the `effectiveVertexOffset` with this
1436 * and add it to the address provided in the patched data segment.
1437 */
1438 dma_desc->stride = binding->stride;
1439
1440 dma_desc->flags = 0;
1441 if (binding->inputRate == VK_VERTEX_INPUT_RATE_INSTANCE)
1442 dma_desc->flags |= PVR_PDS_VERTEX_DMA_FLAGS_INSTANCE_RATE;
1443
1444 /* Size to DMA per vertex attribute. Used to setup src3 in the DDMAD. */
1445 dma_desc->size_in_dwords = attrib_range->count;
1446
1447 /* Vtxin reg offset to start DMAing into. */
1448 dma_desc->destination = attrib_range->start;
1449
1450 /* Will be used by the driver to figure out buffer address to patch in the
1451 * data section. I.e. which binding we should DMA from.
1452 */
1453 dma_desc->binding_index = attribute->binding;
1454
1455 dma_desc->divisor = 1;
1456 if (binding->inputRate == VK_VERTEX_INPUT_RATE_INSTANCE)
1457 dma_desc->divisor = vi->bindings[attribute->binding].divisor;
1458
1459 /* Will be used to generate PDS code that takes care of robust buffer
1460 * access, and later on by the driver to write the correct robustness
1461 * buffer address to DMA the fallback values from.
1462 */
1463 dma_desc->robustness_buffer_offset =
1464 pvr_get_robustness_buffer_format_offset(attribute->format);
1465
1466 /* Used by later on by the driver to figure out if the buffer is being
1467 * accessed out of bounds, for robust buffer access.
1468 */
1469 dma_desc->attrib_size_in_bytes = fmt_description->block.bits / 8;
1470
1471 ++*dma_count;
1472 }
1473 }
1474
1475 static void pvr_graphics_pipeline_setup_fragment_coeff_program(
1476 struct pvr_graphics_pipeline *gfx_pipeline,
1477 nir_shader *fs)
1478 {
1479 struct pvr_fragment_shader_state *fragment_state =
1480 &gfx_pipeline->shader_state.fragment;
1481 struct pvr_pds_coeff_loading_program *frag_coeff_program =
1482 &fragment_state->pds_coeff_program;
1483
1484 uint64_t varyings_used = fs->info.inputs_read &
1485 BITFIELD64_RANGE(VARYING_SLOT_VAR0, MAX_VARYING);
1486 pco_vs_data *vs_data = &gfx_pipeline->vs_data.vs;
1487 pco_fs_data *fs_data = &gfx_pipeline->fs_data.fs;
1488
1489 unsigned fpu = 0;
1490 unsigned dest = 0;
1491
1492 if (fs_data->uses.z) {
1493 pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu],
1494 PDSINST_DOUT_FIELDS_DOUTI_SRC,
1495 douti_src) {
1496 /* TODO: define instead of sizeof(uint16_t). */
1497 douti_src.f32_offset = fs_data->uses.w ? 1 * sizeof(uint16_t) : 0;
1498 douti_src.f16_offset = douti_src.f32_offset;
1499 douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD;
1500 douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_1D;
1501 }
1502
1503 frag_coeff_program->z_iterator = fpu;
1504 frag_coeff_program->destination[fpu++] = dest++;
1505 } else {
1506 frag_coeff_program->z_iterator = ~0u;
1507 }
1508
1509 if (fs_data->uses.w) {
1510 pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu],
1511 PDSINST_DOUT_FIELDS_DOUTI_SRC,
1512 douti_src) {
1513 douti_src.f32_offset = 0;
1514 douti_src.f16_offset = douti_src.f32_offset;
1515 douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD;
1516 douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_1D;
1517 }
1518
1519 frag_coeff_program->destination[fpu++] = dest++;
1520 }
1521
1522 if (fs_data->uses.pntc) {
1523 pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu],
1524 PDSINST_DOUT_FIELDS_DOUTI_SRC,
1525 douti_src) {
1526 douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD;
1527 douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_2D;
1528 douti_src.pointsprite = true;
1529 }
1530
1531 frag_coeff_program->destination[fpu++] = dest;
1532 dest += 2;
1533 }
1534
1535 u_foreach_bit64 (varying, varyings_used) {
1536 nir_variable *var =
1537 nir_find_variable_with_location(fs, nir_var_shader_in, varying);
1538 assert(var);
1539
1540 pco_range *cf_range = &fs_data->varyings[varying];
1541 assert(cf_range->count > 0);
1542 assert(!(cf_range->start % ROGUE_USC_COEFFICIENT_SET_SIZE));
1543 assert(!(cf_range->count % ROGUE_USC_COEFFICIENT_SET_SIZE));
1544
1545 pco_range *vtxout_range = &vs_data->varyings[varying];
1546 assert(vtxout_range->count > 0);
1547 assert(vtxout_range->start >= 4);
1548
1549 if (vtxout_range->count !=
1550 cf_range->count / ROGUE_USC_COEFFICIENT_SET_SIZE) {
1551 pvr_finishme("Mismatch in vs output/fs input counts; "
1552 "amend nir var linking.");
1553 }
1554
1555 unsigned count = vtxout_range->count;
1556
1557 unsigned vtxout = vtxout_range->start;
1558
1559 /* pos.x, pos.y unused. */
1560 vtxout -= 2;
1561
1562 /* pos.z unused. */
1563 if (!fs_data->uses.z)
1564 vtxout -= 1;
1565
1566 /* pos.w unused. */
1567 if (!fs_data->uses.w)
1568 vtxout -= 1;
1569
1570 pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu],
1571 PDSINST_DOUT_FIELDS_DOUTI_SRC,
1572 douti_src) {
1573 /* TODO: define instead of sizeof(uint16_t). */
1574 douti_src.f32_offset = vtxout * sizeof(uint16_t);
1575 /* TODO: f16 support. */
1576 douti_src.f16 = false;
1577 douti_src.f16_offset = douti_src.f32_offset;
1578
1579 switch (var->data.interpolation) {
1580 case INTERP_MODE_SMOOTH:
1581 douti_src.perspective = true;
1582 FALLTHROUGH;
1583
1584 case INTERP_MODE_NOPERSPECTIVE:
1585 douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD;
1586 break;
1587
1588 case INTERP_MODE_FLAT:
1589 /* Shademodel will be set up later for flat. */
1590 BITSET_SET(frag_coeff_program->flat_iter_mask, fpu);
1591 break;
1592
1593 default:
1594 UNREACHABLE("Unimplemented interpolation type.");
1595 }
1596
1597 douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_1D + count - 1;
1598 }
1599
1600 frag_coeff_program->destination[fpu++] =
1601 cf_range->start / ROGUE_USC_COEFFICIENT_SET_SIZE;
1602 }
1603
1604 frag_coeff_program->num_fpu_iterators = fpu;
1605 }
1606
1607 static void set_var(pco_range *allocation_list,
1608 unsigned to,
1609 nir_variable *var,
1610 unsigned dwords_each)
1611 {
1612 unsigned slots = glsl_count_dword_slots(var->type, false);
1613
1614 allocation_list[var->data.location] = (pco_range){
1615 .start = to,
1616 .count = slots * dwords_each,
1617 };
1618 }
1619
1620 static void allocate_var(pco_range *allocation_list,
1621 unsigned *counter,
1622 nir_variable *var,
1623 unsigned dwords_each)
1624 {
1625 unsigned slots = glsl_count_dword_slots(var->type, false);
1626
1627 allocation_list[var->data.location] = (pco_range){
1628 .start = *counter,
1629 .count = slots * dwords_each,
1630 };
1631
1632 *counter += slots * dwords_each;
1633 }
1634
1635 static void try_allocate_var(pco_range *allocation_list,
1636 unsigned *counter,
1637 nir_shader *nir,
1638 uint64_t bitset,
1639 nir_variable_mode mode,
1640 int location,
1641 unsigned dwords_each)
1642 {
1643 if (!(bitset & BITFIELD64_BIT(location)))
1644 return;
1645
1646 nir_variable *var = nir_find_variable_with_location(nir, mode, location);
1647 assert(var);
1648
1649 allocate_var(allocation_list, counter, var, dwords_each);
1650 }
1651
1652 static void allocate_val(pco_range *allocation_list,
1653 unsigned *counter,
1654 unsigned location,
1655 unsigned dwords_each)
1656 {
1657 allocation_list[location] = (pco_range){
1658 .start = *counter,
1659 .count = dwords_each,
1660 };
1661
1662 *counter += dwords_each;
1663 }
1664
1665 static void try_allocate_vars(pco_range *allocation_list,
1666 unsigned *counter,
1667 nir_shader *nir,
1668 uint64_t *bitset,
1669 nir_variable_mode mode,
1670 bool any,
1671 bool f16,
1672 enum glsl_interp_mode interp_mode,
1673 unsigned dwords_each)
1674 {
1675 uint64_t skipped = 0;
1676
1677 while (*bitset) {
1678 int location = u_bit_scan64(bitset);
1679 unsigned accum = 0;
1680
1681 nir_foreach_variable_with_modes (var, nir, mode) {
1682 if (var->data.location != location)
1683 continue;
1684
1685 if (!any && (glsl_type_is_16bit(
1686 glsl_without_array_or_matrix(var->type)) != f16 ||
1687 var->data.interpolation != interp_mode)) {
1688 skipped |= BITFIELD64_BIT(location);
1689 break;
1690 }
1691
1692 accum += glsl_count_dword_slots(var->type, false);
1693 }
1694
1695 if (skipped & BITFIELD64_BIT(location))
1696 continue;
1697
1698 accum *= dwords_each;
1699 allocate_val(allocation_list, counter, location, accum);
1700 }
1701
1702 *bitset |= skipped;
1703 }
1704
1705 static void check_unused_sysvals(
1706 BITSET_WORD system_values_read[static BITSET_WORDS(SYSTEM_VALUE_MAX)])
1707 {
1708 #if MESA_DEBUG
1709 if (!__bitset_is_empty(system_values_read, BITSET_WORDS(SYSTEM_VALUE_MAX))) {
1710 gl_system_value sysval;
1711 BITSET_FOREACH_SET (sysval, system_values_read, SYSTEM_VALUE_MAX) {
1712 debug_printf("unhandled sysval: %s\n", gl_system_value_name(sysval));
1713 }
1714 }
1715 #endif
1716 }
1717
1718 static void pvr_alloc_vs_sysvals(pco_data *data, nir_shader *nir)
1719 {
1720 BITSET_DECLARE(system_values_read, SYSTEM_VALUE_MAX);
1721 BITSET_COPY(system_values_read, nir->info.system_values_read);
1722
1723 gl_system_value sys_vals[] = {
1724 SYSTEM_VALUE_VERTEX_ID, SYSTEM_VALUE_INSTANCE_ID,
1725 SYSTEM_VALUE_BASE_INSTANCE, SYSTEM_VALUE_BASE_VERTEX,
1726 SYSTEM_VALUE_DRAW_ID, SYSTEM_VALUE_VIEW_INDEX,
1727 };
1728
1729 for (unsigned u = 0; u < ARRAY_SIZE(sys_vals); ++u) {
1730 if (BITSET_TEST(system_values_read, sys_vals[u])) {
1731 nir_intrinsic_op op = nir_intrinsic_from_system_value(sys_vals[u]);
1732 unsigned dwords = nir_intrinsic_infos[op].dest_components;
1733 assert(dwords > 0);
1734
1735 allocate_val(data->common.sys_vals,
1736 &data->common.vtxins,
1737 sys_vals[u],
1738 dwords);
1739
1740 BITSET_CLEAR(system_values_read, sys_vals[u]);
1741 }
1742 }
1743
1744 check_unused_sysvals(system_values_read);
1745 assert(BITSET_IS_EMPTY(system_values_read));
1746 }
1747
1748 static void pvr_init_vs_attribs(
1749 pco_data *data,
1750 const VkPipelineVertexInputStateCreateInfo *const vertex_input_state)
1751 {
1752 for (unsigned u = 0; u < vertex_input_state->vertexAttributeDescriptionCount;
1753 ++u) {
1754 const VkVertexInputAttributeDescription *attrib =
1755 &vertex_input_state->pVertexAttributeDescriptions[u];
1756
1757 gl_vert_attrib location = attrib->location + VERT_ATTRIB_GENERIC0;
1758
1759 data->vs.attrib_formats[location] =
1760 vk_format_to_pipe_format(attrib->format);
1761 }
1762 }
1763
1764 static void pvr_alloc_vs_attribs(pco_data *data, nir_shader *nir)
1765 {
1766 nir_foreach_shader_in_variable (var, nir) {
1767 allocate_var(data->vs.attribs, &data->common.vtxins, var, 1);
1768 }
1769 }
1770
1771 static void pvr_alloc_vs_varyings(pco_data *data, nir_shader *nir)
1772 {
1773 uint64_t vars_mask = nir->info.outputs_written &
1774 BITFIELD64_RANGE(VARYING_SLOT_VAR0, MAX_VARYING);
1775
1776 /* Varying ordering is specific. */
1777 try_allocate_var(data->vs.varyings,
1778 &data->vs.vtxouts,
1779 nir,
1780 nir->info.outputs_written,
1781 nir_var_shader_out,
1782 VARYING_SLOT_POS,
1783 1);
1784
1785 /* Save varying counts. */
1786 nir_foreach_variable_with_modes (var, nir, nir_var_shader_out) {
1787 if (var->data.location < VARYING_SLOT_VAR0 ||
1788 var->data.location >= VARYING_SLOT_VAR0 + MAX_VARYING) {
1789 continue;
1790 }
1791
1792 /* TODO: f16 support. */
1793 bool f16 = glsl_type_is_16bit(glsl_without_array_or_matrix(var->type));
1794 assert(!f16);
1795 unsigned components = glsl_get_components(var->type);
1796
1797 switch (var->data.interpolation) {
1798 case INTERP_MODE_NONE:
1799 /* pco_rev_link_nir didn't run; override here. */
1800 var->data.interpolation = INTERP_MODE_SMOOTH;
1801 FALLTHROUGH;
1802
1803 case INTERP_MODE_SMOOTH:
1804 if (f16)
1805 data->vs.f16_smooth += components;
1806 else
1807 data->vs.f32_smooth += components;
1808
1809 break;
1810
1811 case INTERP_MODE_FLAT:
1812 if (f16)
1813 data->vs.f16_flat += components;
1814 else
1815 data->vs.f32_flat += components;
1816
1817 break;
1818
1819 case INTERP_MODE_NOPERSPECTIVE:
1820 if (f16)
1821 data->vs.f16_npc += components;
1822 else
1823 data->vs.f32_npc += components;
1824
1825 break;
1826
1827 default:
1828 UNREACHABLE("");
1829 }
1830 }
1831
1832 for (unsigned f16 = 0; f16 <= 1; ++f16) {
1833 for (enum glsl_interp_mode interp_mode = INTERP_MODE_SMOOTH;
1834 interp_mode <= INTERP_MODE_NOPERSPECTIVE;
1835 ++interp_mode) {
1836 try_allocate_vars(data->vs.varyings,
1837 &data->vs.vtxouts,
1838 nir,
1839 &vars_mask,
1840 nir_var_shader_out,
1841 false,
1842 f16,
1843 interp_mode,
1844 1);
1845 }
1846 }
1847
1848 assert(!vars_mask);
1849
1850 const gl_varying_slot last_slots[] = {
1851 VARYING_SLOT_PSIZ, VARYING_SLOT_VIEWPORT, VARYING_SLOT_LAYER,
1852 VARYING_SLOT_CLIP_DIST0, VARYING_SLOT_CLIP_DIST1,
1853 };
1854
1855 for (unsigned u = 0; u < ARRAY_SIZE(last_slots); ++u) {
1856 try_allocate_var(data->vs.varyings,
1857 &data->vs.vtxouts,
1858 nir,
1859 nir->info.outputs_written,
1860 nir_var_shader_out,
1861 last_slots[u],
1862 1);
1863 }
1864 }
1865
1866 static void pvr_alloc_fs_sysvals(pco_data *data, nir_shader *nir)
1867 {
1868 BITSET_DECLARE(system_values_read, SYSTEM_VALUE_MAX);
1869 BITSET_COPY(system_values_read, nir->info.system_values_read);
1870
1871 gl_system_value sys_vals[] = {
1872 SYSTEM_VALUE_FRONT_FACE,
1873 };
1874
1875 for (unsigned u = 0; u < ARRAY_SIZE(sys_vals); ++u) {
1876 if (BITSET_TEST(system_values_read, sys_vals[u])) {
1877 nir_intrinsic_op op = nir_intrinsic_from_system_value(sys_vals[u]);
1878 unsigned dwords = nir_intrinsic_infos[op].dest_components;
1879 assert(dwords > 0);
1880
1881 allocate_val(data->common.sys_vals,
1882 &data->common.shareds,
1883 sys_vals[u],
1884 dwords);
1885
1886 BITSET_CLEAR(system_values_read, sys_vals[u]);
1887 }
1888 }
1889
1890 /* Clear built-in sysvals. */
1891 gl_system_value builtin_sys_vals[] = {
1892 SYSTEM_VALUE_SAMPLE_ID,
1893 SYSTEM_VALUE_LAYER_ID,
1894 SYSTEM_VALUE_SAMPLE_POS,
1895 SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID,
1896 SYSTEM_VALUE_BARYCENTRIC_LINEAR_COORD,
1897 SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL,
1898 SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE,
1899 SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID,
1900 SYSTEM_VALUE_BARYCENTRIC_PERSP_COORD,
1901 SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL,
1902 SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE,
1903 SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL,
1904 };
1905
1906 for (unsigned u = 0; u < ARRAY_SIZE(builtin_sys_vals); ++u)
1907 BITSET_CLEAR(system_values_read, builtin_sys_vals[u]);
1908
1909 check_unused_sysvals(system_values_read);
1910 assert(BITSET_IS_EMPTY(system_values_read));
1911
1912 data->fs.meta = (pco_range){
1913 .start = data->common.shareds,
1914 .count = 1,
1915 };
1916
1917 ++data->common.shareds;
1918 }
1919
1920 static void pvr_alloc_fs_varyings(pco_data *data, nir_shader *nir)
1921 {
1922 uint64_t vars_mask = nir->info.inputs_read &
1923 BITFIELD64_RANGE(VARYING_SLOT_VAR0, MAX_VARYING);
1924
1925 assert(!data->common.coeffs);
1926
1927 /* Save the z/w locations. */
1928 unsigned zw_count = !!data->fs.uses.z + !!data->fs.uses.w;
1929 allocate_val(data->fs.varyings,
1930 &data->common.coeffs,
1931 VARYING_SLOT_POS,
1932 zw_count * ROGUE_USC_COEFFICIENT_SET_SIZE);
1933
1934 /* If point coords are used, they come after z/w (if present). */
1935 nir_variable *var = nir_find_variable_with_location(nir,
1936 nir_var_shader_in,
1937 VARYING_SLOT_PNTC);
1938 if (var) {
1939 assert(!var->data.location_frac);
1940 unsigned count = glsl_get_components(var->type);
1941 assert(count == 2);
1942
1943 allocate_var(data->fs.varyings,
1944 &data->common.coeffs,
1945 var,
1946 ROGUE_USC_COEFFICIENT_SET_SIZE);
1947
1948 data->fs.uses.pntc = true;
1949 }
1950
1951 /* Allocate the rest of the input varyings. */
1952 try_allocate_vars(data->fs.varyings,
1953 &data->common.coeffs,
1954 nir,
1955 &vars_mask,
1956 nir_var_shader_in,
1957 true,
1958 false,
1959 0,
1960 ROGUE_USC_COEFFICIENT_SET_SIZE);
1961 }
1962
1963 static void
1964 pvr_init_fs_outputs(pco_data *data,
1965 const struct pvr_render_pass *pass,
1966 const struct pvr_render_subpass *const subpass,
1967 const struct pvr_renderpass_hwsetup_subpass *hw_subpass)
1968 {
1969 unsigned u;
1970 pco_fs_data *fs = &data->fs;
1971
1972 for (u = 0; u < subpass->color_count; ++u) {
1973 unsigned idx = subpass->color_attachments[u];
1974 const struct usc_mrt_resource *mrt_resource;
1975 bool tile_buffer;
1976
1977 if (idx == VK_ATTACHMENT_UNUSED)
1978 continue;
1979
1980 gl_frag_result location = FRAG_RESULT_DATA0 + u;
1981 VkFormat vk_format = pass->attachments[idx].vk_format;
1982 fs->output_formats[location] = vk_format_to_pipe_format(vk_format);
1983
1984 mrt_resource = &hw_subpass->setup.mrt_resources[u];
1985 tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
1986
1987 if (tile_buffer) {
1988 fs->num_tile_buffers =
1989 MAX2(fs->num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
1990 fs->output_tile_buffers |= BITFIELD_BIT(u);
1991 }
1992 }
1993
1994 fs->z_replicate = ~0u;
1995 if (hw_subpass->z_replicate >= 0) {
1996 gl_frag_result location = FRAG_RESULT_DATA0 + u;
1997 const struct usc_mrt_resource *mrt_resource =
1998 &hw_subpass->setup.mrt_resources[u];
1999 bool tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
2000
2001 fs->output_formats[location] = PIPE_FORMAT_R32_FLOAT;
2002 fs->z_replicate = location;
2003
2004 if (tile_buffer) {
2005 fs->num_tile_buffers =
2006 MAX2(fs->num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
2007 fs->output_tile_buffers |= BITFIELD_BIT(u);
2008 }
2009 }
2010 }
2011
2012 static void pvr_init_fs_outputs_mrt(pco_data *data,
2013 const struct vk_render_pass_state *rp,
2014 const struct usc_mrt_setup *setup)
2015 {
2016 unsigned u;
2017 pco_fs_data *fs = &data->fs;
2018
2019 if (!vk_render_pass_state_has_attachment_info(rp))
2020 goto early_exit;
2021
2022 for (u = 0; u < PVR_MAX_COLOR_ATTACHMENTS; u++) {
2023 if (!(rp->attachments & MESA_VK_RP_ATTACHMENT_COLOR_BIT(u)))
2024 continue;
2025
2026 const struct usc_mrt_resource *mrt_resource = &setup->mrt_resources[u];
2027 bool tile_buffer;
2028
2029 gl_frag_result location = FRAG_RESULT_DATA0 + u;
2030 VkFormat vk_format = rp->color_attachment_formats[u];
2031 fs->output_formats[location] = vk_format_to_pipe_format(vk_format);
2032
2033 tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
2034
2035 if (tile_buffer) {
2036 fs->num_tile_buffers =
2037 MAX2(fs->num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
2038 fs->output_tile_buffers |= BITFIELD_BIT(u);
2039 }
2040 }
2041
2042 early_exit:
2043 fs->z_replicate = ~0u;
2044 }
2045
2046 static void
2047 pvr_setup_fs_outputs(pco_data *data,
2048 nir_shader *nir,
2049 const struct pvr_render_subpass *const subpass,
2050 const struct pvr_renderpass_hwsetup_subpass *hw_subpass)
2051 {
2052 uint64_t outputs_written = nir->info.outputs_written;
2053 pco_fs_data *fs = &data->fs;
2054
2055 unsigned u;
2056 for (u = 0; u < subpass->color_count; ++u) {
2057 gl_frag_result location = FRAG_RESULT_DATA0 + u;
2058 unsigned idx = subpass->color_attachments[u];
2059 const struct usc_mrt_resource *mrt_resource;
2060 bool tile_buffer;
2061 nir_variable *var;
2062
2063 if (idx == VK_ATTACHMENT_UNUSED)
2064 continue;
2065
2066 var = nir_find_variable_with_location(nir, nir_var_shader_out, location);
2067 if (!var)
2068 continue;
2069
2070 mrt_resource = &hw_subpass->setup.mrt_resources[u];
2071 tile_buffer = fs->output_tile_buffers & BITFIELD_BIT(u);
2072
2073 set_var(fs->outputs,
2074 tile_buffer ? mrt_resource->mem.tile_buffer
2075 : mrt_resource->reg.output_reg,
2076 var,
2077 DIV_ROUND_UP(mrt_resource->intermediate_size, sizeof(uint32_t)));
2078
2079 if (tile_buffer)
2080 fs->outputs[location].offset = mrt_resource->mem.offset_dw;
2081
2082 outputs_written &= ~BITFIELD64_BIT(location);
2083 }
2084
2085 if (hw_subpass->z_replicate >= 0) {
2086 const struct usc_mrt_resource *mrt_resource =
2087 &hw_subpass->setup.mrt_resources[hw_subpass->z_replicate];
2088 gl_frag_result location = FRAG_RESULT_DATA0 + u;
2089 nir_variable *var =
2090 nir_find_variable_with_location(nir, nir_var_shader_out, location);
2091 if (var) {
2092 bool tile_buffer = fs->output_tile_buffers & BITFIELD_BIT(u);
2093
2094 set_var(fs->outputs,
2095 tile_buffer ? mrt_resource->mem.tile_buffer
2096 : mrt_resource->reg.output_reg,
2097 var,
2098 DIV_ROUND_UP(mrt_resource->intermediate_size,
2099 sizeof(uint32_t)));
2100
2101 if (tile_buffer)
2102 fs->outputs[location].offset = mrt_resource->mem.offset_dw;
2103
2104 outputs_written &= ~BITFIELD64_BIT(location);
2105 }
2106 }
2107
2108 assert(!outputs_written);
2109 }
2110
2111 static void pvr_setup_fs_outputs_mrt(pco_data *data,
2112 nir_shader *nir,
2113 const struct usc_mrt_setup *setup)
2114 {
2115 uint64_t outputs_written = nir->info.outputs_written;
2116 pco_fs_data *fs = &data->fs;
2117
2118 unsigned u;
2119 for (u = 0; u < setup->num_render_targets; u++) {
2120 gl_frag_result location = FRAG_RESULT_DATA0 + u;
2121 const struct usc_mrt_resource *mrt_resource = &setup->mrt_resources[u];
2122 bool tile_buffer;
2123 nir_variable *var;
2124
2125 var = nir_find_variable_with_location(nir, nir_var_shader_out, location);
2126 if (!var)
2127 continue;
2128
2129 tile_buffer = fs->output_tile_buffers & BITFIELD_BIT(u);
2130
2131 set_var(fs->outputs,
2132 tile_buffer ? mrt_resource->mem.tile_buffer
2133 : mrt_resource->reg.output_reg,
2134 var,
2135 DIV_ROUND_UP(mrt_resource->intermediate_size, sizeof(uint32_t)));
2136
2137 if (tile_buffer)
2138 fs->outputs[location].offset = mrt_resource->mem.offset_dw;
2139
2140 outputs_written &= ~BITFIELD64_BIT(location);
2141 }
2142
2143 assert(!outputs_written);
2144 }
2145
2146 static void pvr_init_fs_input_attachments(
2147 pco_data *data,
2148 const struct pvr_render_pass *pass,
2149 const struct pvr_render_subpass *const subpass,
2150 const struct pvr_renderpass_hwsetup_subpass *hw_subpass)
2151 {
2152 pco_fs_data *fs = &data->fs;
2153 for (unsigned u = 0; u < subpass->input_count; ++u) {
2154 unsigned idx = subpass->input_attachments[u].attachment_idx;
2155 if (idx == VK_ATTACHMENT_UNUSED)
2156 continue;
2157
2158 bool onchip = hw_subpass->input_access[u].type !=
2159 PVR_RENDERPASS_HWSETUP_INPUT_ACCESS_OFFCHIP;
2160 if (!onchip)
2161 continue;
2162
2163 VkFormat vk_format = pass->attachments[idx].vk_format;
2164 bool has_stencil = vk_format_has_stencil(vk_format);
2165 if (hw_subpass->input_access[u].type ==
2166 PVR_RENDERPASS_HWSETUP_INPUT_ACCESS_ONCHIP_ZREPLICATE) {
2167 vk_format = VK_FORMAT_R32_SFLOAT;
2168 }
2169
2170 fs->ia_formats[u] = vk_format_to_pipe_format(vk_format);
2171 assert(fs->ia_formats[u] != PIPE_FORMAT_NONE);
2172 if (has_stencil)
2173 fs->ia_has_stencil |= BITFIELD_BIT(u);
2174
2175 unsigned mrt_idx = hw_subpass->input_access[u].on_chip_rt;
2176 const struct usc_mrt_resource *mrt_resource =
2177 &hw_subpass->setup.mrt_resources[mrt_idx];
2178
2179 bool tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
2180
2181 if (tile_buffer) {
2182 fs->num_tile_buffers =
2183 MAX2(fs->num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
2184 fs->ia_tile_buffers |= BITFIELD_BIT(u);
2185 }
2186 }
2187 }
2188
2189 static void
2190 pvr_init_fs_input_attachments_mrt(pco_data *data,
2191 const struct vk_render_pass_state *rp,
2192 const struct usc_mrt_setup *setup)
2193 {
2194 pco_fs_data *fs = &data->fs;
2195 for (unsigned u = 0; u < rp->color_attachment_count; u++) {
2196 VkFormat vk_format = rp->color_attachment_formats[u];
2197 bool has_stencil = vk_format_has_stencil(vk_format);
2198
2199 if (vk_format == VK_FORMAT_UNDEFINED)
2200 continue;
2201
2202 fs->ia_formats[u] = vk_format_to_pipe_format(vk_format);
2203 assert(fs->ia_formats[u] != PIPE_FORMAT_NONE);
2204 if (has_stencil)
2205 fs->ia_has_stencil |= BITFIELD_BIT(u);
2206
2207 const struct usc_mrt_resource *mrt_resource = &setup->mrt_resources[u];
2208 bool tile_buffer = mrt_resource->type != USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
2209
2210 if (tile_buffer) {
2211 fs->num_tile_buffers =
2212 MAX2(fs->num_tile_buffers, mrt_resource->mem.tile_buffer + 1);
2213 fs->ia_tile_buffers |= BITFIELD_BIT(u);
2214 }
2215 }
2216 }
2217
2218 static void pvr_init_fs_blend(pco_data *data,
2219 const struct vk_color_blend_state *cb)
2220 {
2221 nir_lower_blend_options *blend_opts = &data->fs.blend_opts;
2222 if (!cb)
2223 return;
2224
2225 blend_opts->logicop_enable = cb->logic_op_enable;
2226 blend_opts->logicop_func = vk_logic_op_to_pipe(cb->logic_op);
2227
2228 unsigned count = cb->attachment_count;
2229 for (unsigned u = 0; u < count; ++u) {
2230 const struct vk_color_blend_attachment_state *rt = &cb->attachments[u];
2231 gl_frag_result location = FRAG_RESULT_DATA0 + u;
2232 blend_opts->rt[u].format = data->fs.output_formats[location];
2233
2234 if (cb->logic_op_enable) {
2235 /* No blending, but we get the colour mask below */
2236 } else if (!rt->blend_enable) {
2237 const nir_lower_blend_channel replace = {
2238 .func = PIPE_BLEND_ADD,
2239 .src_factor = PIPE_BLENDFACTOR_ONE,
2240 .dst_factor = PIPE_BLENDFACTOR_ZERO,
2241 };
2242
2243 blend_opts->rt[u].rgb = replace;
2244 blend_opts->rt[u].alpha = replace;
2245 } else {
2246 blend_opts->rt[u].rgb.func = vk_blend_op_to_pipe(rt->color_blend_op);
2247 blend_opts->rt[u].rgb.src_factor =
2248 vk_blend_factor_to_pipe(rt->src_color_blend_factor);
2249 blend_opts->rt[u].rgb.dst_factor =
2250 vk_blend_factor_to_pipe(rt->dst_color_blend_factor);
2251
2252 blend_opts->rt[u].alpha.func = vk_blend_op_to_pipe(rt->alpha_blend_op);
2253 blend_opts->rt[u].alpha.src_factor =
2254 vk_blend_factor_to_pipe(rt->src_alpha_blend_factor);
2255 blend_opts->rt[u].alpha.dst_factor =
2256 vk_blend_factor_to_pipe(rt->dst_alpha_blend_factor);
2257 }
2258
2259 blend_opts->rt[u].colormask = rt->write_mask;
2260 }
2261 }
2262
2263 static void pvr_setup_fs_input_attachments(
2264 pco_data *data,
2265 nir_shader *nir,
2266 const struct pvr_render_subpass *const subpass,
2267 const struct pvr_renderpass_hwsetup_subpass *hw_subpass)
2268 {
2269 pco_fs_data *fs = &data->fs;
2270 for (unsigned u = 0; u < subpass->input_count; ++u) {
2271 unsigned idx = subpass->input_attachments[u].attachment_idx;
2272 if (idx == VK_ATTACHMENT_UNUSED)
2273 continue;
2274
2275 bool onchip = hw_subpass->input_access[u].type !=
2276 PVR_RENDERPASS_HWSETUP_INPUT_ACCESS_OFFCHIP;
2277 if (!onchip)
2278 continue;
2279
2280 unsigned mrt_idx = hw_subpass->input_access[u].on_chip_rt;
2281 const struct usc_mrt_resource *mrt_resource =
2282 &hw_subpass->setup.mrt_resources[mrt_idx];
2283
2284 bool tile_buffer = fs->ia_tile_buffers & BITFIELD_BIT(u);
2285
2286 fs->ias_onchip[u] = (pco_range){
2287 .start = tile_buffer ? mrt_resource->mem.tile_buffer
2288 : mrt_resource->reg.output_reg,
2289 .count =
2290 DIV_ROUND_UP(mrt_resource->intermediate_size, sizeof(uint32_t)),
2291 };
2292
2293 if (tile_buffer)
2294 fs->ias_onchip[u].offset = mrt_resource->mem.offset_dw;
2295 }
2296 }
2297
2298 static void pvr_setup_fs_input_attachments_mrt(pco_data *data,
2299 nir_shader *nir,
2300 struct usc_mrt_setup *setup)
2301 {
2302 pco_fs_data *fs = &data->fs;
2303 for (unsigned u = 0; u < setup->num_render_targets; ++u) {
2304 const struct usc_mrt_resource *mrt_resource = &setup->mrt_resources[u];
2305
2306 bool tile_buffer = fs->ia_tile_buffers & BITFIELD_BIT(u);
2307
2308 fs->ias_onchip[u] = (pco_range){
2309 .start = tile_buffer ? mrt_resource->mem.tile_buffer
2310 : mrt_resource->reg.output_reg,
2311 .count =
2312 DIV_ROUND_UP(mrt_resource->intermediate_size, sizeof(uint32_t)),
2313 };
2314
2315 if (tile_buffer)
2316 fs->ias_onchip[u].offset = mrt_resource->mem.offset_dw;
2317 }
2318 }
2319
2320 static void pvr_setup_fs_blend(pco_data *data)
2321 {
2322 unsigned num_blend_consts = util_bitcount(data->fs.blend_consts_needed);
2323 if (!num_blend_consts)
2324 return;
2325
2326 data->fs.blend_consts = (pco_range){
2327 .start = data->common.shareds,
2328 .count = num_blend_consts,
2329 };
2330
2331 data->common.shareds += num_blend_consts;
2332 }
2333
2334 static void pvr_init_fs_tile_buffers(pco_data *data)
2335 {
2336 if (!data->fs.num_tile_buffers)
2337 return;
2338
2339 unsigned tile_buffer_addr_dwords =
2340 data->fs.num_tile_buffers * (sizeof(uint64_t) / sizeof(uint32_t));
2341
2342 data->fs.tile_buffers = (pco_range){
2343 .count = tile_buffer_addr_dwords,
2344 .stride = sizeof(uint64_t) / sizeof(uint32_t),
2345 };
2346 }
2347
2348 static void pvr_setup_fs_tile_buffers(pco_data *data)
2349 {
2350 if (!data->fs.tile_buffers.count)
2351 return;
2352
2353 data->fs.tile_buffers.start = data->common.shareds;
2354 data->common.shareds += data->fs.tile_buffers.count;
2355 }
2356
2357 static void pvr_setup_fs_sample_locations(pco_data *data)
2358 {
2359 if (!data->fs.uses.sample_locations)
2360 return;
2361
2362 data->fs.sample_locations.start = data->common.shareds;
2363 data->fs.sample_locations.count = 4; /* TODO */
2364 data->common.shareds += data->fs.sample_locations.count;
2365 }
2366
2367 static void pvr_alloc_cs_sysvals(pco_data *data, nir_shader *nir)
2368 {
2369 BITSET_DECLARE(system_values_read, SYSTEM_VALUE_MAX);
2370 BITSET_COPY(system_values_read, nir->info.system_values_read);
2371
2372 gl_system_value vtxin_sys_vals[] = {
2373 SYSTEM_VALUE_LOCAL_INVOCATION_INDEX,
2374 };
2375
2376 gl_system_value coeff_sys_vals[] = {
2377 SYSTEM_VALUE_WORKGROUP_ID,
2378 SYSTEM_VALUE_NUM_WORKGROUPS,
2379 };
2380
2381 for (unsigned u = 0; u < ARRAY_SIZE(vtxin_sys_vals); ++u) {
2382 if (BITSET_TEST(system_values_read, vtxin_sys_vals[u])) {
2383 nir_intrinsic_op op =
2384 nir_intrinsic_from_system_value(vtxin_sys_vals[u]);
2385 unsigned dwords = nir_intrinsic_infos[op].dest_components;
2386 assert(dwords > 0);
2387
2388 allocate_val(data->common.sys_vals,
2389 &data->common.vtxins,
2390 vtxin_sys_vals[u],
2391 dwords);
2392
2393 BITSET_CLEAR(system_values_read, vtxin_sys_vals[u]);
2394 }
2395 }
2396
2397 for (unsigned u = 0; u < ARRAY_SIZE(coeff_sys_vals); ++u) {
2398 if (BITSET_TEST(system_values_read, coeff_sys_vals[u])) {
2399 nir_intrinsic_op op =
2400 nir_intrinsic_from_system_value(coeff_sys_vals[u]);
2401 unsigned dwords = nir_intrinsic_infos[op].dest_components;
2402 assert(dwords > 0);
2403
2404 if (dwords > 1 && data->common.coeffs & 1)
2405 ++data->common.coeffs;
2406
2407 allocate_val(data->common.sys_vals,
2408 &data->common.coeffs,
2409 coeff_sys_vals[u],
2410 dwords);
2411
2412 BITSET_CLEAR(system_values_read, coeff_sys_vals[u]);
2413 }
2414 }
2415
2416 check_unused_sysvals(system_values_read);
2417 assert(BITSET_IS_EMPTY(system_values_read));
2418 }
2419
2420 static void pvr_alloc_cs_shmem(pco_data *data, nir_shader *nir)
2421 {
2422 assert(!nir->info.cs.has_variable_shared_mem);
2423
2424 data->cs.shmem.start = data->common.coeffs;
2425 data->cs.shmem.count = nir->info.shared_size >> 2;
2426 data->common.coeffs += data->cs.shmem.count;
2427 data->cs.zero_shmem = nir->info.zero_initialize_shared_memory;
2428 }
2429
2430 static void pvr_init_descriptors(pco_data *data,
2431 nir_shader *nir,
2432 struct vk_pipeline_layout *layout)
2433 {
2434 const struct pvr_device *device = vk_to_pvr_device(layout->base.device);
2435 data->common.robust_buffer_access =
2436 device->vk.enabled_features.robustBufferAccess;
2437
2438 data->common.null_descriptor = device->vk.enabled_features.nullDescriptor;
2439
2440 data->common.image_2d_view_of_3d =
2441 device->vk.enabled_features.image2DViewOf3D;
2442
2443 for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) {
2444 const struct pvr_descriptor_set_layout *set_layout =
2445 vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]);
2446 pco_descriptor_set_data *desc_set_data =
2447 &data->common.desc_sets[desc_set];
2448
2449 /* If the descriptor set isn't for this stage, skip it. */
2450 if (!(BITFIELD_BIT(nir->info.stage) & set_layout->stage_flags))
2451 continue;
2452
2453 desc_set_data->binding_count = set_layout->binding_count;
2454 desc_set_data->bindings =
2455 rzalloc_array_size(NULL,
2456 sizeof(*desc_set_data->bindings),
2457 set_layout->binding_count);
2458
2459 for (unsigned binding = 0; binding < set_layout->binding_count;
2460 ++binding) {
2461 const struct pvr_descriptor_set_layout_binding *layout_binding =
2462 &set_layout->bindings[binding];
2463 pco_binding_data *binding_data = &desc_set_data->bindings[binding];
2464
2465 binding_data->is_img_smp = layout_binding->type ==
2466 VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
2467 }
2468 }
2469 }
2470
2471 static void pvr_setup_descriptors(pco_data *data,
2472 nir_shader *nir,
2473 struct vk_pipeline_layout *layout)
2474 {
2475 mesa_shader_stage stage = nir->info.stage;
2476
2477 data->common.shareds = ALIGN_POT(data->common.shareds, 4);
2478
2479 /* Allocate shareds for the descriptors. */
2480 for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) {
2481 const struct pvr_descriptor_set_layout *set_layout =
2482 vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]);
2483 const unsigned desc_set_size_dw = set_layout->size / sizeof(uint32_t);
2484 pco_descriptor_set_data *desc_set_data =
2485 &data->common.desc_sets[desc_set];
2486 pco_range *desc_set_range = &desc_set_data->range;
2487
2488 assert(!(set_layout->size % sizeof(uint32_t)));
2489
2490 /* If the descriptor set isn't for this stage or is unused, skip it. */
2491 if (!(BITFIELD_BIT(stage) & set_layout->stage_flags)) {
2492 assert(!desc_set_data->used);
2493 continue;
2494 }
2495
2496 if (!desc_set_data->used)
2497 continue;
2498
2499 desc_set_range->start = data->common.shareds;
2500 desc_set_range->count = desc_set_size_dw;
2501 data->common.shareds += desc_set_size_dw;
2502
2503 for (unsigned binding = 0; binding < set_layout->binding_count;
2504 ++binding) {
2505 const struct pvr_descriptor_set_layout_binding *layout_binding =
2506 &set_layout->bindings[binding];
2507 pco_binding_data *binding_data = &desc_set_data->bindings[binding];
2508
2509 /* Skip dynamic buffer bindings. */
2510 if (layout_binding->offset == ~0)
2511 continue;
2512
2513 binding_data->range = (pco_range){
2514 .start = desc_set_range->start +
2515 (layout_binding->offset / sizeof(uint32_t)),
2516 .count =
2517 (layout_binding->stride * layout_binding->descriptor_count) /
2518 sizeof(uint32_t),
2519 .stride = layout_binding->stride / sizeof(uint32_t),
2520 };
2521 }
2522 }
2523
2524 /* Allocate shareds for the dynamic descriptors. */
2525 for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) {
2526 const struct pvr_descriptor_set_layout *set_layout =
2527 vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]);
2528 const unsigned desc_set_dynamic_size_dw =
2529 (set_layout->dynamic_buffer_count *
2530 sizeof(struct pvr_buffer_descriptor)) /
2531 sizeof(uint32_t);
2532 pco_descriptor_set_data *desc_set_data =
2533 &data->common.desc_sets[desc_set];
2534 pco_range *desc_set_dynamic_range = &desc_set_data->dynamic_range;
2535
2536 if (!desc_set_dynamic_size_dw)
2537 continue;
2538
2539 /* If the descriptor set isn't for this stage or is unused, skip it. */
2540 if (!(BITFIELD_BIT(stage) & set_layout->stage_flags)) {
2541 assert(!desc_set_data->used);
2542 continue;
2543 }
2544
2545 if (!desc_set_data->used)
2546 continue;
2547
2548 desc_set_dynamic_range->start = data->common.shareds;
2549 desc_set_dynamic_range->count = desc_set_dynamic_size_dw;
2550 data->common.shareds += desc_set_dynamic_size_dw;
2551
2552 for (unsigned binding = 0; binding < set_layout->binding_count;
2553 ++binding) {
2554 const struct pvr_descriptor_set_layout_binding *layout_binding =
2555 &set_layout->bindings[binding];
2556 pco_binding_data *binding_data = &desc_set_data->bindings[binding];
2557
2558 /* Skip non-dynamic bindings. */
2559 if (layout_binding->dynamic_buffer_idx == ~0)
2560 continue;
2561
2562 binding_data->range = (pco_range){
2563 .start = desc_set_dynamic_range->start +
2564 ((layout_binding->dynamic_buffer_idx *
2565 sizeof(struct pvr_buffer_descriptor)) /
2566 sizeof(uint32_t)),
2567 .count =
2568 (layout_binding->stride * layout_binding->descriptor_count) /
2569 sizeof(uint32_t),
2570 .stride = layout_binding->stride / sizeof(uint32_t),
2571 };
2572 }
2573 }
2574
2575 if (data->common.push_consts.used > 0) {
2576 unsigned count = data->common.push_consts.used;
2577
2578 if (count == ~0U) {
2579 count = 0;
2580 for (unsigned u = 0; u < layout->push_range_count; ++u) {
2581 VkPushConstantRange *range = &layout->push_ranges[u];
2582 if (!(mesa_to_vk_shader_stage(stage) & range->stageFlags))
2583 continue;
2584
2585 count = MAX2(count, range->offset + range->size);
2586 }
2587
2588 assert(!(count % 4));
2589 count = count / 4;
2590 }
2591
2592 data->common.push_consts.range = (pco_range){
2593 .start = data->common.shareds,
2594 .count = count,
2595 };
2596
2597 data->common.shareds += count;
2598 }
2599
2600 if (data->common.uses.point_sampler) {
2601 data->common.shareds = ALIGN_POT(data->common.shareds, 4);
2602 data->common.point_sampler = (pco_range){
2603 .start = data->common.shareds,
2604 .count = ROGUE_NUM_TEXSTATE_DWORDS,
2605 };
2606
2607 data->common.shareds += ROGUE_NUM_TEXSTATE_DWORDS;
2608 }
2609
2610 if (data->common.uses.ia_sampler) {
2611 data->common.shareds = ALIGN_POT(data->common.shareds, 4);
2612 data->common.ia_sampler = (pco_range){
2613 .start = data->common.shareds,
2614 .count = ROGUE_NUM_TEXSTATE_DWORDS,
2615 };
2616
2617 data->common.shareds += ROGUE_NUM_TEXSTATE_DWORDS;
2618 }
2619
2620 if (true || data->common.spilled_temps) {
2621 data->common.spill_info = (pco_range){
2622 .start = data->common.shareds,
2623 .count = 3,
2624 };
2625
2626 data->common.shareds += 3;
2627 }
2628
2629 if (data->common.scratch) {
2630 data->common.scratch_info = (pco_range){
2631 .start = data->common.shareds,
2632 .count = 3,
2633 };
2634
2635 data->common.shareds += 3;
2636 }
2637 }
2638
2639 static void
2640 pvr_preprocess_shader_data(pco_data *data,
2641 nir_shader *nir,
2642 const void *pCreateInfo,
2643 struct vk_pipeline_layout *layout,
2644 const struct vk_graphics_pipeline_state *state,
2645 struct usc_mrt_setup *mrt_setup)
2646 {
2647 const VkGraphicsPipelineCreateInfo *pGraphicsCreateInfo = pCreateInfo;
2648
2649 switch (nir->info.stage) {
2650 case MESA_SHADER_VERTEX: {
2651 const VkPipelineVertexInputStateCreateInfo *const vertex_input_state =
2652 pGraphicsCreateInfo->pVertexInputState;
2653
2654 pvr_init_vs_attribs(data, vertex_input_state);
2655 break;
2656 }
2657
2658 case MESA_SHADER_FRAGMENT: {
2659 if (pGraphicsCreateInfo->renderPass) {
2660 VK_FROM_HANDLE(pvr_render_pass, pass, pGraphicsCreateInfo->renderPass);
2661 const struct pvr_render_subpass *const subpass =
2662 &pass->subpasses[pGraphicsCreateInfo->subpass];
2663 const struct pvr_renderpass_hw_map *subpass_map =
2664 &pass->hw_setup->subpass_map[pGraphicsCreateInfo->subpass];
2665 const struct pvr_renderpass_hwsetup_subpass *hw_subpass =
2666 &pass->hw_setup->renders[subpass_map->render]
2667 .subpasses[subpass_map->subpass];
2668
2669 pvr_init_fs_outputs(data, pass, subpass, hw_subpass);
2670 pvr_init_fs_input_attachments(data, pass, subpass, hw_subpass);
2671 } else {
2672 pvr_init_fs_outputs_mrt(data, state->rp, mrt_setup);
2673 pvr_init_fs_input_attachments_mrt(data, state->rp, mrt_setup);
2674 }
2675
2676 pvr_init_fs_blend(data, state->cb);
2677 pvr_init_fs_tile_buffers(data);
2678
2679 data->fs.uses.alpha_to_coverage = false;
2680 if (state->ms)
2681 data->fs.uses.alpha_to_coverage = state->ms->alpha_to_coverage_enable;
2682
2683 if (BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_CB_COLOR_WRITE_ENABLES) ||
2684 (state->cb && state->cb->color_write_enables !=
2685 BITFIELD_MASK(MESA_VK_MAX_COLOR_ATTACHMENTS))) {
2686 data->fs.meta_present.color_write_enable = true;
2687 }
2688
2689 /* TODO: push consts, dynamic state, etc. */
2690 break;
2691 }
2692
2693 case MESA_SHADER_COMPUTE: {
2694 break;
2695 }
2696
2697 default:
2698 UNREACHABLE("");
2699 }
2700
2701 pvr_init_descriptors(data, nir, layout);
2702
2703 /* TODO: common things, like large constants being put into shareds. */
2704 }
2705
2706 static void pvr_postprocess_shader_data(pco_data *data,
2707 nir_shader *nir,
2708 const void *pCreateInfo,
2709 struct vk_pipeline_layout *layout,
2710 struct usc_mrt_setup *mrt_setup)
2711 {
2712 const VkGraphicsPipelineCreateInfo *pGraphicsCreateInfo = pCreateInfo;
2713
2714 switch (nir->info.stage) {
2715 case MESA_SHADER_VERTEX: {
2716 pvr_alloc_vs_sysvals(data, nir);
2717 pvr_alloc_vs_attribs(data, nir);
2718 pvr_alloc_vs_varyings(data, nir);
2719 break;
2720 }
2721
2722 case MESA_SHADER_FRAGMENT: {
2723 pvr_alloc_fs_sysvals(data, nir);
2724 pvr_alloc_fs_varyings(data, nir);
2725
2726 if (pGraphicsCreateInfo->renderPass) {
2727 VK_FROM_HANDLE(pvr_render_pass, pass, pGraphicsCreateInfo->renderPass);
2728 const struct pvr_render_subpass *const subpass =
2729 &pass->subpasses[pGraphicsCreateInfo->subpass];
2730 const struct pvr_renderpass_hw_map *subpass_map =
2731 &pass->hw_setup->subpass_map[pGraphicsCreateInfo->subpass];
2732 const struct pvr_renderpass_hwsetup_subpass *hw_subpass =
2733 &pass->hw_setup->renders[subpass_map->render]
2734 .subpasses[subpass_map->subpass];
2735 pvr_setup_fs_outputs(data, nir, subpass, hw_subpass);
2736 pvr_setup_fs_input_attachments(data, nir, subpass, hw_subpass);
2737 } else {
2738 pvr_setup_fs_outputs_mrt(data, nir, mrt_setup);
2739 pvr_setup_fs_input_attachments_mrt(data, nir, mrt_setup);
2740 }
2741
2742 pvr_setup_fs_blend(data);
2743 pvr_setup_fs_tile_buffers(data);
2744 pvr_setup_fs_sample_locations(data);
2745
2746 /* TODO: push consts, blend consts, dynamic state, etc. */
2747 break;
2748 }
2749
2750 case MESA_SHADER_COMPUTE: {
2751 pvr_alloc_cs_sysvals(data, nir);
2752 pvr_alloc_cs_shmem(data, nir);
2753 break;
2754 }
2755
2756 default:
2757 UNREACHABLE("");
2758 }
2759
2760 pvr_setup_descriptors(data, nir, layout);
2761
2762 /* TODO: common things, like large constants being put into shareds. */
2763 }
2764
2765 static void pvr_early_init_shader_data(pco_data *data,
2766 nir_shader *nir,
2767 const void *pCreateInfo)
2768 {
2769 const VkGraphicsPipelineCreateInfo *pGraphicsCreateInfo = pCreateInfo;
2770
2771 switch (nir->info.stage) {
2772 case MESA_SHADER_VERTEX:
2773 case MESA_SHADER_FRAGMENT: {
2774 VK_FROM_HANDLE(pvr_render_pass, pass, pGraphicsCreateInfo->renderPass);
2775 if (pass) {
2776 data->common.multiview = pass->multiview_enabled;
2777 } else {
2778 const VkPipelineRenderingCreateInfo *ri =
2779 vk_find_struct_const(pGraphicsCreateInfo->pNext,
2780 PIPELINE_RENDERING_CREATE_INFO);
2781 if (ri)
2782 data->common.multiview = !!ri->viewMask;
2783 else
2784 data->common.multiview = false;
2785 }
2786
2787 nir_foreach_variable_with_modes (var, nir, nir_var_system_value) {
2788 if (var->data.location == SYSTEM_VALUE_VIEW_INDEX) {
2789 data->common.multiview = true;
2790 break;
2791 }
2792 }
2793
2794 break;
2795 }
2796
2797 case MESA_SHADER_COMPUTE:
2798 break;
2799
2800 default:
2801 UNREACHABLE("Unsupported stage.");
2802 }
2803 }
2804
2805 static bool pvr_build_fs_passthrough(bool no_fragment_shader,
2806 const struct pvr_device_info *dev_info)
2807 {
2808 bool is_arch_rogue = dev_info->ident.arch == PVR_DEVICE_ARCH_ROGUE;
2809
2810 return is_arch_rogue && no_fragment_shader;
2811 }
2812
2813 /* Compiles and uploads shaders and PDS programs. */
2814 static VkResult
2815 pvr_graphics_pipeline_compile(struct pvr_device *const device,
2816 struct vk_pipeline_cache *cache,
2817 const VkGraphicsPipelineCreateInfo *pCreateInfo,
2818 const VkAllocationCallbacks *const allocator,
2819 struct pvr_graphics_pipeline *const gfx_pipeline,
2820 const struct vk_graphics_pipeline_state *state)
2821 {
2822 struct vk_pipeline_layout *layout = gfx_pipeline->base.layout;
2823 const uint32_t cache_line_size =
2824 pvr_get_slc_cache_line_size(&device->pdevice->dev_info);
2825 VkResult result;
2826
2827 struct pvr_vertex_shader_state *vertex_state =
2828 &gfx_pipeline->shader_state.vertex;
2829 struct pvr_fragment_shader_state *fragment_state =
2830 &gfx_pipeline->shader_state.fragment;
2831 struct usc_mrt_setup mrt_setup = { 0 };
2832
2833 if (!pCreateInfo->renderPass) {
2834 const struct vk_render_pass_state *rp = state->rp;
2835 VkFormat attachment_formats[rp->color_attachment_count];
2836 uint32_t mrt_attachment_map[rp->color_attachment_count];
2837 struct usc_mrt_setup mrt_setup_tmp = { 0 };
2838 uint32_t mrt_count = 0;
2839
2840 for (uint32_t i = 0; i < rp->color_attachment_count; i++) {
2841 if (rp->color_attachment_formats[i] == VK_FORMAT_UNDEFINED) {
2842 mrt_attachment_map[i] = VK_ATTACHMENT_UNUSED;
2843 attachment_formats[mrt_count] = VK_FORMAT_UNDEFINED;
2844 continue;
2845 }
2846
2847 mrt_attachment_map[i] = mrt_count;
2848 attachment_formats[mrt_count++] = rp->color_attachment_formats[i];
2849 }
2850
2851 result = pvr_arch_init_usc_mrt_setup(device,
2852 mrt_count,
2853 attachment_formats,
2854 &mrt_setup_tmp);
2855 if (result != VK_SUCCESS)
2856 return result;
2857
2858 result =
2859 pvr_arch_mrt_setup_partial_init(device,
2860 &mrt_setup,
2861 rp->color_attachment_count,
2862 mrt_setup_tmp.num_output_regs,
2863 mrt_setup_tmp.num_tile_buffers);
2864 if (result != VK_SUCCESS) {
2865 pvr_arch_destroy_mrt_setup(device, &mrt_setup_tmp);
2866 return result;
2867 }
2868
2869 for (uint32_t i = 0; i < rp->color_attachment_count; i++) {
2870 if (mrt_attachment_map[i] != VK_ATTACHMENT_UNUSED) {
2871 memcpy(&mrt_setup.mrt_resources[i],
2872 &mrt_setup_tmp.mrt_resources[mrt_attachment_map[i]],
2873 sizeof(mrt_setup_tmp.mrt_resources[0]));
2874 }
2875 }
2876
2877 pvr_arch_destroy_mrt_setup(device, &mrt_setup_tmp);
2878 }
2879
2880 pco_ctx *pco_ctx = device->pdevice->pco_ctx;
2881
2882 nir_shader *producer = NULL;
2883 nir_shader *consumer = NULL;
2884 pco_data shader_data[MESA_SHADER_STAGES] = { 0 };
2885 pco_data *producer_data = NULL;
2886 nir_shader *nir_shaders[MESA_SHADER_STAGES] = { 0 };
2887 pco_shader *pco_shaders[MESA_SHADER_STAGES] = { 0 };
2888 pco_shader **vs = &pco_shaders[MESA_SHADER_VERTEX];
2889 pco_shader **fs = &pco_shaders[MESA_SHADER_FRAGMENT];
2890 void *shader_mem_ctx = ralloc_context(NULL);
2891
2892 struct pvr_pds_vertex_dma vtx_dma_descriptions[PVR_MAX_VERTEX_ATTRIB_DMAS];
2893 uint32_t vtx_dma_count = 0;
2894 bool build_fs_passthrough = false;
2895
2896 for (mesa_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
2897 size_t stage_index = gfx_pipeline->stage_indices[stage];
2898
2899 /* Skip unused/inactive stages. */
2900 if (stage_index == ~0)
2901 continue;
2902
2903 result =
2904 vk_pipeline_shader_stage_to_nir(&device->vk,
2905 gfx_pipeline->base.pipeline_flags,
2906 &pCreateInfo->pStages[stage_index],
2907 pco_spirv_options(),
2908 pco_nir_options(),
2909 shader_mem_ctx,
2910 &nir_shaders[stage]);
2911 if (result != VK_SUCCESS)
2912 goto err_free_build_context;
2913
2914 pvr_early_init_shader_data(&shader_data[stage],
2915 nir_shaders[stage],
2916 pCreateInfo);
2917 pco_preprocess_nir(pco_ctx, nir_shaders[stage]);
2918 }
2919
2920 for (mesa_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
2921 if (!nir_shaders[stage])
2922 continue;
2923
2924 if (producer)
2925 pco_link_nir(pco_ctx,
2926 producer,
2927 nir_shaders[stage],
2928 producer_data,
2929 &shader_data[stage]);
2930
2931 producer = nir_shaders[stage];
2932 producer_data = &shader_data[stage];
2933 }
2934
2935 for (mesa_shader_stage stage = MESA_SHADER_STAGES; stage-- > 0;) {
2936 if (!nir_shaders[stage])
2937 continue;
2938
2939 if (consumer)
2940 pco_rev_link_nir(pco_ctx, nir_shaders[stage], consumer);
2941
2942 consumer = nir_shaders[stage];
2943 }
2944
2945 /* Check if the fragment passthrough shader should be built. */
2946 build_fs_passthrough =
2947 pvr_build_fs_passthrough(!nir_shaders[MESA_SHADER_FRAGMENT],
2948 &device->pdevice->dev_info);
2949
2950 if (build_fs_passthrough) {
2951 nir_shader *fs_pass = pvr_usc_fs_pfo_passthrough_nir(pco_ctx);
2952 ralloc_steal(shader_mem_ctx, fs_pass);
2953 nir_shaders[MESA_SHADER_FRAGMENT] = fs_pass;
2954 }
2955
2956 for (mesa_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
2957 if (!nir_shaders[stage])
2958 continue;
2959
2960 pvr_preprocess_shader_data(&shader_data[stage],
2961 nir_shaders[stage],
2962 pCreateInfo,
2963 layout,
2964 state,
2965 &mrt_setup);
2966
2967 const struct lower_ycbcr_state ycbcr_state = {
2968 .set_layout_count = layout->set_count,
2969 .set_layouts = layout->set_layouts,
2970 };
2971 NIR_PASS(_,
2972 nir_shaders[stage],
2973 nir_pvr_lower_ycbcr_tex,
2974 lookup_ycbcr_conversion,
2975 &ycbcr_state);
2976 pco_lower_nir(pco_ctx, nir_shaders[stage], &shader_data[stage]);
2977
2978 pco_postprocess_nir(pco_ctx, nir_shaders[stage], &shader_data[stage]);
2979
2980 pvr_postprocess_shader_data(&shader_data[stage],
2981 nir_shaders[stage],
2982 pCreateInfo,
2983 layout,
2984 &mrt_setup);
2985 }
2986
2987 if (!pCreateInfo->renderPass)
2988 pvr_arch_destroy_mrt_setup(device, &mrt_setup);
2989
2990 for (mesa_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
2991 pco_shader **pco = &pco_shaders[stage];
2992
2993 /* Skip unused/inactive stages. */
2994 if (!nir_shaders[stage])
2995 continue;
2996
2997 *pco = pco_trans_nir(pco_ctx,
2998 nir_shaders[stage],
2999 &shader_data[stage],
3000 shader_mem_ctx);
3001 if (!*pco) {
3002 result = VK_ERROR_INITIALIZATION_FAILED;
3003 goto err_free_build_context;
3004 }
3005
3006 pco_process_ir(pco_ctx, *pco);
3007 pco_encode_ir(pco_ctx, *pco);
3008 }
3009
3010 pvr_vertex_state_save(gfx_pipeline, *vs);
3011
3012 if (pCreateInfo->flags & VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR) {
3013 struct pvr_stats stats = pco_get_pvr_stats(*vs);
3014 gfx_pipeline->vs_stats = vk_zalloc2(&device->vk.alloc,
3015 allocator,
3016 sizeof(stats),
3017 8,
3018 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3019
3020 if (!gfx_pipeline->vs_stats)
3021 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
3022
3023 memcpy(gfx_pipeline->vs_stats, &stats, sizeof(stats));
3024 }
3025
3026 if (pCreateInfo->flags &
3027 VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR) {
3028 gfx_pipeline->vs_nir_str =
3029 nir_shader_as_str(nir_shaders[MESA_SHADER_VERTEX], NULL);
3030 }
3031
3032 pvr_graphics_pipeline_setup_vertex_dma(gfx_pipeline,
3033 pCreateInfo->pVertexInputState,
3034 state->vi,
3035 vtx_dma_descriptions,
3036 &vtx_dma_count);
3037
3038 result = pvr_gpu_upload_usc(device,
3039 pco_shader_binary_data(*vs),
3040 pco_shader_binary_size(*vs),
3041 cache_line_size,
3042 &vertex_state->shader_bo);
3043 if (result != VK_SUCCESS)
3044 goto err_free_build_context;
3045
3046 if (*fs) {
3047 pvr_fragment_state_save(gfx_pipeline, *fs);
3048 fragment_state->is_passthrough = build_fs_passthrough;
3049
3050 if (pCreateInfo->flags & VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR) {
3051 struct pvr_stats stats = pco_get_pvr_stats(*fs);
3052 gfx_pipeline->fs_stats = vk_zalloc2(&device->vk.alloc,
3053 allocator,
3054 sizeof(stats),
3055 8,
3056 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3057
3058 if (!gfx_pipeline->fs_stats)
3059 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
3060
3061 memcpy(gfx_pipeline->fs_stats, &stats, sizeof(stats));
3062 }
3063
3064 if (pCreateInfo->flags &
3065 VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR) {
3066 gfx_pipeline->fs_nir_str =
3067 nir_shader_as_str(nir_shaders[MESA_SHADER_FRAGMENT], NULL);
3068 }
3069
3070 pvr_graphics_pipeline_setup_fragment_coeff_program(
3071 gfx_pipeline,
3072 nir_shaders[MESA_SHADER_FRAGMENT]);
3073
3074 result = pvr_gpu_upload_usc(device,
3075 pco_shader_binary_data(*fs),
3076 pco_shader_binary_size(*fs),
3077 cache_line_size,
3078 &fragment_state->shader_bo);
3079 if (result != VK_SUCCESS)
3080 goto err_free_vertex_bo;
3081
3082 result = pvr_pds_coeff_program_create_and_upload(device,
3083 allocator,
3084 fragment_state);
3085 if (result != VK_SUCCESS)
3086 goto err_free_fragment_bo;
3087
3088 result = pvr_pds_fragment_program_create(device,
3089 allocator,
3090 *fs,
3091 fragment_state);
3092 if (result != VK_SUCCESS)
3093 goto err_free_coeff_program;
3094
3095 result = pvr_pds_descriptor_program_create_and_upload(
3096 device,
3097 allocator,
3098 layout,
3099 MESA_SHADER_FRAGMENT,
3100 &gfx_pipeline->fs_data,
3101 &fragment_state->descriptor_state);
3102 if (result != VK_SUCCESS)
3103 goto err_free_frag_program;
3104
3105 /* If not, we need to MAX2() and set
3106 * `fragment_state->stage_state.pds_temps_count` appropriately.
3107 */
3108 assert(fragment_state->descriptor_state.pds_info.temps_required == 0);
3109 }
3110
3111 result = pvr_pds_vertex_attrib_programs_create_and_upload(
3112 device,
3113 allocator,
3114 &gfx_pipeline->vs_data,
3115 vtx_dma_descriptions,
3116 vtx_dma_count,
3117 &vertex_state->pds_attrib_programs);
3118 if (result != VK_SUCCESS)
3119 goto err_free_frag_descriptor_program;
3120
3121 result = pvr_pds_descriptor_program_create_and_upload(
3122 device,
3123 allocator,
3124 layout,
3125 MESA_SHADER_VERTEX,
3126 &gfx_pipeline->vs_data,
3127 &vertex_state->descriptor_state);
3128 if (result != VK_SUCCESS)
3129 goto err_free_vertex_attrib_program;
3130
3131 /* FIXME: When the temp_buffer_total_size is non-zero we need to allocate a
3132 * scratch buffer for both vertex and fragment stage.
3133 * Figure out the best place to do this.
3134 */
3135 /* assert(pvr_pds_descriptor_program_variables.temp_buff_total_size == 0); */
3136 /* TODO: Implement spilling with the above. */
3137
3138 ralloc_free(shader_mem_ctx);
3139
3140 return VK_SUCCESS;
3141
3142 err_free_vertex_attrib_program:
3143 for (uint32_t i = 0; i < ARRAY_SIZE(vertex_state->pds_attrib_programs);
3144 i++) {
3145 struct pvr_pds_attrib_program *const attrib_program =
3146 &vertex_state->pds_attrib_programs[i];
3147
3148 pvr_pds_vertex_attrib_program_destroy(device, allocator, attrib_program);
3149 }
3150 err_free_frag_descriptor_program:
3151 pvr_pds_descriptor_program_destroy(device,
3152 allocator,
3153 &fragment_state->descriptor_state);
3154 err_free_frag_program:
3155 vk_free2(&device->vk.alloc,
3156 allocator,
3157 fragment_state->pds_fragment_program_buffer);
3158 err_free_coeff_program:
3159 vk_free2(&device->vk.alloc,
3160 allocator,
3161 fragment_state->pds_coeff_program_buffer);
3162 err_free_fragment_bo:
3163 pvr_bo_suballoc_free(fragment_state->shader_bo);
3164 err_free_vertex_bo:
3165 pvr_bo_suballoc_free(vertex_state->shader_bo);
3166 err_free_build_context:
3167 ralloc_free(shader_mem_ctx);
3168 if (!pCreateInfo->renderPass)
3169 pvr_arch_destroy_mrt_setup(device, &mrt_setup);
3170 return result;
3171 }
3172
3173 static void
3174 pvr_rendering_info_setup(const VkGraphicsPipelineCreateInfo *const info,
3175 struct vk_multiview_state *mv,
3176 struct vk_render_pass_state *rp)
3177 {
3178 const VkPipelineRenderingCreateInfo *ri =
3179 vk_find_struct_const(info->pNext, PIPELINE_RENDERING_CREATE_INFO);
3180
3181 if (!ri) {
3182 /* From the Vulkan spec for VkPipelineRenderingCreateInfo:
3183 *
3184 * "if this structure is not specified, and the pipeline does not
3185 * include a VkRenderPass, viewMask and colorAttachmentCount are 0, and
3186 * depthAttachmentFormat and stencilAttachmentFormat are
3187 * VK_FORMAT_UNDEFINED.
3188 */
3189 *mv = (struct vk_multiview_state) {
3190 .view_mask = 0,
3191 };
3192 *rp = (struct vk_render_pass_state){
3193 .attachments = 0,
3194 .color_attachment_count = 0,
3195 .depth_attachment_format = VK_FORMAT_UNDEFINED,
3196 .stencil_attachment_format = VK_FORMAT_UNDEFINED,
3197 };
3198
3199 return;
3200 }
3201
3202 mv->view_mask = ri->viewMask;
3203 rp->attachments = 0;
3204
3205 rp->color_attachment_count = ri->colorAttachmentCount;
3206 for (int i = 0; i < ri->colorAttachmentCount; i++) {
3207 rp->color_attachment_formats[i] = ri->pColorAttachmentFormats[i];
3208 if (rp->color_attachment_formats[i] != VK_FORMAT_UNDEFINED)
3209 rp->attachments |= MESA_VK_RP_ATTACHMENT_COLOR_BIT(i);
3210 }
3211
3212 rp->depth_attachment_format = ri->depthAttachmentFormat;
3213 if (ri->depthAttachmentFormat != VK_FORMAT_UNDEFINED)
3214 rp->attachments |= MESA_VK_RP_ATTACHMENT_DEPTH_BIT;
3215
3216 rp->stencil_attachment_format = ri->stencilAttachmentFormat;
3217 if (ri->stencilAttachmentFormat != VK_FORMAT_UNDEFINED)
3218 rp->attachments |= MESA_VK_RP_ATTACHMENT_STENCIL_BIT;
3219 }
3220
3221 static void
3222 pvr_create_renderpass_state(const VkGraphicsPipelineCreateInfo *const info,
3223 struct vk_multiview_state *mv,
3224 struct vk_render_pass_state *rp)
3225 {
3226 if (!info->renderPass)
3227 return pvr_rendering_info_setup(info, mv, rp);
3228
3229 VK_FROM_HANDLE(pvr_render_pass, pass, info->renderPass);
3230 const struct pvr_render_subpass *const subpass =
3231 &pass->subpasses[info->subpass];
3232 enum vk_rp_attachment_flags attachments = 0;
3233
3234 assert(info->subpass < pass->subpass_count);
3235
3236 for (uint32_t i = 0; i < subpass->color_count; i++) {
3237 if (subpass->color_attachments[i] == VK_ATTACHMENT_UNUSED)
3238 continue;
3239
3240 if (pass->attachments[subpass->color_attachments[i]].aspects)
3241 attachments |= MESA_VK_RP_ATTACHMENT_COLOR_0_BIT << i;
3242 }
3243
3244 if (subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED) {
3245 VkImageAspectFlags ds_aspects =
3246 pass->attachments[subpass->depth_stencil_attachment].aspects;
3247 if (ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
3248 attachments |= MESA_VK_RP_ATTACHMENT_DEPTH_BIT;
3249 if (ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT)
3250 attachments |= MESA_VK_RP_ATTACHMENT_STENCIL_BIT;
3251 }
3252
3253 *rp = (struct vk_render_pass_state){
3254 .attachments = attachments,
3255 };
3256 *mv = (struct vk_multiview_state){
3257 .view_mask = subpass->view_mask,
3258 };
3259 }
3260
3261 static VkResult
3262 pvr_graphics_pipeline_init(struct pvr_device *device,
3263 struct vk_pipeline_cache *cache,
3264 const VkGraphicsPipelineCreateInfo *pCreateInfo,
3265 const VkAllocationCallbacks *allocator,
3266 struct pvr_graphics_pipeline *gfx_pipeline)
3267 {
3268 struct vk_dynamic_graphics_state *const dynamic_state =
3269 &gfx_pipeline->dynamic_state;
3270 struct vk_graphics_pipeline_all_state all_state;
3271 struct vk_graphics_pipeline_state state = { 0 };
3272 struct vk_multiview_state mv_state;
3273 struct vk_render_pass_state rp_state;
3274 VkResult result;
3275
3276 pvr_create_renderpass_state(pCreateInfo, &mv_state, &rp_state);
3277
3278 pvr_pipeline_init(device,
3279 PVR_PIPELINE_TYPE_GRAPHICS,
3280 pCreateInfo->layout,
3281 &gfx_pipeline->base);
3282
3283 result = vk_graphics_pipeline_state_fill(&device->vk,
3284 &state,
3285 pCreateInfo,
3286 &mv_state,
3287 &rp_state,
3288 0,
3289 &all_state,
3290 NULL,
3291 0,
3292 NULL);
3293 if (result != VK_SUCCESS)
3294 goto err_pipeline_finish;
3295
3296 vk_dynamic_graphics_state_init(dynamic_state);
3297
3298 /* Load static state into base dynamic state holder. */
3299 vk_dynamic_graphics_state_fill(dynamic_state, &state);
3300
3301 /* The value of ms.rasterization_samples is undefined when
3302 * rasterizer_discard_enable is set, but we need a specific value.
3303 * Fill that in here.
3304 */
3305 if (state.rs->rasterizer_discard_enable)
3306 dynamic_state->ms.rasterization_samples = VK_SAMPLE_COUNT_1_BIT;
3307
3308 memset(gfx_pipeline->stage_indices, ~0, sizeof(gfx_pipeline->stage_indices));
3309
3310 for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) {
3311 VkShaderStageFlagBits vk_stage = pCreateInfo->pStages[i].stage;
3312 mesa_shader_stage gl_stage = vk_to_mesa_shader_stage(vk_stage);
3313 /* From the Vulkan 1.2.192 spec for VkPipelineShaderStageCreateInfo:
3314 *
3315 * "stage must not be VK_SHADER_STAGE_ALL_GRAPHICS,
3316 * or VK_SHADER_STAGE_ALL."
3317 *
3318 * So we don't handle that.
3319 *
3320 * We also don't handle VK_SHADER_STAGE_TESSELLATION_* and
3321 * VK_SHADER_STAGE_GEOMETRY_BIT stages as 'tessellationShader' and
3322 * 'geometryShader' are set to false in the VkPhysicalDeviceFeatures
3323 * structure returned by the driver.
3324 */
3325 switch (pCreateInfo->pStages[i].stage) {
3326 case VK_SHADER_STAGE_VERTEX_BIT:
3327 case VK_SHADER_STAGE_FRAGMENT_BIT:
3328 gfx_pipeline->stage_indices[gl_stage] = i;
3329 break;
3330 default:
3331 UNREACHABLE("Unsupported stage.");
3332 }
3333 }
3334
3335 /* Compiles and uploads shaders and PDS programs. */
3336 result = pvr_graphics_pipeline_compile(device,
3337 cache,
3338 pCreateInfo,
3339 allocator,
3340 gfx_pipeline,
3341 &state);
3342 if (result != VK_SUCCESS)
3343 goto err_pipeline_finish;
3344
3345 return VK_SUCCESS;
3346
3347 err_pipeline_finish:
3348 pvr_pipeline_finish(device, &gfx_pipeline->base);
3349
3350 return result;
3351 }
3352
3353 /* If allocator == NULL, the internal one will be used. */
3354 static VkResult
3355 pvr_graphics_pipeline_create(struct pvr_device *device,
3356 struct vk_pipeline_cache *cache,
3357 const VkGraphicsPipelineCreateInfo *pCreateInfo,
3358 const VkAllocationCallbacks *allocator,
3359 VkPipeline *const pipeline_out)
3360 {
3361 struct pvr_graphics_pipeline *gfx_pipeline;
3362 VkResult result;
3363
3364 gfx_pipeline = vk_zalloc2(&device->vk.alloc,
3365 allocator,
3366 sizeof(*gfx_pipeline),
3367 8,
3368 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
3369 if (!gfx_pipeline)
3370 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
3371
3372 /* Compiles and uploads shaders and PDS programs too. */
3373 result = pvr_graphics_pipeline_init(device,
3374 cache,
3375 pCreateInfo,
3376 allocator,
3377 gfx_pipeline);
3378 if (result != VK_SUCCESS) {
3379 vk_free2(&device->vk.alloc, allocator, gfx_pipeline);
3380 return result;
3381 }
3382
3383 *pipeline_out = pvr_pipeline_to_handle(&gfx_pipeline->base);
3384
3385 return VK_SUCCESS;
3386 }
3387
3388 VkResult PVR_PER_ARCH(CreateGraphicsPipelines)(
3389 VkDevice _device,
3390 VkPipelineCache pipelineCache,
3391 uint32_t createInfoCount,
3392 const VkGraphicsPipelineCreateInfo *pCreateInfos,
3393 const VkAllocationCallbacks *pAllocator,
3394 VkPipeline *pPipelines)
3395 {
3396 VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
3397 VK_FROM_HANDLE(pvr_device, device, _device);
3398 VkResult result = VK_SUCCESS;
3399
3400 for (uint32_t i = 0; i < createInfoCount; i++) {
3401 const VkResult local_result =
3402 pvr_graphics_pipeline_create(device,
3403 cache,
3404 &pCreateInfos[i],
3405 pAllocator,
3406 &pPipelines[i]);
3407 if (local_result != VK_SUCCESS) {
3408 result = local_result;
3409 pPipelines[i] = VK_NULL_HANDLE;
3410 }
3411 }
3412
3413 return result;
3414 }
3415
3416 /*****************************************************************************
3417 Other functions
3418 *****************************************************************************/
3419
3420 void PVR_PER_ARCH(DestroyPipeline)(VkDevice _device,
3421 VkPipeline _pipeline,
3422 const VkAllocationCallbacks *pAllocator)
3423 {
3424 VK_FROM_HANDLE(pvr_pipeline, pipeline, _pipeline);
3425 VK_FROM_HANDLE(pvr_device, device, _device);
3426
3427 if (!pipeline)
3428 return;
3429
3430 switch (pipeline->type) {
3431 case PVR_PIPELINE_TYPE_GRAPHICS: {
3432 struct pvr_graphics_pipeline *const gfx_pipeline =
3433 to_pvr_graphics_pipeline(pipeline);
3434
3435 pvr_graphics_pipeline_destroy(device, pAllocator, gfx_pipeline);
3436 break;
3437 }
3438
3439 case PVR_PIPELINE_TYPE_COMPUTE: {
3440 struct pvr_compute_pipeline *const compute_pipeline =
3441 to_pvr_compute_pipeline(pipeline);
3442
3443 pvr_compute_pipeline_destroy(device, pAllocator, compute_pipeline);
3444 break;
3445 }
3446
3447 default:
3448 UNREACHABLE("Unknown pipeline type.");
3449 }
3450 }
3451
3452 static uint32_t pvr_get_executable_count(struct pvr_pipeline *pipeline)
3453 {
3454 uint32_t exe_count = 0;
3455
3456 switch (pipeline->type) {
3457 case PVR_PIPELINE_TYPE_GRAPHICS: {
3458 struct pvr_graphics_pipeline *const gfx_pipeline =
3459 to_pvr_graphics_pipeline(pipeline);
3460
3461 for (mesa_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
3462 size_t stage_index = gfx_pipeline->stage_indices[stage];
3463
3464 if (stage_index != ~0)
3465 exe_count++;
3466 }
3467 break;
3468 }
3469
3470 case PVR_PIPELINE_TYPE_COMPUTE: {
3471 /* Compute pipelines always only have one executable. */
3472 exe_count = 1;
3473 break;
3474 }
3475
3476 default:
3477 UNREACHABLE("Unknown pipeline type.");
3478 }
3479
3480 return exe_count;
3481 }
3482
3483 VkResult PVR_PER_ARCH(GetPipelineExecutableStatisticsKHR)(
3484 UNUSED VkDevice _device,
3485 const VkPipelineExecutableInfoKHR *pExecutableInfo,
3486 uint32_t *pStatisticCount,
3487 VkPipelineExecutableStatisticKHR *pStatistics)
3488 {
3489 VK_FROM_HANDLE(pvr_pipeline, pipeline, pExecutableInfo->pipeline);
3490 VK_OUTARRAY_MAKE_TYPED(VkPipelineExecutableStatisticKHR,
3491 out,
3492 pStatistics,
3493 pStatisticCount);
3494
3495 switch (pipeline->type) {
3496 case PVR_PIPELINE_TYPE_GRAPHICS: {
3497 struct pvr_graphics_pipeline *const gfx_pipeline =
3498 to_pvr_graphics_pipeline(pipeline);
3499
3500 if (pExecutableInfo->executableIndex == 0) {
3501 vk_add_pvr_stats(out, gfx_pipeline->vs_stats);
3502 } else {
3503 assert(pExecutableInfo->executableIndex == 1);
3504 vk_add_pvr_stats(out, gfx_pipeline->fs_stats);
3505 }
3506 break;
3507 }
3508
3509 case PVR_PIPELINE_TYPE_COMPUTE: {
3510 struct pvr_compute_pipeline *const compute_pipeline =
3511 to_pvr_compute_pipeline(pipeline);
3512
3513 assert(pExecutableInfo->executableIndex == 0);
3514 vk_add_pvr_stats(out, compute_pipeline->cs_stats);
3515 break;
3516 }
3517
3518 default:
3519 UNREACHABLE("Unknown pipeline type.");
3520 }
3521
3522 return vk_outarray_status(&out);
3523 }
3524
3525 VkResult PVR_PER_ARCH(GetPipelineExecutablePropertiesKHR)(
3526 VkDevice _device,
3527 const VkPipelineInfoKHR *pPipelineInfo,
3528 uint32_t *pExecutableCount,
3529 VkPipelineExecutablePropertiesKHR *pProperties)
3530 {
3531 VK_FROM_HANDLE(pvr_device, device, _device);
3532 VK_FROM_HANDLE(pvr_pipeline, pipeline, pPipelineInfo->pipeline);
3533 ASSERTED uint32_t actualExeCount = pvr_get_executable_count(pipeline);
3534
3535 /* Due to individual shaders currently being stored as individual members
3536 * of pvr_graphics_pipeline, if support is added for a new shader stage,
3537 * this code will need to be updated.
3538 */
3539 assert(actualExeCount >= 1 && actualExeCount <= 2);
3540
3541 VK_OUTARRAY_MAKE_TYPED(VkPipelineExecutablePropertiesKHR,
3542 out,
3543 pProperties,
3544 pExecutableCount);
3545
3546 switch (pipeline->type) {
3547 case PVR_PIPELINE_TYPE_GRAPHICS: {
3548 struct pvr_graphics_pipeline *const gfx_pipeline =
3549 to_pvr_graphics_pipeline(pipeline);
3550
3551 vk_outarray_append_typed (VkPipelineExecutablePropertiesKHR,
3552 &out,
3553 vertProps) {
3554 vertProps->stages |= VK_SHADER_STAGE_VERTEX_BIT;
3555 VK_COPY_STR(vertProps->name, "vertex");
3556 VK_COPY_STR(vertProps->description, "Vulkan Vertex Shader");
3557 vertProps->subgroupSize = device->pdevice->vk.properties.subgroupSize;
3558 }
3559
3560 if (gfx_pipeline->stage_indices[MESA_SHADER_FRAGMENT] != ~0) {
3561 vk_outarray_append_typed (VkPipelineExecutablePropertiesKHR,
3562 &out,
3563 fragProps) {
3564 fragProps->stages |= VK_SHADER_STAGE_FRAGMENT_BIT;
3565 VK_COPY_STR(fragProps->name, "fragment");
3566 VK_COPY_STR(fragProps->description, "Vulkan Fragment Shader");
3567 fragProps->subgroupSize =
3568 device->pdevice->vk.properties.subgroupSize;
3569 }
3570 }
3571
3572 break;
3573 }
3574
3575 case PVR_PIPELINE_TYPE_COMPUTE: {
3576 vk_outarray_append_typed (VkPipelineExecutablePropertiesKHR,
3577 &out,
3578 compProps) {
3579 compProps->stages = VK_SHADER_STAGE_COMPUTE_BIT;
3580 VK_COPY_STR(compProps->name, "compute");
3581 VK_COPY_STR(compProps->description, "Vulkan Compute Shader");
3582 compProps->subgroupSize = device->pdevice->vk.properties.subgroupSize;
3583 }
3584 break;
3585 }
3586 default:
3587 UNREACHABLE("Unknown pipeline type.");
3588 }
3589
3590 return vk_outarray_status(&out);
3591 }
3592
3593 static bool
3594 write_ir_text(VkPipelineExecutableInternalRepresentationKHR *ir,
3595 const char *data)
3596 {
3597 ir->isText = VK_TRUE;
3598
3599 size_t data_len = strlen(data) + 1;
3600
3601 if (ir->pData == NULL) {
3602 ir->dataSize = data_len;
3603 return true;
3604 }
3605
3606 strncpy(ir->pData, data, ir->dataSize);
3607 if (ir->dataSize < data_len)
3608 return false;
3609
3610 ir->dataSize = data_len;
3611 return true;
3612 }
3613
3614 VkResult PVR_PER_ARCH(GetPipelineExecutableInternalRepresentationsKHR)(
3615 UNUSED VkDevice _device,
3616 UNUSED const VkPipelineExecutableInfoKHR *pExecutableInfo,
3617 uint32_t *pInternalRepresentationCount,
3618 VkPipelineExecutableInternalRepresentationKHR *pInternalRepresentations)
3619 {
3620 VK_FROM_HANDLE(pvr_pipeline, pipeline, pExecutableInfo->pipeline);
3621 VK_OUTARRAY_MAKE_TYPED(VkPipelineExecutableInternalRepresentationKHR, out,
3622 pInternalRepresentations,
3623 pInternalRepresentationCount);
3624 bool incomplete_text = false;
3625
3626 switch (pipeline->type) {
3627 case PVR_PIPELINE_TYPE_GRAPHICS: {
3628 struct pvr_graphics_pipeline *const gfx_pipeline =
3629 to_pvr_graphics_pipeline(pipeline);
3630
3631 if (pExecutableInfo->executableIndex == 0) {
3632 if (gfx_pipeline->vs_nir_str) {
3633 vk_outarray_append_typed(VkPipelineExecutableInternalRepresentationKHR, &out, ir) {
3634 VK_COPY_STR(ir->name, "Final NIR shader IR");
3635 VK_COPY_STR(ir->description, "Final NIR shader IR to be ingested by the PCO backend");
3636 if (!write_ir_text(ir, gfx_pipeline->vs_nir_str))
3637 incomplete_text = true;
3638 }
3639 }
3640 } else {
3641 assert(pExecutableInfo->executableIndex == 1);
3642
3643 if (gfx_pipeline->fs_nir_str) {
3644 vk_outarray_append_typed(VkPipelineExecutableInternalRepresentationKHR, &out, ir) {
3645 VK_COPY_STR(ir->name, "Final NIR shader IR");
3646 VK_COPY_STR(ir->description, "Final NIR shader IR to be ingested by the PCO backend");
3647 if (!write_ir_text(ir, gfx_pipeline->fs_nir_str))
3648 incomplete_text = true;
3649 }
3650 }
3651 }
3652
3653 break;
3654 }
3655
3656 case PVR_PIPELINE_TYPE_COMPUTE: {
3657 struct pvr_compute_pipeline *const compute_pipeline =
3658 to_pvr_compute_pipeline(pipeline);
3659
3660 assert(pExecutableInfo->executableIndex == 0);
3661
3662 if (compute_pipeline->cs_nir_str) {
3663 vk_outarray_append_typed(VkPipelineExecutableInternalRepresentationKHR, &out, ir) {
3664 VK_COPY_STR(ir->name, "Final NIR shader IR");
3665 VK_COPY_STR(ir->description, "Final NIR shader IR to be ingested by the PCO backend");
3666 if (!write_ir_text(ir, compute_pipeline->cs_nir_str))
3667 incomplete_text = true;
3668 }
3669 }
3670
3671 break;
3672 }
3673
3674 default:
3675 UNREACHABLE("Unknown pipeline type.");
3676 }
3677
3678 return incomplete_text ? VK_INCOMPLETE : vk_outarray_status(&out);
3679 }