Debian Codegraph @

src/imagination/vulkan/pvr_arch_query_compute.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <stddef.h>
27 #include <stdint.h>
28 #include <string.h>
29 #include <vulkan/vulkan.h>
30
31 #include "common/pvr_iface.h"
32 #include "hwdef/rogue_hw_utils.h"
33 #include "pco_uscgen_programs.h"
34 #include "pvr_bo.h"
35 #include "pvr_buffer.h"
36 #include "pvr_cmd_buffer.h"
37 #include "pvr_device.h"
38 #include "pvr_formats.h"
39 #include "pvr_macros.h"
40 #include "pvr_pds.h"
41 #include "pvr_physical_device.h"
42 #include "pvr_pipeline.h"
43 #include "pvr_query.h"
44 #include "pvr_tex_state.h"
45 #include "pvr_types.h"
46 #include "vk_alloc.h"
47 #include "vk_command_pool.h"
48 #include "vk_util.h"
49
50 /* TODO: multicore support/awareness. */
51
52 static inline void pvr_init_primary_compute_pds_program(
53 struct pvr_pds_compute_shader_program *program)
54 {
55 pvr_pds_compute_shader_program_init(program);
56 program->local_input_regs[0] = 0;
57 /* Workgroup id is in reg0. */
58 program->work_group_input_regs[0] = 0;
59 program->flattened_work_groups = true;
60 program->kick_usc = true;
61 }
62
63 static VkResult
64 pvr_create_compute_secondary_prog(struct pvr_device *device,
65 unsigned const_shared_regs,
66 struct pvr_compute_query_shader *query_prog)
67 {
68 const size_t size = PVR_PDS_MAX_DESC_UPLOAD_BYTES;
69 struct pvr_pds_descriptor_program_input sec_pds_program;
70 struct pvr_pds_info *info = &query_prog->info;
71 uint32_t staging_buffer_size;
72 uint32_t *staging_buffer;
73 VkResult result;
74
75 info->entries =
76 vk_zalloc(&device->vk.alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
77 if (!info->entries)
78 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
79
80 info->entries_size_in_bytes = size;
81
82 sec_pds_program = (struct pvr_pds_descriptor_program_input){
83 .buffer_count = 1,
84 .buffers = {
85 [0] = {
86 .buffer_id = 0,
87 .source_offset = 0,
88 .type = PVR_BUFFER_TYPE_COMPILE_TIME,
89 .size_in_dwords = const_shared_regs,
90 .destination = 0,
91 }
92 },
93 };
94
95 pvr_pds_generate_descriptor_upload_program(&sec_pds_program, NULL, info);
96
97 staging_buffer_size = info->code_size_in_dwords;
98
99 staging_buffer = vk_alloc(&device->vk.alloc,
100 PVR_DW_TO_BYTES(staging_buffer_size),
101 8,
102 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
103 if (!staging_buffer) {
104 vk_free(&device->vk.alloc, info->entries);
105 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
106 }
107
108 pvr_pds_generate_descriptor_upload_program(&sec_pds_program,
109 staging_buffer,
110 info);
111
112 assert(info->code_size_in_dwords <= staging_buffer_size);
113
114 /* FIXME: Figure out the define for alignment of 16. */
115 result = pvr_gpu_upload_pds(device,
116 NULL,
117 0,
118 0,
119 staging_buffer,
120 info->code_size_in_dwords,
121 16,
122 16,
123 &query_prog->pds_sec_code);
124 if (result != VK_SUCCESS) {
125 vk_free(&device->vk.alloc, staging_buffer);
126 vk_free(&device->vk.alloc, info->entries);
127 return result;
128 }
129
130 vk_free(&device->vk.alloc, staging_buffer);
131
132 return VK_SUCCESS;
133 }
134
135 static void
136 pvr_destroy_compute_secondary_prog(struct pvr_device *device,
137 struct pvr_compute_query_shader *program)
138 {
139 pvr_bo_suballoc_free(program->pds_sec_code.pvr_bo);
140 vk_free(&device->vk.alloc, program->info.entries);
141 }
142
143 static VkResult pvr_create_compute_query_precomp_program(
144 struct pvr_device *device,
145 enum pco_usclib_program common_program_index,
146 unsigned const_shared_regs,
147 struct pvr_compute_query_shader *query_prog)
148 {
149 const uint32_t cache_line_size =
150 pvr_get_slc_cache_line_size(&device->pdevice->dev_info);
151 struct pvr_pds_compute_shader_program pds_primary_prog = { 0 };
152 const pco_precomp_data *precomp_data;
153 VkResult result;
154
155 memset(query_prog, 0, sizeof(*query_prog));
156
157 precomp_data = (pco_precomp_data *)pco_usclib_common[common_program_index];
158 result = pvr_gpu_upload_usc(device,
159 precomp_data->binary,
160 precomp_data->size_dwords * sizeof(uint32_t),
161 cache_line_size,
162 &query_prog->usc_bo);
163 if (result != VK_SUCCESS)
164 return result;
165
166 pvr_init_primary_compute_pds_program(&pds_primary_prog);
167
168 pvr_pds_setup_doutu(&pds_primary_prog.usc_task_control,
169 query_prog->usc_bo->dev_addr.addr,
170 precomp_data->temps,
171 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
172 false);
173
174 result =
175 pvr_arch_pds_compute_shader_create_and_upload(device,
176 &pds_primary_prog,
177 &query_prog->pds_prim_code);
178 if (result != VK_SUCCESS)
179 goto err_free_usc_bo;
180
181 query_prog->primary_data_size_dw = pds_primary_prog.data_size;
182 query_prog->primary_num_temps = pds_primary_prog.temps_used;
183
184 result =
185 pvr_create_compute_secondary_prog(device, const_shared_regs, query_prog);
186 if (result != VK_SUCCESS)
187 goto err_free_pds_prim_code_bo;
188
189 return VK_SUCCESS;
190
191 err_free_pds_prim_code_bo:
192 pvr_bo_suballoc_free(query_prog->pds_prim_code.pvr_bo);
193
194 err_free_usc_bo:
195 pvr_bo_suballoc_free(query_prog->usc_bo);
196
197 return result;
198 }
199
200 /* TODO: See if we can dedup this with pvr_setup_descriptor_mappings() or
201 * pvr_setup_descriptor_mappings().
202 */
203 static VkResult pvr_write_compute_query_pds_data_section(
204 struct pvr_cmd_buffer *cmd_buffer,
205 const struct pvr_compute_query_shader *query_prog,
206 struct pvr_private_compute_pipeline *pipeline)
207 {
208 const struct pvr_pds_info *const info = &query_prog->info;
209 struct pvr_suballoc_bo *pvr_bo;
210 const uint8_t *entries;
211 uint32_t *dword_buffer;
212 uint64_t *qword_buffer;
213 VkResult result;
214
215 result =
216 pvr_arch_cmd_buffer_alloc_mem(cmd_buffer,
217 cmd_buffer->device->heaps.pds_heap,
218 PVR_DW_TO_BYTES(info->data_size_in_dwords),
219 &pvr_bo);
220 if (result != VK_SUCCESS)
221 return result;
222
223 dword_buffer = (uint32_t *)pvr_bo_suballoc_get_map_addr(pvr_bo);
224 qword_buffer = (uint64_t *)pvr_bo_suballoc_get_map_addr(pvr_bo);
225
226 entries = (uint8_t *)info->entries;
227
228 /* TODO: Remove this when we can test this path and make sure that this is
229 * not needed. If it's needed we should probably be using LITERAL entries for
230 * this instead.
231 */
232 #if !defined(NDEBUG)
233 memset(dword_buffer, 0xFE, PVR_DW_TO_BYTES(info->data_size_in_dwords));
234 #endif /* !defined(NDEBUG) */
235
236 pipeline->pds_shared_update_data_size_dw = info->data_size_in_dwords;
237
238 for (uint32_t i = 0; i < info->entry_count; i++) {
239 const struct pvr_const_map_entry *const entry_header =
240 (struct pvr_const_map_entry *)entries;
241
242 switch (entry_header->type) {
243 case PVR_PDS_CONST_MAP_ENTRY_TYPE_LITERAL32: {
244 const struct pvr_const_map_entry_literal32 *const literal =
245 (struct pvr_const_map_entry_literal32 *)entries;
246
247 PVR_WRITE(dword_buffer,
248 literal->literal_value,
249 literal->const_offset,
250 info->data_size_in_dwords);
251
252 entries += sizeof(*literal);
253 break;
254 }
255 case PVR_PDS_CONST_MAP_ENTRY_TYPE_LITERAL64: {
256 const struct pvr_const_map_entry_literal64 *const literal =
257 (struct pvr_const_map_entry_literal64 *)entries;
258
259 PVR_WRITE(qword_buffer,
260 literal->literal_value,
261 literal->const_offset,
262 info->data_size_in_dwords);
263
264 entries += sizeof(*literal);
265 break;
266 }
267 case PVR_PDS_CONST_MAP_ENTRY_TYPE_DOUTU_ADDRESS: {
268 const struct pvr_const_map_entry_doutu_address *const doutu_addr =
269 (struct pvr_const_map_entry_doutu_address *)entries;
270 const pvr_dev_addr_t exec_addr =
271 PVR_DEV_ADDR_OFFSET(query_prog->pds_sec_code.pvr_bo->dev_addr,
272 query_prog->pds_sec_code.code_offset);
273 uint64_t addr = 0ULL;
274
275 pvr_set_usc_execution_address64(&addr, exec_addr.addr);
276
277 PVR_WRITE(qword_buffer,
278 addr | doutu_addr->doutu_control,
279 doutu_addr->const_offset,
280 info->data_size_in_dwords);
281
282 entries += sizeof(*doutu_addr);
283 break;
284 }
285 case PVR_PDS_CONST_MAP_ENTRY_TYPE_SPECIAL_BUFFER: {
286 const struct pvr_const_map_entry_special_buffer *special_buff_entry =
287 (struct pvr_const_map_entry_special_buffer *)entries;
288
289 switch (special_buff_entry->buffer_type) {
290 case PVR_BUFFER_TYPE_COMPILE_TIME: {
291 uint64_t addr = pipeline->const_buffer_addr.addr;
292
293 PVR_WRITE(qword_buffer,
294 addr,
295 special_buff_entry->const_offset,
296 info->data_size_in_dwords);
297 break;
298 }
299
300 default:
301 UNREACHABLE("Unsupported special buffer type.");
302 }
303
304 entries += sizeof(*special_buff_entry);
305 break;
306 }
307 default:
308 UNREACHABLE("Unsupported data section map");
309 }
310 }
311
312 pipeline->pds_shared_update_data_offset =
313 pvr_bo->dev_addr.addr -
314 cmd_buffer->device->heaps.pds_heap->base_addr.addr;
315
316 return VK_SUCCESS;
317 }
318
319 static void pvr_write_private_compute_dispatch(
320 struct pvr_cmd_buffer *cmd_buffer,
321 struct pvr_private_compute_pipeline *pipeline,
322 uint32_t num_query_indices)
323 {
324 struct pvr_sub_cmd *sub_cmd = cmd_buffer->state.current_sub_cmd;
325 const uint32_t workgroup_size[PVR_WORKGROUP_DIMENSIONS] = {
326 DIV_ROUND_UP(num_query_indices, 32),
327 1,
328 1,
329 };
330
331 assert(sub_cmd->type == PVR_SUB_CMD_TYPE_QUERY);
332
333 pvr_arch_compute_update_shared_private(cmd_buffer,
334 &sub_cmd->compute,
335 pipeline);
336 pvr_arch_compute_update_kernel_private(cmd_buffer,
337 &sub_cmd->compute,
338 pipeline,
339 workgroup_size);
340 pvr_arch_compute_generate_fence(cmd_buffer, &sub_cmd->compute, false);
341 }
342
343 static void
344 pvr_destroy_compute_query_program(struct pvr_device *device,
345 struct pvr_compute_query_shader *program)
346 {
347 pvr_destroy_compute_secondary_prog(device, program);
348 pvr_bo_suballoc_free(program->pds_prim_code.pvr_bo);
349 pvr_bo_suballoc_free(program->usc_bo);
350 }
351
352 VkResult
353 pvr_arch_device_create_compute_query_programs(struct pvr_device *device)
354 {
355 VkResult result;
356
357 result = pvr_create_compute_query_precomp_program(
358 device,
359 CS_QUERY_AVAILABILITY_COMMON,
360 _PVR_QUERY_AVAILABILITY_DATA_COUNT,
361 &device->availability_shader);
362
363 if (result != VK_SUCCESS)
364 return result;
365
366 result =
367 pvr_create_compute_query_precomp_program(device,
368 CS_QUERY_COPY_COMMON,
369 _PVR_QUERY_COPY_DATA_COUNT,
370 &device->copy_results_shader);
371
372 if (result != VK_SUCCESS)
373 goto err_destroy_availability_query_program;
374
375 result =
376 pvr_create_compute_query_precomp_program(device,
377 CS_QUERY_RESET_COMMON,
378 _PVR_QUERY_RESET_DATA_COUNT,
379 &device->reset_queries_shader);
380
381 if (result != VK_SUCCESS)
382 goto err_destroy_copy_results_query_program;
383
384 return VK_SUCCESS;
385
386 err_destroy_copy_results_query_program:
387 pvr_destroy_compute_query_program(device, &device->copy_results_shader);
388
389 err_destroy_availability_query_program:
390 pvr_destroy_compute_query_program(device, &device->availability_shader);
391
392 return result;
393 }
394
395 void pvr_arch_device_destroy_compute_query_programs(struct pvr_device *device)
396 {
397 pvr_destroy_compute_query_program(device, &device->availability_shader);
398 pvr_destroy_compute_query_program(device, &device->copy_results_shader);
399 pvr_destroy_compute_query_program(device, &device->reset_queries_shader);
400 }
401
402 /* TODO: Split this function into per program type functions. */
403 VkResult pvr_arch_add_query_program(struct pvr_cmd_buffer *cmd_buffer,
404 const struct pvr_query_info *query_info)
405 {
406 struct pvr_device *device = cmd_buffer->device;
407 const struct pvr_compute_query_shader *query_prog;
408 struct pvr_private_compute_pipeline pipeline;
409 uint32_t num_query_indices;
410 uint32_t *const_buffer;
411 struct pvr_suballoc_bo *pvr_bo;
412 VkResult result;
413
414 result =
415 pvr_arch_cmd_buffer_start_sub_cmd(cmd_buffer, PVR_SUB_CMD_TYPE_QUERY);
416 if (result != VK_SUCCESS)
417 return result;
418
419 cmd_buffer->state.current_sub_cmd->is_dynamic_render =
420 query_info->is_dynamic_render;
421 cmd_buffer->state.current_sub_cmd->is_suspend = query_info->is_suspend;
422
423 switch (query_info->type) {
424 case PVR_QUERY_TYPE_AVAILABILITY_WRITE:
425 /* Adds a compute shader (fenced on the last 3D) that writes a non-zero
426 * value in availability_bo at every index in index_bo.
427 */
428 query_prog = &device->availability_shader;
429 num_query_indices = query_info->availability_write.num_query_indices;
430 pipeline.const_shared_regs_count = _PVR_QUERY_AVAILABILITY_DATA_COUNT;
431 break;
432
433 case PVR_QUERY_TYPE_COPY_QUERY_RESULTS:
434 /* Adds a compute shader to copy availability and query value data. */
435 query_prog = &device->copy_results_shader;
436 num_query_indices = query_info->copy_query_results.query_count;
437 pipeline.const_shared_regs_count = _PVR_QUERY_COPY_DATA_COUNT;
438 break;
439
440 case PVR_QUERY_TYPE_RESET_QUERY_POOL:
441 /* Adds a compute shader to reset availability and query value data. */
442 query_prog = &device->reset_queries_shader;
443 num_query_indices = query_info->reset_query_pool.query_count;
444 pipeline.const_shared_regs_count = _PVR_QUERY_RESET_DATA_COUNT;
445 break;
446
447 default:
448 UNREACHABLE("Invalid query type");
449 }
450
451 pipeline.pds_code_offset = query_prog->pds_prim_code.code_offset;
452 pipeline.pds_data_offset = query_prog->pds_prim_code.data_offset;
453
454 pipeline.pds_shared_update_code_offset =
455 query_prog->pds_sec_code.code_offset;
456 pipeline.pds_data_size_dw = query_prog->primary_data_size_dw;
457 pipeline.pds_temps_used = query_prog->primary_num_temps;
458
459 /* TODO: set properly. */
460 pipeline.coeff_regs_count = 3;
461 pipeline.unified_store_regs_count = 8;
462
463 const_buffer = vk_alloc(&cmd_buffer->vk.pool->alloc,
464 PVR_DW_TO_BYTES(pipeline.const_shared_regs_count),
465 8,
466 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
467 if (!const_buffer) {
468 return vk_command_buffer_set_error(&cmd_buffer->vk,
469 VK_ERROR_OUT_OF_HOST_MEMORY);
470 }
471
472 switch (query_info->type) {
473 case PVR_QUERY_TYPE_AVAILABILITY_WRITE: {
474 uint64_t index_addr =
475 query_info->availability_write.index_bo->dev_addr.addr;
476
477 uint64_t avail_addr =
478 query_info->availability_write.availability_bo->dev_addr.addr;
479
480 const_buffer[PVR_QUERY_AVAILABILITY_DATA_INDEX_COUNT] = num_query_indices;
481 const_buffer[PVR_QUERY_AVAILABILITY_DATA_INDEX_BO_LO] = index_addr &
482 0xffffffff;
483 const_buffer[PVR_QUERY_AVAILABILITY_DATA_INDEX_BO_HI] = index_addr >> 32;
484 const_buffer[PVR_QUERY_AVAILABILITY_DATA_BO_LO] = avail_addr & 0xffffffff;
485 const_buffer[PVR_QUERY_AVAILABILITY_DATA_BO_HI] = avail_addr >> 32;
486
487 break;
488 }
489
490 case PVR_QUERY_TYPE_COPY_QUERY_RESULTS: {
491 VK_FROM_HANDLE(pvr_query_pool,
492 pool,
493 query_info->copy_query_results.query_pool);
494 VK_FROM_HANDLE(pvr_buffer,
495 buffer,
496 query_info->copy_query_results.dst_buffer);
497
498 pvr_dev_addr_t dev_addr;
499
500 /* Assert if no memory is bound to destination buffer. */
501 assert(buffer->dev_addr.addr);
502
503 uint64_t offset =
504 query_info->copy_query_results.first_query * sizeof(uint32_t);
505
506 dev_addr = PVR_DEV_ADDR_OFFSET(buffer->dev_addr,
507 query_info->copy_query_results.dst_offset);
508 uint64_t dest_addr = dev_addr.addr;
509
510 dev_addr =
511 PVR_DEV_ADDR_OFFSET(pool->availability_buffer->dev_addr, offset);
512 uint64_t avail_addr = dev_addr.addr;
513
514 dev_addr = PVR_DEV_ADDR_OFFSET(pool->result_buffer->dev_addr, offset);
515 uint64_t result_addr = dev_addr.addr;
516
517 const_buffer[PVR_QUERY_COPY_DATA_INDEX_COUNT] = num_query_indices;
518
519 const_buffer[PVR_QUERY_COPY_DATA_DEST_BO_LO] = dest_addr & 0xffffffff;
520 const_buffer[PVR_QUERY_COPY_DATA_DEST_BO_HI] = dest_addr >> 32;
521
522 const_buffer[PVR_QUERY_COPY_DATA_AVAILABILITY_BO_LO] = avail_addr &
523 0xffffffff;
524 const_buffer[PVR_QUERY_COPY_DATA_AVAILABILITY_BO_HI] = avail_addr >> 32;
525
526 const_buffer[PVR_QUERY_COPY_DATA_RESULT_BO_LO] = result_addr & 0xffffffff;
527 const_buffer[PVR_QUERY_COPY_DATA_RESULT_BO_HI] = result_addr >> 32;
528
529 const_buffer[PVR_QUERY_COPY_DATA_DEST_STRIDE] =
530 query_info->copy_query_results.stride;
531
532 const_buffer[PVR_QUERY_COPY_DATA_FLAGS] =
533 query_info->copy_query_results.flags;
534
535 break;
536 }
537
538 case PVR_QUERY_TYPE_RESET_QUERY_POOL: {
539 VK_FROM_HANDLE(pvr_query_pool,
540 pool,
541 query_info->reset_query_pool.query_pool);
542
543 uint64_t offset =
544 query_info->reset_query_pool.first_query * sizeof(uint32_t);
545
546 pvr_dev_addr_t dev_addr =
547 PVR_DEV_ADDR_OFFSET(pool->result_buffer->dev_addr, offset);
548 uint64_t result_addr = dev_addr.addr;
549
550 dev_addr =
551 PVR_DEV_ADDR_OFFSET(pool->availability_buffer->dev_addr, offset);
552 uint64_t avail_addr = dev_addr.addr;
553
554 const_buffer[PVR_QUERY_RESET_DATA_INDEX_COUNT] = num_query_indices;
555 const_buffer[PVR_QUERY_RESET_DATA_RESULT_BO_LO] = result_addr &
556 0xffffffff;
557 const_buffer[PVR_QUERY_RESET_DATA_RESULT_BO_HI] = result_addr >> 32;
558 const_buffer[PVR_QUERY_RESET_DATA_AVAILABILITY_BO_LO] = avail_addr &
559 0xffffffff;
560 const_buffer[PVR_QUERY_RESET_DATA_AVAILABILITY_BO_HI] = avail_addr >> 32;
561
562 break;
563 }
564
565 default:
566 UNREACHABLE("Invalid query type");
567 }
568
569 result = pvr_arch_cmd_buffer_upload_general(
570 cmd_buffer,
571 const_buffer,
572 PVR_DW_TO_BYTES(pipeline.const_shared_regs_count),
573 &pvr_bo);
574 if (result != VK_SUCCESS) {
575 vk_free(&cmd_buffer->vk.pool->alloc, const_buffer);
576
577 return result;
578 }
579
580 pipeline.const_buffer_addr = pvr_bo->dev_addr;
581
582 vk_free(&cmd_buffer->vk.pool->alloc, const_buffer);
583
584 /* PDS data section for the secondary/constant upload. */
585 result = pvr_write_compute_query_pds_data_section(cmd_buffer,
586 query_prog,
587 &pipeline);
588 if (result != VK_SUCCESS)
589 return result;
590
591 pipeline.workgroup_size.width = ROGUE_MAX_INSTANCES_PER_TASK;
592 pipeline.workgroup_size.height = 1;
593 pipeline.workgroup_size.depth = 1;
594
595 pvr_write_private_compute_dispatch(cmd_buffer, &pipeline, num_query_indices);
596
597 return pvr_arch_cmd_buffer_end_sub_cmd(cmd_buffer);
598 }