Debian Codegraph @

src/imagination/vulkan/pvr_arch_queue.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on radv driver which is:
5 * Copyright © 2016 Red Hat.
6 * Copyright © 2016 Bas Nieuwenhuizen
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25 * SOFTWARE.
26 */
27
28 /**
29 * This file implements VkQueue, VkFence, and VkSemaphore
30 */
31
32 #include "pvr_queue.h"
33
34 #include <assert.h>
35 #include <stdbool.h>
36 #include <stddef.h>
37 #include <stdint.h>
38 #include <unistd.h>
39 #include <vulkan/vulkan.h>
40
41 #include "pvr_cmd_buffer.h"
42 #include "pvr_device.h"
43 #include "pvr_entrypoints.h"
44 #include "pvr_job_compute.h"
45 #include "pvr_job_context.h"
46 #include "pvr_job_render.h"
47 #include "pvr_job_transfer.h"
48 #include "pvr_limits.h"
49 #include "pvr_macros.h"
50 #include "pvr_physical_device.h"
51 #include "pvr_pipeline.h"
52
53 #include "util/macros.h"
54 #include "util/u_atomic.h"
55 #include "vk_alloc.h"
56 #include "vk_fence.h"
57 #include "vk_log.h"
58 #include "vk_object.h"
59 #include "vk_queue.h"
60 #include "vk_semaphore.h"
61 #include "vk_sync.h"
62 #include "vk_sync_dummy.h"
63 #include "vk_util.h"
64
65 static VkResult pvr_driver_queue_submit(struct vk_queue *queue,
66 struct vk_queue_submit *submit);
67
68 static VkResult pvr_queue_init(struct pvr_device *device,
69 struct pvr_queue *queue,
70 const VkDeviceQueueCreateInfo *pCreateInfo,
71 uint32_t index_in_family)
72 {
73 struct pvr_transfer_ctx *transfer_ctx;
74 struct pvr_compute_ctx *compute_ctx;
75 struct pvr_compute_ctx *query_ctx;
76 struct pvr_render_ctx *gfx_ctx;
77 VkResult result;
78
79 *queue = (struct pvr_queue){ 0 };
80
81 result =
82 vk_queue_init(&queue->vk, &device->vk, pCreateInfo, index_in_family);
83 if (result != VK_SUCCESS)
84 return result;
85
86 if (device->ws->features.supports_threaded_submit) {
87 result = vk_queue_enable_submit_thread(&queue->vk);
88 if (result != VK_SUCCESS)
89 goto err_vk_queue_finish;
90 }
91
92 result = pvr_arch_transfer_ctx_create(device,
93 PVR_WINSYS_CTX_PRIORITY_MEDIUM,
94 &transfer_ctx);
95 if (result != VK_SUCCESS)
96 goto err_vk_queue_finish;
97
98 result = pvr_arch_compute_ctx_create(device,
99 PVR_WINSYS_CTX_PRIORITY_MEDIUM,
100 &compute_ctx);
101 if (result != VK_SUCCESS)
102 goto err_transfer_ctx_destroy;
103
104 result = pvr_arch_compute_ctx_create(device,
105 PVR_WINSYS_CTX_PRIORITY_MEDIUM,
106 &query_ctx);
107 if (result != VK_SUCCESS)
108 goto err_compute_ctx_destroy;
109
110 result = pvr_arch_render_ctx_create(device,
111 PVR_WINSYS_CTX_PRIORITY_MEDIUM,
112 &gfx_ctx);
113 if (result != VK_SUCCESS)
114 goto err_query_ctx_destroy;
115
116 queue->device = device;
117 queue->gfx_ctx = gfx_ctx;
118 queue->compute_ctx = compute_ctx;
119 queue->query_ctx = query_ctx;
120 queue->transfer_ctx = transfer_ctx;
121
122 queue->vk.driver_submit = pvr_driver_queue_submit;
123
124 return VK_SUCCESS;
125
126 err_query_ctx_destroy:
127 pvr_arch_compute_ctx_destroy(query_ctx);
128
129 err_compute_ctx_destroy:
130 pvr_arch_compute_ctx_destroy(compute_ctx);
131
132 err_transfer_ctx_destroy:
133 pvr_arch_transfer_ctx_destroy(transfer_ctx);
134
135 err_vk_queue_finish:
136 vk_queue_finish(&queue->vk);
137
138 return result;
139 }
140
141 VkResult pvr_arch_queues_create(struct pvr_device *device,
142 const VkDeviceCreateInfo *pCreateInfo)
143 {
144 VkResult result;
145
146 /* Check requested queue families and queues */
147 assert(pCreateInfo->queueCreateInfoCount == 1);
148 assert(pCreateInfo->pQueueCreateInfos[0].queueFamilyIndex == 0);
149 assert(pCreateInfo->pQueueCreateInfos[0].queueCount <= PVR_MAX_QUEUES);
150
151 const VkDeviceQueueCreateInfo *queue_create =
152 &pCreateInfo->pQueueCreateInfos[0];
153
154 device->queues = vk_alloc(&device->vk.alloc,
155 queue_create->queueCount * sizeof(*device->queues),
156 8,
157 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
158 if (!device->queues)
159 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
160
161 device->queue_count = 0;
162
163 for (uint32_t i = 0; i < queue_create->queueCount; i++) {
164 result = pvr_queue_init(device, &device->queues[i], queue_create, i);
165 if (result != VK_SUCCESS)
166 goto err_queues_finish;
167
168 device->queue_count++;
169 }
170
171 return VK_SUCCESS;
172
173 err_queues_finish:
174 pvr_arch_queues_destroy(device);
175 return result;
176 }
177
178 static void pvr_queue_finish(struct pvr_queue *queue)
179 {
180 for (uint32_t i = 0; i < ARRAY_SIZE(queue->next_job_wait_sync); i++) {
181 if (queue->next_job_wait_sync[i])
182 vk_sync_destroy(&queue->device->vk, queue->next_job_wait_sync[i]);
183 }
184
185 for (uint32_t i = 0; i < ARRAY_SIZE(queue->last_job_signal_sync); i++) {
186 if (queue->last_job_signal_sync[i])
187 vk_sync_destroy(&queue->device->vk, queue->last_job_signal_sync[i]);
188 }
189
190 pvr_arch_render_ctx_destroy(queue->gfx_ctx);
191 pvr_arch_compute_ctx_destroy(queue->query_ctx);
192 pvr_arch_compute_ctx_destroy(queue->compute_ctx);
193 pvr_arch_transfer_ctx_destroy(queue->transfer_ctx);
194
195 vk_queue_finish(&queue->vk);
196 }
197
198 void pvr_arch_queues_destroy(struct pvr_device *device)
199 {
200 for (uint32_t q_idx = 0; q_idx < device->queue_count; q_idx++)
201 pvr_queue_finish(&device->queues[q_idx]);
202
203 vk_free(&device->vk.alloc, device->queues);
204 }
205
206 static void pvr_update_job_syncs(struct pvr_device *device,
207 struct pvr_queue *queue,
208 struct vk_sync *new_signal_sync,
209 enum pvr_job_type submitted_job_type)
210 {
211 if (queue->next_job_wait_sync[submitted_job_type]) {
212 vk_sync_destroy(&device->vk,
213 queue->next_job_wait_sync[submitted_job_type]);
214 queue->next_job_wait_sync[submitted_job_type] = NULL;
215 }
216
217 if (queue->last_job_signal_sync[submitted_job_type]) {
218 vk_sync_destroy(&device->vk,
219 queue->last_job_signal_sync[submitted_job_type]);
220 }
221
222 queue->last_job_signal_sync[submitted_job_type] = new_signal_sync;
223 }
224
225 static VkResult
226 pvr_process_graphics_cmd_for_view(struct pvr_device *device,
227 struct pvr_queue *queue,
228 struct pvr_cmd_buffer *cmd_buffer,
229 struct pvr_sub_cmd_gfx *sub_cmd,
230 uint32_t view_index)
231 {
232 pvr_dev_addr_t original_ctrl_stream_addr = { 0 };
233 struct pvr_render_job *job = &sub_cmd->job;
234 struct vk_sync *geom_signal_sync;
235 struct vk_sync *frag_signal_sync = NULL;
236 VkResult result;
237
238 job->ds.addr =
239 PVR_DEV_ADDR_OFFSET(job->ds.addr, job->ds.layer_size * view_index);
240 job->view_state.view_index = view_index;
241
242 result = vk_sync_create(&device->vk,
243 &device->pdevice->ws->syncobj_type,
244 0U,
245 0UL,
246 &geom_signal_sync);
247 if (result != VK_SUCCESS)
248 return result;
249
250 if (job->run_frag) {
251 result = vk_sync_create(&device->vk,
252 &device->pdevice->ws->syncobj_type,
253 0U,
254 0UL,
255 &frag_signal_sync);
256 if (result != VK_SUCCESS)
257 goto err_destroy_geom_sync;
258 }
259
260 /* FIXME: DoShadowLoadOrStore() */
261
262 /* Perform two render submits when using multiple framebuffer layers. The
263 * first submit contains just geometry, while the second only terminates
264 * (and triggers the fragment render if originally specified). This is needed
265 * because the render target cache gets cleared on terminating submits, which
266 * could result in missing primitives.
267 */
268 if (pvr_sub_cmd_gfx_requires_split_submit(sub_cmd)) {
269 /* If fragment work shouldn't be run there's no need for a split,
270 * and if geometry_terminate is false this kick can't have a fragment
271 * stage without another terminating geometry kick.
272 */
273 assert(job->geometry_terminate && job->run_frag);
274
275 /* First submit must not touch fragment work. */
276 job->geometry_terminate = false;
277 job->run_frag = false;
278
279 result = pvr_arch_render_job_submit(
280 queue->gfx_ctx,
281 &sub_cmd->job,
282 queue->next_job_wait_sync[PVR_JOB_TYPE_GEOM],
283 NULL,
284 NULL,
285 NULL);
286
287 job->geometry_terminate = true;
288 job->run_frag = true;
289
290 if (result != VK_SUCCESS)
291 goto err_destroy_frag_sync;
292
293 original_ctrl_stream_addr = job->ctrl_stream_addr;
294
295 /* Second submit contains only a trivial control stream to terminate the
296 * geometry work.
297 */
298 assert(sub_cmd->terminate_ctrl_stream);
299 job->ctrl_stream_addr = sub_cmd->terminate_ctrl_stream->vma->dev_addr;
300 } else if (sub_cmd->view_index_wanted) {
301 original_ctrl_stream_addr = job->ctrl_stream_addr;
302 job->ctrl_stream_addr.addr =
303 sub_cmd->multiview_ctrl_stream->vma->dev_addr.addr +
304 (view_index * PVR_DW_TO_BYTES(sub_cmd->multiview_ctrl_stream_stride));
305 }
306
307 result =
308 pvr_arch_render_job_submit(queue->gfx_ctx,
309 &sub_cmd->job,
310 queue->next_job_wait_sync[PVR_JOB_TYPE_GEOM],
311 queue->next_job_wait_sync[PVR_JOB_TYPE_FRAG],
312 geom_signal_sync,
313 frag_signal_sync);
314
315 if (original_ctrl_stream_addr.addr > 0)
316 job->ctrl_stream_addr = original_ctrl_stream_addr;
317
318 if (result != VK_SUCCESS)
319 goto err_destroy_frag_sync;
320
321 pvr_update_job_syncs(device, queue, geom_signal_sync, PVR_JOB_TYPE_GEOM);
322
323 if (job->run_frag)
324 pvr_update_job_syncs(device, queue, frag_signal_sync, PVR_JOB_TYPE_FRAG);
325
326 /* FIXME: DoShadowLoadOrStore() */
327
328 return VK_SUCCESS;
329
330 err_destroy_frag_sync:
331 if (frag_signal_sync)
332 vk_sync_destroy(&device->vk, frag_signal_sync);
333 err_destroy_geom_sync:
334 vk_sync_destroy(&device->vk, geom_signal_sync);
335
336 return result;
337 }
338
339 static VkResult pvr_process_graphics_cmd(struct pvr_device *device,
340 struct pvr_queue *queue,
341 struct pvr_cmd_buffer *cmd_buffer,
342 struct pvr_sub_cmd_gfx *sub_cmd)
343 {
344 const pvr_dev_addr_t ds_addr = sub_cmd->job.ds.addr;
345
346 u_foreach_bit (view_idx, sub_cmd->view_mask) {
347 VkResult result;
348
349 result = pvr_process_graphics_cmd_for_view(device,
350 queue,
351 cmd_buffer,
352 sub_cmd,
353 view_idx);
354 sub_cmd->job.ds.addr = ds_addr;
355
356 if (result != VK_SUCCESS)
357 return result;
358 }
359
360 return VK_SUCCESS;
361 }
362
363 static VkResult pvr_process_compute_cmd(struct pvr_device *device,
364 struct pvr_queue *queue,
365 struct pvr_sub_cmd_compute *sub_cmd)
366 {
367 struct vk_sync *sync;
368 VkResult result;
369
370 result = vk_sync_create(&device->vk,
371 &device->pdevice->ws->syncobj_type,
372 0U,
373 0UL,
374 &sync);
375 if (result != VK_SUCCESS)
376 return result;
377
378 result = pvr_arch_compute_job_submit(
379 queue->compute_ctx,
380 sub_cmd,
381 queue->next_job_wait_sync[PVR_JOB_TYPE_COMPUTE],
382 sync);
383 if (result != VK_SUCCESS) {
384 vk_sync_destroy(&device->vk, sync);
385 return result;
386 }
387
388 pvr_update_job_syncs(device, queue, sync, PVR_JOB_TYPE_COMPUTE);
389
390 return result;
391 }
392
393 static VkResult pvr_process_transfer_cmds(struct pvr_device *device,
394 struct pvr_queue *queue,
395 struct pvr_sub_cmd_transfer *sub_cmd)
396 {
397 struct vk_sync *sync;
398 VkResult result;
399
400 result = vk_sync_create(&device->vk,
401 &device->pdevice->ws->syncobj_type,
402 0U,
403 0UL,
404 &sync);
405 if (result != VK_SUCCESS)
406 return result;
407
408 result = pvr_arch_transfer_job_submit(
409 queue->transfer_ctx,
410 sub_cmd,
411 queue->next_job_wait_sync[PVR_JOB_TYPE_TRANSFER],
412 sync);
413 if (result != VK_SUCCESS) {
414 vk_sync_destroy(&device->vk, sync);
415 return result;
416 }
417
418 pvr_update_job_syncs(device, queue, sync, PVR_JOB_TYPE_TRANSFER);
419
420 return result;
421 }
422
423 static VkResult pvr_process_query_cmd(struct pvr_device *device,
424 struct pvr_queue *queue,
425 struct pvr_sub_cmd_compute *sub_cmd)
426 {
427 struct vk_sync *sync;
428 VkResult result;
429
430 /* TODO: Currently we add barrier event sub commands to handle the sync
431 * necessary for the different query types. Would we get any speed
432 * up in processing the queue by doing that sync here without using event sub
433 * commands?
434 */
435
436 result = vk_sync_create(&device->vk,
437 &device->pdevice->ws->syncobj_type,
438 0U,
439 0UL,
440 &sync);
441 if (result != VK_SUCCESS)
442 return result;
443
444 result =
445 pvr_arch_compute_job_submit(queue->query_ctx,
446 sub_cmd,
447 queue->next_job_wait_sync[PVR_JOB_TYPE_QUERY],
448 sync);
449 if (result != VK_SUCCESS) {
450 vk_sync_destroy(&device->vk, sync);
451 return result;
452 }
453
454 pvr_update_job_syncs(device, queue, sync, PVR_JOB_TYPE_QUERY);
455
456 return result;
457 }
458
459 static VkResult
460 pvr_process_event_cmd_barrier(struct pvr_device *device,
461 struct pvr_queue *queue,
462 struct pvr_sub_cmd_event_barrier *sub_cmd)
463 {
464 const uint32_t src_mask = sub_cmd->wait_for_stage_mask;
465 const uint32_t dst_mask = sub_cmd->wait_at_stage_mask;
466 struct vk_sync_wait wait_syncs[PVR_JOB_TYPE_MAX + 1];
467 uint32_t src_wait_count = 0;
468 VkResult result;
469
470 assert(!(src_mask &
471 ~(PVR_PIPELINE_STAGE_ALL_BITS | PVR_PIPELINE_STAGE_QUERY_BIT)));
472 assert(!(dst_mask &
473 ~(PVR_PIPELINE_STAGE_ALL_BITS | PVR_PIPELINE_STAGE_QUERY_BIT)));
474
475 u_foreach_bit (stage, src_mask) {
476 if (queue->last_job_signal_sync[stage]) {
477 wait_syncs[src_wait_count++] = (struct vk_sync_wait){
478 .sync = queue->last_job_signal_sync[stage],
479 .stage_mask = ~(VkPipelineStageFlags2)0,
480 .wait_value = 0,
481 };
482 }
483 }
484
485 /* No previous src jobs that need finishing so no need for a barrier. */
486 if (src_wait_count == 0)
487 return VK_SUCCESS;
488
489 u_foreach_bit (stage, dst_mask) {
490 uint32_t wait_count = src_wait_count;
491 struct vk_sync_signal signal;
492 struct vk_sync *signal_sync;
493
494 result = vk_sync_create(&device->vk,
495 &device->pdevice->ws->syncobj_type,
496 0U,
497 0UL,
498 &signal_sync);
499 if (result != VK_SUCCESS)
500 return result;
501
502 signal = (struct vk_sync_signal){
503 .sync = signal_sync,
504 .stage_mask = ~(VkPipelineStageFlags2)0,
505 .signal_value = 0,
506 };
507
508 if (queue->next_job_wait_sync[stage]) {
509 wait_syncs[wait_count++] = (struct vk_sync_wait){
510 .sync = queue->next_job_wait_sync[stage],
511 .stage_mask = ~(VkPipelineStageFlags2)0,
512 .wait_value = 0,
513 };
514 }
515
516 result = device->ws->ops->null_job_submit(device->ws,
517 wait_syncs,
518 wait_count,
519 &signal);
520 if (result != VK_SUCCESS) {
521 vk_sync_destroy(&device->vk, signal_sync);
522 return result;
523 }
524
525 if (queue->next_job_wait_sync[stage])
526 vk_sync_destroy(&device->vk, queue->next_job_wait_sync[stage]);
527
528 queue->next_job_wait_sync[stage] = signal_sync;
529 }
530
531 return VK_SUCCESS;
532 }
533
534 static VkResult
535 pvr_process_event_cmd_set_or_reset(struct pvr_device *device,
536 struct pvr_queue *queue,
537 struct pvr_sub_cmd_event_set_reset *sub_cmd,
538 const enum pvr_event_state new_event_state)
539 {
540 /* Not PVR_JOB_TYPE_MAX since that also includes
541 * PVR_JOB_TYPE_QUERY so no stage in the src mask.
542 */
543 struct vk_sync_wait waits[PVR_NUM_SYNC_PIPELINE_STAGES];
544 struct vk_sync_signal signal;
545 struct vk_sync *signal_sync;
546
547 uint32_t wait_count = 0;
548 VkResult result;
549
550 assert(!(sub_cmd->wait_for_stage_mask & ~PVR_PIPELINE_STAGE_ALL_BITS));
551
552 u_foreach_bit (stage, sub_cmd->wait_for_stage_mask) {
553 if (!queue->last_job_signal_sync[stage])
554 continue;
555
556 waits[wait_count++] = (struct vk_sync_wait){
557 .sync = queue->last_job_signal_sync[stage],
558 .stage_mask = ~(VkPipelineStageFlags2)0,
559 .wait_value = 0,
560 };
561 }
562
563 result = vk_sync_create(&device->vk,
564 &device->pdevice->ws->syncobj_type,
565 0U,
566 0UL,
567 &signal_sync);
568 if (result != VK_SUCCESS)
569 return result;
570
571 signal = (struct vk_sync_signal){
572 .sync = signal_sync,
573 .stage_mask = ~(VkPipelineStageFlags2)0,
574 .signal_value = 0,
575 };
576
577 result =
578 device->ws->ops->null_job_submit(device->ws, waits, wait_count, &signal);
579 if (result != VK_SUCCESS) {
580 vk_sync_destroy(&device->vk, signal_sync);
581 return result;
582 }
583
584 if (sub_cmd->event->sync)
585 vk_sync_destroy(&device->vk, sub_cmd->event->sync);
586
587 sub_cmd->event->sync = signal_sync;
588 sub_cmd->event->state = new_event_state;
589
590 return VK_SUCCESS;
591 }
592
593 static inline VkResult
594 pvr_process_event_cmd_set(struct pvr_device *device,
595 struct pvr_queue *queue,
596 struct pvr_sub_cmd_event_set_reset *sub_cmd)
597 {
598 return pvr_process_event_cmd_set_or_reset(device,
599 queue,
600 sub_cmd,
601 PVR_EVENT_STATE_SET_BY_DEVICE);
602 }
603
604 static inline VkResult
605 pvr_process_event_cmd_reset(struct pvr_device *device,
606 struct pvr_queue *queue,
607 struct pvr_sub_cmd_event_set_reset *sub_cmd)
608 {
609 return pvr_process_event_cmd_set_or_reset(device,
610 queue,
611 sub_cmd,
612 PVR_EVENT_STATE_RESET_BY_DEVICE);
613 }
614
615 /**
616 * \brief Process an event sub command of wait type.
617 *
618 * This sets up barrier syncobjs to create a dependency from the event syncobjs
619 * onto the next job submissions.
620 *
621 * The barriers are setup by taking into consideration each event's dst stage
622 * mask so this is in line with vkCmdWaitEvents2().
623 *
624 * \param[in] device Device to create the syncobjs on.
625 * \param[in] sub_cmd Sub command to process.
626 * \param[in,out] barriers Current barriers as input. Barriers
627 * for the next jobs as output.
628 * \parma[in,out] per_cmd_buffer_syncobjs Completion syncobjs for the command
629 * buffer being processed.
630 */
631 static VkResult
632 pvr_process_event_cmd_wait(struct pvr_device *device,
633 struct pvr_queue *queue,
634 struct pvr_sub_cmd_event_wait *sub_cmd)
635 {
636 uint32_t dst_mask = 0;
637 VkResult result;
638
639 STACK_ARRAY(struct vk_sync_wait, waits, sub_cmd->count + 1);
640 if (!waits)
641 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
642
643 for (uint32_t i = 0; i < sub_cmd->count; i++)
644 dst_mask |= sub_cmd->wait_at_stage_masks[i];
645
646 u_foreach_bit (stage, dst_mask) {
647 struct vk_sync_signal signal;
648 struct vk_sync *signal_sync;
649 uint32_t wait_count = 0;
650
651 for (uint32_t i = 0; i < sub_cmd->count; i++) {
652 if (sub_cmd->wait_at_stage_masks[i] & BITFIELD_BIT(stage)) {
653 waits[wait_count++] = (struct vk_sync_wait){
654 .sync = sub_cmd->events[i]->sync,
655 .stage_mask = ~(VkPipelineStageFlags2)0,
656 .wait_value = 0,
657 };
658 }
659 }
660
661 if (!wait_count)
662 continue;
663
664 if (queue->next_job_wait_sync[stage]) {
665 waits[wait_count++] = (struct vk_sync_wait){
666 .sync = queue->next_job_wait_sync[stage],
667 .stage_mask = ~(VkPipelineStageFlags2)0,
668 .wait_value = 0,
669 };
670 }
671
672 assert(wait_count <= (sub_cmd->count + 1));
673
674 result = vk_sync_create(&device->vk,
675 &device->pdevice->ws->syncobj_type,
676 0U,
677 0UL,
678 &signal_sync);
679 if (result != VK_SUCCESS)
680 goto err_free_waits;
681
682 signal = (struct vk_sync_signal){
683 .sync = signal_sync,
684 .stage_mask = ~(VkPipelineStageFlags2)0,
685 .signal_value = 0,
686 };
687
688 result = device->ws->ops->null_job_submit(device->ws,
689 waits,
690 wait_count,
691 &signal);
692 if (result != VK_SUCCESS) {
693 vk_sync_destroy(&device->vk, signal.sync);
694 goto err_free_waits;
695 }
696
697 if (queue->next_job_wait_sync[stage])
698 vk_sync_destroy(&device->vk, queue->next_job_wait_sync[stage]);
699
700 queue->next_job_wait_sync[stage] = signal.sync;
701 }
702
703 STACK_ARRAY_FINISH(waits);
704
705 return VK_SUCCESS;
706
707 err_free_waits:
708 STACK_ARRAY_FINISH(waits);
709
710 return result;
711 }
712
713 static VkResult pvr_process_event_cmd(struct pvr_device *device,
714 struct pvr_queue *queue,
715 struct pvr_sub_cmd_event *sub_cmd)
716 {
717 switch (sub_cmd->type) {
718 case PVR_EVENT_TYPE_SET:
719 return pvr_process_event_cmd_set(device, queue, &sub_cmd->set_reset);
720 case PVR_EVENT_TYPE_RESET:
721 return pvr_process_event_cmd_reset(device, queue, &sub_cmd->set_reset);
722 case PVR_EVENT_TYPE_WAIT:
723 return pvr_process_event_cmd_wait(device, queue, &sub_cmd->wait);
724 case PVR_EVENT_TYPE_BARRIER:
725 return pvr_process_event_cmd_barrier(device, queue, &sub_cmd->barrier);
726 default:
727 UNREACHABLE("Invalid event sub-command type.");
728 };
729 }
730
731 struct pvr_suspended_data {
732 struct pvr_rt_dataset **rts;
733 bool frag_uses_atomic_ops;
734 bool disable_compute_overlap;
735 bool get_vis_results;
736 const struct pvr_query_pool *query_pool;
737 };
738
739 static VkResult pvr_process_cmd_buffer(struct pvr_device *device,
740 struct pvr_queue *queue,
741 struct pvr_cmd_buffer *cmd_buffer,
742 struct pvr_suspended_data *suspended_data)
743 {
744 VkResult result;
745
746 list_for_each_entry_safe (struct pvr_sub_cmd,
747 sub_cmd,
748 &cmd_buffer->sub_cmds,
749 link) {
750 switch (sub_cmd->type) {
751 case PVR_SUB_CMD_TYPE_GRAPHICS: {
752 /* If the fragment job utilizes queries, for data integrity
753 * it needs to wait for the query to be processed.
754 */
755 if (sub_cmd->gfx.has_query) {
756 struct pvr_sub_cmd_event_barrier barrier = {
757 .wait_for_stage_mask = PVR_PIPELINE_STAGE_QUERY_BIT,
758 .wait_at_stage_mask = PVR_PIPELINE_STAGE_FRAG_BIT,
759 };
760
761 result = pvr_process_event_cmd_barrier(device, queue, &barrier);
762 if (result != VK_SUCCESS)
763 break;
764 }
765
766 if (sub_cmd->gfx.wait_on_previous_transfer) {
767 struct pvr_sub_cmd_event_barrier barrier = {
768 .wait_for_stage_mask = PVR_PIPELINE_STAGE_TRANSFER_BIT,
769 .wait_at_stage_mask = PVR_PIPELINE_STAGE_FRAG_BIT,
770 };
771
772 result = pvr_process_event_cmd_barrier(device, queue, &barrier);
773 if (result != VK_SUCCESS)
774 break;
775 }
776
777 if (sub_cmd->is_resume) {
778 sub_cmd->gfx.job.view_state.rt_datasets = suspended_data->rts;
779 sub_cmd->gfx.job.frag_uses_atomic_ops |=
780 suspended_data->frag_uses_atomic_ops;
781 sub_cmd->gfx.job.disable_compute_overlap |=
782 suspended_data->disable_compute_overlap;
783 sub_cmd->gfx.job.get_vis_results |= suspended_data->get_vis_results;
784 if (!sub_cmd->gfx.query_pool) {
785 sub_cmd->gfx.query_pool = suspended_data->query_pool;
786 } else if (suspended_data->query_pool &&
787 sub_cmd->gfx.query_pool != suspended_data->query_pool) {
788 pvr_finishme(
789 "Emit a barrier between suspending and resuming jobs with different query pools");
790 }
791 }
792
793 if (sub_cmd->is_suspend) {
794 suspended_data->rts = sub_cmd->gfx.job.view_state.rt_datasets;
795 suspended_data->frag_uses_atomic_ops =
796 sub_cmd->gfx.job.frag_uses_atomic_ops;
797 suspended_data->disable_compute_overlap =
798 sub_cmd->gfx.job.disable_compute_overlap;
799 suspended_data->get_vis_results = sub_cmd->gfx.job.get_vis_results;
800 suspended_data->query_pool = sub_cmd->gfx.query_pool;
801 } else {
802 suspended_data->rts = NULL;
803 suspended_data->frag_uses_atomic_ops = false;
804 suspended_data->disable_compute_overlap = false;
805 suspended_data->get_vis_results = false;
806 suspended_data->query_pool = NULL;
807 }
808
809 assert(sub_cmd->gfx.job.view_state.rt_datasets);
810 result =
811 pvr_process_graphics_cmd(device, queue, cmd_buffer, &sub_cmd->gfx);
812 break;
813 }
814
815 case PVR_SUB_CMD_TYPE_COMPUTE:
816 result = pvr_process_compute_cmd(device, queue, &sub_cmd->compute);
817 break;
818
819 case PVR_SUB_CMD_TYPE_TRANSFER: {
820 const bool serialize_with_frag = sub_cmd->transfer.serialize_with_frag;
821
822 if (serialize_with_frag) {
823 struct pvr_sub_cmd_event_barrier barrier = {
824 .wait_for_stage_mask = PVR_PIPELINE_STAGE_FRAG_BIT,
825 .wait_at_stage_mask = PVR_PIPELINE_STAGE_TRANSFER_BIT,
826 };
827
828 result = pvr_process_event_cmd_barrier(device, queue, &barrier);
829 if (result != VK_SUCCESS)
830 break;
831 }
832
833 result = pvr_process_transfer_cmds(device, queue, &sub_cmd->transfer);
834
835 if (serialize_with_frag) {
836 struct pvr_sub_cmd_event_barrier barrier = {
837 .wait_for_stage_mask = PVR_PIPELINE_STAGE_TRANSFER_BIT,
838 .wait_at_stage_mask = PVR_PIPELINE_STAGE_FRAG_BIT,
839 };
840
841 if (result != VK_SUCCESS)
842 break;
843
844 result = pvr_process_event_cmd_barrier(device, queue, &barrier);
845 }
846
847 break;
848 }
849
850 case PVR_SUB_CMD_TYPE_QUERY:
851 result = pvr_process_query_cmd(device, queue, &sub_cmd->compute);
852 break;
853
854 case PVR_SUB_CMD_TYPE_EVENT:
855 result = pvr_process_event_cmd(device, queue, &sub_cmd->event);
856 break;
857
858 default:
859 mesa_loge("Unsupported sub-command type %d", sub_cmd->type);
860 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
861 }
862
863 if (result != VK_SUCCESS)
864 return result;
865
866 p_atomic_inc(&device->global_cmd_buffer_submit_count);
867 }
868
869 return VK_SUCCESS;
870 }
871
872 static VkResult pvr_clear_last_submits_syncs(struct pvr_queue *queue)
873 {
874 struct vk_sync_wait waits[PVR_JOB_TYPE_MAX * 2];
875 uint32_t wait_count = 0;
876 VkResult result;
877
878 for (uint32_t i = 0; i < PVR_JOB_TYPE_MAX; i++) {
879 if (queue->next_job_wait_sync[i]) {
880 waits[wait_count++] = (struct vk_sync_wait){
881 .sync = queue->next_job_wait_sync[i],
882 .stage_mask = ~(VkPipelineStageFlags2)0,
883 .wait_value = 0,
884 };
885 }
886
887 if (queue->last_job_signal_sync[i]) {
888 waits[wait_count++] = (struct vk_sync_wait){
889 .sync = queue->last_job_signal_sync[i],
890 .stage_mask = ~(VkPipelineStageFlags2)0,
891 .wait_value = 0,
892 };
893 }
894 }
895
896 result = vk_sync_wait_many(&queue->device->vk,
897 wait_count,
898 waits,
899 VK_SYNC_WAIT_COMPLETE,
900 UINT64_MAX);
901
902 if (result != VK_SUCCESS)
903 return vk_error(queue, result);
904
905 for (uint32_t i = 0; i < PVR_JOB_TYPE_MAX; i++) {
906 if (queue->next_job_wait_sync[i]) {
907 vk_sync_destroy(&queue->device->vk, queue->next_job_wait_sync[i]);
908 queue->next_job_wait_sync[i] = NULL;
909 }
910
911 if (queue->last_job_signal_sync[i]) {
912 vk_sync_destroy(&queue->device->vk, queue->last_job_signal_sync[i]);
913 queue->last_job_signal_sync[i] = NULL;
914 }
915 }
916
917 return VK_SUCCESS;
918 }
919
920 static VkResult pvr_process_queue_signals(struct pvr_queue *queue,
921 struct vk_sync_signal *signals,
922 uint32_t signal_count)
923 {
924 struct vk_sync_wait signal_waits[PVR_JOB_TYPE_MAX];
925 struct pvr_device *device = queue->device;
926 VkResult result;
927
928 for (uint32_t signal_idx = 0; signal_idx < signal_count; signal_idx++) {
929 struct vk_sync_signal *signal = &signals[signal_idx];
930 const enum pvr_pipeline_stage_bits signal_stage_src =
931 pvr_stage_mask_src(signal->stage_mask);
932 uint32_t wait_count = 0;
933
934 for (uint32_t i = 0; i < PVR_JOB_TYPE_MAX; i++) {
935 /* Exception for query jobs since that's something internal,
936 * so the user provided syncs won't ever have it as a source stage.
937 */
938 if (!(signal_stage_src & BITFIELD_BIT(i)) && i != PVR_JOB_TYPE_QUERY)
939 continue;
940
941 if (!queue->last_job_signal_sync[i])
942 continue;
943
944 signal_waits[wait_count++] = (struct vk_sync_wait){
945 .sync = queue->last_job_signal_sync[i],
946 .stage_mask = ~(VkPipelineStageFlags2)0,
947 .wait_value = 0,
948 };
949 }
950
951 result = device->ws->ops->null_job_submit(device->ws,
952 signal_waits,
953 wait_count,
954 signal);
955 if (result != VK_SUCCESS)
956 return result;
957 }
958
959 return VK_SUCCESS;
960 }
961
962 static VkResult pvr_process_queue_waits(struct pvr_queue *queue,
963 struct vk_sync_wait *waits,
964 uint32_t wait_count)
965 {
966 struct pvr_device *device = queue->device;
967 VkResult result;
968
969 STACK_ARRAY(struct vk_sync_wait, stage_waits, wait_count);
970 if (!stage_waits)
971 return vk_error(queue, VK_ERROR_OUT_OF_HOST_MEMORY);
972
973 for (uint32_t i = 0; i < PVR_JOB_TYPE_MAX; i++) {
974 struct vk_sync_signal next_job_wait_signal_sync;
975 uint32_t stage_wait_count = 0;
976
977 for (uint32_t wait_idx = 0; wait_idx < wait_count; wait_idx++) {
978 if (!(pvr_stage_mask_dst(waits[wait_idx].stage_mask) &
979 BITFIELD_BIT(i))) {
980 continue;
981 }
982
983 stage_waits[stage_wait_count++] = (struct vk_sync_wait){
984 .sync = waits[wait_idx].sync,
985 .stage_mask = ~(VkPipelineStageFlags2)0,
986 .wait_value = waits[wait_idx].wait_value,
987 };
988 }
989
990 if (!stage_wait_count)
991 continue;
992
993 result = vk_sync_create(&device->vk,
994 &device->pdevice->ws->syncobj_type,
995 0U,
996 0UL,
997 &queue->next_job_wait_sync[i]);
998 if (result != VK_SUCCESS)
999 goto err_free_waits;
1000
1001 next_job_wait_signal_sync = (struct vk_sync_signal){
1002 .sync = queue->next_job_wait_sync[i],
1003 .stage_mask = ~(VkPipelineStageFlags2)0,
1004 .signal_value = 0,
1005 };
1006
1007 result = device->ws->ops->null_job_submit(device->ws,
1008 stage_waits,
1009 stage_wait_count,
1010 &next_job_wait_signal_sync);
1011 if (result != VK_SUCCESS)
1012 goto err_free_waits;
1013 }
1014
1015 STACK_ARRAY_FINISH(stage_waits);
1016
1017 return VK_SUCCESS;
1018
1019 err_free_waits:
1020 STACK_ARRAY_FINISH(stage_waits);
1021
1022 return result;
1023 }
1024
1025 static VkResult pvr_driver_queue_submit(struct vk_queue *queue,
1026 struct vk_queue_submit *submit)
1027 {
1028 struct pvr_queue *driver_queue = container_of(queue, struct pvr_queue, vk);
1029 struct pvr_device *device = driver_queue->device;
1030 VkResult result;
1031
1032 result = pvr_clear_last_submits_syncs(driver_queue);
1033 if (result != VK_SUCCESS)
1034 return result;
1035
1036 if (submit->wait_count) {
1037 result = pvr_process_queue_waits(driver_queue,
1038 submit->waits,
1039 submit->wait_count);
1040 if (result != VK_SUCCESS)
1041 return result;
1042 }
1043
1044 struct pvr_suspended_data suspended_data = {0};
1045
1046 for (uint32_t i = 0U; i < submit->command_buffer_count; i++) {
1047 result = pvr_process_cmd_buffer(
1048 device,
1049 driver_queue,
1050 container_of(submit->command_buffers[i], struct pvr_cmd_buffer, vk),
1051 &suspended_data);
1052 if (result != VK_SUCCESS)
1053 return result;
1054 }
1055
1056 assert(suspended_data.rts == NULL && "suspended graphics job never resumed");
1057
1058 result = pvr_process_queue_signals(driver_queue,
1059 submit->signals,
1060 submit->signal_count);
1061 if (result != VK_SUCCESS)
1062 return result;
1063
1064 return VK_SUCCESS;
1065 }
1066
1067 VkResult PVR_PER_ARCH(QueueBindSparse)(VkQueue _queue,
1068 uint32_t bindInfoCount,
1069 const VkBindSparseInfo *pBindInfo,
1070 VkFence fence)
1071 {
1072 return VK_SUCCESS;
1073 }