Debian Codegraph @

src/imagination/vulkan/pvr_device.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on anv driver which is:
5 * Copyright © 2015 Intel Corporation
6 *
7 * based in part on v3dv driver which is:
8 * Copyright © 2019 Raspberry Pi
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice (including the next
18 * paragraph) shall be included in all copies or substantial portions of the
19 * Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 * SOFTWARE.
28 */
29
30 #include "pvr_device.h"
31
32 #include <assert.h>
33 #include <fcntl.h>
34 #include <inttypes.h>
35 #include <stdbool.h>
36 #include <stddef.h>
37 #include <stdint.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <vulkan/vulkan.h>
42
43 #include "hwdef/pvr_hw_utils.h"
44 #include "hwdef/rogue_hw_utils.h"
45 #include "pvr_bo.h"
46 #include "pvr_buffer.h"
47 #include "pvr_entrypoints.h"
48 #include "pvr_framebuffer.h"
49 #include "pvr_free_list.h"
50 #include "pvr_hw_pass.h"
51 #include "pvr_image.h"
52 #include "pvr_macros.h"
53 #include "pvr_pass.h"
54 #include "pvr_pds.h"
55 #include "pvr_physical_device.h"
56 #include "pvr_rt_dataset.h"
57 #include "pvr_types.h"
58 #include "pvr_usc.h"
59 #include "pvr_util.h"
60 #include "pvr_winsys.h"
61 #include "pvr_wsi.h"
62 #include "util/log.h"
63 #include "util/macros.h"
64 #include "util/mesa-blake3.h"
65 #include "util/os_misc.h"
66 #include "util/u_math.h"
67 #include "vk_device_memory.h"
68 #include "vk_extensions.h"
69 #include "vk_log.h"
70 #include "vk_object.h"
71 #include "vk_physical_device_features.h"
72 #include "vk_physical_device_properties.h"
73 #include "vk_sampler.h"
74 #include "vk_util.h"
75
76 /** Gets the amount of memory to allocate per-core for a tile buffer. */
77 static uint32_t
78 pvr_get_tile_buffer_size_per_core(const struct pvr_device *device)
79 {
80 uint32_t clusters =
81 PVR_GET_FEATURE_VALUE(&device->pdevice->dev_info, num_clusters, 1U);
82
83 /* Round the number of clusters up to the next power of two. */
84 if (!PVR_HAS_FEATURE(&device->pdevice->dev_info, tile_per_usc))
85 clusters = util_next_power_of_two(clusters);
86
87 /* Tile buffer is (total number of partitions across all clusters) * 16 * 16
88 * (quadrant size in pixels).
89 */
90 return device->pdevice->dev_runtime_info.total_reserved_partition_size *
91 clusters * sizeof(uint32_t);
92 }
93
94 /**
95 * Gets the amount of memory to allocate for a tile buffer on the current BVNC.
96 */
97 static uint32_t pvr_get_tile_buffer_size(const struct pvr_device *device)
98 {
99 /* On a multicore system duplicate the buffer for each core. */
100 /* TODO: Optimise tile buffer size to use core_count, not max_num_cores. */
101 return pvr_get_tile_buffer_size_per_core(device) *
102 rogue_get_max_num_cores(&device->pdevice->dev_info);
103 }
104
105 /**
106 * \brief Ensures that a certain amount of tile buffers are allocated.
107 *
108 * Make sure that \p capacity amount of tile buffers are allocated. If less were
109 * present, append new tile buffers of \p size_in_bytes each to reach the quota.
110 */
111 VkResult pvr_device_tile_buffer_ensure_cap(struct pvr_device *device,
112 uint32_t capacity)
113 {
114 uint32_t size_in_bytes = pvr_get_tile_buffer_size(device);
115 struct pvr_device_tile_buffer_state *tile_buffer_state =
116 &device->tile_buffer_state;
117 const uint32_t cache_line_size =
118 pvr_get_slc_cache_line_size(&device->pdevice->dev_info);
119 VkResult result;
120
121 simple_mtx_lock(&tile_buffer_state->mtx);
122
123 /* Clamping in release and asserting in debug. */
124 assert(capacity <= ARRAY_SIZE(tile_buffer_state->buffers));
125 capacity = CLAMP(capacity,
126 tile_buffer_state->buffer_count,
127 ARRAY_SIZE(tile_buffer_state->buffers));
128
129 /* TODO: Implement bo multialloc? To reduce the amount of syscalls and
130 * allocations.
131 */
132 for (uint32_t i = tile_buffer_state->buffer_count; i < capacity; i++) {
133 result = pvr_bo_alloc(device,
134 device->heaps.general_heap,
135 size_in_bytes,
136 cache_line_size,
137 0,
138 &tile_buffer_state->buffers[i]);
139 if (result != VK_SUCCESS) {
140 for (uint32_t j = tile_buffer_state->buffer_count; j < i; j++)
141 pvr_bo_free(device, tile_buffer_state->buffers[j]);
142
143 goto err_release_lock;
144 }
145 }
146
147 tile_buffer_state->buffer_count = capacity;
148
149 simple_mtx_unlock(&tile_buffer_state->mtx);
150
151 return VK_SUCCESS;
152
153 err_release_lock:
154 simple_mtx_unlock(&tile_buffer_state->mtx);
155
156 return result;
157 }
158
159 void pvr_rstate_entry_add(struct pvr_device *device,
160 struct pvr_render_state *rstate)
161 {
162 simple_mtx_lock(&device->rs_mtx);
163 list_addtail(&rstate->link, &device->render_states);
164 simple_mtx_unlock(&device->rs_mtx);
165 }
166
167 void pvr_rstate_entry_remove(struct pvr_device *device,
168 const struct pvr_render_state *rstate)
169 {
170 simple_mtx_lock(&device->rs_mtx);
171 assert(rstate);
172
173 list_for_each_entry_safe (struct pvr_render_state,
174 entry,
175 &device->render_states,
176 link) {
177 if (entry != rstate)
178 continue;
179
180 pvr_render_state_cleanup(device, &device->vk.alloc, rstate);
181 list_del(&entry->link);
182
183 vk_free(&device->vk.alloc, entry);
184 }
185
186 simple_mtx_unlock(&device->rs_mtx);
187 }
188
189 VkResult pvr_AllocateMemory(VkDevice _device,
190 const VkMemoryAllocateInfo *pAllocateInfo,
191 const VkAllocationCallbacks *pAllocator,
192 VkDeviceMemory *pMem)
193 {
194 const VkImportMemoryFdInfoKHR *fd_info = NULL;
195 VK_FROM_HANDLE(pvr_device, device, _device);
196 enum pvr_winsys_bo_type type = PVR_WINSYS_BO_TYPE_GPU;
197 struct pvr_device_memory *mem;
198 VkResult result;
199
200 assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
201 assert(pAllocateInfo->allocationSize > 0);
202
203 const VkMemoryType *mem_type =
204 &device->pdevice->memory.memoryTypes[pAllocateInfo->memoryTypeIndex];
205 const VkMemoryHeap *mem_heap =
206 &device->pdevice->memory.memoryHeaps[mem_type->heapIndex];
207
208 VkDeviceSize aligned_alloc_size =
209 ALIGN_POT(pAllocateInfo->allocationSize, device->ws->page_size);
210
211 if (aligned_alloc_size > mem_heap->size)
212 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
213
214 mem = vk_device_memory_create(&device->vk,
215 pAllocateInfo,
216 pAllocator,
217 sizeof(*mem));
218 if (!mem)
219 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
220
221 vk_foreach_struct_const (ext, pAllocateInfo->pNext) {
222 switch ((unsigned)ext->sType) {
223 case VK_STRUCTURE_TYPE_WSI_MEMORY_ALLOCATE_INFO_MESA:
224 if (device->ws->display_fd >= 0)
225 type = PVR_WINSYS_BO_TYPE_DISPLAY;
226 break;
227 case VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR:
228 fd_info = (void *)ext;
229 break;
230 case VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO:
231 break;
232 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO:
233 /* We don't have particular optimizations associated with memory
234 * allocations that won't be suballocated to multiple resources.
235 */
236 break;
237 case VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO:
238 /* We're not yet using any of the flags provided. */
239 break;
240 default:
241 vk_debug_ignored_stype(ext->sType);
242 break;
243 }
244 }
245
246 if (fd_info && fd_info->handleType) {
247 assert(
248 fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
249 fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
250
251 result = device->ws->ops->buffer_create_from_fd(device->ws,
252 fd_info->fd,
253 &mem->bo);
254 if (result != VK_SUCCESS)
255 goto err_vk_device_memory_destroy;
256
257 /* For security purposes, we reject importing the bo if it's smaller
258 * than the requested allocation size. This prevents a malicious client
259 * from passing a buffer to a trusted client, lying about the size, and
260 * telling the trusted client to try and texture from an image that goes
261 * out-of-bounds. This sort of thing could lead to GPU hangs or worse
262 * in the trusted client. The trusted client can protect itself against
263 * this sort of attack but only if it can trust the buffer size.
264 */
265 if (aligned_alloc_size > mem->bo->size) {
266 result = vk_errorf(device,
267 VK_ERROR_INVALID_EXTERNAL_HANDLE,
268 "Aligned requested size too large for the given fd "
269 "%" PRIu64 "B > %" PRIu64 "B",
270 pAllocateInfo->allocationSize,
271 mem->bo->size);
272 device->ws->ops->buffer_destroy(mem->bo);
273 goto err_vk_device_memory_destroy;
274 }
275
276 /* From the Vulkan spec:
277 *
278 * "Importing memory from a file descriptor transfers ownership of
279 * the file descriptor from the application to the Vulkan
280 * implementation. The application must not perform any operations on
281 * the file descriptor after a successful import."
282 *
283 * If the import fails, we leave the file descriptor open.
284 */
285 close(fd_info->fd);
286 } else {
287 /* Align physical allocations to the page size of the heap that will be
288 * used when binding device memory (see pvr_bind_memory()) to ensure the
289 * entire allocation can be mapped.
290 */
291 const uint64_t alignment = device->heaps.general_heap->page_size;
292
293 /* FIXME: Need to determine the flags based on
294 * device->pdevice->memory.memoryTypes[pAllocateInfo->memoryTypeIndex].propertyFlags.
295 *
296 * The alternative would be to store the flags alongside the memory
297 * types as an array that's indexed by pAllocateInfo->memoryTypeIndex so
298 * that they can be looked up.
299 */
300 result = device->ws->ops->buffer_create(device->ws,
301 pAllocateInfo->allocationSize,
302 alignment,
303 type,
304 PVR_WINSYS_BO_FLAG_CPU_ACCESS,
305 &mem->bo);
306 if (result != VK_SUCCESS)
307 goto err_vk_device_memory_destroy;
308 }
309
310 *pMem = pvr_device_memory_to_handle(mem);
311
312 return VK_SUCCESS;
313
314 err_vk_device_memory_destroy:
315 vk_device_memory_destroy(&device->vk, pAllocator, &mem->vk);
316
317 return result;
318 }
319
320 VkResult pvr_GetMemoryFdKHR(VkDevice _device,
321 const VkMemoryGetFdInfoKHR *pGetFdInfo,
322 int *pFd)
323 {
324 VK_FROM_HANDLE(pvr_device, device, _device);
325 VK_FROM_HANDLE(pvr_device_memory, mem, pGetFdInfo->memory);
326
327 assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR);
328
329 assert(
330 pGetFdInfo->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
331 pGetFdInfo->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
332
333 return device->ws->ops->buffer_get_fd(mem->bo, pFd);
334 }
335
336 VkResult
337 pvr_GetMemoryFdPropertiesKHR(VkDevice _device,
338 VkExternalMemoryHandleTypeFlagBits handleType,
339 int fd,
340 VkMemoryFdPropertiesKHR *pMemoryFdProperties)
341 {
342 VK_FROM_HANDLE(pvr_device, device, _device);
343
344 switch (handleType) {
345 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT:
346 /* FIXME: This should only allow memory types having
347 * VK_MEMORY_PROPERTY_HOST_CACHED_BIT flag set, as
348 * dma-buf should be imported using cacheable memory types,
349 * given exporter's mmap will always map it as cacheable.
350 * Ref:
351 * https://www.kernel.org/doc/html/latest/driver-api/dma-buf.html#c.dma_buf_ops
352 */
353 pMemoryFdProperties->memoryTypeBits =
354 (1 << device->pdevice->memory.memoryTypeCount) - 1;
355 return VK_SUCCESS;
356 default:
357 return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE);
358 }
359 }
360
361 void pvr_FreeMemory(VkDevice _device,
362 VkDeviceMemory _mem,
363 const VkAllocationCallbacks *pAllocator)
364 {
365 VK_FROM_HANDLE(pvr_device, device, _device);
366 VK_FROM_HANDLE(pvr_device_memory, mem, _mem);
367
368 if (!mem)
369 return;
370
371 /* From the Vulkan spec (§11.2.13. Freeing Device Memory):
372 * If a memory object is mapped at the time it is freed, it is implicitly
373 * unmapped.
374 */
375 if (mem->bo->map)
376 device->ws->ops->buffer_unmap(mem->bo, false);
377
378 device->ws->ops->buffer_destroy(mem->bo);
379
380 vk_device_memory_destroy(&device->vk, pAllocator, &mem->vk);
381 }
382
383 VkResult pvr_MapMemory2(VkDevice _device,
384 const VkMemoryMapInfo *pMemoryMapInfo,
385 void **ppData)
386 {
387 VK_FROM_HANDLE(pvr_device, device, _device);
388 VK_FROM_HANDLE(pvr_device_memory, mem, pMemoryMapInfo->memory);
389 VkDeviceSize offset;
390 VkDeviceSize size;
391 VkResult result;
392
393 if (!mem) {
394 *ppData = NULL;
395 return VK_SUCCESS;
396 }
397
398 offset = pMemoryMapInfo->offset;
399 size = vk_device_memory_range(&mem->vk, offset, pMemoryMapInfo->size);
400
401 void *addr = NULL;
402 if (pMemoryMapInfo->flags & VK_MEMORY_MAP_PLACED_BIT_EXT) {
403 const VkMemoryMapPlacedInfoEXT *placed_info =
404 vk_find_struct_const(pMemoryMapInfo->pNext,
405 MEMORY_MAP_PLACED_INFO_EXT);
406 addr = placed_info->pPlacedAddress;
407 }
408
409 /* From the Vulkan spec version 1.0.32 docs for MapMemory:
410 *
411 * * If size is not equal to VK_WHOLE_SIZE, size must be greater than 0
412 * assert(size != 0);
413 * * If size is not equal to VK_WHOLE_SIZE, size must be less than or
414 * equal to the size of the memory minus offset
415 */
416
417 assert(size > 0);
418 assert(offset + size <= mem->bo->size);
419
420 /* From the Vulkan 1.2.194 spec:
421 *
422 * "memory must not be currently host mapped"
423 */
424 if (mem->bo->map != NULL) {
425 return vk_errorf(device,
426 VK_ERROR_MEMORY_MAP_FAILED,
427 "Memory object already mapped.");
428 }
429
430 vk_foreach_struct_const (ext, pMemoryMapInfo->pNext) {
431 vk_debug_ignored_stype(ext->sType);
432 }
433
434 /* Map it all at once */
435 result = device->ws->ops->buffer_map(mem->bo, addr);
436 if (result != VK_SUCCESS)
437 return result;
438
439 *ppData = (uint8_t *)mem->bo->map + offset;
440
441 return VK_SUCCESS;
442 }
443
444 VkResult pvr_UnmapMemory2(VkDevice _device,
445 const VkMemoryUnmapInfo *pMemoryUnmapInfo)
446 {
447 VK_FROM_HANDLE(pvr_device, device, _device);
448 VK_FROM_HANDLE(pvr_device_memory, mem, pMemoryUnmapInfo->memory);
449
450 if (mem && mem->bo->map) {
451 bool reserve =
452 !!(pMemoryUnmapInfo->flags & VK_MEMORY_UNMAP_RESERVE_BIT_EXT);
453 return device->ws->ops->buffer_unmap(mem->bo, reserve);
454 }
455
456 return VK_SUCCESS;
457 }
458
459 VkResult pvr_FlushMappedMemoryRanges(VkDevice _device,
460 uint32_t memoryRangeCount,
461 const VkMappedMemoryRange *pMemoryRanges)
462 {
463 return VK_SUCCESS;
464 }
465
466 VkResult
467 pvr_InvalidateMappedMemoryRanges(VkDevice _device,
468 uint32_t memoryRangeCount,
469 const VkMappedMemoryRange *pMemoryRanges)
470 {
471 return VK_SUCCESS;
472 }
473
474 void pvr_GetImageSparseMemoryRequirements2(
475 VkDevice device,
476 const VkImageSparseMemoryRequirementsInfo2 *pInfo,
477 uint32_t *pSparseMemoryRequirementCount,
478 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
479 {
480 *pSparseMemoryRequirementCount = 0;
481 }
482
483 void pvr_GetDeviceMemoryCommitment(VkDevice device,
484 VkDeviceMemory memory,
485 VkDeviceSize *pCommittedMemoryInBytes)
486 {
487 *pCommittedMemoryInBytes = 0;
488 }
489
490 VkResult pvr_bind_memory(struct pvr_device *device,
491 struct pvr_device_memory *mem,
492 VkDeviceSize offset,
493 VkDeviceSize size,
494 VkDeviceSize alignment,
495 struct pvr_winsys_vma **const vma_out,
496 pvr_dev_addr_t *const dev_addr_out)
497 {
498 VkDeviceSize virt_size =
499 size + (offset & (device->heaps.general_heap->page_size - 1));
500 struct pvr_winsys_vma *vma;
501 pvr_dev_addr_t dev_addr;
502 VkResult result;
503
504 /* Valid usage:
505 *
506 * "memoryOffset must be an integer multiple of the alignment member of
507 * the VkMemoryRequirements structure returned from a call to
508 * vkGetBufferMemoryRequirements with buffer"
509 *
510 * "memoryOffset must be an integer multiple of the alignment member of
511 * the VkMemoryRequirements structure returned from a call to
512 * vkGetImageMemoryRequirements with image"
513 */
514 assert(offset % alignment == 0);
515 assert(offset < mem->bo->size);
516
517 result = device->ws->ops->heap_alloc(device->heaps.general_heap,
518 virt_size,
519 alignment,
520 &vma);
521 if (result != VK_SUCCESS)
522 goto err_out;
523
524 result = device->ws->ops->vma_map(vma, mem->bo, offset, size, &dev_addr);
525 if (result != VK_SUCCESS)
526 goto err_free_vma;
527
528 *dev_addr_out = dev_addr;
529 *vma_out = vma;
530
531 return VK_SUCCESS;
532
533 err_free_vma:
534 device->ws->ops->heap_free(vma);
535
536 err_out:
537 return result;
538 }
539
540 void pvr_unbind_memory(struct pvr_device *device, struct pvr_winsys_vma *vma)
541 {
542 device->ws->ops->vma_unmap(vma);
543 device->ws->ops->heap_free(vma);
544 }
545
546 VkResult pvr_BindBufferMemory2(VkDevice _device,
547 uint32_t bindInfoCount,
548 const VkBindBufferMemoryInfo *pBindInfos)
549 {
550 VK_FROM_HANDLE(pvr_device, device, _device);
551 uint32_t i;
552
553 for (i = 0; i < bindInfoCount; i++) {
554 VK_FROM_HANDLE(pvr_device_memory, mem, pBindInfos[i].memory);
555 VK_FROM_HANDLE(pvr_buffer, buffer, pBindInfos[i].buffer);
556
557 VkResult result = pvr_bind_memory(device,
558 mem,
559 pBindInfos[i].memoryOffset,
560 buffer->vk.size,
561 buffer->alignment,
562 &buffer->vma,
563 &buffer->dev_addr);
564 if (result != VK_SUCCESS) {
565 while (i--) {
566 VK_FROM_HANDLE(pvr_buffer, buffer, pBindInfos[i].buffer);
567 pvr_unbind_memory(device, buffer->vma);
568 }
569
570 return result;
571 }
572 }
573
574 return VK_SUCCESS;
575 }
576
577 /* Event functions. */
578
579 VkResult pvr_CreateEvent(VkDevice _device,
580 const VkEventCreateInfo *pCreateInfo,
581 const VkAllocationCallbacks *pAllocator,
582 VkEvent *pEvent)
583 {
584 VK_FROM_HANDLE(pvr_device, device, _device);
585
586 struct pvr_event *event = vk_object_alloc(&device->vk,
587 pAllocator,
588 sizeof(*event),
589 VK_OBJECT_TYPE_EVENT);
590 if (!event)
591 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
592
593 event->sync = NULL;
594 event->state = PVR_EVENT_STATE_RESET_BY_HOST;
595
596 *pEvent = pvr_event_to_handle(event);
597
598 return VK_SUCCESS;
599 }
600
601 void pvr_DestroyEvent(VkDevice _device,
602 VkEvent _event,
603 const VkAllocationCallbacks *pAllocator)
604 {
605 VK_FROM_HANDLE(pvr_device, device, _device);
606 VK_FROM_HANDLE(pvr_event, event, _event);
607
608 if (!event)
609 return;
610
611 if (event->sync)
612 vk_sync_destroy(&device->vk, event->sync);
613
614 vk_object_free(&device->vk, pAllocator, event);
615 }
616
617 VkResult pvr_GetEventStatus(VkDevice _device, VkEvent _event)
618 {
619 VK_FROM_HANDLE(pvr_device, device, _device);
620 VK_FROM_HANDLE(pvr_event, event, _event);
621 VkResult result;
622
623 switch (event->state) {
624 case PVR_EVENT_STATE_SET_BY_DEVICE:
625 if (!event->sync)
626 return VK_EVENT_RESET;
627
628 result =
629 vk_sync_wait(&device->vk, event->sync, 0U, VK_SYNC_WAIT_COMPLETE, 0);
630 result = (result == VK_SUCCESS) ? VK_EVENT_SET : VK_EVENT_RESET;
631 break;
632
633 case PVR_EVENT_STATE_RESET_BY_DEVICE:
634 if (!event->sync)
635 return VK_EVENT_RESET;
636
637 result =
638 vk_sync_wait(&device->vk, event->sync, 0U, VK_SYNC_WAIT_COMPLETE, 0);
639 result = (result == VK_SUCCESS) ? VK_EVENT_RESET : VK_EVENT_SET;
640 break;
641
642 case PVR_EVENT_STATE_SET_BY_HOST:
643 result = VK_EVENT_SET;
644 break;
645
646 case PVR_EVENT_STATE_RESET_BY_HOST:
647 result = VK_EVENT_RESET;
648 break;
649
650 default:
651 UNREACHABLE("Event object in unknown state");
652 }
653
654 return result;
655 }
656
657 VkResult pvr_SetEvent(VkDevice _device, VkEvent _event)
658 {
659 VK_FROM_HANDLE(pvr_event, event, _event);
660
661 if (event->sync) {
662 VK_FROM_HANDLE(pvr_device, device, _device);
663
664 const VkResult result = vk_sync_signal(&device->vk, event->sync, 0);
665 if (result != VK_SUCCESS)
666 return result;
667 }
668
669 event->state = PVR_EVENT_STATE_SET_BY_HOST;
670
671 return VK_SUCCESS;
672 }
673
674 VkResult pvr_ResetEvent(VkDevice _device, VkEvent _event)
675 {
676 VK_FROM_HANDLE(pvr_event, event, _event);
677
678 if (event->sync) {
679 VK_FROM_HANDLE(pvr_device, device, _device);
680
681 const VkResult result = vk_sync_reset(&device->vk, event->sync);
682 if (result != VK_SUCCESS)
683 return result;
684 }
685
686 event->state = PVR_EVENT_STATE_RESET_BY_HOST;
687
688 return VK_SUCCESS;
689 }
690
691 /* Buffer functions. */
692
693 VkResult pvr_CreateBuffer(VkDevice _device,
694 const VkBufferCreateInfo *pCreateInfo,
695 const VkAllocationCallbacks *pAllocator,
696 VkBuffer *pBuffer)
697 {
698 VK_FROM_HANDLE(pvr_device, device, _device);
699 const uint32_t alignment = device->pdevice->ws->page_size;
700 struct pvr_buffer *buffer;
701
702 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
703 assert(pCreateInfo->usage != 0);
704
705 /* We check against (ULONG_MAX - alignment) to prevent overflow issues */
706 if (pCreateInfo->size >= ULONG_MAX - alignment)
707 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
708
709 buffer =
710 vk_buffer_create(&device->vk, pCreateInfo, pAllocator, sizeof(*buffer));
711 if (!buffer)
712 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
713
714 buffer->alignment = alignment;
715
716 *pBuffer = pvr_buffer_to_handle(buffer);
717
718 return VK_SUCCESS;
719 }
720
721 VkDeviceAddress
722 pvr_GetBufferDeviceAddress(UNUSED VkDevice device,
723 const VkBufferDeviceAddressInfo *pInfo)
724 {
725 VK_FROM_HANDLE(pvr_buffer, buffer, pInfo->buffer);
726
727 return buffer->dev_addr.addr;
728 }
729
730 uint64_t
731 pvr_GetBufferOpaqueCaptureAddress(UNUSED VkDevice device,
732 UNUSED const VkBufferDeviceAddressInfo *pInfo)
733 {
734 pvr_finishme("Missing support for bufferDeviceAddressCaptureReplay");
735 return 0;
736 }
737
738 uint64_t pvr_GetDeviceMemoryOpaqueCaptureAddress(
739 UNUSED VkDevice device,
740 UNUSED const VkDeviceMemoryOpaqueCaptureAddressInfo *pInfo)
741 {
742 pvr_finishme("Missing support for bufferDeviceAddressCaptureReplay");
743 return 0;
744 }
745
746 void pvr_DestroyBuffer(VkDevice _device,
747 VkBuffer _buffer,
748 const VkAllocationCallbacks *pAllocator)
749 {
750 VK_FROM_HANDLE(pvr_device, device, _device);
751 VK_FROM_HANDLE(pvr_buffer, buffer, _buffer);
752
753 if (!buffer)
754 return;
755
756 if (buffer->vma)
757 pvr_unbind_memory(device, buffer->vma);
758
759 vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk);
760 }
761
762 VkResult pvr_gpu_upload(struct pvr_device *device,
763 struct pvr_winsys_heap *heap,
764 const void *data,
765 size_t size,
766 uint64_t alignment,
767 struct pvr_suballoc_bo **const pvr_bo_out)
768 {
769 struct pvr_suballoc_bo *suballoc_bo = NULL;
770 struct pvr_suballocator *allocator;
771 VkResult result;
772 void *map;
773
774 assert(size > 0);
775
776 if (heap == device->heaps.general_heap)
777 allocator = &device->suballoc_general;
778 else if (heap == device->heaps.pds_heap)
779 allocator = &device->suballoc_pds;
780 else if (heap == device->heaps.transfer_frag_heap)
781 allocator = &device->suballoc_transfer;
782 else if (heap == device->heaps.usc_heap)
783 allocator = &device->suballoc_usc;
784 else
785 UNREACHABLE("Unknown heap type");
786
787 result = pvr_bo_suballoc(allocator, size, alignment, false, &suballoc_bo);
788 if (result != VK_SUCCESS)
789 return result;
790
791 map = pvr_bo_suballoc_get_map_addr(suballoc_bo);
792 if (data)
793 memcpy(map, data, size);
794
795 *pvr_bo_out = suballoc_bo;
796
797 return VK_SUCCESS;
798 }
799
800 VkResult pvr_gpu_upload_usc(struct pvr_device *device,
801 const void *code,
802 size_t code_size,
803 uint64_t code_alignment,
804 struct pvr_suballoc_bo **const pvr_bo_out)
805 {
806 struct pvr_suballoc_bo *suballoc_bo = NULL;
807 VkResult result;
808 void *map;
809
810 assert(code_size > 0);
811
812 /* The USC will prefetch the next instruction, so over allocate by 1
813 * instruction to prevent reading off the end of a page into a potentially
814 * unallocated page.
815 */
816 result = pvr_bo_suballoc(&device->suballoc_usc,
817 code_size + ROGUE_MAX_INSTR_BYTES,
818 code_alignment,
819 false,
820 &suballoc_bo);
821 if (result != VK_SUCCESS)
822 return result;
823
824 map = pvr_bo_suballoc_get_map_addr(suballoc_bo);
825 memcpy(map, code, code_size);
826
827 *pvr_bo_out = suballoc_bo;
828
829 return VK_SUCCESS;
830 }
831
832 /**
833 * \brief Upload PDS program data and code segments from host memory to device
834 * memory.
835 *
836 * \param[in] device Logical device pointer.
837 * \param[in] data Pointer to PDS data segment to upload.
838 * \param[in] data_size_dwords Size of PDS data segment in dwords.
839 * \param[in] data_alignment Required alignment of the PDS data segment in
840 * bytes. Must be a power of two.
841 * \param[in] code Pointer to PDS code segment to upload.
842 * \param[in] code_size_dwords Size of PDS code segment in dwords.
843 * \param[in] code_alignment Required alignment of the PDS code segment in
844 * bytes. Must be a power of two.
845 * \param[in] min_alignment Minimum alignment of the bo holding the PDS
846 * program in bytes.
847 * \param[out] pds_upload_out On success will be initialized based on the
848 * uploaded PDS program.
849 * \return VK_SUCCESS on success, or error code otherwise.
850 */
851 VkResult pvr_gpu_upload_pds(struct pvr_device *device,
852 const uint32_t *data,
853 uint32_t data_size_dwords,
854 uint32_t data_alignment,
855 const uint32_t *code,
856 uint32_t code_size_dwords,
857 uint32_t code_alignment,
858 uint64_t min_alignment,
859 struct pvr_pds_upload *const pds_upload_out)
860 {
861 /* All alignment and sizes below are in bytes. */
862 const size_t data_size = PVR_DW_TO_BYTES(data_size_dwords);
863 const size_t code_size = PVR_DW_TO_BYTES(code_size_dwords);
864 const uint64_t data_aligned_size = ALIGN_POT(data_size, data_alignment);
865 const uint64_t code_aligned_size = ALIGN_POT(code_size, code_alignment);
866 const uint32_t code_offset = ALIGN_POT(data_aligned_size, code_alignment);
867 const uint64_t bo_alignment = MAX2(min_alignment, data_alignment);
868 const uint64_t bo_size = (!!code) ? (code_offset + code_aligned_size)
869 : data_aligned_size;
870 VkResult result;
871 void *map;
872
873 assert(code || data);
874 assert(!code || (code_size_dwords != 0 && code_alignment != 0));
875 assert(!data || (data_size_dwords != 0 && data_alignment != 0));
876
877 result = pvr_bo_suballoc(&device->suballoc_pds,
878 bo_size,
879 bo_alignment,
880 true,
881 &pds_upload_out->pvr_bo);
882 if (result != VK_SUCCESS)
883 return result;
884
885 map = pvr_bo_suballoc_get_map_addr(pds_upload_out->pvr_bo);
886
887 if (data) {
888 memcpy(map, data, data_size);
889
890 pds_upload_out->data_offset = pds_upload_out->pvr_bo->dev_addr.addr -
891 device->heaps.pds_heap->base_addr.addr;
892
893 /* Store data size in dwords. */
894 assert(data_aligned_size % 4 == 0);
895 pds_upload_out->data_size = data_aligned_size / 4;
896 } else {
897 pds_upload_out->data_offset = 0;
898 pds_upload_out->data_size = 0;
899 }
900
901 if (code) {
902 memcpy((uint8_t *)map + code_offset, code, code_size);
903
904 pds_upload_out->code_offset =
905 (pds_upload_out->pvr_bo->dev_addr.addr + code_offset) -
906 device->heaps.pds_heap->base_addr.addr;
907
908 /* Store code size in dwords. */
909 assert(code_aligned_size % 4 == 0);
910 pds_upload_out->code_size = code_aligned_size / 4;
911 } else {
912 pds_upload_out->code_offset = 0;
913 pds_upload_out->code_size = 0;
914 }
915
916 return VK_SUCCESS;
917 }
918
919 void pvr_render_targets_fini(struct pvr_render_target *render_targets,
920 uint32_t render_targets_count)
921 {
922 for (uint32_t i = 0; i < render_targets_count; i++) {
923 pvr_render_targets_datasets_destroy(&render_targets[i]);
924 pthread_mutex_destroy(&render_targets[i].mutex);
925 }
926 }
927
928 void pvr_render_state_cleanup(struct pvr_device *device,
929 const VkAllocationCallbacks *pAllocator,
930 const struct pvr_render_state *rstate)
931 {
932 if (!rstate)
933 return;
934
935 for (uint32_t i = 0; i < rstate->render_count; i++) {
936 pvr_spm_finish_bgobj_state(device,
937 &rstate->spm_bgobj_state_per_render[i]);
938
939 pvr_spm_finish_eot_state(device, &rstate->spm_eot_state_per_render[i]);
940 }
941
942 pvr_spm_scratch_buffer_release(device, rstate->scratch_buffer);
943 pvr_render_targets_fini(rstate->render_targets,
944 rstate->render_targets_count);
945 pvr_bo_suballoc_free(rstate->ppp_state_bo);
946 vk_free2(&device->vk.alloc, pAllocator, rstate->render_targets);
947 }
948
949 void pvr_GetBufferMemoryRequirements2(
950 VkDevice _device,
951 const VkBufferMemoryRequirementsInfo2 *pInfo,
952 VkMemoryRequirements2 *pMemoryRequirements)
953 {
954 VK_FROM_HANDLE(pvr_buffer, buffer, pInfo->buffer);
955 VK_FROM_HANDLE(pvr_device, device, _device);
956 uint64_t size;
957
958 /* The Vulkan 1.0.166 spec says:
959 *
960 * memoryTypeBits is a bitmask and contains one bit set for every
961 * supported memory type for the resource. Bit 'i' is set if and only
962 * if the memory type 'i' in the VkPhysicalDeviceMemoryProperties
963 * structure for the physical device is supported for the resource.
964 *
965 * All types are currently supported for buffers.
966 */
967 pMemoryRequirements->memoryRequirements.memoryTypeBits =
968 (1ul << device->pdevice->memory.memoryTypeCount) - 1;
969
970 pMemoryRequirements->memoryRequirements.alignment = buffer->alignment;
971
972 size = buffer->vk.size;
973
974 if (size % device->ws->page_size == 0 ||
975 size % device->ws->page_size >
976 device->ws->page_size - PVR_BUFFER_MEMORY_PADDING_SIZE) {
977 /* TODO: We can save memory by having one extra virtual page mapped
978 * in and having the first and last virtual page mapped to the first
979 * physical address.
980 */
981 size += PVR_BUFFER_MEMORY_PADDING_SIZE;
982 }
983
984 /* Use align64 to prevent overflow for large buffers (> 4GB). */
985 pMemoryRequirements->memoryRequirements.size =
986 align64(size, buffer->alignment);
987
988 vk_foreach_struct (ext, pMemoryRequirements->pNext) {
989 switch (ext->sType) {
990 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
991 VkMemoryDedicatedRequirements *req =
992 (VkMemoryDedicatedRequirements *)ext;
993
994 req->requiresDedicatedAllocation = false;
995 req->prefersDedicatedAllocation = false;
996 break;
997 }
998 default:
999 vk_debug_ignored_stype(ext->sType);
1000 break;
1001 }
1002 }
1003 }
1004
1005 void pvr_GetDeviceBufferMemoryRequirements(
1006 VkDevice _device,
1007 const VkDeviceBufferMemoryRequirements *pInfo,
1008 VkMemoryRequirements2 *pMemoryRequirements)
1009 {
1010 VK_FROM_HANDLE(pvr_device, device, _device);
1011 struct pvr_buffer buffer = { 0 };
1012
1013 /* Initialize a minimal buffer structure */
1014 vk_buffer_init(&device->vk, &buffer.vk, pInfo->pCreateInfo);
1015 buffer.alignment = device->pdevice->ws->page_size;
1016
1017 VkBufferMemoryRequirementsInfo2 buffer_info = {
1018 .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2,
1019 .buffer = pvr_buffer_to_handle(&buffer),
1020 };
1021
1022 pvr_GetBufferMemoryRequirements2(_device, &buffer_info, pMemoryRequirements);
1023
1024 /* Clean up the temporary buffer */
1025 vk_buffer_finish(&buffer.vk);
1026 }
1027
1028 void pvr_GetDeviceImageMemoryRequirements(
1029 VkDevice _device,
1030 const VkDeviceImageMemoryRequirements *pInfo,
1031 VkMemoryRequirements2 *pMemoryRequirements)
1032 {
1033 VK_FROM_HANDLE(pvr_device, device, _device);
1034 struct pvr_image image = { 0 };
1035
1036 vk_image_init(&device->vk, &image.vk, pInfo->pCreateInfo);
1037 pvr_image_init(device, pInfo->pCreateInfo, &image);
1038
1039 VkImageMemoryRequirementsInfo2 image_info = {
1040 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
1041 .image = pvr_image_to_handle(&image),
1042 };
1043
1044 pvr_GetImageMemoryRequirements2(_device, &image_info, pMemoryRequirements);
1045
1046 pvr_image_fini(device, &image);
1047 vk_image_finish(&image.vk);
1048 }
1049
1050 void pvr_GetDeviceImageSparseMemoryRequirements(
1051 VkDevice device,
1052 const VkDeviceImageMemoryRequirements *pInfo,
1053 uint32_t *pSparseMemoryRequirementCount,
1054 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1055 {
1056 /* Sparse images are not yet supported */
1057 *pSparseMemoryRequirementCount = 0;
1058 }
1059
1060 void pvr_GetImageMemoryRequirements2(VkDevice _device,
1061 const VkImageMemoryRequirementsInfo2 *pInfo,
1062 VkMemoryRequirements2 *pMemoryRequirements)
1063 {
1064 VK_FROM_HANDLE(pvr_device, device, _device);
1065 VK_FROM_HANDLE(pvr_image, image, pInfo->image);
1066
1067 /* The Vulkan 1.0.166 spec says:
1068 *
1069 * memoryTypeBits is a bitmask and contains one bit set for every
1070 * supported memory type for the resource. Bit 'i' is set if and only
1071 * if the memory type 'i' in the VkPhysicalDeviceMemoryProperties
1072 * structure for the physical device is supported for the resource.
1073 *
1074 * All types are currently supported for images.
1075 */
1076 const uint32_t memory_types =
1077 (1ul << device->pdevice->memory.memoryTypeCount) - 1;
1078
1079 /* TODO: The returned size is aligned here in case of arrays/CEM (as is done
1080 * in GetImageMemoryRequirements()), but this should be known at image
1081 * creation time (pCreateInfo->arrayLayers > 1). This is confirmed in
1082 * ImageCreate()/ImageGetMipMapOffsetInBytes() where it aligns the size to
1083 * 4096 if pCreateInfo->arrayLayers > 1. So is the alignment here actually
1084 * necessary? If not, what should it be when pCreateInfo->arrayLayers == 1?
1085 *
1086 * Note: Presumably the 4096 alignment requirement comes from the Vulkan
1087 * driver setting RGX_CR_TPU_TAG_CEM_4K_FACE_PACKING_EN when setting up
1088 * render and compute jobs.
1089 */
1090 pMemoryRequirements->memoryRequirements.alignment = image->alignment;
1091 pMemoryRequirements->memoryRequirements.size =
1092 align64(image->total_size, image->alignment);
1093 pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
1094
1095 vk_foreach_struct (ext, pMemoryRequirements->pNext) {
1096 switch (ext->sType) {
1097 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
1098 bool has_ext_handle_types = image->vk.external_handle_types != 0;
1099 VkMemoryDedicatedRequirements *req =
1100 (VkMemoryDedicatedRequirements *)ext;
1101
1102 req->prefersDedicatedAllocation = has_ext_handle_types;
1103 req->requiresDedicatedAllocation = has_ext_handle_types;
1104 break;
1105 }
1106 default:
1107 vk_debug_ignored_stype(ext->sType);
1108 break;
1109 }
1110 }
1111 }