Debian Codegraph @

src/imagination/vulkan/winsys/pvrsrvkm/pvr_srv_bo.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <fcntl.h>
26 #include <stdbool.h>
27 #include <stddef.h>
28 #include <stdint.h>
29 #include <sys/mman.h>
30 #include <xf86drm.h>
31
32 #include "pvr_macros.h"
33 #include "pvr_srv.h"
34 #include "pvr_srv_bo.h"
35 #include "pvr_srv_bridge.h"
36 #include "pvr_types.h"
37 #include "pvr_winsys_helper.h"
38 #include "util/u_atomic.h"
39 #include "util/bitscan.h"
40 #include "util/macros.h"
41 #include "util/u_math.h"
42 #include "vk_alloc.h"
43 #include "vk_log.h"
44
45 /* Note: This function does not have an associated pvr_srv_free_display_pmr
46 * function, use pvr_srv_free_pmr instead.
47 */
48 static VkResult pvr_srv_alloc_display_pmr(struct pvr_srv_winsys *srv_ws,
49 uint64_t size,
50 uint64_t srv_flags,
51 void **const pmr_out,
52 uint32_t *const handle_out)
53 {
54 uint64_t aligment_out;
55 uint64_t size_out;
56 VkResult result;
57 uint32_t handle;
58 int ret;
59 int fd;
60
61 result =
62 pvr_winsys_helper_display_buffer_create(&srv_ws->base, size, &handle);
63 if (result != VK_SUCCESS)
64 return result;
65
66 ret = drmPrimeHandleToFD(srv_ws->base.display_fd, handle, DRM_CLOEXEC | DRM_RDWR, &fd);
67 if (ret) {
68 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
69 goto err_display_buffer_destroy;
70 }
71
72 result = pvr_srv_physmem_import_dmabuf(srv_ws->base.render_fd,
73 fd,
74 srv_flags,
75 pmr_out,
76 &size_out,
77 &aligment_out);
78
79 assert(size_out >= size);
80 assert(aligment_out == srv_ws->base.page_size);
81
82 /* close fd, not needed anymore */
83 close(fd);
84
85 if (result != VK_SUCCESS)
86 goto err_display_buffer_destroy;
87
88 *handle_out = handle;
89
90 return VK_SUCCESS;
91
92 err_display_buffer_destroy:
93 pvr_winsys_helper_display_buffer_destroy(&srv_ws->base, handle);
94
95 return result;
96 }
97
98 static void buffer_acquire(struct pvr_srv_winsys_bo *srv_bo)
99 {
100 p_atomic_inc(&srv_bo->ref_count);
101 }
102
103 static void buffer_release(struct pvr_srv_winsys_bo *srv_bo)
104 {
105 struct pvr_winsys *ws;
106
107 /* If all references were dropped the pmr can be freed and unlocked */
108 if (p_atomic_dec_return(&srv_bo->ref_count) != 0)
109 return;
110
111 ws = srv_bo->base.ws;
112 pvr_srv_free_pmr(ws->render_fd, srv_bo->pmr);
113
114 if (srv_bo->is_display_buffer)
115 pvr_winsys_helper_display_buffer_destroy(ws, srv_bo->handle);
116
117 vk_free(ws->alloc, srv_bo);
118 }
119
120 static uint64_t pvr_srv_get_alloc_flags(uint32_t ws_flags)
121 {
122 /* TODO: For now we assume that buffers should always be accessible to the
123 * kernel and that the PVR_WINSYS_BO_FLAG_CPU_ACCESS flag only applies to
124 * userspace mappings. Check to see if there's any situations where we
125 * wouldn't want this to be the case.
126 */
127 uint64_t srv_flags =
128 PVR_SRV_MEMALLOCFLAG_GPU_READABLE | PVR_SRV_MEMALLOCFLAG_GPU_WRITEABLE |
129 PVR_SRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
130 PVR_SRV_MEMALLOCFLAG_CPU_UNCACHED_WC | PVR_SRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
131
132 if (ws_flags & PVR_WINSYS_BO_FLAG_CPU_ACCESS) {
133 srv_flags |= PVR_SRV_MEMALLOCFLAG_CPU_READABLE |
134 PVR_SRV_MEMALLOCFLAG_CPU_WRITEABLE;
135 }
136
137 if (ws_flags & PVR_WINSYS_BO_FLAG_GPU_UNCACHED)
138 srv_flags |= PVR_SRV_MEMALLOCFLAG_GPU_UNCACHED;
139 else
140 srv_flags |= PVR_SRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT;
141
142 if (ws_flags & PVR_WINSYS_BO_FLAG_PM_FW_PROTECT)
143 srv_flags |= PVR_SRV_MEMALLOCFLAG_DEVICE_FLAG(PM_FW_PROTECT);
144
145 return srv_flags;
146 }
147
148 VkResult pvr_srv_winsys_buffer_create(struct pvr_winsys *ws,
149 uint64_t size,
150 uint64_t alignment,
151 enum pvr_winsys_bo_type type,
152 uint32_t ws_flags,
153 struct pvr_winsys_bo **const bo_out)
154 {
155 const uint64_t srv_flags = pvr_srv_get_alloc_flags(ws_flags);
156 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws);
157 struct pvr_srv_winsys_bo *srv_bo;
158 VkResult result;
159
160 assert(util_is_power_of_two_nonzero64(alignment));
161
162 /* Kernel will page align the size, we do the same here so we have access to
163 * all the allocated memory.
164 */
165 alignment = MAX2(alignment, ws->page_size);
166 size = ALIGN_POT(size, alignment);
167
168 srv_bo = vk_zalloc(ws->alloc,
169 sizeof(*srv_bo),
170 8,
171 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
172 if (!srv_bo)
173 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
174
175 srv_bo->is_display_buffer = (type == PVR_WINSYS_BO_TYPE_DISPLAY);
176 if (srv_bo->is_display_buffer) {
177 result = pvr_srv_alloc_display_pmr(srv_ws,
178 size,
179 srv_flags &
180 PVR_SRV_MEMALLOCFLAGS_PMRFLAGSMASK,
181 &srv_bo->pmr,
182 &srv_bo->handle);
183
184 srv_bo->base.is_imported = true;
185 } else {
186 result =
187 pvr_srv_alloc_pmr(ws->render_fd,
188 size,
189 size,
190 1,
191 1,
192 ws->log2_page_size,
193 (srv_flags & PVR_SRV_MEMALLOCFLAGS_PMRFLAGSMASK),
194 getpid(),
195 &srv_bo->pmr);
196 }
197
198 if (result != VK_SUCCESS)
199 goto err_vk_free_srv_bo;
200
201 srv_bo->base.size = size;
202 srv_bo->base.ws = ws;
203 srv_bo->flags = srv_flags;
204
205 p_atomic_set(&srv_bo->ref_count, 1);
206
207 *bo_out = &srv_bo->base;
208
209 return VK_SUCCESS;
210
211 err_vk_free_srv_bo:
212 vk_free(ws->alloc, srv_bo);
213
214 return result;
215 }
216
217 VkResult
218 pvr_srv_winsys_buffer_create_from_fd(struct pvr_winsys *ws,
219 int fd,
220 struct pvr_winsys_bo **const bo_out)
221 {
222 /* FIXME: PVR_SRV_MEMALLOCFLAG_CPU_UNCACHED_WC should be changed to
223 * PVR_SRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT, as dma-buf is always mapped
224 * as cacheable by the exporter. Flags are not passed to the exporter and it
225 * doesn't really change the behavior, but these can be used for internal
226 * checking so it should reflect the correct cachability of the buffer.
227 * Ref: pvr_GetMemoryFdPropertiesKHR
228 * https://www.kernel.org/doc/html/latest/driver-api/dma-buf.html#c.dma_buf_ops
229 */
230 static const uint64_t srv_flags =
231 PVR_SRV_MEMALLOCFLAG_CPU_READABLE | PVR_SRV_MEMALLOCFLAG_CPU_WRITEABLE |
232 PVR_SRV_MEMALLOCFLAG_CPU_UNCACHED_WC | PVR_SRV_MEMALLOCFLAG_GPU_READABLE |
233 PVR_SRV_MEMALLOCFLAG_GPU_WRITEABLE |
234 PVR_SRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT;
235 struct pvr_srv_winsys_bo *srv_bo;
236 uint64_t aligment_out;
237 uint64_t size_out;
238 VkResult result;
239
240 srv_bo = vk_zalloc(ws->alloc,
241 sizeof(*srv_bo),
242 8,
243 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
244 if (!srv_bo)
245 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
246
247 result = pvr_srv_physmem_import_dmabuf(ws->render_fd,
248 fd,
249 srv_flags,
250 &srv_bo->pmr,
251 &size_out,
252 &aligment_out);
253 if (result != VK_SUCCESS)
254 goto err_vk_free_srv_bo;
255
256 assert(aligment_out == ws->page_size);
257
258 srv_bo->base.ws = ws;
259 srv_bo->base.size = size_out;
260 srv_bo->base.is_imported = true;
261 srv_bo->flags = srv_flags;
262
263 p_atomic_set(&srv_bo->ref_count, 1);
264
265 *bo_out = &srv_bo->base;
266
267 return VK_SUCCESS;
268
269 err_vk_free_srv_bo:
270 vk_free(ws->alloc, srv_bo);
271
272 return result;
273 }
274
275 void pvr_srv_winsys_buffer_destroy(struct pvr_winsys_bo *bo)
276 {
277 struct pvr_srv_winsys_bo *srv_bo = to_pvr_srv_winsys_bo(bo);
278
279 buffer_release(srv_bo);
280 }
281
282 VkResult pvr_srv_winsys_buffer_get_fd(struct pvr_winsys_bo *bo,
283 int *const fd_out)
284 {
285 struct pvr_srv_winsys_bo *srv_bo = to_pvr_srv_winsys_bo(bo);
286 struct pvr_winsys *ws = bo->ws;
287 int ret;
288
289 if (!srv_bo->is_display_buffer)
290 return pvr_srv_physmem_export_dmabuf(ws->render_fd, srv_bo->pmr, fd_out);
291
292 /* For display buffers, export using saved buffer handle */
293 ret = drmPrimeHandleToFD(ws->display_fd, srv_bo->handle, DRM_CLOEXEC | DRM_RDWR, fd_out);
294 if (ret)
295 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
296
297 return VK_SUCCESS;
298 }
299
300 VkResult pvr_srv_winsys_buffer_map(struct pvr_winsys_bo *bo, void *addr)
301 {
302 struct pvr_srv_winsys_bo *srv_bo = to_pvr_srv_winsys_bo(bo);
303 struct pvr_winsys *ws = bo->ws;
304 const int prot =
305 (srv_bo->flags & PVR_SRV_MEMALLOCFLAG_CPU_WRITEABLE ? PROT_WRITE : 0) |
306 (srv_bo->flags & PVR_SRV_MEMALLOCFLAG_CPU_READABLE ? PROT_READ : 0);
307 VkResult result;
308
309 /* assert if memory is already mapped */
310 assert(!bo->map);
311
312 /* Map the full PMR to CPU space */
313 result = pvr_mmap(addr,
314 bo->size,
315 prot,
316 MAP_SHARED,
317 ws->render_fd,
318 (off_t)srv_bo->pmr << ws->log2_page_size,
319 &bo->map);
320 if (result != VK_SUCCESS) {
321 bo->map = NULL;
322 return result;
323 }
324
325 VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, true));
326
327 buffer_acquire(srv_bo);
328
329 return VK_SUCCESS;
330 }
331
332 VkResult pvr_srv_winsys_buffer_unmap(struct pvr_winsys_bo *bo, bool reserve)
333 {
334 struct pvr_srv_winsys_bo *srv_bo = to_pvr_srv_winsys_bo(bo);
335 VkResult result;
336
337 /* output error if trying to unmap memory that is not previously mapped */
338 assert(bo->map);
339
340 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
341
342 /* Unmap the whole PMR from CPU space */
343 result = pvr_munmap(bo->map, bo->size, reserve);
344
345 bo->map = NULL;
346
347 buffer_release(srv_bo);
348
349 return result;
350 }
351
352 /* This function must be used to allocate from a heap carveout and must only be
353 * used within the winsys code. This also means whoever is using it, must know
354 * what they are doing.
355 */
356 VkResult pvr_srv_heap_alloc_carveout(struct pvr_winsys_heap *heap,
357 const pvr_dev_addr_t carveout_dev_addr,
358 uint64_t size,
359 uint64_t alignment,
360 struct pvr_winsys_vma **const vma_out)
361 {
362 struct pvr_srv_winsys_heap *srv_heap = to_pvr_srv_winsys_heap(heap);
363 struct pvr_winsys *ws = heap->ws;
364 struct pvr_srv_winsys_vma *srv_vma;
365 VkResult result;
366
367 assert(util_is_power_of_two_nonzero64(alignment));
368
369 /* pvr_srv_winsys_buffer_create() page aligns the size. We must do the same
370 * here to ensure enough heap space is allocated to be able to map the
371 * buffer to the GPU.
372 */
373 alignment = MAX2(alignment, heap->ws->page_size);
374 size = ALIGN_POT(size, alignment);
375
376 srv_vma = vk_alloc(ws->alloc,
377 sizeof(*srv_vma),
378 8,
379 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
380 if (!srv_vma) {
381 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
382 goto err_out;
383 }
384
385 /* Just check address is correct and aligned, locking is not required as
386 * user is responsible to provide a distinct address.
387 */
388 if (carveout_dev_addr.addr < heap->base_addr.addr ||
389 carveout_dev_addr.addr + size >
390 heap->base_addr.addr + heap->static_data_carveout_size ||
391 carveout_dev_addr.addr & ((ws->page_size) - 1)) {
392 result = vk_error(NULL, VK_ERROR_INITIALIZATION_FAILED);
393 goto err_vk_free_srv_vma;
394 }
395
396 /* Reserve the virtual range in the MMU and create a mapping structure */
397 result = pvr_srv_int_reserve_addr(ws->render_fd,
398 srv_heap->server_heap,
399 carveout_dev_addr,
400 size,
401 &srv_vma->reservation);
402 if (result != VK_SUCCESS)
403 goto err_vk_free_srv_vma;
404
405 srv_vma->base.dev_addr = carveout_dev_addr;
406 srv_vma->base.bo = NULL;
407 srv_vma->base.heap = heap;
408 srv_vma->base.size = size;
409
410 p_atomic_inc(&srv_heap->base.ref_count);
411
412 *vma_out = &srv_vma->base;
413
414 return VK_SUCCESS;
415
416 err_vk_free_srv_vma:
417 vk_free(ws->alloc, srv_vma);
418
419 err_out:
420 return result;
421 }
422
423 VkResult pvr_srv_winsys_heap_alloc(struct pvr_winsys_heap *heap,
424 uint64_t size,
425 uint64_t alignment,
426 struct pvr_winsys_vma **const vma_out)
427 {
428 struct pvr_srv_winsys_heap *const srv_heap = to_pvr_srv_winsys_heap(heap);
429 struct pvr_srv_winsys *const srv_ws = to_pvr_srv_winsys(heap->ws);
430 struct pvr_srv_winsys_vma *srv_vma;
431 VkResult result;
432
433 srv_vma = vk_alloc(srv_ws->base.alloc,
434 sizeof(*srv_vma),
435 8,
436 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
437 if (!srv_vma) {
438 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
439 goto err_out;
440 }
441
442 result = pvr_winsys_helper_heap_alloc(heap, size, alignment, &srv_vma->base);
443 if (result != VK_SUCCESS)
444 goto err_pvr_srv_free_vma;
445
446 /* Reserve the virtual range in the MMU and create a mapping structure. */
447 result = pvr_srv_int_reserve_addr(srv_ws->base.render_fd,
448 srv_heap->server_heap,
449 srv_vma->base.dev_addr,
450 srv_vma->base.size,
451 &srv_vma->reservation);
452 if (result != VK_SUCCESS)
453 goto err_pvr_srv_free_allocation;
454
455 *vma_out = &srv_vma->base;
456
457 return VK_SUCCESS;
458
459 err_pvr_srv_free_allocation:
460 pvr_winsys_helper_heap_free(&srv_vma->base);
461
462 err_pvr_srv_free_vma:
463 vk_free(srv_ws->base.alloc, srv_vma);
464
465 err_out:
466 return result;
467 }
468
469 void pvr_srv_winsys_heap_free(struct pvr_winsys_vma *vma)
470 {
471 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(vma->heap->ws);
472 struct pvr_srv_winsys_vma *srv_vma = to_pvr_srv_winsys_vma(vma);
473
474 /* A vma with an existing device mapping should not be freed. */
475 assert(!vma->bo);
476
477 /* Remove mapping handle and underlying reservation. */
478 pvr_srv_int_unreserve_addr(srv_ws->base.render_fd, srv_vma->reservation);
479
480 /* Check if we are dealing with carveout address range. */
481 if (vma->dev_addr.addr <
482 (vma->heap->base_addr.addr + vma->heap->static_data_carveout_size)) {
483 /* For the carveout addresses just decrement the reference count. */
484 p_atomic_dec(&vma->heap->ref_count);
485 } else {
486 /* Free allocated virtual space. */
487 pvr_winsys_helper_heap_free(vma);
488 }
489
490 vk_free(srv_ws->base.alloc, srv_vma);
491 }
492
493 /* * We assume the vma has been allocated with extra space to accommodate the
494 * offset.
495 * * The offset passed in is unchanged and can be used to calculate the extra
496 * size that needs to be mapped and final device virtual address.
497 */
498 VkResult pvr_srv_winsys_vma_map(struct pvr_winsys_vma *vma,
499 struct pvr_winsys_bo *bo,
500 uint64_t offset,
501 uint64_t size,
502 pvr_dev_addr_t *const dev_addr_out)
503 {
504 struct pvr_srv_winsys_vma *srv_vma = to_pvr_srv_winsys_vma(vma);
505 struct pvr_srv_winsys_bo *srv_bo = to_pvr_srv_winsys_bo(bo);
506 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(bo->ws);
507 const uint64_t srv_flags = srv_bo->flags &
508 PVR_SRV_MEMALLOCFLAGS_VIRTUAL_MASK;
509 const uint32_t virt_offset = offset & (vma->heap->page_size - 1);
510 const uint64_t aligned_virt_size =
511 ALIGN_POT(virt_offset + size, vma->heap->page_size);
512 VkResult result;
513
514 /* Address should not be mapped already */
515 assert(!vma->bo);
516
517 if (srv_bo->is_display_buffer) {
518 struct pvr_srv_winsys_heap *srv_heap = to_pvr_srv_winsys_heap(vma->heap);
519
520 /* In case of display buffers, we only support to map whole PMR */
521 if (offset != 0 || bo->size != ALIGN_POT(size, srv_ws->base.page_size) ||
522 vma->size != bo->size) {
523 return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED);
524 }
525
526 /* Map the requested pmr */
527 result = pvr_srv_int_map_pmr(srv_ws->base.render_fd,
528 srv_heap->server_heap,
529 srv_vma->reservation,
530 srv_bo->pmr,
531 srv_flags,
532 &srv_vma->mapping);
533
534 } else {
535 const uint32_t phys_page_offset = (offset - virt_offset) >>
536 srv_ws->base.log2_page_size;
537 const uint32_t phys_page_count = aligned_virt_size >>
538 srv_ws->base.log2_page_size;
539
540 /* Check if bo and vma can accommodate the given size and offset */
541 if (ALIGN_POT(offset + size, vma->heap->page_size) > bo->size ||
542 aligned_virt_size > vma->size) {
543 return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED);
544 }
545
546 /* Map the requested pages */
547 result = pvr_srv_int_map_pages(srv_ws->base.render_fd,
548 srv_vma->reservation,
549 srv_bo->pmr,
550 phys_page_count,
551 phys_page_offset,
552 srv_flags,
553 vma->dev_addr);
554 }
555
556 if (result != VK_SUCCESS)
557 return result;
558
559 buffer_acquire(srv_bo);
560
561 vma->bo = bo;
562 vma->bo_offset = offset;
563 vma->mapped_size = aligned_virt_size;
564
565 if (dev_addr_out)
566 *dev_addr_out = PVR_DEV_ADDR_OFFSET(vma->dev_addr, virt_offset);
567
568 return VK_SUCCESS;
569 }
570
571 void pvr_srv_winsys_vma_unmap(struct pvr_winsys_vma *vma)
572 {
573 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(vma->heap->ws);
574 struct pvr_srv_winsys_vma *srv_vma = to_pvr_srv_winsys_vma(vma);
575 struct pvr_srv_winsys_bo *srv_bo;
576
577 /* Address should be mapped */
578 assert(vma->bo);
579
580 srv_bo = to_pvr_srv_winsys_bo(vma->bo);
581
582 if (srv_bo->is_display_buffer) {
583 /* Unmap the requested pmr */
584 pvr_srv_int_unmap_pmr(srv_ws->base.render_fd, srv_vma->mapping);
585 } else {
586 /* Unmap requested pages */
587 pvr_srv_int_unmap_pages(srv_ws->base.render_fd,
588 srv_vma->reservation,
589 vma->dev_addr,
590 vma->mapped_size >> srv_ws->base.log2_page_size);
591 }
592
593 buffer_release(srv_bo);
594
595 vma->bo = NULL;
596 }