Debian Codegraph @

src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <fcntl.h>
26 #include <stdbool.h>
27 #include <stddef.h>
28 #include <stdint.h>
29 #include <sys/mman.h>
30 #include <sys/types.h>
31 #include <vulkan/vulkan.h>
32 #include <unistd.h>
33 #include <xf86drm.h>
34
35 #include "drm-uapi/pvr_drm.h"
36 #include "pvr_drm.h"
37 #include "pvr_drm_bo.h"
38 #include "pvr_macros.h"
39 #include "pvr_winsys_helper.h"
40 #include "util/bitscan.h"
41 #include "util/macros.h"
42 #include "vk_alloc.h"
43 #include "vk_log.h"
44
45 static VkResult pvr_drm_create_gem_bo(struct pvr_drm_winsys *drm_ws,
46 uint32_t drm_flags,
47 uint64_t size,
48 uint32_t *const handle_out)
49 {
50 struct drm_pvr_ioctl_create_bo_args args = {
51 .size = size,
52 .flags = drm_flags,
53 };
54 VkResult result;
55
56 result = pvr_ioctlf(drm_ws->base.render_fd,
57 DRM_IOCTL_PVR_CREATE_BO,
58 &args,
59 VK_ERROR_OUT_OF_DEVICE_MEMORY,
60 "Failed to create gem bo");
61 if (result != VK_SUCCESS)
62 return result;
63
64 *handle_out = args.handle;
65
66 return VK_SUCCESS;
67 }
68
69 static VkResult pvr_drm_destroy_gem_bo(struct pvr_drm_winsys *drm_ws,
70 uint32_t handle)
71 {
72 struct drm_gem_close args = {
73 .handle = handle,
74 };
75
76 /* The kernel driver doesn't have a corresponding DRM_IOCTL_PVR_DESTROY_BO
77 * IOCTL as DRM provides a common IOCTL for doing this.
78 */
79 return pvr_ioctlf(drm_ws->base.render_fd,
80 DRM_IOCTL_GEM_CLOSE,
81 &args,
82 VK_ERROR_UNKNOWN,
83 "Failed to destroy gem bo");
84 }
85
86 static VkResult pvr_drm_get_bo_mmap_offset(struct pvr_drm_winsys *drm_ws,
87 uint32_t handle,
88 uint64_t *const offset_out)
89 {
90 struct drm_pvr_ioctl_get_bo_mmap_offset_args args = {
91 .handle = handle,
92 };
93 VkResult result;
94
95 result = pvr_ioctl(drm_ws->base.render_fd,
96 DRM_IOCTL_PVR_GET_BO_MMAP_OFFSET,
97 &args,
98 VK_ERROR_MEMORY_MAP_FAILED);
99 if (result != VK_SUCCESS)
100 return result;
101
102 *offset_out = args.offset;
103
104 return VK_SUCCESS;
105 }
106
107 static void pvr_drm_buffer_acquire(struct pvr_drm_winsys_bo *drm_bo)
108 {
109 p_atomic_inc(&drm_bo->ref_count);
110 }
111
112 static void pvr_drm_buffer_release(struct pvr_drm_winsys_bo *drm_bo)
113 {
114 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(drm_bo->base.ws);
115
116 u_rwlock_rdlock(&drm_ws->dmabuf_bo_lock);
117
118 if (p_atomic_dec_return(&drm_bo->ref_count) == 0) {
119 uint32_t handle = drm_bo->handle;
120
121 /* Our BO structs are stored in a sparse array in the winsys structure,
122 * so we don't want to free the BO pointer, instead we want to reset it
123 * to 0, to signal that array entry as being free.
124 *
125 * We must do the reset before we actually free the BO in the kernel,
126 * since otherwise there is a chance the application creates another BO in
127 * a different thread and gets the same array entry, causing a race.
128 */
129 memset(drm_bo, 0, sizeof(*drm_bo));
130
131 pvr_drm_destroy_gem_bo(drm_ws, handle);
132 }
133
134 u_rwlock_rdunlock(&drm_ws->dmabuf_bo_lock);
135 }
136
137 static VkResult
138 pvr_drm_display_buffer_create(struct pvr_drm_winsys *drm_ws,
139 uint64_t size,
140 struct pvr_winsys_bo **const bo_out)
141 {
142 uint32_t handle;
143 VkResult result;
144 int ret;
145 int fd;
146
147 result =
148 pvr_winsys_helper_display_buffer_create(&drm_ws->base, size, &handle);
149 if (result != VK_SUCCESS)
150 return result;
151
152 ret = drmPrimeHandleToFD(drm_ws->base.display_fd, handle, DRM_CLOEXEC | DRM_RDWR, &fd);
153 pvr_winsys_helper_display_buffer_destroy(&drm_ws->base, handle);
154 if (ret)
155 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
156
157 result = pvr_drm_winsys_buffer_create_from_fd(&drm_ws->base, fd, bo_out);
158 close(fd);
159 if (result != VK_SUCCESS)
160 return result;
161
162 assert((*bo_out)->size >= size);
163
164 return VK_SUCCESS;
165 }
166
167 static uint64_t pvr_drm_get_alloc_flags(uint32_t ws_flags)
168 {
169 uint64_t drm_flags = 0U;
170
171 if (ws_flags & PVR_WINSYS_BO_FLAG_GPU_UNCACHED)
172 drm_flags |= DRM_PVR_BO_BYPASS_DEVICE_CACHE;
173
174 if (ws_flags & PVR_WINSYS_BO_FLAG_PM_FW_PROTECT)
175 drm_flags |= DRM_PVR_BO_PM_FW_PROTECT;
176
177 if (ws_flags & PVR_WINSYS_BO_FLAG_CPU_ACCESS)
178 drm_flags |= DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS;
179
180 return drm_flags;
181 }
182
183 static inline struct pvr_drm_winsys_bo *
184 pvr_drm_winsys_lookup_bo(struct pvr_drm_winsys *drm_ws, uint32_t handle)
185 {
186 return (struct pvr_drm_winsys_bo *)util_sparse_array_get(&drm_ws->bo_map,
187 handle);
188 }
189
190 VkResult pvr_drm_winsys_buffer_create(struct pvr_winsys *ws,
191 uint64_t size,
192 uint64_t alignment,
193 enum pvr_winsys_bo_type type,
194 uint32_t ws_flags,
195 struct pvr_winsys_bo **const bo_out)
196 {
197 const uint64_t drm_flags = pvr_drm_get_alloc_flags(ws_flags);
198 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws);
199 struct pvr_drm_winsys_bo *drm_bo;
200 uint32_t handle = 0;
201 VkResult result;
202
203 assert(util_is_power_of_two_nonzero64(alignment));
204 size = ALIGN_POT(size, alignment);
205 size = ALIGN_POT(size, ws->page_size);
206
207 if (type == PVR_WINSYS_BO_TYPE_DISPLAY)
208 return pvr_drm_display_buffer_create(drm_ws, size, bo_out);
209
210 result = pvr_drm_create_gem_bo(drm_ws, drm_flags, size, &handle);
211 if (result != VK_SUCCESS)
212 return result;
213
214 drm_bo = pvr_drm_winsys_lookup_bo(drm_ws, handle);
215 assert(drm_bo && drm_bo->handle == 0);
216
217 drm_bo->base.size = size;
218 drm_bo->base.ws = ws;
219 drm_bo->handle = handle;
220 drm_bo->flags = drm_flags;
221
222 p_atomic_set(&drm_bo->ref_count, 1);
223
224 *bo_out = &drm_bo->base;
225
226 return VK_SUCCESS;
227 }
228
229 VkResult
230 pvr_drm_winsys_buffer_create_from_fd(struct pvr_winsys *ws,
231 int fd,
232 struct pvr_winsys_bo **const bo_out)
233 {
234 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws);
235 struct pvr_drm_winsys_bo *drm_bo;
236 uint32_t handle;
237 off_t size;
238 int ret;
239
240 size = lseek(fd, 0, SEEK_END);
241 if (size == (off_t)-1)
242 return vk_error(NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE);
243
244 /* Importing the same dma-buf several times will yield the same GEM
245 * handle. Thus, there is a potential race when destroying a BO and importing
246 * the same dma-buf from different threads. We must not permit the creation
247 * of a dma-buf BO and its release to happen in parallel.
248 */
249 u_rwlock_wrlock(&drm_ws->dmabuf_bo_lock);
250
251 ret = drmPrimeFDToHandle(ws->render_fd, fd, &handle);
252 if (ret) {
253 u_rwlock_wrunlock(&drm_ws->dmabuf_bo_lock);
254
255 return vk_error(NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE);
256 }
257
258 drm_bo = pvr_drm_winsys_lookup_bo(drm_ws, handle);
259
260 if (p_atomic_read(&drm_bo->ref_count) == 0) {
261 drm_bo->base.ws = ws;
262 drm_bo->base.size = (uint64_t)size;
263 drm_bo->base.is_imported = true;
264 drm_bo->handle = handle;
265
266 p_atomic_set(&drm_bo->ref_count, 1);
267 } else {
268 pvr_drm_buffer_acquire(drm_bo);
269 }
270
271 u_rwlock_wrunlock(&drm_ws->dmabuf_bo_lock);
272
273 *bo_out = &drm_bo->base;
274
275 return VK_SUCCESS;
276 }
277
278 void pvr_drm_winsys_buffer_destroy(struct pvr_winsys_bo *bo)
279 {
280 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
281
282 pvr_drm_buffer_release(drm_bo);
283 }
284
285 VkResult pvr_drm_winsys_buffer_get_fd(struct pvr_winsys_bo *bo,
286 int *const fd_out)
287 {
288 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
289 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(bo->ws);
290 int ret;
291
292 ret = drmPrimeHandleToFD(drm_ws->base.render_fd,
293 drm_bo->handle,
294 DRM_CLOEXEC | DRM_RDWR,
295 fd_out);
296 if (ret)
297 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
298
299 return VK_SUCCESS;
300 }
301
302 VkResult pvr_drm_winsys_buffer_map(struct pvr_winsys_bo *bo, void *addr)
303 {
304 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
305 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(bo->ws);
306 uint64_t offset = 0;
307 void *map = NULL;
308 VkResult result;
309
310 assert(!bo->map);
311
312 result = pvr_drm_get_bo_mmap_offset(drm_ws, drm_bo->handle, &offset);
313 if (result != VK_SUCCESS)
314 goto err_out;
315
316 result = pvr_mmap(addr,
317 bo->size,
318 PROT_READ | PROT_WRITE,
319 MAP_SHARED,
320 drm_ws->base.render_fd,
321 offset,
322 &map);
323 if (result != VK_SUCCESS)
324 goto err_out;
325
326 VG(VALGRIND_MALLOCLIKE_BLOCK(map, bo->size, 0, true));
327
328 pvr_drm_buffer_acquire(drm_bo);
329 bo->map = map;
330
331 return VK_SUCCESS;
332
333 err_out:
334 return result;
335 }
336
337 VkResult pvr_drm_winsys_buffer_unmap(struct pvr_winsys_bo *bo, bool reserve)
338 {
339 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
340 VkResult result;
341
342 assert(bo->map);
343
344 result = pvr_munmap(bo->map, bo->size, reserve);
345
346 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
347
348 bo->map = NULL;
349
350 pvr_drm_buffer_release(drm_bo);
351
352 return result;
353 }
354
355 /* This function must be used to allocate from a heap carveout and must only be
356 * used within the winsys code. This also means whoever is using it, must know
357 * what they are doing.
358 */
359 VkResult pvr_drm_heap_alloc_carveout(struct pvr_winsys_heap *const heap,
360 const pvr_dev_addr_t carveout_dev_addr,
361 uint64_t size,
362 uint64_t alignment,
363 struct pvr_winsys_vma **const vma_out)
364 {
365 const struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(heap->ws);
366 struct pvr_drm_winsys_vma *drm_vma;
367 VkResult result;
368
369 assert(util_is_power_of_two_nonzero64(alignment));
370
371 drm_vma = vk_zalloc(drm_ws->base.alloc,
372 sizeof(*drm_vma),
373 8,
374 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
375 if (!drm_vma) {
376 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
377 goto err_out;
378 }
379
380 /* The powervr kernel mode driver returns a page aligned size when
381 * allocating buffers.
382 */
383 alignment = MAX2(alignment, heap->page_size);
384 size = ALIGN_POT(size, alignment);
385
386 /* TODO: Should we keep track of the allocations in the carveout? */
387
388 drm_vma->base.dev_addr = carveout_dev_addr;
389 drm_vma->base.heap = heap;
390 drm_vma->base.size = size;
391
392 p_atomic_inc(&heap->ref_count);
393
394 *vma_out = &drm_vma->base;
395
396 return VK_SUCCESS;
397
398 err_out:
399 return result;
400 }
401
402 VkResult pvr_drm_winsys_heap_alloc(struct pvr_winsys_heap *heap,
403 uint64_t size,
404 uint64_t alignment,
405 struct pvr_winsys_vma **const vma_out)
406 {
407 const struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(heap->ws);
408 struct pvr_drm_winsys_vma *drm_vma;
409 VkResult result;
410
411 drm_vma = vk_alloc(drm_ws->base.alloc,
412 sizeof(*drm_vma),
413 8,
414 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
415 if (!drm_vma) {
416 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
417 goto err_out;
418 }
419
420 result = pvr_winsys_helper_heap_alloc(heap, size, alignment, &drm_vma->base);
421 if (result != VK_SUCCESS)
422 goto err_free_vma;
423
424 *vma_out = &drm_vma->base;
425
426 return VK_SUCCESS;
427
428 err_free_vma:
429 vk_free(drm_ws->base.alloc, drm_vma);
430
431 err_out:
432 return result;
433 }
434
435 void pvr_drm_winsys_heap_free(struct pvr_winsys_vma *vma)
436 {
437 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(vma->heap->ws);
438 struct pvr_drm_winsys_vma *drm_vma = to_pvr_drm_winsys_vma(vma);
439 const uint64_t carveout_addr = vma->heap->static_data_carveout_addr.addr;
440
441 /* A vma with an existing device mapping should not be freed. */
442 assert(!drm_vma->base.bo);
443
444 /* Check if we are dealing with carveout address range. */
445 if (vma->dev_addr.addr >= carveout_addr &&
446 vma->dev_addr.addr <
447 (carveout_addr + vma->heap->static_data_carveout_size)) {
448 /* For the carveout addresses just decrement the reference count. */
449 p_atomic_dec(&vma->heap->ref_count);
450 } else {
451 /* Free allocated virtual space. */
452 pvr_winsys_helper_heap_free(vma);
453 }
454
455 vk_free(drm_ws->base.alloc, drm_vma);
456 }
457
458 VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma,
459 struct pvr_winsys_bo *bo,
460 uint64_t offset,
461 uint64_t size,
462 pvr_dev_addr_t *const dev_addr_out)
463 {
464 struct pvr_drm_winsys_bo *const drm_bo = to_pvr_drm_winsys_bo(bo);
465 struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(bo->ws);
466 const uint32_t virt_offset = offset & (vma->heap->page_size - 1);
467 const uint64_t aligned_virt_size =
468 ALIGN_POT(virt_offset + size, vma->heap->page_size);
469 const uint32_t phys_page_offset = offset - virt_offset;
470
471 struct drm_pvr_ioctl_vm_map_args args = { .device_addr = vma->dev_addr.addr,
472 .flags = 0U,
473 .handle = drm_bo->handle,
474 .offset = phys_page_offset,
475 .size = aligned_virt_size,
476 .vm_context_handle =
477 drm_ws->vm_context };
478
479 VkResult result;
480
481 /* Address should not be mapped already. */
482 assert(!vma->bo);
483
484 /* Check if bo and vma can accommodate the given size and offset. */
485 if (ALIGN_POT(offset + size, vma->heap->page_size) > bo->size ||
486 aligned_virt_size > vma->size) {
487 return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED);
488 }
489
490 result = pvr_ioctl(drm_ws->base.render_fd,
491 DRM_IOCTL_PVR_VM_MAP,
492 &args,
493 VK_ERROR_MEMORY_MAP_FAILED);
494 if (result != VK_SUCCESS)
495 return result;
496
497 pvr_drm_buffer_acquire(drm_bo);
498
499 vma->bo = &drm_bo->base;
500 vma->bo_offset = offset;
501 vma->mapped_size = aligned_virt_size;
502
503 if (dev_addr_out)
504 *dev_addr_out = PVR_DEV_ADDR_OFFSET(vma->dev_addr, virt_offset);
505
506 return VK_SUCCESS;
507 }
508
509 void pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma *vma)
510 {
511 struct pvr_drm_winsys_bo *const drm_bo = to_pvr_drm_winsys_bo(vma->bo);
512 struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(vma->bo->ws);
513
514 struct drm_pvr_ioctl_vm_unmap_args args = {
515 .vm_context_handle = drm_ws->vm_context,
516 .device_addr = vma->dev_addr.addr,
517 .size = vma->mapped_size,
518 };
519
520 /* Address should be mapped. */
521 assert(vma->bo);
522
523 pvr_ioctlf(drm_ws->base.render_fd,
524 DRM_IOCTL_PVR_VM_UNMAP,
525 &args,
526 VK_ERROR_UNKNOWN,
527 "Unmap failed");
528
529 pvr_drm_buffer_release(drm_bo);
530
531 vma->bo = NULL;
532 }