Debian Codegraph @

src/imagination/vulkan/pvr_bo.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on tu driver which is:
5 * Copyright © 2022 Google LLC
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * SOFTWARE.
25 */
26
27 #include <assert.h>
28 #include <limits.h>
29 #include <stddef.h>
30 #include <stdint.h>
31 #include <stdio.h>
32 #include <vulkan/vulkan.h>
33
34 #if defined(HAVE_VALGRIND)
35 # include <valgrind.h>
36 # include <memcheck.h>
37 #endif
38
39 #include "hwdef/rogue_hw_utils.h"
40 #include "pvr_bo.h"
41 #include "pvr_debug.h"
42 #include "pvr_device.h"
43 #include "pvr_dump.h"
44 #include "pvr_macros.h"
45 #include "pvr_physical_device.h"
46 #include "pvr_types.h"
47 #include "pvr_util.h"
48 #include "pvr_winsys.h"
49 #include "util/macros.h"
50 #include "util/rb_tree.h"
51 #include "util/simple_mtx.h"
52 #include "util/u_debug.h"
53 #include "vk_alloc.h"
54 #include "vk_log.h"
55
56 struct pvr_bo_store {
57 struct rb_tree tree;
58 simple_mtx_t mutex;
59 uint32_t size;
60 };
61
62 struct pvr_bo_store_entry {
63 struct rb_node node;
64 struct pvr_bo bo;
65 };
66
67 #define entry_from_node(node_) \
68 container_of(node_, struct pvr_bo_store_entry, node)
69 #define entry_from_bo(bo_) container_of(bo_, struct pvr_bo_store_entry, bo)
70
71 static inline int pvr_dev_addr_cmp(const pvr_dev_addr_t a,
72 const pvr_dev_addr_t b)
73 {
74 const uint64_t addr_a = a.addr;
75 const uint64_t addr_b = b.addr;
76
77 if (addr_a < addr_b)
78 return 1;
79 else if (addr_a > addr_b)
80 return -1;
81 else
82 return 0;
83 }
84
85 /* Borrowed from pandecode. Using this comparator allows us to lookup intervals
86 * in the RB-tree without storing extra information.
87 */
88 static inline int pvr_bo_store_entry_cmp_key(const struct rb_node *node,
89 const void *const key)
90 {
91 const struct pvr_winsys_vma *const vma = entry_from_node(node)->bo.vma;
92 const pvr_dev_addr_t addr = *(const pvr_dev_addr_t *)key;
93
94 if (addr.addr >= vma->dev_addr.addr &&
95 addr.addr < (vma->dev_addr.addr + vma->size)) {
96 return 0;
97 }
98
99 return pvr_dev_addr_cmp(vma->dev_addr, addr);
100 }
101
102 static inline int pvr_bo_store_entry_cmp(const struct rb_node *const a,
103 const struct rb_node *const b)
104 {
105 return pvr_dev_addr_cmp(entry_from_node(a)->bo.vma->dev_addr,
106 entry_from_node(b)->bo.vma->dev_addr);
107 }
108
109 VkResult pvr_bo_store_create(struct pvr_device *device)
110 {
111 struct pvr_bo_store *store;
112
113 if (!PVR_IS_DEBUG_SET(TRACK_BOS)) {
114 device->bo_store = NULL;
115 return VK_SUCCESS;
116 }
117
118 store = vk_alloc(&device->vk.alloc,
119 sizeof(*store),
120 8,
121 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
122 if (!store)
123 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
124
125 rb_tree_init(&store->tree);
126 store->size = 0;
127 simple_mtx_init(&store->mutex, mtx_plain);
128
129 device->bo_store = store;
130
131 return VK_SUCCESS;
132 }
133
134 void pvr_bo_store_destroy(struct pvr_device *device)
135 {
136 struct pvr_bo_store *store = device->bo_store;
137
138 if (likely(!store))
139 return;
140
141 if (unlikely(!rb_tree_is_empty(&store->tree))) {
142 debug_warning("Non-empty BO store destroyed; dump follows");
143 pvr_bo_store_dump(device);
144 }
145
146 simple_mtx_destroy(&store->mutex);
147
148 vk_free(&device->vk.alloc, store);
149
150 device->bo_store = NULL;
151 }
152
153 static void pvr_bo_store_insert(struct pvr_bo_store *const store,
154 struct pvr_bo *const bo)
155 {
156 if (likely(!store))
157 return;
158
159 simple_mtx_lock(&store->mutex);
160 rb_tree_insert(&store->tree,
161 &entry_from_bo(bo)->node,
162 pvr_bo_store_entry_cmp);
163 store->size++;
164 simple_mtx_unlock(&store->mutex);
165 }
166
167 static void pvr_bo_store_remove(struct pvr_bo_store *const store,
168 struct pvr_bo *const bo)
169 {
170 if (likely(!store))
171 return;
172
173 simple_mtx_lock(&store->mutex);
174 rb_tree_remove(&store->tree, &entry_from_bo(bo)->node);
175 store->size--;
176 simple_mtx_unlock(&store->mutex);
177 }
178
179 struct pvr_bo *pvr_bo_store_lookup(struct pvr_device *const device,
180 const pvr_dev_addr_t addr)
181 {
182 struct pvr_bo_store *const store = device->bo_store;
183 struct rb_node *node;
184
185 if (unlikely(!store))
186 return NULL;
187
188 simple_mtx_lock(&store->mutex);
189 node = rb_tree_search(&store->tree, &addr, pvr_bo_store_entry_cmp_key);
190 simple_mtx_unlock(&store->mutex);
191
192 if (!node)
193 return NULL;
194
195 return &entry_from_node(node)->bo;
196 }
197
198 static void pvr_bo_dump_line(struct pvr_dump_ctx *const ctx,
199 const struct pvr_bo *bo,
200 const uint32_t index,
201 const uint32_t nr_bos_log10)
202 {
203 static const char *const pretty_sizes[64 + 1] = {
204 "", "1 B", "2 B", "4 B", "8 B", "16 B",
205 "32 B", "64 B", "128 B", "256 B", "512 B", "1 KiB",
206 "2 KiB", "4 KiB", "8 KiB", "16 KiB", "32 KiB", "64 KiB",
207 "128 KiB", "256 KiB", "512 KiB", "1 MiB", "2 MiB", "4 MiB",
208 "8 MiB", "16 MiB", "32 MiB", "64 MiB", "128 MiB", "256 MiB",
209 "512 MiB", "1 GiB", "2 GiB", "4 GiB", "8 GiB", "16 GiB",
210 "32 GiB", "64 GiB", "128 GiB", "256 GiB", "512 GiB", "1 TiB",
211 "2 TiB", "4 TiB", "8 TiB", "16 TiB", "32 TiB", "64 TiB",
212 "128 TiB", "256 TiB", "512 TiB", "1 PiB", "2 PiB", "4 PiB",
213 "8 PiB", "16 PiB", "32 PiB", "64 PiB", "128 PiB", "256 PiB",
214 "512 PiB", "1 EiB", "2 EiB", "4 EiB", "8 EiB",
215 };
216
217 const uint64_t size = bo->vma->size;
218 const uint32_t size_log2 =
219 util_is_power_of_two_or_zero64(size) ? util_last_bit(size) : 0;
220
221 pvr_dump_println(ctx,
222 "[%0*" PRIu32 "] " PVR_DEV_ADDR_FMT " -> %*p "
223 "(%s%s0x%" PRIx64 " bytes)",
224 nr_bos_log10,
225 index,
226 bo->vma->dev_addr.addr,
227 (int)sizeof(void *) * 2 + 2, /* nr hex digits + 0x prefix */
228 bo->bo->map,
229 pretty_sizes[size_log2],
230 size_log2 ? ", " : "",
231 size);
232 }
233
234 bool pvr_bo_store_dump(struct pvr_device *const device)
235 {
236 struct pvr_bo_store *const store = device->bo_store;
237 const uint32_t nr_bos = store->size;
238 const uint32_t nr_bos_log10 = u32_dec_digits(nr_bos);
239 struct pvr_dump_ctx ctx;
240 uint32_t bo_idx = 0;
241
242 if (unlikely(!store)) {
243 debug_warning("Requested BO store dump, but no BO store is present.");
244 return false;
245 }
246
247 pvr_dump_begin(&ctx, stderr, "BO STORE", 1);
248
249 pvr_dump_println(&ctx, "Dumping %" PRIu32 " BO store entries...", nr_bos);
250
251 pvr_dump_indent(&ctx);
252 rb_tree_foreach_safe (struct pvr_bo_store_entry, entry, &store->tree, node) {
253 pvr_bo_dump_line(&ctx, &entry->bo, bo_idx++, nr_bos_log10);
254 }
255 pvr_dump_dedent(&ctx);
256
257 return pvr_dump_end(&ctx);
258 }
259
260 void pvr_bo_list_dump(struct pvr_dump_ctx *const ctx,
261 const struct list_head *const bo_list,
262 const uint32_t nr_bos)
263 {
264 const uint32_t real_nr_bos = nr_bos ? nr_bos : list_length(bo_list);
265 const uint32_t nr_bos_log10 = u32_dec_digits(real_nr_bos);
266 uint32_t bo_idx = 0;
267
268 list_for_each_entry (struct pvr_bo, bo, bo_list, link) {
269 pvr_bo_dump_line(ctx, bo, bo_idx++, nr_bos_log10);
270 }
271 }
272
273 static uint32_t pvr_bo_alloc_to_winsys_flags(uint64_t flags)
274 {
275 uint32_t ws_flags = 0;
276
277 if (flags & (PVR_BO_ALLOC_FLAG_CPU_ACCESS | PVR_BO_ALLOC_FLAG_CPU_MAPPED))
278 ws_flags |= PVR_WINSYS_BO_FLAG_CPU_ACCESS;
279
280 if (flags & PVR_BO_ALLOC_FLAG_GPU_UNCACHED)
281 ws_flags |= PVR_WINSYS_BO_FLAG_GPU_UNCACHED;
282
283 if (flags & PVR_BO_ALLOC_FLAG_PM_FW_PROTECT)
284 ws_flags |= PVR_WINSYS_BO_FLAG_PM_FW_PROTECT;
285
286 return ws_flags;
287 }
288
289 static inline struct pvr_bo *
290 pvr_bo_alloc_bo(const struct pvr_device *const device)
291 {
292 size_t size;
293 void *ptr;
294
295 if (unlikely(device->bo_store))
296 size = sizeof(struct pvr_bo_store_entry);
297 else
298 size = sizeof(struct pvr_bo);
299
300 ptr =
301 vk_alloc(&device->vk.alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
302 if (unlikely(!ptr))
303 return NULL;
304
305 if (unlikely(device->bo_store))
306 return &((struct pvr_bo_store_entry *)ptr)->bo;
307 else
308 return (struct pvr_bo *)ptr;
309 }
310
311 static inline void pvr_bo_free_bo(const struct pvr_device *const device,
312 struct pvr_bo *const bo)
313 {
314 void *ptr;
315
316 if (unlikely(device->bo_store))
317 ptr = entry_from_bo(bo);
318 else
319 ptr = bo;
320
321 vk_free(&device->vk.alloc, ptr);
322 }
323
324 /**
325 * \brief Helper interface to allocate a GPU buffer and map it to both host and
326 * device virtual memory. Host mapping is conditional and is controlled by
327 * flags.
328 *
329 * \param[in] device Logical device pointer.
330 * \param[in] heap Heap to allocate device virtual address from.
331 * \param[in] size Size of buffer to allocate.
332 * \param[in] alignment Required alignment of the allocation. Must be a power
333 * of two.
334 * \param[in] flags Controls allocation, CPU and GPU mapping behavior
335 * using PVR_BO_ALLOC_FLAG_*.
336 * \param[out] pvr_bo_out On success output buffer is returned in this pointer.
337 * \return VK_SUCCESS on success, or error code otherwise.
338 *
339 * \sa #pvr_bo_free()
340 */
341 VkResult pvr_bo_alloc(struct pvr_device *device,
342 struct pvr_winsys_heap *heap,
343 uint64_t size,
344 uint64_t alignment,
345 uint64_t flags,
346 struct pvr_bo **const pvr_bo_out)
347 {
348 struct pvr_bo *pvr_bo;
349 VkResult result;
350
351 pvr_bo = pvr_bo_alloc_bo(device);
352 if (!pvr_bo) {
353 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
354 goto err_out;
355 }
356
357 pvr_bo->ref_count = 1;
358
359 result = device->ws->ops->buffer_create(device->ws,
360 size,
361 alignment,
362 PVR_WINSYS_BO_TYPE_GPU,
363 pvr_bo_alloc_to_winsys_flags(flags),
364 &pvr_bo->bo);
365 if (result != VK_SUCCESS)
366 goto err_free_bo;
367
368 if (flags & PVR_BO_ALLOC_FLAG_CPU_MAPPED) {
369 result = device->ws->ops->buffer_map(pvr_bo->bo, NULL);
370 if (result != VK_SUCCESS)
371 goto err_buffer_destroy;
372
373 VG(VALGRIND_MAKE_MEM_DEFINED(pvr_bo->bo->map, pvr_bo->bo->size));
374 }
375
376 result = device->ws->ops->heap_alloc(heap, size, alignment, &pvr_bo->vma);
377 if (result != VK_SUCCESS)
378 goto err_buffer_unmap;
379
380 result = device->ws->ops->vma_map(pvr_bo->vma, pvr_bo->bo, 0, size, NULL);
381 if (result != VK_SUCCESS)
382 goto err_heap_free;
383
384 pvr_bo_store_insert(device->bo_store, pvr_bo);
385 *pvr_bo_out = pvr_bo;
386
387 return VK_SUCCESS;
388
389 err_heap_free:
390 device->ws->ops->heap_free(pvr_bo->vma);
391
392 err_buffer_unmap:
393 if (flags & PVR_BO_ALLOC_FLAG_CPU_MAPPED)
394 device->ws->ops->buffer_unmap(pvr_bo->bo, false);
395
396 err_buffer_destroy:
397 device->ws->ops->buffer_destroy(pvr_bo->bo);
398
399 err_free_bo:
400 pvr_bo_free_bo(device, pvr_bo);
401
402 err_out:
403 return result;
404 }
405
406 /**
407 * \brief Interface to map the buffer into host virtual address space.
408 *
409 * Buffer should have been created with the #PVR_BO_ALLOC_FLAG_CPU_ACCESS
410 * flag. It should also not already be mapped or it should have been unmapped
411 * using #pvr_bo_cpu_unmap() before mapping again.
412 *
413 * \param[in] device Logical device pointer.
414 * \param[in] pvr_bo Buffer to map.
415 * \return Valid host virtual address on success, or NULL otherwise.
416 *
417 * \sa #pvr_bo_alloc(), #PVR_BO_ALLOC_FLAG_CPU_MAPPED
418 */
419 VkResult pvr_bo_cpu_map(struct pvr_device *device, struct pvr_bo *pvr_bo)
420 {
421 assert(!pvr_bo->bo->map);
422
423 return device->ws->ops->buffer_map(pvr_bo->bo, NULL);
424 }
425
426 /**
427 * \brief Interface to unmap the buffer from host virtual address space.
428 *
429 * Buffer should have a valid mapping, created either using #pvr_bo_cpu_map() or
430 * by passing #PVR_BO_ALLOC_FLAG_CPU_MAPPED flag to #pvr_bo_alloc() at
431 * allocation time.
432 *
433 * Buffer can be remapped using #pvr_bo_cpu_map().
434 *
435 * \param[in] device Logical device pointer.
436 * \param[in] pvr_bo Buffer to unmap.
437 */
438 void pvr_bo_cpu_unmap(struct pvr_device *device, struct pvr_bo *pvr_bo)
439 {
440 struct pvr_winsys_bo *bo = pvr_bo->bo;
441
442 assert(bo->map);
443
444 #if defined(HAVE_VALGRIND)
445 if (!bo->vbits)
446 bo->vbits = vk_alloc(&device->vk.alloc,
447 bo->size,
448 8,
449 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
450 if (bo->vbits) {
451 unsigned ret = VALGRIND_GET_VBITS(bo->map, bo->vbits, bo->size);
452 if (ret != 0 && ret != 1)
453 mesa_loge("Failed to get vbits; expect bad valgrind results.");
454 } else {
455 mesa_loge("Failed to alloc vbits storage; expect bad valgrind results.");
456 }
457 #endif /* defined(HAVE_VALGRIND) */
458
459 device->ws->ops->buffer_unmap(bo, false);
460 }
461
462 /**
463 * \brief Interface to free the buffer object.
464 *
465 * \param[in] device Logical device pointer.
466 * \param[in] pvr_bo Buffer to free.
467 *
468 * \sa #pvr_bo_alloc()
469 */
470 void pvr_bo_free(struct pvr_device *device, struct pvr_bo *pvr_bo)
471 {
472 if (!pvr_bo)
473 return;
474
475 if (!p_atomic_dec_zero(&pvr_bo->ref_count))
476 return;
477
478 #if defined(HAVE_VALGRIND)
479 vk_free(&device->vk.alloc, pvr_bo->bo->vbits);
480 #endif /* defined(HAVE_VALGRIND) */
481
482 pvr_bo_store_remove(device->bo_store, pvr_bo);
483
484 device->ws->ops->vma_unmap(pvr_bo->vma);
485 device->ws->ops->heap_free(pvr_bo->vma);
486
487 if (pvr_bo->bo->map)
488 device->ws->ops->buffer_unmap(pvr_bo->bo, false);
489
490 device->ws->ops->buffer_destroy(pvr_bo->bo);
491
492 pvr_bo_free_bo(device, pvr_bo);
493 }
494
495 /**
496 * \brief Interface to initialize a pvr_suballocator.
497 *
498 * \param[in] allocator Sub-allocator to initialize.
499 * \param[in] heap Heap to sub-allocate device virtual address from.
500 * \param[in] device Logical device pointer.
501 * \param[in] default_size Minimum size used for pvr bo(s).
502 *
503 * \sa #pvr_bo_suballocator_fini()
504 */
505 void pvr_bo_suballocator_init(struct pvr_suballocator *allocator,
506 struct pvr_winsys_heap *heap,
507 struct pvr_device *device,
508 uint32_t default_size)
509 {
510 *allocator = (struct pvr_suballocator){
511 .device = device,
512 .default_size = default_size,
513 .heap = heap,
514 };
515
516 simple_mtx_init(&allocator->mtx, mtx_plain);
517 }
518
519 /**
520 * \brief Interface to destroy a pvr_suballocator.
521 *
522 * \param[in] allocator Sub-allocator to clean-up.
523 *
524 * \sa #pvr_bo_suballocator_init()
525 */
526 void pvr_bo_suballocator_fini(struct pvr_suballocator *allocator)
527 {
528 pvr_bo_free(allocator->device, allocator->bo);
529 pvr_bo_free(allocator->device, allocator->bo_cached);
530
531 simple_mtx_destroy(&allocator->mtx);
532 }
533
534 static inline struct pvr_bo *pvr_bo_get_ref(struct pvr_bo *bo)
535 {
536 p_atomic_inc(&bo->ref_count);
537
538 return bo;
539 }
540
541 /**
542 * \brief Interface to sub-allocate buffer objects.
543 *
544 * \param[in] allocator Sub-allocator used to make a sub-allocation.
545 * \param[in] size Size of buffer to sub-alllocate.
546 * \param[in] align Required alignment of the allocation. Must be
547 * a power of two.
548 * \param[in] zero_on_alloc Require memory for the sub-allocation to be 0.
549 * \param[out] suballoc_bo_out On success points to the sub-allocated buffer
550 * object.
551 * \return VK_SUCCESS on success, or error code otherwise.
552 *
553 * \sa # pvr_bo_suballoc_free()
554 */
555 VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator,
556 uint32_t size,
557 uint32_t align,
558 bool zero_on_alloc,
559 struct pvr_suballoc_bo **const suballoc_bo_out)
560 {
561 const struct pvr_device_info *dev_info =
562 &allocator->device->pdevice->dev_info;
563 const uint32_t cache_line_size = pvr_get_slc_cache_line_size(dev_info);
564 struct pvr_suballoc_bo *suballoc_bo;
565 uint32_t alloc_size, aligned_size;
566 VkResult result;
567
568 suballoc_bo = vk_alloc(&allocator->device->vk.alloc,
569 sizeof(*suballoc_bo),
570 8,
571 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
572 if (!suballoc_bo)
573 return vk_error(allocator->device, VK_ERROR_OUT_OF_HOST_MEMORY);
574
575 /* This cache line value is used for all type of allocations (i.e. USC, PDS,
576 * transfer and general), so always align them to at least the size of the
577 * cache line.
578 */
579 align = MAX2(align, cache_line_size);
580 assert(util_is_power_of_two_nonzero(align));
581 aligned_size = ALIGN_POT(size, align);
582
583 simple_mtx_lock(&allocator->mtx);
584
585 if (allocator->bo) {
586 uint32_t aligned_offset = ALIGN_POT(allocator->next_offset, align);
587
588 if (aligned_offset + aligned_size <= allocator->bo->bo->size) {
589 suballoc_bo->allocator = allocator;
590 suballoc_bo->bo = pvr_bo_get_ref(allocator->bo);
591 suballoc_bo->dev_addr =
592 PVR_DEV_ADDR_OFFSET(allocator->bo->vma->dev_addr, aligned_offset);
593 suballoc_bo->offset = aligned_offset;
594 suballoc_bo->size = aligned_size;
595
596 allocator->next_offset = aligned_offset + aligned_size;
597
598 if (zero_on_alloc)
599 memset(pvr_bo_suballoc_get_map_addr(suballoc_bo), 0, aligned_size);
600
601 *suballoc_bo_out = suballoc_bo;
602 simple_mtx_unlock(&allocator->mtx);
603
604 return VK_SUCCESS;
605 } else {
606 pvr_bo_free(allocator->device, allocator->bo);
607 allocator->bo = NULL;
608 }
609 }
610
611 alloc_size = MAX2(aligned_size, ALIGN_POT(allocator->default_size, align));
612
613 if (allocator->bo_cached) {
614 struct pvr_winsys_bo *bo_cached = allocator->bo_cached->bo;
615
616 if (alloc_size <= bo_cached->size)
617 allocator->bo = allocator->bo_cached;
618 else
619 pvr_bo_free(allocator->device, allocator->bo_cached);
620
621 allocator->bo_cached = NULL;
622 }
623
624 if (!allocator->bo) {
625 result = pvr_bo_alloc(allocator->device,
626 allocator->heap,
627 alloc_size,
628 align,
629 PVR_BO_ALLOC_FLAG_CPU_MAPPED,
630 &allocator->bo);
631 if (result != VK_SUCCESS) {
632 vk_free(&allocator->device->vk.alloc, suballoc_bo);
633 simple_mtx_unlock(&allocator->mtx);
634 return result;
635 }
636 }
637
638 suballoc_bo->allocator = allocator;
639 suballoc_bo->bo = pvr_bo_get_ref(allocator->bo);
640 suballoc_bo->dev_addr = allocator->bo->vma->dev_addr;
641 suballoc_bo->offset = 0;
642 suballoc_bo->size = aligned_size;
643
644 allocator->next_offset = aligned_size;
645
646 if (zero_on_alloc)
647 memset(pvr_bo_suballoc_get_map_addr(suballoc_bo), 0, aligned_size);
648
649 *suballoc_bo_out = suballoc_bo;
650 simple_mtx_unlock(&allocator->mtx);
651
652 return VK_SUCCESS;
653 }
654
655 /**
656 * \brief Interface to free a sub-allocated buffer object.
657 *
658 * \param[in] suballoc_bo Sub-allocated buffer object to free.
659 *
660 * \sa #pvr_bo_suballoc()
661 */
662 void pvr_bo_suballoc_free(struct pvr_suballoc_bo *suballoc_bo)
663 {
664 if (!suballoc_bo)
665 return;
666
667 simple_mtx_lock(&suballoc_bo->allocator->mtx);
668
669 if (p_atomic_read(&suballoc_bo->bo->ref_count) == 1 &&
670 !suballoc_bo->allocator->bo_cached) {
671 suballoc_bo->allocator->bo_cached = suballoc_bo->bo;
672 } else {
673 pvr_bo_free(suballoc_bo->allocator->device, suballoc_bo->bo);
674 }
675
676 simple_mtx_unlock(&suballoc_bo->allocator->mtx);
677
678 vk_free(&suballoc_bo->allocator->device->vk.alloc, suballoc_bo);
679 }
680
681 /**
682 * \brief Interface to retrieve sub-allocated memory offset from the host
683 * virtual address space.
684 *
685 * \param[in] suballoc_bo Sub-allocated buffer object pointer.
686 *
687 * \return Valid host virtual address on success.
688 *
689 * \sa #pvr_bo_suballoc()
690 */
691 void *pvr_bo_suballoc_get_map_addr(const struct pvr_suballoc_bo *suballoc_bo)
692 {
693 const struct pvr_bo *pvr_bo = suballoc_bo->bo;
694
695 assert((uint8_t *)pvr_bo->bo->map + suballoc_bo->offset <
696 (uint8_t *)pvr_bo->bo->map + pvr_bo->bo->size);
697
698 return (uint8_t *)pvr_bo->bo->map + suballoc_bo->offset;
699 }
700
701 #if defined(HAVE_VALGRIND)
702 VkResult pvr_bo_cpu_map_unchanged(struct pvr_device *device,
703 struct pvr_bo *pvr_bo)
704 {
705 VkResult result = pvr_bo_cpu_map(device, pvr_bo);
706 if (result == VK_SUCCESS) {
707 unsigned ret = VALGRIND_SET_VBITS(pvr_bo->bo->map,
708 pvr_bo->bo->vbits,
709 pvr_bo->bo->size);
710 if (ret != 0 && ret != 1)
711 mesa_loge("Failed to set vbits; expect bad valgrind results.");
712 }
713
714 return result;
715 }
716 #endif /* defined(HAVE_VALGRIND) */