Debian Codegraph @

third_party/vkmemalloc/src/vk_mem_alloc.h

raw
1 //
2 // Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25
26 /** \mainpage Vulkan Memory Allocator
27
28 <b>Version 3.0.0-development</b> (2020-06-24)
29
30 Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. \n
31 License: MIT
32
33 Documentation of all members: vk_mem_alloc.h
34
35 \section main_table_of_contents Table of contents
36
37 - <b>User guide</b>
38 - \subpage quick_start
39 - [Project setup](@ref quick_start_project_setup)
40 - [Initialization](@ref quick_start_initialization)
41 - [Resource allocation](@ref quick_start_resource_allocation)
42 - \subpage choosing_memory_type
43 - [Usage](@ref choosing_memory_type_usage)
44 - [Required and preferred flags](@ref choosing_memory_type_required_preferred_flags)
45 - [Explicit memory types](@ref choosing_memory_type_explicit_memory_types)
46 - [Custom memory pools](@ref choosing_memory_type_custom_memory_pools)
47 - [Dedicated allocations](@ref choosing_memory_type_dedicated_allocations)
48 - \subpage memory_mapping
49 - [Mapping functions](@ref memory_mapping_mapping_functions)
50 - [Persistently mapped memory](@ref memory_mapping_persistently_mapped_memory)
51 - [Cache flush and invalidate](@ref memory_mapping_cache_control)
52 - [Finding out if memory is mappable](@ref memory_mapping_finding_if_memory_mappable)
53 - \subpage staying_within_budget
54 - [Querying for budget](@ref staying_within_budget_querying_for_budget)
55 - [Controlling memory usage](@ref staying_within_budget_controlling_memory_usage)
56 - \subpage resource_aliasing
57 - \subpage custom_memory_pools
58 - [Choosing memory type index](@ref custom_memory_pools_MemTypeIndex)
59 - [Linear allocation algorithm](@ref linear_algorithm)
60 - [Free-at-once](@ref linear_algorithm_free_at_once)
61 - [Stack](@ref linear_algorithm_stack)
62 - [Double stack](@ref linear_algorithm_double_stack)
63 - [Ring buffer](@ref linear_algorithm_ring_buffer)
64 - [Buddy allocation algorithm](@ref buddy_algorithm)
65 - \subpage defragmentation
66 - [Defragmenting CPU memory](@ref defragmentation_cpu)
67 - [Defragmenting GPU memory](@ref defragmentation_gpu)
68 - [Additional notes](@ref defragmentation_additional_notes)
69 - [Writing custom allocation algorithm](@ref defragmentation_custom_algorithm)
70 - \subpage lost_allocations
71 - \subpage statistics
72 - [Numeric statistics](@ref statistics_numeric_statistics)
73 - [JSON dump](@ref statistics_json_dump)
74 - \subpage allocation_annotation
75 - [Allocation user data](@ref allocation_user_data)
76 - [Allocation names](@ref allocation_names)
77 - \subpage debugging_memory_usage
78 - [Memory initialization](@ref debugging_memory_usage_initialization)
79 - [Margins](@ref debugging_memory_usage_margins)
80 - [Corruption detection](@ref debugging_memory_usage_corruption_detection)
81 - \subpage record_and_replay
82 - \subpage usage_patterns
83 - [Common mistakes](@ref usage_patterns_common_mistakes)
84 - [Simple patterns](@ref usage_patterns_simple)
85 - [Advanced patterns](@ref usage_patterns_advanced)
86 - \subpage configuration
87 - [Pointers to Vulkan functions](@ref config_Vulkan_functions)
88 - [Custom host memory allocator](@ref custom_memory_allocator)
89 - [Device memory allocation callbacks](@ref allocation_callbacks)
90 - [Device heap memory limit](@ref heap_memory_limit)
91 - \subpage vk_khr_dedicated_allocation
92 - \subpage enabling_buffer_device_address
93 - \subpage vk_amd_device_coherent_memory
94 - \subpage general_considerations
95 - [Thread safety](@ref general_considerations_thread_safety)
96 - [Validation layer warnings](@ref general_considerations_validation_layer_warnings)
97 - [Allocation algorithm](@ref general_considerations_allocation_algorithm)
98 - [Features not supported](@ref general_considerations_features_not_supported)
99
100 \section main_see_also See also
101
102 - [Product page on GPUOpen](https://gpuopen.com/gaming-product/vulkan-memory-allocator/)
103 - [Source repository on GitHub](https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator)
104
105
106
107
108 \page quick_start Quick start
109
110 \section quick_start_project_setup Project setup
111
112 Vulkan Memory Allocator comes in form of a "stb-style" single header file.
113 You don't need to build it as a separate library project.
114 You can add this file directly to your project and submit it to code repository next to your other source files.
115
116 "Single header" doesn't mean that everything is contained in C/C++ declarations,
117 like it tends to be in case of inline functions or C++ templates.
118 It means that implementation is bundled with interface in a single file and needs to be extracted using preprocessor macro.
119 If you don't do it properly, you will get linker errors.
120
121 To do it properly:
122
123 -# Include "vk_mem_alloc.h" file in each CPP file where you want to use the library.
124 This includes declarations of all members of the library.
125 -# In exacly one CPP file define following macro before this include.
126 It enables also internal definitions.
127
128 \code
129 #define VMA_IMPLEMENTATION
130 #include "vk_mem_alloc.h"
131 \endcode
132
133 It may be a good idea to create dedicated CPP file just for this purpose.
134
135 Note on language: This library is written in C++, but has C-compatible interface.
136 Thus you can include and use vk_mem_alloc.h in C or C++ code, but full
137 implementation with `VMA_IMPLEMENTATION` macro must be compiled as C++, NOT as C.
138
139 Please note that this library includes header `<vulkan/vulkan.h>`, which in turn
140 includes `<windows.h>` on Windows. If you need some specific macros defined
141 before including these headers (like `WIN32_LEAN_AND_MEAN` or
142 `WINVER` for Windows, `VK_USE_PLATFORM_WIN32_KHR` for Vulkan), you must define
143 them before every `#include` of this library.
144
145
146 \section quick_start_initialization Initialization
147
148 At program startup:
149
150 -# Initialize Vulkan to have `VkPhysicalDevice`, `VkDevice` and `VkInstance` object.
151 -# Fill VmaAllocatorCreateInfo structure and create #VmaAllocator object by
152 calling vmaCreateAllocator().
153
154 \code
155 VmaAllocatorCreateInfo allocatorInfo = {};
156 allocatorInfo.physicalDevice = physicalDevice;
157 allocatorInfo.device = device;
158 allocatorInfo.instance = instance;
159
160 VmaAllocator allocator;
161 vmaCreateAllocator(&allocatorInfo, &allocator);
162 \endcode
163
164 \section quick_start_resource_allocation Resource allocation
165
166 When you want to create a buffer or image:
167
168 -# Fill `VkBufferCreateInfo` / `VkImageCreateInfo` structure.
169 -# Fill VmaAllocationCreateInfo structure.
170 -# Call vmaCreateBuffer() / vmaCreateImage() to get `VkBuffer`/`VkImage` with memory
171 already allocated and bound to it.
172
173 \code
174 VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
175 bufferInfo.size = 65536;
176 bufferInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
177
178 VmaAllocationCreateInfo allocInfo = {};
179 allocInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
180
181 VkBuffer buffer;
182 VmaAllocation allocation;
183 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
184 \endcode
185
186 Don't forget to destroy your objects when no longer needed:
187
188 \code
189 vmaDestroyBuffer(allocator, buffer, allocation);
190 vmaDestroyAllocator(allocator);
191 \endcode
192
193
194 \page choosing_memory_type Choosing memory type
195
196 Physical devices in Vulkan support various combinations of memory heaps and
197 types. Help with choosing correct and optimal memory type for your specific
198 resource is one of the key features of this library. You can use it by filling
199 appropriate members of VmaAllocationCreateInfo structure, as described below.
200 You can also combine multiple methods.
201
202 -# If you just want to find memory type index that meets your requirements, you
203 can use function: vmaFindMemoryTypeIndex(), vmaFindMemoryTypeIndexForBufferInfo(),
204 vmaFindMemoryTypeIndexForImageInfo().
205 -# If you want to allocate a region of device memory without association with any
206 specific image or buffer, you can use function vmaAllocateMemory(). Usage of
207 this function is not recommended and usually not needed.
208 vmaAllocateMemoryPages() function is also provided for creating multiple allocations at once,
209 which may be useful for sparse binding.
210 -# If you already have a buffer or an image created, you want to allocate memory
211 for it and then you will bind it yourself, you can use function
212 vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage().
213 For binding you should use functions: vmaBindBufferMemory(), vmaBindImageMemory()
214 or their extended versions: vmaBindBufferMemory2(), vmaBindImageMemory2().
215 -# If you want to create a buffer or an image, allocate memory for it and bind
216 them together, all in one call, you can use function vmaCreateBuffer(),
217 vmaCreateImage(). This is the easiest and recommended way to use this library.
218
219 When using 3. or 4., the library internally queries Vulkan for memory types
220 supported for that buffer or image (function `vkGetBufferMemoryRequirements()`)
221 and uses only one of these types.
222
223 If no memory type can be found that meets all the requirements, these functions
224 return `VK_ERROR_FEATURE_NOT_PRESENT`.
225
226 You can leave VmaAllocationCreateInfo structure completely filled with zeros.
227 It means no requirements are specified for memory type.
228 It is valid, although not very useful.
229
230 \section choosing_memory_type_usage Usage
231
232 The easiest way to specify memory requirements is to fill member
233 VmaAllocationCreateInfo::usage using one of the values of enum #VmaMemoryUsage.
234 It defines high level, common usage types.
235 For more details, see description of this enum.
236
237 For example, if you want to create a uniform buffer that will be filled using
238 transfer only once or infrequently and used for rendering every frame, you can
239 do it using following code:
240
241 \code
242 VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
243 bufferInfo.size = 65536;
244 bufferInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
245
246 VmaAllocationCreateInfo allocInfo = {};
247 allocInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
248
249 VkBuffer buffer;
250 VmaAllocation allocation;
251 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
252 \endcode
253
254 \section choosing_memory_type_required_preferred_flags Required and preferred flags
255
256 You can specify more detailed requirements by filling members
257 VmaAllocationCreateInfo::requiredFlags and VmaAllocationCreateInfo::preferredFlags
258 with a combination of bits from enum `VkMemoryPropertyFlags`. For example,
259 if you want to create a buffer that will be persistently mapped on host (so it
260 must be `HOST_VISIBLE`) and preferably will also be `HOST_COHERENT` and `HOST_CACHED`,
261 use following code:
262
263 \code
264 VmaAllocationCreateInfo allocInfo = {};
265 allocInfo.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
266 allocInfo.preferredFlags = VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
267 allocInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
268
269 VkBuffer buffer;
270 VmaAllocation allocation;
271 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
272 \endcode
273
274 A memory type is chosen that has all the required flags and as many preferred
275 flags set as possible.
276
277 If you use VmaAllocationCreateInfo::usage, it is just internally converted to
278 a set of required and preferred flags.
279
280 \section choosing_memory_type_explicit_memory_types Explicit memory types
281
282 If you inspected memory types available on the physical device and you have
283 a preference for memory types that you want to use, you can fill member
284 VmaAllocationCreateInfo::memoryTypeBits. It is a bit mask, where each bit set
285 means that a memory type with that index is allowed to be used for the
286 allocation. Special value 0, just like `UINT32_MAX`, means there are no
287 restrictions to memory type index.
288
289 Please note that this member is NOT just a memory type index.
290 Still you can use it to choose just one, specific memory type.
291 For example, if you already determined that your buffer should be created in
292 memory type 2, use following code:
293
294 \code
295 uint32_t memoryTypeIndex = 2;
296
297 VmaAllocationCreateInfo allocInfo = {};
298 allocInfo.memoryTypeBits = 1u << memoryTypeIndex;
299
300 VkBuffer buffer;
301 VmaAllocation allocation;
302 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
303 \endcode
304
305
306 \section choosing_memory_type_custom_memory_pools Custom memory pools
307
308 If you allocate from custom memory pool, all the ways of specifying memory
309 requirements described above are not applicable and the aforementioned members
310 of VmaAllocationCreateInfo structure are ignored. Memory type is selected
311 explicitly when creating the pool and then used to make all the allocations from
312 that pool. For further details, see \ref custom_memory_pools.
313
314 \section choosing_memory_type_dedicated_allocations Dedicated allocations
315
316 Memory for allocations is reserved out of larger block of `VkDeviceMemory`
317 allocated from Vulkan internally. That's the main feature of this whole library.
318 You can still request a separate memory block to be created for an allocation,
319 just like you would do in a trivial solution without using any allocator.
320 In that case, a buffer or image is always bound to that memory at offset 0.
321 This is called a "dedicated allocation".
322 You can explicitly request it by using flag #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT.
323 The library can also internally decide to use dedicated allocation in some cases, e.g.:
324
325 - When the size of the allocation is large.
326 - When [VK_KHR_dedicated_allocation](@ref vk_khr_dedicated_allocation) extension is enabled
327 and it reports that dedicated allocation is required or recommended for the resource.
328 - When allocation of next big memory block fails due to not enough device memory,
329 but allocation with the exact requested size succeeds.
330
331
332 \page memory_mapping Memory mapping
333
334 To "map memory" in Vulkan means to obtain a CPU pointer to `VkDeviceMemory`,
335 to be able to read from it or write to it in CPU code.
336 Mapping is possible only of memory allocated from a memory type that has
337 `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag.
338 Functions `vkMapMemory()`, `vkUnmapMemory()` are designed for this purpose.
339 You can use them directly with memory allocated by this library,
340 but it is not recommended because of following issue:
341 Mapping the same `VkDeviceMemory` block multiple times is illegal - only one mapping at a time is allowed.
342 This includes mapping disjoint regions. Mapping is not reference-counted internally by Vulkan.
343 Because of this, Vulkan Memory Allocator provides following facilities:
344
345 \section memory_mapping_mapping_functions Mapping functions
346
347 The library provides following functions for mapping of a specific #VmaAllocation: vmaMapMemory(), vmaUnmapMemory().
348 They are safer and more convenient to use than standard Vulkan functions.
349 You can map an allocation multiple times simultaneously - mapping is reference-counted internally.
350 You can also map different allocations simultaneously regardless of whether they use the same `VkDeviceMemory` block.
351 The way it's implemented is that the library always maps entire memory block, not just region of the allocation.
352 For further details, see description of vmaMapMemory() function.
353 Example:
354
355 \code
356 // Having these objects initialized:
357
358 struct ConstantBuffer
359 {
360 ...
361 };
362 ConstantBuffer constantBufferData;
363
364 VmaAllocator allocator;
365 VkBuffer constantBuffer;
366 VmaAllocation constantBufferAllocation;
367
368 // You can map and fill your buffer using following code:
369
370 void* mappedData;
371 vmaMapMemory(allocator, constantBufferAllocation, &mappedData);
372 memcpy(mappedData, &constantBufferData, sizeof(constantBufferData));
373 vmaUnmapMemory(allocator, constantBufferAllocation);
374 \endcode
375
376 When mapping, you may see a warning from Vulkan validation layer similar to this one:
377
378 <i>Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used.</i>
379
380 It happens because the library maps entire `VkDeviceMemory` block, where different
381 types of images and buffers may end up together, especially on GPUs with unified memory like Intel.
382 You can safely ignore it if you are sure you access only memory of the intended
383 object that you wanted to map.
384
385
386 \section memory_mapping_persistently_mapped_memory Persistently mapped memory
387
388 Kepping your memory persistently mapped is generally OK in Vulkan.
389 You don't need to unmap it before using its data on the GPU.
390 The library provides a special feature designed for that:
391 Allocations made with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag set in
392 VmaAllocationCreateInfo::flags stay mapped all the time,
393 so you can just access CPU pointer to it any time
394 without a need to call any "map" or "unmap" function.
395 Example:
396
397 \code
398 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
399 bufCreateInfo.size = sizeof(ConstantBuffer);
400 bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
401
402 VmaAllocationCreateInfo allocCreateInfo = {};
403 allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY;
404 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
405
406 VkBuffer buf;
407 VmaAllocation alloc;
408 VmaAllocationInfo allocInfo;
409 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
410
411 // Buffer is already mapped. You can access its memory.
412 memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
413 \endcode
414
415 There are some exceptions though, when you should consider mapping memory only for a short period of time:
416
417 - When operating system is Windows 7 or 8.x (Windows 10 is not affected because it uses WDDM2),
418 device is discrete AMD GPU,
419 and memory type is the special 256 MiB pool of `DEVICE_LOCAL + HOST_VISIBLE` memory
420 (selected when you use #VMA_MEMORY_USAGE_CPU_TO_GPU),
421 then whenever a memory block allocated from this memory type stays mapped
422 for the time of any call to `vkQueueSubmit()` or `vkQueuePresentKHR()`, this
423 block is migrated by WDDM to system RAM, which degrades performance. It doesn't
424 matter if that particular memory block is actually used by the command buffer
425 being submitted.
426 - On Mac/MoltenVK there is a known bug - [Issue #175](https://github.com/KhronosGroup/MoltenVK/issues/175)
427 which requires unmapping before GPU can see updated texture.
428 - Keeping many large memory blocks mapped may impact performance or stability of some debugging tools.
429
430 \section memory_mapping_cache_control Cache flush and invalidate
431
432 Memory in Vulkan doesn't need to be unmapped before using it on GPU,
433 but unless a memory types has `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` flag set,
434 you need to manually **invalidate** cache before reading of mapped pointer
435 and **flush** cache after writing to mapped pointer.
436 Map/unmap operations don't do that automatically.
437 Vulkan provides following functions for this purpose `vkFlushMappedMemoryRanges()`,
438 `vkInvalidateMappedMemoryRanges()`, but this library provides more convenient
439 functions that refer to given allocation object: vmaFlushAllocation(),
440 vmaInvalidateAllocation(),
441 or multiple objects at once: vmaFlushAllocations(), vmaInvalidateAllocations().
442
443 Regions of memory specified for flush/invalidate must be aligned to
444 `VkPhysicalDeviceLimits::nonCoherentAtomSize`. This is automatically ensured by the library.
445 In any memory type that is `HOST_VISIBLE` but not `HOST_COHERENT`, all allocations
446 within blocks are aligned to this value, so their offsets are always multiply of
447 `nonCoherentAtomSize` and two different allocations never share same "line" of this size.
448
449 Please note that memory allocated with #VMA_MEMORY_USAGE_CPU_ONLY is guaranteed to be `HOST_COHERENT`.
450
451 Also, Windows drivers from all 3 **PC** GPU vendors (AMD, Intel, NVIDIA)
452 currently provide `HOST_COHERENT` flag on all memory types that are
453 `HOST_VISIBLE`, so on this platform you may not need to bother.
454
455 \section memory_mapping_finding_if_memory_mappable Finding out if memory is mappable
456
457 It may happen that your allocation ends up in memory that is `HOST_VISIBLE` (available for mapping)
458 despite it wasn't explicitly requested.
459 For example, application may work on integrated graphics with unified memory (like Intel) or
460 allocation from video memory might have failed, so the library chose system memory as fallback.
461
462 You can detect this case and map such allocation to access its memory on CPU directly,
463 instead of launching a transfer operation.
464 In order to do that: inspect `allocInfo.memoryType`, call vmaGetMemoryTypeProperties(),
465 and look for `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag in properties of that memory type.
466
467 \code
468 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
469 bufCreateInfo.size = sizeof(ConstantBuffer);
470 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
471
472 VmaAllocationCreateInfo allocCreateInfo = {};
473 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
474 allocCreateInfo.preferredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
475
476 VkBuffer buf;
477 VmaAllocation alloc;
478 VmaAllocationInfo allocInfo;
479 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
480
481 VkMemoryPropertyFlags memFlags;
482 vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags);
483 if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
484 {
485 // Allocation ended up in mappable memory. You can map it and access it directly.
486 void* mappedData;
487 vmaMapMemory(allocator, alloc, &mappedData);
488 memcpy(mappedData, &constantBufferData, sizeof(constantBufferData));
489 vmaUnmapMemory(allocator, alloc);
490 }
491 else
492 {
493 // Allocation ended up in non-mappable memory.
494 // You need to create CPU-side buffer in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer.
495 }
496 \endcode
497
498 You can even use #VMA_ALLOCATION_CREATE_MAPPED_BIT flag while creating allocations
499 that are not necessarily `HOST_VISIBLE` (e.g. using #VMA_MEMORY_USAGE_GPU_ONLY).
500 If the allocation ends up in memory type that is `HOST_VISIBLE`, it will be persistently mapped and you can use it directly.
501 If not, the flag is just ignored.
502 Example:
503
504 \code
505 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
506 bufCreateInfo.size = sizeof(ConstantBuffer);
507 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
508
509 VmaAllocationCreateInfo allocCreateInfo = {};
510 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
511 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
512
513 VkBuffer buf;
514 VmaAllocation alloc;
515 VmaAllocationInfo allocInfo;
516 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
517
518 if(allocInfo.pMappedData != nullptr)
519 {
520 // Allocation ended up in mappable memory.
521 // It's persistently mapped. You can access it directly.
522 memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
523 }
524 else
525 {
526 // Allocation ended up in non-mappable memory.
527 // You need to create CPU-side buffer in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer.
528 }
529 \endcode
530
531
532 \page staying_within_budget Staying within budget
533
534 When developing a graphics-intensive game or program, it is important to avoid allocating
535 more GPU memory than it's physically available. When the memory is over-committed,
536 various bad things can happen, depending on the specific GPU, graphics driver, and
537 operating system:
538
539 - It may just work without any problems.
540 - The application may slow down because some memory blocks are moved to system RAM
541 and the GPU has to access them through PCI Express bus.
542 - A new allocation may take very long time to complete, even few seconds, and possibly
543 freeze entire system.
544 - The new allocation may fail with `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
545 - It may even result in GPU crash (TDR), observed as `VK_ERROR_DEVICE_LOST`
546 returned somewhere later.
547
548 \section staying_within_budget_querying_for_budget Querying for budget
549
550 To query for current memory usage and available budget, use function vmaGetBudget().
551 Returned structure #VmaBudget contains quantities expressed in bytes, per Vulkan memory heap.
552
553 Please note that this function returns different information and works faster than
554 vmaCalculateStats(). vmaGetBudget() can be called every frame or even before every
555 allocation, while vmaCalculateStats() is intended to be used rarely,
556 only to obtain statistical information, e.g. for debugging purposes.
557
558 It is recommended to use <b>VK_EXT_memory_budget</b> device extension to obtain information
559 about the budget from Vulkan device. VMA is able to use this extension automatically.
560 When not enabled, the allocator behaves same way, but then it estimates current usage
561 and available budget based on its internal information and Vulkan memory heap sizes,
562 which may be less precise. In order to use this extension:
563
564 1. Make sure extensions VK_EXT_memory_budget and VK_KHR_get_physical_device_properties2
565 required by it are available and enable them. Please note that the first is a device
566 extension and the second is instance extension!
567 2. Use flag #VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT when creating #VmaAllocator object.
568 3. Make sure to call vmaSetCurrentFrameIndex() every frame. Budget is queried from
569 Vulkan inside of it to avoid overhead of querying it with every allocation.
570
571 \section staying_within_budget_controlling_memory_usage Controlling memory usage
572
573 There are many ways in which you can try to stay within the budget.
574
575 First, when making new allocation requires allocating a new memory block, the library
576 tries not to exceed the budget automatically. If a block with default recommended size
577 (e.g. 256 MB) would go over budget, a smaller block is allocated, possibly even
578 dedicated memory for just this resource.
579
580 If the size of the requested resource plus current memory usage is more than the
581 budget, by default the library still tries to create it, leaving it to the Vulkan
582 implementation whether the allocation succeeds or fails. You can change this behavior
583 by using #VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT flag. With it, the allocation is
584 not made if it would exceed the budget or if the budget is already exceeded.
585 Some other allocations become lost instead to make room for it, if the mechanism of
586 [lost allocations](@ref lost_allocations) is used.
587 If that is not possible, the allocation fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
588 Example usage pattern may be to pass the #VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT flag
589 when creating resources that are not essential for the application (e.g. the texture
590 of a specific object) and not to pass it when creating critically important resources
591 (e.g. render targets).
592
593 Finally, you can also use #VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT flag to make sure
594 a new allocation is created only when it fits inside one of the existing memory blocks.
595 If it would require to allocate a new block, if fails instead with `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
596 This also ensures that the function call is very fast because it never goes to Vulkan
597 to obtain a new block.
598
599 Please note that creating \ref custom_memory_pools with VmaPoolCreateInfo::minBlockCount
600 set to more than 0 will try to allocate memory blocks without checking whether they
601 fit within budget.
602
603
604 \page resource_aliasing Resource aliasing (overlap)
605
606 New explicit graphics APIs (Vulkan and Direct3D 12), thanks to manual memory
607 management, give an opportunity to alias (overlap) multiple resources in the
608 same region of memory - a feature not available in the old APIs (Direct3D 11, OpenGL).
609 It can be useful to save video memory, but it must be used with caution.
610
611 For example, if you know the flow of your whole render frame in advance, you
612 are going to use some intermediate textures or buffers only during a small range of render passes,
613 and you know these ranges don't overlap in time, you can bind these resources to
614 the same place in memory, even if they have completely different parameters (width, height, format etc.).
615
616 ![Resource aliasing (overlap)](../gfx/Aliasing.png)
617
618 Such scenario is possible using VMA, but you need to create your images manually.
619 Then you need to calculate parameters of an allocation to be made using formula:
620
621 - allocation size = max(size of each image)
622 - allocation alignment = max(alignment of each image)
623 - allocation memoryTypeBits = bitwise AND(memoryTypeBits of each image)
624
625 Following example shows two different images bound to the same place in memory,
626 allocated to fit largest of them.
627
628 \code
629 // A 512x512 texture to be sampled.
630 VkImageCreateInfo img1CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
631 img1CreateInfo.imageType = VK_IMAGE_TYPE_2D;
632 img1CreateInfo.extent.width = 512;
633 img1CreateInfo.extent.height = 512;
634 img1CreateInfo.extent.depth = 1;
635 img1CreateInfo.mipLevels = 10;
636 img1CreateInfo.arrayLayers = 1;
637 img1CreateInfo.format = VK_FORMAT_R8G8B8A8_SRGB;
638 img1CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
639 img1CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
640 img1CreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
641 img1CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT;
642
643 // A full screen texture to be used as color attachment.
644 VkImageCreateInfo img2CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
645 img2CreateInfo.imageType = VK_IMAGE_TYPE_2D;
646 img2CreateInfo.extent.width = 1920;
647 img2CreateInfo.extent.height = 1080;
648 img2CreateInfo.extent.depth = 1;
649 img2CreateInfo.mipLevels = 1;
650 img2CreateInfo.arrayLayers = 1;
651 img2CreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
652 img2CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
653 img2CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
654 img2CreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
655 img2CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT;
656
657 VkImage img1;
658 res = vkCreateImage(device, &img1CreateInfo, nullptr, &img1);
659 VkImage img2;
660 res = vkCreateImage(device, &img2CreateInfo, nullptr, &img2);
661
662 VkMemoryRequirements img1MemReq;
663 vkGetImageMemoryRequirements(device, img1, &img1MemReq);
664 VkMemoryRequirements img2MemReq;
665 vkGetImageMemoryRequirements(device, img2, &img2MemReq);
666
667 VkMemoryRequirements finalMemReq = {};
668 finalMemReq.size = std::max(img1MemReq.size, img2MemReq.size);
669 finalMemReq.alignment = std::max(img1MemReq.alignment, img2MemReq.alignment);
670 finalMemReq.memoryTypeBits = img1MemReq.memoryTypeBits & img2MemReq.memoryTypeBits;
671 // Validate if(finalMemReq.memoryTypeBits != 0)
672
673 VmaAllocationCreateInfo allocCreateInfo = {};
674 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
675
676 VmaAllocation alloc;
677 res = vmaAllocateMemory(allocator, &finalMemReq, &allocCreateInfo, &alloc, nullptr);
678
679 res = vmaBindImageMemory(allocator, alloc, img1);
680 res = vmaBindImageMemory(allocator, alloc, img2);
681
682 // You can use img1, img2 here, but not at the same time!
683
684 vmaFreeMemory(allocator, alloc);
685 vkDestroyImage(allocator, img2, nullptr);
686 vkDestroyImage(allocator, img1, nullptr);
687 \endcode
688
689 Remember that using resouces that alias in memory requires proper synchronization.
690 You need to issue a memory barrier to make sure commands that use `img1` and `img2`
691 don't overlap on GPU timeline.
692 You also need to treat a resource after aliasing as uninitialized - containing garbage data.
693 For example, if you use `img1` and then want to use `img2`, you need to issue
694 an image memory barrier for `img2` with `oldLayout` = `VK_IMAGE_LAYOUT_UNDEFINED`.
695
696 Additional considerations:
697
698 - Vulkan also allows to interpret contents of memory between aliasing resources consistently in some cases.
699 See chapter 11.8. "Memory Aliasing" of Vulkan specification or `VK_IMAGE_CREATE_ALIAS_BIT` flag.
700 - You can create more complex layout where different images and buffers are bound
701 at different offsets inside one large allocation. For example, one can imagine
702 a big texture used in some render passes, aliasing with a set of many small buffers
703 used between in some further passes. To bind a resource at non-zero offset of an allocation,
704 use vmaBindBufferMemory2() / vmaBindImageMemory2().
705 - Before allocating memory for the resources you want to alias, check `memoryTypeBits`
706 returned in memory requirements of each resource to make sure the bits overlap.
707 Some GPUs may expose multiple memory types suitable e.g. only for buffers or
708 images with `COLOR_ATTACHMENT` usage, so the sets of memory types supported by your
709 resources may be disjoint. Aliasing them is not possible in that case.
710
711
712 \page custom_memory_pools Custom memory pools
713
714 A memory pool contains a number of `VkDeviceMemory` blocks.
715 The library automatically creates and manages default pool for each memory type available on the device.
716 Default memory pool automatically grows in size.
717 Size of allocated blocks is also variable and managed automatically.
718
719 You can create custom pool and allocate memory out of it.
720 It can be useful if you want to:
721
722 - Keep certain kind of allocations separate from others.
723 - Enforce particular, fixed size of Vulkan memory blocks.
724 - Limit maximum amount of Vulkan memory allocated for that pool.
725 - Reserve minimum or fixed amount of Vulkan memory always preallocated for that pool.
726
727 To use custom memory pools:
728
729 -# Fill VmaPoolCreateInfo structure.
730 -# Call vmaCreatePool() to obtain #VmaPool handle.
731 -# When making an allocation, set VmaAllocationCreateInfo::pool to this handle.
732 You don't need to specify any other parameters of this structure, like `usage`.
733
734 Example:
735
736 \code
737 // Create a pool that can have at most 2 blocks, 128 MiB each.
738 VmaPoolCreateInfo poolCreateInfo = {};
739 poolCreateInfo.memoryTypeIndex = ...
740 poolCreateInfo.blockSize = 128ull * 1024 * 1024;
741 poolCreateInfo.maxBlockCount = 2;
742
743 VmaPool pool;
744 vmaCreatePool(allocator, &poolCreateInfo, &pool);
745
746 // Allocate a buffer out of it.
747 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
748 bufCreateInfo.size = 1024;
749 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
750
751 VmaAllocationCreateInfo allocCreateInfo = {};
752 allocCreateInfo.pool = pool;
753
754 VkBuffer buf;
755 VmaAllocation alloc;
756 VmaAllocationInfo allocInfo;
757 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
758 \endcode
759
760 You have to free all allocations made from this pool before destroying it.
761
762 \code
763 vmaDestroyBuffer(allocator, buf, alloc);
764 vmaDestroyPool(allocator, pool);
765 \endcode
766
767 \section custom_memory_pools_MemTypeIndex Choosing memory type index
768
769 When creating a pool, you must explicitly specify memory type index.
770 To find the one suitable for your buffers or images, you can use helper functions
771 vmaFindMemoryTypeIndexForBufferInfo(), vmaFindMemoryTypeIndexForImageInfo().
772 You need to provide structures with example parameters of buffers or images
773 that you are going to create in that pool.
774
775 \code
776 VkBufferCreateInfo exampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
777 exampleBufCreateInfo.size = 1024; // Whatever.
778 exampleBufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; // Change if needed.
779
780 VmaAllocationCreateInfo allocCreateInfo = {};
781 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; // Change if needed.
782
783 uint32_t memTypeIndex;
784 vmaFindMemoryTypeIndexForBufferInfo(allocator, &exampleBufCreateInfo, &allocCreateInfo, &memTypeIndex);
785
786 VmaPoolCreateInfo poolCreateInfo = {};
787 poolCreateInfo.memoryTypeIndex = memTypeIndex;
788 // ...
789 \endcode
790
791 When creating buffers/images allocated in that pool, provide following parameters:
792
793 - `VkBufferCreateInfo`: Prefer to pass same parameters as above.
794 Otherwise you risk creating resources in a memory type that is not suitable for them, which may result in undefined behavior.
795 Using different `VK_BUFFER_USAGE_` flags may work, but you shouldn't create images in a pool intended for buffers
796 or the other way around.
797 - VmaAllocationCreateInfo: You don't need to pass same parameters. Fill only `pool` member.
798 Other members are ignored anyway.
799
800 \section linear_algorithm Linear allocation algorithm
801
802 Each Vulkan memory block managed by this library has accompanying metadata that
803 keeps track of used and unused regions. By default, the metadata structure and
804 algorithm tries to find best place for new allocations among free regions to
805 optimize memory usage. This way you can allocate and free objects in any order.
806
807 ![Default allocation algorithm](../gfx/Linear_allocator_1_algo_default.png)
808
809 Sometimes there is a need to use simpler, linear allocation algorithm. You can
810 create custom pool that uses such algorithm by adding flag
811 #VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT to VmaPoolCreateInfo::flags while creating
812 #VmaPool object. Then an alternative metadata management is used. It always
813 creates new allocations after last one and doesn't reuse free regions after
814 allocations freed in the middle. It results in better allocation performance and
815 less memory consumed by metadata.
816
817 ![Linear allocation algorithm](../gfx/Linear_allocator_2_algo_linear.png)
818
819 With this one flag, you can create a custom pool that can be used in many ways:
820 free-at-once, stack, double stack, and ring buffer. See below for details.
821
822 \subsection linear_algorithm_free_at_once Free-at-once
823
824 In a pool that uses linear algorithm, you still need to free all the allocations
825 individually, e.g. by using vmaFreeMemory() or vmaDestroyBuffer(). You can free
826 them in any order. New allocations are always made after last one - free space
827 in the middle is not reused. However, when you release all the allocation and
828 the pool becomes empty, allocation starts from the beginning again. This way you
829 can use linear algorithm to speed up creation of allocations that you are going
830 to release all at once.
831
832 ![Free-at-once](../gfx/Linear_allocator_3_free_at_once.png)
833
834 This mode is also available for pools created with VmaPoolCreateInfo::maxBlockCount
835 value that allows multiple memory blocks.
836
837 \subsection linear_algorithm_stack Stack
838
839 When you free an allocation that was created last, its space can be reused.
840 Thanks to this, if you always release allocations in the order opposite to their
841 creation (LIFO - Last In First Out), you can achieve behavior of a stack.
842
843 ![Stack](../gfx/Linear_allocator_4_stack.png)
844
845 This mode is also available for pools created with VmaPoolCreateInfo::maxBlockCount
846 value that allows multiple memory blocks.
847
848 \subsection linear_algorithm_double_stack Double stack
849
850 The space reserved by a custom pool with linear algorithm may be used by two
851 stacks:
852
853 - First, default one, growing up from offset 0.
854 - Second, "upper" one, growing down from the end towards lower offsets.
855
856 To make allocation from upper stack, add flag #VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT
857 to VmaAllocationCreateInfo::flags.
858
859 ![Double stack](../gfx/Linear_allocator_7_double_stack.png)
860
861 Double stack is available only in pools with one memory block -
862 VmaPoolCreateInfo::maxBlockCount must be 1. Otherwise behavior is undefined.
863
864 When the two stacks' ends meet so there is not enough space between them for a
865 new allocation, such allocation fails with usual
866 `VK_ERROR_OUT_OF_DEVICE_MEMORY` error.
867
868 \subsection linear_algorithm_ring_buffer Ring buffer
869
870 When you free some allocations from the beginning and there is not enough free space
871 for a new one at the end of a pool, allocator's "cursor" wraps around to the
872 beginning and starts allocation there. Thanks to this, if you always release
873 allocations in the same order as you created them (FIFO - First In First Out),
874 you can achieve behavior of a ring buffer / queue.
875
876 ![Ring buffer](../gfx/Linear_allocator_5_ring_buffer.png)
877
878 Pools with linear algorithm support [lost allocations](@ref lost_allocations) when used as ring buffer.
879 If there is not enough free space for a new allocation, but existing allocations
880 from the front of the queue can become lost, they become lost and the allocation
881 succeeds.
882
883 ![Ring buffer with lost allocations](../gfx/Linear_allocator_6_ring_buffer_lost.png)
884
885 Ring buffer is available only in pools with one memory block -
886 VmaPoolCreateInfo::maxBlockCount must be 1. Otherwise behavior is undefined.
887
888 \section buddy_algorithm Buddy allocation algorithm
889
890 There is another allocation algorithm that can be used with custom pools, called
891 "buddy". Its internal data structure is based on a tree of blocks, each having
892 size that is a power of two and a half of its parent's size. When you want to
893 allocate memory of certain size, a free node in the tree is located. If it's too
894 large, it is recursively split into two halves (called "buddies"). However, if
895 requested allocation size is not a power of two, the size of a tree node is
896 aligned up to the nearest power of two and the remaining space is wasted. When
897 two buddy nodes become free, they are merged back into one larger node.
898
899 ![Buddy allocator](../gfx/Buddy_allocator.png)
900
901 The advantage of buddy allocation algorithm over default algorithm is faster
902 allocation and deallocation, as well as smaller external fragmentation. The
903 disadvantage is more wasted space (internal fragmentation).
904
905 For more information, please read ["Buddy memory allocation" on Wikipedia](https://en.wikipedia.org/wiki/Buddy_memory_allocation)
906 or other sources that describe this concept in general.
907
908 To use buddy allocation algorithm with a custom pool, add flag
909 #VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT to VmaPoolCreateInfo::flags while creating
910 #VmaPool object.
911
912 Several limitations apply to pools that use buddy algorithm:
913
914 - It is recommended to use VmaPoolCreateInfo::blockSize that is a power of two.
915 Otherwise, only largest power of two smaller than the size is used for
916 allocations. The remaining space always stays unused.
917 - [Margins](@ref debugging_memory_usage_margins) and
918 [corruption detection](@ref debugging_memory_usage_corruption_detection)
919 don't work in such pools.
920 - [Lost allocations](@ref lost_allocations) don't work in such pools. You can
921 use them, but they never become lost. Support may be added in the future.
922 - [Defragmentation](@ref defragmentation) doesn't work with allocations made from
923 such pool.
924
925 \page defragmentation Defragmentation
926
927 Interleaved allocations and deallocations of many objects of varying size can
928 cause fragmentation over time, which can lead to a situation where the library is unable
929 to find a continuous range of free memory for a new allocation despite there is
930 enough free space, just scattered across many small free ranges between existing
931 allocations.
932
933 To mitigate this problem, you can use defragmentation feature:
934 structure #VmaDefragmentationInfo2, function vmaDefragmentationBegin(), vmaDefragmentationEnd().
935 Given set of allocations,
936 this function can move them to compact used memory, ensure more continuous free
937 space and possibly also free some `VkDeviceMemory` blocks.
938
939 What the defragmentation does is:
940
941 - Updates #VmaAllocation objects to point to new `VkDeviceMemory` and offset.
942 After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or
943 VmaAllocationInfo::offset changes. You must query them again using
944 vmaGetAllocationInfo() if you need them.
945 - Moves actual data in memory.
946
947 What it doesn't do, so you need to do it yourself:
948
949 - Recreate buffers and images that were bound to allocations that were defragmented and
950 bind them with their new places in memory.
951 You must use `vkDestroyBuffer()`, `vkDestroyImage()`,
952 `vkCreateBuffer()`, `vkCreateImage()`, vmaBindBufferMemory(), vmaBindImageMemory()
953 for that purpose and NOT vmaDestroyBuffer(),
954 vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage(), because you don't need to
955 destroy or create allocation objects!
956 - Recreate views and update descriptors that point to these buffers and images.
957
958 \section defragmentation_cpu Defragmenting CPU memory
959
960 Following example demonstrates how you can run defragmentation on CPU.
961 Only allocations created in memory types that are `HOST_VISIBLE` can be defragmented.
962 Others are ignored.
963
964 The way it works is:
965
966 - It temporarily maps entire memory blocks when necessary.
967 - It moves data using `memmove()` function.
968
969 \code
970 // Given following variables already initialized:
971 VkDevice device;
972 VmaAllocator allocator;
973 std::vector<VkBuffer> buffers;
974 std::vector<VmaAllocation> allocations;
975
976
977 const uint32_t allocCount = (uint32_t)allocations.size();
978 std::vector<VkBool32> allocationsChanged(allocCount);
979
980 VmaDefragmentationInfo2 defragInfo = {};
981 defragInfo.allocationCount = allocCount;
982 defragInfo.pAllocations = allocations.data();
983 defragInfo.pAllocationsChanged = allocationsChanged.data();
984 defragInfo.maxCpuBytesToMove = VK_WHOLE_SIZE; // No limit.
985 defragInfo.maxCpuAllocationsToMove = UINT32_MAX; // No limit.
986
987 VmaDefragmentationContext defragCtx;
988 vmaDefragmentationBegin(allocator, &defragInfo, nullptr, &defragCtx);
989 vmaDefragmentationEnd(allocator, defragCtx);
990
991 for(uint32_t i = 0; i < allocCount; ++i)
992 {
993 if(allocationsChanged[i])
994 {
995 // Destroy buffer that is immutably bound to memory region which is no longer valid.
996 vkDestroyBuffer(device, buffers[i], nullptr);
997
998 // Create new buffer with same parameters.
999 VkBufferCreateInfo bufferInfo = ...;
1000 vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]);
1001
1002 // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning.
1003
1004 // Bind new buffer to new memory region. Data contained in it is already moved.
1005 VmaAllocationInfo allocInfo;
1006 vmaGetAllocationInfo(allocator, allocations[i], &allocInfo);
1007 vmaBindBufferMemory(allocator, allocations[i], buffers[i]);
1008 }
1009 }
1010 \endcode
1011
1012 Setting VmaDefragmentationInfo2::pAllocationsChanged is optional.
1013 This output array tells whether particular allocation in VmaDefragmentationInfo2::pAllocations at the same index
1014 has been modified during defragmentation.
1015 You can pass null, but you then need to query every allocation passed to defragmentation
1016 for new parameters using vmaGetAllocationInfo() if you might need to recreate and rebind a buffer or image associated with it.
1017
1018 If you use [Custom memory pools](@ref choosing_memory_type_custom_memory_pools),
1019 you can fill VmaDefragmentationInfo2::poolCount and VmaDefragmentationInfo2::pPools
1020 instead of VmaDefragmentationInfo2::allocationCount and VmaDefragmentationInfo2::pAllocations
1021 to defragment all allocations in given pools.
1022 You cannot use VmaDefragmentationInfo2::pAllocationsChanged in that case.
1023 You can also combine both methods.
1024
1025 \section defragmentation_gpu Defragmenting GPU memory
1026
1027 It is also possible to defragment allocations created in memory types that are not `HOST_VISIBLE`.
1028 To do that, you need to pass a command buffer that meets requirements as described in
1029 VmaDefragmentationInfo2::commandBuffer. The way it works is:
1030
1031 - It creates temporary buffers and binds them to entire memory blocks when necessary.
1032 - It issues `vkCmdCopyBuffer()` to passed command buffer.
1033
1034 Example:
1035
1036 \code
1037 // Given following variables already initialized:
1038 VkDevice device;
1039 VmaAllocator allocator;
1040 VkCommandBuffer commandBuffer;
1041 std::vector<VkBuffer> buffers;
1042 std::vector<VmaAllocation> allocations;
1043
1044
1045 const uint32_t allocCount = (uint32_t)allocations.size();
1046 std::vector<VkBool32> allocationsChanged(allocCount);
1047
1048 VkCommandBufferBeginInfo cmdBufBeginInfo = ...;
1049 vkBeginCommandBuffer(commandBuffer, &cmdBufBeginInfo);
1050
1051 VmaDefragmentationInfo2 defragInfo = {};
1052 defragInfo.allocationCount = allocCount;
1053 defragInfo.pAllocations = allocations.data();
1054 defragInfo.pAllocationsChanged = allocationsChanged.data();
1055 defragInfo.maxGpuBytesToMove = VK_WHOLE_SIZE; // Notice it's "GPU" this time.
1056 defragInfo.maxGpuAllocationsToMove = UINT32_MAX; // Notice it's "GPU" this time.
1057 defragInfo.commandBuffer = commandBuffer;
1058
1059 VmaDefragmentationContext defragCtx;
1060 vmaDefragmentationBegin(allocator, &defragInfo, nullptr, &defragCtx);
1061
1062 vkEndCommandBuffer(commandBuffer);
1063
1064 // Submit commandBuffer.
1065 // Wait for a fence that ensures commandBuffer execution finished.
1066
1067 vmaDefragmentationEnd(allocator, defragCtx);
1068
1069 for(uint32_t i = 0; i < allocCount; ++i)
1070 {
1071 if(allocationsChanged[i])
1072 {
1073 // Destroy buffer that is immutably bound to memory region which is no longer valid.
1074 vkDestroyBuffer(device, buffers[i], nullptr);
1075
1076 // Create new buffer with same parameters.
1077 VkBufferCreateInfo bufferInfo = ...;
1078 vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]);
1079
1080 // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning.
1081
1082 // Bind new buffer to new memory region. Data contained in it is already moved.
1083 VmaAllocationInfo allocInfo;
1084 vmaGetAllocationInfo(allocator, allocations[i], &allocInfo);
1085 vmaBindBufferMemory(allocator, allocations[i], buffers[i]);
1086 }
1087 }
1088 \endcode
1089
1090 You can combine these two methods by specifying non-zero `maxGpu*` as well as `maxCpu*` parameters.
1091 The library automatically chooses best method to defragment each memory pool.
1092
1093 You may try not to block your entire program to wait until defragmentation finishes,
1094 but do it in the background, as long as you carefully fullfill requirements described
1095 in function vmaDefragmentationBegin().
1096
1097 \section defragmentation_additional_notes Additional notes
1098
1099 It is only legal to defragment allocations bound to:
1100
1101 - buffers
1102 - images created with `VK_IMAGE_CREATE_ALIAS_BIT`, `VK_IMAGE_TILING_LINEAR`, and
1103 being currently in `VK_IMAGE_LAYOUT_GENERAL` or `VK_IMAGE_LAYOUT_PREINITIALIZED`.
1104
1105 Defragmentation of images created with `VK_IMAGE_TILING_OPTIMAL` or in any other
1106 layout may give undefined results.
1107
1108 If you defragment allocations bound to images, new images to be bound to new
1109 memory region after defragmentation should be created with `VK_IMAGE_LAYOUT_PREINITIALIZED`
1110 and then transitioned to their original layout from before defragmentation if
1111 needed using an image memory barrier.
1112
1113 While using defragmentation, you may experience validation layer warnings, which you just need to ignore.
1114 See [Validation layer warnings](@ref general_considerations_validation_layer_warnings).
1115
1116 Please don't expect memory to be fully compacted after defragmentation.
1117 Algorithms inside are based on some heuristics that try to maximize number of Vulkan
1118 memory blocks to make totally empty to release them, as well as to maximimze continuous
1119 empty space inside remaining blocks, while minimizing the number and size of allocations that
1120 need to be moved. Some fragmentation may still remain - this is normal.
1121
1122 \section defragmentation_custom_algorithm Writing custom defragmentation algorithm
1123
1124 If you want to implement your own, custom defragmentation algorithm,
1125 there is infrastructure prepared for that,
1126 but it is not exposed through the library API - you need to hack its source code.
1127 Here are steps needed to do this:
1128
1129 -# Main thing you need to do is to define your own class derived from base abstract
1130 class `VmaDefragmentationAlgorithm` and implement your version of its pure virtual methods.
1131 See definition and comments of this class for details.
1132 -# Your code needs to interact with device memory block metadata.
1133 If you need more access to its data than it's provided by its public interface,
1134 declare your new class as a friend class e.g. in class `VmaBlockMetadata_Generic`.
1135 -# If you want to create a flag that would enable your algorithm or pass some additional
1136 flags to configure it, add them to `VmaDefragmentationFlagBits` and use them in
1137 VmaDefragmentationInfo2::flags.
1138 -# Modify function `VmaBlockVectorDefragmentationContext::Begin` to create object
1139 of your new class whenever needed.
1140
1141
1142 \page lost_allocations Lost allocations
1143
1144 If your game oversubscribes video memory, if may work OK in previous-generation
1145 graphics APIs (DirectX 9, 10, 11, OpenGL) because resources are automatically
1146 paged to system RAM. In Vulkan you can't do it because when you run out of
1147 memory, an allocation just fails. If you have more data (e.g. textures) that can
1148 fit into VRAM and you don't need it all at once, you may want to upload them to
1149 GPU on demand and "push out" ones that are not used for a long time to make room
1150 for the new ones, effectively using VRAM (or a cartain memory pool) as a form of
1151 cache. Vulkan Memory Allocator can help you with that by supporting a concept of
1152 "lost allocations".
1153
1154 To create an allocation that can become lost, include #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
1155 flag in VmaAllocationCreateInfo::flags. Before using a buffer or image bound to
1156 such allocation in every new frame, you need to query it if it's not lost.
1157 To check it, call vmaTouchAllocation().
1158 If the allocation is lost, you should not use it or buffer/image bound to it.
1159 You mustn't forget to destroy this allocation and this buffer/image.
1160 vmaGetAllocationInfo() can also be used for checking status of the allocation.
1161 Allocation is lost when returned VmaAllocationInfo::deviceMemory == `VK_NULL_HANDLE`.
1162
1163 To create an allocation that can make some other allocations lost to make room
1164 for it, use #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag. You will
1165 usually use both flags #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT and
1166 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT at the same time.
1167
1168 Warning! Current implementation uses quite naive, brute force algorithm,
1169 which can make allocation calls that use #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
1170 flag quite slow. A new, more optimal algorithm and data structure to speed this
1171 up is planned for the future.
1172
1173 <b>Q: When interleaving creation of new allocations with usage of existing ones,
1174 how do you make sure that an allocation won't become lost while it's used in the
1175 current frame?</b>
1176
1177 It is ensured because vmaTouchAllocation() / vmaGetAllocationInfo() not only returns allocation
1178 status/parameters and checks whether it's not lost, but when it's not, it also
1179 atomically marks it as used in the current frame, which makes it impossible to
1180 become lost in that frame. It uses lockless algorithm, so it works fast and
1181 doesn't involve locking any internal mutex.
1182
1183 <b>Q: What if my allocation may still be in use by the GPU when it's rendering a
1184 previous frame while I already submit new frame on the CPU?</b>
1185
1186 You can make sure that allocations "touched" by vmaTouchAllocation() / vmaGetAllocationInfo() will not
1187 become lost for a number of additional frames back from the current one by
1188 specifying this number as VmaAllocatorCreateInfo::frameInUseCount (for default
1189 memory pool) and VmaPoolCreateInfo::frameInUseCount (for custom pool).
1190
1191 <b>Q: How do you inform the library when new frame starts?</b>
1192
1193 You need to call function vmaSetCurrentFrameIndex().
1194
1195 Example code:
1196
1197 \code
1198 struct MyBuffer
1199 {
1200 VkBuffer m_Buf = nullptr;
1201 VmaAllocation m_Alloc = nullptr;
1202
1203 // Called when the buffer is really needed in the current frame.
1204 void EnsureBuffer();
1205 };
1206
1207 void MyBuffer::EnsureBuffer()
1208 {
1209 // Buffer has been created.
1210 if(m_Buf != VK_NULL_HANDLE)
1211 {
1212 // Check if its allocation is not lost + mark it as used in current frame.
1213 if(vmaTouchAllocation(allocator, m_Alloc))
1214 {
1215 // It's all OK - safe to use m_Buf.
1216 return;
1217 }
1218 }
1219
1220 // Buffer not yet exists or lost - destroy and recreate it.
1221
1222 vmaDestroyBuffer(allocator, m_Buf, m_Alloc);
1223
1224 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
1225 bufCreateInfo.size = 1024;
1226 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
1227
1228 VmaAllocationCreateInfo allocCreateInfo = {};
1229 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
1230 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT |
1231 VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT;
1232
1233 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &m_Buf, &m_Alloc, nullptr);
1234 }
1235 \endcode
1236
1237 When using lost allocations, you may see some Vulkan validation layer warnings
1238 about overlapping regions of memory bound to different kinds of buffers and
1239 images. This is still valid as long as you implement proper handling of lost
1240 allocations (like in the example above) and don't use them.
1241
1242 You can create an allocation that is already in lost state from the beginning using function
1243 vmaCreateLostAllocation(). It may be useful if you need a "dummy" allocation that is not null.
1244
1245 You can call function vmaMakePoolAllocationsLost() to set all eligible allocations
1246 in a specified custom pool to lost state.
1247 Allocations that have been "touched" in current frame or VmaPoolCreateInfo::frameInUseCount frames back
1248 cannot become lost.
1249
1250 <b>Q: Can I touch allocation that cannot become lost?</b>
1251
1252 Yes, although it has no visible effect.
1253 Calls to vmaGetAllocationInfo() and vmaTouchAllocation() update last use frame index
1254 also for allocations that cannot become lost, but the only way to observe it is to dump
1255 internal allocator state using vmaBuildStatsString().
1256 You can use this feature for debugging purposes to explicitly mark allocations that you use
1257 in current frame and then analyze JSON dump to see for how long each allocation stays unused.
1258
1259
1260 \page statistics Statistics
1261
1262 This library contains functions that return information about its internal state,
1263 especially the amount of memory allocated from Vulkan.
1264 Please keep in mind that these functions need to traverse all internal data structures
1265 to gather these information, so they may be quite time-consuming.
1266 Don't call them too often.
1267
1268 \section statistics_numeric_statistics Numeric statistics
1269
1270 You can query for overall statistics of the allocator using function vmaCalculateStats().
1271 Information are returned using structure #VmaStats.
1272 It contains #VmaStatInfo - number of allocated blocks, number of allocations
1273 (occupied ranges in these blocks), number of unused (free) ranges in these blocks,
1274 number of bytes used and unused (but still allocated from Vulkan) and other information.
1275 They are summed across memory heaps, memory types and total for whole allocator.
1276
1277 You can query for statistics of a custom pool using function vmaGetPoolStats().
1278 Information are returned using structure #VmaPoolStats.
1279
1280 You can query for information about specific allocation using function vmaGetAllocationInfo().
1281 It fill structure #VmaAllocationInfo.
1282
1283 \section statistics_json_dump JSON dump
1284
1285 You can dump internal state of the allocator to a string in JSON format using function vmaBuildStatsString().
1286 The result is guaranteed to be correct JSON.
1287 It uses ANSI encoding.
1288 Any strings provided by user (see [Allocation names](@ref allocation_names))
1289 are copied as-is and properly escaped for JSON, so if they use UTF-8, ISO-8859-2 or any other encoding,
1290 this JSON string can be treated as using this encoding.
1291 It must be freed using function vmaFreeStatsString().
1292
1293 The format of this JSON string is not part of official documentation of the library,
1294 but it will not change in backward-incompatible way without increasing library major version number
1295 and appropriate mention in changelog.
1296
1297 The JSON string contains all the data that can be obtained using vmaCalculateStats().
1298 It can also contain detailed map of allocated memory blocks and their regions -
1299 free and occupied by allocations.
1300 This allows e.g. to visualize the memory or assess fragmentation.
1301
1302
1303 \page allocation_annotation Allocation names and user data
1304
1305 \section allocation_user_data Allocation user data
1306
1307 You can annotate allocations with your own information, e.g. for debugging purposes.
1308 To do that, fill VmaAllocationCreateInfo::pUserData field when creating
1309 an allocation. It's an opaque `void*` pointer. You can use it e.g. as a pointer,
1310 some handle, index, key, ordinal number or any other value that would associate
1311 the allocation with your custom metadata.
1312
1313 \code
1314 VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
1315 // Fill bufferInfo...
1316
1317 MyBufferMetadata* pMetadata = CreateBufferMetadata();
1318
1319 VmaAllocationCreateInfo allocCreateInfo = {};
1320 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
1321 allocCreateInfo.pUserData = pMetadata;
1322
1323 VkBuffer buffer;
1324 VmaAllocation allocation;
1325 vmaCreateBuffer(allocator, &bufferInfo, &allocCreateInfo, &buffer, &allocation, nullptr);
1326 \endcode
1327
1328 The pointer may be later retrieved as VmaAllocationInfo::pUserData:
1329
1330 \code
1331 VmaAllocationInfo allocInfo;
1332 vmaGetAllocationInfo(allocator, allocation, &allocInfo);
1333 MyBufferMetadata* pMetadata = (MyBufferMetadata*)allocInfo.pUserData;
1334 \endcode
1335
1336 It can also be changed using function vmaSetAllocationUserData().
1337
1338 Values of (non-zero) allocations' `pUserData` are printed in JSON report created by
1339 vmaBuildStatsString(), in hexadecimal form.
1340
1341 \section allocation_names Allocation names
1342
1343 There is alternative mode available where `pUserData` pointer is used to point to
1344 a null-terminated string, giving a name to the allocation. To use this mode,
1345 set #VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT flag in VmaAllocationCreateInfo::flags.
1346 Then `pUserData` passed as VmaAllocationCreateInfo::pUserData or argument to
1347 vmaSetAllocationUserData() must be either null or pointer to a null-terminated string.
1348 The library creates internal copy of the string, so the pointer you pass doesn't need
1349 to be valid for whole lifetime of the allocation. You can free it after the call.
1350
1351 \code
1352 VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
1353 // Fill imageInfo...
1354
1355 std::string imageName = "Texture: ";
1356 imageName += fileName;
1357
1358 VmaAllocationCreateInfo allocCreateInfo = {};
1359 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
1360 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT;
1361 allocCreateInfo.pUserData = imageName.c_str();
1362
1363 VkImage image;
1364 VmaAllocation allocation;
1365 vmaCreateImage(allocator, &imageInfo, &allocCreateInfo, &image, &allocation, nullptr);
1366 \endcode
1367
1368 The value of `pUserData` pointer of the allocation will be different than the one
1369 you passed when setting allocation's name - pointing to a buffer managed
1370 internally that holds copy of the string.
1371
1372 \code
1373 VmaAllocationInfo allocInfo;
1374 vmaGetAllocationInfo(allocator, allocation, &allocInfo);
1375 const char* imageName = (const char*)allocInfo.pUserData;
1376 printf("Image name: %s\n", imageName);
1377 \endcode
1378
1379 That string is also printed in JSON report created by vmaBuildStatsString().
1380
1381 \note Passing string name to VMA allocation doesn't automatically set it to the Vulkan buffer or image created with it.
1382 You must do it manually using an extension like VK_EXT_debug_utils, which is independent of this library.
1383
1384
1385 \page debugging_memory_usage Debugging incorrect memory usage
1386
1387 If you suspect a bug with memory usage, like usage of uninitialized memory or
1388 memory being overwritten out of bounds of an allocation,
1389 you can use debug features of this library to verify this.
1390
1391 \section debugging_memory_usage_initialization Memory initialization
1392
1393 If you experience a bug with incorrect and nondeterministic data in your program and you suspect uninitialized memory to be used,
1394 you can enable automatic memory initialization to verify this.
1395 To do it, define macro `VMA_DEBUG_INITIALIZE_ALLOCATIONS` to 1.
1396
1397 \code
1398 #define VMA_DEBUG_INITIALIZE_ALLOCATIONS 1
1399 #include "vk_mem_alloc.h"
1400 \endcode
1401
1402 It makes memory of all new allocations initialized to bit pattern `0xDCDCDCDC`.
1403 Before an allocation is destroyed, its memory is filled with bit pattern `0xEFEFEFEF`.
1404 Memory is automatically mapped and unmapped if necessary.
1405
1406 If you find these values while debugging your program, good chances are that you incorrectly
1407 read Vulkan memory that is allocated but not initialized, or already freed, respectively.
1408
1409 Memory initialization works only with memory types that are `HOST_VISIBLE`.
1410 It works also with dedicated allocations.
1411 It doesn't work with allocations created with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag,
1412 as they cannot be mapped.
1413
1414 \section debugging_memory_usage_margins Margins
1415
1416 By default, allocations are laid out in memory blocks next to each other if possible
1417 (considering required alignment, `bufferImageGranularity`, and `nonCoherentAtomSize`).
1418
1419 ![Allocations without margin](../gfx/Margins_1.png)
1420
1421 Define macro `VMA_DEBUG_MARGIN` to some non-zero value (e.g. 16) to enforce specified
1422 number of bytes as a margin before and after every allocation.
1423
1424 \code
1425 #define VMA_DEBUG_MARGIN 16
1426 #include "vk_mem_alloc.h"
1427 \endcode
1428
1429 ![Allocations with margin](../gfx/Margins_2.png)
1430
1431 If your bug goes away after enabling margins, it means it may be caused by memory
1432 being overwritten outside of allocation boundaries. It is not 100% certain though.
1433 Change in application behavior may also be caused by different order and distribution
1434 of allocations across memory blocks after margins are applied.
1435
1436 The margin is applied also before first and after last allocation in a block.
1437 It may occur only once between two adjacent allocations.
1438
1439 Margins work with all types of memory.
1440
1441 Margin is applied only to allocations made out of memory blocks and not to dedicated
1442 allocations, which have their own memory block of specific size.
1443 It is thus not applied to allocations made using #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT flag
1444 or those automatically decided to put into dedicated allocations, e.g. due to its
1445 large size or recommended by VK_KHR_dedicated_allocation extension.
1446 Margins are also not active in custom pools created with #VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT flag.
1447
1448 Margins appear in [JSON dump](@ref statistics_json_dump) as part of free space.
1449
1450 Note that enabling margins increases memory usage and fragmentation.
1451
1452 \section debugging_memory_usage_corruption_detection Corruption detection
1453
1454 You can additionally define macro `VMA_DEBUG_DETECT_CORRUPTION` to 1 to enable validation
1455 of contents of the margins.
1456
1457 \code
1458 #define VMA_DEBUG_MARGIN 16
1459 #define VMA_DEBUG_DETECT_CORRUPTION 1
1460 #include "vk_mem_alloc.h"
1461 \endcode
1462
1463 When this feature is enabled, number of bytes specified as `VMA_DEBUG_MARGIN`
1464 (it must be multiply of 4) before and after every allocation is filled with a magic number.
1465 This idea is also know as "canary".
1466 Memory is automatically mapped and unmapped if necessary.
1467
1468 This number is validated automatically when the allocation is destroyed.
1469 If it's not equal to the expected value, `VMA_ASSERT()` is executed.
1470 It clearly means that either CPU or GPU overwritten the memory outside of boundaries of the allocation,
1471 which indicates a serious bug.
1472
1473 You can also explicitly request checking margins of all allocations in all memory blocks
1474 that belong to specified memory types by using function vmaCheckCorruption(),
1475 or in memory blocks that belong to specified custom pool, by using function
1476 vmaCheckPoolCorruption().
1477
1478 Margin validation (corruption detection) works only for memory types that are
1479 `HOST_VISIBLE` and `HOST_COHERENT`.
1480
1481
1482 \page record_and_replay Record and replay
1483
1484 \section record_and_replay_introduction Introduction
1485
1486 While using the library, sequence of calls to its functions together with their
1487 parameters can be recorded to a file and later replayed using standalone player
1488 application. It can be useful to:
1489
1490 - Test correctness - check if same sequence of calls will not cause crash or
1491 failures on a target platform.
1492 - Gather statistics - see number of allocations, peak memory usage, number of
1493 calls etc.
1494 - Benchmark performance - see how much time it takes to replay the whole
1495 sequence.
1496
1497 \section record_and_replay_usage Usage
1498
1499 Recording functionality is disabled by default.
1500 To enable it, define following macro before every include of this library:
1501
1502 \code
1503 #define VMA_RECORDING_ENABLED 1
1504 \endcode
1505
1506 <b>To record sequence of calls to a file:</b> Fill in
1507 VmaAllocatorCreateInfo::pRecordSettings member while creating #VmaAllocator
1508 object. File is opened and written during whole lifetime of the allocator.
1509
1510 <b>To replay file:</b> Use VmaReplay - standalone command-line program.
1511 Precompiled binary can be found in "bin" directory.
1512 Its source can be found in "src/VmaReplay" directory.
1513 Its project is generated by Premake.
1514 Command line syntax is printed when the program is launched without parameters.
1515 Basic usage:
1516
1517 VmaReplay.exe MyRecording.csv
1518
1519 <b>Documentation of file format</b> can be found in file: "docs/Recording file format.md".
1520 It's a human-readable, text file in CSV format (Comma Separated Values).
1521
1522 \section record_and_replay_additional_considerations Additional considerations
1523
1524 - Replaying file that was recorded on a different GPU (with different parameters
1525 like `bufferImageGranularity`, `nonCoherentAtomSize`, and especially different
1526 set of memory heaps and types) may give different performance and memory usage
1527 results, as well as issue some warnings and errors.
1528 - Current implementation of recording in VMA, as well as VmaReplay application, is
1529 coded and tested only on Windows. Inclusion of recording code is driven by
1530 `VMA_RECORDING_ENABLED` macro. Support for other platforms should be easy to
1531 add. Contributions are welcomed.
1532
1533
1534 \page usage_patterns Recommended usage patterns
1535
1536 See also slides from talk:
1537 [Sawicki, Adam. Advanced Graphics Techniques Tutorial: Memory management in Vulkan and DX12. Game Developers Conference, 2018](https://www.gdcvault.com/play/1025458/Advanced-Graphics-Techniques-Tutorial-New)
1538
1539
1540 \section usage_patterns_common_mistakes Common mistakes
1541
1542 <b>Use of CPU_TO_GPU instead of CPU_ONLY memory</b>
1543
1544 #VMA_MEMORY_USAGE_CPU_TO_GPU is recommended only for resources that will be
1545 mapped and written by the CPU, as well as read directly by the GPU - like some
1546 buffers or textures updated every frame (dynamic). If you create a staging copy
1547 of a resource to be written by CPU and then used as a source of transfer to
1548 another resource placed in the GPU memory, that staging resource should be
1549 created with #VMA_MEMORY_USAGE_CPU_ONLY. Please read the descriptions of these
1550 enums carefully for details.
1551
1552 <b>Unnecessary use of custom pools</b>
1553
1554 \ref custom_memory_pools may be useful for special purposes - when you want to
1555 keep certain type of resources separate e.g. to reserve minimum amount of memory
1556 for them, limit maximum amount of memory they can occupy, or make some of them
1557 push out the other through the mechanism of \ref lost_allocations. For most
1558 resources this is not needed and so it is not recommended to create #VmaPool
1559 objects and allocations out of them. Allocating from the default pool is sufficient.
1560
1561 \section usage_patterns_simple Simple patterns
1562
1563 \subsection usage_patterns_simple_render_targets Render targets
1564
1565 <b>When:</b>
1566 Any resources that you frequently write and read on GPU,
1567 e.g. images used as color attachments (aka "render targets"), depth-stencil attachments,
1568 images/buffers used as storage image/buffer (aka "Unordered Access View (UAV)").
1569
1570 <b>What to do:</b>
1571 Create them in video memory that is fastest to access from GPU using
1572 #VMA_MEMORY_USAGE_GPU_ONLY.
1573
1574 Consider using [VK_KHR_dedicated_allocation](@ref vk_khr_dedicated_allocation) extension
1575 and/or manually creating them as dedicated allocations using #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT,
1576 especially if they are large or if you plan to destroy and recreate them e.g. when
1577 display resolution changes.
1578 Prefer to create such resources first and all other GPU resources (like textures and vertex buffers) later.
1579
1580 \subsection usage_patterns_simple_immutable_resources Immutable resources
1581
1582 <b>When:</b>
1583 Any resources that you fill on CPU only once (aka "immutable") or infrequently
1584 and then read frequently on GPU,
1585 e.g. textures, vertex and index buffers, constant buffers that don't change often.
1586
1587 <b>What to do:</b>
1588 Create them in video memory that is fastest to access from GPU using
1589 #VMA_MEMORY_USAGE_GPU_ONLY.
1590
1591 To initialize content of such resource, create a CPU-side (aka "staging") copy of it
1592 in system memory - #VMA_MEMORY_USAGE_CPU_ONLY, map it, fill it,
1593 and submit a transfer from it to the GPU resource.
1594 You can keep the staging copy if you need it for another upload transfer in the future.
1595 If you don't, you can destroy it or reuse this buffer for uploading different resource
1596 after the transfer finishes.
1597
1598 Prefer to create just buffers in system memory rather than images, even for uploading textures.
1599 Use `vkCmdCopyBufferToImage()`.
1600 Dont use images with `VK_IMAGE_TILING_LINEAR`.
1601
1602 \subsection usage_patterns_dynamic_resources Dynamic resources
1603
1604 <b>When:</b>
1605 Any resources that change frequently (aka "dynamic"), e.g. every frame or every draw call,
1606 written on CPU, read on GPU.
1607
1608 <b>What to do:</b>
1609 Create them using #VMA_MEMORY_USAGE_CPU_TO_GPU.
1610 You can map it and write to it directly on CPU, as well as read from it on GPU.
1611
1612 This is a more complex situation. Different solutions are possible,
1613 and the best one depends on specific GPU type, but you can use this simple approach for the start.
1614 Prefer to write to such resource sequentially (e.g. using `memcpy`).
1615 Don't perform random access or any reads from it on CPU, as it may be very slow.
1616 Also note that textures written directly from the host through a mapped pointer need to be in LINEAR not OPTIMAL layout.
1617
1618 \subsection usage_patterns_readback Readback
1619
1620 <b>When:</b>
1621 Resources that contain data written by GPU that you want to read back on CPU,
1622 e.g. results of some computations.
1623
1624 <b>What to do:</b>
1625 Create them using #VMA_MEMORY_USAGE_GPU_TO_CPU.
1626 You can write to them directly on GPU, as well as map and read them on CPU.
1627
1628 \section usage_patterns_advanced Advanced patterns
1629
1630 \subsection usage_patterns_integrated_graphics Detecting integrated graphics
1631
1632 You can support integrated graphics (like Intel HD Graphics, AMD APU) better
1633 by detecting it in Vulkan.
1634 To do it, call `vkGetPhysicalDeviceProperties()`, inspect
1635 `VkPhysicalDeviceProperties::deviceType` and look for `VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`.
1636 When you find it, you can assume that memory is unified and all memory types are comparably fast
1637 to access from GPU, regardless of `VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT`.
1638
1639 You can then sum up sizes of all available memory heaps and treat them as useful for
1640 your GPU resources, instead of only `DEVICE_LOCAL` ones.
1641 You can also prefer to create your resources in memory types that are `HOST_VISIBLE` to map them
1642 directly instead of submitting explicit transfer (see below).
1643
1644 \subsection usage_patterns_direct_vs_transfer Direct access versus transfer
1645
1646 For resources that you frequently write on CPU and read on GPU, many solutions are possible:
1647
1648 -# Create one copy in video memory using #VMA_MEMORY_USAGE_GPU_ONLY,
1649 second copy in system memory using #VMA_MEMORY_USAGE_CPU_ONLY and submit explicit transfer each time.
1650 -# Create just a single copy using #VMA_MEMORY_USAGE_CPU_TO_GPU, map it and fill it on CPU,
1651 read it directly on GPU.
1652 -# Create just a single copy using #VMA_MEMORY_USAGE_CPU_ONLY, map it and fill it on CPU,
1653 read it directly on GPU.
1654
1655 Which solution is the most efficient depends on your resource and especially on the GPU.
1656 It is best to measure it and then make the decision.
1657 Some general recommendations:
1658
1659 - On integrated graphics use (2) or (3) to avoid unnecesary time and memory overhead
1660 related to using a second copy and making transfer.
1661 - For small resources (e.g. constant buffers) use (2).
1662 Discrete AMD cards have special 256 MiB pool of video memory that is directly mappable.
1663 Even if the resource ends up in system memory, its data may be cached on GPU after first
1664 fetch over PCIe bus.
1665 - For larger resources (e.g. textures), decide between (1) and (2).
1666 You may want to differentiate NVIDIA and AMD, e.g. by looking for memory type that is
1667 both `DEVICE_LOCAL` and `HOST_VISIBLE`. When you find it, use (2), otherwise use (1).
1668
1669 Similarly, for resources that you frequently write on GPU and read on CPU, multiple
1670 solutions are possible:
1671
1672 -# Create one copy in video memory using #VMA_MEMORY_USAGE_GPU_ONLY,
1673 second copy in system memory using #VMA_MEMORY_USAGE_GPU_TO_CPU and submit explicit tranfer each time.
1674 -# Create just single copy using #VMA_MEMORY_USAGE_GPU_TO_CPU, write to it directly on GPU,
1675 map it and read it on CPU.
1676
1677 You should take some measurements to decide which option is faster in case of your specific
1678 resource.
1679
1680 Note that textures accessed directly from the host through a mapped pointer need to be in LINEAR layout,
1681 which may slow down their usage on the device.
1682 Textures accessed only by the device and transfer operations can use OPTIMAL layout.
1683
1684 If you don't want to specialize your code for specific types of GPUs, you can still make
1685 an simple optimization for cases when your resource ends up in mappable memory to use it
1686 directly in this case instead of creating CPU-side staging copy.
1687 For details see [Finding out if memory is mappable](@ref memory_mapping_finding_if_memory_mappable).
1688
1689
1690 \page configuration Configuration
1691
1692 Please check "CONFIGURATION SECTION" in the code to find macros that you can define
1693 before each include of this file or change directly in this file to provide
1694 your own implementation of basic facilities like assert, `min()` and `max()` functions,
1695 mutex, atomic etc.
1696 The library uses its own implementation of containers by default, but you can switch to using
1697 STL containers instead.
1698
1699 For example, define `VMA_ASSERT(expr)` before including the library to provide
1700 custom implementation of the assertion, compatible with your project.
1701 By default it is defined to standard C `assert(expr)` in `_DEBUG` configuration
1702 and empty otherwise.
1703
1704 \section config_Vulkan_functions Pointers to Vulkan functions
1705
1706 There are multiple ways to import pointers to Vulkan functions in the library.
1707 In the simplest case you don't need to do anything.
1708 If the compilation or linking of your program or the initialization of the #VmaAllocator
1709 doesn't work for you, you can try to reconfigure it.
1710
1711 First, the allocator tries to fetch pointers to Vulkan functions linked statically,
1712 like this:
1713
1714 \code
1715 m_VulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkAllocateMemory;
1716 \endcode
1717
1718 If you want to disable this feature, set configuration macro: `#define VMA_STATIC_VULKAN_FUNCTIONS 0`.
1719
1720 Second, you can provide the pointers yourself by setting member VmaAllocatorCreateInfo::pVulkanFunctions.
1721 You can fetch them e.g. using functions `vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` or
1722 by using a helper library like [volk](https://github.com/zeux/volk).
1723
1724 Third, VMA tries to fetch remaining pointers that are still null by calling
1725 `vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` on its own.
1726 If you want to disable this feature, set configuration macro: `#define VMA_DYNAMIC_VULKAN_FUNCTIONS 0`.
1727
1728 Finally, all the function pointers required by the library (considering selected
1729 Vulkan version and enabled extensions) are checked with `VMA_ASSERT` if they are not null.
1730
1731
1732 \section custom_memory_allocator Custom host memory allocator
1733
1734 If you use custom allocator for CPU memory rather than default operator `new`
1735 and `delete` from C++, you can make this library using your allocator as well
1736 by filling optional member VmaAllocatorCreateInfo::pAllocationCallbacks. These
1737 functions will be passed to Vulkan, as well as used by the library itself to
1738 make any CPU-side allocations.
1739
1740 \section allocation_callbacks Device memory allocation callbacks
1741
1742 The library makes calls to `vkAllocateMemory()` and `vkFreeMemory()` internally.
1743 You can setup callbacks to be informed about these calls, e.g. for the purpose
1744 of gathering some statistics. To do it, fill optional member
1745 VmaAllocatorCreateInfo::pDeviceMemoryCallbacks.
1746
1747 \section heap_memory_limit Device heap memory limit
1748
1749 When device memory of certain heap runs out of free space, new allocations may
1750 fail (returning error code) or they may succeed, silently pushing some existing
1751 memory blocks from GPU VRAM to system RAM (which degrades performance). This
1752 behavior is implementation-dependant - it depends on GPU vendor and graphics
1753 driver.
1754
1755 On AMD cards it can be controlled while creating Vulkan device object by using
1756 VK_AMD_memory_overallocation_behavior extension, if available.
1757
1758 Alternatively, if you want to test how your program behaves with limited amount of Vulkan device
1759 memory available without switching your graphics card to one that really has
1760 smaller VRAM, you can use a feature of this library intended for this purpose.
1761 To do it, fill optional member VmaAllocatorCreateInfo::pHeapSizeLimit.
1762
1763
1764
1765 \page vk_khr_dedicated_allocation VK_KHR_dedicated_allocation
1766
1767 VK_KHR_dedicated_allocation is a Vulkan extension which can be used to improve
1768 performance on some GPUs. It augments Vulkan API with possibility to query
1769 driver whether it prefers particular buffer or image to have its own, dedicated
1770 allocation (separate `VkDeviceMemory` block) for better efficiency - to be able
1771 to do some internal optimizations.
1772
1773 The extension is supported by this library. It will be used automatically when
1774 enabled. To enable it:
1775
1776 1 . When creating Vulkan device, check if following 2 device extensions are
1777 supported (call `vkEnumerateDeviceExtensionProperties()`).
1778 If yes, enable them (fill `VkDeviceCreateInfo::ppEnabledExtensionNames`).
1779
1780 - VK_KHR_get_memory_requirements2
1781 - VK_KHR_dedicated_allocation
1782
1783 If you enabled these extensions:
1784
1785 2 . Use #VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT flag when creating
1786 your #VmaAllocator`to inform the library that you enabled required extensions
1787 and you want the library to use them.
1788
1789 \code
1790 allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT;
1791
1792 vmaCreateAllocator(&allocatorInfo, &allocator);
1793 \endcode
1794
1795 That's all. The extension will be automatically used whenever you create a
1796 buffer using vmaCreateBuffer() or image using vmaCreateImage().
1797
1798 When using the extension together with Vulkan Validation Layer, you will receive
1799 warnings like this:
1800
1801 vkBindBufferMemory(): Binding memory to buffer 0x33 but vkGetBufferMemoryRequirements() has not been called on that buffer.
1802
1803 It is OK, you should just ignore it. It happens because you use function
1804 `vkGetBufferMemoryRequirements2KHR()` instead of standard
1805 `vkGetBufferMemoryRequirements()`, while the validation layer seems to be
1806 unaware of it.
1807
1808 To learn more about this extension, see:
1809
1810 - [VK_KHR_dedicated_allocation in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap44.html#VK_KHR_dedicated_allocation)
1811 - [VK_KHR_dedicated_allocation unofficial manual](http://asawicki.info/articles/VK_KHR_dedicated_allocation.php5)
1812
1813
1814
1815 \page vk_amd_device_coherent_memory VK_AMD_device_coherent_memory
1816
1817 VK_AMD_device_coherent_memory is a device extension that enables access to
1818 additional memory types with `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` and
1819 `VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD` flag. It is useful mostly for
1820 allocation of buffers intended for writing "breadcrumb markers" in between passes
1821 or draw calls, which in turn are useful for debugging GPU crash/hang/TDR cases.
1822
1823 When the extension is available but has not been enabled, Vulkan physical device
1824 still exposes those memory types, but their usage is forbidden. VMA automatically
1825 takes care of that - it returns `VK_ERROR_FEATURE_NOT_PRESENT` when an attempt
1826 to allocate memory of such type is made.
1827
1828 If you want to use this extension in connection with VMA, follow these steps:
1829
1830 \section vk_amd_device_coherent_memory_initialization Initialization
1831
1832 1) Call `vkEnumerateDeviceExtensionProperties` for the physical device.
1833 Check if the extension is supported - if returned array of `VkExtensionProperties` contains "VK_AMD_device_coherent_memory".
1834
1835 2) Call `vkGetPhysicalDeviceFeatures2` for the physical device instead of old `vkGetPhysicalDeviceFeatures`.
1836 Attach additional structure `VkPhysicalDeviceCoherentMemoryFeaturesAMD` to `VkPhysicalDeviceFeatures2::pNext` to be returned.
1837 Check if the device feature is really supported - check if `VkPhysicalDeviceCoherentMemoryFeaturesAMD::deviceCoherentMemory` is true.
1838
1839 3) While creating device with `vkCreateDevice`, enable this extension - add "VK_AMD_device_coherent_memory"
1840 to the list passed as `VkDeviceCreateInfo::ppEnabledExtensionNames`.
1841
1842 4) While creating the device, also don't set `VkDeviceCreateInfo::pEnabledFeatures`.
1843 Fill in `VkPhysicalDeviceFeatures2` structure instead and pass it as `VkDeviceCreateInfo::pNext`.
1844 Enable this device feature - attach additional structure `VkPhysicalDeviceCoherentMemoryFeaturesAMD` to
1845 `VkPhysicalDeviceFeatures2::pNext` and set its member `deviceCoherentMemory` to `VK_TRUE`.
1846
1847 5) While creating #VmaAllocator with vmaCreateAllocator() inform VMA that you
1848 have enabled this extension and feature - add #VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT
1849 to VmaAllocatorCreateInfo::flags.
1850
1851 \section vk_amd_device_coherent_memory_usage Usage
1852
1853 After following steps described above, you can create VMA allocations and custom pools
1854 out of the special `DEVICE_COHERENT` and `DEVICE_UNCACHED` memory types on eligible
1855 devices. There are multiple ways to do it, for example:
1856
1857 - You can request or prefer to allocate out of such memory types by adding
1858 `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` to VmaAllocationCreateInfo::requiredFlags
1859 or VmaAllocationCreateInfo::preferredFlags. Those flags can be freely mixed with
1860 other ways of \ref choosing_memory_type, like setting VmaAllocationCreateInfo::usage.
1861 - If you manually found memory type index to use for this purpose, force allocation
1862 from this specific index by setting VmaAllocationCreateInfo::memoryTypeBits `= 1u << index`.
1863
1864 \section vk_amd_device_coherent_memory_more_information More information
1865
1866 To learn more about this extension, see [VK_AMD_device_coherent_memory in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap44.html#VK_AMD_device_coherent_memory)
1867
1868 Example use of this extension can be found in the code of the sample and test suite
1869 accompanying this library.
1870
1871
1872 \page enabling_buffer_device_address Enabling buffer device address
1873
1874 Device extension VK_KHR_buffer_device_address
1875 allow to fetch raw GPU pointer to a buffer and pass it for usage in a shader code.
1876 It is promoted to core Vulkan 1.2.
1877
1878 If you want to use this feature in connection with VMA, follow these steps:
1879
1880 \section enabling_buffer_device_address_initialization Initialization
1881
1882 1) (For Vulkan version < 1.2) Call `vkEnumerateDeviceExtensionProperties` for the physical device.
1883 Check if the extension is supported - if returned array of `VkExtensionProperties` contains
1884 "VK_KHR_buffer_device_address".
1885
1886 2) Call `vkGetPhysicalDeviceFeatures2` for the physical device instead of old `vkGetPhysicalDeviceFeatures`.
1887 Attach additional structure `VkPhysicalDeviceBufferDeviceAddressFeatures*` to `VkPhysicalDeviceFeatures2::pNext` to be returned.
1888 Check if the device feature is really supported - check if `VkPhysicalDeviceBufferDeviceAddressFeatures*::bufferDeviceAddress` is true.
1889
1890 3) (For Vulkan version < 1.2) While creating device with `vkCreateDevice`, enable this extension - add
1891 "VK_KHR_buffer_device_address" to the list passed as `VkDeviceCreateInfo::ppEnabledExtensionNames`.
1892
1893 4) While creating the device, also don't set `VkDeviceCreateInfo::pEnabledFeatures`.
1894 Fill in `VkPhysicalDeviceFeatures2` structure instead and pass it as `VkDeviceCreateInfo::pNext`.
1895 Enable this device feature - attach additional structure `VkPhysicalDeviceBufferDeviceAddressFeatures*` to
1896 `VkPhysicalDeviceFeatures2::pNext` and set its member `bufferDeviceAddress` to `VK_TRUE`.
1897
1898 5) While creating #VmaAllocator with vmaCreateAllocator() inform VMA that you
1899 have enabled this feature - add #VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT
1900 to VmaAllocatorCreateInfo::flags.
1901
1902 \section enabling_buffer_device_address_usage Usage
1903
1904 After following steps described above, you can create buffers with `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*` using VMA.
1905 The library automatically adds `VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT*` to
1906 allocated memory blocks wherever it might be needed.
1907
1908 Please note that the library supports only `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*`.
1909 The second part of this functionality related to "capture and replay" is not supported,
1910 as it is intended for usage in debugging tools like RenderDoc, not in everyday Vulkan usage.
1911
1912 \section enabling_buffer_device_address_more_information More information
1913
1914 To learn more about this extension, see [VK_KHR_buffer_device_address in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap46.html#VK_KHR_buffer_device_address)
1915
1916 Example use of this extension can be found in the code of the sample and test suite
1917 accompanying this library.
1918
1919 \page general_considerations General considerations
1920
1921 \section general_considerations_thread_safety Thread safety
1922
1923 - The library has no global state, so separate #VmaAllocator objects can be used
1924 independently.
1925 There should be no need to create multiple such objects though - one per `VkDevice` is enough.
1926 - By default, all calls to functions that take #VmaAllocator as first parameter
1927 are safe to call from multiple threads simultaneously because they are
1928 synchronized internally when needed.
1929 - When the allocator is created with #VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
1930 flag, calls to functions that take such #VmaAllocator object must be
1931 synchronized externally.
1932 - Access to a #VmaAllocation object must be externally synchronized. For example,
1933 you must not call vmaGetAllocationInfo() and vmaMapMemory() from different
1934 threads at the same time if you pass the same #VmaAllocation object to these
1935 functions.
1936
1937 \section general_considerations_validation_layer_warnings Validation layer warnings
1938
1939 When using this library, you can meet following types of warnings issued by
1940 Vulkan validation layer. They don't necessarily indicate a bug, so you may need
1941 to just ignore them.
1942
1943 - *vkBindBufferMemory(): Binding memory to buffer 0xeb8e4 but vkGetBufferMemoryRequirements() has not been called on that buffer.*
1944 - It happens when VK_KHR_dedicated_allocation extension is enabled.
1945 `vkGetBufferMemoryRequirements2KHR` function is used instead, while validation layer seems to be unaware of it.
1946 - *Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used.*
1947 - It happens when you map a buffer or image, because the library maps entire
1948 `VkDeviceMemory` block, where different types of images and buffers may end
1949 up together, especially on GPUs with unified memory like Intel.
1950 - *Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug.*
1951 - It happens when you use lost allocations, and a new image or buffer is
1952 created in place of an existing object that bacame lost.
1953 - It may happen also when you use [defragmentation](@ref defragmentation).
1954
1955 \section general_considerations_allocation_algorithm Allocation algorithm
1956
1957 The library uses following algorithm for allocation, in order:
1958
1959 -# Try to find free range of memory in existing blocks.
1960 -# If failed, try to create a new block of `VkDeviceMemory`, with preferred block size.
1961 -# If failed, try to create such block with size/2, size/4, size/8.
1962 -# If failed and #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag was
1963 specified, try to find space in existing blocks, possilby making some other
1964 allocations lost.
1965 -# If failed, try to allocate separate `VkDeviceMemory` for this allocation,
1966 just like when you use #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT.
1967 -# If failed, choose other memory type that meets the requirements specified in
1968 VmaAllocationCreateInfo and go to point 1.
1969 -# If failed, return `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
1970
1971 \section general_considerations_features_not_supported Features not supported
1972
1973 Features deliberately excluded from the scope of this library:
1974
1975 - Data transfer. Uploading (straming) and downloading data of buffers and images
1976 between CPU and GPU memory and related synchronization is responsibility of the user.
1977 Defining some "texture" object that would automatically stream its data from a
1978 staging copy in CPU memory to GPU memory would rather be a feature of another,
1979 higher-level library implemented on top of VMA.
1980 - Allocations for imported/exported external memory. They tend to require
1981 explicit memory type index and dedicated allocation anyway, so they don't
1982 interact with main features of this library. Such special purpose allocations
1983 should be made manually, using `vkCreateBuffer()` and `vkAllocateMemory()`.
1984 - Recreation of buffers and images. Although the library has functions for
1985 buffer and image creation (vmaCreateBuffer(), vmaCreateImage()), you need to
1986 recreate these objects yourself after defragmentation. That's because the big
1987 structures `VkBufferCreateInfo`, `VkImageCreateInfo` are not stored in
1988 #VmaAllocation object.
1989 - Handling CPU memory allocation failures. When dynamically creating small C++
1990 objects in CPU memory (not Vulkan memory), allocation failures are not checked
1991 and handled gracefully, because that would complicate code significantly and
1992 is usually not needed in desktop PC applications anyway.
1993 Success of an allocation is just checked with an assert.
1994 - Code free of any compiler warnings. Maintaining the library to compile and
1995 work correctly on so many different platforms is hard enough. Being free of
1996 any warnings, on any version of any compiler, is simply not feasible.
1997 - This is a C++ library with C interface.
1998 Bindings or ports to any other programming languages are welcomed as external projects and
1999 are not going to be included into this repository.
2000
2001 */
2002
2003 #if VMA_RECORDING_ENABLED
2004 #include <chrono>
2005 #if defined(_WIN32)
2006 #include <windows.h>
2007 #else
2008 #include <sstream>
2009 #include <thread>
2010 #endif
2011 #endif
2012
2013 #ifdef __cplusplus
2014 extern "C" {
2015 #endif
2016
2017 /*
2018 Define this macro to 0/1 to disable/enable support for recording functionality,
2019 available through VmaAllocatorCreateInfo::pRecordSettings.
2020 */
2021 #ifndef VMA_RECORDING_ENABLED
2022 #define VMA_RECORDING_ENABLED 0
2023 #endif
2024
2025 #ifndef NOMINMAX
2026 #define NOMINMAX // For windows.h
2027 #endif
2028
2029 #if defined(__ANDROID__) && defined(VK_NO_PROTOTYPES) && VMA_STATIC_VULKAN_FUNCTIONS
2030 extern PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr;
2031 extern PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr;
2032 extern PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
2033 extern PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
2034 extern PFN_vkAllocateMemory vkAllocateMemory;
2035 extern PFN_vkFreeMemory vkFreeMemory;
2036 extern PFN_vkMapMemory vkMapMemory;
2037 extern PFN_vkUnmapMemory vkUnmapMemory;
2038 extern PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
2039 extern PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
2040 extern PFN_vkBindBufferMemory vkBindBufferMemory;
2041 extern PFN_vkBindImageMemory vkBindImageMemory;
2042 extern PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
2043 extern PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
2044 extern PFN_vkCreateBuffer vkCreateBuffer;
2045 extern PFN_vkDestroyBuffer vkDestroyBuffer;
2046 extern PFN_vkCreateImage vkCreateImage;
2047 extern PFN_vkDestroyImage vkDestroyImage;
2048 extern PFN_vkCmdCopyBuffer vkCmdCopyBuffer;
2049 #if VMA_VULKAN_VERSION >= 1001000
2050 extern PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2;
2051 extern PFN_vkGetImageMemoryRequirements2 vkGetImageMemoryRequirements2;
2052 extern PFN_vkBindBufferMemory2 vkBindBufferMemory2;
2053 extern PFN_vkBindImageMemory2 vkBindImageMemory2;
2054 extern PFN_vkGetPhysicalDeviceMemoryProperties2 vkGetPhysicalDeviceMemoryProperties2;
2055 #endif // #if VMA_VULKAN_VERSION >= 1001000
2056 #endif // #if defined(__ANDROID__) && VMA_STATIC_VULKAN_FUNCTIONS && VK_NO_PROTOTYPES
2057
2058 #ifndef VULKAN_H_
2059 #include <vulkan/vulkan.h>
2060 #endif
2061
2062 // Define this macro to declare maximum supported Vulkan version in format AAABBBCCC,
2063 // where AAA = major, BBB = minor, CCC = patch.
2064 // If you want to use version > 1.0, it still needs to be enabled via VmaAllocatorCreateInfo::vulkanApiVersion.
2065 #if !defined(VMA_VULKAN_VERSION)
2066 #if defined(VK_VERSION_1_2)
2067 #define VMA_VULKAN_VERSION 1002000
2068 #elif defined(VK_VERSION_1_1)
2069 #define VMA_VULKAN_VERSION 1001000
2070 #else
2071 #define VMA_VULKAN_VERSION 1000000
2072 #endif
2073 #endif
2074
2075 #if !defined(VMA_DEDICATED_ALLOCATION)
2076 #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
2077 #define VMA_DEDICATED_ALLOCATION 1
2078 #else
2079 #define VMA_DEDICATED_ALLOCATION 0
2080 #endif
2081 #endif
2082
2083 #if !defined(VMA_BIND_MEMORY2)
2084 #if VK_KHR_bind_memory2
2085 #define VMA_BIND_MEMORY2 1
2086 #else
2087 #define VMA_BIND_MEMORY2 0
2088 #endif
2089 #endif
2090
2091 #if !defined(VMA_MEMORY_BUDGET)
2092 #if VK_EXT_memory_budget && (VK_KHR_get_physical_device_properties2 || VMA_VULKAN_VERSION >= 1001000)
2093 #define VMA_MEMORY_BUDGET 1
2094 #else
2095 #define VMA_MEMORY_BUDGET 0
2096 #endif
2097 #endif
2098
2099 // Defined to 1 when VK_KHR_buffer_device_address device extension or equivalent core Vulkan 1.2 feature is defined in its headers.
2100 #if !defined(VMA_BUFFER_DEVICE_ADDRESS)
2101 #if VK_KHR_buffer_device_address || VMA_VULKAN_VERSION >= 1002000
2102 #define VMA_BUFFER_DEVICE_ADDRESS 1
2103 #else
2104 #define VMA_BUFFER_DEVICE_ADDRESS 0
2105 #endif
2106 #endif
2107
2108 // Define these macros to decorate all public functions with additional code,
2109 // before and after returned type, appropriately. This may be useful for
2110 // exporting the functions when compiling VMA as a separate library. Example:
2111 // #define VMA_CALL_PRE __declspec(dllexport)
2112 // #define VMA_CALL_POST __cdecl
2113 #ifndef VMA_CALL_PRE
2114 #define VMA_CALL_PRE
2115 #endif
2116 #ifndef VMA_CALL_POST
2117 #define VMA_CALL_POST
2118 #endif
2119
2120 // Define this macro to decorate pointers with an attribute specifying the
2121 // length of the array they point to if they are not null.
2122 //
2123 // The length may be one of
2124 // - The name of another parameter in the argument list where the pointer is declared
2125 // - The name of another member in the struct where the pointer is declared
2126 // - The name of a member of a struct type, meaning the value of that member in
2127 // the context of the call. For example
2128 // VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryHeapCount"),
2129 // this means the number of memory heaps available in the device associated
2130 // with the VmaAllocator being dealt with.
2131 #ifndef VMA_LEN_IF_NOT_NULL
2132 #define VMA_LEN_IF_NOT_NULL(len)
2133 #endif
2134
2135 // The VMA_NULLABLE macro is defined to be _Nullable when compiling with Clang.
2136 // see: https://clang.llvm.org/docs/AttributeReference.html#nullable
2137 #ifndef VMA_NULLABLE
2138 #ifdef __clang__
2139 #define VMA_NULLABLE _Nullable
2140 #else
2141 #define VMA_NULLABLE
2142 #endif
2143 #endif
2144
2145 // The VMA_NOT_NULL macro is defined to be _Nonnull when compiling with Clang.
2146 // see: https://clang.llvm.org/docs/AttributeReference.html#nonnull
2147 #ifndef VMA_NOT_NULL
2148 #ifdef __clang__
2149 #define VMA_NOT_NULL _Nonnull
2150 #else
2151 #define VMA_NOT_NULL
2152 #endif
2153 #endif
2154
2155 // If non-dispatchable handles are represented as pointers then we can give
2156 // then nullability annotations
2157 #ifndef VMA_NOT_NULL_NON_DISPATCHABLE
2158 #if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined (_M_IA64) || defined(__aarch64__) || defined(__powerpc64__)
2159 #define VMA_NOT_NULL_NON_DISPATCHABLE VMA_NOT_NULL
2160 #else
2161 #define VMA_NOT_NULL_NON_DISPATCHABLE
2162 #endif
2163 #endif
2164
2165 #ifndef VMA_NULLABLE_NON_DISPATCHABLE
2166 #if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined (_M_IA64) || defined(__aarch64__) || defined(__powerpc64__)
2167 #define VMA_NULLABLE_NON_DISPATCHABLE VMA_NULLABLE
2168 #else
2169 #define VMA_NULLABLE_NON_DISPATCHABLE
2170 #endif
2171 #endif
2172
2173 /** \struct VmaAllocator
2174 \brief Represents main object of this library initialized.
2175
2176 Fill structure #VmaAllocatorCreateInfo and call function vmaCreateAllocator() to create it.
2177 Call function vmaDestroyAllocator() to destroy it.
2178
2179 It is recommended to create just one object of this type per `VkDevice` object,
2180 right after Vulkan is initialized and keep it alive until before Vulkan device is destroyed.
2181 */
2182 VK_DEFINE_HANDLE(VmaAllocator)
2183
2184 /// Callback function called after successful vkAllocateMemory.
2185 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
2186 VmaAllocator VMA_NOT_NULL allocator,
2187 uint32_t memoryType,
2188 VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory,
2189 VkDeviceSize size,
2190 void* VMA_NULLABLE pUserData);
2191 /// Callback function called before vkFreeMemory.
2192 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
2193 VmaAllocator VMA_NOT_NULL allocator,
2194 uint32_t memoryType,
2195 VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory,
2196 VkDeviceSize size,
2197 void* VMA_NULLABLE pUserData);
2198
2199 /** \brief Set of callbacks that the library will call for `vkAllocateMemory` and `vkFreeMemory`.
2200
2201 Provided for informative purpose, e.g. to gather statistics about number of
2202 allocations or total amount of memory allocated in Vulkan.
2203
2204 Used in VmaAllocatorCreateInfo::pDeviceMemoryCallbacks.
2205 */
2206 typedef struct VmaDeviceMemoryCallbacks {
2207 /// Optional, can be null.
2208 PFN_vmaAllocateDeviceMemoryFunction VMA_NULLABLE pfnAllocate;
2209 /// Optional, can be null.
2210 PFN_vmaFreeDeviceMemoryFunction VMA_NULLABLE pfnFree;
2211 /// Optional, can be null.
2212 void* VMA_NULLABLE pUserData;
2213 } VmaDeviceMemoryCallbacks;
2214
2215 /// Flags for created #VmaAllocator.
2216 typedef enum VmaAllocatorCreateFlagBits {
2217 /** \brief Allocator and all objects created from it will not be synchronized internally, so you must guarantee they are used from only one thread at a time or synchronized externally by you.
2218
2219 Using this flag may increase performance because internal mutexes are not used.
2220 */
2221 VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
2222 /** \brief Enables usage of VK_KHR_dedicated_allocation extension.
2223
2224 The flag works only if VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_0`.
2225 When it's `VK_API_VERSION_1_1`, the flag is ignored because the extension has been promoted to Vulkan 1.1.
2226
2227 Using this extenion will automatically allocate dedicated blocks of memory for
2228 some buffers and images instead of suballocating place for them out of bigger
2229 memory blocks (as if you explicitly used #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
2230 flag) when it is recommended by the driver. It may improve performance on some
2231 GPUs.
2232
2233 You may set this flag only if you found out that following device extensions are
2234 supported, you enabled them while creating Vulkan device passed as
2235 VmaAllocatorCreateInfo::device, and you want them to be used internally by this
2236 library:
2237
2238 - VK_KHR_get_memory_requirements2 (device extension)
2239 - VK_KHR_dedicated_allocation (device extension)
2240
2241 When this flag is set, you can experience following warnings reported by Vulkan
2242 validation layer. You can ignore them.
2243
2244 > vkBindBufferMemory(): Binding memory to buffer 0x2d but vkGetBufferMemoryRequirements() has not been called on that buffer.
2245 */
2246 VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
2247 /**
2248 Enables usage of VK_KHR_bind_memory2 extension.
2249
2250 The flag works only if VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_0`.
2251 When it's `VK_API_VERSION_1_1`, the flag is ignored because the extension has been promoted to Vulkan 1.1.
2252
2253 You may set this flag only if you found out that this device extension is supported,
2254 you enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device,
2255 and you want it to be used internally by this library.
2256
2257 The extension provides functions `vkBindBufferMemory2KHR` and `vkBindImageMemory2KHR`,
2258 which allow to pass a chain of `pNext` structures while binding.
2259 This flag is required if you use `pNext` parameter in vmaBindBufferMemory2() or vmaBindImageMemory2().
2260 */
2261 VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT = 0x00000004,
2262 /**
2263 Enables usage of VK_EXT_memory_budget extension.
2264
2265 You may set this flag only if you found out that this device extension is supported,
2266 you enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device,
2267 and you want it to be used internally by this library, along with another instance extension
2268 VK_KHR_get_physical_device_properties2, which is required by it (or Vulkan 1.1, where this extension is promoted).
2269
2270 The extension provides query for current memory usage and budget, which will probably
2271 be more accurate than an estimation used by the library otherwise.
2272 */
2273 VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT = 0x00000008,
2274 /**
2275 Enables usage of VK_AMD_device_coherent_memory extension.
2276
2277 You may set this flag only if you:
2278
2279 - found out that this device extension is supported and enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device,
2280 - checked that `VkPhysicalDeviceCoherentMemoryFeaturesAMD::deviceCoherentMemory` is true and set it while creating the Vulkan device,
2281 - want it to be used internally by this library.
2282
2283 The extension and accompanying device feature provide access to memory types with
2284 `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` and `VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD` flags.
2285 They are useful mostly for writing breadcrumb markers - a common method for debugging GPU crash/hang/TDR.
2286
2287 When the extension is not enabled, such memory types are still enumerated, but their usage is illegal.
2288 To protect from this error, if you don't create the allocator with this flag, it will refuse to allocate any memory or create a custom pool in such memory type,
2289 returning `VK_ERROR_FEATURE_NOT_PRESENT`.
2290 */
2291 VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT = 0x00000010,
2292 /**
2293 Enables usage of "buffer device address" feature, which allows you to use function
2294 `vkGetBufferDeviceAddress*` to get raw GPU pointer to a buffer and pass it for usage inside a shader.
2295
2296 You may set this flag only if you:
2297
2298 1. (For Vulkan version < 1.2) Found as available and enabled device extension
2299 VK_KHR_buffer_device_address.
2300 This extension is promoted to core Vulkan 1.2.
2301 2. Found as available and enabled device feature `VkPhysicalDeviceBufferDeviceAddressFeatures*::bufferDeviceAddress`.
2302
2303 When this flag is set, you can create buffers with `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*` using VMA.
2304 The library automatically adds `VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT*` to
2305 allocated memory blocks wherever it might be needed.
2306
2307 For more information, see documentation chapter \ref enabling_buffer_device_address.
2308 */
2309 VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT = 0x00000020,
2310
2311 VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
2312 } VmaAllocatorCreateFlagBits;
2313 typedef VkFlags VmaAllocatorCreateFlags;
2314
2315 /** \brief Pointers to some Vulkan functions - a subset used by the library.
2316
2317 Used in VmaAllocatorCreateInfo::pVulkanFunctions.
2318 */
2319 typedef struct VmaVulkanFunctions {
2320 PFN_vkGetPhysicalDeviceProperties VMA_NULLABLE vkGetPhysicalDeviceProperties;
2321 PFN_vkGetPhysicalDeviceMemoryProperties VMA_NULLABLE vkGetPhysicalDeviceMemoryProperties;
2322 PFN_vkAllocateMemory VMA_NULLABLE vkAllocateMemory;
2323 PFN_vkFreeMemory VMA_NULLABLE vkFreeMemory;
2324 PFN_vkMapMemory VMA_NULLABLE vkMapMemory;
2325 PFN_vkUnmapMemory VMA_NULLABLE vkUnmapMemory;
2326 PFN_vkFlushMappedMemoryRanges VMA_NULLABLE vkFlushMappedMemoryRanges;
2327 PFN_vkInvalidateMappedMemoryRanges VMA_NULLABLE vkInvalidateMappedMemoryRanges;
2328 PFN_vkBindBufferMemory VMA_NULLABLE vkBindBufferMemory;
2329 PFN_vkBindImageMemory VMA_NULLABLE vkBindImageMemory;
2330 PFN_vkGetBufferMemoryRequirements VMA_NULLABLE vkGetBufferMemoryRequirements;
2331 PFN_vkGetImageMemoryRequirements VMA_NULLABLE vkGetImageMemoryRequirements;
2332 PFN_vkCreateBuffer VMA_NULLABLE vkCreateBuffer;
2333 PFN_vkDestroyBuffer VMA_NULLABLE vkDestroyBuffer;
2334 PFN_vkCreateImage VMA_NULLABLE vkCreateImage;
2335 PFN_vkDestroyImage VMA_NULLABLE vkDestroyImage;
2336 PFN_vkCmdCopyBuffer VMA_NULLABLE vkCmdCopyBuffer;
2337 #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
2338 PFN_vkGetBufferMemoryRequirements2KHR VMA_NULLABLE vkGetBufferMemoryRequirements2KHR;
2339 PFN_vkGetImageMemoryRequirements2KHR VMA_NULLABLE vkGetImageMemoryRequirements2KHR;
2340 #endif
2341 #if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000
2342 PFN_vkBindBufferMemory2KHR VMA_NULLABLE vkBindBufferMemory2KHR;
2343 PFN_vkBindImageMemory2KHR VMA_NULLABLE vkBindImageMemory2KHR;
2344 #endif
2345 #if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000
2346 PFN_vkGetPhysicalDeviceMemoryProperties2KHR VMA_NULLABLE vkGetPhysicalDeviceMemoryProperties2KHR;
2347 #endif
2348 } VmaVulkanFunctions;
2349
2350 /// Flags to be used in VmaRecordSettings::flags.
2351 typedef enum VmaRecordFlagBits {
2352 /** \brief Enables flush after recording every function call.
2353
2354 Enable it if you expect your application to crash, which may leave recording file truncated.
2355 It may degrade performance though.
2356 */
2357 VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
2358
2359 VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
2360 } VmaRecordFlagBits;
2361 typedef VkFlags VmaRecordFlags;
2362
2363 /// Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings.
2364 typedef struct VmaRecordSettings
2365 {
2366 /// Flags for recording. Use #VmaRecordFlagBits enum.
2367 VmaRecordFlags flags;
2368 /** \brief Path to the file that should be written by the recording.
2369
2370 Suggested extension: "csv".
2371 If the file already exists, it will be overwritten.
2372 It will be opened for the whole time #VmaAllocator object is alive.
2373 If opening this file fails, creation of the whole allocator object fails.
2374 */
2375 const char* VMA_NOT_NULL pFilePath;
2376 } VmaRecordSettings;
2377
2378 /// Description of a Allocator to be created.
2379 typedef struct VmaAllocatorCreateInfo
2380 {
2381 /// Flags for created allocator. Use #VmaAllocatorCreateFlagBits enum.
2382 VmaAllocatorCreateFlags flags;
2383 /// Vulkan physical device.
2384 /** It must be valid throughout whole lifetime of created allocator. */
2385 VkPhysicalDevice VMA_NOT_NULL physicalDevice;
2386 /// Vulkan device.
2387 /** It must be valid throughout whole lifetime of created allocator. */
2388 VkDevice VMA_NOT_NULL device;
2389 /// Preferred size of a single `VkDeviceMemory` block to be allocated from large heaps > 1 GiB. Optional.
2390 /** Set to 0 to use default, which is currently 256 MiB. */
2391 VkDeviceSize preferredLargeHeapBlockSize;
2392 /// Custom CPU memory allocation callbacks. Optional.
2393 /** Optional, can be null. When specified, will also be used for all CPU-side memory allocations. */
2394 const VkAllocationCallbacks* VMA_NULLABLE pAllocationCallbacks;
2395 /// Informative callbacks for `vkAllocateMemory`, `vkFreeMemory`. Optional.
2396 /** Optional, can be null. */
2397 const VmaDeviceMemoryCallbacks* VMA_NULLABLE pDeviceMemoryCallbacks;
2398 /** \brief Maximum number of additional frames that are in use at the same time as current frame.
2399
2400 This value is used only when you make allocations with
2401 VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become
2402 lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount.
2403
2404 For example, if you double-buffer your command buffers, so resources used for
2405 rendering in previous frame may still be in use by the GPU at the moment you
2406 allocate resources needed for the current frame, set this value to 1.
2407
2408 If you want to allow any allocations other than used in the current frame to
2409 become lost, set this value to 0.
2410 */
2411 uint32_t frameInUseCount;
2412 /** \brief Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out of particular Vulkan memory heap.
2413
2414 If not NULL, it must be a pointer to an array of
2415 `VkPhysicalDeviceMemoryProperties::memoryHeapCount` elements, defining limit on
2416 maximum number of bytes that can be allocated out of particular Vulkan memory
2417 heap.
2418
2419 Any of the elements may be equal to `VK_WHOLE_SIZE`, which means no limit on that
2420 heap. This is also the default in case of `pHeapSizeLimit` = NULL.
2421
2422 If there is a limit defined for a heap:
2423
2424 - If user tries to allocate more memory from that heap using this allocator,
2425 the allocation fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
2426 - If the limit is smaller than heap size reported in `VkMemoryHeap::size`, the
2427 value of this limit will be reported instead when using vmaGetMemoryProperties().
2428
2429 Warning! Using this feature may not be equivalent to installing a GPU with
2430 smaller amount of memory, because graphics driver doesn't necessary fail new
2431 allocations with `VK_ERROR_OUT_OF_DEVICE_MEMORY` result when memory capacity is
2432 exceeded. It may return success and just silently migrate some device memory
2433 blocks to system RAM. This driver behavior can also be controlled using
2434 VK_AMD_memory_overallocation_behavior extension.
2435 */
2436 const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryHeapCount") pHeapSizeLimit;
2437
2438 /** \brief Pointers to Vulkan functions. Can be null.
2439
2440 For details see [Pointers to Vulkan functions](@ref config_Vulkan_functions).
2441 */
2442 const VmaVulkanFunctions* VMA_NULLABLE pVulkanFunctions;
2443 /** \brief Parameters for recording of VMA calls. Can be null.
2444
2445 If not null, it enables recording of calls to VMA functions to a file.
2446 If support for recording is not enabled using `VMA_RECORDING_ENABLED` macro,
2447 creation of the allocator object fails with `VK_ERROR_FEATURE_NOT_PRESENT`.
2448 */
2449 const VmaRecordSettings* VMA_NULLABLE pRecordSettings;
2450 /** \brief Handle to Vulkan instance object.
2451
2452 Starting from version 3.0.0 this member is no longer optional, it must be set!
2453 */
2454 VkInstance VMA_NOT_NULL instance;
2455 /** \brief Optional. The highest version of Vulkan that the application is designed to use.
2456
2457 It must be a value in the format as created by macro `VK_MAKE_VERSION` or a constant like: `VK_API_VERSION_1_1`, `VK_API_VERSION_1_0`.
2458 The patch version number specified is ignored. Only the major and minor versions are considered.
2459 It must be less or equal (preferably equal) to value as passed to `vkCreateInstance` as `VkApplicationInfo::apiVersion`.
2460 Only versions 1.0 and 1.1 are supported by the current implementation.
2461 Leaving it initialized to zero is equivalent to `VK_API_VERSION_1_0`.
2462 */
2463 uint32_t vulkanApiVersion;
2464 } VmaAllocatorCreateInfo;
2465
2466 /// Creates Allocator object.
2467 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAllocator(
2468 const VmaAllocatorCreateInfo* VMA_NOT_NULL pCreateInfo,
2469 VmaAllocator VMA_NULLABLE * VMA_NOT_NULL pAllocator);
2470
2471 /// Destroys allocator object.
2472 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyAllocator(
2473 VmaAllocator VMA_NULLABLE allocator);
2474
2475 /** \brief Information about existing #VmaAllocator object.
2476 */
2477 typedef struct VmaAllocatorInfo
2478 {
2479 /** \brief Handle to Vulkan instance object.
2480
2481 This is the same value as has been passed through VmaAllocatorCreateInfo::instance.
2482 */
2483 VkInstance VMA_NOT_NULL instance;
2484 /** \brief Handle to Vulkan physical device object.
2485
2486 This is the same value as has been passed through VmaAllocatorCreateInfo::physicalDevice.
2487 */
2488 VkPhysicalDevice VMA_NOT_NULL physicalDevice;
2489 /** \brief Handle to Vulkan device object.
2490
2491 This is the same value as has been passed through VmaAllocatorCreateInfo::device.
2492 */
2493 VkDevice VMA_NOT_NULL device;
2494 } VmaAllocatorInfo;
2495
2496 /** \brief Returns information about existing #VmaAllocator object - handle to Vulkan device etc.
2497
2498 It might be useful if you want to keep just the #VmaAllocator handle and fetch other required handles to
2499 `VkPhysicalDevice`, `VkDevice` etc. every time using this function.
2500 */
2501 VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocatorInfo(VmaAllocator VMA_NOT_NULL allocator, VmaAllocatorInfo* VMA_NOT_NULL pAllocatorInfo);
2502
2503 /**
2504 PhysicalDeviceProperties are fetched from physicalDevice by the allocator.
2505 You can access it here, without fetching it again on your own.
2506 */
2507 VMA_CALL_PRE void VMA_CALL_POST vmaGetPhysicalDeviceProperties(
2508 VmaAllocator VMA_NOT_NULL allocator,
2509 const VkPhysicalDeviceProperties* VMA_NULLABLE * VMA_NOT_NULL ppPhysicalDeviceProperties);
2510
2511 /**
2512 PhysicalDeviceMemoryProperties are fetched from physicalDevice by the allocator.
2513 You can access it here, without fetching it again on your own.
2514 */
2515 VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryProperties(
2516 VmaAllocator VMA_NOT_NULL allocator,
2517 const VkPhysicalDeviceMemoryProperties* VMA_NULLABLE * VMA_NOT_NULL ppPhysicalDeviceMemoryProperties);
2518
2519 /**
2520 \brief Given Memory Type Index, returns Property Flags of this memory type.
2521
2522 This is just a convenience function. Same information can be obtained using
2523 vmaGetMemoryProperties().
2524 */
2525 VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryTypeProperties(
2526 VmaAllocator VMA_NOT_NULL allocator,
2527 uint32_t memoryTypeIndex,
2528 VkMemoryPropertyFlags* VMA_NOT_NULL pFlags);
2529
2530 /** \brief Sets index of the current frame.
2531
2532 This function must be used if you make allocations with
2533 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT and
2534 #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flags to inform the allocator
2535 when a new frame begins. Allocations queried using vmaGetAllocationInfo() cannot
2536 become lost in the current frame.
2537 */
2538 VMA_CALL_PRE void VMA_CALL_POST vmaSetCurrentFrameIndex(
2539 VmaAllocator VMA_NOT_NULL allocator,
2540 uint32_t frameIndex);
2541
2542 /** \brief Calculated statistics of memory usage in entire allocator.
2543 */
2544 typedef struct VmaStatInfo
2545 {
2546 /// Number of `VkDeviceMemory` Vulkan memory blocks allocated.
2547 uint32_t blockCount;
2548 /// Number of #VmaAllocation allocation objects allocated.
2549 uint32_t allocationCount;
2550 /// Number of free ranges of memory between allocations.
2551 uint32_t unusedRangeCount;
2552 /// Total number of bytes occupied by all allocations.
2553 VkDeviceSize usedBytes;
2554 /// Total number of bytes occupied by unused ranges.
2555 VkDeviceSize unusedBytes;
2556 VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
2557 VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
2558 } VmaStatInfo;
2559
2560 /// General statistics from current state of Allocator.
2561 typedef struct VmaStats
2562 {
2563 VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
2564 VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
2565 VmaStatInfo total;
2566 } VmaStats;
2567
2568 /** \brief Retrieves statistics from current state of the Allocator.
2569
2570 This function is called "calculate" not "get" because it has to traverse all
2571 internal data structures, so it may be quite slow. For faster but more brief statistics
2572 suitable to be called every frame or every allocation, use vmaGetBudget().
2573
2574 Note that when using allocator from multiple threads, returned information may immediately
2575 become outdated.
2576 */
2577 VMA_CALL_PRE void VMA_CALL_POST vmaCalculateStats(
2578 VmaAllocator VMA_NOT_NULL allocator,
2579 VmaStats* VMA_NOT_NULL pStats);
2580
2581 /** \brief Statistics of current memory usage and available budget, in bytes, for specific memory heap.
2582 */
2583 typedef struct VmaBudget
2584 {
2585 /** \brief Sum size of all `VkDeviceMemory` blocks allocated from particular heap, in bytes.
2586 */
2587 VkDeviceSize blockBytes;
2588
2589 /** \brief Sum size of all allocations created in particular heap, in bytes.
2590
2591 Usually less or equal than `blockBytes`.
2592 Difference `blockBytes - allocationBytes` is the amount of memory allocated but unused -
2593 available for new allocations or wasted due to fragmentation.
2594
2595 It might be greater than `blockBytes` if there are some allocations in lost state, as they account
2596 to this value as well.
2597 */
2598 VkDeviceSize allocationBytes;
2599
2600 /** \brief Estimated current memory usage of the program, in bytes.
2601
2602 Fetched from system using `VK_EXT_memory_budget` extension if enabled.
2603
2604 It might be different than `blockBytes` (usually higher) due to additional implicit objects
2605 also occupying the memory, like swapchain, pipelines, descriptor heaps, command buffers, or
2606 `VkDeviceMemory` blocks allocated outside of this library, if any.
2607 */
2608 VkDeviceSize usage;
2609
2610 /** \brief Estimated amount of memory available to the program, in bytes.
2611
2612 Fetched from system using `VK_EXT_memory_budget` extension if enabled.
2613
2614 It might be different (most probably smaller) than `VkMemoryHeap::size[heapIndex]` due to factors
2615 external to the program, like other programs also consuming system resources.
2616 Difference `budget - usage` is the amount of additional memory that can probably
2617 be allocated without problems. Exceeding the budget may result in various problems.
2618 */
2619 VkDeviceSize budget;
2620 } VmaBudget;
2621
2622 /** \brief Retrieves information about current memory budget for all memory heaps.
2623
2624 \param[out] pBudget Must point to array with number of elements at least equal to number of memory heaps in physical device used.
2625
2626 This function is called "get" not "calculate" because it is very fast, suitable to be called
2627 every frame or every allocation. For more detailed statistics use vmaCalculateStats().
2628
2629 Note that when using allocator from multiple threads, returned information may immediately
2630 become outdated.
2631 */
2632 VMA_CALL_PRE void VMA_CALL_POST vmaGetBudget(
2633 VmaAllocator VMA_NOT_NULL allocator,
2634 VmaBudget* VMA_NOT_NULL pBudget);
2635
2636 #ifndef VMA_STATS_STRING_ENABLED
2637 #define VMA_STATS_STRING_ENABLED 1
2638 #endif
2639
2640 #if VMA_STATS_STRING_ENABLED
2641
2642 /// Builds and returns statistics as string in JSON format.
2643 /** @param[out] ppStatsString Must be freed using vmaFreeStatsString() function.
2644 */
2645 VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString(
2646 VmaAllocator VMA_NOT_NULL allocator,
2647 char* VMA_NULLABLE * VMA_NOT_NULL ppStatsString,
2648 VkBool32 detailedMap);
2649
2650 VMA_CALL_PRE void VMA_CALL_POST vmaFreeStatsString(
2651 VmaAllocator VMA_NOT_NULL allocator,
2652 char* VMA_NULLABLE pStatsString);
2653
2654 #endif // #if VMA_STATS_STRING_ENABLED
2655
2656 /** \struct VmaPool
2657 \brief Represents custom memory pool
2658
2659 Fill structure VmaPoolCreateInfo and call function vmaCreatePool() to create it.
2660 Call function vmaDestroyPool() to destroy it.
2661
2662 For more information see [Custom memory pools](@ref choosing_memory_type_custom_memory_pools).
2663 */
2664 VK_DEFINE_HANDLE(VmaPool)
2665
2666 typedef enum VmaMemoryUsage
2667 {
2668 /** No intended memory usage specified.
2669 Use other members of VmaAllocationCreateInfo to specify your requirements.
2670 */
2671 VMA_MEMORY_USAGE_UNKNOWN = 0,
2672 /** Memory will be used on device only, so fast access from the device is preferred.
2673 It usually means device-local GPU (video) memory.
2674 No need to be mappable on host.
2675 It is roughly equivalent of `D3D12_HEAP_TYPE_DEFAULT`.
2676
2677 Usage:
2678
2679 - Resources written and read by device, e.g. images used as attachments.
2680 - Resources transferred from host once (immutable) or infrequently and read by
2681 device multiple times, e.g. textures to be sampled, vertex buffers, uniform
2682 (constant) buffers, and majority of other types of resources used on GPU.
2683
2684 Allocation may still end up in `HOST_VISIBLE` memory on some implementations.
2685 In such case, you are free to map it.
2686 You can use #VMA_ALLOCATION_CREATE_MAPPED_BIT with this usage type.
2687 */
2688 VMA_MEMORY_USAGE_GPU_ONLY = 1,
2689 /** Memory will be mappable on host.
2690 It usually means CPU (system) memory.
2691 Guarantees to be `HOST_VISIBLE` and `HOST_COHERENT`.
2692 CPU access is typically uncached. Writes may be write-combined.
2693 Resources created in this pool may still be accessible to the device, but access to them can be slow.
2694 It is roughly equivalent of `D3D12_HEAP_TYPE_UPLOAD`.
2695
2696 Usage: Staging copy of resources used as transfer source.
2697 */
2698 VMA_MEMORY_USAGE_CPU_ONLY = 2,
2699 /**
2700 Memory that is both mappable on host (guarantees to be `HOST_VISIBLE`) and preferably fast to access by GPU.
2701 CPU access is typically uncached. Writes may be write-combined.
2702
2703 Usage: Resources written frequently by host (dynamic), read by device. E.g. textures (with LINEAR layout), vertex buffers, uniform buffers updated every frame or every draw call.
2704 */
2705 VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
2706 /** Memory mappable on host (guarantees to be `HOST_VISIBLE`) and cached.
2707 It is roughly equivalent of `D3D12_HEAP_TYPE_READBACK`.
2708
2709 Usage:
2710
2711 - Resources written by device, read by host - results of some computations, e.g. screen capture, average scene luminance for HDR tone mapping.
2712 - Any resources read or accessed randomly on host, e.g. CPU-side copy of vertex buffer used as source of transfer, but also used for collision detection.
2713 */
2714 VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
2715 /** CPU memory - memory that is preferably not `DEVICE_LOCAL`, but also not guaranteed to be `HOST_VISIBLE`.
2716
2717 Usage: Staging copy of resources moved from GPU memory to CPU memory as part
2718 of custom paging/residency mechanism, to be moved back to GPU memory when needed.
2719 */
2720 VMA_MEMORY_USAGE_CPU_COPY = 5,
2721 /** Lazily allocated GPU memory having `VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT`.
2722 Exists mostly on mobile platforms. Using it on desktop PC or other GPUs with no such memory type present will fail the allocation.
2723
2724 Usage: Memory for transient attachment images (color attachments, depth attachments etc.), created with `VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT`.
2725
2726 Allocations with this usage are always created as dedicated - it implies #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT.
2727 */
2728 VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED = 6,
2729
2730 VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
2731 } VmaMemoryUsage;
2732
2733 /// Flags to be passed as VmaAllocationCreateInfo::flags.
2734 typedef enum VmaAllocationCreateFlagBits {
2735 /** \brief Set this flag if the allocation should have its own memory block.
2736
2737 Use it for special, big resources, like fullscreen images used as attachments.
2738
2739 You should not use this flag if VmaAllocationCreateInfo::pool is not null.
2740 */
2741 VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
2742
2743 /** \brief Set this flag to only try to allocate from existing `VkDeviceMemory` blocks and never create new such block.
2744
2745 If new allocation cannot be placed in any of the existing blocks, allocation
2746 fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY` error.
2747
2748 You should not use #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT and
2749 #VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT at the same time. It makes no sense.
2750
2751 If VmaAllocationCreateInfo::pool is not null, this flag is implied and ignored. */
2752 VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
2753 /** \brief Set this flag to use a memory that will be persistently mapped and retrieve pointer to it.
2754
2755 Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData.
2756
2757 It is valid to use this flag for allocation made from memory type that is not
2758 `HOST_VISIBLE`. This flag is then ignored and memory is not mapped. This is
2759 useful if you need an allocation that is efficient to use on GPU
2760 (`DEVICE_LOCAL`) and still want to map it directly if possible on platforms that
2761 support it (e.g. Intel GPU).
2762
2763 You should not use this flag together with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT.
2764 */
2765 VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
2766 /** Allocation created with this flag can become lost as a result of another
2767 allocation with #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag, so you
2768 must check it before use.
2769
2770 To check if allocation is not lost, call vmaGetAllocationInfo() and check if
2771 VmaAllocationInfo::deviceMemory is not `VK_NULL_HANDLE`.
2772
2773 For details about supporting lost allocations, see Lost Allocations
2774 chapter of User Guide on Main Page.
2775
2776 You should not use this flag together with #VMA_ALLOCATION_CREATE_MAPPED_BIT.
2777 */
2778 VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
2779 /** While creating allocation using this flag, other allocations that were
2780 created with flag #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT can become lost.
2781
2782 For details about supporting lost allocations, see Lost Allocations
2783 chapter of User Guide on Main Page.
2784 */
2785 VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
2786 /** Set this flag to treat VmaAllocationCreateInfo::pUserData as pointer to a
2787 null-terminated string. Instead of copying pointer value, a local copy of the
2788 string is made and stored in allocation's `pUserData`. The string is automatically
2789 freed together with the allocation. It is also used in vmaBuildStatsString().
2790 */
2791 VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
2792 /** Allocation will be created from upper stack in a double stack pool.
2793
2794 This flag is only allowed for custom pools created with #VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT flag.
2795 */
2796 VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT = 0x00000040,
2797 /** Create both buffer/image and allocation, but don't bind them together.
2798 It is useful when you want to bind yourself to do some more advanced binding, e.g. using some extensions.
2799 The flag is meaningful only with functions that bind by default: vmaCreateBuffer(), vmaCreateImage().
2800 Otherwise it is ignored.
2801 */
2802 VMA_ALLOCATION_CREATE_DONT_BIND_BIT = 0x00000080,
2803 /** Create allocation only if additional device memory required for it, if any, won't exceed
2804 memory budget. Otherwise return `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
2805 */
2806 VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT = 0x00000100,
2807
2808 /** Allocation strategy that chooses smallest possible free range for the
2809 allocation.
2810 */
2811 VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT = 0x00010000,
2812 /** Allocation strategy that chooses biggest possible free range for the
2813 allocation.
2814 */
2815 VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT = 0x00020000,
2816 /** Allocation strategy that chooses first suitable free range for the
2817 allocation.
2818
2819 "First" doesn't necessarily means the one with smallest offset in memory,
2820 but rather the one that is easiest and fastest to find.
2821 */
2822 VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT = 0x00040000,
2823
2824 /** Allocation strategy that tries to minimize memory usage.
2825 */
2826 VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT = VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT,
2827 /** Allocation strategy that tries to minimize allocation time.
2828 */
2829 VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT = VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT,
2830 /** Allocation strategy that tries to minimize memory fragmentation.
2831 */
2832 VMA_ALLOCATION_CREATE_STRATEGY_MIN_FRAGMENTATION_BIT = VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT,
2833
2834 /** A bit mask to extract only `STRATEGY` bits from entire set of flags.
2835 */
2836 VMA_ALLOCATION_CREATE_STRATEGY_MASK =
2837 VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT |
2838 VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT |
2839 VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT,
2840
2841 VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
2842 } VmaAllocationCreateFlagBits;
2843 typedef VkFlags VmaAllocationCreateFlags;
2844
2845 typedef struct VmaAllocationCreateInfo
2846 {
2847 /// Use #VmaAllocationCreateFlagBits enum.
2848 VmaAllocationCreateFlags flags;
2849 /** \brief Intended usage of memory.
2850
2851 You can leave #VMA_MEMORY_USAGE_UNKNOWN if you specify memory requirements in other way. \n
2852 If `pool` is not null, this member is ignored.
2853 */
2854 VmaMemoryUsage usage;
2855 /** \brief Flags that must be set in a Memory Type chosen for an allocation.
2856
2857 Leave 0 if you specify memory requirements in other way. \n
2858 If `pool` is not null, this member is ignored.*/
2859 VkMemoryPropertyFlags requiredFlags;
2860 /** \brief Flags that preferably should be set in a memory type chosen for an allocation.
2861
2862 Set to 0 if no additional flags are prefered. \n
2863 If `pool` is not null, this member is ignored. */
2864 VkMemoryPropertyFlags preferredFlags;
2865 /** \brief Bitmask containing one bit set for every memory type acceptable for this allocation.
2866
2867 Value 0 is equivalent to `UINT32_MAX` - it means any memory type is accepted if
2868 it meets other requirements specified by this structure, with no further
2869 restrictions on memory type index. \n
2870 If `pool` is not null, this member is ignored.
2871 */
2872 uint32_t memoryTypeBits;
2873 /** \brief Pool that this allocation should be created in.
2874
2875 Leave `VK_NULL_HANDLE` to allocate from default pool. If not null, members:
2876 `usage`, `requiredFlags`, `preferredFlags`, `memoryTypeBits` are ignored.
2877 */
2878 VmaPool VMA_NULLABLE pool;
2879 /** \brief Custom general-purpose pointer that will be stored in #VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData().
2880
2881 If #VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is used, it must be either
2882 null or pointer to a null-terminated string. The string will be then copied to
2883 internal buffer, so it doesn't need to be valid after allocation call.
2884 */
2885 void* VMA_NULLABLE pUserData;
2886 } VmaAllocationCreateInfo;
2887
2888 /**
2889 \brief Helps to find memoryTypeIndex, given memoryTypeBits and VmaAllocationCreateInfo.
2890
2891 This algorithm tries to find a memory type that:
2892
2893 - Is allowed by memoryTypeBits.
2894 - Contains all the flags from pAllocationCreateInfo->requiredFlags.
2895 - Matches intended usage.
2896 - Has as many flags from pAllocationCreateInfo->preferredFlags as possible.
2897
2898 \return Returns VK_ERROR_FEATURE_NOT_PRESENT if not found. Receiving such result
2899 from this function or any other allocating function probably means that your
2900 device doesn't support any memory type with requested features for the specific
2901 type of resource you want to use it for. Please check parameters of your
2902 resource, like image layout (OPTIMAL versus LINEAR) or mip level count.
2903 */
2904 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex(
2905 VmaAllocator VMA_NOT_NULL allocator,
2906 uint32_t memoryTypeBits,
2907 const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo,
2908 uint32_t* VMA_NOT_NULL pMemoryTypeIndex);
2909
2910 /**
2911 \brief Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo.
2912
2913 It can be useful e.g. to determine value to be used as VmaPoolCreateInfo::memoryTypeIndex.
2914 It internally creates a temporary, dummy buffer that never has memory bound.
2915 It is just a convenience function, equivalent to calling:
2916
2917 - `vkCreateBuffer`
2918 - `vkGetBufferMemoryRequirements`
2919 - `vmaFindMemoryTypeIndex`
2920 - `vkDestroyBuffer`
2921 */
2922 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForBufferInfo(
2923 VmaAllocator VMA_NOT_NULL allocator,
2924 const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo,
2925 const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo,
2926 uint32_t* VMA_NOT_NULL pMemoryTypeIndex);
2927
2928 /**
2929 \brief Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo.
2930
2931 It can be useful e.g. to determine value to be used as VmaPoolCreateInfo::memoryTypeIndex.
2932 It internally creates a temporary, dummy image that never has memory bound.
2933 It is just a convenience function, equivalent to calling:
2934
2935 - `vkCreateImage`
2936 - `vkGetImageMemoryRequirements`
2937 - `vmaFindMemoryTypeIndex`
2938 - `vkDestroyImage`
2939 */
2940 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForImageInfo(
2941 VmaAllocator VMA_NOT_NULL allocator,
2942 const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo,
2943 const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo,
2944 uint32_t* VMA_NOT_NULL pMemoryTypeIndex);
2945
2946 /// Flags to be passed as VmaPoolCreateInfo::flags.
2947 typedef enum VmaPoolCreateFlagBits {
2948 /** \brief Use this flag if you always allocate only buffers and linear images or only optimal images out of this pool and so Buffer-Image Granularity can be ignored.
2949
2950 This is an optional optimization flag.
2951
2952 If you always allocate using vmaCreateBuffer(), vmaCreateImage(),
2953 vmaAllocateMemoryForBuffer(), then you don't need to use it because allocator
2954 knows exact type of your allocations so it can handle Buffer-Image Granularity
2955 in the optimal way.
2956
2957 If you also allocate using vmaAllocateMemoryForImage() or vmaAllocateMemory(),
2958 exact type of such allocations is not known, so allocator must be conservative
2959 in handling Buffer-Image Granularity, which can lead to suboptimal allocation
2960 (wasted memory). In that case, if you can make sure you always allocate only
2961 buffers and linear images or only optimal images out of this pool, use this flag
2962 to make allocator disregard Buffer-Image Granularity and so make allocations
2963 faster and more optimal.
2964 */
2965 VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
2966
2967 /** \brief Enables alternative, linear allocation algorithm in this pool.
2968
2969 Specify this flag to enable linear allocation algorithm, which always creates
2970 new allocations after last one and doesn't reuse space from allocations freed in
2971 between. It trades memory consumption for simplified algorithm and data
2972 structure, which has better performance and uses less memory for metadata.
2973
2974 By using this flag, you can achieve behavior of free-at-once, stack,
2975 ring buffer, and double stack. For details, see documentation chapter
2976 \ref linear_algorithm.
2977
2978 When using this flag, you must specify VmaPoolCreateInfo::maxBlockCount == 1 (or 0 for default).
2979
2980 For more details, see [Linear allocation algorithm](@ref linear_algorithm).
2981 */
2982 VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT = 0x00000004,
2983
2984 /** \brief Enables alternative, buddy allocation algorithm in this pool.
2985
2986 It operates on a tree of blocks, each having size that is a power of two and
2987 a half of its parent's size. Comparing to default algorithm, this one provides
2988 faster allocation and deallocation and decreased external fragmentation,
2989 at the expense of more memory wasted (internal fragmentation).
2990
2991 For more details, see [Buddy allocation algorithm](@ref buddy_algorithm).
2992 */
2993 VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT = 0x00000008,
2994
2995 /** Bit mask to extract only `ALGORITHM` bits from entire set of flags.
2996 */
2997 VMA_POOL_CREATE_ALGORITHM_MASK =
2998 VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT |
2999 VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT,
3000
3001 VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
3002 } VmaPoolCreateFlagBits;
3003 typedef VkFlags VmaPoolCreateFlags;
3004
3005 /** \brief Describes parameter of created #VmaPool.
3006 */
3007 typedef struct VmaPoolCreateInfo {
3008 /** \brief Vulkan memory type index to allocate this pool from.
3009 */
3010 uint32_t memoryTypeIndex;
3011 /** \brief Use combination of #VmaPoolCreateFlagBits.
3012 */
3013 VmaPoolCreateFlags flags;
3014 /** \brief Size of a single `VkDeviceMemory` block to be allocated as part of this pool, in bytes. Optional.
3015
3016 Specify nonzero to set explicit, constant size of memory blocks used by this
3017 pool.
3018
3019 Leave 0 to use default and let the library manage block sizes automatically.
3020 Sizes of particular blocks may vary.
3021 */
3022 VkDeviceSize blockSize;
3023 /** \brief Minimum number of blocks to be always allocated in this pool, even if they stay empty.
3024
3025 Set to 0 to have no preallocated blocks and allow the pool be completely empty.
3026 */
3027 size_t minBlockCount;
3028 /** \brief Maximum number of blocks that can be allocated in this pool. Optional.
3029
3030 Set to 0 to use default, which is `SIZE_MAX`, which means no limit.
3031
3032 Set to same value as VmaPoolCreateInfo::minBlockCount to have fixed amount of memory allocated
3033 throughout whole lifetime of this pool.
3034 */
3035 size_t maxBlockCount;
3036 /** \brief Maximum number of additional frames that are in use at the same time as current frame.
3037
3038 This value is used only when you make allocations with
3039 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become
3040 lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount.
3041
3042 For example, if you double-buffer your command buffers, so resources used for
3043 rendering in previous frame may still be in use by the GPU at the moment you
3044 allocate resources needed for the current frame, set this value to 1.
3045
3046 If you want to allow any allocations other than used in the current frame to
3047 become lost, set this value to 0.
3048 */
3049 uint32_t frameInUseCount;
3050 } VmaPoolCreateInfo;
3051
3052 /** \brief Describes parameter of existing #VmaPool.
3053 */
3054 typedef struct VmaPoolStats {
3055 /** \brief Total amount of `VkDeviceMemory` allocated from Vulkan for this pool, in bytes.
3056 */
3057 VkDeviceSize size;
3058 /** \brief Total number of bytes in the pool not used by any #VmaAllocation.
3059 */
3060 VkDeviceSize unusedSize;
3061 /** \brief Number of #VmaAllocation objects created from this pool that were not destroyed or lost.
3062 */
3063 size_t allocationCount;
3064 /** \brief Number of continuous memory ranges in the pool not used by any #VmaAllocation.
3065 */
3066 size_t unusedRangeCount;
3067 /** \brief Size of the largest continuous free memory region available for new allocation.
3068
3069 Making a new allocation of that size is not guaranteed to succeed because of
3070 possible additional margin required to respect alignment and buffer/image
3071 granularity.
3072 */
3073 VkDeviceSize unusedRangeSizeMax;
3074 /** \brief Number of `VkDeviceMemory` blocks allocated for this pool.
3075 */
3076 size_t blockCount;
3077 } VmaPoolStats;
3078
3079 /** \brief Allocates Vulkan device memory and creates #VmaPool object.
3080
3081 @param allocator Allocator object.
3082 @param pCreateInfo Parameters of pool to create.
3083 @param[out] pPool Handle to created pool.
3084 */
3085 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreatePool(
3086 VmaAllocator VMA_NOT_NULL allocator,
3087 const VmaPoolCreateInfo* VMA_NOT_NULL pCreateInfo,
3088 VmaPool VMA_NULLABLE * VMA_NOT_NULL pPool);
3089
3090 /** \brief Destroys #VmaPool object and frees Vulkan device memory.
3091 */
3092 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool(
3093 VmaAllocator VMA_NOT_NULL allocator,
3094 VmaPool VMA_NULLABLE pool);
3095
3096 /** \brief Retrieves statistics of existing #VmaPool object.
3097
3098 @param allocator Allocator object.
3099 @param pool Pool object.
3100 @param[out] pPoolStats Statistics of specified pool.
3101 */
3102 VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolStats(
3103 VmaAllocator VMA_NOT_NULL allocator,
3104 VmaPool VMA_NOT_NULL pool,
3105 VmaPoolStats* VMA_NOT_NULL pPoolStats);
3106
3107 /** \brief Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now.
3108
3109 @param allocator Allocator object.
3110 @param pool Pool.
3111 @param[out] pLostAllocationCount Number of allocations marked as lost. Optional - pass null if you don't need this information.
3112 */
3113 VMA_CALL_PRE void VMA_CALL_POST vmaMakePoolAllocationsLost(
3114 VmaAllocator VMA_NOT_NULL allocator,
3115 VmaPool VMA_NOT_NULL pool,
3116 size_t* VMA_NULLABLE pLostAllocationCount);
3117
3118 /** \brief Checks magic number in margins around all allocations in given memory pool in search for corruptions.
3119
3120 Corruption detection is enabled only when `VMA_DEBUG_DETECT_CORRUPTION` macro is defined to nonzero,
3121 `VMA_DEBUG_MARGIN` is defined to nonzero and the pool is created in memory type that is
3122 `HOST_VISIBLE` and `HOST_COHERENT`. For more information, see [Corruption detection](@ref debugging_memory_usage_corruption_detection).
3123
3124 Possible return values:
3125
3126 - `VK_ERROR_FEATURE_NOT_PRESENT` - corruption detection is not enabled for specified pool.
3127 - `VK_SUCCESS` - corruption detection has been performed and succeeded.
3128 - `VK_ERROR_VALIDATION_FAILED_EXT` - corruption detection has been performed and found memory corruptions around one of the allocations.
3129 `VMA_ASSERT` is also fired in that case.
3130 - Other value: Error returned by Vulkan, e.g. memory mapping failure.
3131 */
3132 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckPoolCorruption(VmaAllocator VMA_NOT_NULL allocator, VmaPool VMA_NOT_NULL pool);
3133
3134 /** \brief Retrieves name of a custom pool.
3135
3136 After the call `ppName` is either null or points to an internally-owned null-terminated string
3137 containing name of the pool that was previously set. The pointer becomes invalid when the pool is
3138 destroyed or its name is changed using vmaSetPoolName().
3139 */
3140 VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName(
3141 VmaAllocator VMA_NOT_NULL allocator,
3142 VmaPool VMA_NOT_NULL pool,
3143 const char* VMA_NULLABLE * VMA_NOT_NULL ppName);
3144
3145 /** \brief Sets name of a custom pool.
3146
3147 `pName` can be either null or pointer to a null-terminated string with new name for the pool.
3148 Function makes internal copy of the string, so it can be changed or freed immediately after this call.
3149 */
3150 VMA_CALL_PRE void VMA_CALL_POST vmaSetPoolName(
3151 VmaAllocator VMA_NOT_NULL allocator,
3152 VmaPool VMA_NOT_NULL pool,
3153 const char* VMA_NULLABLE pName);
3154
3155 /** \struct VmaAllocation
3156 \brief Represents single memory allocation.
3157
3158 It may be either dedicated block of `VkDeviceMemory` or a specific region of a bigger block of this type
3159 plus unique offset.
3160
3161 There are multiple ways to create such object.
3162 You need to fill structure VmaAllocationCreateInfo.
3163 For more information see [Choosing memory type](@ref choosing_memory_type).
3164
3165 Although the library provides convenience functions that create Vulkan buffer or image,
3166 allocate memory for it and bind them together,
3167 binding of the allocation to a buffer or an image is out of scope of the allocation itself.
3168 Allocation object can exist without buffer/image bound,
3169 binding can be done manually by the user, and destruction of it can be done
3170 independently of destruction of the allocation.
3171
3172 The object also remembers its size and some other information.
3173 To retrieve this information, use function vmaGetAllocationInfo() and inspect
3174 returned structure VmaAllocationInfo.
3175
3176 Some kinds allocations can be in lost state.
3177 For more information, see [Lost allocations](@ref lost_allocations).
3178 */
3179 VK_DEFINE_HANDLE(VmaAllocation)
3180
3181 /** \brief Parameters of #VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
3182 */
3183 typedef struct VmaAllocationInfo {
3184 /** \brief Memory type index that this allocation was allocated from.
3185
3186 It never changes.
3187 */
3188 uint32_t memoryType;
3189 /** \brief Handle to Vulkan memory object.
3190
3191 Same memory object can be shared by multiple allocations.
3192
3193 It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost.
3194
3195 If the allocation is lost, it is equal to `VK_NULL_HANDLE`.
3196 */
3197 VkDeviceMemory VMA_NULLABLE_NON_DISPATCHABLE deviceMemory;
3198 /** \brief Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
3199
3200 It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost.
3201 */
3202 VkDeviceSize offset;
3203 /** \brief Size of this allocation, in bytes.
3204
3205 It never changes, unless allocation is lost.
3206
3207 \note Allocation size returned in this variable may be greater than the size
3208 requested for the resource e.g. as `VkBufferCreateInfo::size`. Whole size of the
3209 allocation is accessible for operations on memory e.g. using a pointer after
3210 mapping with vmaMapMemory(), but operations on the resource e.g. using
3211 `vkCmdCopyBuffer` must be limited to the size of the resource.
3212 */
3213 VkDeviceSize size;
3214 /** \brief Pointer to the beginning of this allocation as mapped data.
3215
3216 If the allocation hasn't been mapped using vmaMapMemory() and hasn't been
3217 created with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag, this value is null.
3218
3219 It can change after call to vmaMapMemory(), vmaUnmapMemory().
3220 It can also change after call to vmaDefragment() if this allocation is passed to the function.
3221 */
3222 void* VMA_NULLABLE pMappedData;
3223 /** \brief Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData().
3224
3225 It can change after call to vmaSetAllocationUserData() for this allocation.
3226 */
3227 void* VMA_NULLABLE pUserData;
3228 } VmaAllocationInfo;
3229
3230 /** \brief General purpose memory allocation.
3231
3232 @param[out] pAllocation Handle to allocated memory.
3233 @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
3234
3235 You should free the memory using vmaFreeMemory() or vmaFreeMemoryPages().
3236
3237 It is recommended to use vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage(),
3238 vmaCreateBuffer(), vmaCreateImage() instead whenever possible.
3239 */
3240 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory(
3241 VmaAllocator VMA_NOT_NULL allocator,
3242 const VkMemoryRequirements* VMA_NOT_NULL pVkMemoryRequirements,
3243 const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo,
3244 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation,
3245 VmaAllocationInfo* VMA_NULLABLE pAllocationInfo);
3246
3247 /** \brief General purpose memory allocation for multiple allocation objects at once.
3248
3249 @param allocator Allocator object.
3250 @param pVkMemoryRequirements Memory requirements for each allocation.
3251 @param pCreateInfo Creation parameters for each alloction.
3252 @param allocationCount Number of allocations to make.
3253 @param[out] pAllocations Pointer to array that will be filled with handles to created allocations.
3254 @param[out] pAllocationInfo Optional. Pointer to array that will be filled with parameters of created allocations.
3255
3256 You should free the memory using vmaFreeMemory() or vmaFreeMemoryPages().
3257
3258 Word "pages" is just a suggestion to use this function to allocate pieces of memory needed for sparse binding.
3259 It is just a general purpose allocation function able to make multiple allocations at once.
3260 It may be internally optimized to be more efficient than calling vmaAllocateMemory() `allocationCount` times.
3261
3262 All allocations are made using same parameters. All of them are created out of the same memory pool and type.
3263 If any allocation fails, all allocations already made within this function call are also freed, so that when
3264 returned result is not `VK_SUCCESS`, `pAllocation` array is always entirely filled with `VK_NULL_HANDLE`.
3265 */
3266 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages(
3267 VmaAllocator VMA_NOT_NULL allocator,
3268 const VkMemoryRequirements* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pVkMemoryRequirements,
3269 const VmaAllocationCreateInfo* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pCreateInfo,
3270 size_t allocationCount,
3271 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations,
3272 VmaAllocationInfo* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationInfo);
3273
3274 /**
3275 @param[out] pAllocation Handle to allocated memory.
3276 @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
3277
3278 You should free the memory using vmaFreeMemory().
3279 */
3280 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer(
3281 VmaAllocator VMA_NOT_NULL allocator,
3282 VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer,
3283 const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo,
3284 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation,
3285 VmaAllocationInfo* VMA_NULLABLE pAllocationInfo);
3286
3287 /// Function similar to vmaAllocateMemoryForBuffer().
3288 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage(
3289 VmaAllocator VMA_NOT_NULL allocator,
3290 VkImage VMA_NOT_NULL_NON_DISPATCHABLE image,
3291 const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo,
3292 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation,
3293 VmaAllocationInfo* VMA_NULLABLE pAllocationInfo);
3294
3295 /** \brief Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
3296
3297 Passing `VK_NULL_HANDLE` as `allocation` is valid. Such function call is just skipped.
3298 */
3299 VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory(
3300 VmaAllocator VMA_NOT_NULL allocator,
3301 const VmaAllocation VMA_NULLABLE allocation);
3302
3303 /** \brief Frees memory and destroys multiple allocations.
3304
3305 Word "pages" is just a suggestion to use this function to free pieces of memory used for sparse binding.
3306 It is just a general purpose function to free memory and destroy allocations made using e.g. vmaAllocateMemory(),
3307 vmaAllocateMemoryPages() and other functions.
3308 It may be internally optimized to be more efficient than calling vmaFreeMemory() `allocationCount` times.
3309
3310 Allocations in `pAllocations` array can come from any memory pools and types.
3311 Passing `VK_NULL_HANDLE` as elements of `pAllocations` array is valid. Such entries are just skipped.
3312 */
3313 VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages(
3314 VmaAllocator VMA_NOT_NULL allocator,
3315 size_t allocationCount,
3316 const VmaAllocation VMA_NULLABLE * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations);
3317
3318 /** \brief Deprecated.
3319
3320 \deprecated
3321 In version 2.2.0 it used to try to change allocation's size without moving or reallocating it.
3322 In current version it returns `VK_SUCCESS` only if `newSize` equals current allocation's size.
3323 Otherwise returns `VK_ERROR_OUT_OF_POOL_MEMORY`, indicating that allocation's size could not be changed.
3324 */
3325 VMA_CALL_PRE VkResult VMA_CALL_POST vmaResizeAllocation(
3326 VmaAllocator VMA_NOT_NULL allocator,
3327 VmaAllocation VMA_NOT_NULL allocation,
3328 VkDeviceSize newSize);
3329
3330 /** \brief Returns current information about specified allocation and atomically marks it as used in current frame.
3331
3332 Current paramters of given allocation are returned in `pAllocationInfo`.
3333
3334 This function also atomically "touches" allocation - marks it as used in current frame,
3335 just like vmaTouchAllocation().
3336 If the allocation is in lost state, `pAllocationInfo->deviceMemory == VK_NULL_HANDLE`.
3337
3338 Although this function uses atomics and doesn't lock any mutex, so it should be quite efficient,
3339 you can avoid calling it too often.
3340
3341 - You can retrieve same VmaAllocationInfo structure while creating your resource, from function
3342 vmaCreateBuffer(), vmaCreateImage(). You can remember it if you are sure parameters don't change
3343 (e.g. due to defragmentation or allocation becoming lost).
3344 - If you just want to check if allocation is not lost, vmaTouchAllocation() will work faster.
3345 */
3346 VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocationInfo(
3347 VmaAllocator VMA_NOT_NULL allocator,
3348 VmaAllocation VMA_NOT_NULL allocation,
3349 VmaAllocationInfo* VMA_NOT_NULL pAllocationInfo);
3350
3351 /** \brief Returns `VK_TRUE` if allocation is not lost and atomically marks it as used in current frame.
3352
3353 If the allocation has been created with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag,
3354 this function returns `VK_TRUE` if it's not in lost state, so it can still be used.
3355 It then also atomically "touches" the allocation - marks it as used in current frame,
3356 so that you can be sure it won't become lost in current frame or next `frameInUseCount` frames.
3357
3358 If the allocation is in lost state, the function returns `VK_FALSE`.
3359 Memory of such allocation, as well as buffer or image bound to it, should not be used.
3360 Lost allocation and the buffer/image still need to be destroyed.
3361
3362 If the allocation has been created without #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag,
3363 this function always returns `VK_TRUE`.
3364 */
3365 VMA_CALL_PRE VkBool32 VMA_CALL_POST vmaTouchAllocation(
3366 VmaAllocator VMA_NOT_NULL allocator,
3367 VmaAllocation VMA_NOT_NULL allocation);
3368
3369 /** \brief Sets pUserData in given allocation to new value.
3370
3371 If the allocation was created with VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT,
3372 pUserData must be either null, or pointer to a null-terminated string. The function
3373 makes local copy of the string and sets it as allocation's `pUserData`. String
3374 passed as pUserData doesn't need to be valid for whole lifetime of the allocation -
3375 you can free it after this call. String previously pointed by allocation's
3376 pUserData is freed from memory.
3377
3378 If the flag was not used, the value of pointer `pUserData` is just copied to
3379 allocation's `pUserData`. It is opaque, so you can use it however you want - e.g.
3380 as a pointer, ordinal number or some handle to you own data.
3381 */
3382 VMA_CALL_PRE void VMA_CALL_POST vmaSetAllocationUserData(
3383 VmaAllocator VMA_NOT_NULL allocator,
3384 VmaAllocation VMA_NOT_NULL allocation,
3385 void* VMA_NULLABLE pUserData);
3386
3387 /** \brief Creates new allocation that is in lost state from the beginning.
3388
3389 It can be useful if you need a dummy, non-null allocation.
3390
3391 You still need to destroy created object using vmaFreeMemory().
3392
3393 Returned allocation is not tied to any specific memory pool or memory type and
3394 not bound to any image or buffer. It has size = 0. It cannot be turned into
3395 a real, non-empty allocation.
3396 */
3397 VMA_CALL_PRE void VMA_CALL_POST vmaCreateLostAllocation(
3398 VmaAllocator VMA_NOT_NULL allocator,
3399 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation);
3400
3401 /** \brief Maps memory represented by given allocation and returns pointer to it.
3402
3403 Maps memory represented by given allocation to make it accessible to CPU code.
3404 When succeeded, `*ppData` contains pointer to first byte of this memory.
3405 If the allocation is part of bigger `VkDeviceMemory` block, the pointer is
3406 correctly offseted to the beginning of region assigned to this particular
3407 allocation.
3408
3409 Mapping is internally reference-counted and synchronized, so despite raw Vulkan
3410 function `vkMapMemory()` cannot be used to map same block of `VkDeviceMemory`
3411 multiple times simultaneously, it is safe to call this function on allocations
3412 assigned to the same memory block. Actual Vulkan memory will be mapped on first
3413 mapping and unmapped on last unmapping.
3414
3415 If the function succeeded, you must call vmaUnmapMemory() to unmap the
3416 allocation when mapping is no longer needed or before freeing the allocation, at
3417 the latest.
3418
3419 It also safe to call this function multiple times on the same allocation. You
3420 must call vmaUnmapMemory() same number of times as you called vmaMapMemory().
3421
3422 It is also safe to call this function on allocation created with
3423 #VMA_ALLOCATION_CREATE_MAPPED_BIT flag. Its memory stays mapped all the time.
3424 You must still call vmaUnmapMemory() same number of times as you called
3425 vmaMapMemory(). You must not call vmaUnmapMemory() additional time to free the
3426 "0-th" mapping made automatically due to #VMA_ALLOCATION_CREATE_MAPPED_BIT flag.
3427
3428 This function fails when used on allocation made in memory type that is not
3429 `HOST_VISIBLE`.
3430
3431 This function always fails when called for allocation that was created with
3432 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocations cannot be
3433 mapped.
3434
3435 This function doesn't automatically flush or invalidate caches.
3436 If the allocation is made from a memory types that is not `HOST_COHERENT`,
3437 you also need to use vmaInvalidateAllocation() / vmaFlushAllocation(), as required by Vulkan specification.
3438 */
3439 VMA_CALL_PRE VkResult VMA_CALL_POST vmaMapMemory(
3440 VmaAllocator VMA_NOT_NULL allocator,
3441 VmaAllocation VMA_NOT_NULL allocation,
3442 void* VMA_NULLABLE * VMA_NOT_NULL ppData);
3443
3444 /** \brief Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
3445
3446 For details, see description of vmaMapMemory().
3447
3448 This function doesn't automatically flush or invalidate caches.
3449 If the allocation is made from a memory types that is not `HOST_COHERENT`,
3450 you also need to use vmaInvalidateAllocation() / vmaFlushAllocation(), as required by Vulkan specification.
3451 */
3452 VMA_CALL_PRE void VMA_CALL_POST vmaUnmapMemory(
3453 VmaAllocator VMA_NOT_NULL allocator,
3454 VmaAllocation VMA_NOT_NULL allocation);
3455
3456 /** \brief Flushes memory of given allocation.
3457
3458 Calls `vkFlushMappedMemoryRanges()` for memory associated with given range of given allocation.
3459 It needs to be called after writing to a mapped memory for memory types that are not `HOST_COHERENT`.
3460 Unmap operation doesn't do that automatically.
3461
3462 - `offset` must be relative to the beginning of allocation.
3463 - `size` can be `VK_WHOLE_SIZE`. It means all memory from `offset` the the end of given allocation.
3464 - `offset` and `size` don't have to be aligned.
3465 They are internally rounded down/up to multiply of `nonCoherentAtomSize`.
3466 - If `size` is 0, this call is ignored.
3467 - If memory type that the `allocation` belongs to is not `HOST_VISIBLE` or it is `HOST_COHERENT`,
3468 this call is ignored.
3469
3470 Warning! `offset` and `size` are relative to the contents of given `allocation`.
3471 If you mean whole allocation, you can pass 0 and `VK_WHOLE_SIZE`, respectively.
3472 Do not pass allocation's offset as `offset`!!!
3473
3474 This function returns the `VkResult` from `vkFlushMappedMemoryRanges` if it is
3475 called, otherwise `VK_SUCCESS`.
3476 */
3477 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocation(
3478 VmaAllocator VMA_NOT_NULL allocator,
3479 VmaAllocation VMA_NOT_NULL allocation,
3480 VkDeviceSize offset,
3481 VkDeviceSize size);
3482
3483 /** \brief Invalidates memory of given allocation.
3484
3485 Calls `vkInvalidateMappedMemoryRanges()` for memory associated with given range of given allocation.
3486 It needs to be called before reading from a mapped memory for memory types that are not `HOST_COHERENT`.
3487 Map operation doesn't do that automatically.
3488
3489 - `offset` must be relative to the beginning of allocation.
3490 - `size` can be `VK_WHOLE_SIZE`. It means all memory from `offset` the the end of given allocation.
3491 - `offset` and `size` don't have to be aligned.
3492 They are internally rounded down/up to multiply of `nonCoherentAtomSize`.
3493 - If `size` is 0, this call is ignored.
3494 - If memory type that the `allocation` belongs to is not `HOST_VISIBLE` or it is `HOST_COHERENT`,
3495 this call is ignored.
3496
3497 Warning! `offset` and `size` are relative to the contents of given `allocation`.
3498 If you mean whole allocation, you can pass 0 and `VK_WHOLE_SIZE`, respectively.
3499 Do not pass allocation's offset as `offset`!!!
3500
3501 This function returns the `VkResult` from `vkInvalidateMappedMemoryRanges` if
3502 it is called, otherwise `VK_SUCCESS`.
3503 */
3504 VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocation(
3505 VmaAllocator VMA_NOT_NULL allocator,
3506 VmaAllocation VMA_NOT_NULL allocation,
3507 VkDeviceSize offset,
3508 VkDeviceSize size);
3509
3510 /** \brief Flushes memory of given set of allocations.
3511
3512 Calls `vkFlushMappedMemoryRanges()` for memory associated with given ranges of given allocations.
3513 For more information, see documentation of vmaFlushAllocation().
3514
3515 \param allocator
3516 \param allocationCount
3517 \param allocations
3518 \param offsets If not null, it must point to an array of offsets of regions to flush, relative to the beginning of respective allocations. Null means all ofsets are zero.
3519 \param sizes If not null, it must point to an array of sizes of regions to flush in respective allocations. Null means `VK_WHOLE_SIZE` for all allocations.
3520
3521 This function returns the `VkResult` from `vkFlushMappedMemoryRanges` if it is
3522 called, otherwise `VK_SUCCESS`.
3523 */
3524 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocations(
3525 VmaAllocator VMA_NOT_NULL allocator,
3526 uint32_t allocationCount,
3527 const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) allocations,
3528 const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) offsets,
3529 const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) sizes);
3530
3531 /** \brief Invalidates memory of given set of allocations.
3532
3533 Calls `vkInvalidateMappedMemoryRanges()` for memory associated with given ranges of given allocations.
3534 For more information, see documentation of vmaInvalidateAllocation().
3535
3536 \param allocator
3537 \param allocationCount
3538 \param allocations
3539 \param offsets If not null, it must point to an array of offsets of regions to flush, relative to the beginning of respective allocations. Null means all ofsets are zero.
3540 \param sizes If not null, it must point to an array of sizes of regions to flush in respective allocations. Null means `VK_WHOLE_SIZE` for all allocations.
3541
3542 This function returns the `VkResult` from `vkInvalidateMappedMemoryRanges` if it is
3543 called, otherwise `VK_SUCCESS`.
3544 */
3545 VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocations(
3546 VmaAllocator VMA_NOT_NULL allocator,
3547 uint32_t allocationCount,
3548 const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) allocations,
3549 const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) offsets,
3550 const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) sizes);
3551
3552 /** \brief Checks magic number in margins around all allocations in given memory types (in both default and custom pools) in search for corruptions.
3553
3554 @param memoryTypeBits Bit mask, where each bit set means that a memory type with that index should be checked.
3555
3556 Corruption detection is enabled only when `VMA_DEBUG_DETECT_CORRUPTION` macro is defined to nonzero,
3557 `VMA_DEBUG_MARGIN` is defined to nonzero and only for memory types that are
3558 `HOST_VISIBLE` and `HOST_COHERENT`. For more information, see [Corruption detection](@ref debugging_memory_usage_corruption_detection).
3559
3560 Possible return values:
3561
3562 - `VK_ERROR_FEATURE_NOT_PRESENT` - corruption detection is not enabled for any of specified memory types.
3563 - `VK_SUCCESS` - corruption detection has been performed and succeeded.
3564 - `VK_ERROR_VALIDATION_FAILED_EXT` - corruption detection has been performed and found memory corruptions around one of the allocations.
3565 `VMA_ASSERT` is also fired in that case.
3566 - Other value: Error returned by Vulkan, e.g. memory mapping failure.
3567 */
3568 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeBits);
3569
3570 /** \struct VmaDefragmentationContext
3571 \brief Represents Opaque object that represents started defragmentation process.
3572
3573 Fill structure #VmaDefragmentationInfo2 and call function vmaDefragmentationBegin() to create it.
3574 Call function vmaDefragmentationEnd() to destroy it.
3575 */
3576 VK_DEFINE_HANDLE(VmaDefragmentationContext)
3577
3578 /// Flags to be used in vmaDefragmentationBegin(). None at the moment. Reserved for future use.
3579 typedef enum VmaDefragmentationFlagBits {
3580 VMA_DEFRAGMENTATION_FLAG_INCREMENTAL = 0x1,
3581 VMA_DEFRAGMENTATION_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
3582 } VmaDefragmentationFlagBits;
3583 typedef VkFlags VmaDefragmentationFlags;
3584
3585 /** \brief Parameters for defragmentation.
3586
3587 To be used with function vmaDefragmentationBegin().
3588 */
3589 typedef struct VmaDefragmentationInfo2 {
3590 /** \brief Reserved for future use. Should be 0.
3591 */
3592 VmaDefragmentationFlags flags;
3593 /** \brief Number of allocations in `pAllocations` array.
3594 */
3595 uint32_t allocationCount;
3596 /** \brief Pointer to array of allocations that can be defragmented.
3597
3598 The array should have `allocationCount` elements.
3599 The array should not contain nulls.
3600 Elements in the array should be unique - same allocation cannot occur twice.
3601 It is safe to pass allocations that are in the lost state - they are ignored.
3602 All allocations not present in this array are considered non-moveable during this defragmentation.
3603 */
3604 const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations;
3605 /** \brief Optional, output. Pointer to array that will be filled with information whether the allocation at certain index has been changed during defragmentation.
3606
3607 The array should have `allocationCount` elements.
3608 You can pass null if you are not interested in this information.
3609 */
3610 VkBool32* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationsChanged;
3611 /** \brief Numer of pools in `pPools` array.
3612 */
3613 uint32_t poolCount;
3614 /** \brief Either null or pointer to array of pools to be defragmented.
3615
3616 All the allocations in the specified pools can be moved during defragmentation
3617 and there is no way to check if they were really moved as in `pAllocationsChanged`,
3618 so you must query all the allocations in all these pools for new `VkDeviceMemory`
3619 and offset using vmaGetAllocationInfo() if you might need to recreate buffers
3620 and images bound to them.
3621
3622 The array should have `poolCount` elements.
3623 The array should not contain nulls.
3624 Elements in the array should be unique - same pool cannot occur twice.
3625
3626 Using this array is equivalent to specifying all allocations from the pools in `pAllocations`.
3627 It might be more efficient.
3628 */
3629 const VmaPool VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(poolCount) pPools;
3630 /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places using transfers on CPU side, like `memcpy()`, `memmove()`.
3631
3632 `VK_WHOLE_SIZE` means no limit.
3633 */
3634 VkDeviceSize maxCpuBytesToMove;
3635 /** \brief Maximum number of allocations that can be moved to a different place using transfers on CPU side, like `memcpy()`, `memmove()`.
3636
3637 `UINT32_MAX` means no limit.
3638 */
3639 uint32_t maxCpuAllocationsToMove;
3640 /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places using transfers on GPU side, posted to `commandBuffer`.
3641
3642 `VK_WHOLE_SIZE` means no limit.
3643 */
3644 VkDeviceSize maxGpuBytesToMove;
3645 /** \brief Maximum number of allocations that can be moved to a different place using transfers on GPU side, posted to `commandBuffer`.
3646
3647 `UINT32_MAX` means no limit.
3648 */
3649 uint32_t maxGpuAllocationsToMove;
3650 /** \brief Optional. Command buffer where GPU copy commands will be posted.
3651
3652 If not null, it must be a valid command buffer handle that supports Transfer queue type.
3653 It must be in the recording state and outside of a render pass instance.
3654 You need to submit it and make sure it finished execution before calling vmaDefragmentationEnd().
3655
3656 Passing null means that only CPU defragmentation will be performed.
3657 */
3658 VkCommandBuffer VMA_NULLABLE commandBuffer;
3659 } VmaDefragmentationInfo2;
3660
3661 typedef struct VmaDefragmentationPassMoveInfo {
3662 VmaAllocation VMA_NOT_NULL allocation;
3663 VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory;
3664 VkDeviceSize offset;
3665 } VmaDefragmentationPassMoveInfo;
3666
3667 /** \brief Parameters for incremental defragmentation steps.
3668
3669 To be used with function vmaBeginDefragmentationPass().
3670 */
3671 typedef struct VmaDefragmentationPassInfo {
3672 uint32_t moveCount;
3673 VmaDefragmentationPassMoveInfo* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(moveCount) pMoves;
3674 } VmaDefragmentationPassInfo;
3675
3676 /** \brief Deprecated. Optional configuration parameters to be passed to function vmaDefragment().
3677
3678 \deprecated This is a part of the old interface. It is recommended to use structure #VmaDefragmentationInfo2 and function vmaDefragmentationBegin() instead.
3679 */
3680 typedef struct VmaDefragmentationInfo {
3681 /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places.
3682
3683 Default is `VK_WHOLE_SIZE`, which means no limit.
3684 */
3685 VkDeviceSize maxBytesToMove;
3686 /** \brief Maximum number of allocations that can be moved to different place.
3687
3688 Default is `UINT32_MAX`, which means no limit.
3689 */
3690 uint32_t maxAllocationsToMove;
3691 } VmaDefragmentationInfo;
3692
3693 /** \brief Statistics returned by function vmaDefragment(). */
3694 typedef struct VmaDefragmentationStats {
3695 /// Total number of bytes that have been copied while moving allocations to different places.
3696 VkDeviceSize bytesMoved;
3697 /// Total number of bytes that have been released to the system by freeing empty `VkDeviceMemory` objects.
3698 VkDeviceSize bytesFreed;
3699 /// Number of allocations that have been moved to different places.
3700 uint32_t allocationsMoved;
3701 /// Number of empty `VkDeviceMemory` objects that have been released to the system.
3702 uint32_t deviceMemoryBlocksFreed;
3703 } VmaDefragmentationStats;
3704
3705 /** \brief Begins defragmentation process.
3706
3707 @param allocator Allocator object.
3708 @param pInfo Structure filled with parameters of defragmentation.
3709 @param[out] pStats Optional. Statistics of defragmentation. You can pass null if you are not interested in this information.
3710 @param[out] pContext Context object that must be passed to vmaDefragmentationEnd() to finish defragmentation.
3711 @return `VK_SUCCESS` and `*pContext == null` if defragmentation finished within this function call. `VK_NOT_READY` and `*pContext != null` if defragmentation has been started and you need to call vmaDefragmentationEnd() to finish it. Negative value in case of error.
3712
3713 Use this function instead of old, deprecated vmaDefragment().
3714
3715 Warning! Between the call to vmaDefragmentationBegin() and vmaDefragmentationEnd():
3716
3717 - You should not use any of allocations passed as `pInfo->pAllocations` or
3718 any allocations that belong to pools passed as `pInfo->pPools`,
3719 including calling vmaGetAllocationInfo(), vmaTouchAllocation(), or access
3720 their data.
3721 - Some mutexes protecting internal data structures may be locked, so trying to
3722 make or free any allocations, bind buffers or images, map memory, or launch
3723 another simultaneous defragmentation in between may cause stall (when done on
3724 another thread) or deadlock (when done on the same thread), unless you are
3725 100% sure that defragmented allocations are in different pools.
3726 - Information returned via `pStats` and `pInfo->pAllocationsChanged` are undefined.
3727 They become valid after call to vmaDefragmentationEnd().
3728 - If `pInfo->commandBuffer` is not null, you must submit that command buffer
3729 and make sure it finished execution before calling vmaDefragmentationEnd().
3730
3731 For more information and important limitations regarding defragmentation, see documentation chapter:
3732 [Defragmentation](@ref defragmentation).
3733 */
3734 VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationBegin(
3735 VmaAllocator VMA_NOT_NULL allocator,
3736 const VmaDefragmentationInfo2* VMA_NOT_NULL pInfo,
3737 VmaDefragmentationStats* VMA_NULLABLE pStats,
3738 VmaDefragmentationContext VMA_NULLABLE * VMA_NOT_NULL pContext);
3739
3740 /** \brief Ends defragmentation process.
3741
3742 Use this function to finish defragmentation started by vmaDefragmentationBegin().
3743 It is safe to pass `context == null`. The function then does nothing.
3744 */
3745 VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationEnd(
3746 VmaAllocator VMA_NOT_NULL allocator,
3747 VmaDefragmentationContext VMA_NULLABLE context);
3748
3749 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass(
3750 VmaAllocator VMA_NOT_NULL allocator,
3751 VmaDefragmentationContext VMA_NULLABLE context,
3752 VmaDefragmentationPassInfo* VMA_NOT_NULL pInfo
3753 );
3754 VMA_CALL_PRE VkResult VMA_CALL_POST vmaEndDefragmentationPass(
3755 VmaAllocator VMA_NOT_NULL allocator,
3756 VmaDefragmentationContext VMA_NULLABLE context
3757 );
3758
3759 /** \brief Deprecated. Compacts memory by moving allocations.
3760
3761 @param pAllocations Array of allocations that can be moved during this compation.
3762 @param allocationCount Number of elements in pAllocations and pAllocationsChanged arrays.
3763 @param[out] pAllocationsChanged Array of boolean values that will indicate whether matching allocation in pAllocations array has been moved. This parameter is optional. Pass null if you don't need this information.
3764 @param pDefragmentationInfo Configuration parameters. Optional - pass null to use default values.
3765 @param[out] pDefragmentationStats Statistics returned by the function. Optional - pass null if you don't need this information.
3766 @return `VK_SUCCESS` if completed, negative error code in case of error.
3767
3768 \deprecated This is a part of the old interface. It is recommended to use structure #VmaDefragmentationInfo2 and function vmaDefragmentationBegin() instead.
3769
3770 This function works by moving allocations to different places (different
3771 `VkDeviceMemory` objects and/or different offsets) in order to optimize memory
3772 usage. Only allocations that are in `pAllocations` array can be moved. All other
3773 allocations are considered nonmovable in this call. Basic rules:
3774
3775 - Only allocations made in memory types that have
3776 `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` and `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT`
3777 flags can be compacted. You may pass other allocations but it makes no sense -
3778 these will never be moved.
3779 - Custom pools created with #VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT or
3780 #VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT flag are not defragmented. Allocations
3781 passed to this function that come from such pools are ignored.
3782 - Allocations created with #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT or
3783 created as dedicated allocations for any other reason are also ignored.
3784 - Both allocations made with or without #VMA_ALLOCATION_CREATE_MAPPED_BIT
3785 flag can be compacted. If not persistently mapped, memory will be mapped
3786 temporarily inside this function if needed.
3787 - You must not pass same #VmaAllocation object multiple times in `pAllocations` array.
3788
3789 The function also frees empty `VkDeviceMemory` blocks.
3790
3791 Warning: This function may be time-consuming, so you shouldn't call it too often
3792 (like after every resource creation/destruction).
3793 You can call it on special occasions (like when reloading a game level or
3794 when you just destroyed a lot of objects). Calling it every frame may be OK, but
3795 you should measure that on your platform.
3796
3797 For more information, see [Defragmentation](@ref defragmentation) chapter.
3798 */
3799 VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragment(
3800 VmaAllocator VMA_NOT_NULL allocator,
3801 const VmaAllocation VMA_NOT_NULL * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations,
3802 size_t allocationCount,
3803 VkBool32* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationsChanged,
3804 const VmaDefragmentationInfo* VMA_NULLABLE pDefragmentationInfo,
3805 VmaDefragmentationStats* VMA_NULLABLE pDefragmentationStats);
3806
3807 /** \brief Binds buffer to allocation.
3808
3809 Binds specified buffer to region of memory represented by specified allocation.
3810 Gets `VkDeviceMemory` handle and offset from the allocation.
3811 If you want to create a buffer, allocate memory for it and bind them together separately,
3812 you should use this function for binding instead of standard `vkBindBufferMemory()`,
3813 because it ensures proper synchronization so that when a `VkDeviceMemory` object is used by multiple
3814 allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from multiple threads simultaneously
3815 (which is illegal in Vulkan).
3816
3817 It is recommended to use function vmaCreateBuffer() instead of this one.
3818 */
3819 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory(
3820 VmaAllocator VMA_NOT_NULL allocator,
3821 VmaAllocation VMA_NOT_NULL allocation,
3822 VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer);
3823
3824 /** \brief Binds buffer to allocation with additional parameters.
3825
3826 @param allocationLocalOffset Additional offset to be added while binding, relative to the beginnig of the `allocation`. Normally it should be 0.
3827 @param pNext A chain of structures to be attached to `VkBindBufferMemoryInfoKHR` structure used internally. Normally it should be null.
3828
3829 This function is similar to vmaBindBufferMemory(), but it provides additional parameters.
3830
3831 If `pNext` is not null, #VmaAllocator object must have been created with #VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT flag
3832 or with VmaAllocatorCreateInfo::vulkanApiVersion `>= VK_API_VERSION_1_1`. Otherwise the call fails.
3833 */
3834 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory2(
3835 VmaAllocator VMA_NOT_NULL allocator,
3836 VmaAllocation VMA_NOT_NULL allocation,
3837 VkDeviceSize allocationLocalOffset,
3838 VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer,
3839 const void* VMA_NULLABLE pNext);
3840
3841 /** \brief Binds image to allocation.
3842
3843 Binds specified image to region of memory represented by specified allocation.
3844 Gets `VkDeviceMemory` handle and offset from the allocation.
3845 If you want to create an image, allocate memory for it and bind them together separately,
3846 you should use this function for binding instead of standard `vkBindImageMemory()`,
3847 because it ensures proper synchronization so that when a `VkDeviceMemory` object is used by multiple
3848 allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from multiple threads simultaneously
3849 (which is illegal in Vulkan).
3850
3851 It is recommended to use function vmaCreateImage() instead of this one.
3852 */
3853 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory(
3854 VmaAllocator VMA_NOT_NULL allocator,
3855 VmaAllocation VMA_NOT_NULL allocation,
3856 VkImage VMA_NOT_NULL_NON_DISPATCHABLE image);
3857
3858 /** \brief Binds image to allocation with additional parameters.
3859
3860 @param allocationLocalOffset Additional offset to be added while binding, relative to the beginnig of the `allocation`. Normally it should be 0.
3861 @param pNext A chain of structures to be attached to `VkBindImageMemoryInfoKHR` structure used internally. Normally it should be null.
3862
3863 This function is similar to vmaBindImageMemory(), but it provides additional parameters.
3864
3865 If `pNext` is not null, #VmaAllocator object must have been created with #VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT flag
3866 or with VmaAllocatorCreateInfo::vulkanApiVersion `>= VK_API_VERSION_1_1`. Otherwise the call fails.
3867 */
3868 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory2(
3869 VmaAllocator VMA_NOT_NULL allocator,
3870 VmaAllocation VMA_NOT_NULL allocation,
3871 VkDeviceSize allocationLocalOffset,
3872 VkImage VMA_NOT_NULL_NON_DISPATCHABLE image,
3873 const void* VMA_NULLABLE pNext);
3874
3875 /**
3876 @param[out] pBuffer Buffer that was created.
3877 @param[out] pAllocation Allocation that was created.
3878 @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
3879
3880 This function automatically:
3881
3882 -# Creates buffer.
3883 -# Allocates appropriate memory for it.
3884 -# Binds the buffer with the memory.
3885
3886 If any of these operations fail, buffer and allocation are not created,
3887 returned value is negative error code, *pBuffer and *pAllocation are null.
3888
3889 If the function succeeded, you must destroy both buffer and allocation when you
3890 no longer need them using either convenience function vmaDestroyBuffer() or
3891 separately, using `vkDestroyBuffer()` and vmaFreeMemory().
3892
3893 If VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT flag was used,
3894 VK_KHR_dedicated_allocation extension is used internally to query driver whether
3895 it requires or prefers the new buffer to have dedicated allocation. If yes,
3896 and if dedicated allocation is possible (VmaAllocationCreateInfo::pool is null
3897 and VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT is not used), it creates dedicated
3898 allocation for this buffer, just like when using
3899 VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT.
3900 */
3901 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer(
3902 VmaAllocator VMA_NOT_NULL allocator,
3903 const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo,
3904 const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo,
3905 VkBuffer VMA_NULLABLE_NON_DISPATCHABLE * VMA_NOT_NULL pBuffer,
3906 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation,
3907 VmaAllocationInfo* VMA_NULLABLE pAllocationInfo);
3908
3909 /** \brief Destroys Vulkan buffer and frees allocated memory.
3910
3911 This is just a convenience function equivalent to:
3912
3913 \code
3914 vkDestroyBuffer(device, buffer, allocationCallbacks);
3915 vmaFreeMemory(allocator, allocation);
3916 \endcode
3917
3918 It it safe to pass null as buffer and/or allocation.
3919 */
3920 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyBuffer(
3921 VmaAllocator VMA_NOT_NULL allocator,
3922 VkBuffer VMA_NULLABLE_NON_DISPATCHABLE buffer,
3923 VmaAllocation VMA_NULLABLE allocation);
3924
3925 /// Function similar to vmaCreateBuffer().
3926 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage(
3927 VmaAllocator VMA_NOT_NULL allocator,
3928 const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo,
3929 const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo,
3930 VkImage VMA_NULLABLE_NON_DISPATCHABLE * VMA_NOT_NULL pImage,
3931 VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation,
3932 VmaAllocationInfo* VMA_NULLABLE pAllocationInfo);
3933
3934 /** \brief Destroys Vulkan image and frees allocated memory.
3935
3936 This is just a convenience function equivalent to:
3937
3938 \code
3939 vkDestroyImage(device, image, allocationCallbacks);
3940 vmaFreeMemory(allocator, allocation);
3941 \endcode
3942
3943 It it safe to pass null as image and/or allocation.
3944 */
3945 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyImage(
3946 VmaAllocator VMA_NOT_NULL allocator,
3947 VkImage VMA_NULLABLE_NON_DISPATCHABLE image,
3948 VmaAllocation VMA_NULLABLE allocation);
3949
3950 #ifdef __cplusplus
3951 }
3952 #endif
3953
3954 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
3955
3956 // For Visual Studio IntelliSense.
3957 #if defined(__cplusplus) && defined(__INTELLISENSE__)
3958 #define VMA_IMPLEMENTATION
3959 #endif
3960
3961 #ifdef VMA_IMPLEMENTATION
3962 #undef VMA_IMPLEMENTATION
3963
3964 #include <cstdint>
3965 #include <cstdlib>
3966 #include <cstring>
3967 #include <utility>
3968
3969 /*******************************************************************************
3970 CONFIGURATION SECTION
3971
3972 Define some of these macros before each #include of this header or change them
3973 here if you need other then default behavior depending on your environment.
3974 */
3975
3976 /*
3977 Define this macro to 1 to make the library fetch pointers to Vulkan functions
3978 internally, like:
3979
3980 vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
3981 */
3982 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
3983 #define VMA_STATIC_VULKAN_FUNCTIONS 1
3984 #endif
3985
3986 /*
3987 Define this macro to 1 to make the library fetch pointers to Vulkan functions
3988 internally, like:
3989
3990 vulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkGetDeviceProcAddr(m_hDevice, vkAllocateMemory);
3991 */
3992 #if !defined(VMA_DYNAMIC_VULKAN_FUNCTIONS)
3993 #define VMA_DYNAMIC_VULKAN_FUNCTIONS 1
3994 #if defined(VK_NO_PROTOTYPES)
3995 extern PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr;
3996 extern PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr;
3997 #endif
3998 #endif
3999
4000 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
4001 //#define VMA_USE_STL_CONTAINERS 1
4002
4003 /* Set this macro to 1 to make the library including and using STL containers:
4004 std::pair, std::vector, std::list, std::unordered_map.
4005
4006 Set it to 0 or undefined to make the library using its own implementation of
4007 the containers.
4008 */
4009 #if VMA_USE_STL_CONTAINERS
4010 #define VMA_USE_STL_VECTOR 1
4011 #define VMA_USE_STL_UNORDERED_MAP 1
4012 #define VMA_USE_STL_LIST 1
4013 #endif
4014
4015 #ifndef VMA_USE_STL_SHARED_MUTEX
4016 // Compiler conforms to C++17.
4017 #if __cplusplus >= 201703L
4018 #define VMA_USE_STL_SHARED_MUTEX 1
4019 // Visual studio defines __cplusplus properly only when passed additional parameter: /Zc:__cplusplus
4020 // Otherwise it's always 199711L, despite shared_mutex works since Visual Studio 2015 Update 2.
4021 // See: https://blogs.msdn.microsoft.com/vcblog/2018/04/09/msvc-now-correctly-reports-__cplusplus/
4022 #elif defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 190023918 && __cplusplus == 199711L && _MSVC_LANG >= 201703L
4023 #define VMA_USE_STL_SHARED_MUTEX 1
4024 #else
4025 #define VMA_USE_STL_SHARED_MUTEX 0
4026 #endif
4027 #endif
4028
4029 /*
4030 THESE INCLUDES ARE NOT ENABLED BY DEFAULT.
4031 Library has its own container implementation.
4032 */
4033 #if VMA_USE_STL_VECTOR
4034 #include <vector>
4035 #endif
4036
4037 #if VMA_USE_STL_UNORDERED_MAP
4038 #include <unordered_map>
4039 #endif
4040
4041 #if VMA_USE_STL_LIST
4042 #include <list>
4043 #endif
4044
4045 /*
4046 Following headers are used in this CONFIGURATION section only, so feel free to
4047 remove them if not needed.
4048 */
4049 #include <cassert> // for assert
4050 #include <algorithm> // for min, max
4051 #include <mutex>
4052
4053 #ifndef VMA_NULL
4054 // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
4055 #define VMA_NULL nullptr
4056 #endif
4057
4058 #if defined(__ANDROID_API__) && (__ANDROID_API__ < 16)
4059 #include <cstdlib>
4060 void *vma_aligned_alloc(size_t alignment, size_t size)
4061 {
4062 // alignment must be >= sizeof(void*)
4063 if(alignment < sizeof(void*))
4064 {
4065 alignment = sizeof(void*);
4066 }
4067
4068 return memalign(alignment, size);
4069 }
4070 #elif defined(__APPLE__) || defined(__ANDROID__) || (defined(__linux__) && defined(__GLIBCXX__) && !defined(_GLIBCXX_HAVE_ALIGNED_ALLOC))
4071 #include <cstdlib>
4072
4073 #if defined(__APPLE__)
4074 #include <AvailabilityMacros.h>
4075 #endif
4076
4077 void *vma_aligned_alloc(size_t alignment, size_t size)
4078 {
4079 #if defined(__APPLE__) && (defined(MAC_OS_X_VERSION_10_16) || defined(__IPHONE_14_0))
4080 #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_16 || __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_14_0
4081 // For C++14, usr/include/malloc/_malloc.h declares aligned_alloc()) only
4082 // with the MacOSX11.0 SDK in Xcode 12 (which is what adds
4083 // MAC_OS_X_VERSION_10_16), even though the function is marked
4084 // availabe for 10.15. That's why the preprocessor checks for 10.16 but
4085 // the __builtin_available checks for 10.15.
4086 // People who use C++17 could call aligned_alloc with the 10.15 SDK already.
4087 if (__builtin_available(macOS 10.15, iOS 13, *))
4088 return aligned_alloc(alignment, size);
4089 #endif
4090 #endif
4091 // alignment must be >= sizeof(void*)
4092 if(alignment < sizeof(void*))
4093 {
4094 alignment = sizeof(void*);
4095 }
4096
4097 void *pointer;
4098 if(posix_memalign(&pointer, alignment, size) == 0)
4099 return pointer;
4100 return VMA_NULL;
4101 }
4102 #elif defined(_WIN32)
4103 void *vma_aligned_alloc(size_t alignment, size_t size)
4104 {
4105 return _aligned_malloc(size, alignment);
4106 }
4107 #else
4108 void *vma_aligned_alloc(size_t alignment, size_t size)
4109 {
4110 return aligned_alloc(alignment, size);
4111 }
4112 #endif
4113
4114 // If your compiler is not compatible with C++11 and definition of
4115 // aligned_alloc() function is missing, uncommeting following line may help:
4116
4117 //#include <malloc.h>
4118
4119 // Normal assert to check for programmer's errors, especially in Debug configuration.
4120 #ifndef VMA_ASSERT
4121 #ifdef NDEBUG
4122 #define VMA_ASSERT(expr)
4123 #else
4124 #define VMA_ASSERT(expr) assert(expr)
4125 #endif
4126 #endif
4127
4128 // Assert that will be called very often, like inside data structures e.g. operator[].
4129 // Making it non-empty can make program slow.
4130 #ifndef VMA_HEAVY_ASSERT
4131 #ifdef NDEBUG
4132 #define VMA_HEAVY_ASSERT(expr)
4133 #else
4134 #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
4135 #endif
4136 #endif
4137
4138 #ifndef VMA_ALIGN_OF
4139 #define VMA_ALIGN_OF(type) (__alignof(type))
4140 #endif
4141
4142 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
4143 #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) vma_aligned_alloc((alignment), (size))
4144 #endif
4145
4146 #ifndef VMA_SYSTEM_FREE
4147 #if defined(_WIN32)
4148 #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
4149 #else
4150 #define VMA_SYSTEM_FREE(ptr) free(ptr)
4151 #endif
4152 #endif
4153
4154 #ifndef VMA_MIN
4155 #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
4156 #endif
4157
4158 #ifndef VMA_MAX
4159 #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
4160 #endif
4161
4162 #ifndef VMA_SWAP
4163 #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
4164 #endif
4165
4166 #ifndef VMA_SORT
4167 #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
4168 #endif
4169
4170 #ifndef VMA_DEBUG_LOG
4171 #define VMA_DEBUG_LOG(format, ...)
4172 /*
4173 #define VMA_DEBUG_LOG(format, ...) do { \
4174 printf(format, __VA_ARGS__); \
4175 printf("\n"); \
4176 } while(false)
4177 */
4178 #endif
4179
4180 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
4181 #if VMA_STATS_STRING_ENABLED
4182 static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
4183 {
4184 snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
4185 }
4186 static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
4187 {
4188 snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
4189 }
4190 static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
4191 {
4192 snprintf(outStr, strLen, "%p", ptr);
4193 }
4194 #endif
4195
4196 #ifndef VMA_MUTEX
4197 class VmaMutex
4198 {
4199 public:
4200 void Lock() { m_Mutex.lock(); }
4201 void Unlock() { m_Mutex.unlock(); }
4202 bool TryLock() { return m_Mutex.try_lock(); }
4203 private:
4204 std::mutex m_Mutex;
4205 };
4206 #define VMA_MUTEX VmaMutex
4207 #endif
4208
4209 // Read-write mutex, where "read" is shared access, "write" is exclusive access.
4210 #ifndef VMA_RW_MUTEX
4211 #if VMA_USE_STL_SHARED_MUTEX
4212 // Use std::shared_mutex from C++17.
4213 #include <shared_mutex>
4214 class VmaRWMutex
4215 {
4216 public:
4217 void LockRead() { m_Mutex.lock_shared(); }
4218 void UnlockRead() { m_Mutex.unlock_shared(); }
4219 bool TryLockRead() { return m_Mutex.try_lock_shared(); }
4220 void LockWrite() { m_Mutex.lock(); }
4221 void UnlockWrite() { m_Mutex.unlock(); }
4222 bool TryLockWrite() { return m_Mutex.try_lock(); }
4223 private:
4224 std::shared_mutex m_Mutex;
4225 };
4226 #define VMA_RW_MUTEX VmaRWMutex
4227 #elif defined(_WIN32) && defined(WINVER) && WINVER >= 0x0600
4228 // Use SRWLOCK from WinAPI.
4229 // Minimum supported client = Windows Vista, server = Windows Server 2008.
4230 class VmaRWMutex
4231 {
4232 public:
4233 VmaRWMutex() { InitializeSRWLock(&m_Lock); }
4234 void LockRead() { AcquireSRWLockShared(&m_Lock); }
4235 void UnlockRead() { ReleaseSRWLockShared(&m_Lock); }
4236 bool TryLockRead() { return TryAcquireSRWLockShared(&m_Lock) != FALSE; }
4237 void LockWrite() { AcquireSRWLockExclusive(&m_Lock); }
4238 void UnlockWrite() { ReleaseSRWLockExclusive(&m_Lock); }
4239 bool TryLockWrite() { return TryAcquireSRWLockExclusive(&m_Lock) != FALSE; }
4240 private:
4241 SRWLOCK m_Lock;
4242 };
4243 #define VMA_RW_MUTEX VmaRWMutex
4244 #else
4245 // Less efficient fallback: Use normal mutex.
4246 class VmaRWMutex
4247 {
4248 public:
4249 void LockRead() { m_Mutex.Lock(); }
4250 void UnlockRead() { m_Mutex.Unlock(); }
4251 bool TryLockRead() { return m_Mutex.TryLock(); }
4252 void LockWrite() { m_Mutex.Lock(); }
4253 void UnlockWrite() { m_Mutex.Unlock(); }
4254 bool TryLockWrite() { return m_Mutex.TryLock(); }
4255 private:
4256 VMA_MUTEX m_Mutex;
4257 };
4258 #define VMA_RW_MUTEX VmaRWMutex
4259 #endif // #if VMA_USE_STL_SHARED_MUTEX
4260 #endif // #ifndef VMA_RW_MUTEX
4261
4262 /*
4263 If providing your own implementation, you need to implement a subset of std::atomic.
4264 */
4265 #ifndef VMA_ATOMIC_UINT32
4266 #include <atomic>
4267 #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
4268 #endif
4269
4270 #ifndef VMA_ATOMIC_UINT64
4271 #include <atomic>
4272 #define VMA_ATOMIC_UINT64 std::atomic<uint64_t>
4273 #endif
4274
4275 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
4276 /**
4277 Every allocation will have its own memory block.
4278 Define to 1 for debugging purposes only.
4279 */
4280 #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
4281 #endif
4282
4283 #ifndef VMA_DEBUG_ALIGNMENT
4284 /**
4285 Minimum alignment of all allocations, in bytes.
4286 Set to more than 1 for debugging purposes only. Must be power of two.
4287 */
4288 #define VMA_DEBUG_ALIGNMENT (1)
4289 #endif
4290
4291 #ifndef VMA_DEBUG_MARGIN
4292 /**
4293 Minimum margin before and after every allocation, in bytes.
4294 Set nonzero for debugging purposes only.
4295 */
4296 #define VMA_DEBUG_MARGIN (0)
4297 #endif
4298
4299 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
4300 /**
4301 Define this macro to 1 to automatically fill new allocations and destroyed
4302 allocations with some bit pattern.
4303 */
4304 #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
4305 #endif
4306
4307 #ifndef VMA_DEBUG_DETECT_CORRUPTION
4308 /**
4309 Define this macro to 1 together with non-zero value of VMA_DEBUG_MARGIN to
4310 enable writing magic value to the margin before and after every allocation and
4311 validating it, so that memory corruptions (out-of-bounds writes) are detected.
4312 */
4313 #define VMA_DEBUG_DETECT_CORRUPTION (0)
4314 #endif
4315
4316 #ifndef VMA_DEBUG_GLOBAL_MUTEX
4317 /**
4318 Set this to 1 for debugging purposes only, to enable single mutex protecting all
4319 entry calls to the library. Can be useful for debugging multithreading issues.
4320 */
4321 #define VMA_DEBUG_GLOBAL_MUTEX (0)
4322 #endif
4323
4324 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
4325 /**
4326 Minimum value for VkPhysicalDeviceLimits::bufferImageGranularity.
4327 Set to more than 1 for debugging purposes only. Must be power of two.
4328 */
4329 #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
4330 #endif
4331
4332 #ifndef VMA_SMALL_HEAP_MAX_SIZE
4333 /// Maximum size of a memory heap in Vulkan to consider it "small".
4334 #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
4335 #endif
4336
4337 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
4338 /// Default size of a block allocated as single VkDeviceMemory from a "large" heap.
4339 #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
4340 #endif
4341
4342 #ifndef VMA_CLASS_NO_COPY
4343 #define VMA_CLASS_NO_COPY(className) \
4344 private: \
4345 className(const className&) = delete; \
4346 className& operator=(const className&) = delete;
4347 #endif
4348
4349 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
4350
4351 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
4352 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
4353
4354 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
4355 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
4356
4357 /*******************************************************************************
4358 END OF CONFIGURATION
4359 */
4360
4361 // # Copy of some Vulkan definitions so we don't need to check their existence just to handle few constants.
4362
4363 static const uint32_t VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY = 0x00000040;
4364 static const uint32_t VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY = 0x00000080;
4365 static const uint32_t VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_COPY = 0x00020000;
4366
4367 static const uint32_t VMA_ALLOCATION_INTERNAL_STRATEGY_MIN_OFFSET = 0x10000000u;
4368
4369 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
4370 VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
4371
4372 // Returns number of bits set to 1 in (v).
4373 static inline uint32_t VmaCountBitsSet(uint32_t v)
4374 {
4375 uint32_t c = v - ((v >> 1) & 0x55555555);
4376 c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
4377 c = ((c >> 4) + c) & 0x0F0F0F0F;
4378 c = ((c >> 8) + c) & 0x00FF00FF;
4379 c = ((c >> 16) + c) & 0x0000FFFF;
4380 return c;
4381 }
4382
4383 /*
4384 Returns true if given number is a power of two.
4385 T must be unsigned integer number or signed integer but always nonnegative.
4386 For 0 returns true.
4387 */
4388 template <typename T>
4389 inline bool VmaIsPow2(T x)
4390 {
4391 return (x & (x-1)) == 0;
4392 }
4393
4394 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
4395 // Use types like uint32_t, uint64_t as T.
4396 template <typename T>
4397 static inline T VmaAlignUp(T val, T alignment)
4398 {
4399 VMA_HEAVY_ASSERT(VmaIsPow2(alignment));
4400 return (val + alignment - 1) & ~(alignment - 1);
4401 }
4402 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
4403 // Use types like uint32_t, uint64_t as T.
4404 template <typename T>
4405 static inline T VmaAlignDown(T val, T alignment)
4406 {
4407 VMA_HEAVY_ASSERT(VmaIsPow2(alignment));
4408 return val & ~(alignment - 1);
4409 }
4410
4411 // Division with mathematical rounding to nearest number.
4412 template <typename T>
4413 static inline T VmaRoundDiv(T x, T y)
4414 {
4415 return (x + (y / (T)2)) / y;
4416 }
4417
4418 // Returns smallest power of 2 greater or equal to v.
4419 static inline uint32_t VmaNextPow2(uint32_t v)
4420 {
4421 v--;
4422 v |= v >> 1;
4423 v |= v >> 2;
4424 v |= v >> 4;
4425 v |= v >> 8;
4426 v |= v >> 16;
4427 v++;
4428 return v;
4429 }
4430 static inline uint64_t VmaNextPow2(uint64_t v)
4431 {
4432 v--;
4433 v |= v >> 1;
4434 v |= v >> 2;
4435 v |= v >> 4;
4436 v |= v >> 8;
4437 v |= v >> 16;
4438 v |= v >> 32;
4439 v++;
4440 return v;
4441 }
4442
4443 // Returns largest power of 2 less or equal to v.
4444 static inline uint32_t VmaPrevPow2(uint32_t v)
4445 {
4446 v |= v >> 1;
4447 v |= v >> 2;
4448 v |= v >> 4;
4449 v |= v >> 8;
4450 v |= v >> 16;
4451 v = v ^ (v >> 1);
4452 return v;
4453 }
4454 static inline uint64_t VmaPrevPow2(uint64_t v)
4455 {
4456 v |= v >> 1;
4457 v |= v >> 2;
4458 v |= v >> 4;
4459 v |= v >> 8;
4460 v |= v >> 16;
4461 v |= v >> 32;
4462 v = v ^ (v >> 1);
4463 return v;
4464 }
4465
4466 static inline bool VmaStrIsEmpty(const char* pStr)
4467 {
4468 return pStr == VMA_NULL || *pStr == '\0';
4469 }
4470
4471 #if VMA_STATS_STRING_ENABLED
4472
4473 static const char* VmaAlgorithmToStr(uint32_t algorithm)
4474 {
4475 switch(algorithm)
4476 {
4477 case VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT:
4478 return "Linear";
4479 case VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT:
4480 return "Buddy";
4481 case 0:
4482 return "Default";
4483 default:
4484 VMA_ASSERT(0);
4485 return "";
4486 }
4487 }
4488
4489 #endif // #if VMA_STATS_STRING_ENABLED
4490
4491 #ifndef VMA_SORT
4492
4493 template<typename Iterator, typename Compare>
4494 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
4495 {
4496 Iterator centerValue = end; --centerValue;
4497 Iterator insertIndex = beg;
4498 for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
4499 {
4500 if(cmp(*memTypeIndex, *centerValue))
4501 {
4502 if(insertIndex != memTypeIndex)
4503 {
4504 VMA_SWAP(*memTypeIndex, *insertIndex);
4505 }
4506 ++insertIndex;
4507 }
4508 }
4509 if(insertIndex != centerValue)
4510 {
4511 VMA_SWAP(*insertIndex, *centerValue);
4512 }
4513 return insertIndex;
4514 }
4515
4516 template<typename Iterator, typename Compare>
4517 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
4518 {
4519 if(beg < end)
4520 {
4521 Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
4522 VmaQuickSort<Iterator, Compare>(beg, it, cmp);
4523 VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
4524 }
4525 }
4526
4527 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
4528
4529 #endif // #ifndef VMA_SORT
4530
4531 /*
4532 Returns true if two memory blocks occupy overlapping pages.
4533 ResourceA must be in less memory offset than ResourceB.
4534
4535 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
4536 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
4537 */
4538 static inline bool VmaBlocksOnSamePage(
4539 VkDeviceSize resourceAOffset,
4540 VkDeviceSize resourceASize,
4541 VkDeviceSize resourceBOffset,
4542 VkDeviceSize pageSize)
4543 {
4544 VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
4545 VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
4546 VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
4547 VkDeviceSize resourceBStart = resourceBOffset;
4548 VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
4549 return resourceAEndPage == resourceBStartPage;
4550 }
4551
4552 enum VmaSuballocationType
4553 {
4554 VMA_SUBALLOCATION_TYPE_FREE = 0,
4555 VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
4556 VMA_SUBALLOCATION_TYPE_BUFFER = 2,
4557 VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
4558 VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
4559 VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
4560 VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
4561 };
4562
4563 /*
4564 Returns true if given suballocation types could conflict and must respect
4565 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
4566 or linear image and another one is optimal image. If type is unknown, behave
4567 conservatively.
4568 */
4569 static inline bool VmaIsBufferImageGranularityConflict(
4570 VmaSuballocationType suballocType1,
4571 VmaSuballocationType suballocType2)
4572 {
4573 if(suballocType1 > suballocType2)
4574 {
4575 VMA_SWAP(suballocType1, suballocType2);
4576 }
4577
4578 switch(suballocType1)
4579 {
4580 case VMA_SUBALLOCATION_TYPE_FREE:
4581 return false;
4582 case VMA_SUBALLOCATION_TYPE_UNKNOWN:
4583 return true;
4584 case VMA_SUBALLOCATION_TYPE_BUFFER:
4585 return
4586 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
4587 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
4588 case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
4589 return
4590 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
4591 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
4592 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
4593 case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
4594 return
4595 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
4596 case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
4597 return false;
4598 default:
4599 VMA_ASSERT(0);
4600 return true;
4601 }
4602 }
4603
4604 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
4605 {
4606 #if VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_DETECT_CORRUPTION
4607 uint32_t* pDst = (uint32_t*)((char*)pData + offset);
4608 const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
4609 for(size_t i = 0; i < numberCount; ++i, ++pDst)
4610 {
4611 *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
4612 }
4613 #else
4614 // no-op
4615 #endif
4616 }
4617
4618 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
4619 {
4620 #if VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_DETECT_CORRUPTION
4621 const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
4622 const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
4623 for(size_t i = 0; i < numberCount; ++i, ++pSrc)
4624 {
4625 if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
4626 {
4627 return false;
4628 }
4629 }
4630 #endif
4631 return true;
4632 }
4633
4634 /*
4635 Fills structure with parameters of an example buffer to be used for transfers
4636 during GPU memory defragmentation.
4637 */
4638 static void VmaFillGpuDefragmentationBufferCreateInfo(VkBufferCreateInfo& outBufCreateInfo)
4639 {
4640 memset(&outBufCreateInfo, 0, sizeof(outBufCreateInfo));
4641 outBufCreateInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
4642 outBufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4643 outBufCreateInfo.size = (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE; // Example size.
4644 }
4645
4646 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
4647 struct VmaMutexLock
4648 {
4649 VMA_CLASS_NO_COPY(VmaMutexLock)
4650 public:
4651 VmaMutexLock(VMA_MUTEX& mutex, bool useMutex = true) :
4652 m_pMutex(useMutex ? &mutex : VMA_NULL)
4653 { if(m_pMutex) { m_pMutex->Lock(); } }
4654 ~VmaMutexLock()
4655 { if(m_pMutex) { m_pMutex->Unlock(); } }
4656 private:
4657 VMA_MUTEX* m_pMutex;
4658 };
4659
4660 // Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for reading.
4661 struct VmaMutexLockRead
4662 {
4663 VMA_CLASS_NO_COPY(VmaMutexLockRead)
4664 public:
4665 VmaMutexLockRead(VMA_RW_MUTEX& mutex, bool useMutex) :
4666 m_pMutex(useMutex ? &mutex : VMA_NULL)
4667 { if(m_pMutex) { m_pMutex->LockRead(); } }
4668 ~VmaMutexLockRead() { if(m_pMutex) { m_pMutex->UnlockRead(); } }
4669 private:
4670 VMA_RW_MUTEX* m_pMutex;
4671 };
4672
4673 // Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for writing.
4674 struct VmaMutexLockWrite
4675 {
4676 VMA_CLASS_NO_COPY(VmaMutexLockWrite)
4677 public:
4678 VmaMutexLockWrite(VMA_RW_MUTEX& mutex, bool useMutex) :
4679 m_pMutex(useMutex ? &mutex : VMA_NULL)
4680 { if(m_pMutex) { m_pMutex->LockWrite(); } }
4681 ~VmaMutexLockWrite() { if(m_pMutex) { m_pMutex->UnlockWrite(); } }
4682 private:
4683 VMA_RW_MUTEX* m_pMutex;
4684 };
4685
4686 #if VMA_DEBUG_GLOBAL_MUTEX
4687 static VMA_MUTEX gDebugGlobalMutex;
4688 #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
4689 #else
4690 #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
4691 #endif
4692
4693 // Minimum size of a free suballocation to register it in the free suballocation collection.
4694 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
4695
4696 /*
4697 Performs binary search and returns iterator to first element that is greater or
4698 equal to (key), according to comparison (cmp).
4699
4700 Cmp should return true if first argument is less than second argument.
4701
4702 Returned value is the found element, if present in the collection or place where
4703 new element with value (key) should be inserted.
4704 */
4705 template <typename CmpLess, typename IterT, typename KeyT>
4706 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, const CmpLess& cmp)
4707 {
4708 size_t down = 0, up = (end - beg);
4709 while(down < up)
4710 {
4711 const size_t mid = (down + up) / 2;
4712 if(cmp(*(beg+mid), key))
4713 {
4714 down = mid + 1;
4715 }
4716 else
4717 {
4718 up = mid;
4719 }
4720 }
4721 return beg + down;
4722 }
4723
4724 template<typename CmpLess, typename IterT, typename KeyT>
4725 IterT VmaBinaryFindSorted(const IterT& beg, const IterT& end, const KeyT& value, const CmpLess& cmp)
4726 {
4727 IterT it = VmaBinaryFindFirstNotLess<CmpLess, IterT, KeyT>(
4728 beg, end, value, cmp);
4729 if(it == end ||
4730 (!cmp(*it, value) && !cmp(value, *it)))
4731 {
4732 return it;
4733 }
4734 return end;
4735 }
4736
4737 /*
4738 Returns true if all pointers in the array are not-null and unique.
4739 Warning! O(n^2) complexity. Use only inside VMA_HEAVY_ASSERT.
4740 T must be pointer type, e.g. VmaAllocation, VmaPool.
4741 */
4742 template<typename T>
4743 static bool VmaValidatePointerArray(uint32_t count, const T* arr)
4744 {
4745 for(uint32_t i = 0; i < count; ++i)
4746 {
4747 const T iPtr = arr[i];
4748 if(iPtr == VMA_NULL)
4749 {
4750 return false;
4751 }
4752 for(uint32_t j = i + 1; j < count; ++j)
4753 {
4754 if(iPtr == arr[j])
4755 {
4756 return false;
4757 }
4758 }
4759 }
4760 return true;
4761 }
4762
4763 template<typename MainT, typename NewT>
4764 static inline void VmaPnextChainPushFront(MainT* mainStruct, NewT* newStruct)
4765 {
4766 newStruct->pNext = mainStruct->pNext;
4767 mainStruct->pNext = newStruct;
4768 }
4769
4770 ////////////////////////////////////////////////////////////////////////////////
4771 // Memory allocation
4772
4773 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
4774 {
4775 void* result = VMA_NULL;
4776 if((pAllocationCallbacks != VMA_NULL) &&
4777 (pAllocationCallbacks->pfnAllocation != VMA_NULL))
4778 {
4779 result = (*pAllocationCallbacks->pfnAllocation)(
4780 pAllocationCallbacks->pUserData,
4781 size,
4782 alignment,
4783 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
4784 }
4785 else
4786 {
4787 result = VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
4788 }
4789 VMA_ASSERT(result != VMA_NULL && "CPU memory allocation failed.");
4790 return result;
4791 }
4792
4793 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
4794 {
4795 if((pAllocationCallbacks != VMA_NULL) &&
4796 (pAllocationCallbacks->pfnFree != VMA_NULL))
4797 {
4798 (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
4799 }
4800 else
4801 {
4802 VMA_SYSTEM_FREE(ptr);
4803 }
4804 }
4805
4806 template<typename T>
4807 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
4808 {
4809 return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
4810 }
4811
4812 template<typename T>
4813 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
4814 {
4815 return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
4816 }
4817
4818 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
4819
4820 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
4821
4822 template<typename T>
4823 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
4824 {
4825 ptr->~T();
4826 VmaFree(pAllocationCallbacks, ptr);
4827 }
4828
4829 template<typename T>
4830 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
4831 {
4832 if(ptr != VMA_NULL)
4833 {
4834 for(size_t i = count; i--; )
4835 {
4836 ptr[i].~T();
4837 }
4838 VmaFree(pAllocationCallbacks, ptr);
4839 }
4840 }
4841
4842 static char* VmaCreateStringCopy(const VkAllocationCallbacks* allocs, const char* srcStr)
4843 {
4844 if(srcStr != VMA_NULL)
4845 {
4846 const size_t len = strlen(srcStr);
4847 char* const result = vma_new_array(allocs, char, len + 1);
4848 memcpy(result, srcStr, len + 1);
4849 return result;
4850 }
4851 else
4852 {
4853 return VMA_NULL;
4854 }
4855 }
4856
4857 static void VmaFreeString(const VkAllocationCallbacks* allocs, char* str)
4858 {
4859 if(str != VMA_NULL)
4860 {
4861 const size_t len = strlen(str);
4862 vma_delete_array(allocs, str, len + 1);
4863 }
4864 }
4865
4866 // STL-compatible allocator.
4867 template<typename T>
4868 class VmaStlAllocator
4869 {
4870 public:
4871 const VkAllocationCallbacks* const m_pCallbacks;
4872 typedef T value_type;
4873
4874 VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
4875 template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
4876
4877 T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
4878 void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
4879
4880 template<typename U>
4881 bool operator==(const VmaStlAllocator<U>& rhs) const
4882 {
4883 return m_pCallbacks == rhs.m_pCallbacks;
4884 }
4885 template<typename U>
4886 bool operator!=(const VmaStlAllocator<U>& rhs) const
4887 {
4888 return m_pCallbacks != rhs.m_pCallbacks;
4889 }
4890
4891 VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
4892 };
4893
4894 #if VMA_USE_STL_VECTOR
4895
4896 #define VmaVector std::vector
4897
4898 template<typename T, typename allocatorT>
4899 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
4900 {
4901 vec.insert(vec.begin() + index, item);
4902 }
4903
4904 template<typename T, typename allocatorT>
4905 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
4906 {
4907 vec.erase(vec.begin() + index);
4908 }
4909
4910 #else // #if VMA_USE_STL_VECTOR
4911
4912 /* Class with interface compatible with subset of std::vector.
4913 T must be POD because constructors and destructors are not called and memcpy is
4914 used for these objects. */
4915 template<typename T, typename AllocatorT>
4916 class VmaVector
4917 {
4918 public:
4919 typedef T value_type;
4920
4921 VmaVector(const AllocatorT& allocator) :
4922 m_Allocator(allocator),
4923 m_pArray(VMA_NULL),
4924 m_Count(0),
4925 m_Capacity(0)
4926 {
4927 }
4928
4929 VmaVector(size_t count, const AllocatorT& allocator) :
4930 m_Allocator(allocator),
4931 m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
4932 m_Count(count),
4933 m_Capacity(count)
4934 {
4935 }
4936
4937 // This version of the constructor is here for compatibility with pre-C++14 std::vector.
4938 // value is unused.
4939 VmaVector(size_t count, const T& value, const AllocatorT& allocator)
4940 : VmaVector(count, allocator) {}
4941
4942 VmaVector(const VmaVector<T, AllocatorT>& src) :
4943 m_Allocator(src.m_Allocator),
4944 m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
4945 m_Count(src.m_Count),
4946 m_Capacity(src.m_Count)
4947 {
4948 if(m_Count != 0)
4949 {
4950 memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
4951 }
4952 }
4953
4954 ~VmaVector()
4955 {
4956 VmaFree(m_Allocator.m_pCallbacks, m_pArray);
4957 }
4958
4959 VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
4960 {
4961 if(&rhs != this)
4962 {
4963 resize(rhs.m_Count);
4964 if(m_Count != 0)
4965 {
4966 memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
4967 }
4968 }
4969 return *this;
4970 }
4971
4972 bool empty() const { return m_Count == 0; }
4973 size_t size() const { return m_Count; }
4974 T* data() { return m_pArray; }
4975 const T* data() const { return m_pArray; }
4976
4977 T& operator[](size_t index)
4978 {
4979 VMA_HEAVY_ASSERT(index < m_Count);
4980 return m_pArray[index];
4981 }
4982 const T& operator[](size_t index) const
4983 {
4984 VMA_HEAVY_ASSERT(index < m_Count);
4985 return m_pArray[index];
4986 }
4987
4988 T& front()
4989 {
4990 VMA_HEAVY_ASSERT(m_Count > 0);
4991 return m_pArray[0];
4992 }
4993 const T& front() const
4994 {
4995 VMA_HEAVY_ASSERT(m_Count > 0);
4996 return m_pArray[0];
4997 }
4998 T& back()
4999 {
5000 VMA_HEAVY_ASSERT(m_Count > 0);
5001 return m_pArray[m_Count - 1];
5002 }
5003 const T& back() const
5004 {
5005 VMA_HEAVY_ASSERT(m_Count > 0);
5006 return m_pArray[m_Count - 1];
5007 }
5008
5009 void reserve(size_t newCapacity, bool freeMemory = false)
5010 {
5011 newCapacity = VMA_MAX(newCapacity, m_Count);
5012
5013 if((newCapacity < m_Capacity) && !freeMemory)
5014 {
5015 newCapacity = m_Capacity;
5016 }
5017
5018 if(newCapacity != m_Capacity)
5019 {
5020 T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
5021 if(m_Count != 0)
5022 {
5023 memcpy(newArray, m_pArray, m_Count * sizeof(T));
5024 }
5025 VmaFree(m_Allocator.m_pCallbacks, m_pArray);
5026 m_Capacity = newCapacity;
5027 m_pArray = newArray;
5028 }
5029 }
5030
5031 void resize(size_t newCount, bool freeMemory = false)
5032 {
5033 size_t newCapacity = m_Capacity;
5034 if(newCount > m_Capacity)
5035 {
5036 newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
5037 }
5038 else if(freeMemory)
5039 {
5040 newCapacity = newCount;
5041 }
5042
5043 if(newCapacity != m_Capacity)
5044 {
5045 T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
5046 const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
5047 if(elementsToCopy != 0)
5048 {
5049 memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
5050 }
5051 VmaFree(m_Allocator.m_pCallbacks, m_pArray);
5052 m_Capacity = newCapacity;
5053 m_pArray = newArray;
5054 }
5055
5056 m_Count = newCount;
5057 }
5058
5059 void clear(bool freeMemory = false)
5060 {
5061 resize(0, freeMemory);
5062 }
5063
5064 void insert(size_t index, const T& src)
5065 {
5066 VMA_HEAVY_ASSERT(index <= m_Count);
5067 const size_t oldCount = size();
5068 resize(oldCount + 1);
5069 if(index < oldCount)
5070 {
5071 memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
5072 }
5073 m_pArray[index] = src;
5074 }
5075
5076 void remove(size_t index)
5077 {
5078 VMA_HEAVY_ASSERT(index < m_Count);
5079 const size_t oldCount = size();
5080 if(index < oldCount - 1)
5081 {
5082 memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
5083 }
5084 resize(oldCount - 1);
5085 }
5086
5087 void push_back(const T& src)
5088 {
5089 const size_t newIndex = size();
5090 resize(newIndex + 1);
5091 m_pArray[newIndex] = src;
5092 }
5093
5094 void pop_back()
5095 {
5096 VMA_HEAVY_ASSERT(m_Count > 0);
5097 resize(size() - 1);
5098 }
5099
5100 void push_front(const T& src)
5101 {
5102 insert(0, src);
5103 }
5104
5105 void pop_front()
5106 {
5107 VMA_HEAVY_ASSERT(m_Count > 0);
5108 remove(0);
5109 }
5110
5111 typedef T* iterator;
5112
5113 iterator begin() { return m_pArray; }
5114 iterator end() { return m_pArray + m_Count; }
5115
5116 private:
5117 AllocatorT m_Allocator;
5118 T* m_pArray;
5119 size_t m_Count;
5120 size_t m_Capacity;
5121 };
5122
5123 template<typename T, typename allocatorT>
5124 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
5125 {
5126 vec.insert(index, item);
5127 }
5128
5129 template<typename T, typename allocatorT>
5130 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
5131 {
5132 vec.remove(index);
5133 }
5134
5135 #endif // #if VMA_USE_STL_VECTOR
5136
5137 template<typename CmpLess, typename VectorT>
5138 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
5139 {
5140 const size_t indexToInsert = VmaBinaryFindFirstNotLess(
5141 vector.data(),
5142 vector.data() + vector.size(),
5143 value,
5144 CmpLess()) - vector.data();
5145 VmaVectorInsert(vector, indexToInsert, value);
5146 return indexToInsert;
5147 }
5148
5149 template<typename CmpLess, typename VectorT>
5150 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
5151 {
5152 CmpLess comparator;
5153 typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
5154 vector.begin(),
5155 vector.end(),
5156 value,
5157 comparator);
5158 if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
5159 {
5160 size_t indexToRemove = it - vector.begin();
5161 VmaVectorRemove(vector, indexToRemove);
5162 return true;
5163 }
5164 return false;
5165 }
5166
5167 ////////////////////////////////////////////////////////////////////////////////
5168 // class VmaSmallVector
5169
5170 /*
5171 This is a vector (a variable-sized array), optimized for the case when the array is small.
5172
5173 It contains some number of elements in-place, which allows it to avoid heap allocation
5174 when the actual number of elements is below that threshold. This allows normal "small"
5175 cases to be fast without losing generality for large inputs.
5176 */
5177
5178 template<typename T, typename AllocatorT, size_t N>
5179 class VmaSmallVector
5180 {
5181 public:
5182 typedef T value_type;
5183
5184 VmaSmallVector(const AllocatorT& allocator) :
5185 m_Count(0),
5186 m_DynamicArray(allocator)
5187 {
5188 }
5189 VmaSmallVector(size_t count, const AllocatorT& allocator) :
5190 m_Count(count),
5191 m_DynamicArray(count > N ? count : 0, allocator)
5192 {
5193 }
5194 template<typename SrcT, typename SrcAllocatorT, size_t SrcN>
5195 VmaSmallVector(const VmaSmallVector<SrcT, SrcAllocatorT, SrcN>& src) = delete;
5196 template<typename SrcT, typename SrcAllocatorT, size_t SrcN>
5197 VmaSmallVector<T, AllocatorT, N>& operator=(const VmaSmallVector<SrcT, SrcAllocatorT, SrcN>& rhs) = delete;
5198
5199 bool empty() const { return m_Count == 0; }
5200 size_t size() const { return m_Count; }
5201 T* data() { return m_Count > N ? m_DynamicArray.data() : m_StaticArray; }
5202 const T* data() const { return m_Count > N ? m_DynamicArray.data() : m_StaticArray; }
5203
5204 T& operator[](size_t index)
5205 {
5206 VMA_HEAVY_ASSERT(index < m_Count);
5207 return data()[index];
5208 }
5209 const T& operator[](size_t index) const
5210 {
5211 VMA_HEAVY_ASSERT(index < m_Count);
5212 return data()[index];
5213 }
5214
5215 T& front()
5216 {
5217 VMA_HEAVY_ASSERT(m_Count > 0);
5218 return data()[0];
5219 }
5220 const T& front() const
5221 {
5222 VMA_HEAVY_ASSERT(m_Count > 0);
5223 return data()[0];
5224 }
5225 T& back()
5226 {
5227 VMA_HEAVY_ASSERT(m_Count > 0);
5228 return data()[m_Count - 1];
5229 }
5230 const T& back() const
5231 {
5232 VMA_HEAVY_ASSERT(m_Count > 0);
5233 return data()[m_Count - 1];
5234 }
5235
5236 void resize(size_t newCount, bool freeMemory = false)
5237 {
5238 if(newCount > N && m_Count > N)
5239 {
5240 // Any direction, staying in m_DynamicArray
5241 m_DynamicArray.resize(newCount, freeMemory);
5242 }
5243 else if(newCount > N && m_Count <= N)
5244 {
5245 // Growing, moving from m_StaticArray to m_DynamicArray
5246 m_DynamicArray.resize(newCount, freeMemory);
5247 if(m_Count > 0)
5248 {
5249 memcpy(m_DynamicArray.data(), m_StaticArray, m_Count * sizeof(T));
5250 }
5251 }
5252 else if(newCount <= N && m_Count > N)
5253 {
5254 // Shrinking, moving from m_DynamicArray to m_StaticArray
5255 if(newCount > 0)
5256 {
5257 memcpy(m_StaticArray, m_DynamicArray.data(), newCount * sizeof(T));
5258 }
5259 m_DynamicArray.resize(0, freeMemory);
5260 }
5261 else
5262 {
5263 // Any direction, staying in m_StaticArray - nothing to do here
5264 }
5265 m_Count = newCount;
5266 }
5267
5268 void clear(bool freeMemory = false)
5269 {
5270 m_DynamicArray.clear(freeMemory);
5271 m_Count = 0;
5272 }
5273
5274 void insert(size_t index, const T& src)
5275 {
5276 VMA_HEAVY_ASSERT(index <= m_Count);
5277 const size_t oldCount = size();
5278 resize(oldCount + 1);
5279 T* const dataPtr = data();
5280 if(index < oldCount)
5281 {
5282 // I know, this could be more optimal for case where memmove can be memcpy directly from m_StaticArray to m_DynamicArray.
5283 memmove(dataPtr + (index + 1), dataPtr + index, (oldCount - index) * sizeof(T));
5284 }
5285 dataPtr[index] = src;
5286 }
5287
5288 void remove(size_t index)
5289 {
5290 VMA_HEAVY_ASSERT(index < m_Count);
5291 const size_t oldCount = size();
5292 if(index < oldCount - 1)
5293 {
5294 // I know, this could be more optimal for case where memmove can be memcpy directly from m_DynamicArray to m_StaticArray.
5295 T* const dataPtr = data();
5296 memmove(dataPtr + index, dataPtr + (index + 1), (oldCount - index - 1) * sizeof(T));
5297 }
5298 resize(oldCount - 1);
5299 }
5300
5301 void push_back(const T& src)
5302 {
5303 const size_t newIndex = size();
5304 resize(newIndex + 1);
5305 data()[newIndex] = src;
5306 }
5307
5308 void pop_back()
5309 {
5310 VMA_HEAVY_ASSERT(m_Count > 0);
5311 resize(size() - 1);
5312 }
5313
5314 void push_front(const T& src)
5315 {
5316 insert(0, src);
5317 }
5318
5319 void pop_front()
5320 {
5321 VMA_HEAVY_ASSERT(m_Count > 0);
5322 remove(0);
5323 }
5324
5325 typedef T* iterator;
5326
5327 iterator begin() { return data(); }
5328 iterator end() { return data() + m_Count; }
5329
5330 private:
5331 size_t m_Count;
5332 T m_StaticArray[N]; // Used when m_Size <= N
5333 VmaVector<T, AllocatorT> m_DynamicArray; // Used when m_Size > N
5334 };
5335
5336 ////////////////////////////////////////////////////////////////////////////////
5337 // class VmaPoolAllocator
5338
5339 /*
5340 Allocator for objects of type T using a list of arrays (pools) to speed up
5341 allocation. Number of elements that can be allocated is not bounded because
5342 allocator can create multiple blocks.
5343 */
5344 template<typename T>
5345 class VmaPoolAllocator
5346 {
5347 VMA_CLASS_NO_COPY(VmaPoolAllocator)
5348 public:
5349 VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, uint32_t firstBlockCapacity);
5350 ~VmaPoolAllocator();
5351 template<typename... Types> T* Alloc(Types... args);
5352 void Free(T* ptr);
5353
5354 private:
5355 union Item
5356 {
5357 uint32_t NextFreeIndex;
5358 alignas(T) char Value[sizeof(T)];
5359 };
5360
5361 struct ItemBlock
5362 {
5363 Item* pItems;
5364 uint32_t Capacity;
5365 uint32_t FirstFreeIndex;
5366 };
5367
5368 const VkAllocationCallbacks* m_pAllocationCallbacks;
5369 const uint32_t m_FirstBlockCapacity;
5370 VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
5371
5372 ItemBlock& CreateNewBlock();
5373 };
5374
5375 template<typename T>
5376 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, uint32_t firstBlockCapacity) :
5377 m_pAllocationCallbacks(pAllocationCallbacks),
5378 m_FirstBlockCapacity(firstBlockCapacity),
5379 m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
5380 {
5381 VMA_ASSERT(m_FirstBlockCapacity > 1);
5382 }
5383
5384 template<typename T>
5385 VmaPoolAllocator<T>::~VmaPoolAllocator()
5386 {
5387 for(size_t i = m_ItemBlocks.size(); i--; )
5388 vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemBlocks[i].Capacity);
5389 m_ItemBlocks.clear();
5390 }
5391
5392 template<typename T>
5393 template<typename... Types> T* VmaPoolAllocator<T>::Alloc(Types... args)
5394 {
5395 for(size_t i = m_ItemBlocks.size(); i--; )
5396 {
5397 ItemBlock& block = m_ItemBlocks[i];
5398 // This block has some free items: Use first one.
5399 if(block.FirstFreeIndex != UINT32_MAX)
5400 {
5401 Item* const pItem = &block.pItems[block.FirstFreeIndex];
5402 block.FirstFreeIndex = pItem->NextFreeIndex;
5403 T* result = (T*)&pItem->Value;
5404 new(result)T(std::forward<Types>(args)...); // Explicit constructor call.
5405 return result;
5406 }
5407 }
5408
5409 // No block has free item: Create new one and use it.
5410 ItemBlock& newBlock = CreateNewBlock();
5411 Item* const pItem = &newBlock.pItems[0];
5412 newBlock.FirstFreeIndex = pItem->NextFreeIndex;
5413 T* result = (T*)&pItem->Value;
5414 new(result)T(std::forward<Types>(args)...); // Explicit constructor call.
5415 return result;
5416 }
5417
5418 template<typename T>
5419 void VmaPoolAllocator<T>::Free(T* ptr)
5420 {
5421 // Search all memory blocks to find ptr.
5422 for(size_t i = m_ItemBlocks.size(); i--; )
5423 {
5424 ItemBlock& block = m_ItemBlocks[i];
5425
5426 // Casting to union.
5427 Item* pItemPtr;
5428 memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
5429
5430 // Check if pItemPtr is in address range of this block.
5431 if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + block.Capacity))
5432 {
5433 ptr->~T(); // Explicit destructor call.
5434 const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
5435 pItemPtr->NextFreeIndex = block.FirstFreeIndex;
5436 block.FirstFreeIndex = index;
5437 return;
5438 }
5439 }
5440 VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
5441 }
5442
5443 template<typename T>
5444 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
5445 {
5446 const uint32_t newBlockCapacity = m_ItemBlocks.empty() ?
5447 m_FirstBlockCapacity : m_ItemBlocks.back().Capacity * 3 / 2;
5448
5449 const ItemBlock newBlock = {
5450 vma_new_array(m_pAllocationCallbacks, Item, newBlockCapacity),
5451 newBlockCapacity,
5452 0 };
5453
5454 m_ItemBlocks.push_back(newBlock);
5455
5456 // Setup singly-linked list of all free items in this block.
5457 for(uint32_t i = 0; i < newBlockCapacity - 1; ++i)
5458 newBlock.pItems[i].NextFreeIndex = i + 1;
5459 newBlock.pItems[newBlockCapacity - 1].NextFreeIndex = UINT32_MAX;
5460 return m_ItemBlocks.back();
5461 }
5462
5463 ////////////////////////////////////////////////////////////////////////////////
5464 // class VmaRawList, VmaList
5465
5466 #if VMA_USE_STL_LIST
5467
5468 #define VmaList std::list
5469
5470 #else // #if VMA_USE_STL_LIST
5471
5472 template<typename T>
5473 struct VmaListItem
5474 {
5475 VmaListItem* pPrev;
5476 VmaListItem* pNext;
5477 T Value;
5478 };
5479
5480 // Doubly linked list.
5481 template<typename T>
5482 class VmaRawList
5483 {
5484 VMA_CLASS_NO_COPY(VmaRawList)
5485 public:
5486 typedef VmaListItem<T> ItemType;
5487
5488 VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
5489 ~VmaRawList();
5490 void Clear();
5491
5492 size_t GetCount() const { return m_Count; }
5493 bool IsEmpty() const { return m_Count == 0; }
5494
5495 ItemType* Front() { return m_pFront; }
5496 const ItemType* Front() const { return m_pFront; }
5497 ItemType* Back() { return m_pBack; }
5498 const ItemType* Back() const { return m_pBack; }
5499
5500 ItemType* PushBack();
5501 ItemType* PushFront();
5502 ItemType* PushBack(const T& value);
5503 ItemType* PushFront(const T& value);
5504 void PopBack();
5505 void PopFront();
5506
5507 // Item can be null - it means PushBack.
5508 ItemType* InsertBefore(ItemType* pItem);
5509 // Item can be null - it means PushFront.
5510 ItemType* InsertAfter(ItemType* pItem);
5511
5512 ItemType* InsertBefore(ItemType* pItem, const T& value);
5513 ItemType* InsertAfter(ItemType* pItem, const T& value);
5514
5515 void Remove(ItemType* pItem);
5516
5517 private:
5518 const VkAllocationCallbacks* const m_pAllocationCallbacks;
5519 VmaPoolAllocator<ItemType> m_ItemAllocator;
5520 ItemType* m_pFront;
5521 ItemType* m_pBack;
5522 size_t m_Count;
5523 };
5524
5525 template<typename T>
5526 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
5527 m_pAllocationCallbacks(pAllocationCallbacks),
5528 m_ItemAllocator(pAllocationCallbacks, 128),
5529 m_pFront(VMA_NULL),
5530 m_pBack(VMA_NULL),
5531 m_Count(0)
5532 {
5533 }
5534
5535 template<typename T>
5536 VmaRawList<T>::~VmaRawList()
5537 {
5538 // Intentionally not calling Clear, because that would be unnecessary
5539 // computations to return all items to m_ItemAllocator as free.
5540 }
5541
5542 template<typename T>
5543 void VmaRawList<T>::Clear()
5544 {
5545 if(IsEmpty() == false)
5546 {
5547 ItemType* pItem = m_pBack;
5548 while(pItem != VMA_NULL)
5549 {
5550 ItemType* const pPrevItem = pItem->pPrev;
5551 m_ItemAllocator.Free(pItem);
5552 pItem = pPrevItem;
5553 }
5554 m_pFront = VMA_NULL;
5555 m_pBack = VMA_NULL;
5556 m_Count = 0;
5557 }
5558 }
5559
5560 template<typename T>
5561 VmaListItem<T>* VmaRawList<T>::PushBack()
5562 {
5563 ItemType* const pNewItem = m_ItemAllocator.Alloc();
5564 pNewItem->pNext = VMA_NULL;
5565 if(IsEmpty())
5566 {
5567 pNewItem->pPrev = VMA_NULL;
5568 m_pFront = pNewItem;
5569 m_pBack = pNewItem;
5570 m_Count = 1;
5571 }
5572 else
5573 {
5574 pNewItem->pPrev = m_pBack;
5575 m_pBack->pNext = pNewItem;
5576 m_pBack = pNewItem;
5577 ++m_Count;
5578 }
5579 return pNewItem;
5580 }
5581
5582 template<typename T>
5583 VmaListItem<T>* VmaRawList<T>::PushFront()
5584 {
5585 ItemType* const pNewItem = m_ItemAllocator.Alloc();
5586 pNewItem->pPrev = VMA_NULL;
5587 if(IsEmpty())
5588 {
5589 pNewItem->pNext = VMA_NULL;
5590 m_pFront = pNewItem;
5591 m_pBack = pNewItem;
5592 m_Count = 1;
5593 }
5594 else
5595 {
5596 pNewItem->pNext = m_pFront;
5597 m_pFront->pPrev = pNewItem;
5598 m_pFront = pNewItem;
5599 ++m_Count;
5600 }
5601 return pNewItem;
5602 }
5603
5604 template<typename T>
5605 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
5606 {
5607 ItemType* const pNewItem = PushBack();
5608 pNewItem->Value = value;
5609 return pNewItem;
5610 }
5611
5612 template<typename T>
5613 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
5614 {
5615 ItemType* const pNewItem = PushFront();
5616 pNewItem->Value = value;
5617 return pNewItem;
5618 }
5619
5620 template<typename T>
5621 void VmaRawList<T>::PopBack()
5622 {
5623 VMA_HEAVY_ASSERT(m_Count > 0);
5624 ItemType* const pBackItem = m_pBack;
5625 ItemType* const pPrevItem = pBackItem->pPrev;
5626 if(pPrevItem != VMA_NULL)
5627 {
5628 pPrevItem->pNext = VMA_NULL;
5629 }
5630 m_pBack = pPrevItem;
5631 m_ItemAllocator.Free(pBackItem);
5632 --m_Count;
5633 }
5634
5635 template<typename T>
5636 void VmaRawList<T>::PopFront()
5637 {
5638 VMA_HEAVY_ASSERT(m_Count > 0);
5639 ItemType* const pFrontItem = m_pFront;
5640 ItemType* const pNextItem = pFrontItem->pNext;
5641 if(pNextItem != VMA_NULL)
5642 {
5643 pNextItem->pPrev = VMA_NULL;
5644 }
5645 m_pFront = pNextItem;
5646 m_ItemAllocator.Free(pFrontItem);
5647 --m_Count;
5648 }
5649
5650 template<typename T>
5651 void VmaRawList<T>::Remove(ItemType* pItem)
5652 {
5653 VMA_HEAVY_ASSERT(pItem != VMA_NULL);
5654 VMA_HEAVY_ASSERT(m_Count > 0);
5655
5656 if(pItem->pPrev != VMA_NULL)
5657 {
5658 pItem->pPrev->pNext = pItem->pNext;
5659 }
5660 else
5661 {
5662 VMA_HEAVY_ASSERT(m_pFront == pItem);
5663 m_pFront = pItem->pNext;
5664 }
5665
5666 if(pItem->pNext != VMA_NULL)
5667 {
5668 pItem->pNext->pPrev = pItem->pPrev;
5669 }
5670 else
5671 {
5672 VMA_HEAVY_ASSERT(m_pBack == pItem);
5673 m_pBack = pItem->pPrev;
5674 }
5675
5676 m_ItemAllocator.Free(pItem);
5677 --m_Count;
5678 }
5679
5680 template<typename T>
5681 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
5682 {
5683 if(pItem != VMA_NULL)
5684 {
5685 ItemType* const prevItem = pItem->pPrev;
5686 ItemType* const newItem = m_ItemAllocator.Alloc();
5687 newItem->pPrev = prevItem;
5688 newItem->pNext = pItem;
5689 pItem->pPrev = newItem;
5690 if(prevItem != VMA_NULL)
5691 {
5692 prevItem->pNext = newItem;
5693 }
5694 else
5695 {
5696 VMA_HEAVY_ASSERT(m_pFront == pItem);
5697 m_pFront = newItem;
5698 }
5699 ++m_Count;
5700 return newItem;
5701 }
5702 else
5703 return PushBack();
5704 }
5705
5706 template<typename T>
5707 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
5708 {
5709 if(pItem != VMA_NULL)
5710 {
5711 ItemType* const nextItem = pItem->pNext;
5712 ItemType* const newItem = m_ItemAllocator.Alloc();
5713 newItem->pNext = nextItem;
5714 newItem->pPrev = pItem;
5715 pItem->pNext = newItem;
5716 if(nextItem != VMA_NULL)
5717 {
5718 nextItem->pPrev = newItem;
5719 }
5720 else
5721 {
5722 VMA_HEAVY_ASSERT(m_pBack == pItem);
5723 m_pBack = newItem;
5724 }
5725 ++m_Count;
5726 return newItem;
5727 }
5728 else
5729 return PushFront();
5730 }
5731
5732 template<typename T>
5733 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
5734 {
5735 ItemType* const newItem = InsertBefore(pItem);
5736 newItem->Value = value;
5737 return newItem;
5738 }
5739
5740 template<typename T>
5741 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
5742 {
5743 ItemType* const newItem = InsertAfter(pItem);
5744 newItem->Value = value;
5745 return newItem;
5746 }
5747
5748 template<typename T, typename AllocatorT>
5749 class VmaList
5750 {
5751 VMA_CLASS_NO_COPY(VmaList)
5752 public:
5753 class iterator
5754 {
5755 public:
5756 iterator() :
5757 m_pList(VMA_NULL),
5758 m_pItem(VMA_NULL)
5759 {
5760 }
5761
5762 T& operator*() const
5763 {
5764 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
5765 return m_pItem->Value;
5766 }
5767 T* operator->() const
5768 {
5769 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
5770 return &m_pItem->Value;
5771 }
5772
5773 iterator& operator++()
5774 {
5775 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
5776 m_pItem = m_pItem->pNext;
5777 return *this;
5778 }
5779 iterator& operator--()
5780 {
5781 if(m_pItem != VMA_NULL)
5782 {
5783 m_pItem = m_pItem->pPrev;
5784 }
5785 else
5786 {
5787 VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
5788 m_pItem = m_pList->Back();
5789 }
5790 return *this;
5791 }
5792
5793 iterator operator++(int)
5794 {
5795 iterator result = *this;
5796 ++*this;
5797 return result;
5798 }
5799 iterator operator--(int)
5800 {
5801 iterator result = *this;
5802 --*this;
5803 return result;
5804 }
5805
5806 bool operator==(const iterator& rhs) const
5807 {
5808 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
5809 return m_pItem == rhs.m_pItem;
5810 }
5811 bool operator!=(const iterator& rhs) const
5812 {
5813 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
5814 return m_pItem != rhs.m_pItem;
5815 }
5816
5817 private:
5818 VmaRawList<T>* m_pList;
5819 VmaListItem<T>* m_pItem;
5820
5821 iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
5822 m_pList(pList),
5823 m_pItem(pItem)
5824 {
5825 }
5826
5827 friend class VmaList<T, AllocatorT>;
5828 };
5829
5830 class const_iterator
5831 {
5832 public:
5833 const_iterator() :
5834 m_pList(VMA_NULL),
5835 m_pItem(VMA_NULL)
5836 {
5837 }
5838
5839 const_iterator(const iterator& src) :
5840 m_pList(src.m_pList),
5841 m_pItem(src.m_pItem)
5842 {
5843 }
5844
5845 const T& operator*() const
5846 {
5847 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
5848 return m_pItem->Value;
5849 }
5850 const T* operator->() const
5851 {
5852 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
5853 return &m_pItem->Value;
5854 }
5855
5856 const_iterator& operator++()
5857 {
5858 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
5859 m_pItem = m_pItem->pNext;
5860 return *this;
5861 }
5862 const_iterator& operator--()
5863 {
5864 if(m_pItem != VMA_NULL)
5865 {
5866 m_pItem = m_pItem->pPrev;
5867 }
5868 else
5869 {
5870 VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
5871 m_pItem = m_pList->Back();
5872 }
5873 return *this;
5874 }
5875
5876 const_iterator operator++(int)
5877 {
5878 const_iterator result = *this;
5879 ++*this;
5880 return result;
5881 }
5882 const_iterator operator--(int)
5883 {
5884 const_iterator result = *this;
5885 --*this;
5886 return result;
5887 }
5888
5889 bool operator==(const const_iterator& rhs) const
5890 {
5891 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
5892 return m_pItem == rhs.m_pItem;
5893 }
5894 bool operator!=(const const_iterator& rhs) const
5895 {
5896 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
5897 return m_pItem != rhs.m_pItem;
5898 }
5899
5900 private:
5901 const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
5902 m_pList(pList),
5903 m_pItem(pItem)
5904 {
5905 }
5906
5907 const VmaRawList<T>* m_pList;
5908 const VmaListItem<T>* m_pItem;
5909
5910 friend class VmaList<T, AllocatorT>;
5911 };
5912
5913 VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
5914
5915 bool empty() const { return m_RawList.IsEmpty(); }
5916 size_t size() const { return m_RawList.GetCount(); }
5917
5918 iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
5919 iterator end() { return iterator(&m_RawList, VMA_NULL); }
5920
5921 const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
5922 const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
5923
5924 void clear() { m_RawList.Clear(); }
5925 void push_back(const T& value) { m_RawList.PushBack(value); }
5926 void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
5927 iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
5928
5929 private:
5930 VmaRawList<T> m_RawList;
5931 };
5932
5933 #endif // #if VMA_USE_STL_LIST
5934
5935 ////////////////////////////////////////////////////////////////////////////////
5936 // class VmaMap
5937
5938 // Unused in this version.
5939 #if 0
5940
5941 #if VMA_USE_STL_UNORDERED_MAP
5942
5943 #define VmaPair std::pair
5944
5945 #define VMA_MAP_TYPE(KeyT, ValueT) \
5946 std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
5947
5948 #else // #if VMA_USE_STL_UNORDERED_MAP
5949
5950 template<typename T1, typename T2>
5951 struct VmaPair
5952 {
5953 T1 first;
5954 T2 second;
5955
5956 VmaPair() : first(), second() { }
5957 VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
5958 };
5959
5960 /* Class compatible with subset of interface of std::unordered_map.
5961 KeyT, ValueT must be POD because they will be stored in VmaVector.
5962 */
5963 template<typename KeyT, typename ValueT>
5964 class VmaMap
5965 {
5966 public:
5967 typedef VmaPair<KeyT, ValueT> PairType;
5968 typedef PairType* iterator;
5969
5970 VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
5971
5972 iterator begin() { return m_Vector.begin(); }
5973 iterator end() { return m_Vector.end(); }
5974
5975 void insert(const PairType& pair);
5976 iterator find(const KeyT& key);
5977 void erase(iterator it);
5978
5979 private:
5980 VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
5981 };
5982
5983 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
5984
5985 template<typename FirstT, typename SecondT>
5986 struct VmaPairFirstLess
5987 {
5988 bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
5989 {
5990 return lhs.first < rhs.first;
5991 }
5992 bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
5993 {
5994 return lhs.first < rhsFirst;
5995 }
5996 };
5997
5998 template<typename KeyT, typename ValueT>
5999 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
6000 {
6001 const size_t indexToInsert = VmaBinaryFindFirstNotLess(
6002 m_Vector.data(),
6003 m_Vector.data() + m_Vector.size(),
6004 pair,
6005 VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
6006 VmaVectorInsert(m_Vector, indexToInsert, pair);
6007 }
6008
6009 template<typename KeyT, typename ValueT>
6010 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
6011 {
6012 PairType* it = VmaBinaryFindFirstNotLess(
6013 m_Vector.data(),
6014 m_Vector.data() + m_Vector.size(),
6015 key,
6016 VmaPairFirstLess<KeyT, ValueT>());
6017 if((it != m_Vector.end()) && (it->first == key))
6018 {
6019 return it;
6020 }
6021 else
6022 {
6023 return m_Vector.end();
6024 }
6025 }
6026
6027 template<typename KeyT, typename ValueT>
6028 void VmaMap<KeyT, ValueT>::erase(iterator it)
6029 {
6030 VmaVectorRemove(m_Vector, it - m_Vector.begin());
6031 }
6032
6033 #endif // #if VMA_USE_STL_UNORDERED_MAP
6034
6035 #endif // #if 0
6036
6037 ////////////////////////////////////////////////////////////////////////////////
6038
6039 class VmaDeviceMemoryBlock;
6040
6041 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
6042
6043 struct VmaAllocation_T
6044 {
6045 private:
6046 static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
6047
6048 enum FLAGS
6049 {
6050 FLAG_USER_DATA_STRING = 0x01,
6051 };
6052
6053 public:
6054 enum ALLOCATION_TYPE
6055 {
6056 ALLOCATION_TYPE_NONE,
6057 ALLOCATION_TYPE_BLOCK,
6058 ALLOCATION_TYPE_DEDICATED,
6059 };
6060
6061 /*
6062 This struct is allocated using VmaPoolAllocator.
6063 */
6064
6065 VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
6066 m_Alignment{1},
6067 m_Size{0},
6068 m_pUserData{VMA_NULL},
6069 m_LastUseFrameIndex{currentFrameIndex},
6070 m_MemoryTypeIndex{0},
6071 m_Type{(uint8_t)ALLOCATION_TYPE_NONE},
6072 m_SuballocationType{(uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN},
6073 m_MapCount{0},
6074 m_Flags{userDataString ? (uint8_t)FLAG_USER_DATA_STRING : (uint8_t)0}
6075 {
6076 #if VMA_STATS_STRING_ENABLED
6077 m_CreationFrameIndex = currentFrameIndex;
6078 m_BufferImageUsage = 0;
6079 #endif
6080 }
6081
6082 ~VmaAllocation_T()
6083 {
6084 VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
6085
6086 // Check if owned string was freed.
6087 VMA_ASSERT(m_pUserData == VMA_NULL);
6088 }
6089
6090 void InitBlockAllocation(
6091 VmaDeviceMemoryBlock* block,
6092 VkDeviceSize offset,
6093 VkDeviceSize alignment,
6094 VkDeviceSize size,
6095 uint32_t memoryTypeIndex,
6096 VmaSuballocationType suballocationType,
6097 bool mapped,
6098 bool canBecomeLost)
6099 {
6100 VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
6101 VMA_ASSERT(block != VMA_NULL);
6102 m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
6103 m_Alignment = alignment;
6104 m_Size = size;
6105 m_MemoryTypeIndex = memoryTypeIndex;
6106 m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
6107 m_SuballocationType = (uint8_t)suballocationType;
6108 m_BlockAllocation.m_Block = block;
6109 m_BlockAllocation.m_Offset = offset;
6110 m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
6111 }
6112
6113 void InitLost()
6114 {
6115 VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
6116 VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
6117 m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
6118 m_MemoryTypeIndex = 0;
6119 m_BlockAllocation.m_Block = VMA_NULL;
6120 m_BlockAllocation.m_Offset = 0;
6121 m_BlockAllocation.m_CanBecomeLost = true;
6122 }
6123
6124 void ChangeBlockAllocation(
6125 VmaAllocator hAllocator,
6126 VmaDeviceMemoryBlock* block,
6127 VkDeviceSize offset);
6128
6129 void ChangeOffset(VkDeviceSize newOffset);
6130
6131 // pMappedData not null means allocation is created with MAPPED flag.
6132 void InitDedicatedAllocation(
6133 uint32_t memoryTypeIndex,
6134 VkDeviceMemory hMemory,
6135 VmaSuballocationType suballocationType,
6136 void* pMappedData,
6137 VkDeviceSize size)
6138 {
6139 VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
6140 VMA_ASSERT(hMemory != VK_NULL_HANDLE);
6141 m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
6142 m_Alignment = 0;
6143 m_Size = size;
6144 m_MemoryTypeIndex = memoryTypeIndex;
6145 m_SuballocationType = (uint8_t)suballocationType;
6146 m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
6147 m_DedicatedAllocation.m_hMemory = hMemory;
6148 m_DedicatedAllocation.m_pMappedData = pMappedData;
6149 }
6150
6151 ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
6152 VkDeviceSize GetAlignment() const { return m_Alignment; }
6153 VkDeviceSize GetSize() const { return m_Size; }
6154 bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
6155 void* GetUserData() const { return m_pUserData; }
6156 void SetUserData(VmaAllocator hAllocator, void* pUserData);
6157 VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
6158
6159 VmaDeviceMemoryBlock* GetBlock() const
6160 {
6161 VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
6162 return m_BlockAllocation.m_Block;
6163 }
6164 VkDeviceSize GetOffset() const;
6165 VkDeviceMemory GetMemory() const;
6166 uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
6167 bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
6168 void* GetMappedData() const;
6169 bool CanBecomeLost() const;
6170
6171 uint32_t GetLastUseFrameIndex() const
6172 {
6173 return m_LastUseFrameIndex.load();
6174 }
6175 bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
6176 {
6177 return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
6178 }
6179 /*
6180 - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
6181 makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
6182 - Else, returns false.
6183
6184 If hAllocation is already lost, assert - you should not call it then.
6185 If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
6186 */
6187 bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
6188
6189 void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
6190 {
6191 VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
6192 outInfo.blockCount = 1;
6193 outInfo.allocationCount = 1;
6194 outInfo.unusedRangeCount = 0;
6195 outInfo.usedBytes = m_Size;
6196 outInfo.unusedBytes = 0;
6197 outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
6198 outInfo.unusedRangeSizeMin = UINT64_MAX;
6199 outInfo.unusedRangeSizeMax = 0;
6200 }
6201
6202 void BlockAllocMap();
6203 void BlockAllocUnmap();
6204 VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
6205 void DedicatedAllocUnmap(VmaAllocator hAllocator);
6206
6207 #if VMA_STATS_STRING_ENABLED
6208 uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
6209 uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
6210
6211 void InitBufferImageUsage(uint32_t bufferImageUsage)
6212 {
6213 VMA_ASSERT(m_BufferImageUsage == 0);
6214 m_BufferImageUsage = bufferImageUsage;
6215 }
6216
6217 void PrintParameters(class VmaJsonWriter& json) const;
6218 #endif
6219
6220 private:
6221 VkDeviceSize m_Alignment;
6222 VkDeviceSize m_Size;
6223 void* m_pUserData;
6224 VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
6225 uint32_t m_MemoryTypeIndex;
6226 uint8_t m_Type; // ALLOCATION_TYPE
6227 uint8_t m_SuballocationType; // VmaSuballocationType
6228 // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
6229 // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
6230 uint8_t m_MapCount;
6231 uint8_t m_Flags; // enum FLAGS
6232
6233 // Allocation out of VmaDeviceMemoryBlock.
6234 struct BlockAllocation
6235 {
6236 VmaDeviceMemoryBlock* m_Block;
6237 VkDeviceSize m_Offset;
6238 bool m_CanBecomeLost;
6239 };
6240
6241 // Allocation for an object that has its own private VkDeviceMemory.
6242 struct DedicatedAllocation
6243 {
6244 VkDeviceMemory m_hMemory;
6245 void* m_pMappedData; // Not null means memory is mapped.
6246 };
6247
6248 union
6249 {
6250 // Allocation out of VmaDeviceMemoryBlock.
6251 BlockAllocation m_BlockAllocation;
6252 // Allocation for an object that has its own private VkDeviceMemory.
6253 DedicatedAllocation m_DedicatedAllocation;
6254 };
6255
6256 #if VMA_STATS_STRING_ENABLED
6257 uint32_t m_CreationFrameIndex;
6258 uint32_t m_BufferImageUsage; // 0 if unknown.
6259 #endif
6260
6261 void FreeUserDataString(VmaAllocator hAllocator);
6262 };
6263
6264 /*
6265 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
6266 allocated memory block or free.
6267 */
6268 struct VmaSuballocation
6269 {
6270 VkDeviceSize offset;
6271 VkDeviceSize size;
6272 VmaAllocation hAllocation;
6273 VmaSuballocationType type;
6274 };
6275
6276 // Comparator for offsets.
6277 struct VmaSuballocationOffsetLess
6278 {
6279 bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const
6280 {
6281 return lhs.offset < rhs.offset;
6282 }
6283 };
6284 struct VmaSuballocationOffsetGreater
6285 {
6286 bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const
6287 {
6288 return lhs.offset > rhs.offset;
6289 }
6290 };
6291
6292 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
6293
6294 // Cost of one additional allocation lost, as equivalent in bytes.
6295 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
6296
6297 enum class VmaAllocationRequestType
6298 {
6299 Normal,
6300 // Used by "Linear" algorithm.
6301 UpperAddress,
6302 EndOf1st,
6303 EndOf2nd,
6304 };
6305
6306 /*
6307 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
6308
6309 If canMakeOtherLost was false:
6310 - item points to a FREE suballocation.
6311 - itemsToMakeLostCount is 0.
6312
6313 If canMakeOtherLost was true:
6314 - item points to first of sequence of suballocations, which are either FREE,
6315 or point to VmaAllocations that can become lost.
6316 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
6317 the requested allocation to succeed.
6318 */
6319 struct VmaAllocationRequest
6320 {
6321 VkDeviceSize offset;
6322 VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
6323 VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
6324 VmaSuballocationList::iterator item;
6325 size_t itemsToMakeLostCount;
6326 void* customData;
6327 VmaAllocationRequestType type;
6328
6329 VkDeviceSize CalcCost() const
6330 {
6331 return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
6332 }
6333 };
6334
6335 /*
6336 Data structure used for bookkeeping of allocations and unused ranges of memory
6337 in a single VkDeviceMemory block.
6338 */
6339 class VmaBlockMetadata
6340 {
6341 public:
6342 VmaBlockMetadata(VmaAllocator hAllocator);
6343 virtual ~VmaBlockMetadata() { }
6344 virtual void Init(VkDeviceSize size) { m_Size = size; }
6345
6346 // Validates all data structures inside this object. If not valid, returns false.
6347 virtual bool Validate() const = 0;
6348 VkDeviceSize GetSize() const { return m_Size; }
6349 virtual size_t GetAllocationCount() const = 0;
6350 virtual VkDeviceSize GetSumFreeSize() const = 0;
6351 virtual VkDeviceSize GetUnusedRangeSizeMax() const = 0;
6352 // Returns true if this block is empty - contains only single free suballocation.
6353 virtual bool IsEmpty() const = 0;
6354
6355 virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const = 0;
6356 // Shouldn't modify blockCount.
6357 virtual void AddPoolStats(VmaPoolStats& inoutStats) const = 0;
6358
6359 #if VMA_STATS_STRING_ENABLED
6360 virtual void PrintDetailedMap(class VmaJsonWriter& json) const = 0;
6361 #endif
6362
6363 // Tries to find a place for suballocation with given parameters inside this block.
6364 // If succeeded, fills pAllocationRequest and returns true.
6365 // If failed, returns false.
6366 virtual bool CreateAllocationRequest(
6367 uint32_t currentFrameIndex,
6368 uint32_t frameInUseCount,
6369 VkDeviceSize bufferImageGranularity,
6370 VkDeviceSize allocSize,
6371 VkDeviceSize allocAlignment,
6372 bool upperAddress,
6373 VmaSuballocationType allocType,
6374 bool canMakeOtherLost,
6375 // Always one of VMA_ALLOCATION_CREATE_STRATEGY_* or VMA_ALLOCATION_INTERNAL_STRATEGY_* flags.
6376 uint32_t strategy,
6377 VmaAllocationRequest* pAllocationRequest) = 0;
6378
6379 virtual bool MakeRequestedAllocationsLost(
6380 uint32_t currentFrameIndex,
6381 uint32_t frameInUseCount,
6382 VmaAllocationRequest* pAllocationRequest) = 0;
6383
6384 virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) = 0;
6385
6386 virtual VkResult CheckCorruption(const void* pBlockData) = 0;
6387
6388 // Makes actual allocation based on request. Request must already be checked and valid.
6389 virtual void Alloc(
6390 const VmaAllocationRequest& request,
6391 VmaSuballocationType type,
6392 VkDeviceSize allocSize,
6393 VmaAllocation hAllocation) = 0;
6394
6395 // Frees suballocation assigned to given memory region.
6396 virtual void Free(const VmaAllocation allocation) = 0;
6397 virtual void FreeAtOffset(VkDeviceSize offset) = 0;
6398
6399 protected:
6400 const VkAllocationCallbacks* GetAllocationCallbacks() const { return m_pAllocationCallbacks; }
6401
6402 #if VMA_STATS_STRING_ENABLED
6403 void PrintDetailedMap_Begin(class VmaJsonWriter& json,
6404 VkDeviceSize unusedBytes,
6405 size_t allocationCount,
6406 size_t unusedRangeCount) const;
6407 void PrintDetailedMap_Allocation(class VmaJsonWriter& json,
6408 VkDeviceSize offset,
6409 VmaAllocation hAllocation) const;
6410 void PrintDetailedMap_UnusedRange(class VmaJsonWriter& json,
6411 VkDeviceSize offset,
6412 VkDeviceSize size) const;
6413 void PrintDetailedMap_End(class VmaJsonWriter& json) const;
6414 #endif
6415
6416 private:
6417 VkDeviceSize m_Size;
6418 const VkAllocationCallbacks* m_pAllocationCallbacks;
6419 };
6420
6421 #define VMA_VALIDATE(cond) do { if(!(cond)) { \
6422 VMA_ASSERT(0 && "Validation failed: " #cond); \
6423 return false; \
6424 } } while(false)
6425
6426 class VmaBlockMetadata_Generic : public VmaBlockMetadata
6427 {
6428 VMA_CLASS_NO_COPY(VmaBlockMetadata_Generic)
6429 public:
6430 VmaBlockMetadata_Generic(VmaAllocator hAllocator);
6431 virtual ~VmaBlockMetadata_Generic();
6432 virtual void Init(VkDeviceSize size);
6433
6434 virtual bool Validate() const;
6435 virtual size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
6436 virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
6437 virtual VkDeviceSize GetUnusedRangeSizeMax() const;
6438 virtual bool IsEmpty() const;
6439
6440 virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
6441 virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
6442
6443 #if VMA_STATS_STRING_ENABLED
6444 virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
6445 #endif
6446
6447 virtual bool CreateAllocationRequest(
6448 uint32_t currentFrameIndex,
6449 uint32_t frameInUseCount,
6450 VkDeviceSize bufferImageGranularity,
6451 VkDeviceSize allocSize,
6452 VkDeviceSize allocAlignment,
6453 bool upperAddress,
6454 VmaSuballocationType allocType,
6455 bool canMakeOtherLost,
6456 uint32_t strategy,
6457 VmaAllocationRequest* pAllocationRequest);
6458
6459 virtual bool MakeRequestedAllocationsLost(
6460 uint32_t currentFrameIndex,
6461 uint32_t frameInUseCount,
6462 VmaAllocationRequest* pAllocationRequest);
6463
6464 virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
6465
6466 virtual VkResult CheckCorruption(const void* pBlockData);
6467
6468 virtual void Alloc(
6469 const VmaAllocationRequest& request,
6470 VmaSuballocationType type,
6471 VkDeviceSize allocSize,
6472 VmaAllocation hAllocation);
6473
6474 virtual void Free(const VmaAllocation allocation);
6475 virtual void FreeAtOffset(VkDeviceSize offset);
6476
6477 ////////////////////////////////////////////////////////////////////////////////
6478 // For defragmentation
6479
6480 bool IsBufferImageGranularityConflictPossible(
6481 VkDeviceSize bufferImageGranularity,
6482 VmaSuballocationType& inOutPrevSuballocType) const;
6483
6484 private:
6485 friend class VmaDefragmentationAlgorithm_Generic;
6486 friend class VmaDefragmentationAlgorithm_Fast;
6487
6488 uint32_t m_FreeCount;
6489 VkDeviceSize m_SumFreeSize;
6490 VmaSuballocationList m_Suballocations;
6491 // Suballocations that are free and have size greater than certain threshold.
6492 // Sorted by size, ascending.
6493 VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
6494
6495 bool ValidateFreeSuballocationList() const;
6496
6497 // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
6498 // If yes, fills pOffset and returns true. If no, returns false.
6499 bool CheckAllocation(
6500 uint32_t currentFrameIndex,
6501 uint32_t frameInUseCount,
6502 VkDeviceSize bufferImageGranularity,
6503 VkDeviceSize allocSize,
6504 VkDeviceSize allocAlignment,
6505 VmaSuballocationType allocType,
6506 VmaSuballocationList::const_iterator suballocItem,
6507 bool canMakeOtherLost,
6508 VkDeviceSize* pOffset,
6509 size_t* itemsToMakeLostCount,
6510 VkDeviceSize* pSumFreeSize,
6511 VkDeviceSize* pSumItemSize) const;
6512 // Given free suballocation, it merges it with following one, which must also be free.
6513 void MergeFreeWithNext(VmaSuballocationList::iterator item);
6514 // Releases given suballocation, making it free.
6515 // Merges it with adjacent free suballocations if applicable.
6516 // Returns iterator to new free suballocation at this place.
6517 VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
6518 // Given free suballocation, it inserts it into sorted list of
6519 // m_FreeSuballocationsBySize if it's suitable.
6520 void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
6521 // Given free suballocation, it removes it from sorted list of
6522 // m_FreeSuballocationsBySize if it's suitable.
6523 void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
6524 };
6525
6526 /*
6527 Allocations and their references in internal data structure look like this:
6528
6529 if(m_2ndVectorMode == SECOND_VECTOR_EMPTY):
6530
6531 0 +-------+
6532 | |
6533 | |
6534 | |
6535 +-------+
6536 | Alloc | 1st[m_1stNullItemsBeginCount]
6537 +-------+
6538 | Alloc | 1st[m_1stNullItemsBeginCount + 1]
6539 +-------+
6540 | ... |
6541 +-------+
6542 | Alloc | 1st[1st.size() - 1]
6543 +-------+
6544 | |
6545 | |
6546 | |
6547 GetSize() +-------+
6548
6549 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER):
6550
6551 0 +-------+
6552 | Alloc | 2nd[0]
6553 +-------+
6554 | Alloc | 2nd[1]
6555 +-------+
6556 | ... |
6557 +-------+
6558 | Alloc | 2nd[2nd.size() - 1]
6559 +-------+
6560 | |
6561 | |
6562 | |
6563 +-------+
6564 | Alloc | 1st[m_1stNullItemsBeginCount]
6565 +-------+
6566 | Alloc | 1st[m_1stNullItemsBeginCount + 1]
6567 +-------+
6568 | ... |
6569 +-------+
6570 | Alloc | 1st[1st.size() - 1]
6571 +-------+
6572 | |
6573 GetSize() +-------+
6574
6575 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK):
6576
6577 0 +-------+
6578 | |
6579 | |
6580 | |
6581 +-------+
6582 | Alloc | 1st[m_1stNullItemsBeginCount]
6583 +-------+
6584 | Alloc | 1st[m_1stNullItemsBeginCount + 1]
6585 +-------+
6586 | ... |
6587 +-------+
6588 | Alloc | 1st[1st.size() - 1]
6589 +-------+
6590 | |
6591 | |
6592 | |
6593 +-------+
6594 | Alloc | 2nd[2nd.size() - 1]
6595 +-------+
6596 | ... |
6597 +-------+
6598 | Alloc | 2nd[1]
6599 +-------+
6600 | Alloc | 2nd[0]
6601 GetSize() +-------+
6602
6603 */
6604 class VmaBlockMetadata_Linear : public VmaBlockMetadata
6605 {
6606 VMA_CLASS_NO_COPY(VmaBlockMetadata_Linear)
6607 public:
6608 VmaBlockMetadata_Linear(VmaAllocator hAllocator);
6609 virtual ~VmaBlockMetadata_Linear();
6610 virtual void Init(VkDeviceSize size);
6611
6612 virtual bool Validate() const;
6613 virtual size_t GetAllocationCount() const;
6614 virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
6615 virtual VkDeviceSize GetUnusedRangeSizeMax() const;
6616 virtual bool IsEmpty() const { return GetAllocationCount() == 0; }
6617
6618 virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
6619 virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
6620
6621 #if VMA_STATS_STRING_ENABLED
6622 virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
6623 #endif
6624
6625 virtual bool CreateAllocationRequest(
6626 uint32_t currentFrameIndex,
6627 uint32_t frameInUseCount,
6628 VkDeviceSize bufferImageGranularity,
6629 VkDeviceSize allocSize,
6630 VkDeviceSize allocAlignment,
6631 bool upperAddress,
6632 VmaSuballocationType allocType,
6633 bool canMakeOtherLost,
6634 uint32_t strategy,
6635 VmaAllocationRequest* pAllocationRequest);
6636
6637 virtual bool MakeRequestedAllocationsLost(
6638 uint32_t currentFrameIndex,
6639 uint32_t frameInUseCount,
6640 VmaAllocationRequest* pAllocationRequest);
6641
6642 virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
6643
6644 virtual VkResult CheckCorruption(const void* pBlockData);
6645
6646 virtual void Alloc(
6647 const VmaAllocationRequest& request,
6648 VmaSuballocationType type,
6649 VkDeviceSize allocSize,
6650 VmaAllocation hAllocation);
6651
6652 virtual void Free(const VmaAllocation allocation);
6653 virtual void FreeAtOffset(VkDeviceSize offset);
6654
6655 private:
6656 /*
6657 There are two suballocation vectors, used in ping-pong way.
6658 The one with index m_1stVectorIndex is called 1st.
6659 The one with index (m_1stVectorIndex ^ 1) is called 2nd.
6660 2nd can be non-empty only when 1st is not empty.
6661 When 2nd is not empty, m_2ndVectorMode indicates its mode of operation.
6662 */
6663 typedef VmaVector< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > SuballocationVectorType;
6664
6665 enum SECOND_VECTOR_MODE
6666 {
6667 SECOND_VECTOR_EMPTY,
6668 /*
6669 Suballocations in 2nd vector are created later than the ones in 1st, but they
6670 all have smaller offset.
6671 */
6672 SECOND_VECTOR_RING_BUFFER,
6673 /*
6674 Suballocations in 2nd vector are upper side of double stack.
6675 They all have offsets higher than those in 1st vector.
6676 Top of this stack means smaller offsets, but higher indices in this vector.
6677 */
6678 SECOND_VECTOR_DOUBLE_STACK,
6679 };
6680
6681 VkDeviceSize m_SumFreeSize;
6682 SuballocationVectorType m_Suballocations0, m_Suballocations1;
6683 uint32_t m_1stVectorIndex;
6684 SECOND_VECTOR_MODE m_2ndVectorMode;
6685
6686 SuballocationVectorType& AccessSuballocations1st() { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; }
6687 SuballocationVectorType& AccessSuballocations2nd() { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; }
6688 const SuballocationVectorType& AccessSuballocations1st() const { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; }
6689 const SuballocationVectorType& AccessSuballocations2nd() const { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; }
6690
6691 // Number of items in 1st vector with hAllocation = null at the beginning.
6692 size_t m_1stNullItemsBeginCount;
6693 // Number of other items in 1st vector with hAllocation = null somewhere in the middle.
6694 size_t m_1stNullItemsMiddleCount;
6695 // Number of items in 2nd vector with hAllocation = null.
6696 size_t m_2ndNullItemsCount;
6697
6698 bool ShouldCompact1st() const;
6699 void CleanupAfterFree();
6700
6701 bool CreateAllocationRequest_LowerAddress(
6702 uint32_t currentFrameIndex,
6703 uint32_t frameInUseCount,
6704 VkDeviceSize bufferImageGranularity,
6705 VkDeviceSize allocSize,
6706 VkDeviceSize allocAlignment,
6707 VmaSuballocationType allocType,
6708 bool canMakeOtherLost,
6709 uint32_t strategy,
6710 VmaAllocationRequest* pAllocationRequest);
6711 bool CreateAllocationRequest_UpperAddress(
6712 uint32_t currentFrameIndex,
6713 uint32_t frameInUseCount,
6714 VkDeviceSize bufferImageGranularity,
6715 VkDeviceSize allocSize,
6716 VkDeviceSize allocAlignment,
6717 VmaSuballocationType allocType,
6718 bool canMakeOtherLost,
6719 uint32_t strategy,
6720 VmaAllocationRequest* pAllocationRequest);
6721 };
6722
6723 /*
6724 - GetSize() is the original size of allocated memory block.
6725 - m_UsableSize is this size aligned down to a power of two.
6726 All allocations and calculations happen relative to m_UsableSize.
6727 - GetUnusableSize() is the difference between them.
6728 It is repoted as separate, unused range, not available for allocations.
6729
6730 Node at level 0 has size = m_UsableSize.
6731 Each next level contains nodes with size 2 times smaller than current level.
6732 m_LevelCount is the maximum number of levels to use in the current object.
6733 */
6734 class VmaBlockMetadata_Buddy : public VmaBlockMetadata
6735 {
6736 VMA_CLASS_NO_COPY(VmaBlockMetadata_Buddy)
6737 public:
6738 VmaBlockMetadata_Buddy(VmaAllocator hAllocator);
6739 virtual ~VmaBlockMetadata_Buddy();
6740 virtual void Init(VkDeviceSize size);
6741
6742 virtual bool Validate() const;
6743 virtual size_t GetAllocationCount() const { return m_AllocationCount; }
6744 virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize + GetUnusableSize(); }
6745 virtual VkDeviceSize GetUnusedRangeSizeMax() const;
6746 virtual bool IsEmpty() const { return m_Root->type == Node::TYPE_FREE; }
6747
6748 virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
6749 virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
6750
6751 #if VMA_STATS_STRING_ENABLED
6752 virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
6753 #endif
6754
6755 virtual bool CreateAllocationRequest(
6756 uint32_t currentFrameIndex,
6757 uint32_t frameInUseCount,
6758 VkDeviceSize bufferImageGranularity,
6759 VkDeviceSize allocSize,
6760 VkDeviceSize allocAlignment,
6761 bool upperAddress,
6762 VmaSuballocationType allocType,
6763 bool canMakeOtherLost,
6764 uint32_t strategy,
6765 VmaAllocationRequest* pAllocationRequest);
6766
6767 virtual bool MakeRequestedAllocationsLost(
6768 uint32_t currentFrameIndex,
6769 uint32_t frameInUseCount,
6770 VmaAllocationRequest* pAllocationRequest);
6771
6772 virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
6773
6774 virtual VkResult CheckCorruption(const void* pBlockData) { return VK_ERROR_FEATURE_NOT_PRESENT; }
6775
6776 virtual void Alloc(
6777 const VmaAllocationRequest& request,
6778 VmaSuballocationType type,
6779 VkDeviceSize allocSize,
6780 VmaAllocation hAllocation);
6781
6782 virtual void Free(const VmaAllocation allocation) { FreeAtOffset(allocation, allocation->GetOffset()); }
6783 virtual void FreeAtOffset(VkDeviceSize offset) { FreeAtOffset(VMA_NULL, offset); }
6784
6785 private:
6786 static const VkDeviceSize MIN_NODE_SIZE = 32;
6787 static const size_t MAX_LEVELS = 30;
6788
6789 struct ValidationContext
6790 {
6791 size_t calculatedAllocationCount;
6792 size_t calculatedFreeCount;
6793 VkDeviceSize calculatedSumFreeSize;
6794
6795 ValidationContext() :
6796 calculatedAllocationCount(0),
6797 calculatedFreeCount(0),
6798 calculatedSumFreeSize(0) { }
6799 };
6800
6801 struct Node
6802 {
6803 VkDeviceSize offset;
6804 enum TYPE
6805 {
6806 TYPE_FREE,
6807 TYPE_ALLOCATION,
6808 TYPE_SPLIT,
6809 TYPE_COUNT
6810 } type;
6811 Node* parent;
6812 Node* buddy;
6813
6814 union
6815 {
6816 struct
6817 {
6818 Node* prev;
6819 Node* next;
6820 } free;
6821 struct
6822 {
6823 VmaAllocation alloc;
6824 } allocation;
6825 struct
6826 {
6827 Node* leftChild;
6828 } split;
6829 };
6830 };
6831
6832 // Size of the memory block aligned down to a power of two.
6833 VkDeviceSize m_UsableSize;
6834 uint32_t m_LevelCount;
6835
6836 Node* m_Root;
6837 struct {
6838 Node* front;
6839 Node* back;
6840 } m_FreeList[MAX_LEVELS];
6841 // Number of nodes in the tree with type == TYPE_ALLOCATION.
6842 size_t m_AllocationCount;
6843 // Number of nodes in the tree with type == TYPE_FREE.
6844 size_t m_FreeCount;
6845 // This includes space wasted due to internal fragmentation. Doesn't include unusable size.
6846 VkDeviceSize m_SumFreeSize;
6847
6848 VkDeviceSize GetUnusableSize() const { return GetSize() - m_UsableSize; }
6849 void DeleteNode(Node* node);
6850 bool ValidateNode(ValidationContext& ctx, const Node* parent, const Node* curr, uint32_t level, VkDeviceSize levelNodeSize) const;
6851 uint32_t AllocSizeToLevel(VkDeviceSize allocSize) const;
6852 inline VkDeviceSize LevelToNodeSize(uint32_t level) const { return m_UsableSize >> level; }
6853 // Alloc passed just for validation. Can be null.
6854 void FreeAtOffset(VmaAllocation alloc, VkDeviceSize offset);
6855 void CalcAllocationStatInfoNode(VmaStatInfo& outInfo, const Node* node, VkDeviceSize levelNodeSize) const;
6856 // Adds node to the front of FreeList at given level.
6857 // node->type must be FREE.
6858 // node->free.prev, next can be undefined.
6859 void AddToFreeListFront(uint32_t level, Node* node);
6860 // Removes node from FreeList at given level.
6861 // node->type must be FREE.
6862 // node->free.prev, next stay untouched.
6863 void RemoveFromFreeList(uint32_t level, Node* node);
6864
6865 #if VMA_STATS_STRING_ENABLED
6866 void PrintDetailedMapNode(class VmaJsonWriter& json, const Node* node, VkDeviceSize levelNodeSize) const;
6867 #endif
6868 };
6869
6870 /*
6871 Represents a single block of device memory (`VkDeviceMemory`) with all the
6872 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
6873
6874 Thread-safety: This class must be externally synchronized.
6875 */
6876 class VmaDeviceMemoryBlock
6877 {
6878 VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
6879 public:
6880 VmaBlockMetadata* m_pMetadata;
6881
6882 VmaDeviceMemoryBlock(VmaAllocator hAllocator);
6883
6884 ~VmaDeviceMemoryBlock()
6885 {
6886 VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
6887 VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
6888 }
6889
6890 // Always call after construction.
6891 void Init(
6892 VmaAllocator hAllocator,
6893 VmaPool hParentPool,
6894 uint32_t newMemoryTypeIndex,
6895 VkDeviceMemory newMemory,
6896 VkDeviceSize newSize,
6897 uint32_t id,
6898 uint32_t algorithm);
6899 // Always call before destruction.
6900 void Destroy(VmaAllocator allocator);
6901
6902 VmaPool GetParentPool() const { return m_hParentPool; }
6903 VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
6904 uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
6905 uint32_t GetId() const { return m_Id; }
6906 void* GetMappedData() const { return m_pMappedData; }
6907
6908 // Validates all data structures inside this object. If not valid, returns false.
6909 bool Validate() const;
6910
6911 VkResult CheckCorruption(VmaAllocator hAllocator);
6912
6913 // ppData can be null.
6914 VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
6915 void Unmap(VmaAllocator hAllocator, uint32_t count);
6916
6917 VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
6918 VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
6919
6920 VkResult BindBufferMemory(
6921 const VmaAllocator hAllocator,
6922 const VmaAllocation hAllocation,
6923 VkDeviceSize allocationLocalOffset,
6924 VkBuffer hBuffer,
6925 const void* pNext);
6926 VkResult BindImageMemory(
6927 const VmaAllocator hAllocator,
6928 const VmaAllocation hAllocation,
6929 VkDeviceSize allocationLocalOffset,
6930 VkImage hImage,
6931 const void* pNext);
6932
6933 private:
6934 VmaPool m_hParentPool; // VK_NULL_HANDLE if not belongs to custom pool.
6935 uint32_t m_MemoryTypeIndex;
6936 uint32_t m_Id;
6937 VkDeviceMemory m_hMemory;
6938
6939 /*
6940 Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
6941 Also protects m_MapCount, m_pMappedData.
6942 Allocations, deallocations, any change in m_pMetadata is protected by parent's VmaBlockVector::m_Mutex.
6943 */
6944 VMA_MUTEX m_Mutex;
6945 uint32_t m_MapCount;
6946 void* m_pMappedData;
6947 };
6948
6949 struct VmaPointerLess
6950 {
6951 bool operator()(const void* lhs, const void* rhs) const
6952 {
6953 return lhs < rhs;
6954 }
6955 };
6956
6957 struct VmaDefragmentationMove
6958 {
6959 size_t srcBlockIndex;
6960 size_t dstBlockIndex;
6961 VkDeviceSize srcOffset;
6962 VkDeviceSize dstOffset;
6963 VkDeviceSize size;
6964 VmaAllocation hAllocation;
6965 VmaDeviceMemoryBlock* pSrcBlock;
6966 VmaDeviceMemoryBlock* pDstBlock;
6967 };
6968
6969 class VmaDefragmentationAlgorithm;
6970
6971 /*
6972 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
6973 Vulkan memory type.
6974
6975 Synchronized internally with a mutex.
6976 */
6977 struct VmaBlockVector
6978 {
6979 VMA_CLASS_NO_COPY(VmaBlockVector)
6980 public:
6981 VmaBlockVector(
6982 VmaAllocator hAllocator,
6983 VmaPool hParentPool,
6984 uint32_t memoryTypeIndex,
6985 VkDeviceSize preferredBlockSize,
6986 size_t minBlockCount,
6987 size_t maxBlockCount,
6988 VkDeviceSize bufferImageGranularity,
6989 uint32_t frameInUseCount,
6990 bool explicitBlockSize,
6991 uint32_t algorithm);
6992 ~VmaBlockVector();
6993
6994 VkResult CreateMinBlocks();
6995
6996 VmaAllocator GetAllocator() const { return m_hAllocator; }
6997 VmaPool GetParentPool() const { return m_hParentPool; }
6998 bool IsCustomPool() const { return m_hParentPool != VMA_NULL; }
6999 uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
7000 VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
7001 VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
7002 uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
7003 uint32_t GetAlgorithm() const { return m_Algorithm; }
7004
7005 void GetPoolStats(VmaPoolStats* pStats);
7006
7007 bool IsEmpty();
7008 bool IsCorruptionDetectionEnabled() const;
7009
7010 VkResult Allocate(
7011 uint32_t currentFrameIndex,
7012 VkDeviceSize size,
7013 VkDeviceSize alignment,
7014 const VmaAllocationCreateInfo& createInfo,
7015 VmaSuballocationType suballocType,
7016 size_t allocationCount,
7017 VmaAllocation* pAllocations);
7018
7019 void Free(const VmaAllocation hAllocation);
7020
7021 // Adds statistics of this BlockVector to pStats.
7022 void AddStats(VmaStats* pStats);
7023
7024 #if VMA_STATS_STRING_ENABLED
7025 void PrintDetailedMap(class VmaJsonWriter& json);
7026 #endif
7027
7028 void MakePoolAllocationsLost(
7029 uint32_t currentFrameIndex,
7030 size_t* pLostAllocationCount);
7031 VkResult CheckCorruption();
7032
7033 // Saves results in pCtx->res.
7034 void Defragment(
7035 class VmaBlockVectorDefragmentationContext* pCtx,
7036 VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags,
7037 VkDeviceSize& maxCpuBytesToMove, uint32_t& maxCpuAllocationsToMove,
7038 VkDeviceSize& maxGpuBytesToMove, uint32_t& maxGpuAllocationsToMove,
7039 VkCommandBuffer commandBuffer);
7040 void DefragmentationEnd(
7041 class VmaBlockVectorDefragmentationContext* pCtx,
7042 uint32_t flags,
7043 VmaDefragmentationStats* pStats);
7044
7045 uint32_t ProcessDefragmentations(
7046 class VmaBlockVectorDefragmentationContext *pCtx,
7047 VmaDefragmentationPassMoveInfo* pMove, uint32_t maxMoves);
7048
7049 void CommitDefragmentations(
7050 class VmaBlockVectorDefragmentationContext *pCtx,
7051 VmaDefragmentationStats* pStats);
7052
7053 ////////////////////////////////////////////////////////////////////////////////
7054 // To be used only while the m_Mutex is locked. Used during defragmentation.
7055
7056 size_t GetBlockCount() const { return m_Blocks.size(); }
7057 VmaDeviceMemoryBlock* GetBlock(size_t index) const { return m_Blocks[index]; }
7058 size_t CalcAllocationCount() const;
7059 bool IsBufferImageGranularityConflictPossible() const;
7060
7061 private:
7062 friend class VmaDefragmentationAlgorithm_Generic;
7063
7064 const VmaAllocator m_hAllocator;
7065 const VmaPool m_hParentPool;
7066 const uint32_t m_MemoryTypeIndex;
7067 const VkDeviceSize m_PreferredBlockSize;
7068 const size_t m_MinBlockCount;
7069 const size_t m_MaxBlockCount;
7070 const VkDeviceSize m_BufferImageGranularity;
7071 const uint32_t m_FrameInUseCount;
7072 const bool m_ExplicitBlockSize;
7073 const uint32_t m_Algorithm;
7074 VMA_RW_MUTEX m_Mutex;
7075
7076 /* There can be at most one allocation that is completely empty (except when minBlockCount > 0) -
7077 a hysteresis to avoid pessimistic case of alternating creation and destruction of a VkDeviceMemory. */
7078 bool m_HasEmptyBlock;
7079 // Incrementally sorted by sumFreeSize, ascending.
7080 VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
7081 uint32_t m_NextBlockId;
7082
7083 VkDeviceSize CalcMaxBlockSize() const;
7084
7085 // Finds and removes given block from vector.
7086 void Remove(VmaDeviceMemoryBlock* pBlock);
7087
7088 // Performs single step in sorting m_Blocks. They may not be fully sorted
7089 // after this call.
7090 void IncrementallySortBlocks();
7091
7092 VkResult AllocatePage(
7093 uint32_t currentFrameIndex,
7094 VkDeviceSize size,
7095 VkDeviceSize alignment,
7096 const VmaAllocationCreateInfo& createInfo,
7097 VmaSuballocationType suballocType,
7098 VmaAllocation* pAllocation);
7099
7100 // To be used only without CAN_MAKE_OTHER_LOST flag.
7101 VkResult AllocateFromBlock(
7102 VmaDeviceMemoryBlock* pBlock,
7103 uint32_t currentFrameIndex,
7104 VkDeviceSize size,
7105 VkDeviceSize alignment,
7106 VmaAllocationCreateFlags allocFlags,
7107 void* pUserData,
7108 VmaSuballocationType suballocType,
7109 uint32_t strategy,
7110 VmaAllocation* pAllocation);
7111
7112 VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
7113
7114 // Saves result to pCtx->res.
7115 void ApplyDefragmentationMovesCpu(
7116 class VmaBlockVectorDefragmentationContext* pDefragCtx,
7117 const VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves);
7118 // Saves result to pCtx->res.
7119 void ApplyDefragmentationMovesGpu(
7120 class VmaBlockVectorDefragmentationContext* pDefragCtx,
7121 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
7122 VkCommandBuffer commandBuffer);
7123
7124 /*
7125 Used during defragmentation. pDefragmentationStats is optional. It's in/out
7126 - updated with new data.
7127 */
7128 void FreeEmptyBlocks(VmaDefragmentationStats* pDefragmentationStats);
7129
7130 void UpdateHasEmptyBlock();
7131 };
7132
7133 struct VmaPool_T
7134 {
7135 VMA_CLASS_NO_COPY(VmaPool_T)
7136 public:
7137 VmaBlockVector m_BlockVector;
7138
7139 VmaPool_T(
7140 VmaAllocator hAllocator,
7141 const VmaPoolCreateInfo& createInfo,
7142 VkDeviceSize preferredBlockSize);
7143 ~VmaPool_T();
7144
7145 uint32_t GetId() const { return m_Id; }
7146 void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
7147
7148 const char* GetName() const { return m_Name; }
7149 void SetName(const char* pName);
7150
7151 #if VMA_STATS_STRING_ENABLED
7152 //void PrintDetailedMap(class VmaStringBuilder& sb);
7153 #endif
7154
7155 private:
7156 uint32_t m_Id;
7157 char* m_Name;
7158 };
7159
7160 /*
7161 Performs defragmentation:
7162
7163 - Updates `pBlockVector->m_pMetadata`.
7164 - Updates allocations by calling ChangeBlockAllocation() or ChangeOffset().
7165 - Does not move actual data, only returns requested moves as `moves`.
7166 */
7167 class VmaDefragmentationAlgorithm
7168 {
7169 VMA_CLASS_NO_COPY(VmaDefragmentationAlgorithm)
7170 public:
7171 VmaDefragmentationAlgorithm(
7172 VmaAllocator hAllocator,
7173 VmaBlockVector* pBlockVector,
7174 uint32_t currentFrameIndex) :
7175 m_hAllocator(hAllocator),
7176 m_pBlockVector(pBlockVector),
7177 m_CurrentFrameIndex(currentFrameIndex)
7178 {
7179 }
7180 virtual ~VmaDefragmentationAlgorithm()
7181 {
7182 }
7183
7184 virtual void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) = 0;
7185 virtual void AddAll() = 0;
7186
7187 virtual VkResult Defragment(
7188 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
7189 VkDeviceSize maxBytesToMove,
7190 uint32_t maxAllocationsToMove,
7191 VmaDefragmentationFlags flags) = 0;
7192
7193 virtual VkDeviceSize GetBytesMoved() const = 0;
7194 virtual uint32_t GetAllocationsMoved() const = 0;
7195
7196 protected:
7197 VmaAllocator const m_hAllocator;
7198 VmaBlockVector* const m_pBlockVector;
7199 const uint32_t m_CurrentFrameIndex;
7200
7201 struct AllocationInfo
7202 {
7203 VmaAllocation m_hAllocation;
7204 VkBool32* m_pChanged;
7205
7206 AllocationInfo() :
7207 m_hAllocation(VK_NULL_HANDLE),
7208 m_pChanged(VMA_NULL)
7209 {
7210 }
7211 AllocationInfo(VmaAllocation hAlloc, VkBool32* pChanged) :
7212 m_hAllocation(hAlloc),
7213 m_pChanged(pChanged)
7214 {
7215 }
7216 };
7217 };
7218
7219 class VmaDefragmentationAlgorithm_Generic : public VmaDefragmentationAlgorithm
7220 {
7221 VMA_CLASS_NO_COPY(VmaDefragmentationAlgorithm_Generic)
7222 public:
7223 VmaDefragmentationAlgorithm_Generic(
7224 VmaAllocator hAllocator,
7225 VmaBlockVector* pBlockVector,
7226 uint32_t currentFrameIndex,
7227 bool overlappingMoveSupported);
7228 virtual ~VmaDefragmentationAlgorithm_Generic();
7229
7230 virtual void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
7231 virtual void AddAll() { m_AllAllocations = true; }
7232
7233 virtual VkResult Defragment(
7234 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
7235 VkDeviceSize maxBytesToMove,
7236 uint32_t maxAllocationsToMove,
7237 VmaDefragmentationFlags flags);
7238
7239 virtual VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
7240 virtual uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
7241
7242 private:
7243 uint32_t m_AllocationCount;
7244 bool m_AllAllocations;
7245
7246 VkDeviceSize m_BytesMoved;
7247 uint32_t m_AllocationsMoved;
7248
7249 struct AllocationInfoSizeGreater
7250 {
7251 bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
7252 {
7253 return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
7254 }
7255 };
7256
7257 struct AllocationInfoOffsetGreater
7258 {
7259 bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
7260 {
7261 return lhs.m_hAllocation->GetOffset() > rhs.m_hAllocation->GetOffset();
7262 }
7263 };
7264
7265 struct BlockInfo
7266 {
7267 size_t m_OriginalBlockIndex;
7268 VmaDeviceMemoryBlock* m_pBlock;
7269 bool m_HasNonMovableAllocations;
7270 VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
7271
7272 BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
7273 m_OriginalBlockIndex(SIZE_MAX),
7274 m_pBlock(VMA_NULL),
7275 m_HasNonMovableAllocations(true),
7276 m_Allocations(pAllocationCallbacks)
7277 {
7278 }
7279
7280 void CalcHasNonMovableAllocations()
7281 {
7282 const size_t blockAllocCount = m_pBlock->m_pMetadata->GetAllocationCount();
7283 const size_t defragmentAllocCount = m_Allocations.size();
7284 m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
7285 }
7286
7287 void SortAllocationsBySizeDescending()
7288 {
7289 VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
7290 }
7291
7292 void SortAllocationsByOffsetDescending()
7293 {
7294 VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoOffsetGreater());
7295 }
7296 };
7297
7298 struct BlockPointerLess
7299 {
7300 bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
7301 {
7302 return pLhsBlockInfo->m_pBlock < pRhsBlock;
7303 }
7304 bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
7305 {
7306 return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
7307 }
7308 };
7309
7310 // 1. Blocks with some non-movable allocations go first.
7311 // 2. Blocks with smaller sumFreeSize go first.
7312 struct BlockInfoCompareMoveDestination
7313 {
7314 bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
7315 {
7316 if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
7317 {
7318 return true;
7319 }
7320 if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
7321 {
7322 return false;
7323 }
7324 if(pLhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize())
7325 {
7326 return true;
7327 }
7328 return false;
7329 }
7330 };
7331
7332 typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
7333 BlockInfoVector m_Blocks;
7334
7335 VkResult DefragmentRound(
7336 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
7337 VkDeviceSize maxBytesToMove,
7338 uint32_t maxAllocationsToMove,
7339 bool freeOldAllocations);
7340
7341 size_t CalcBlocksWithNonMovableCount() const;
7342
7343 static bool MoveMakesSense(
7344 size_t dstBlockIndex, VkDeviceSize dstOffset,
7345 size_t srcBlockIndex, VkDeviceSize srcOffset);
7346 };
7347
7348 class VmaDefragmentationAlgorithm_Fast : public VmaDefragmentationAlgorithm
7349 {
7350 VMA_CLASS_NO_COPY(VmaDefragmentationAlgorithm_Fast)
7351 public:
7352 VmaDefragmentationAlgorithm_Fast(
7353 VmaAllocator hAllocator,
7354 VmaBlockVector* pBlockVector,
7355 uint32_t currentFrameIndex,
7356 bool overlappingMoveSupported);
7357 virtual ~VmaDefragmentationAlgorithm_Fast();
7358
7359 virtual void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged) { ++m_AllocationCount; }
7360 virtual void AddAll() { m_AllAllocations = true; }
7361
7362 virtual VkResult Defragment(
7363 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
7364 VkDeviceSize maxBytesToMove,
7365 uint32_t maxAllocationsToMove,
7366 VmaDefragmentationFlags flags);
7367
7368 virtual VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
7369 virtual uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
7370
7371 private:
7372 struct BlockInfo
7373 {
7374 size_t origBlockIndex;
7375 };
7376
7377 class FreeSpaceDatabase
7378 {
7379 public:
7380 FreeSpaceDatabase()
7381 {
7382 FreeSpace s = {};
7383 s.blockInfoIndex = SIZE_MAX;
7384 for(size_t i = 0; i < MAX_COUNT; ++i)
7385 {
7386 m_FreeSpaces[i] = s;
7387 }
7388 }
7389
7390 void Register(size_t blockInfoIndex, VkDeviceSize offset, VkDeviceSize size)
7391 {
7392 if(size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
7393 {
7394 return;
7395 }
7396
7397 // Find first invalid or the smallest structure.
7398 size_t bestIndex = SIZE_MAX;
7399 for(size_t i = 0; i < MAX_COUNT; ++i)
7400 {
7401 // Empty structure.
7402 if(m_FreeSpaces[i].blockInfoIndex == SIZE_MAX)
7403 {
7404 bestIndex = i;
7405 break;
7406 }
7407 if(m_FreeSpaces[i].size < size &&
7408 (bestIndex == SIZE_MAX || m_FreeSpaces[bestIndex].size > m_FreeSpaces[i].size))
7409 {
7410 bestIndex = i;
7411 }
7412 }
7413
7414 if(bestIndex != SIZE_MAX)
7415 {
7416 m_FreeSpaces[bestIndex].blockInfoIndex = blockInfoIndex;
7417 m_FreeSpaces[bestIndex].offset = offset;
7418 m_FreeSpaces[bestIndex].size = size;
7419 }
7420 }
7421
7422 bool Fetch(VkDeviceSize alignment, VkDeviceSize size,
7423 size_t& outBlockInfoIndex, VkDeviceSize& outDstOffset)
7424 {
7425 size_t bestIndex = SIZE_MAX;
7426 VkDeviceSize bestFreeSpaceAfter = 0;
7427 for(size_t i = 0; i < MAX_COUNT; ++i)
7428 {
7429 // Structure is valid.
7430 if(m_FreeSpaces[i].blockInfoIndex != SIZE_MAX)
7431 {
7432 const VkDeviceSize dstOffset = VmaAlignUp(m_FreeSpaces[i].offset, alignment);
7433 // Allocation fits into this structure.
7434 if(dstOffset + size <= m_FreeSpaces[i].offset + m_FreeSpaces[i].size)
7435 {
7436 const VkDeviceSize freeSpaceAfter = (m_FreeSpaces[i].offset + m_FreeSpaces[i].size) -
7437 (dstOffset + size);
7438 if(bestIndex == SIZE_MAX || freeSpaceAfter > bestFreeSpaceAfter)
7439 {
7440 bestIndex = i;
7441 bestFreeSpaceAfter = freeSpaceAfter;
7442 }
7443 }
7444 }
7445 }
7446
7447 if(bestIndex != SIZE_MAX)
7448 {
7449 outBlockInfoIndex = m_FreeSpaces[bestIndex].blockInfoIndex;
7450 outDstOffset = VmaAlignUp(m_FreeSpaces[bestIndex].offset, alignment);
7451
7452 if(bestFreeSpaceAfter >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
7453 {
7454 // Leave this structure for remaining empty space.
7455 const VkDeviceSize alignmentPlusSize = (outDstOffset - m_FreeSpaces[bestIndex].offset) + size;
7456 m_FreeSpaces[bestIndex].offset += alignmentPlusSize;
7457 m_FreeSpaces[bestIndex].size -= alignmentPlusSize;
7458 }
7459 else
7460 {
7461 // This structure becomes invalid.
7462 m_FreeSpaces[bestIndex].blockInfoIndex = SIZE_MAX;
7463 }
7464
7465 return true;
7466 }
7467
7468 return false;
7469 }
7470
7471 private:
7472 static const size_t MAX_COUNT = 4;
7473
7474 struct FreeSpace
7475 {
7476 size_t blockInfoIndex; // SIZE_MAX means this structure is invalid.
7477 VkDeviceSize offset;
7478 VkDeviceSize size;
7479 } m_FreeSpaces[MAX_COUNT];
7480 };
7481
7482 const bool m_OverlappingMoveSupported;
7483
7484 uint32_t m_AllocationCount;
7485 bool m_AllAllocations;
7486
7487 VkDeviceSize m_BytesMoved;
7488 uint32_t m_AllocationsMoved;
7489
7490 VmaVector< BlockInfo, VmaStlAllocator<BlockInfo> > m_BlockInfos;
7491
7492 void PreprocessMetadata();
7493 void PostprocessMetadata();
7494 void InsertSuballoc(VmaBlockMetadata_Generic* pMetadata, const VmaSuballocation& suballoc);
7495 };
7496
7497 struct VmaBlockDefragmentationContext
7498 {
7499 enum BLOCK_FLAG
7500 {
7501 BLOCK_FLAG_USED = 0x00000001,
7502 };
7503 uint32_t flags;
7504 VkBuffer hBuffer;
7505 };
7506
7507 class VmaBlockVectorDefragmentationContext
7508 {
7509 VMA_CLASS_NO_COPY(VmaBlockVectorDefragmentationContext)
7510 public:
7511 VkResult res;
7512 bool mutexLocked;
7513 VmaVector< VmaBlockDefragmentationContext, VmaStlAllocator<VmaBlockDefragmentationContext> > blockContexts;
7514 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> > defragmentationMoves;
7515 uint32_t defragmentationMovesProcessed;
7516 uint32_t defragmentationMovesCommitted;
7517 bool hasDefragmentationPlan;
7518
7519 VmaBlockVectorDefragmentationContext(
7520 VmaAllocator hAllocator,
7521 VmaPool hCustomPool, // Optional.
7522 VmaBlockVector* pBlockVector,
7523 uint32_t currFrameIndex);
7524 ~VmaBlockVectorDefragmentationContext();
7525
7526 VmaPool GetCustomPool() const { return m_hCustomPool; }
7527 VmaBlockVector* GetBlockVector() const { return m_pBlockVector; }
7528 VmaDefragmentationAlgorithm* GetAlgorithm() const { return m_pAlgorithm; }
7529
7530 void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
7531 void AddAll() { m_AllAllocations = true; }
7532
7533 void Begin(bool overlappingMoveSupported, VmaDefragmentationFlags flags);
7534
7535 private:
7536 const VmaAllocator m_hAllocator;
7537 // Null if not from custom pool.
7538 const VmaPool m_hCustomPool;
7539 // Redundant, for convenience not to fetch from m_hCustomPool->m_BlockVector or m_hAllocator->m_pBlockVectors.
7540 VmaBlockVector* const m_pBlockVector;
7541 const uint32_t m_CurrFrameIndex;
7542 // Owner of this object.
7543 VmaDefragmentationAlgorithm* m_pAlgorithm;
7544
7545 struct AllocInfo
7546 {
7547 VmaAllocation hAlloc;
7548 VkBool32* pChanged;
7549 };
7550 // Used between constructor and Begin.
7551 VmaVector< AllocInfo, VmaStlAllocator<AllocInfo> > m_Allocations;
7552 bool m_AllAllocations;
7553 };
7554
7555 struct VmaDefragmentationContext_T
7556 {
7557 private:
7558 VMA_CLASS_NO_COPY(VmaDefragmentationContext_T)
7559 public:
7560 VmaDefragmentationContext_T(
7561 VmaAllocator hAllocator,
7562 uint32_t currFrameIndex,
7563 uint32_t flags,
7564 VmaDefragmentationStats* pStats);
7565 ~VmaDefragmentationContext_T();
7566
7567 void AddPools(uint32_t poolCount, const VmaPool* pPools);
7568 void AddAllocations(
7569 uint32_t allocationCount,
7570 const VmaAllocation* pAllocations,
7571 VkBool32* pAllocationsChanged);
7572
7573 /*
7574 Returns:
7575 - `VK_SUCCESS` if succeeded and object can be destroyed immediately.
7576 - `VK_NOT_READY` if succeeded but the object must remain alive until vmaDefragmentationEnd().
7577 - Negative value if error occured and object can be destroyed immediately.
7578 */
7579 VkResult Defragment(
7580 VkDeviceSize maxCpuBytesToMove, uint32_t maxCpuAllocationsToMove,
7581 VkDeviceSize maxGpuBytesToMove, uint32_t maxGpuAllocationsToMove,
7582 VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags);
7583
7584 VkResult DefragmentPassBegin(VmaDefragmentationPassInfo* pInfo);
7585 VkResult DefragmentPassEnd();
7586
7587 private:
7588 const VmaAllocator m_hAllocator;
7589 const uint32_t m_CurrFrameIndex;
7590 const uint32_t m_Flags;
7591 VmaDefragmentationStats* const m_pStats;
7592
7593 VkDeviceSize m_MaxCpuBytesToMove;
7594 uint32_t m_MaxCpuAllocationsToMove;
7595 VkDeviceSize m_MaxGpuBytesToMove;
7596 uint32_t m_MaxGpuAllocationsToMove;
7597
7598 // Owner of these objects.
7599 VmaBlockVectorDefragmentationContext* m_DefaultPoolContexts[VK_MAX_MEMORY_TYPES];
7600 // Owner of these objects.
7601 VmaVector< VmaBlockVectorDefragmentationContext*, VmaStlAllocator<VmaBlockVectorDefragmentationContext*> > m_CustomPoolContexts;
7602 };
7603
7604 #if VMA_RECORDING_ENABLED
7605
7606 class VmaRecorder
7607 {
7608 public:
7609 VmaRecorder();
7610 VkResult Init(const VmaRecordSettings& settings, bool useMutex);
7611 void WriteConfiguration(
7612 const VkPhysicalDeviceProperties& devProps,
7613 const VkPhysicalDeviceMemoryProperties& memProps,
7614 uint32_t vulkanApiVersion,
7615 bool dedicatedAllocationExtensionEnabled,
7616 bool bindMemory2ExtensionEnabled,
7617 bool memoryBudgetExtensionEnabled,
7618 bool deviceCoherentMemoryExtensionEnabled);
7619 ~VmaRecorder();
7620
7621 void RecordCreateAllocator(uint32_t frameIndex);
7622 void RecordDestroyAllocator(uint32_t frameIndex);
7623 void RecordCreatePool(uint32_t frameIndex,
7624 const VmaPoolCreateInfo& createInfo,
7625 VmaPool pool);
7626 void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
7627 void RecordAllocateMemory(uint32_t frameIndex,
7628 const VkMemoryRequirements& vkMemReq,
7629 const VmaAllocationCreateInfo& createInfo,
7630 VmaAllocation allocation);
7631 void RecordAllocateMemoryPages(uint32_t frameIndex,
7632 const VkMemoryRequirements& vkMemReq,
7633 const VmaAllocationCreateInfo& createInfo,
7634 uint64_t allocationCount,
7635 const VmaAllocation* pAllocations);
7636 void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
7637 const VkMemoryRequirements& vkMemReq,
7638 bool requiresDedicatedAllocation,
7639 bool prefersDedicatedAllocation,
7640 const VmaAllocationCreateInfo& createInfo,
7641 VmaAllocation allocation);
7642 void RecordAllocateMemoryForImage(uint32_t frameIndex,
7643 const VkMemoryRequirements& vkMemReq,
7644 bool requiresDedicatedAllocation,
7645 bool prefersDedicatedAllocation,
7646 const VmaAllocationCreateInfo& createInfo,
7647 VmaAllocation allocation);
7648 void RecordFreeMemory(uint32_t frameIndex,
7649 VmaAllocation allocation);
7650 void RecordFreeMemoryPages(uint32_t frameIndex,
7651 uint64_t allocationCount,
7652 const VmaAllocation* pAllocations);
7653 void RecordSetAllocationUserData(uint32_t frameIndex,
7654 VmaAllocation allocation,
7655 const void* pUserData);
7656 void RecordCreateLostAllocation(uint32_t frameIndex,
7657 VmaAllocation allocation);
7658 void RecordMapMemory(uint32_t frameIndex,
7659 VmaAllocation allocation);
7660 void RecordUnmapMemory(uint32_t frameIndex,
7661 VmaAllocation allocation);
7662 void RecordFlushAllocation(uint32_t frameIndex,
7663 VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
7664 void RecordInvalidateAllocation(uint32_t frameIndex,
7665 VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
7666 void RecordCreateBuffer(uint32_t frameIndex,
7667 const VkBufferCreateInfo& bufCreateInfo,
7668 const VmaAllocationCreateInfo& allocCreateInfo,
7669 VmaAllocation allocation);
7670 void RecordCreateImage(uint32_t frameIndex,
7671 const VkImageCreateInfo& imageCreateInfo,
7672 const VmaAllocationCreateInfo& allocCreateInfo,
7673 VmaAllocation allocation);
7674 void RecordDestroyBuffer(uint32_t frameIndex,
7675 VmaAllocation allocation);
7676 void RecordDestroyImage(uint32_t frameIndex,
7677 VmaAllocation allocation);
7678 void RecordTouchAllocation(uint32_t frameIndex,
7679 VmaAllocation allocation);
7680 void RecordGetAllocationInfo(uint32_t frameIndex,
7681 VmaAllocation allocation);
7682 void RecordMakePoolAllocationsLost(uint32_t frameIndex,
7683 VmaPool pool);
7684 void RecordDefragmentationBegin(uint32_t frameIndex,
7685 const VmaDefragmentationInfo2& info,
7686 VmaDefragmentationContext ctx);
7687 void RecordDefragmentationEnd(uint32_t frameIndex,
7688 VmaDefragmentationContext ctx);
7689 void RecordSetPoolName(uint32_t frameIndex,
7690 VmaPool pool,
7691 const char* name);
7692
7693 private:
7694 struct CallParams
7695 {
7696 uint32_t threadId;
7697 double time;
7698 };
7699
7700 class UserDataString
7701 {
7702 public:
7703 UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
7704 const char* GetString() const { return m_Str; }
7705
7706 private:
7707 char m_PtrStr[17];
7708 const char* m_Str;
7709 };
7710
7711 bool m_UseMutex;
7712 VmaRecordFlags m_Flags;
7713 FILE* m_File;
7714 VMA_MUTEX m_FileMutex;
7715 std::chrono::time_point<std::chrono::high_resolution_clock> m_RecordingStartTime;
7716
7717 void GetBasicParams(CallParams& outParams);
7718
7719 // T must be a pointer type, e.g. VmaAllocation, VmaPool.
7720 template<typename T>
7721 void PrintPointerList(uint64_t count, const T* pItems)
7722 {
7723 if(count)
7724 {
7725 fprintf(m_File, "%p", pItems[0]);
7726 for(uint64_t i = 1; i < count; ++i)
7727 {
7728 fprintf(m_File, " %p", pItems[i]);
7729 }
7730 }
7731 }
7732
7733 void PrintPointerList(uint64_t count, const VmaAllocation* pItems);
7734 void Flush();
7735 };
7736
7737 #endif // #if VMA_RECORDING_ENABLED
7738
7739 /*
7740 Thread-safe wrapper over VmaPoolAllocator free list, for allocation of VmaAllocation_T objects.
7741 */
7742 class VmaAllocationObjectAllocator
7743 {
7744 VMA_CLASS_NO_COPY(VmaAllocationObjectAllocator)
7745 public:
7746 VmaAllocationObjectAllocator(const VkAllocationCallbacks* pAllocationCallbacks);
7747
7748 template<typename... Types> VmaAllocation Allocate(Types... args);
7749 void Free(VmaAllocation hAlloc);
7750
7751 private:
7752 VMA_MUTEX m_Mutex;
7753 VmaPoolAllocator<VmaAllocation_T> m_Allocator;
7754 };
7755
7756 struct VmaCurrentBudgetData
7757 {
7758 VMA_ATOMIC_UINT64 m_BlockBytes[VK_MAX_MEMORY_HEAPS];
7759 VMA_ATOMIC_UINT64 m_AllocationBytes[VK_MAX_MEMORY_HEAPS];
7760
7761 #if VMA_MEMORY_BUDGET
7762 VMA_ATOMIC_UINT32 m_OperationsSinceBudgetFetch;
7763 VMA_RW_MUTEX m_BudgetMutex;
7764 uint64_t m_VulkanUsage[VK_MAX_MEMORY_HEAPS];
7765 uint64_t m_VulkanBudget[VK_MAX_MEMORY_HEAPS];
7766 uint64_t m_BlockBytesAtBudgetFetch[VK_MAX_MEMORY_HEAPS];
7767 #endif // #if VMA_MEMORY_BUDGET
7768
7769 VmaCurrentBudgetData()
7770 {
7771 for(uint32_t heapIndex = 0; heapIndex < VK_MAX_MEMORY_HEAPS; ++heapIndex)
7772 {
7773 m_BlockBytes[heapIndex] = 0;
7774 m_AllocationBytes[heapIndex] = 0;
7775 #if VMA_MEMORY_BUDGET
7776 m_VulkanUsage[heapIndex] = 0;
7777 m_VulkanBudget[heapIndex] = 0;
7778 m_BlockBytesAtBudgetFetch[heapIndex] = 0;
7779 #endif
7780 }
7781
7782 #if VMA_MEMORY_BUDGET
7783 m_OperationsSinceBudgetFetch = 0;
7784 #endif
7785 }
7786
7787 void AddAllocation(uint32_t heapIndex, VkDeviceSize allocationSize)
7788 {
7789 m_AllocationBytes[heapIndex] += allocationSize;
7790 #if VMA_MEMORY_BUDGET
7791 ++m_OperationsSinceBudgetFetch;
7792 #endif
7793 }
7794
7795 void RemoveAllocation(uint32_t heapIndex, VkDeviceSize allocationSize)
7796 {
7797 VMA_ASSERT(m_AllocationBytes[heapIndex] >= allocationSize); // DELME
7798 m_AllocationBytes[heapIndex] -= allocationSize;
7799 #if VMA_MEMORY_BUDGET
7800 ++m_OperationsSinceBudgetFetch;
7801 #endif
7802 }
7803 };
7804
7805 // Main allocator object.
7806 struct VmaAllocator_T
7807 {
7808 VMA_CLASS_NO_COPY(VmaAllocator_T)
7809 public:
7810 bool m_UseMutex;
7811 uint32_t m_VulkanApiVersion;
7812 bool m_UseKhrDedicatedAllocation; // Can be set only if m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0).
7813 bool m_UseKhrBindMemory2; // Can be set only if m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0).
7814 bool m_UseExtMemoryBudget;
7815 bool m_UseAmdDeviceCoherentMemory;
7816 bool m_UseKhrBufferDeviceAddress;
7817 VkDevice m_hDevice;
7818 VkInstance m_hInstance;
7819 bool m_AllocationCallbacksSpecified;
7820 VkAllocationCallbacks m_AllocationCallbacks;
7821 VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
7822 VmaAllocationObjectAllocator m_AllocationObjectAllocator;
7823
7824 // Each bit (1 << i) is set if HeapSizeLimit is enabled for that heap, so cannot allocate more than the heap size.
7825 uint32_t m_HeapSizeLimitMask;
7826
7827 VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
7828 VkPhysicalDeviceMemoryProperties m_MemProps;
7829
7830 // Default pools.
7831 VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
7832
7833 // Each vector is sorted by memory (handle value).
7834 typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
7835 AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
7836 VMA_RW_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
7837
7838 VmaCurrentBudgetData m_Budget;
7839
7840 VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
7841 VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
7842 ~VmaAllocator_T();
7843
7844 const VkAllocationCallbacks* GetAllocationCallbacks() const
7845 {
7846 return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
7847 }
7848 const VmaVulkanFunctions& GetVulkanFunctions() const
7849 {
7850 return m_VulkanFunctions;
7851 }
7852
7853 VkPhysicalDevice GetPhysicalDevice() const { return m_PhysicalDevice; }
7854
7855 VkDeviceSize GetBufferImageGranularity() const
7856 {
7857 return VMA_MAX(
7858 static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
7859 m_PhysicalDeviceProperties.limits.bufferImageGranularity);
7860 }
7861
7862 uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
7863 uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
7864
7865 uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
7866 {
7867 VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
7868 return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
7869 }
7870 // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
7871 bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
7872 {
7873 return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
7874 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7875 }
7876 // Minimum alignment for all allocations in specific memory type.
7877 VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
7878 {
7879 return IsMemoryTypeNonCoherent(memTypeIndex) ?
7880 VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
7881 (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
7882 }
7883
7884 bool IsIntegratedGpu() const
7885 {
7886 return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
7887 }
7888
7889 uint32_t GetGlobalMemoryTypeBits() const { return m_GlobalMemoryTypeBits; }
7890
7891 #if VMA_RECORDING_ENABLED
7892 VmaRecorder* GetRecorder() const { return m_pRecorder; }
7893 #endif
7894
7895 void GetBufferMemoryRequirements(
7896 VkBuffer hBuffer,
7897 VkMemoryRequirements& memReq,
7898 bool& requiresDedicatedAllocation,
7899 bool& prefersDedicatedAllocation) const;
7900 void GetImageMemoryRequirements(
7901 VkImage hImage,
7902 VkMemoryRequirements& memReq,
7903 bool& requiresDedicatedAllocation,
7904 bool& prefersDedicatedAllocation) const;
7905
7906 // Main allocation function.
7907 VkResult AllocateMemory(
7908 const VkMemoryRequirements& vkMemReq,
7909 bool requiresDedicatedAllocation,
7910 bool prefersDedicatedAllocation,
7911 VkBuffer dedicatedBuffer,
7912 VkBufferUsageFlags dedicatedBufferUsage, // UINT32_MAX when unknown.
7913 VkImage dedicatedImage,
7914 const VmaAllocationCreateInfo& createInfo,
7915 VmaSuballocationType suballocType,
7916 size_t allocationCount,
7917 VmaAllocation* pAllocations);
7918
7919 // Main deallocation function.
7920 void FreeMemory(
7921 size_t allocationCount,
7922 const VmaAllocation* pAllocations);
7923
7924 VkResult ResizeAllocation(
7925 const VmaAllocation alloc,
7926 VkDeviceSize newSize);
7927
7928 void CalculateStats(VmaStats* pStats);
7929
7930 void GetBudget(
7931 VmaBudget* outBudget, uint32_t firstHeap, uint32_t heapCount);
7932
7933 #if VMA_STATS_STRING_ENABLED
7934 void PrintDetailedMap(class VmaJsonWriter& json);
7935 #endif
7936
7937 VkResult DefragmentationBegin(
7938 const VmaDefragmentationInfo2& info,
7939 VmaDefragmentationStats* pStats,
7940 VmaDefragmentationContext* pContext);
7941 VkResult DefragmentationEnd(
7942 VmaDefragmentationContext context);
7943
7944 VkResult DefragmentationPassBegin(
7945 VmaDefragmentationPassInfo* pInfo,
7946 VmaDefragmentationContext context);
7947 VkResult DefragmentationPassEnd(
7948 VmaDefragmentationContext context);
7949
7950 void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
7951 bool TouchAllocation(VmaAllocation hAllocation);
7952
7953 VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
7954 void DestroyPool(VmaPool pool);
7955 void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
7956
7957 void SetCurrentFrameIndex(uint32_t frameIndex);
7958 uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
7959
7960 void MakePoolAllocationsLost(
7961 VmaPool hPool,
7962 size_t* pLostAllocationCount);
7963 VkResult CheckPoolCorruption(VmaPool hPool);
7964 VkResult CheckCorruption(uint32_t memoryTypeBits);
7965
7966 void CreateLostAllocation(VmaAllocation* pAllocation);
7967
7968 // Call to Vulkan function vkAllocateMemory with accompanying bookkeeping.
7969 VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
7970 // Call to Vulkan function vkFreeMemory with accompanying bookkeeping.
7971 void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
7972 // Call to Vulkan function vkBindBufferMemory or vkBindBufferMemory2KHR.
7973 VkResult BindVulkanBuffer(
7974 VkDeviceMemory memory,
7975 VkDeviceSize memoryOffset,
7976 VkBuffer buffer,
7977 const void* pNext);
7978 // Call to Vulkan function vkBindImageMemory or vkBindImageMemory2KHR.
7979 VkResult BindVulkanImage(
7980 VkDeviceMemory memory,
7981 VkDeviceSize memoryOffset,
7982 VkImage image,
7983 const void* pNext);
7984
7985 VkResult Map(VmaAllocation hAllocation, void** ppData);
7986 void Unmap(VmaAllocation hAllocation);
7987
7988 VkResult BindBufferMemory(
7989 VmaAllocation hAllocation,
7990 VkDeviceSize allocationLocalOffset,
7991 VkBuffer hBuffer,
7992 const void* pNext);
7993 VkResult BindImageMemory(
7994 VmaAllocation hAllocation,
7995 VkDeviceSize allocationLocalOffset,
7996 VkImage hImage,
7997 const void* pNext);
7998
7999 VkResult FlushOrInvalidateAllocation(
8000 VmaAllocation hAllocation,
8001 VkDeviceSize offset, VkDeviceSize size,
8002 VMA_CACHE_OPERATION op);
8003 VkResult FlushOrInvalidateAllocations(
8004 uint32_t allocationCount,
8005 const VmaAllocation* allocations,
8006 const VkDeviceSize* offsets, const VkDeviceSize* sizes,
8007 VMA_CACHE_OPERATION op);
8008
8009 void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
8010
8011 /*
8012 Returns bit mask of memory types that can support defragmentation on GPU as
8013 they support creation of required buffer for copy operations.
8014 */
8015 uint32_t GetGpuDefragmentationMemoryTypeBits();
8016
8017 private:
8018 VkDeviceSize m_PreferredLargeHeapBlockSize;
8019
8020 VkPhysicalDevice m_PhysicalDevice;
8021 VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
8022 VMA_ATOMIC_UINT32 m_GpuDefragmentationMemoryTypeBits; // UINT32_MAX means uninitialized.
8023
8024 VMA_RW_MUTEX m_PoolsMutex;
8025 // Protected by m_PoolsMutex. Sorted by pointer value.
8026 VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
8027 uint32_t m_NextPoolId;
8028
8029 VmaVulkanFunctions m_VulkanFunctions;
8030
8031 // Global bit mask AND-ed with any memoryTypeBits to disallow certain memory types.
8032 uint32_t m_GlobalMemoryTypeBits;
8033
8034 #if VMA_RECORDING_ENABLED
8035 VmaRecorder* m_pRecorder;
8036 #endif
8037
8038 void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
8039
8040 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
8041 void ImportVulkanFunctions_Static();
8042 #endif
8043
8044 void ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVulkanFunctions);
8045
8046 #if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1
8047 void ImportVulkanFunctions_Dynamic();
8048 #endif
8049
8050 void ValidateVulkanFunctions();
8051
8052 VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
8053
8054 VkResult AllocateMemoryOfType(
8055 VkDeviceSize size,
8056 VkDeviceSize alignment,
8057 bool dedicatedAllocation,
8058 VkBuffer dedicatedBuffer,
8059 VkBufferUsageFlags dedicatedBufferUsage,
8060 VkImage dedicatedImage,
8061 const VmaAllocationCreateInfo& createInfo,
8062 uint32_t memTypeIndex,
8063 VmaSuballocationType suballocType,
8064 size_t allocationCount,
8065 VmaAllocation* pAllocations);
8066
8067 // Helper function only to be used inside AllocateDedicatedMemory.
8068 VkResult AllocateDedicatedMemoryPage(
8069 VkDeviceSize size,
8070 VmaSuballocationType suballocType,
8071 uint32_t memTypeIndex,
8072 const VkMemoryAllocateInfo& allocInfo,
8073 bool map,
8074 bool isUserDataString,
8075 void* pUserData,
8076 VmaAllocation* pAllocation);
8077
8078 // Allocates and registers new VkDeviceMemory specifically for dedicated allocations.
8079 VkResult AllocateDedicatedMemory(
8080 VkDeviceSize size,
8081 VmaSuballocationType suballocType,
8082 uint32_t memTypeIndex,
8083 bool withinBudget,
8084 bool map,
8085 bool isUserDataString,
8086 void* pUserData,
8087 VkBuffer dedicatedBuffer,
8088 VkBufferUsageFlags dedicatedBufferUsage,
8089 VkImage dedicatedImage,
8090 size_t allocationCount,
8091 VmaAllocation* pAllocations);
8092
8093 void FreeDedicatedMemory(const VmaAllocation allocation);
8094
8095 /*
8096 Calculates and returns bit mask of memory types that can support defragmentation
8097 on GPU as they support creation of required buffer for copy operations.
8098 */
8099 uint32_t CalculateGpuDefragmentationMemoryTypeBits() const;
8100
8101 uint32_t CalculateGlobalMemoryTypeBits() const;
8102
8103 bool GetFlushOrInvalidateRange(
8104 VmaAllocation allocation,
8105 VkDeviceSize offset, VkDeviceSize size,
8106 VkMappedMemoryRange& outRange) const;
8107
8108 #if VMA_MEMORY_BUDGET
8109 void UpdateVulkanBudget();
8110 #endif // #if VMA_MEMORY_BUDGET
8111 };
8112
8113 ////////////////////////////////////////////////////////////////////////////////
8114 // Memory allocation #2 after VmaAllocator_T definition
8115
8116 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
8117 {
8118 return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
8119 }
8120
8121 static void VmaFree(VmaAllocator hAllocator, void* ptr)
8122 {
8123 VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
8124 }
8125
8126 template<typename T>
8127 static T* VmaAllocate(VmaAllocator hAllocator)
8128 {
8129 return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
8130 }
8131
8132 template<typename T>
8133 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
8134 {
8135 return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
8136 }
8137
8138 template<typename T>
8139 static void vma_delete(VmaAllocator hAllocator, T* ptr)
8140 {
8141 if(ptr != VMA_NULL)
8142 {
8143 ptr->~T();
8144 VmaFree(hAllocator, ptr);
8145 }
8146 }
8147
8148 template<typename T>
8149 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
8150 {
8151 if(ptr != VMA_NULL)
8152 {
8153 for(size_t i = count; i--; )
8154 ptr[i].~T();
8155 VmaFree(hAllocator, ptr);
8156 }
8157 }
8158
8159 ////////////////////////////////////////////////////////////////////////////////
8160 // VmaStringBuilder
8161
8162 #if VMA_STATS_STRING_ENABLED
8163
8164 class VmaStringBuilder
8165 {
8166 public:
8167 VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
8168 size_t GetLength() const { return m_Data.size(); }
8169 const char* GetData() const { return m_Data.data(); }
8170
8171 void Add(char ch) { m_Data.push_back(ch); }
8172 void Add(const char* pStr);
8173 void AddNewLine() { Add('\n'); }
8174 void AddNumber(uint32_t num);
8175 void AddNumber(uint64_t num);
8176 void AddPointer(const void* ptr);
8177
8178 private:
8179 VmaVector< char, VmaStlAllocator<char> > m_Data;
8180 };
8181
8182 void VmaStringBuilder::Add(const char* pStr)
8183 {
8184 const size_t strLen = strlen(pStr);
8185 if(strLen > 0)
8186 {
8187 const size_t oldCount = m_Data.size();
8188 m_Data.resize(oldCount + strLen);
8189 memcpy(m_Data.data() + oldCount, pStr, strLen);
8190 }
8191 }
8192
8193 void VmaStringBuilder::AddNumber(uint32_t num)
8194 {
8195 char buf[11];
8196 buf[10] = '\0';
8197 char *p = &buf[10];
8198 do
8199 {
8200 *--p = '0' + (num % 10);
8201 num /= 10;
8202 }
8203 while(num);
8204 Add(p);
8205 }
8206
8207 void VmaStringBuilder::AddNumber(uint64_t num)
8208 {
8209 char buf[21];
8210 buf[20] = '\0';
8211 char *p = &buf[20];
8212 do
8213 {
8214 *--p = '0' + (num % 10);
8215 num /= 10;
8216 }
8217 while(num);
8218 Add(p);
8219 }
8220
8221 void VmaStringBuilder::AddPointer(const void* ptr)
8222 {
8223 char buf[21];
8224 VmaPtrToStr(buf, sizeof(buf), ptr);
8225 Add(buf);
8226 }
8227
8228 #endif // #if VMA_STATS_STRING_ENABLED
8229
8230 ////////////////////////////////////////////////////////////////////////////////
8231 // VmaJsonWriter
8232
8233 #if VMA_STATS_STRING_ENABLED
8234
8235 class VmaJsonWriter
8236 {
8237 VMA_CLASS_NO_COPY(VmaJsonWriter)
8238 public:
8239 VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
8240 ~VmaJsonWriter();
8241
8242 void BeginObject(bool singleLine = false);
8243 void EndObject();
8244
8245 void BeginArray(bool singleLine = false);
8246 void EndArray();
8247
8248 void WriteString(const char* pStr);
8249 void BeginString(const char* pStr = VMA_NULL);
8250 void ContinueString(const char* pStr);
8251 void ContinueString(uint32_t n);
8252 void ContinueString(uint64_t n);
8253 void ContinueString_Pointer(const void* ptr);
8254 void EndString(const char* pStr = VMA_NULL);
8255
8256 void WriteNumber(uint32_t n);
8257 void WriteNumber(uint64_t n);
8258 void WriteBool(bool b);
8259 void WriteNull();
8260
8261 private:
8262 static const char* const INDENT;
8263
8264 enum COLLECTION_TYPE
8265 {
8266 COLLECTION_TYPE_OBJECT,
8267 COLLECTION_TYPE_ARRAY,
8268 };
8269 struct StackItem
8270 {
8271 COLLECTION_TYPE type;
8272 uint32_t valueCount;
8273 bool singleLineMode;
8274 };
8275
8276 VmaStringBuilder& m_SB;
8277 VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
8278 bool m_InsideString;
8279
8280 void BeginValue(bool isString);
8281 void WriteIndent(bool oneLess = false);
8282 };
8283
8284 const char* const VmaJsonWriter::INDENT = " ";
8285
8286 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
8287 m_SB(sb),
8288 m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
8289 m_InsideString(false)
8290 {
8291 }
8292
8293 VmaJsonWriter::~VmaJsonWriter()
8294 {
8295 VMA_ASSERT(!m_InsideString);
8296 VMA_ASSERT(m_Stack.empty());
8297 }
8298
8299 void VmaJsonWriter::BeginObject(bool singleLine)
8300 {
8301 VMA_ASSERT(!m_InsideString);
8302
8303 BeginValue(false);
8304 m_SB.Add('{');
8305
8306 StackItem item;
8307 item.type = COLLECTION_TYPE_OBJECT;
8308 item.valueCount = 0;
8309 item.singleLineMode = singleLine;
8310 m_Stack.push_back(item);
8311 }
8312
8313 void VmaJsonWriter::EndObject()
8314 {
8315 VMA_ASSERT(!m_InsideString);
8316
8317 WriteIndent(true);
8318 m_SB.Add('}');
8319
8320 VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
8321 m_Stack.pop_back();
8322 }
8323
8324 void VmaJsonWriter::BeginArray(bool singleLine)
8325 {
8326 VMA_ASSERT(!m_InsideString);
8327
8328 BeginValue(false);
8329 m_SB.Add('[');
8330
8331 StackItem item;
8332 item.type = COLLECTION_TYPE_ARRAY;
8333 item.valueCount = 0;
8334 item.singleLineMode = singleLine;
8335 m_Stack.push_back(item);
8336 }
8337
8338 void VmaJsonWriter::EndArray()
8339 {
8340 VMA_ASSERT(!m_InsideString);
8341
8342 WriteIndent(true);
8343 m_SB.Add(']');
8344
8345 VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
8346 m_Stack.pop_back();
8347 }
8348
8349 void VmaJsonWriter::WriteString(const char* pStr)
8350 {
8351 BeginString(pStr);
8352 EndString();
8353 }
8354
8355 void VmaJsonWriter::BeginString(const char* pStr)
8356 {
8357 VMA_ASSERT(!m_InsideString);
8358
8359 BeginValue(true);
8360 m_SB.Add('"');
8361 m_InsideString = true;
8362 if(pStr != VMA_NULL && pStr[0] != '\0')
8363 {
8364 ContinueString(pStr);
8365 }
8366 }
8367
8368 void VmaJsonWriter::ContinueString(const char* pStr)
8369 {
8370 VMA_ASSERT(m_InsideString);
8371
8372 const size_t strLen = strlen(pStr);
8373 for(size_t i = 0; i < strLen; ++i)
8374 {
8375 char ch = pStr[i];
8376 if(ch == '\\')
8377 {
8378 m_SB.Add("\\\\");
8379 }
8380 else if(ch == '"')
8381 {
8382 m_SB.Add("\\\"");
8383 }
8384 else if(ch >= 32)
8385 {
8386 m_SB.Add(ch);
8387 }
8388 else switch(ch)
8389 {
8390 case '\b':
8391 m_SB.Add("\\b");
8392 break;
8393 case '\f':
8394 m_SB.Add("\\f");
8395 break;
8396 case '\n':
8397 m_SB.Add("\\n");
8398 break;
8399 case '\r':
8400 m_SB.Add("\\r");
8401 break;
8402 case '\t':
8403 m_SB.Add("\\t");
8404 break;
8405 default:
8406 VMA_ASSERT(0 && "Character not currently supported.");
8407 break;
8408 }
8409 }
8410 }
8411
8412 void VmaJsonWriter::ContinueString(uint32_t n)
8413 {
8414 VMA_ASSERT(m_InsideString);
8415 m_SB.AddNumber(n);
8416 }
8417
8418 void VmaJsonWriter::ContinueString(uint64_t n)
8419 {
8420 VMA_ASSERT(m_InsideString);
8421 m_SB.AddNumber(n);
8422 }
8423
8424 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
8425 {
8426 VMA_ASSERT(m_InsideString);
8427 m_SB.AddPointer(ptr);
8428 }
8429
8430 void VmaJsonWriter::EndString(const char* pStr)
8431 {
8432 VMA_ASSERT(m_InsideString);
8433 if(pStr != VMA_NULL && pStr[0] != '\0')
8434 {
8435 ContinueString(pStr);
8436 }
8437 m_SB.Add('"');
8438 m_InsideString = false;
8439 }
8440
8441 void VmaJsonWriter::WriteNumber(uint32_t n)
8442 {
8443 VMA_ASSERT(!m_InsideString);
8444 BeginValue(false);
8445 m_SB.AddNumber(n);
8446 }
8447
8448 void VmaJsonWriter::WriteNumber(uint64_t n)
8449 {
8450 VMA_ASSERT(!m_InsideString);
8451 BeginValue(false);
8452 m_SB.AddNumber(n);
8453 }
8454
8455 void VmaJsonWriter::WriteBool(bool b)
8456 {
8457 VMA_ASSERT(!m_InsideString);
8458 BeginValue(false);
8459 m_SB.Add(b ? "true" : "false");
8460 }
8461
8462 void VmaJsonWriter::WriteNull()
8463 {
8464 VMA_ASSERT(!m_InsideString);
8465 BeginValue(false);
8466 m_SB.Add("null");
8467 }
8468
8469 void VmaJsonWriter::BeginValue(bool isString)
8470 {
8471 if(!m_Stack.empty())
8472 {
8473 StackItem& currItem = m_Stack.back();
8474 if(currItem.type == COLLECTION_TYPE_OBJECT &&
8475 currItem.valueCount % 2 == 0)
8476 {
8477 VMA_ASSERT(isString);
8478 }
8479
8480 if(currItem.type == COLLECTION_TYPE_OBJECT &&
8481 currItem.valueCount % 2 != 0)
8482 {
8483 m_SB.Add(": ");
8484 }
8485 else if(currItem.valueCount > 0)
8486 {
8487 m_SB.Add(", ");
8488 WriteIndent();
8489 }
8490 else
8491 {
8492 WriteIndent();
8493 }
8494 ++currItem.valueCount;
8495 }
8496 }
8497
8498 void VmaJsonWriter::WriteIndent(bool oneLess)
8499 {
8500 if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
8501 {
8502 m_SB.AddNewLine();
8503
8504 size_t count = m_Stack.size();
8505 if(count > 0 && oneLess)
8506 {
8507 --count;
8508 }
8509 for(size_t i = 0; i < count; ++i)
8510 {
8511 m_SB.Add(INDENT);
8512 }
8513 }
8514 }
8515
8516 #endif // #if VMA_STATS_STRING_ENABLED
8517
8518 ////////////////////////////////////////////////////////////////////////////////
8519
8520 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
8521 {
8522 if(IsUserDataString())
8523 {
8524 VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
8525
8526 FreeUserDataString(hAllocator);
8527
8528 if(pUserData != VMA_NULL)
8529 {
8530 m_pUserData = VmaCreateStringCopy(hAllocator->GetAllocationCallbacks(), (const char*)pUserData);
8531 }
8532 }
8533 else
8534 {
8535 m_pUserData = pUserData;
8536 }
8537 }
8538
8539 void VmaAllocation_T::ChangeBlockAllocation(
8540 VmaAllocator hAllocator,
8541 VmaDeviceMemoryBlock* block,
8542 VkDeviceSize offset)
8543 {
8544 VMA_ASSERT(block != VMA_NULL);
8545 VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
8546
8547 // Move mapping reference counter from old block to new block.
8548 if(block != m_BlockAllocation.m_Block)
8549 {
8550 uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
8551 if(IsPersistentMap())
8552 ++mapRefCount;
8553 m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
8554 block->Map(hAllocator, mapRefCount, VMA_NULL);
8555 }
8556
8557 m_BlockAllocation.m_Block = block;
8558 m_BlockAllocation.m_Offset = offset;
8559 }
8560
8561 void VmaAllocation_T::ChangeOffset(VkDeviceSize newOffset)
8562 {
8563 VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
8564 m_BlockAllocation.m_Offset = newOffset;
8565 }
8566
8567 VkDeviceSize VmaAllocation_T::GetOffset() const
8568 {
8569 switch(m_Type)
8570 {
8571 case ALLOCATION_TYPE_BLOCK:
8572 return m_BlockAllocation.m_Offset;
8573 case ALLOCATION_TYPE_DEDICATED:
8574 return 0;
8575 default:
8576 VMA_ASSERT(0);
8577 return 0;
8578 }
8579 }
8580
8581 VkDeviceMemory VmaAllocation_T::GetMemory() const
8582 {
8583 switch(m_Type)
8584 {
8585 case ALLOCATION_TYPE_BLOCK:
8586 return m_BlockAllocation.m_Block->GetDeviceMemory();
8587 case ALLOCATION_TYPE_DEDICATED:
8588 return m_DedicatedAllocation.m_hMemory;
8589 default:
8590 VMA_ASSERT(0);
8591 return VK_NULL_HANDLE;
8592 }
8593 }
8594
8595 void* VmaAllocation_T::GetMappedData() const
8596 {
8597 switch(m_Type)
8598 {
8599 case ALLOCATION_TYPE_BLOCK:
8600 if(m_MapCount != 0)
8601 {
8602 void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
8603 VMA_ASSERT(pBlockData != VMA_NULL);
8604 return (char*)pBlockData + m_BlockAllocation.m_Offset;
8605 }
8606 else
8607 {
8608 return VMA_NULL;
8609 }
8610 break;
8611 case ALLOCATION_TYPE_DEDICATED:
8612 VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
8613 return m_DedicatedAllocation.m_pMappedData;
8614 default:
8615 VMA_ASSERT(0);
8616 return VMA_NULL;
8617 }
8618 }
8619
8620 bool VmaAllocation_T::CanBecomeLost() const
8621 {
8622 switch(m_Type)
8623 {
8624 case ALLOCATION_TYPE_BLOCK:
8625 return m_BlockAllocation.m_CanBecomeLost;
8626 case ALLOCATION_TYPE_DEDICATED:
8627 return false;
8628 default:
8629 VMA_ASSERT(0);
8630 return false;
8631 }
8632 }
8633
8634 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
8635 {
8636 VMA_ASSERT(CanBecomeLost());
8637
8638 /*
8639 Warning: This is a carefully designed algorithm.
8640 Do not modify unless you really know what you're doing :)
8641 */
8642 uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
8643 for(;;)
8644 {
8645 if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
8646 {
8647 VMA_ASSERT(0);
8648 return false;
8649 }
8650 else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
8651 {
8652 return false;
8653 }
8654 else // Last use time earlier than current time.
8655 {
8656 if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
8657 {
8658 // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
8659 // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
8660 return true;
8661 }
8662 }
8663 }
8664 }
8665
8666 #if VMA_STATS_STRING_ENABLED
8667
8668 // Correspond to values of enum VmaSuballocationType.
8669 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
8670 "FREE",
8671 "UNKNOWN",
8672 "BUFFER",
8673 "IMAGE_UNKNOWN",
8674 "IMAGE_LINEAR",
8675 "IMAGE_OPTIMAL",
8676 };
8677
8678 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
8679 {
8680 json.WriteString("Type");
8681 json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
8682
8683 json.WriteString("Size");
8684 json.WriteNumber(m_Size);
8685
8686 if(m_pUserData != VMA_NULL)
8687 {
8688 json.WriteString("UserData");
8689 if(IsUserDataString())
8690 {
8691 json.WriteString((const char*)m_pUserData);
8692 }
8693 else
8694 {
8695 json.BeginString();
8696 json.ContinueString_Pointer(m_pUserData);
8697 json.EndString();
8698 }
8699 }
8700
8701 json.WriteString("CreationFrameIndex");
8702 json.WriteNumber(m_CreationFrameIndex);
8703
8704 json.WriteString("LastUseFrameIndex");
8705 json.WriteNumber(GetLastUseFrameIndex());
8706
8707 if(m_BufferImageUsage != 0)
8708 {
8709 json.WriteString("Usage");
8710 json.WriteNumber(m_BufferImageUsage);
8711 }
8712 }
8713
8714 #endif
8715
8716 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
8717 {
8718 VMA_ASSERT(IsUserDataString());
8719 VmaFreeString(hAllocator->GetAllocationCallbacks(), (char*)m_pUserData);
8720 m_pUserData = VMA_NULL;
8721 }
8722
8723 void VmaAllocation_T::BlockAllocMap()
8724 {
8725 VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
8726
8727 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
8728 {
8729 ++m_MapCount;
8730 }
8731 else
8732 {
8733 VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
8734 }
8735 }
8736
8737 void VmaAllocation_T::BlockAllocUnmap()
8738 {
8739 VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
8740
8741 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
8742 {
8743 --m_MapCount;
8744 }
8745 else
8746 {
8747 VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
8748 }
8749 }
8750
8751 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
8752 {
8753 VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
8754
8755 if(m_MapCount != 0)
8756 {
8757 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
8758 {
8759 VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
8760 *ppData = m_DedicatedAllocation.m_pMappedData;
8761 ++m_MapCount;
8762 return VK_SUCCESS;
8763 }
8764 else
8765 {
8766 VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
8767 return VK_ERROR_MEMORY_MAP_FAILED;
8768 }
8769 }
8770 else
8771 {
8772 VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
8773 hAllocator->m_hDevice,
8774 m_DedicatedAllocation.m_hMemory,
8775 0, // offset
8776 VK_WHOLE_SIZE,
8777 0, // flags
8778 ppData);
8779 if(result == VK_SUCCESS)
8780 {
8781 m_DedicatedAllocation.m_pMappedData = *ppData;
8782 m_MapCount = 1;
8783 }
8784 return result;
8785 }
8786 }
8787
8788 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
8789 {
8790 VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
8791
8792 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
8793 {
8794 --m_MapCount;
8795 if(m_MapCount == 0)
8796 {
8797 m_DedicatedAllocation.m_pMappedData = VMA_NULL;
8798 (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
8799 hAllocator->m_hDevice,
8800 m_DedicatedAllocation.m_hMemory);
8801 }
8802 }
8803 else
8804 {
8805 VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
8806 }
8807 }
8808
8809 #if VMA_STATS_STRING_ENABLED
8810
8811 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
8812 {
8813 json.BeginObject();
8814
8815 json.WriteString("Blocks");
8816 json.WriteNumber(stat.blockCount);
8817
8818 json.WriteString("Allocations");
8819 json.WriteNumber(stat.allocationCount);
8820
8821 json.WriteString("UnusedRanges");
8822 json.WriteNumber(stat.unusedRangeCount);
8823
8824 json.WriteString("UsedBytes");
8825 json.WriteNumber(stat.usedBytes);
8826
8827 json.WriteString("UnusedBytes");
8828 json.WriteNumber(stat.unusedBytes);
8829
8830 if(stat.allocationCount > 1)
8831 {
8832 json.WriteString("AllocationSize");
8833 json.BeginObject(true);
8834 json.WriteString("Min");
8835 json.WriteNumber(stat.allocationSizeMin);
8836 json.WriteString("Avg");
8837 json.WriteNumber(stat.allocationSizeAvg);
8838 json.WriteString("Max");
8839 json.WriteNumber(stat.allocationSizeMax);
8840 json.EndObject();
8841 }
8842
8843 if(stat.unusedRangeCount > 1)
8844 {
8845 json.WriteString("UnusedRangeSize");
8846 json.BeginObject(true);
8847 json.WriteString("Min");
8848 json.WriteNumber(stat.unusedRangeSizeMin);
8849 json.WriteString("Avg");
8850 json.WriteNumber(stat.unusedRangeSizeAvg);
8851 json.WriteString("Max");
8852 json.WriteNumber(stat.unusedRangeSizeMax);
8853 json.EndObject();
8854 }
8855
8856 json.EndObject();
8857 }
8858
8859 #endif // #if VMA_STATS_STRING_ENABLED
8860
8861 struct VmaSuballocationItemSizeLess
8862 {
8863 bool operator()(
8864 const VmaSuballocationList::iterator lhs,
8865 const VmaSuballocationList::iterator rhs) const
8866 {
8867 return lhs->size < rhs->size;
8868 }
8869 bool operator()(
8870 const VmaSuballocationList::iterator lhs,
8871 VkDeviceSize rhsSize) const
8872 {
8873 return lhs->size < rhsSize;
8874 }
8875 };
8876
8877
8878 ////////////////////////////////////////////////////////////////////////////////
8879 // class VmaBlockMetadata
8880
8881 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
8882 m_Size(0),
8883 m_pAllocationCallbacks(hAllocator->GetAllocationCallbacks())
8884 {
8885 }
8886
8887 #if VMA_STATS_STRING_ENABLED
8888
8889 void VmaBlockMetadata::PrintDetailedMap_Begin(class VmaJsonWriter& json,
8890 VkDeviceSize unusedBytes,
8891 size_t allocationCount,
8892 size_t unusedRangeCount) const
8893 {
8894 json.BeginObject();
8895
8896 json.WriteString("TotalBytes");
8897 json.WriteNumber(GetSize());
8898
8899 json.WriteString("UnusedBytes");
8900 json.WriteNumber(unusedBytes);
8901
8902 json.WriteString("Allocations");
8903 json.WriteNumber((uint64_t)allocationCount);
8904
8905 json.WriteString("UnusedRanges");
8906 json.WriteNumber((uint64_t)unusedRangeCount);
8907
8908 json.WriteString("Suballocations");
8909 json.BeginArray();
8910 }
8911
8912 void VmaBlockMetadata::PrintDetailedMap_Allocation(class VmaJsonWriter& json,
8913 VkDeviceSize offset,
8914 VmaAllocation hAllocation) const
8915 {
8916 json.BeginObject(true);
8917
8918 json.WriteString("Offset");
8919 json.WriteNumber(offset);
8920
8921 hAllocation->PrintParameters(json);
8922
8923 json.EndObject();
8924 }
8925
8926 void VmaBlockMetadata::PrintDetailedMap_UnusedRange(class VmaJsonWriter& json,
8927 VkDeviceSize offset,
8928 VkDeviceSize size) const
8929 {
8930 json.BeginObject(true);
8931
8932 json.WriteString("Offset");
8933 json.WriteNumber(offset);
8934
8935 json.WriteString("Type");
8936 json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
8937
8938 json.WriteString("Size");
8939 json.WriteNumber(size);
8940
8941 json.EndObject();
8942 }
8943
8944 void VmaBlockMetadata::PrintDetailedMap_End(class VmaJsonWriter& json) const
8945 {
8946 json.EndArray();
8947 json.EndObject();
8948 }
8949
8950 #endif // #if VMA_STATS_STRING_ENABLED
8951
8952 ////////////////////////////////////////////////////////////////////////////////
8953 // class VmaBlockMetadata_Generic
8954
8955 VmaBlockMetadata_Generic::VmaBlockMetadata_Generic(VmaAllocator hAllocator) :
8956 VmaBlockMetadata(hAllocator),
8957 m_FreeCount(0),
8958 m_SumFreeSize(0),
8959 m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
8960 m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
8961 {
8962 }
8963
8964 VmaBlockMetadata_Generic::~VmaBlockMetadata_Generic()
8965 {
8966 }
8967
8968 void VmaBlockMetadata_Generic::Init(VkDeviceSize size)
8969 {
8970 VmaBlockMetadata::Init(size);
8971
8972 m_FreeCount = 1;
8973 m_SumFreeSize = size;
8974
8975 VmaSuballocation suballoc = {};
8976 suballoc.offset = 0;
8977 suballoc.size = size;
8978 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8979 suballoc.hAllocation = VK_NULL_HANDLE;
8980
8981 VMA_ASSERT(size > VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER);
8982 m_Suballocations.push_back(suballoc);
8983 VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
8984 --suballocItem;
8985 m_FreeSuballocationsBySize.push_back(suballocItem);
8986 }
8987
8988 bool VmaBlockMetadata_Generic::Validate() const
8989 {
8990 VMA_VALIDATE(!m_Suballocations.empty());
8991
8992 // Expected offset of new suballocation as calculated from previous ones.
8993 VkDeviceSize calculatedOffset = 0;
8994 // Expected number of free suballocations as calculated from traversing their list.
8995 uint32_t calculatedFreeCount = 0;
8996 // Expected sum size of free suballocations as calculated from traversing their list.
8997 VkDeviceSize calculatedSumFreeSize = 0;
8998 // Expected number of free suballocations that should be registered in
8999 // m_FreeSuballocationsBySize calculated from traversing their list.
9000 size_t freeSuballocationsToRegister = 0;
9001 // True if previous visited suballocation was free.
9002 bool prevFree = false;
9003
9004 for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
9005 suballocItem != m_Suballocations.cend();
9006 ++suballocItem)
9007 {
9008 const VmaSuballocation& subAlloc = *suballocItem;
9009
9010 // Actual offset of this suballocation doesn't match expected one.
9011 VMA_VALIDATE(subAlloc.offset == calculatedOffset);
9012
9013 const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
9014 // Two adjacent free suballocations are invalid. They should be merged.
9015 VMA_VALIDATE(!prevFree || !currFree);
9016
9017 VMA_VALIDATE(currFree == (subAlloc.hAllocation == VK_NULL_HANDLE));
9018
9019 if(currFree)
9020 {
9021 calculatedSumFreeSize += subAlloc.size;
9022 ++calculatedFreeCount;
9023 if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
9024 {
9025 ++freeSuballocationsToRegister;
9026 }
9027
9028 // Margin required between allocations - every free space must be at least that large.
9029 VMA_VALIDATE(subAlloc.size >= VMA_DEBUG_MARGIN);
9030 }
9031 else
9032 {
9033 VMA_VALIDATE(subAlloc.hAllocation->GetOffset() == subAlloc.offset);
9034 VMA_VALIDATE(subAlloc.hAllocation->GetSize() == subAlloc.size);
9035
9036 // Margin required between allocations - previous allocation must be free.
9037 VMA_VALIDATE(VMA_DEBUG_MARGIN == 0 || prevFree);
9038 }
9039
9040 calculatedOffset += subAlloc.size;
9041 prevFree = currFree;
9042 }
9043
9044 // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
9045 // match expected one.
9046 VMA_VALIDATE(m_FreeSuballocationsBySize.size() == freeSuballocationsToRegister);
9047
9048 VkDeviceSize lastSize = 0;
9049 for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
9050 {
9051 VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
9052
9053 // Only free suballocations can be registered in m_FreeSuballocationsBySize.
9054 VMA_VALIDATE(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE);
9055 // They must be sorted by size ascending.
9056 VMA_VALIDATE(suballocItem->size >= lastSize);
9057
9058 lastSize = suballocItem->size;
9059 }
9060
9061 // Check if totals match calculacted values.
9062 VMA_VALIDATE(ValidateFreeSuballocationList());
9063 VMA_VALIDATE(calculatedOffset == GetSize());
9064 VMA_VALIDATE(calculatedSumFreeSize == m_SumFreeSize);
9065 VMA_VALIDATE(calculatedFreeCount == m_FreeCount);
9066
9067 return true;
9068 }
9069
9070 VkDeviceSize VmaBlockMetadata_Generic::GetUnusedRangeSizeMax() const
9071 {
9072 if(!m_FreeSuballocationsBySize.empty())
9073 {
9074 return m_FreeSuballocationsBySize.back()->size;
9075 }
9076 else
9077 {
9078 return 0;
9079 }
9080 }
9081
9082 bool VmaBlockMetadata_Generic::IsEmpty() const
9083 {
9084 return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
9085 }
9086
9087 void VmaBlockMetadata_Generic::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
9088 {
9089 outInfo.blockCount = 1;
9090
9091 const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
9092 outInfo.allocationCount = rangeCount - m_FreeCount;
9093 outInfo.unusedRangeCount = m_FreeCount;
9094
9095 outInfo.unusedBytes = m_SumFreeSize;
9096 outInfo.usedBytes = GetSize() - outInfo.unusedBytes;
9097
9098 outInfo.allocationSizeMin = UINT64_MAX;
9099 outInfo.allocationSizeMax = 0;
9100 outInfo.unusedRangeSizeMin = UINT64_MAX;
9101 outInfo.unusedRangeSizeMax = 0;
9102
9103 for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
9104 suballocItem != m_Suballocations.cend();
9105 ++suballocItem)
9106 {
9107 const VmaSuballocation& suballoc = *suballocItem;
9108 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
9109 {
9110 outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
9111 outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
9112 }
9113 else
9114 {
9115 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
9116 outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
9117 }
9118 }
9119 }
9120
9121 void VmaBlockMetadata_Generic::AddPoolStats(VmaPoolStats& inoutStats) const
9122 {
9123 const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
9124
9125 inoutStats.size += GetSize();
9126 inoutStats.unusedSize += m_SumFreeSize;
9127 inoutStats.allocationCount += rangeCount - m_FreeCount;
9128 inoutStats.unusedRangeCount += m_FreeCount;
9129 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
9130 }
9131
9132 #if VMA_STATS_STRING_ENABLED
9133
9134 void VmaBlockMetadata_Generic::PrintDetailedMap(class VmaJsonWriter& json) const
9135 {
9136 PrintDetailedMap_Begin(json,
9137 m_SumFreeSize, // unusedBytes
9138 m_Suballocations.size() - (size_t)m_FreeCount, // allocationCount
9139 m_FreeCount); // unusedRangeCount
9140
9141 size_t i = 0;
9142 for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
9143 suballocItem != m_Suballocations.cend();
9144 ++suballocItem, ++i)
9145 {
9146 if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
9147 {
9148 PrintDetailedMap_UnusedRange(json, suballocItem->offset, suballocItem->size);
9149 }
9150 else
9151 {
9152 PrintDetailedMap_Allocation(json, suballocItem->offset, suballocItem->hAllocation);
9153 }
9154 }
9155
9156 PrintDetailedMap_End(json);
9157 }
9158
9159 #endif // #if VMA_STATS_STRING_ENABLED
9160
9161 bool VmaBlockMetadata_Generic::CreateAllocationRequest(
9162 uint32_t currentFrameIndex,
9163 uint32_t frameInUseCount,
9164 VkDeviceSize bufferImageGranularity,
9165 VkDeviceSize allocSize,
9166 VkDeviceSize allocAlignment,
9167 bool upperAddress,
9168 VmaSuballocationType allocType,
9169 bool canMakeOtherLost,
9170 uint32_t strategy,
9171 VmaAllocationRequest* pAllocationRequest)
9172 {
9173 VMA_ASSERT(allocSize > 0);
9174 VMA_ASSERT(!upperAddress);
9175 VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
9176 VMA_ASSERT(pAllocationRequest != VMA_NULL);
9177 VMA_HEAVY_ASSERT(Validate());
9178
9179 pAllocationRequest->type = VmaAllocationRequestType::Normal;
9180
9181 // There is not enough total free space in this block to fullfill the request: Early return.
9182 if(canMakeOtherLost == false &&
9183 m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
9184 {
9185 return false;
9186 }
9187
9188 // New algorithm, efficiently searching freeSuballocationsBySize.
9189 const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
9190 if(freeSuballocCount > 0)
9191 {
9192 if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT)
9193 {
9194 // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
9195 VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
9196 m_FreeSuballocationsBySize.data(),
9197 m_FreeSuballocationsBySize.data() + freeSuballocCount,
9198 allocSize + 2 * VMA_DEBUG_MARGIN,
9199 VmaSuballocationItemSizeLess());
9200 size_t index = it - m_FreeSuballocationsBySize.data();
9201 for(; index < freeSuballocCount; ++index)
9202 {
9203 if(CheckAllocation(
9204 currentFrameIndex,
9205 frameInUseCount,
9206 bufferImageGranularity,
9207 allocSize,
9208 allocAlignment,
9209 allocType,
9210 m_FreeSuballocationsBySize[index],
9211 false, // canMakeOtherLost
9212 &pAllocationRequest->offset,
9213 &pAllocationRequest->itemsToMakeLostCount,
9214 &pAllocationRequest->sumFreeSize,
9215 &pAllocationRequest->sumItemSize))
9216 {
9217 pAllocationRequest->item = m_FreeSuballocationsBySize[index];
9218 return true;
9219 }
9220 }
9221 }
9222 else if(strategy == VMA_ALLOCATION_INTERNAL_STRATEGY_MIN_OFFSET)
9223 {
9224 for(VmaSuballocationList::iterator it = m_Suballocations.begin();
9225 it != m_Suballocations.end();
9226 ++it)
9227 {
9228 if(it->type == VMA_SUBALLOCATION_TYPE_FREE && CheckAllocation(
9229 currentFrameIndex,
9230 frameInUseCount,
9231 bufferImageGranularity,
9232 allocSize,
9233 allocAlignment,
9234 allocType,
9235 it,
9236 false, // canMakeOtherLost
9237 &pAllocationRequest->offset,
9238 &pAllocationRequest->itemsToMakeLostCount,
9239 &pAllocationRequest->sumFreeSize,
9240 &pAllocationRequest->sumItemSize))
9241 {
9242 pAllocationRequest->item = it;
9243 return true;
9244 }
9245 }
9246 }
9247 else // WORST_FIT, FIRST_FIT
9248 {
9249 // Search staring from biggest suballocations.
9250 for(size_t index = freeSuballocCount; index--; )
9251 {
9252 if(CheckAllocation(
9253 currentFrameIndex,
9254 frameInUseCount,
9255 bufferImageGranularity,
9256 allocSize,
9257 allocAlignment,
9258 allocType,
9259 m_FreeSuballocationsBySize[index],
9260 false, // canMakeOtherLost
9261 &pAllocationRequest->offset,
9262 &pAllocationRequest->itemsToMakeLostCount,
9263 &pAllocationRequest->sumFreeSize,
9264 &pAllocationRequest->sumItemSize))
9265 {
9266 pAllocationRequest->item = m_FreeSuballocationsBySize[index];
9267 return true;
9268 }
9269 }
9270 }
9271 }
9272
9273 if(canMakeOtherLost)
9274 {
9275 // Brute-force algorithm. TODO: Come up with something better.
9276
9277 bool found = false;
9278 VmaAllocationRequest tmpAllocRequest = {};
9279 tmpAllocRequest.type = VmaAllocationRequestType::Normal;
9280 for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
9281 suballocIt != m_Suballocations.end();
9282 ++suballocIt)
9283 {
9284 if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
9285 suballocIt->hAllocation->CanBecomeLost())
9286 {
9287 if(CheckAllocation(
9288 currentFrameIndex,
9289 frameInUseCount,
9290 bufferImageGranularity,
9291 allocSize,
9292 allocAlignment,
9293 allocType,
9294 suballocIt,
9295 canMakeOtherLost,
9296 &tmpAllocRequest.offset,
9297 &tmpAllocRequest.itemsToMakeLostCount,
9298 &tmpAllocRequest.sumFreeSize,
9299 &tmpAllocRequest.sumItemSize))
9300 {
9301 if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT)
9302 {
9303 *pAllocationRequest = tmpAllocRequest;
9304 pAllocationRequest->item = suballocIt;
9305 break;
9306 }
9307 if(!found || tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
9308 {
9309 *pAllocationRequest = tmpAllocRequest;
9310 pAllocationRequest->item = suballocIt;
9311 found = true;
9312 }
9313 }
9314 }
9315 }
9316
9317 return found;
9318 }
9319
9320 return false;
9321 }
9322
9323 bool VmaBlockMetadata_Generic::MakeRequestedAllocationsLost(
9324 uint32_t currentFrameIndex,
9325 uint32_t frameInUseCount,
9326 VmaAllocationRequest* pAllocationRequest)
9327 {
9328 VMA_ASSERT(pAllocationRequest && pAllocationRequest->type == VmaAllocationRequestType::Normal);
9329
9330 while(pAllocationRequest->itemsToMakeLostCount > 0)
9331 {
9332 if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
9333 {
9334 ++pAllocationRequest->item;
9335 }
9336 VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
9337 VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
9338 VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
9339 if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
9340 {
9341 pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
9342 --pAllocationRequest->itemsToMakeLostCount;
9343 }
9344 else
9345 {
9346 return false;
9347 }
9348 }
9349
9350 VMA_HEAVY_ASSERT(Validate());
9351 VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
9352 VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
9353
9354 return true;
9355 }
9356
9357 uint32_t VmaBlockMetadata_Generic::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
9358 {
9359 uint32_t lostAllocationCount = 0;
9360 for(VmaSuballocationList::iterator it = m_Suballocations.begin();
9361 it != m_Suballocations.end();
9362 ++it)
9363 {
9364 if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
9365 it->hAllocation->CanBecomeLost() &&
9366 it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
9367 {
9368 it = FreeSuballocation(it);
9369 ++lostAllocationCount;
9370 }
9371 }
9372 return lostAllocationCount;
9373 }
9374
9375 VkResult VmaBlockMetadata_Generic::CheckCorruption(const void* pBlockData)
9376 {
9377 for(VmaSuballocationList::iterator it = m_Suballocations.begin();
9378 it != m_Suballocations.end();
9379 ++it)
9380 {
9381 if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
9382 {
9383 if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
9384 {
9385 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
9386 return VK_ERROR_VALIDATION_FAILED_EXT;
9387 }
9388 if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
9389 {
9390 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
9391 return VK_ERROR_VALIDATION_FAILED_EXT;
9392 }
9393 }
9394 }
9395
9396 return VK_SUCCESS;
9397 }
9398
9399 void VmaBlockMetadata_Generic::Alloc(
9400 const VmaAllocationRequest& request,
9401 VmaSuballocationType type,
9402 VkDeviceSize allocSize,
9403 VmaAllocation hAllocation)
9404 {
9405 VMA_ASSERT(request.type == VmaAllocationRequestType::Normal);
9406 VMA_ASSERT(request.item != m_Suballocations.end());
9407 VmaSuballocation& suballoc = *request.item;
9408 // Given suballocation is a free block.
9409 VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
9410 // Given offset is inside this suballocation.
9411 VMA_ASSERT(request.offset >= suballoc.offset);
9412 const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
9413 VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
9414 const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
9415
9416 // Unregister this free suballocation from m_FreeSuballocationsBySize and update
9417 // it to become used.
9418 UnregisterFreeSuballocation(request.item);
9419
9420 suballoc.offset = request.offset;
9421 suballoc.size = allocSize;
9422 suballoc.type = type;
9423 suballoc.hAllocation = hAllocation;
9424
9425 // If there are any free bytes remaining at the end, insert new free suballocation after current one.
9426 if(paddingEnd)
9427 {
9428 VmaSuballocation paddingSuballoc = {};
9429 paddingSuballoc.offset = request.offset + allocSize;
9430 paddingSuballoc.size = paddingEnd;
9431 paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
9432 VmaSuballocationList::iterator next = request.item;
9433 ++next;
9434 const VmaSuballocationList::iterator paddingEndItem =
9435 m_Suballocations.insert(next, paddingSuballoc);
9436 RegisterFreeSuballocation(paddingEndItem);
9437 }
9438
9439 // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
9440 if(paddingBegin)
9441 {
9442 VmaSuballocation paddingSuballoc = {};
9443 paddingSuballoc.offset = request.offset - paddingBegin;
9444 paddingSuballoc.size = paddingBegin;
9445 paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
9446 const VmaSuballocationList::iterator paddingBeginItem =
9447 m_Suballocations.insert(request.item, paddingSuballoc);
9448 RegisterFreeSuballocation(paddingBeginItem);
9449 }
9450
9451 // Update totals.
9452 m_FreeCount = m_FreeCount - 1;
9453 if(paddingBegin > 0)
9454 {
9455 ++m_FreeCount;
9456 }
9457 if(paddingEnd > 0)
9458 {
9459 ++m_FreeCount;
9460 }
9461 m_SumFreeSize -= allocSize;
9462 }
9463
9464 void VmaBlockMetadata_Generic::Free(const VmaAllocation allocation)
9465 {
9466 for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
9467 suballocItem != m_Suballocations.end();
9468 ++suballocItem)
9469 {
9470 VmaSuballocation& suballoc = *suballocItem;
9471 if(suballoc.hAllocation == allocation)
9472 {
9473 FreeSuballocation(suballocItem);
9474 VMA_HEAVY_ASSERT(Validate());
9475 return;
9476 }
9477 }
9478 VMA_ASSERT(0 && "Not found!");
9479 }
9480
9481 void VmaBlockMetadata_Generic::FreeAtOffset(VkDeviceSize offset)
9482 {
9483 for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
9484 suballocItem != m_Suballocations.end();
9485 ++suballocItem)
9486 {
9487 VmaSuballocation& suballoc = *suballocItem;
9488 if(suballoc.offset == offset)
9489 {
9490 FreeSuballocation(suballocItem);
9491 return;
9492 }
9493 }
9494 VMA_ASSERT(0 && "Not found!");
9495 }
9496
9497 bool VmaBlockMetadata_Generic::ValidateFreeSuballocationList() const
9498 {
9499 VkDeviceSize lastSize = 0;
9500 for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
9501 {
9502 const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
9503
9504 VMA_VALIDATE(it->type == VMA_SUBALLOCATION_TYPE_FREE);
9505 VMA_VALIDATE(it->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER);
9506 VMA_VALIDATE(it->size >= lastSize);
9507 lastSize = it->size;
9508 }
9509 return true;
9510 }
9511
9512 bool VmaBlockMetadata_Generic::CheckAllocation(
9513 uint32_t currentFrameIndex,
9514 uint32_t frameInUseCount,
9515 VkDeviceSize bufferImageGranularity,
9516 VkDeviceSize allocSize,
9517 VkDeviceSize allocAlignment,
9518 VmaSuballocationType allocType,
9519 VmaSuballocationList::const_iterator suballocItem,
9520 bool canMakeOtherLost,
9521 VkDeviceSize* pOffset,
9522 size_t* itemsToMakeLostCount,
9523 VkDeviceSize* pSumFreeSize,
9524 VkDeviceSize* pSumItemSize) const
9525 {
9526 VMA_ASSERT(allocSize > 0);
9527 VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
9528 VMA_ASSERT(suballocItem != m_Suballocations.cend());
9529 VMA_ASSERT(pOffset != VMA_NULL);
9530
9531 *itemsToMakeLostCount = 0;
9532 *pSumFreeSize = 0;
9533 *pSumItemSize = 0;
9534
9535 if(canMakeOtherLost)
9536 {
9537 if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
9538 {
9539 *pSumFreeSize = suballocItem->size;
9540 }
9541 else
9542 {
9543 if(suballocItem->hAllocation->CanBecomeLost() &&
9544 suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
9545 {
9546 ++*itemsToMakeLostCount;
9547 *pSumItemSize = suballocItem->size;
9548 }
9549 else
9550 {
9551 return false;
9552 }
9553 }
9554
9555 // Remaining size is too small for this request: Early return.
9556 if(GetSize() - suballocItem->offset < allocSize)
9557 {
9558 return false;
9559 }
9560
9561 // Start from offset equal to beginning of this suballocation.
9562 *pOffset = suballocItem->offset;
9563
9564 // Apply VMA_DEBUG_MARGIN at the beginning.
9565 if(VMA_DEBUG_MARGIN > 0)
9566 {
9567 *pOffset += VMA_DEBUG_MARGIN;
9568 }
9569
9570 // Apply alignment.
9571 *pOffset = VmaAlignUp(*pOffset, allocAlignment);
9572
9573 // Check previous suballocations for BufferImageGranularity conflicts.
9574 // Make bigger alignment if necessary.
9575 if(bufferImageGranularity > 1)
9576 {
9577 bool bufferImageGranularityConflict = false;
9578 VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
9579 while(prevSuballocItem != m_Suballocations.cbegin())
9580 {
9581 --prevSuballocItem;
9582 const VmaSuballocation& prevSuballoc = *prevSuballocItem;
9583 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
9584 {
9585 if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
9586 {
9587 bufferImageGranularityConflict = true;
9588 break;
9589 }
9590 }
9591 else
9592 // Already on previous page.
9593 break;
9594 }
9595 if(bufferImageGranularityConflict)
9596 {
9597 *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
9598 }
9599 }
9600
9601 // Now that we have final *pOffset, check if we are past suballocItem.
9602 // If yes, return false - this function should be called for another suballocItem as starting point.
9603 if(*pOffset >= suballocItem->offset + suballocItem->size)
9604 {
9605 return false;
9606 }
9607
9608 // Calculate padding at the beginning based on current offset.
9609 const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
9610
9611 // Calculate required margin at the end.
9612 const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
9613
9614 const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
9615 // Another early return check.
9616 if(suballocItem->offset + totalSize > GetSize())
9617 {
9618 return false;
9619 }
9620
9621 // Advance lastSuballocItem until desired size is reached.
9622 // Update itemsToMakeLostCount.
9623 VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
9624 if(totalSize > suballocItem->size)
9625 {
9626 VkDeviceSize remainingSize = totalSize - suballocItem->size;
9627 while(remainingSize > 0)
9628 {
9629 ++lastSuballocItem;
9630 if(lastSuballocItem == m_Suballocations.cend())
9631 {
9632 return false;
9633 }
9634 if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
9635 {
9636 *pSumFreeSize += lastSuballocItem->size;
9637 }
9638 else
9639 {
9640 VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
9641 if(lastSuballocItem->hAllocation->CanBecomeLost() &&
9642 lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
9643 {
9644 ++*itemsToMakeLostCount;
9645 *pSumItemSize += lastSuballocItem->size;
9646 }
9647 else
9648 {
9649 return false;
9650 }
9651 }
9652 remainingSize = (lastSuballocItem->size < remainingSize) ?
9653 remainingSize - lastSuballocItem->size : 0;
9654 }
9655 }
9656
9657 // Check next suballocations for BufferImageGranularity conflicts.
9658 // If conflict exists, we must mark more allocations lost or fail.
9659 if(bufferImageGranularity > 1)
9660 {
9661 VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
9662 ++nextSuballocItem;
9663 while(nextSuballocItem != m_Suballocations.cend())
9664 {
9665 const VmaSuballocation& nextSuballoc = *nextSuballocItem;
9666 if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
9667 {
9668 if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
9669 {
9670 VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
9671 if(nextSuballoc.hAllocation->CanBecomeLost() &&
9672 nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
9673 {
9674 ++*itemsToMakeLostCount;
9675 }
9676 else
9677 {
9678 return false;
9679 }
9680 }
9681 }
9682 else
9683 {
9684 // Already on next page.
9685 break;
9686 }
9687 ++nextSuballocItem;
9688 }
9689 }
9690 }
9691 else
9692 {
9693 const VmaSuballocation& suballoc = *suballocItem;
9694 VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
9695
9696 *pSumFreeSize = suballoc.size;
9697
9698 // Size of this suballocation is too small for this request: Early return.
9699 if(suballoc.size < allocSize)
9700 {
9701 return false;
9702 }
9703
9704 // Start from offset equal to beginning of this suballocation.
9705 *pOffset = suballoc.offset;
9706
9707 // Apply VMA_DEBUG_MARGIN at the beginning.
9708 if(VMA_DEBUG_MARGIN > 0)
9709 {
9710 *pOffset += VMA_DEBUG_MARGIN;
9711 }
9712
9713 // Apply alignment.
9714 *pOffset = VmaAlignUp(*pOffset, allocAlignment);
9715
9716 // Check previous suballocations for BufferImageGranularity conflicts.
9717 // Make bigger alignment if necessary.
9718 if(bufferImageGranularity > 1)
9719 {
9720 bool bufferImageGranularityConflict = false;
9721 VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
9722 while(prevSuballocItem != m_Suballocations.cbegin())
9723 {
9724 --prevSuballocItem;
9725 const VmaSuballocation& prevSuballoc = *prevSuballocItem;
9726 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
9727 {
9728 if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
9729 {
9730 bufferImageGranularityConflict = true;
9731 break;
9732 }
9733 }
9734 else
9735 // Already on previous page.
9736 break;
9737 }
9738 if(bufferImageGranularityConflict)
9739 {
9740 *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
9741 }
9742 }
9743
9744 // Calculate padding at the beginning based on current offset.
9745 const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
9746
9747 // Calculate required margin at the end.
9748 const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
9749
9750 // Fail if requested size plus margin before and after is bigger than size of this suballocation.
9751 if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
9752 {
9753 return false;
9754 }
9755
9756 // Check next suballocations for BufferImageGranularity conflicts.
9757 // If conflict exists, allocation cannot be made here.
9758 if(bufferImageGranularity > 1)
9759 {
9760 VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
9761 ++nextSuballocItem;
9762 while(nextSuballocItem != m_Suballocations.cend())
9763 {
9764 const VmaSuballocation& nextSuballoc = *nextSuballocItem;
9765 if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
9766 {
9767 if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
9768 {
9769 return false;
9770 }
9771 }
9772 else
9773 {
9774 // Already on next page.
9775 break;
9776 }
9777 ++nextSuballocItem;
9778 }
9779 }
9780 }
9781
9782 // All tests passed: Success. pOffset is already filled.
9783 return true;
9784 }
9785
9786 void VmaBlockMetadata_Generic::MergeFreeWithNext(VmaSuballocationList::iterator item)
9787 {
9788 VMA_ASSERT(item != m_Suballocations.end());
9789 VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
9790
9791 VmaSuballocationList::iterator nextItem = item;
9792 ++nextItem;
9793 VMA_ASSERT(nextItem != m_Suballocations.end());
9794 VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
9795
9796 item->size += nextItem->size;
9797 --m_FreeCount;
9798 m_Suballocations.erase(nextItem);
9799 }
9800
9801 VmaSuballocationList::iterator VmaBlockMetadata_Generic::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
9802 {
9803 // Change this suballocation to be marked as free.
9804 VmaSuballocation& suballoc = *suballocItem;
9805 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
9806 suballoc.hAllocation = VK_NULL_HANDLE;
9807
9808 // Update totals.
9809 ++m_FreeCount;
9810 m_SumFreeSize += suballoc.size;
9811
9812 // Merge with previous and/or next suballocation if it's also free.
9813 bool mergeWithNext = false;
9814 bool mergeWithPrev = false;
9815
9816 VmaSuballocationList::iterator nextItem = suballocItem;
9817 ++nextItem;
9818 if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
9819 {
9820 mergeWithNext = true;
9821 }
9822
9823 VmaSuballocationList::iterator prevItem = suballocItem;
9824 if(suballocItem != m_Suballocations.begin())
9825 {
9826 --prevItem;
9827 if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
9828 {
9829 mergeWithPrev = true;
9830 }
9831 }
9832
9833 if(mergeWithNext)
9834 {
9835 UnregisterFreeSuballocation(nextItem);
9836 MergeFreeWithNext(suballocItem);
9837 }
9838
9839 if(mergeWithPrev)
9840 {
9841 UnregisterFreeSuballocation(prevItem);
9842 MergeFreeWithNext(prevItem);
9843 RegisterFreeSuballocation(prevItem);
9844 return prevItem;
9845 }
9846 else
9847 {
9848 RegisterFreeSuballocation(suballocItem);
9849 return suballocItem;
9850 }
9851 }
9852
9853 void VmaBlockMetadata_Generic::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
9854 {
9855 VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
9856 VMA_ASSERT(item->size > 0);
9857
9858 // You may want to enable this validation at the beginning or at the end of
9859 // this function, depending on what do you want to check.
9860 VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
9861
9862 if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
9863 {
9864 if(m_FreeSuballocationsBySize.empty())
9865 {
9866 m_FreeSuballocationsBySize.push_back(item);
9867 }
9868 else
9869 {
9870 VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
9871 }
9872 }
9873
9874 //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
9875 }
9876
9877
9878 void VmaBlockMetadata_Generic::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
9879 {
9880 VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
9881 VMA_ASSERT(item->size > 0);
9882
9883 // You may want to enable this validation at the beginning or at the end of
9884 // this function, depending on what do you want to check.
9885 VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
9886
9887 if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
9888 {
9889 VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
9890 m_FreeSuballocationsBySize.data(),
9891 m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
9892 item,
9893 VmaSuballocationItemSizeLess());
9894 for(size_t index = it - m_FreeSuballocationsBySize.data();
9895 index < m_FreeSuballocationsBySize.size();
9896 ++index)
9897 {
9898 if(m_FreeSuballocationsBySize[index] == item)
9899 {
9900 VmaVectorRemove(m_FreeSuballocationsBySize, index);
9901 return;
9902 }
9903 VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
9904 }
9905 VMA_ASSERT(0 && "Not found.");
9906 }
9907
9908 //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
9909 }
9910
9911 bool VmaBlockMetadata_Generic::IsBufferImageGranularityConflictPossible(
9912 VkDeviceSize bufferImageGranularity,
9913 VmaSuballocationType& inOutPrevSuballocType) const
9914 {
9915 if(bufferImageGranularity == 1 || IsEmpty())
9916 {
9917 return false;
9918 }
9919
9920 VkDeviceSize minAlignment = VK_WHOLE_SIZE;
9921 bool typeConflictFound = false;
9922 for(VmaSuballocationList::const_iterator it = m_Suballocations.cbegin();
9923 it != m_Suballocations.cend();
9924 ++it)
9925 {
9926 const VmaSuballocationType suballocType = it->type;
9927 if(suballocType != VMA_SUBALLOCATION_TYPE_FREE)
9928 {
9929 minAlignment = VMA_MIN(minAlignment, it->hAllocation->GetAlignment());
9930 if(VmaIsBufferImageGranularityConflict(inOutPrevSuballocType, suballocType))
9931 {
9932 typeConflictFound = true;
9933 }
9934 inOutPrevSuballocType = suballocType;
9935 }
9936 }
9937
9938 return typeConflictFound || minAlignment >= bufferImageGranularity;
9939 }
9940
9941 ////////////////////////////////////////////////////////////////////////////////
9942 // class VmaBlockMetadata_Linear
9943
9944 VmaBlockMetadata_Linear::VmaBlockMetadata_Linear(VmaAllocator hAllocator) :
9945 VmaBlockMetadata(hAllocator),
9946 m_SumFreeSize(0),
9947 m_Suballocations0(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
9948 m_Suballocations1(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
9949 m_1stVectorIndex(0),
9950 m_2ndVectorMode(SECOND_VECTOR_EMPTY),
9951 m_1stNullItemsBeginCount(0),
9952 m_1stNullItemsMiddleCount(0),
9953 m_2ndNullItemsCount(0)
9954 {
9955 }
9956
9957 VmaBlockMetadata_Linear::~VmaBlockMetadata_Linear()
9958 {
9959 }
9960
9961 void VmaBlockMetadata_Linear::Init(VkDeviceSize size)
9962 {
9963 VmaBlockMetadata::Init(size);
9964 m_SumFreeSize = size;
9965 }
9966
9967 bool VmaBlockMetadata_Linear::Validate() const
9968 {
9969 const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
9970 const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
9971
9972 VMA_VALIDATE(suballocations2nd.empty() == (m_2ndVectorMode == SECOND_VECTOR_EMPTY));
9973 VMA_VALIDATE(!suballocations1st.empty() ||
9974 suballocations2nd.empty() ||
9975 m_2ndVectorMode != SECOND_VECTOR_RING_BUFFER);
9976
9977 if(!suballocations1st.empty())
9978 {
9979 // Null item at the beginning should be accounted into m_1stNullItemsBeginCount.
9980 VMA_VALIDATE(suballocations1st[m_1stNullItemsBeginCount].hAllocation != VK_NULL_HANDLE);
9981 // Null item at the end should be just pop_back().
9982 VMA_VALIDATE(suballocations1st.back().hAllocation != VK_NULL_HANDLE);
9983 }
9984 if(!suballocations2nd.empty())
9985 {
9986 // Null item at the end should be just pop_back().
9987 VMA_VALIDATE(suballocations2nd.back().hAllocation != VK_NULL_HANDLE);
9988 }
9989
9990 VMA_VALIDATE(m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount <= suballocations1st.size());
9991 VMA_VALIDATE(m_2ndNullItemsCount <= suballocations2nd.size());
9992
9993 VkDeviceSize sumUsedSize = 0;
9994 const size_t suballoc1stCount = suballocations1st.size();
9995 VkDeviceSize offset = VMA_DEBUG_MARGIN;
9996
9997 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
9998 {
9999 const size_t suballoc2ndCount = suballocations2nd.size();
10000 size_t nullItem2ndCount = 0;
10001 for(size_t i = 0; i < suballoc2ndCount; ++i)
10002 {
10003 const VmaSuballocation& suballoc = suballocations2nd[i];
10004 const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
10005
10006 VMA_VALIDATE(currFree == (suballoc.hAllocation == VK_NULL_HANDLE));
10007 VMA_VALIDATE(suballoc.offset >= offset);
10008
10009 if(!currFree)
10010 {
10011 VMA_VALIDATE(suballoc.hAllocation->GetOffset() == suballoc.offset);
10012 VMA_VALIDATE(suballoc.hAllocation->GetSize() == suballoc.size);
10013 sumUsedSize += suballoc.size;
10014 }
10015 else
10016 {
10017 ++nullItem2ndCount;
10018 }
10019
10020 offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
10021 }
10022
10023 VMA_VALIDATE(nullItem2ndCount == m_2ndNullItemsCount);
10024 }
10025
10026 for(size_t i = 0; i < m_1stNullItemsBeginCount; ++i)
10027 {
10028 const VmaSuballocation& suballoc = suballocations1st[i];
10029 VMA_VALIDATE(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE &&
10030 suballoc.hAllocation == VK_NULL_HANDLE);
10031 }
10032
10033 size_t nullItem1stCount = m_1stNullItemsBeginCount;
10034
10035 for(size_t i = m_1stNullItemsBeginCount; i < suballoc1stCount; ++i)
10036 {
10037 const VmaSuballocation& suballoc = suballocations1st[i];
10038 const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
10039
10040 VMA_VALIDATE(currFree == (suballoc.hAllocation == VK_NULL_HANDLE));
10041 VMA_VALIDATE(suballoc.offset >= offset);
10042 VMA_VALIDATE(i >= m_1stNullItemsBeginCount || currFree);
10043
10044 if(!currFree)
10045 {
10046 VMA_VALIDATE(suballoc.hAllocation->GetOffset() == suballoc.offset);
10047 VMA_VALIDATE(suballoc.hAllocation->GetSize() == suballoc.size);
10048 sumUsedSize += suballoc.size;
10049 }
10050 else
10051 {
10052 ++nullItem1stCount;
10053 }
10054
10055 offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
10056 }
10057 VMA_VALIDATE(nullItem1stCount == m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount);
10058
10059 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
10060 {
10061 const size_t suballoc2ndCount = suballocations2nd.size();
10062 size_t nullItem2ndCount = 0;
10063 for(size_t i = suballoc2ndCount; i--; )
10064 {
10065 const VmaSuballocation& suballoc = suballocations2nd[i];
10066 const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
10067
10068 VMA_VALIDATE(currFree == (suballoc.hAllocation == VK_NULL_HANDLE));
10069 VMA_VALIDATE(suballoc.offset >= offset);
10070
10071 if(!currFree)
10072 {
10073 VMA_VALIDATE(suballoc.hAllocation->GetOffset() == suballoc.offset);
10074 VMA_VALIDATE(suballoc.hAllocation->GetSize() == suballoc.size);
10075 sumUsedSize += suballoc.size;
10076 }
10077 else
10078 {
10079 ++nullItem2ndCount;
10080 }
10081
10082 offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
10083 }
10084
10085 VMA_VALIDATE(nullItem2ndCount == m_2ndNullItemsCount);
10086 }
10087
10088 VMA_VALIDATE(offset <= GetSize());
10089 VMA_VALIDATE(m_SumFreeSize == GetSize() - sumUsedSize);
10090
10091 return true;
10092 }
10093
10094 size_t VmaBlockMetadata_Linear::GetAllocationCount() const
10095 {
10096 return AccessSuballocations1st().size() - (m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount) +
10097 AccessSuballocations2nd().size() - m_2ndNullItemsCount;
10098 }
10099
10100 VkDeviceSize VmaBlockMetadata_Linear::GetUnusedRangeSizeMax() const
10101 {
10102 const VkDeviceSize size = GetSize();
10103
10104 /*
10105 We don't consider gaps inside allocation vectors with freed allocations because
10106 they are not suitable for reuse in linear allocator. We consider only space that
10107 is available for new allocations.
10108 */
10109 if(IsEmpty())
10110 {
10111 return size;
10112 }
10113
10114 const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
10115
10116 switch(m_2ndVectorMode)
10117 {
10118 case SECOND_VECTOR_EMPTY:
10119 /*
10120 Available space is after end of 1st, as well as before beginning of 1st (which
10121 whould make it a ring buffer).
10122 */
10123 {
10124 const size_t suballocations1stCount = suballocations1st.size();
10125 VMA_ASSERT(suballocations1stCount > m_1stNullItemsBeginCount);
10126 const VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount];
10127 const VmaSuballocation& lastSuballoc = suballocations1st[suballocations1stCount - 1];
10128 return VMA_MAX(
10129 firstSuballoc.offset,
10130 size - (lastSuballoc.offset + lastSuballoc.size));
10131 }
10132 break;
10133
10134 case SECOND_VECTOR_RING_BUFFER:
10135 /*
10136 Available space is only between end of 2nd and beginning of 1st.
10137 */
10138 {
10139 const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
10140 const VmaSuballocation& lastSuballoc2nd = suballocations2nd.back();
10141 const VmaSuballocation& firstSuballoc1st = suballocations1st[m_1stNullItemsBeginCount];
10142 return firstSuballoc1st.offset - (lastSuballoc2nd.offset + lastSuballoc2nd.size);
10143 }
10144 break;
10145
10146 case SECOND_VECTOR_DOUBLE_STACK:
10147 /*
10148 Available space is only between end of 1st and top of 2nd.
10149 */
10150 {
10151 const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
10152 const VmaSuballocation& topSuballoc2nd = suballocations2nd.back();
10153 const VmaSuballocation& lastSuballoc1st = suballocations1st.back();
10154 return topSuballoc2nd.offset - (lastSuballoc1st.offset + lastSuballoc1st.size);
10155 }
10156 break;
10157
10158 default:
10159 VMA_ASSERT(0);
10160 return 0;
10161 }
10162 }
10163
10164 void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
10165 {
10166 const VkDeviceSize size = GetSize();
10167 const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
10168 const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
10169 const size_t suballoc1stCount = suballocations1st.size();
10170 const size_t suballoc2ndCount = suballocations2nd.size();
10171
10172 outInfo.blockCount = 1;
10173 outInfo.allocationCount = (uint32_t)GetAllocationCount();
10174 outInfo.unusedRangeCount = 0;
10175 outInfo.usedBytes = 0;
10176 outInfo.allocationSizeMin = UINT64_MAX;
10177 outInfo.allocationSizeMax = 0;
10178 outInfo.unusedRangeSizeMin = UINT64_MAX;
10179 outInfo.unusedRangeSizeMax = 0;
10180
10181 VkDeviceSize lastOffset = 0;
10182
10183 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
10184 {
10185 const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
10186 size_t nextAlloc2ndIndex = 0;
10187 while(lastOffset < freeSpace2ndTo1stEnd)
10188 {
10189 // Find next non-null allocation or move nextAllocIndex to the end.
10190 while(nextAlloc2ndIndex < suballoc2ndCount &&
10191 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10192 {
10193 ++nextAlloc2ndIndex;
10194 }
10195
10196 // Found non-null allocation.
10197 if(nextAlloc2ndIndex < suballoc2ndCount)
10198 {
10199 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10200
10201 // 1. Process free space before this allocation.
10202 if(lastOffset < suballoc.offset)
10203 {
10204 // There is free space from lastOffset to suballoc.offset.
10205 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10206 ++outInfo.unusedRangeCount;
10207 outInfo.unusedBytes += unusedRangeSize;
10208 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
10209 outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
10210 }
10211
10212 // 2. Process this allocation.
10213 // There is allocation with suballoc.offset, suballoc.size.
10214 outInfo.usedBytes += suballoc.size;
10215 outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
10216 outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
10217
10218 // 3. Prepare for next iteration.
10219 lastOffset = suballoc.offset + suballoc.size;
10220 ++nextAlloc2ndIndex;
10221 }
10222 // We are at the end.
10223 else
10224 {
10225 // There is free space from lastOffset to freeSpace2ndTo1stEnd.
10226 if(lastOffset < freeSpace2ndTo1stEnd)
10227 {
10228 const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
10229 ++outInfo.unusedRangeCount;
10230 outInfo.unusedBytes += unusedRangeSize;
10231 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
10232 outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
10233 }
10234
10235 // End of loop.
10236 lastOffset = freeSpace2ndTo1stEnd;
10237 }
10238 }
10239 }
10240
10241 size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
10242 const VkDeviceSize freeSpace1stTo2ndEnd =
10243 m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
10244 while(lastOffset < freeSpace1stTo2ndEnd)
10245 {
10246 // Find next non-null allocation or move nextAllocIndex to the end.
10247 while(nextAlloc1stIndex < suballoc1stCount &&
10248 suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
10249 {
10250 ++nextAlloc1stIndex;
10251 }
10252
10253 // Found non-null allocation.
10254 if(nextAlloc1stIndex < suballoc1stCount)
10255 {
10256 const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
10257
10258 // 1. Process free space before this allocation.
10259 if(lastOffset < suballoc.offset)
10260 {
10261 // There is free space from lastOffset to suballoc.offset.
10262 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10263 ++outInfo.unusedRangeCount;
10264 outInfo.unusedBytes += unusedRangeSize;
10265 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
10266 outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
10267 }
10268
10269 // 2. Process this allocation.
10270 // There is allocation with suballoc.offset, suballoc.size.
10271 outInfo.usedBytes += suballoc.size;
10272 outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
10273 outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
10274
10275 // 3. Prepare for next iteration.
10276 lastOffset = suballoc.offset + suballoc.size;
10277 ++nextAlloc1stIndex;
10278 }
10279 // We are at the end.
10280 else
10281 {
10282 // There is free space from lastOffset to freeSpace1stTo2ndEnd.
10283 if(lastOffset < freeSpace1stTo2ndEnd)
10284 {
10285 const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
10286 ++outInfo.unusedRangeCount;
10287 outInfo.unusedBytes += unusedRangeSize;
10288 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
10289 outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
10290 }
10291
10292 // End of loop.
10293 lastOffset = freeSpace1stTo2ndEnd;
10294 }
10295 }
10296
10297 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
10298 {
10299 size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
10300 while(lastOffset < size)
10301 {
10302 // Find next non-null allocation or move nextAllocIndex to the end.
10303 while(nextAlloc2ndIndex != SIZE_MAX &&
10304 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10305 {
10306 --nextAlloc2ndIndex;
10307 }
10308
10309 // Found non-null allocation.
10310 if(nextAlloc2ndIndex != SIZE_MAX)
10311 {
10312 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10313
10314 // 1. Process free space before this allocation.
10315 if(lastOffset < suballoc.offset)
10316 {
10317 // There is free space from lastOffset to suballoc.offset.
10318 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10319 ++outInfo.unusedRangeCount;
10320 outInfo.unusedBytes += unusedRangeSize;
10321 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
10322 outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
10323 }
10324
10325 // 2. Process this allocation.
10326 // There is allocation with suballoc.offset, suballoc.size.
10327 outInfo.usedBytes += suballoc.size;
10328 outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
10329 outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
10330
10331 // 3. Prepare for next iteration.
10332 lastOffset = suballoc.offset + suballoc.size;
10333 --nextAlloc2ndIndex;
10334 }
10335 // We are at the end.
10336 else
10337 {
10338 // There is free space from lastOffset to size.
10339 if(lastOffset < size)
10340 {
10341 const VkDeviceSize unusedRangeSize = size - lastOffset;
10342 ++outInfo.unusedRangeCount;
10343 outInfo.unusedBytes += unusedRangeSize;
10344 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
10345 outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
10346 }
10347
10348 // End of loop.
10349 lastOffset = size;
10350 }
10351 }
10352 }
10353
10354 outInfo.unusedBytes = size - outInfo.usedBytes;
10355 }
10356
10357 void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const
10358 {
10359 const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
10360 const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
10361 const VkDeviceSize size = GetSize();
10362 const size_t suballoc1stCount = suballocations1st.size();
10363 const size_t suballoc2ndCount = suballocations2nd.size();
10364
10365 inoutStats.size += size;
10366
10367 VkDeviceSize lastOffset = 0;
10368
10369 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
10370 {
10371 const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
10372 size_t nextAlloc2ndIndex = m_1stNullItemsBeginCount;
10373 while(lastOffset < freeSpace2ndTo1stEnd)
10374 {
10375 // Find next non-null allocation or move nextAlloc2ndIndex to the end.
10376 while(nextAlloc2ndIndex < suballoc2ndCount &&
10377 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10378 {
10379 ++nextAlloc2ndIndex;
10380 }
10381
10382 // Found non-null allocation.
10383 if(nextAlloc2ndIndex < suballoc2ndCount)
10384 {
10385 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10386
10387 // 1. Process free space before this allocation.
10388 if(lastOffset < suballoc.offset)
10389 {
10390 // There is free space from lastOffset to suballoc.offset.
10391 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10392 inoutStats.unusedSize += unusedRangeSize;
10393 ++inoutStats.unusedRangeCount;
10394 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
10395 }
10396
10397 // 2. Process this allocation.
10398 // There is allocation with suballoc.offset, suballoc.size.
10399 ++inoutStats.allocationCount;
10400
10401 // 3. Prepare for next iteration.
10402 lastOffset = suballoc.offset + suballoc.size;
10403 ++nextAlloc2ndIndex;
10404 }
10405 // We are at the end.
10406 else
10407 {
10408 if(lastOffset < freeSpace2ndTo1stEnd)
10409 {
10410 // There is free space from lastOffset to freeSpace2ndTo1stEnd.
10411 const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
10412 inoutStats.unusedSize += unusedRangeSize;
10413 ++inoutStats.unusedRangeCount;
10414 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
10415 }
10416
10417 // End of loop.
10418 lastOffset = freeSpace2ndTo1stEnd;
10419 }
10420 }
10421 }
10422
10423 size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
10424 const VkDeviceSize freeSpace1stTo2ndEnd =
10425 m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
10426 while(lastOffset < freeSpace1stTo2ndEnd)
10427 {
10428 // Find next non-null allocation or move nextAllocIndex to the end.
10429 while(nextAlloc1stIndex < suballoc1stCount &&
10430 suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
10431 {
10432 ++nextAlloc1stIndex;
10433 }
10434
10435 // Found non-null allocation.
10436 if(nextAlloc1stIndex < suballoc1stCount)
10437 {
10438 const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
10439
10440 // 1. Process free space before this allocation.
10441 if(lastOffset < suballoc.offset)
10442 {
10443 // There is free space from lastOffset to suballoc.offset.
10444 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10445 inoutStats.unusedSize += unusedRangeSize;
10446 ++inoutStats.unusedRangeCount;
10447 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
10448 }
10449
10450 // 2. Process this allocation.
10451 // There is allocation with suballoc.offset, suballoc.size.
10452 ++inoutStats.allocationCount;
10453
10454 // 3. Prepare for next iteration.
10455 lastOffset = suballoc.offset + suballoc.size;
10456 ++nextAlloc1stIndex;
10457 }
10458 // We are at the end.
10459 else
10460 {
10461 if(lastOffset < freeSpace1stTo2ndEnd)
10462 {
10463 // There is free space from lastOffset to freeSpace1stTo2ndEnd.
10464 const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
10465 inoutStats.unusedSize += unusedRangeSize;
10466 ++inoutStats.unusedRangeCount;
10467 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
10468 }
10469
10470 // End of loop.
10471 lastOffset = freeSpace1stTo2ndEnd;
10472 }
10473 }
10474
10475 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
10476 {
10477 size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
10478 while(lastOffset < size)
10479 {
10480 // Find next non-null allocation or move nextAlloc2ndIndex to the end.
10481 while(nextAlloc2ndIndex != SIZE_MAX &&
10482 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10483 {
10484 --nextAlloc2ndIndex;
10485 }
10486
10487 // Found non-null allocation.
10488 if(nextAlloc2ndIndex != SIZE_MAX)
10489 {
10490 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10491
10492 // 1. Process free space before this allocation.
10493 if(lastOffset < suballoc.offset)
10494 {
10495 // There is free space from lastOffset to suballoc.offset.
10496 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10497 inoutStats.unusedSize += unusedRangeSize;
10498 ++inoutStats.unusedRangeCount;
10499 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
10500 }
10501
10502 // 2. Process this allocation.
10503 // There is allocation with suballoc.offset, suballoc.size.
10504 ++inoutStats.allocationCount;
10505
10506 // 3. Prepare for next iteration.
10507 lastOffset = suballoc.offset + suballoc.size;
10508 --nextAlloc2ndIndex;
10509 }
10510 // We are at the end.
10511 else
10512 {
10513 if(lastOffset < size)
10514 {
10515 // There is free space from lastOffset to size.
10516 const VkDeviceSize unusedRangeSize = size - lastOffset;
10517 inoutStats.unusedSize += unusedRangeSize;
10518 ++inoutStats.unusedRangeCount;
10519 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
10520 }
10521
10522 // End of loop.
10523 lastOffset = size;
10524 }
10525 }
10526 }
10527 }
10528
10529 #if VMA_STATS_STRING_ENABLED
10530 void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const
10531 {
10532 const VkDeviceSize size = GetSize();
10533 const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
10534 const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
10535 const size_t suballoc1stCount = suballocations1st.size();
10536 const size_t suballoc2ndCount = suballocations2nd.size();
10537
10538 // FIRST PASS
10539
10540 size_t unusedRangeCount = 0;
10541 VkDeviceSize usedBytes = 0;
10542
10543 VkDeviceSize lastOffset = 0;
10544
10545 size_t alloc2ndCount = 0;
10546 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
10547 {
10548 const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
10549 size_t nextAlloc2ndIndex = 0;
10550 while(lastOffset < freeSpace2ndTo1stEnd)
10551 {
10552 // Find next non-null allocation or move nextAlloc2ndIndex to the end.
10553 while(nextAlloc2ndIndex < suballoc2ndCount &&
10554 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10555 {
10556 ++nextAlloc2ndIndex;
10557 }
10558
10559 // Found non-null allocation.
10560 if(nextAlloc2ndIndex < suballoc2ndCount)
10561 {
10562 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10563
10564 // 1. Process free space before this allocation.
10565 if(lastOffset < suballoc.offset)
10566 {
10567 // There is free space from lastOffset to suballoc.offset.
10568 ++unusedRangeCount;
10569 }
10570
10571 // 2. Process this allocation.
10572 // There is allocation with suballoc.offset, suballoc.size.
10573 ++alloc2ndCount;
10574 usedBytes += suballoc.size;
10575
10576 // 3. Prepare for next iteration.
10577 lastOffset = suballoc.offset + suballoc.size;
10578 ++nextAlloc2ndIndex;
10579 }
10580 // We are at the end.
10581 else
10582 {
10583 if(lastOffset < freeSpace2ndTo1stEnd)
10584 {
10585 // There is free space from lastOffset to freeSpace2ndTo1stEnd.
10586 ++unusedRangeCount;
10587 }
10588
10589 // End of loop.
10590 lastOffset = freeSpace2ndTo1stEnd;
10591 }
10592 }
10593 }
10594
10595 size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
10596 size_t alloc1stCount = 0;
10597 const VkDeviceSize freeSpace1stTo2ndEnd =
10598 m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
10599 while(lastOffset < freeSpace1stTo2ndEnd)
10600 {
10601 // Find next non-null allocation or move nextAllocIndex to the end.
10602 while(nextAlloc1stIndex < suballoc1stCount &&
10603 suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
10604 {
10605 ++nextAlloc1stIndex;
10606 }
10607
10608 // Found non-null allocation.
10609 if(nextAlloc1stIndex < suballoc1stCount)
10610 {
10611 const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
10612
10613 // 1. Process free space before this allocation.
10614 if(lastOffset < suballoc.offset)
10615 {
10616 // There is free space from lastOffset to suballoc.offset.
10617 ++unusedRangeCount;
10618 }
10619
10620 // 2. Process this allocation.
10621 // There is allocation with suballoc.offset, suballoc.size.
10622 ++alloc1stCount;
10623 usedBytes += suballoc.size;
10624
10625 // 3. Prepare for next iteration.
10626 lastOffset = suballoc.offset + suballoc.size;
10627 ++nextAlloc1stIndex;
10628 }
10629 // We are at the end.
10630 else
10631 {
10632 if(lastOffset < size)
10633 {
10634 // There is free space from lastOffset to freeSpace1stTo2ndEnd.
10635 ++unusedRangeCount;
10636 }
10637
10638 // End of loop.
10639 lastOffset = freeSpace1stTo2ndEnd;
10640 }
10641 }
10642
10643 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
10644 {
10645 size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
10646 while(lastOffset < size)
10647 {
10648 // Find next non-null allocation or move nextAlloc2ndIndex to the end.
10649 while(nextAlloc2ndIndex != SIZE_MAX &&
10650 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10651 {
10652 --nextAlloc2ndIndex;
10653 }
10654
10655 // Found non-null allocation.
10656 if(nextAlloc2ndIndex != SIZE_MAX)
10657 {
10658 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10659
10660 // 1. Process free space before this allocation.
10661 if(lastOffset < suballoc.offset)
10662 {
10663 // There is free space from lastOffset to suballoc.offset.
10664 ++unusedRangeCount;
10665 }
10666
10667 // 2. Process this allocation.
10668 // There is allocation with suballoc.offset, suballoc.size.
10669 ++alloc2ndCount;
10670 usedBytes += suballoc.size;
10671
10672 // 3. Prepare for next iteration.
10673 lastOffset = suballoc.offset + suballoc.size;
10674 --nextAlloc2ndIndex;
10675 }
10676 // We are at the end.
10677 else
10678 {
10679 if(lastOffset < size)
10680 {
10681 // There is free space from lastOffset to size.
10682 ++unusedRangeCount;
10683 }
10684
10685 // End of loop.
10686 lastOffset = size;
10687 }
10688 }
10689 }
10690
10691 const VkDeviceSize unusedBytes = size - usedBytes;
10692 PrintDetailedMap_Begin(json, unusedBytes, alloc1stCount + alloc2ndCount, unusedRangeCount);
10693
10694 // SECOND PASS
10695 lastOffset = 0;
10696
10697 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
10698 {
10699 const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
10700 size_t nextAlloc2ndIndex = 0;
10701 while(lastOffset < freeSpace2ndTo1stEnd)
10702 {
10703 // Find next non-null allocation or move nextAlloc2ndIndex to the end.
10704 while(nextAlloc2ndIndex < suballoc2ndCount &&
10705 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10706 {
10707 ++nextAlloc2ndIndex;
10708 }
10709
10710 // Found non-null allocation.
10711 if(nextAlloc2ndIndex < suballoc2ndCount)
10712 {
10713 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10714
10715 // 1. Process free space before this allocation.
10716 if(lastOffset < suballoc.offset)
10717 {
10718 // There is free space from lastOffset to suballoc.offset.
10719 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10720 PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
10721 }
10722
10723 // 2. Process this allocation.
10724 // There is allocation with suballoc.offset, suballoc.size.
10725 PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
10726
10727 // 3. Prepare for next iteration.
10728 lastOffset = suballoc.offset + suballoc.size;
10729 ++nextAlloc2ndIndex;
10730 }
10731 // We are at the end.
10732 else
10733 {
10734 if(lastOffset < freeSpace2ndTo1stEnd)
10735 {
10736 // There is free space from lastOffset to freeSpace2ndTo1stEnd.
10737 const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
10738 PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
10739 }
10740
10741 // End of loop.
10742 lastOffset = freeSpace2ndTo1stEnd;
10743 }
10744 }
10745 }
10746
10747 nextAlloc1stIndex = m_1stNullItemsBeginCount;
10748 while(lastOffset < freeSpace1stTo2ndEnd)
10749 {
10750 // Find next non-null allocation or move nextAllocIndex to the end.
10751 while(nextAlloc1stIndex < suballoc1stCount &&
10752 suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
10753 {
10754 ++nextAlloc1stIndex;
10755 }
10756
10757 // Found non-null allocation.
10758 if(nextAlloc1stIndex < suballoc1stCount)
10759 {
10760 const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
10761
10762 // 1. Process free space before this allocation.
10763 if(lastOffset < suballoc.offset)
10764 {
10765 // There is free space from lastOffset to suballoc.offset.
10766 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10767 PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
10768 }
10769
10770 // 2. Process this allocation.
10771 // There is allocation with suballoc.offset, suballoc.size.
10772 PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
10773
10774 // 3. Prepare for next iteration.
10775 lastOffset = suballoc.offset + suballoc.size;
10776 ++nextAlloc1stIndex;
10777 }
10778 // We are at the end.
10779 else
10780 {
10781 if(lastOffset < freeSpace1stTo2ndEnd)
10782 {
10783 // There is free space from lastOffset to freeSpace1stTo2ndEnd.
10784 const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
10785 PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
10786 }
10787
10788 // End of loop.
10789 lastOffset = freeSpace1stTo2ndEnd;
10790 }
10791 }
10792
10793 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
10794 {
10795 size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
10796 while(lastOffset < size)
10797 {
10798 // Find next non-null allocation or move nextAlloc2ndIndex to the end.
10799 while(nextAlloc2ndIndex != SIZE_MAX &&
10800 suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
10801 {
10802 --nextAlloc2ndIndex;
10803 }
10804
10805 // Found non-null allocation.
10806 if(nextAlloc2ndIndex != SIZE_MAX)
10807 {
10808 const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
10809
10810 // 1. Process free space before this allocation.
10811 if(lastOffset < suballoc.offset)
10812 {
10813 // There is free space from lastOffset to suballoc.offset.
10814 const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
10815 PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
10816 }
10817
10818 // 2. Process this allocation.
10819 // There is allocation with suballoc.offset, suballoc.size.
10820 PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
10821
10822 // 3. Prepare for next iteration.
10823 lastOffset = suballoc.offset + suballoc.size;
10824 --nextAlloc2ndIndex;
10825 }
10826 // We are at the end.
10827 else
10828 {
10829 if(lastOffset < size)
10830 {
10831 // There is free space from lastOffset to size.
10832 const VkDeviceSize unusedRangeSize = size - lastOffset;
10833 PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
10834 }
10835
10836 // End of loop.
10837 lastOffset = size;
10838 }
10839 }
10840 }
10841
10842 PrintDetailedMap_End(json);
10843 }
10844 #endif // #if VMA_STATS_STRING_ENABLED
10845
10846 bool VmaBlockMetadata_Linear::CreateAllocationRequest(
10847 uint32_t currentFrameIndex,
10848 uint32_t frameInUseCount,
10849 VkDeviceSize bufferImageGranularity,
10850 VkDeviceSize allocSize,
10851 VkDeviceSize allocAlignment,
10852 bool upperAddress,
10853 VmaSuballocationType allocType,
10854 bool canMakeOtherLost,
10855 uint32_t strategy,
10856 VmaAllocationRequest* pAllocationRequest)
10857 {
10858 VMA_ASSERT(allocSize > 0);
10859 VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
10860 VMA_ASSERT(pAllocationRequest != VMA_NULL);
10861 VMA_HEAVY_ASSERT(Validate());
10862 return upperAddress ?
10863 CreateAllocationRequest_UpperAddress(
10864 currentFrameIndex, frameInUseCount, bufferImageGranularity,
10865 allocSize, allocAlignment, allocType, canMakeOtherLost, strategy, pAllocationRequest) :
10866 CreateAllocationRequest_LowerAddress(
10867 currentFrameIndex, frameInUseCount, bufferImageGranularity,
10868 allocSize, allocAlignment, allocType, canMakeOtherLost, strategy, pAllocationRequest);
10869 }
10870
10871 bool VmaBlockMetadata_Linear::CreateAllocationRequest_UpperAddress(
10872 uint32_t currentFrameIndex,
10873 uint32_t frameInUseCount,
10874 VkDeviceSize bufferImageGranularity,
10875 VkDeviceSize allocSize,
10876 VkDeviceSize allocAlignment,
10877 VmaSuballocationType allocType,
10878 bool canMakeOtherLost,
10879 uint32_t strategy,
10880 VmaAllocationRequest* pAllocationRequest)
10881 {
10882 const VkDeviceSize size = GetSize();
10883 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
10884 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
10885
10886 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
10887 {
10888 VMA_ASSERT(0 && "Trying to use pool with linear algorithm as double stack, while it is already being used as ring buffer.");
10889 return false;
10890 }
10891
10892 // Try to allocate before 2nd.back(), or end of block if 2nd.empty().
10893 if(allocSize > size)
10894 {
10895 return false;
10896 }
10897 VkDeviceSize resultBaseOffset = size - allocSize;
10898 if(!suballocations2nd.empty())
10899 {
10900 const VmaSuballocation& lastSuballoc = suballocations2nd.back();
10901 resultBaseOffset = lastSuballoc.offset - allocSize;
10902 if(allocSize > lastSuballoc.offset)
10903 {
10904 return false;
10905 }
10906 }
10907
10908 // Start from offset equal to end of free space.
10909 VkDeviceSize resultOffset = resultBaseOffset;
10910
10911 // Apply VMA_DEBUG_MARGIN at the end.
10912 if(VMA_DEBUG_MARGIN > 0)
10913 {
10914 if(resultOffset < VMA_DEBUG_MARGIN)
10915 {
10916 return false;
10917 }
10918 resultOffset -= VMA_DEBUG_MARGIN;
10919 }
10920
10921 // Apply alignment.
10922 resultOffset = VmaAlignDown(resultOffset, allocAlignment);
10923
10924 // Check next suballocations from 2nd for BufferImageGranularity conflicts.
10925 // Make bigger alignment if necessary.
10926 if(bufferImageGranularity > 1 && !suballocations2nd.empty())
10927 {
10928 bool bufferImageGranularityConflict = false;
10929 for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; )
10930 {
10931 const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex];
10932 if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
10933 {
10934 if(VmaIsBufferImageGranularityConflict(nextSuballoc.type, allocType))
10935 {
10936 bufferImageGranularityConflict = true;
10937 break;
10938 }
10939 }
10940 else
10941 // Already on previous page.
10942 break;
10943 }
10944 if(bufferImageGranularityConflict)
10945 {
10946 resultOffset = VmaAlignDown(resultOffset, bufferImageGranularity);
10947 }
10948 }
10949
10950 // There is enough free space.
10951 const VkDeviceSize endOf1st = !suballocations1st.empty() ?
10952 suballocations1st.back().offset + suballocations1st.back().size :
10953 0;
10954 if(endOf1st + VMA_DEBUG_MARGIN <= resultOffset)
10955 {
10956 // Check previous suballocations for BufferImageGranularity conflicts.
10957 // If conflict exists, allocation cannot be made here.
10958 if(bufferImageGranularity > 1)
10959 {
10960 for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; )
10961 {
10962 const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex];
10963 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
10964 {
10965 if(VmaIsBufferImageGranularityConflict(allocType, prevSuballoc.type))
10966 {
10967 return false;
10968 }
10969 }
10970 else
10971 {
10972 // Already on next page.
10973 break;
10974 }
10975 }
10976 }
10977
10978 // All tests passed: Success.
10979 pAllocationRequest->offset = resultOffset;
10980 pAllocationRequest->sumFreeSize = resultBaseOffset + allocSize - endOf1st;
10981 pAllocationRequest->sumItemSize = 0;
10982 // pAllocationRequest->item unused.
10983 pAllocationRequest->itemsToMakeLostCount = 0;
10984 pAllocationRequest->type = VmaAllocationRequestType::UpperAddress;
10985 return true;
10986 }
10987
10988 return false;
10989 }
10990
10991 bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress(
10992 uint32_t currentFrameIndex,
10993 uint32_t frameInUseCount,
10994 VkDeviceSize bufferImageGranularity,
10995 VkDeviceSize allocSize,
10996 VkDeviceSize allocAlignment,
10997 VmaSuballocationType allocType,
10998 bool canMakeOtherLost,
10999 uint32_t strategy,
11000 VmaAllocationRequest* pAllocationRequest)
11001 {
11002 const VkDeviceSize size = GetSize();
11003 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11004 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11005
11006 if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
11007 {
11008 // Try to allocate at the end of 1st vector.
11009
11010 VkDeviceSize resultBaseOffset = 0;
11011 if(!suballocations1st.empty())
11012 {
11013 const VmaSuballocation& lastSuballoc = suballocations1st.back();
11014 resultBaseOffset = lastSuballoc.offset + lastSuballoc.size;
11015 }
11016
11017 // Start from offset equal to beginning of free space.
11018 VkDeviceSize resultOffset = resultBaseOffset;
11019
11020 // Apply VMA_DEBUG_MARGIN at the beginning.
11021 if(VMA_DEBUG_MARGIN > 0)
11022 {
11023 resultOffset += VMA_DEBUG_MARGIN;
11024 }
11025
11026 // Apply alignment.
11027 resultOffset = VmaAlignUp(resultOffset, allocAlignment);
11028
11029 // Check previous suballocations for BufferImageGranularity conflicts.
11030 // Make bigger alignment if necessary.
11031 if(bufferImageGranularity > 1 && !suballocations1st.empty())
11032 {
11033 bool bufferImageGranularityConflict = false;
11034 for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; )
11035 {
11036 const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex];
11037 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
11038 {
11039 if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
11040 {
11041 bufferImageGranularityConflict = true;
11042 break;
11043 }
11044 }
11045 else
11046 // Already on previous page.
11047 break;
11048 }
11049 if(bufferImageGranularityConflict)
11050 {
11051 resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity);
11052 }
11053 }
11054
11055 const VkDeviceSize freeSpaceEnd = m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ?
11056 suballocations2nd.back().offset : size;
11057
11058 // There is enough free space at the end after alignment.
11059 if(resultOffset + allocSize + VMA_DEBUG_MARGIN <= freeSpaceEnd)
11060 {
11061 // Check next suballocations for BufferImageGranularity conflicts.
11062 // If conflict exists, allocation cannot be made here.
11063 if(bufferImageGranularity > 1 && m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
11064 {
11065 for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; )
11066 {
11067 const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex];
11068 if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
11069 {
11070 if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
11071 {
11072 return false;
11073 }
11074 }
11075 else
11076 {
11077 // Already on previous page.
11078 break;
11079 }
11080 }
11081 }
11082
11083 // All tests passed: Success.
11084 pAllocationRequest->offset = resultOffset;
11085 pAllocationRequest->sumFreeSize = freeSpaceEnd - resultBaseOffset;
11086 pAllocationRequest->sumItemSize = 0;
11087 // pAllocationRequest->item, customData unused.
11088 pAllocationRequest->type = VmaAllocationRequestType::EndOf1st;
11089 pAllocationRequest->itemsToMakeLostCount = 0;
11090 return true;
11091 }
11092 }
11093
11094 // Wrap-around to end of 2nd vector. Try to allocate there, watching for the
11095 // beginning of 1st vector as the end of free space.
11096 if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
11097 {
11098 VMA_ASSERT(!suballocations1st.empty());
11099
11100 VkDeviceSize resultBaseOffset = 0;
11101 if(!suballocations2nd.empty())
11102 {
11103 const VmaSuballocation& lastSuballoc = suballocations2nd.back();
11104 resultBaseOffset = lastSuballoc.offset + lastSuballoc.size;
11105 }
11106
11107 // Start from offset equal to beginning of free space.
11108 VkDeviceSize resultOffset = resultBaseOffset;
11109
11110 // Apply VMA_DEBUG_MARGIN at the beginning.
11111 if(VMA_DEBUG_MARGIN > 0)
11112 {
11113 resultOffset += VMA_DEBUG_MARGIN;
11114 }
11115
11116 // Apply alignment.
11117 resultOffset = VmaAlignUp(resultOffset, allocAlignment);
11118
11119 // Check previous suballocations for BufferImageGranularity conflicts.
11120 // Make bigger alignment if necessary.
11121 if(bufferImageGranularity > 1 && !suballocations2nd.empty())
11122 {
11123 bool bufferImageGranularityConflict = false;
11124 for(size_t prevSuballocIndex = suballocations2nd.size(); prevSuballocIndex--; )
11125 {
11126 const VmaSuballocation& prevSuballoc = suballocations2nd[prevSuballocIndex];
11127 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
11128 {
11129 if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
11130 {
11131 bufferImageGranularityConflict = true;
11132 break;
11133 }
11134 }
11135 else
11136 // Already on previous page.
11137 break;
11138 }
11139 if(bufferImageGranularityConflict)
11140 {
11141 resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity);
11142 }
11143 }
11144
11145 pAllocationRequest->itemsToMakeLostCount = 0;
11146 pAllocationRequest->sumItemSize = 0;
11147 size_t index1st = m_1stNullItemsBeginCount;
11148
11149 if(canMakeOtherLost)
11150 {
11151 while(index1st < suballocations1st.size() &&
11152 resultOffset + allocSize + VMA_DEBUG_MARGIN > suballocations1st[index1st].offset)
11153 {
11154 // Next colliding allocation at the beginning of 1st vector found. Try to make it lost.
11155 const VmaSuballocation& suballoc = suballocations1st[index1st];
11156 if(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE)
11157 {
11158 // No problem.
11159 }
11160 else
11161 {
11162 VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE);
11163 if(suballoc.hAllocation->CanBecomeLost() &&
11164 suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
11165 {
11166 ++pAllocationRequest->itemsToMakeLostCount;
11167 pAllocationRequest->sumItemSize += suballoc.size;
11168 }
11169 else
11170 {
11171 return false;
11172 }
11173 }
11174 ++index1st;
11175 }
11176
11177 // Check next suballocations for BufferImageGranularity conflicts.
11178 // If conflict exists, we must mark more allocations lost or fail.
11179 if(bufferImageGranularity > 1)
11180 {
11181 while(index1st < suballocations1st.size())
11182 {
11183 const VmaSuballocation& suballoc = suballocations1st[index1st];
11184 if(VmaBlocksOnSamePage(resultOffset, allocSize, suballoc.offset, bufferImageGranularity))
11185 {
11186 if(suballoc.hAllocation != VK_NULL_HANDLE)
11187 {
11188 // Not checking actual VmaIsBufferImageGranularityConflict(allocType, suballoc.type).
11189 if(suballoc.hAllocation->CanBecomeLost() &&
11190 suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
11191 {
11192 ++pAllocationRequest->itemsToMakeLostCount;
11193 pAllocationRequest->sumItemSize += suballoc.size;
11194 }
11195 else
11196 {
11197 return false;
11198 }
11199 }
11200 }
11201 else
11202 {
11203 // Already on next page.
11204 break;
11205 }
11206 ++index1st;
11207 }
11208 }
11209
11210 // Special case: There is not enough room at the end for this allocation, even after making all from the 1st lost.
11211 if(index1st == suballocations1st.size() &&
11212 resultOffset + allocSize + VMA_DEBUG_MARGIN > size)
11213 {
11214 // TODO: This is a known bug that it's not yet implemented and the allocation is failing.
11215 VMA_DEBUG_LOG("Unsupported special case in custom pool with linear allocation algorithm used as ring buffer with allocations that can be lost.");
11216 }
11217 }
11218
11219 // There is enough free space at the end after alignment.
11220 if((index1st == suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN <= size) ||
11221 (index1st < suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN <= suballocations1st[index1st].offset))
11222 {
11223 // Check next suballocations for BufferImageGranularity conflicts.
11224 // If conflict exists, allocation cannot be made here.
11225 if(bufferImageGranularity > 1)
11226 {
11227 for(size_t nextSuballocIndex = index1st;
11228 nextSuballocIndex < suballocations1st.size();
11229 nextSuballocIndex++)
11230 {
11231 const VmaSuballocation& nextSuballoc = suballocations1st[nextSuballocIndex];
11232 if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
11233 {
11234 if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
11235 {
11236 return false;
11237 }
11238 }
11239 else
11240 {
11241 // Already on next page.
11242 break;
11243 }
11244 }
11245 }
11246
11247 // All tests passed: Success.
11248 pAllocationRequest->offset = resultOffset;
11249 pAllocationRequest->sumFreeSize =
11250 (index1st < suballocations1st.size() ? suballocations1st[index1st].offset : size)
11251 - resultBaseOffset
11252 - pAllocationRequest->sumItemSize;
11253 pAllocationRequest->type = VmaAllocationRequestType::EndOf2nd;
11254 // pAllocationRequest->item, customData unused.
11255 return true;
11256 }
11257 }
11258
11259 return false;
11260 }
11261
11262 bool VmaBlockMetadata_Linear::MakeRequestedAllocationsLost(
11263 uint32_t currentFrameIndex,
11264 uint32_t frameInUseCount,
11265 VmaAllocationRequest* pAllocationRequest)
11266 {
11267 if(pAllocationRequest->itemsToMakeLostCount == 0)
11268 {
11269 return true;
11270 }
11271
11272 VMA_ASSERT(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER);
11273
11274 // We always start from 1st.
11275 SuballocationVectorType* suballocations = &AccessSuballocations1st();
11276 size_t index = m_1stNullItemsBeginCount;
11277 size_t madeLostCount = 0;
11278 while(madeLostCount < pAllocationRequest->itemsToMakeLostCount)
11279 {
11280 if(index == suballocations->size())
11281 {
11282 index = 0;
11283 // If we get to the end of 1st, we wrap around to beginning of 2nd of 1st.
11284 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
11285 {
11286 suballocations = &AccessSuballocations2nd();
11287 }
11288 // else: m_2ndVectorMode == SECOND_VECTOR_EMPTY:
11289 // suballocations continues pointing at AccessSuballocations1st().
11290 VMA_ASSERT(!suballocations->empty());
11291 }
11292 VmaSuballocation& suballoc = (*suballocations)[index];
11293 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
11294 {
11295 VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE);
11296 VMA_ASSERT(suballoc.hAllocation->CanBecomeLost());
11297 if(suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
11298 {
11299 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
11300 suballoc.hAllocation = VK_NULL_HANDLE;
11301 m_SumFreeSize += suballoc.size;
11302 if(suballocations == &AccessSuballocations1st())
11303 {
11304 ++m_1stNullItemsMiddleCount;
11305 }
11306 else
11307 {
11308 ++m_2ndNullItemsCount;
11309 }
11310 ++madeLostCount;
11311 }
11312 else
11313 {
11314 return false;
11315 }
11316 }
11317 ++index;
11318 }
11319
11320 CleanupAfterFree();
11321 //VMA_HEAVY_ASSERT(Validate()); // Already called by ClanupAfterFree().
11322
11323 return true;
11324 }
11325
11326 uint32_t VmaBlockMetadata_Linear::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
11327 {
11328 uint32_t lostAllocationCount = 0;
11329
11330 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11331 for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i)
11332 {
11333 VmaSuballocation& suballoc = suballocations1st[i];
11334 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE &&
11335 suballoc.hAllocation->CanBecomeLost() &&
11336 suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
11337 {
11338 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
11339 suballoc.hAllocation = VK_NULL_HANDLE;
11340 ++m_1stNullItemsMiddleCount;
11341 m_SumFreeSize += suballoc.size;
11342 ++lostAllocationCount;
11343 }
11344 }
11345
11346 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11347 for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i)
11348 {
11349 VmaSuballocation& suballoc = suballocations2nd[i];
11350 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE &&
11351 suballoc.hAllocation->CanBecomeLost() &&
11352 suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
11353 {
11354 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
11355 suballoc.hAllocation = VK_NULL_HANDLE;
11356 ++m_2ndNullItemsCount;
11357 m_SumFreeSize += suballoc.size;
11358 ++lostAllocationCount;
11359 }
11360 }
11361
11362 if(lostAllocationCount)
11363 {
11364 CleanupAfterFree();
11365 }
11366
11367 return lostAllocationCount;
11368 }
11369
11370 VkResult VmaBlockMetadata_Linear::CheckCorruption(const void* pBlockData)
11371 {
11372 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11373 for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i)
11374 {
11375 const VmaSuballocation& suballoc = suballocations1st[i];
11376 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
11377 {
11378 if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN))
11379 {
11380 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
11381 return VK_ERROR_VALIDATION_FAILED_EXT;
11382 }
11383 if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size))
11384 {
11385 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
11386 return VK_ERROR_VALIDATION_FAILED_EXT;
11387 }
11388 }
11389 }
11390
11391 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11392 for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i)
11393 {
11394 const VmaSuballocation& suballoc = suballocations2nd[i];
11395 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
11396 {
11397 if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN))
11398 {
11399 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
11400 return VK_ERROR_VALIDATION_FAILED_EXT;
11401 }
11402 if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size))
11403 {
11404 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
11405 return VK_ERROR_VALIDATION_FAILED_EXT;
11406 }
11407 }
11408 }
11409
11410 return VK_SUCCESS;
11411 }
11412
11413 void VmaBlockMetadata_Linear::Alloc(
11414 const VmaAllocationRequest& request,
11415 VmaSuballocationType type,
11416 VkDeviceSize allocSize,
11417 VmaAllocation hAllocation)
11418 {
11419 const VmaSuballocation newSuballoc = { request.offset, allocSize, hAllocation, type };
11420
11421 switch(request.type)
11422 {
11423 case VmaAllocationRequestType::UpperAddress:
11424 {
11425 VMA_ASSERT(m_2ndVectorMode != SECOND_VECTOR_RING_BUFFER &&
11426 "CRITICAL ERROR: Trying to use linear allocator as double stack while it was already used as ring buffer.");
11427 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11428 suballocations2nd.push_back(newSuballoc);
11429 m_2ndVectorMode = SECOND_VECTOR_DOUBLE_STACK;
11430 }
11431 break;
11432 case VmaAllocationRequestType::EndOf1st:
11433 {
11434 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11435
11436 VMA_ASSERT(suballocations1st.empty() ||
11437 request.offset >= suballocations1st.back().offset + suballocations1st.back().size);
11438 // Check if it fits before the end of the block.
11439 VMA_ASSERT(request.offset + allocSize <= GetSize());
11440
11441 suballocations1st.push_back(newSuballoc);
11442 }
11443 break;
11444 case VmaAllocationRequestType::EndOf2nd:
11445 {
11446 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11447 // New allocation at the end of 2-part ring buffer, so before first allocation from 1st vector.
11448 VMA_ASSERT(!suballocations1st.empty() &&
11449 request.offset + allocSize <= suballocations1st[m_1stNullItemsBeginCount].offset);
11450 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11451
11452 switch(m_2ndVectorMode)
11453 {
11454 case SECOND_VECTOR_EMPTY:
11455 // First allocation from second part ring buffer.
11456 VMA_ASSERT(suballocations2nd.empty());
11457 m_2ndVectorMode = SECOND_VECTOR_RING_BUFFER;
11458 break;
11459 case SECOND_VECTOR_RING_BUFFER:
11460 // 2-part ring buffer is already started.
11461 VMA_ASSERT(!suballocations2nd.empty());
11462 break;
11463 case SECOND_VECTOR_DOUBLE_STACK:
11464 VMA_ASSERT(0 && "CRITICAL ERROR: Trying to use linear allocator as ring buffer while it was already used as double stack.");
11465 break;
11466 default:
11467 VMA_ASSERT(0);
11468 }
11469
11470 suballocations2nd.push_back(newSuballoc);
11471 }
11472 break;
11473 default:
11474 VMA_ASSERT(0 && "CRITICAL INTERNAL ERROR.");
11475 }
11476
11477 m_SumFreeSize -= newSuballoc.size;
11478 }
11479
11480 void VmaBlockMetadata_Linear::Free(const VmaAllocation allocation)
11481 {
11482 FreeAtOffset(allocation->GetOffset());
11483 }
11484
11485 void VmaBlockMetadata_Linear::FreeAtOffset(VkDeviceSize offset)
11486 {
11487 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11488 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11489
11490 if(!suballocations1st.empty())
11491 {
11492 // First allocation: Mark it as next empty at the beginning.
11493 VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount];
11494 if(firstSuballoc.offset == offset)
11495 {
11496 firstSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
11497 firstSuballoc.hAllocation = VK_NULL_HANDLE;
11498 m_SumFreeSize += firstSuballoc.size;
11499 ++m_1stNullItemsBeginCount;
11500 CleanupAfterFree();
11501 return;
11502 }
11503 }
11504
11505 // Last allocation in 2-part ring buffer or top of upper stack (same logic).
11506 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ||
11507 m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
11508 {
11509 VmaSuballocation& lastSuballoc = suballocations2nd.back();
11510 if(lastSuballoc.offset == offset)
11511 {
11512 m_SumFreeSize += lastSuballoc.size;
11513 suballocations2nd.pop_back();
11514 CleanupAfterFree();
11515 return;
11516 }
11517 }
11518 // Last allocation in 1st vector.
11519 else if(m_2ndVectorMode == SECOND_VECTOR_EMPTY)
11520 {
11521 VmaSuballocation& lastSuballoc = suballocations1st.back();
11522 if(lastSuballoc.offset == offset)
11523 {
11524 m_SumFreeSize += lastSuballoc.size;
11525 suballocations1st.pop_back();
11526 CleanupAfterFree();
11527 return;
11528 }
11529 }
11530
11531 // Item from the middle of 1st vector.
11532 {
11533 VmaSuballocation refSuballoc;
11534 refSuballoc.offset = offset;
11535 // Rest of members stays uninitialized intentionally for better performance.
11536 SuballocationVectorType::iterator it = VmaBinaryFindSorted(
11537 suballocations1st.begin() + m_1stNullItemsBeginCount,
11538 suballocations1st.end(),
11539 refSuballoc,
11540 VmaSuballocationOffsetLess());
11541 if(it != suballocations1st.end())
11542 {
11543 it->type = VMA_SUBALLOCATION_TYPE_FREE;
11544 it->hAllocation = VK_NULL_HANDLE;
11545 ++m_1stNullItemsMiddleCount;
11546 m_SumFreeSize += it->size;
11547 CleanupAfterFree();
11548 return;
11549 }
11550 }
11551
11552 if(m_2ndVectorMode != SECOND_VECTOR_EMPTY)
11553 {
11554 // Item from the middle of 2nd vector.
11555 VmaSuballocation refSuballoc;
11556 refSuballoc.offset = offset;
11557 // Rest of members stays uninitialized intentionally for better performance.
11558 SuballocationVectorType::iterator it = m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ?
11559 VmaBinaryFindSorted(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc, VmaSuballocationOffsetLess()) :
11560 VmaBinaryFindSorted(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc, VmaSuballocationOffsetGreater());
11561 if(it != suballocations2nd.end())
11562 {
11563 it->type = VMA_SUBALLOCATION_TYPE_FREE;
11564 it->hAllocation = VK_NULL_HANDLE;
11565 ++m_2ndNullItemsCount;
11566 m_SumFreeSize += it->size;
11567 CleanupAfterFree();
11568 return;
11569 }
11570 }
11571
11572 VMA_ASSERT(0 && "Allocation to free not found in linear allocator!");
11573 }
11574
11575 bool VmaBlockMetadata_Linear::ShouldCompact1st() const
11576 {
11577 const size_t nullItemCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount;
11578 const size_t suballocCount = AccessSuballocations1st().size();
11579 return suballocCount > 32 && nullItemCount * 2 >= (suballocCount - nullItemCount) * 3;
11580 }
11581
11582 void VmaBlockMetadata_Linear::CleanupAfterFree()
11583 {
11584 SuballocationVectorType& suballocations1st = AccessSuballocations1st();
11585 SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
11586
11587 if(IsEmpty())
11588 {
11589 suballocations1st.clear();
11590 suballocations2nd.clear();
11591 m_1stNullItemsBeginCount = 0;
11592 m_1stNullItemsMiddleCount = 0;
11593 m_2ndNullItemsCount = 0;
11594 m_2ndVectorMode = SECOND_VECTOR_EMPTY;
11595 }
11596 else
11597 {
11598 const size_t suballoc1stCount = suballocations1st.size();
11599 const size_t nullItem1stCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount;
11600 VMA_ASSERT(nullItem1stCount <= suballoc1stCount);
11601
11602 // Find more null items at the beginning of 1st vector.
11603 while(m_1stNullItemsBeginCount < suballoc1stCount &&
11604 suballocations1st[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
11605 {
11606 ++m_1stNullItemsBeginCount;
11607 --m_1stNullItemsMiddleCount;
11608 }
11609
11610 // Find more null items at the end of 1st vector.
11611 while(m_1stNullItemsMiddleCount > 0 &&
11612 suballocations1st.back().hAllocation == VK_NULL_HANDLE)
11613 {
11614 --m_1stNullItemsMiddleCount;
11615 suballocations1st.pop_back();
11616 }
11617
11618 // Find more null items at the end of 2nd vector.
11619 while(m_2ndNullItemsCount > 0 &&
11620 suballocations2nd.back().hAllocation == VK_NULL_HANDLE)
11621 {
11622 --m_2ndNullItemsCount;
11623 suballocations2nd.pop_back();
11624 }
11625
11626 // Find more null items at the beginning of 2nd vector.
11627 while(m_2ndNullItemsCount > 0 &&
11628 suballocations2nd[0].hAllocation == VK_NULL_HANDLE)
11629 {
11630 --m_2ndNullItemsCount;
11631 VmaVectorRemove(suballocations2nd, 0);
11632 }
11633
11634 if(ShouldCompact1st())
11635 {
11636 const size_t nonNullItemCount = suballoc1stCount - nullItem1stCount;
11637 size_t srcIndex = m_1stNullItemsBeginCount;
11638 for(size_t dstIndex = 0; dstIndex < nonNullItemCount; ++dstIndex)
11639 {
11640 while(suballocations1st[srcIndex].hAllocation == VK_NULL_HANDLE)
11641 {
11642 ++srcIndex;
11643 }
11644 if(dstIndex != srcIndex)
11645 {
11646 suballocations1st[dstIndex] = suballocations1st[srcIndex];
11647 }
11648 ++srcIndex;
11649 }
11650 suballocations1st.resize(nonNullItemCount);
11651 m_1stNullItemsBeginCount = 0;
11652 m_1stNullItemsMiddleCount = 0;
11653 }
11654
11655 // 2nd vector became empty.
11656 if(suballocations2nd.empty())
11657 {
11658 m_2ndVectorMode = SECOND_VECTOR_EMPTY;
11659 }
11660
11661 // 1st vector became empty.
11662 if(suballocations1st.size() - m_1stNullItemsBeginCount == 0)
11663 {
11664 suballocations1st.clear();
11665 m_1stNullItemsBeginCount = 0;
11666
11667 if(!suballocations2nd.empty() && m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
11668 {
11669 // Swap 1st with 2nd. Now 2nd is empty.
11670 m_2ndVectorMode = SECOND_VECTOR_EMPTY;
11671 m_1stNullItemsMiddleCount = m_2ndNullItemsCount;
11672 while(m_1stNullItemsBeginCount < suballocations2nd.size() &&
11673 suballocations2nd[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
11674 {
11675 ++m_1stNullItemsBeginCount;
11676 --m_1stNullItemsMiddleCount;
11677 }
11678 m_2ndNullItemsCount = 0;
11679 m_1stVectorIndex ^= 1;
11680 }
11681 }
11682 }
11683
11684 VMA_HEAVY_ASSERT(Validate());
11685 }
11686
11687
11688 ////////////////////////////////////////////////////////////////////////////////
11689 // class VmaBlockMetadata_Buddy
11690
11691 VmaBlockMetadata_Buddy::VmaBlockMetadata_Buddy(VmaAllocator hAllocator) :
11692 VmaBlockMetadata(hAllocator),
11693 m_Root(VMA_NULL),
11694 m_AllocationCount(0),
11695 m_FreeCount(1),
11696 m_SumFreeSize(0)
11697 {
11698 memset(m_FreeList, 0, sizeof(m_FreeList));
11699 }
11700
11701 VmaBlockMetadata_Buddy::~VmaBlockMetadata_Buddy()
11702 {
11703 DeleteNode(m_Root);
11704 }
11705
11706 void VmaBlockMetadata_Buddy::Init(VkDeviceSize size)
11707 {
11708 VmaBlockMetadata::Init(size);
11709
11710 m_UsableSize = VmaPrevPow2(size);
11711 m_SumFreeSize = m_UsableSize;
11712
11713 // Calculate m_LevelCount.
11714 m_LevelCount = 1;
11715 while(m_LevelCount < MAX_LEVELS &&
11716 LevelToNodeSize(m_LevelCount) >= MIN_NODE_SIZE)
11717 {
11718 ++m_LevelCount;
11719 }
11720
11721 Node* rootNode = vma_new(GetAllocationCallbacks(), Node)();
11722 rootNode->offset = 0;
11723 rootNode->type = Node::TYPE_FREE;
11724 rootNode->parent = VMA_NULL;
11725 rootNode->buddy = VMA_NULL;
11726
11727 m_Root = rootNode;
11728 AddToFreeListFront(0, rootNode);
11729 }
11730
11731 bool VmaBlockMetadata_Buddy::Validate() const
11732 {
11733 // Validate tree.
11734 ValidationContext ctx;
11735 if(!ValidateNode(ctx, VMA_NULL, m_Root, 0, LevelToNodeSize(0)))
11736 {
11737 VMA_VALIDATE(false && "ValidateNode failed.");
11738 }
11739 VMA_VALIDATE(m_AllocationCount == ctx.calculatedAllocationCount);
11740 VMA_VALIDATE(m_SumFreeSize == ctx.calculatedSumFreeSize);
11741
11742 // Validate free node lists.
11743 for(uint32_t level = 0; level < m_LevelCount; ++level)
11744 {
11745 VMA_VALIDATE(m_FreeList[level].front == VMA_NULL ||
11746 m_FreeList[level].front->free.prev == VMA_NULL);
11747
11748 for(Node* node = m_FreeList[level].front;
11749 node != VMA_NULL;
11750 node = node->free.next)
11751 {
11752 VMA_VALIDATE(node->type == Node::TYPE_FREE);
11753
11754 if(node->free.next == VMA_NULL)
11755 {
11756 VMA_VALIDATE(m_FreeList[level].back == node);
11757 }
11758 else
11759 {
11760 VMA_VALIDATE(node->free.next->free.prev == node);
11761 }
11762 }
11763 }
11764
11765 // Validate that free lists ar higher levels are empty.
11766 for(uint32_t level = m_LevelCount; level < MAX_LEVELS; ++level)
11767 {
11768 VMA_VALIDATE(m_FreeList[level].front == VMA_NULL && m_FreeList[level].back == VMA_NULL);
11769 }
11770
11771 return true;
11772 }
11773
11774 VkDeviceSize VmaBlockMetadata_Buddy::GetUnusedRangeSizeMax() const
11775 {
11776 for(uint32_t level = 0; level < m_LevelCount; ++level)
11777 {
11778 if(m_FreeList[level].front != VMA_NULL)
11779 {
11780 return LevelToNodeSize(level);
11781 }
11782 }
11783 return 0;
11784 }
11785
11786 void VmaBlockMetadata_Buddy::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
11787 {
11788 const VkDeviceSize unusableSize = GetUnusableSize();
11789
11790 outInfo.blockCount = 1;
11791
11792 outInfo.allocationCount = outInfo.unusedRangeCount = 0;
11793 outInfo.usedBytes = outInfo.unusedBytes = 0;
11794
11795 outInfo.allocationSizeMax = outInfo.unusedRangeSizeMax = 0;
11796 outInfo.allocationSizeMin = outInfo.unusedRangeSizeMin = UINT64_MAX;
11797 outInfo.allocationSizeAvg = outInfo.unusedRangeSizeAvg = 0; // Unused.
11798
11799 CalcAllocationStatInfoNode(outInfo, m_Root, LevelToNodeSize(0));
11800
11801 if(unusableSize > 0)
11802 {
11803 ++outInfo.unusedRangeCount;
11804 outInfo.unusedBytes += unusableSize;
11805 outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, unusableSize);
11806 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusableSize);
11807 }
11808 }
11809
11810 void VmaBlockMetadata_Buddy::AddPoolStats(VmaPoolStats& inoutStats) const
11811 {
11812 const VkDeviceSize unusableSize = GetUnusableSize();
11813
11814 inoutStats.size += GetSize();
11815 inoutStats.unusedSize += m_SumFreeSize + unusableSize;
11816 inoutStats.allocationCount += m_AllocationCount;
11817 inoutStats.unusedRangeCount += m_FreeCount;
11818 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
11819
11820 if(unusableSize > 0)
11821 {
11822 ++inoutStats.unusedRangeCount;
11823 // Not updating inoutStats.unusedRangeSizeMax with unusableSize because this space is not available for allocations.
11824 }
11825 }
11826
11827 #if VMA_STATS_STRING_ENABLED
11828
11829 void VmaBlockMetadata_Buddy::PrintDetailedMap(class VmaJsonWriter& json) const
11830 {
11831 // TODO optimize
11832 VmaStatInfo stat;
11833 CalcAllocationStatInfo(stat);
11834
11835 PrintDetailedMap_Begin(
11836 json,
11837 stat.unusedBytes,
11838 stat.allocationCount,
11839 stat.unusedRangeCount);
11840
11841 PrintDetailedMapNode(json, m_Root, LevelToNodeSize(0));
11842
11843 const VkDeviceSize unusableSize = GetUnusableSize();
11844 if(unusableSize > 0)
11845 {
11846 PrintDetailedMap_UnusedRange(json,
11847 m_UsableSize, // offset
11848 unusableSize); // size
11849 }
11850
11851 PrintDetailedMap_End(json);
11852 }
11853
11854 #endif // #if VMA_STATS_STRING_ENABLED
11855
11856 bool VmaBlockMetadata_Buddy::CreateAllocationRequest(
11857 uint32_t currentFrameIndex,
11858 uint32_t frameInUseCount,
11859 VkDeviceSize bufferImageGranularity,
11860 VkDeviceSize allocSize,
11861 VkDeviceSize allocAlignment,
11862 bool upperAddress,
11863 VmaSuballocationType allocType,
11864 bool canMakeOtherLost,
11865 uint32_t strategy,
11866 VmaAllocationRequest* pAllocationRequest)
11867 {
11868 VMA_ASSERT(!upperAddress && "VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT can be used only with linear algorithm.");
11869
11870 // Simple way to respect bufferImageGranularity. May be optimized some day.
11871 // Whenever it might be an OPTIMAL image...
11872 if(allocType == VMA_SUBALLOCATION_TYPE_UNKNOWN ||
11873 allocType == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
11874 allocType == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL)
11875 {
11876 allocAlignment = VMA_MAX(allocAlignment, bufferImageGranularity);
11877 allocSize = VMA_MAX(allocSize, bufferImageGranularity);
11878 }
11879
11880 if(allocSize > m_UsableSize)
11881 {
11882 return false;
11883 }
11884
11885 const uint32_t targetLevel = AllocSizeToLevel(allocSize);
11886 for(uint32_t level = targetLevel + 1; level--; )
11887 {
11888 for(Node* freeNode = m_FreeList[level].front;
11889 freeNode != VMA_NULL;
11890 freeNode = freeNode->free.next)
11891 {
11892 if(freeNode->offset % allocAlignment == 0)
11893 {
11894 pAllocationRequest->type = VmaAllocationRequestType::Normal;
11895 pAllocationRequest->offset = freeNode->offset;
11896 pAllocationRequest->sumFreeSize = LevelToNodeSize(level);
11897 pAllocationRequest->sumItemSize = 0;
11898 pAllocationRequest->itemsToMakeLostCount = 0;
11899 pAllocationRequest->customData = (void*)(uintptr_t)level;
11900 return true;
11901 }
11902 }
11903 }
11904
11905 return false;
11906 }
11907
11908 bool VmaBlockMetadata_Buddy::MakeRequestedAllocationsLost(
11909 uint32_t currentFrameIndex,
11910 uint32_t frameInUseCount,
11911 VmaAllocationRequest* pAllocationRequest)
11912 {
11913 /*
11914 Lost allocations are not supported in buddy allocator at the moment.
11915 Support might be added in the future.
11916 */
11917 return pAllocationRequest->itemsToMakeLostCount == 0;
11918 }
11919
11920 uint32_t VmaBlockMetadata_Buddy::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
11921 {
11922 /*
11923 Lost allocations are not supported in buddy allocator at the moment.
11924 Support might be added in the future.
11925 */
11926 return 0;
11927 }
11928
11929 void VmaBlockMetadata_Buddy::Alloc(
11930 const VmaAllocationRequest& request,
11931 VmaSuballocationType type,
11932 VkDeviceSize allocSize,
11933 VmaAllocation hAllocation)
11934 {
11935 VMA_ASSERT(request.type == VmaAllocationRequestType::Normal);
11936
11937 const uint32_t targetLevel = AllocSizeToLevel(allocSize);
11938 uint32_t currLevel = (uint32_t)(uintptr_t)request.customData;
11939
11940 Node* currNode = m_FreeList[currLevel].front;
11941 VMA_ASSERT(currNode != VMA_NULL && currNode->type == Node::TYPE_FREE);
11942 while(currNode->offset != request.offset)
11943 {
11944 currNode = currNode->free.next;
11945 VMA_ASSERT(currNode != VMA_NULL && currNode->type == Node::TYPE_FREE);
11946 }
11947
11948 // Go down, splitting free nodes.
11949 while(currLevel < targetLevel)
11950 {
11951 // currNode is already first free node at currLevel.
11952 // Remove it from list of free nodes at this currLevel.
11953 RemoveFromFreeList(currLevel, currNode);
11954
11955 const uint32_t childrenLevel = currLevel + 1;
11956
11957 // Create two free sub-nodes.
11958 Node* leftChild = vma_new(GetAllocationCallbacks(), Node)();
11959 Node* rightChild = vma_new(GetAllocationCallbacks(), Node)();
11960
11961 leftChild->offset = currNode->offset;
11962 leftChild->type = Node::TYPE_FREE;
11963 leftChild->parent = currNode;
11964 leftChild->buddy = rightChild;
11965
11966 rightChild->offset = currNode->offset + LevelToNodeSize(childrenLevel);
11967 rightChild->type = Node::TYPE_FREE;
11968 rightChild->parent = currNode;
11969 rightChild->buddy = leftChild;
11970
11971 // Convert current currNode to split type.
11972 currNode->type = Node::TYPE_SPLIT;
11973 currNode->split.leftChild = leftChild;
11974
11975 // Add child nodes to free list. Order is important!
11976 AddToFreeListFront(childrenLevel, rightChild);
11977 AddToFreeListFront(childrenLevel, leftChild);
11978
11979 ++m_FreeCount;
11980 //m_SumFreeSize -= LevelToNodeSize(currLevel) % 2; // Useful only when level node sizes can be non power of 2.
11981 ++currLevel;
11982 currNode = m_FreeList[currLevel].front;
11983
11984 /*
11985 We can be sure that currNode, as left child of node previously split,
11986 also fullfills the alignment requirement.
11987 */
11988 }
11989
11990 // Remove from free list.
11991 VMA_ASSERT(currLevel == targetLevel &&
11992 currNode != VMA_NULL &&
11993 currNode->type == Node::TYPE_FREE);
11994 RemoveFromFreeList(currLevel, currNode);
11995
11996 // Convert to allocation node.
11997 currNode->type = Node::TYPE_ALLOCATION;
11998 currNode->allocation.alloc = hAllocation;
11999
12000 ++m_AllocationCount;
12001 --m_FreeCount;
12002 m_SumFreeSize -= allocSize;
12003 }
12004
12005 void VmaBlockMetadata_Buddy::DeleteNode(Node* node)
12006 {
12007 if(node->type == Node::TYPE_SPLIT)
12008 {
12009 DeleteNode(node->split.leftChild->buddy);
12010 DeleteNode(node->split.leftChild);
12011 }
12012
12013 vma_delete(GetAllocationCallbacks(), node);
12014 }
12015
12016 bool VmaBlockMetadata_Buddy::ValidateNode(ValidationContext& ctx, const Node* parent, const Node* curr, uint32_t level, VkDeviceSize levelNodeSize) const
12017 {
12018 VMA_VALIDATE(level < m_LevelCount);
12019 VMA_VALIDATE(curr->parent == parent);
12020 VMA_VALIDATE((curr->buddy == VMA_NULL) == (parent == VMA_NULL));
12021 VMA_VALIDATE(curr->buddy == VMA_NULL || curr->buddy->buddy == curr);
12022 switch(curr->type)
12023 {
12024 case Node::TYPE_FREE:
12025 // curr->free.prev, next are validated separately.
12026 ctx.calculatedSumFreeSize += levelNodeSize;
12027 ++ctx.calculatedFreeCount;
12028 break;
12029 case Node::TYPE_ALLOCATION:
12030 ++ctx.calculatedAllocationCount;
12031 ctx.calculatedSumFreeSize += levelNodeSize - curr->allocation.alloc->GetSize();
12032 VMA_VALIDATE(curr->allocation.alloc != VK_NULL_HANDLE);
12033 break;
12034 case Node::TYPE_SPLIT:
12035 {
12036 const uint32_t childrenLevel = level + 1;
12037 const VkDeviceSize childrenLevelNodeSize = levelNodeSize / 2;
12038 const Node* const leftChild = curr->split.leftChild;
12039 VMA_VALIDATE(leftChild != VMA_NULL);
12040 VMA_VALIDATE(leftChild->offset == curr->offset);
12041 if(!ValidateNode(ctx, curr, leftChild, childrenLevel, childrenLevelNodeSize))
12042 {
12043 VMA_VALIDATE(false && "ValidateNode for left child failed.");
12044 }
12045 const Node* const rightChild = leftChild->buddy;
12046 VMA_VALIDATE(rightChild->offset == curr->offset + childrenLevelNodeSize);
12047 if(!ValidateNode(ctx, curr, rightChild, childrenLevel, childrenLevelNodeSize))
12048 {
12049 VMA_VALIDATE(false && "ValidateNode for right child failed.");
12050 }
12051 }
12052 break;
12053 default:
12054 return false;
12055 }
12056
12057 return true;
12058 }
12059
12060 uint32_t VmaBlockMetadata_Buddy::AllocSizeToLevel(VkDeviceSize allocSize) const
12061 {
12062 // I know this could be optimized somehow e.g. by using std::log2p1 from C++20.
12063 uint32_t level = 0;
12064 VkDeviceSize currLevelNodeSize = m_UsableSize;
12065 VkDeviceSize nextLevelNodeSize = currLevelNodeSize >> 1;
12066 while(allocSize <= nextLevelNodeSize && level + 1 < m_LevelCount)
12067 {
12068 ++level;
12069 currLevelNodeSize = nextLevelNodeSize;
12070 nextLevelNodeSize = currLevelNodeSize >> 1;
12071 }
12072 return level;
12073 }
12074
12075 void VmaBlockMetadata_Buddy::FreeAtOffset(VmaAllocation alloc, VkDeviceSize offset)
12076 {
12077 // Find node and level.
12078 Node* node = m_Root;
12079 VkDeviceSize nodeOffset = 0;
12080 uint32_t level = 0;
12081 VkDeviceSize levelNodeSize = LevelToNodeSize(0);
12082 while(node->type == Node::TYPE_SPLIT)
12083 {
12084 const VkDeviceSize nextLevelSize = levelNodeSize >> 1;
12085 if(offset < nodeOffset + nextLevelSize)
12086 {
12087 node = node->split.leftChild;
12088 }
12089 else
12090 {
12091 node = node->split.leftChild->buddy;
12092 nodeOffset += nextLevelSize;
12093 }
12094 ++level;
12095 levelNodeSize = nextLevelSize;
12096 }
12097
12098 VMA_ASSERT(node != VMA_NULL && node->type == Node::TYPE_ALLOCATION);
12099 VMA_ASSERT(alloc == VK_NULL_HANDLE || node->allocation.alloc == alloc);
12100
12101 ++m_FreeCount;
12102 --m_AllocationCount;
12103 m_SumFreeSize += alloc->GetSize();
12104
12105 node->type = Node::TYPE_FREE;
12106
12107 // Join free nodes if possible.
12108 while(level > 0 && node->buddy->type == Node::TYPE_FREE)
12109 {
12110 RemoveFromFreeList(level, node->buddy);
12111 Node* const parent = node->parent;
12112
12113 vma_delete(GetAllocationCallbacks(), node->buddy);
12114 vma_delete(GetAllocationCallbacks(), node);
12115 parent->type = Node::TYPE_FREE;
12116
12117 node = parent;
12118 --level;
12119 //m_SumFreeSize += LevelToNodeSize(level) % 2; // Useful only when level node sizes can be non power of 2.
12120 --m_FreeCount;
12121 }
12122
12123 AddToFreeListFront(level, node);
12124 }
12125
12126 void VmaBlockMetadata_Buddy::CalcAllocationStatInfoNode(VmaStatInfo& outInfo, const Node* node, VkDeviceSize levelNodeSize) const
12127 {
12128 switch(node->type)
12129 {
12130 case Node::TYPE_FREE:
12131 ++outInfo.unusedRangeCount;
12132 outInfo.unusedBytes += levelNodeSize;
12133 outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, levelNodeSize);
12134 outInfo.unusedRangeSizeMin = VMA_MAX(outInfo.unusedRangeSizeMin, levelNodeSize);
12135 break;
12136 case Node::TYPE_ALLOCATION:
12137 {
12138 const VkDeviceSize allocSize = node->allocation.alloc->GetSize();
12139 ++outInfo.allocationCount;
12140 outInfo.usedBytes += allocSize;
12141 outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, allocSize);
12142 outInfo.allocationSizeMin = VMA_MAX(outInfo.allocationSizeMin, allocSize);
12143
12144 const VkDeviceSize unusedRangeSize = levelNodeSize - allocSize;
12145 if(unusedRangeSize > 0)
12146 {
12147 ++outInfo.unusedRangeCount;
12148 outInfo.unusedBytes += unusedRangeSize;
12149 outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, unusedRangeSize);
12150 outInfo.unusedRangeSizeMin = VMA_MAX(outInfo.unusedRangeSizeMin, unusedRangeSize);
12151 }
12152 }
12153 break;
12154 case Node::TYPE_SPLIT:
12155 {
12156 const VkDeviceSize childrenNodeSize = levelNodeSize / 2;
12157 const Node* const leftChild = node->split.leftChild;
12158 CalcAllocationStatInfoNode(outInfo, leftChild, childrenNodeSize);
12159 const Node* const rightChild = leftChild->buddy;
12160 CalcAllocationStatInfoNode(outInfo, rightChild, childrenNodeSize);
12161 }
12162 break;
12163 default:
12164 VMA_ASSERT(0);
12165 }
12166 }
12167
12168 void VmaBlockMetadata_Buddy::AddToFreeListFront(uint32_t level, Node* node)
12169 {
12170 VMA_ASSERT(node->type == Node::TYPE_FREE);
12171
12172 // List is empty.
12173 Node* const frontNode = m_FreeList[level].front;
12174 if(frontNode == VMA_NULL)
12175 {
12176 VMA_ASSERT(m_FreeList[level].back == VMA_NULL);
12177 node->free.prev = node->free.next = VMA_NULL;
12178 m_FreeList[level].front = m_FreeList[level].back = node;
12179 }
12180 else
12181 {
12182 VMA_ASSERT(frontNode->free.prev == VMA_NULL);
12183 node->free.prev = VMA_NULL;
12184 node->free.next = frontNode;
12185 frontNode->free.prev = node;
12186 m_FreeList[level].front = node;
12187 }
12188 }
12189
12190 void VmaBlockMetadata_Buddy::RemoveFromFreeList(uint32_t level, Node* node)
12191 {
12192 VMA_ASSERT(m_FreeList[level].front != VMA_NULL);
12193
12194 // It is at the front.
12195 if(node->free.prev == VMA_NULL)
12196 {
12197 VMA_ASSERT(m_FreeList[level].front == node);
12198 m_FreeList[level].front = node->free.next;
12199 }
12200 else
12201 {
12202 Node* const prevFreeNode = node->free.prev;
12203 VMA_ASSERT(prevFreeNode->free.next == node);
12204 prevFreeNode->free.next = node->free.next;
12205 }
12206
12207 // It is at the back.
12208 if(node->free.next == VMA_NULL)
12209 {
12210 VMA_ASSERT(m_FreeList[level].back == node);
12211 m_FreeList[level].back = node->free.prev;
12212 }
12213 else
12214 {
12215 Node* const nextFreeNode = node->free.next;
12216 VMA_ASSERT(nextFreeNode->free.prev == node);
12217 nextFreeNode->free.prev = node->free.prev;
12218 }
12219 }
12220
12221 #if VMA_STATS_STRING_ENABLED
12222 void VmaBlockMetadata_Buddy::PrintDetailedMapNode(class VmaJsonWriter& json, const Node* node, VkDeviceSize levelNodeSize) const
12223 {
12224 switch(node->type)
12225 {
12226 case Node::TYPE_FREE:
12227 PrintDetailedMap_UnusedRange(json, node->offset, levelNodeSize);
12228 break;
12229 case Node::TYPE_ALLOCATION:
12230 {
12231 PrintDetailedMap_Allocation(json, node->offset, node->allocation.alloc);
12232 const VkDeviceSize allocSize = node->allocation.alloc->GetSize();
12233 if(allocSize < levelNodeSize)
12234 {
12235 PrintDetailedMap_UnusedRange(json, node->offset + allocSize, levelNodeSize - allocSize);
12236 }
12237 }
12238 break;
12239 case Node::TYPE_SPLIT:
12240 {
12241 const VkDeviceSize childrenNodeSize = levelNodeSize / 2;
12242 const Node* const leftChild = node->split.leftChild;
12243 PrintDetailedMapNode(json, leftChild, childrenNodeSize);
12244 const Node* const rightChild = leftChild->buddy;
12245 PrintDetailedMapNode(json, rightChild, childrenNodeSize);
12246 }
12247 break;
12248 default:
12249 VMA_ASSERT(0);
12250 }
12251 }
12252 #endif // #if VMA_STATS_STRING_ENABLED
12253
12254
12255 ////////////////////////////////////////////////////////////////////////////////
12256 // class VmaDeviceMemoryBlock
12257
12258 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
12259 m_pMetadata(VMA_NULL),
12260 m_MemoryTypeIndex(UINT32_MAX),
12261 m_Id(0),
12262 m_hMemory(VK_NULL_HANDLE),
12263 m_MapCount(0),
12264 m_pMappedData(VMA_NULL)
12265 {
12266 }
12267
12268 void VmaDeviceMemoryBlock::Init(
12269 VmaAllocator hAllocator,
12270 VmaPool hParentPool,
12271 uint32_t newMemoryTypeIndex,
12272 VkDeviceMemory newMemory,
12273 VkDeviceSize newSize,
12274 uint32_t id,
12275 uint32_t algorithm)
12276 {
12277 VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
12278
12279 m_hParentPool = hParentPool;
12280 m_MemoryTypeIndex = newMemoryTypeIndex;
12281 m_Id = id;
12282 m_hMemory = newMemory;
12283
12284 switch(algorithm)
12285 {
12286 case VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT:
12287 m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Linear)(hAllocator);
12288 break;
12289 case VMA_POOL_CREATE_BUDDY_ALGORITHM_BIT:
12290 m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Buddy)(hAllocator);
12291 break;
12292 default:
12293 VMA_ASSERT(0);
12294 // Fall-through.
12295 case 0:
12296 m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Generic)(hAllocator);
12297 }
12298 m_pMetadata->Init(newSize);
12299 }
12300
12301 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
12302 {
12303 // This is the most important assert in the entire library.
12304 // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
12305 VMA_ASSERT(m_pMetadata->IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
12306
12307 VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
12308 allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_pMetadata->GetSize(), m_hMemory);
12309 m_hMemory = VK_NULL_HANDLE;
12310
12311 vma_delete(allocator, m_pMetadata);
12312 m_pMetadata = VMA_NULL;
12313 }
12314
12315 bool VmaDeviceMemoryBlock::Validate() const
12316 {
12317 VMA_VALIDATE((m_hMemory != VK_NULL_HANDLE) &&
12318 (m_pMetadata->GetSize() != 0));
12319
12320 return m_pMetadata->Validate();
12321 }
12322
12323 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
12324 {
12325 void* pData = nullptr;
12326 VkResult res = Map(hAllocator, 1, &pData);
12327 if(res != VK_SUCCESS)
12328 {
12329 return res;
12330 }
12331
12332 res = m_pMetadata->CheckCorruption(pData);
12333
12334 Unmap(hAllocator, 1);
12335
12336 return res;
12337 }
12338
12339 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
12340 {
12341 if(count == 0)
12342 {
12343 return VK_SUCCESS;
12344 }
12345
12346 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
12347 if(m_MapCount != 0)
12348 {
12349 m_MapCount += count;
12350 VMA_ASSERT(m_pMappedData != VMA_NULL);
12351 if(ppData != VMA_NULL)
12352 {
12353 *ppData = m_pMappedData;
12354 }
12355 return VK_SUCCESS;
12356 }
12357 else
12358 {
12359 VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
12360 hAllocator->m_hDevice,
12361 m_hMemory,
12362 0, // offset
12363 VK_WHOLE_SIZE,
12364 0, // flags
12365 &m_pMappedData);
12366 if(result == VK_SUCCESS)
12367 {
12368 if(ppData != VMA_NULL)
12369 {
12370 *ppData = m_pMappedData;
12371 }
12372 m_MapCount = count;
12373 }
12374 return result;
12375 }
12376 }
12377
12378 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
12379 {
12380 if(count == 0)
12381 {
12382 return;
12383 }
12384
12385 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
12386 if(m_MapCount >= count)
12387 {
12388 m_MapCount -= count;
12389 if(m_MapCount == 0)
12390 {
12391 m_pMappedData = VMA_NULL;
12392 (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
12393 }
12394 }
12395 else
12396 {
12397 VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
12398 }
12399 }
12400
12401 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
12402 {
12403 VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
12404 VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
12405
12406 void* pData;
12407 VkResult res = Map(hAllocator, 1, &pData);
12408 if(res != VK_SUCCESS)
12409 {
12410 return res;
12411 }
12412
12413 VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
12414 VmaWriteMagicValue(pData, allocOffset + allocSize);
12415
12416 Unmap(hAllocator, 1);
12417
12418 return VK_SUCCESS;
12419 }
12420
12421 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
12422 {
12423 VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
12424 VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
12425
12426 void* pData;
12427 VkResult res = Map(hAllocator, 1, &pData);
12428 if(res != VK_SUCCESS)
12429 {
12430 return res;
12431 }
12432
12433 if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
12434 {
12435 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
12436 }
12437 else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
12438 {
12439 VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
12440 }
12441
12442 Unmap(hAllocator, 1);
12443
12444 return VK_SUCCESS;
12445 }
12446
12447 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
12448 const VmaAllocator hAllocator,
12449 const VmaAllocation hAllocation,
12450 VkDeviceSize allocationLocalOffset,
12451 VkBuffer hBuffer,
12452 const void* pNext)
12453 {
12454 VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
12455 hAllocation->GetBlock() == this);
12456 VMA_ASSERT(allocationLocalOffset < hAllocation->GetSize() &&
12457 "Invalid allocationLocalOffset. Did you forget that this offset is relative to the beginning of the allocation, not the whole memory block?");
12458 const VkDeviceSize memoryOffset = hAllocation->GetOffset() + allocationLocalOffset;
12459 // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
12460 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
12461 return hAllocator->BindVulkanBuffer(m_hMemory, memoryOffset, hBuffer, pNext);
12462 }
12463
12464 VkResult VmaDeviceMemoryBlock::BindImageMemory(
12465 const VmaAllocator hAllocator,
12466 const VmaAllocation hAllocation,
12467 VkDeviceSize allocationLocalOffset,
12468 VkImage hImage,
12469 const void* pNext)
12470 {
12471 VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
12472 hAllocation->GetBlock() == this);
12473 VMA_ASSERT(allocationLocalOffset < hAllocation->GetSize() &&
12474 "Invalid allocationLocalOffset. Did you forget that this offset is relative to the beginning of the allocation, not the whole memory block?");
12475 const VkDeviceSize memoryOffset = hAllocation->GetOffset() + allocationLocalOffset;
12476 // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
12477 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
12478 return hAllocator->BindVulkanImage(m_hMemory, memoryOffset, hImage, pNext);
12479 }
12480
12481 static void InitStatInfo(VmaStatInfo& outInfo)
12482 {
12483 memset(&outInfo, 0, sizeof(outInfo));
12484 outInfo.allocationSizeMin = UINT64_MAX;
12485 outInfo.unusedRangeSizeMin = UINT64_MAX;
12486 }
12487
12488 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
12489 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
12490 {
12491 inoutInfo.blockCount += srcInfo.blockCount;
12492 inoutInfo.allocationCount += srcInfo.allocationCount;
12493 inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
12494 inoutInfo.usedBytes += srcInfo.usedBytes;
12495 inoutInfo.unusedBytes += srcInfo.unusedBytes;
12496 inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
12497 inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
12498 inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
12499 inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
12500 }
12501
12502 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
12503 {
12504 inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
12505 VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
12506 inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
12507 VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
12508 }
12509
12510 VmaPool_T::VmaPool_T(
12511 VmaAllocator hAllocator,
12512 const VmaPoolCreateInfo& createInfo,
12513 VkDeviceSize preferredBlockSize) :
12514 m_BlockVector(
12515 hAllocator,
12516 this, // hParentPool
12517 createInfo.memoryTypeIndex,
12518 createInfo.blockSize != 0 ? createInfo.blockSize : preferredBlockSize,
12519 createInfo.minBlockCount,
12520 createInfo.maxBlockCount,
12521 (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
12522 createInfo.frameInUseCount,
12523 createInfo.blockSize != 0, // explicitBlockSize
12524 createInfo.flags & VMA_POOL_CREATE_ALGORITHM_MASK), // algorithm
12525 m_Id(0),
12526 m_Name(VMA_NULL)
12527 {
12528 }
12529
12530 VmaPool_T::~VmaPool_T()
12531 {
12532 }
12533
12534 void VmaPool_T::SetName(const char* pName)
12535 {
12536 const VkAllocationCallbacks* allocs = m_BlockVector.GetAllocator()->GetAllocationCallbacks();
12537 VmaFreeString(allocs, m_Name);
12538
12539 if(pName != VMA_NULL)
12540 {
12541 m_Name = VmaCreateStringCopy(allocs, pName);
12542 }
12543 else
12544 {
12545 m_Name = VMA_NULL;
12546 }
12547 }
12548
12549 #if VMA_STATS_STRING_ENABLED
12550
12551 #endif // #if VMA_STATS_STRING_ENABLED
12552
12553 VmaBlockVector::VmaBlockVector(
12554 VmaAllocator hAllocator,
12555 VmaPool hParentPool,
12556 uint32_t memoryTypeIndex,
12557 VkDeviceSize preferredBlockSize,
12558 size_t minBlockCount,
12559 size_t maxBlockCount,
12560 VkDeviceSize bufferImageGranularity,
12561 uint32_t frameInUseCount,
12562 bool explicitBlockSize,
12563 uint32_t algorithm) :
12564 m_hAllocator(hAllocator),
12565 m_hParentPool(hParentPool),
12566 m_MemoryTypeIndex(memoryTypeIndex),
12567 m_PreferredBlockSize(preferredBlockSize),
12568 m_MinBlockCount(minBlockCount),
12569 m_MaxBlockCount(maxBlockCount),
12570 m_BufferImageGranularity(bufferImageGranularity),
12571 m_FrameInUseCount(frameInUseCount),
12572 m_ExplicitBlockSize(explicitBlockSize),
12573 m_Algorithm(algorithm),
12574 m_HasEmptyBlock(false),
12575 m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
12576 m_NextBlockId(0)
12577 {
12578 }
12579
12580 VmaBlockVector::~VmaBlockVector()
12581 {
12582 for(size_t i = m_Blocks.size(); i--; )
12583 {
12584 m_Blocks[i]->Destroy(m_hAllocator);
12585 vma_delete(m_hAllocator, m_Blocks[i]);
12586 }
12587 }
12588
12589 VkResult VmaBlockVector::CreateMinBlocks()
12590 {
12591 for(size_t i = 0; i < m_MinBlockCount; ++i)
12592 {
12593 VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
12594 if(res != VK_SUCCESS)
12595 {
12596 return res;
12597 }
12598 }
12599 return VK_SUCCESS;
12600 }
12601
12602 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
12603 {
12604 VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex);
12605
12606 const size_t blockCount = m_Blocks.size();
12607
12608 pStats->size = 0;
12609 pStats->unusedSize = 0;
12610 pStats->allocationCount = 0;
12611 pStats->unusedRangeCount = 0;
12612 pStats->unusedRangeSizeMax = 0;
12613 pStats->blockCount = blockCount;
12614
12615 for(uint32_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
12616 {
12617 const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
12618 VMA_ASSERT(pBlock);
12619 VMA_HEAVY_ASSERT(pBlock->Validate());
12620 pBlock->m_pMetadata->AddPoolStats(*pStats);
12621 }
12622 }
12623
12624 bool VmaBlockVector::IsEmpty()
12625 {
12626 VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex);
12627 return m_Blocks.empty();
12628 }
12629
12630 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
12631 {
12632 const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
12633 return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
12634 (VMA_DEBUG_MARGIN > 0) &&
12635 (m_Algorithm == 0 || m_Algorithm == VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) &&
12636 (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
12637 }
12638
12639 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
12640
12641 VkResult VmaBlockVector::Allocate(
12642 uint32_t currentFrameIndex,
12643 VkDeviceSize size,
12644 VkDeviceSize alignment,
12645 const VmaAllocationCreateInfo& createInfo,
12646 VmaSuballocationType suballocType,
12647 size_t allocationCount,
12648 VmaAllocation* pAllocations)
12649 {
12650 size_t allocIndex;
12651 VkResult res = VK_SUCCESS;
12652
12653 if(IsCorruptionDetectionEnabled())
12654 {
12655 size = VmaAlignUp<VkDeviceSize>(size, sizeof(VMA_CORRUPTION_DETECTION_MAGIC_VALUE));
12656 alignment = VmaAlignUp<VkDeviceSize>(alignment, sizeof(VMA_CORRUPTION_DETECTION_MAGIC_VALUE));
12657 }
12658
12659 {
12660 VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex);
12661 for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
12662 {
12663 res = AllocatePage(
12664 currentFrameIndex,
12665 size,
12666 alignment,
12667 createInfo,
12668 suballocType,
12669 pAllocations + allocIndex);
12670 if(res != VK_SUCCESS)
12671 {
12672 break;
12673 }
12674 }
12675 }
12676
12677 if(res != VK_SUCCESS)
12678 {
12679 // Free all already created allocations.
12680 while(allocIndex--)
12681 {
12682 Free(pAllocations[allocIndex]);
12683 }
12684 memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount);
12685 }
12686
12687 return res;
12688 }
12689
12690 VkResult VmaBlockVector::AllocatePage(
12691 uint32_t currentFrameIndex,
12692 VkDeviceSize size,
12693 VkDeviceSize alignment,
12694 const VmaAllocationCreateInfo& createInfo,
12695 VmaSuballocationType suballocType,
12696 VmaAllocation* pAllocation)
12697 {
12698 const bool isUpperAddress = (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0;
12699 bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
12700 const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
12701 const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
12702
12703 VkDeviceSize freeMemory;
12704 {
12705 const uint32_t heapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex);
12706 VmaBudget heapBudget = {};
12707 m_hAllocator->GetBudget(&heapBudget, heapIndex, 1);
12708 freeMemory = (heapBudget.usage < heapBudget.budget) ? (heapBudget.budget - heapBudget.usage) : 0;
12709 }
12710
12711 const bool canFallbackToDedicated = !IsCustomPool();
12712 const bool canCreateNewBlock =
12713 ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
12714 (m_Blocks.size() < m_MaxBlockCount) &&
12715 (freeMemory >= size || !canFallbackToDedicated);
12716 uint32_t strategy = createInfo.flags & VMA_ALLOCATION_CREATE_STRATEGY_MASK;
12717
12718 // If linearAlgorithm is used, canMakeOtherLost is available only when used as ring buffer.
12719 // Which in turn is available only when maxBlockCount = 1.
12720 if(m_Algorithm == VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT && m_MaxBlockCount > 1)
12721 {
12722 canMakeOtherLost = false;
12723 }
12724
12725 // Upper address can only be used with linear allocator and within single memory block.
12726 if(isUpperAddress &&
12727 (m_Algorithm != VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT || m_MaxBlockCount > 1))
12728 {
12729 return VK_ERROR_FEATURE_NOT_PRESENT;
12730 }
12731
12732 // Validate strategy.
12733 switch(strategy)
12734 {
12735 case 0:
12736 strategy = VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT;
12737 break;
12738 case VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT:
12739 case VMA_ALLOCATION_CREATE_STRATEGY_WORST_FIT_BIT:
12740 case VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT:
12741 break;
12742 default:
12743 return VK_ERROR_FEATURE_NOT_PRESENT;
12744 }
12745
12746 // Early reject: requested allocation size is larger that maximum block size for this block vector.
12747 if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
12748 {
12749 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
12750 }
12751
12752 /*
12753 Under certain condition, this whole section can be skipped for optimization, so
12754 we move on directly to trying to allocate with canMakeOtherLost. That's the case
12755 e.g. for custom pools with linear algorithm.
12756 */
12757 if(!canMakeOtherLost || canCreateNewBlock)
12758 {
12759 // 1. Search existing allocations. Try to allocate without making other allocations lost.
12760 VmaAllocationCreateFlags allocFlagsCopy = createInfo.flags;
12761 allocFlagsCopy &= ~VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT;
12762
12763 if(m_Algorithm == VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT)
12764 {
12765 // Use only last block.
12766 if(!m_Blocks.empty())
12767 {
12768 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks.back();
12769 VMA_ASSERT(pCurrBlock);
12770 VkResult res = AllocateFromBlock(
12771 pCurrBlock,
12772 currentFrameIndex,
12773 size,
12774 alignment,
12775 allocFlagsCopy,
12776 createInfo.pUserData,
12777 suballocType,
12778 strategy,
12779 pAllocation);
12780 if(res == VK_SUCCESS)
12781 {
12782 VMA_DEBUG_LOG(" Returned from last block #%u", pCurrBlock->GetId());
12783 return VK_SUCCESS;
12784 }
12785 }
12786 }
12787 else
12788 {
12789 if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT)
12790 {
12791 // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
12792 for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
12793 {
12794 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
12795 VMA_ASSERT(pCurrBlock);
12796 VkResult res = AllocateFromBlock(
12797 pCurrBlock,
12798 currentFrameIndex,
12799 size,
12800 alignment,
12801 allocFlagsCopy,
12802 createInfo.pUserData,
12803 suballocType,
12804 strategy,
12805 pAllocation);
12806 if(res == VK_SUCCESS)
12807 {
12808 VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId());
12809 return VK_SUCCESS;
12810 }
12811 }
12812 }
12813 else // WORST_FIT, FIRST_FIT
12814 {
12815 // Backward order in m_Blocks - prefer blocks with largest amount of free space.
12816 for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
12817 {
12818 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
12819 VMA_ASSERT(pCurrBlock);
12820 VkResult res = AllocateFromBlock(
12821 pCurrBlock,
12822 currentFrameIndex,
12823 size,
12824 alignment,
12825 allocFlagsCopy,
12826 createInfo.pUserData,
12827 suballocType,
12828 strategy,
12829 pAllocation);
12830 if(res == VK_SUCCESS)
12831 {
12832 VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId());
12833 return VK_SUCCESS;
12834 }
12835 }
12836 }
12837 }
12838
12839 // 2. Try to create new block.
12840 if(canCreateNewBlock)
12841 {
12842 // Calculate optimal size for new block.
12843 VkDeviceSize newBlockSize = m_PreferredBlockSize;
12844 uint32_t newBlockSizeShift = 0;
12845 const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
12846
12847 if(!m_ExplicitBlockSize)
12848 {
12849 // Allocate 1/8, 1/4, 1/2 as first blocks.
12850 const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
12851 for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
12852 {
12853 const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
12854 if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
12855 {
12856 newBlockSize = smallerNewBlockSize;
12857 ++newBlockSizeShift;
12858 }
12859 else
12860 {
12861 break;
12862 }
12863 }
12864 }
12865
12866 size_t newBlockIndex = 0;
12867 VkResult res = (newBlockSize <= freeMemory || !canFallbackToDedicated) ?
12868 CreateBlock(newBlockSize, &newBlockIndex) : VK_ERROR_OUT_OF_DEVICE_MEMORY;
12869 // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
12870 if(!m_ExplicitBlockSize)
12871 {
12872 while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
12873 {
12874 const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
12875 if(smallerNewBlockSize >= size)
12876 {
12877 newBlockSize = smallerNewBlockSize;
12878 ++newBlockSizeShift;
12879 res = (newBlockSize <= freeMemory || !canFallbackToDedicated) ?
12880 CreateBlock(newBlockSize, &newBlockIndex) : VK_ERROR_OUT_OF_DEVICE_MEMORY;
12881 }
12882 else
12883 {
12884 break;
12885 }
12886 }
12887 }
12888
12889 if(res == VK_SUCCESS)
12890 {
12891 VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
12892 VMA_ASSERT(pBlock->m_pMetadata->GetSize() >= size);
12893
12894 res = AllocateFromBlock(
12895 pBlock,
12896 currentFrameIndex,
12897 size,
12898 alignment,
12899 allocFlagsCopy,
12900 createInfo.pUserData,
12901 suballocType,
12902 strategy,
12903 pAllocation);
12904 if(res == VK_SUCCESS)
12905 {
12906 VMA_DEBUG_LOG(" Created new block #%u Size=%llu", pBlock->GetId(), newBlockSize);
12907 return VK_SUCCESS;
12908 }
12909 else
12910 {
12911 // Allocation from new block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
12912 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
12913 }
12914 }
12915 }
12916 }
12917
12918 // 3. Try to allocate from existing blocks with making other allocations lost.
12919 if(canMakeOtherLost)
12920 {
12921 uint32_t tryIndex = 0;
12922 for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
12923 {
12924 VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
12925 VmaAllocationRequest bestRequest = {};
12926 VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
12927
12928 // 1. Search existing allocations.
12929 if(strategy == VMA_ALLOCATION_CREATE_STRATEGY_BEST_FIT_BIT)
12930 {
12931 // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
12932 for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
12933 {
12934 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
12935 VMA_ASSERT(pCurrBlock);
12936 VmaAllocationRequest currRequest = {};
12937 if(pCurrBlock->m_pMetadata->CreateAllocationRequest(
12938 currentFrameIndex,
12939 m_FrameInUseCount,
12940 m_BufferImageGranularity,
12941 size,
12942 alignment,
12943 (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0,
12944 suballocType,
12945 canMakeOtherLost,
12946 strategy,
12947 &currRequest))
12948 {
12949 const VkDeviceSize currRequestCost = currRequest.CalcCost();
12950 if(pBestRequestBlock == VMA_NULL ||
12951 currRequestCost < bestRequestCost)
12952 {
12953 pBestRequestBlock = pCurrBlock;
12954 bestRequest = currRequest;
12955 bestRequestCost = currRequestCost;
12956
12957 if(bestRequestCost == 0)
12958 {
12959 break;
12960 }
12961 }
12962 }
12963 }
12964 }
12965 else // WORST_FIT, FIRST_FIT
12966 {
12967 // Backward order in m_Blocks - prefer blocks with largest amount of free space.
12968 for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
12969 {
12970 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
12971 VMA_ASSERT(pCurrBlock);
12972 VmaAllocationRequest currRequest = {};
12973 if(pCurrBlock->m_pMetadata->CreateAllocationRequest(
12974 currentFrameIndex,
12975 m_FrameInUseCount,
12976 m_BufferImageGranularity,
12977 size,
12978 alignment,
12979 (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0,
12980 suballocType,
12981 canMakeOtherLost,
12982 strategy,
12983 &currRequest))
12984 {
12985 const VkDeviceSize currRequestCost = currRequest.CalcCost();
12986 if(pBestRequestBlock == VMA_NULL ||
12987 currRequestCost < bestRequestCost ||
12988 strategy == VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT)
12989 {
12990 pBestRequestBlock = pCurrBlock;
12991 bestRequest = currRequest;
12992 bestRequestCost = currRequestCost;
12993
12994 if(bestRequestCost == 0 ||
12995 strategy == VMA_ALLOCATION_CREATE_STRATEGY_FIRST_FIT_BIT)
12996 {
12997 break;
12998 }
12999 }
13000 }
13001 }
13002 }
13003
13004 if(pBestRequestBlock != VMA_NULL)
13005 {
13006 if(mapped)
13007 {
13008 VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
13009 if(res != VK_SUCCESS)
13010 {
13011 return res;
13012 }
13013 }
13014
13015 if(pBestRequestBlock->m_pMetadata->MakeRequestedAllocationsLost(
13016 currentFrameIndex,
13017 m_FrameInUseCount,
13018 &bestRequest))
13019 {
13020 // Allocate from this pBlock.
13021 *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(currentFrameIndex, isUserDataString);
13022 pBestRequestBlock->m_pMetadata->Alloc(bestRequest, suballocType, size, *pAllocation);
13023 UpdateHasEmptyBlock();
13024 (*pAllocation)->InitBlockAllocation(
13025 pBestRequestBlock,
13026 bestRequest.offset,
13027 alignment,
13028 size,
13029 m_MemoryTypeIndex,
13030 suballocType,
13031 mapped,
13032 (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
13033 VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
13034 VMA_DEBUG_LOG(" Returned from existing block");
13035 (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
13036 m_hAllocator->m_Budget.AddAllocation(m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex), size);
13037 if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
13038 {
13039 m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
13040 }
13041 if(IsCorruptionDetectionEnabled())
13042 {
13043 VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
13044 VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
13045 }
13046 return VK_SUCCESS;
13047 }
13048 // else: Some allocations must have been touched while we are here. Next try.
13049 }
13050 else
13051 {
13052 // Could not find place in any of the blocks - break outer loop.
13053 break;
13054 }
13055 }
13056 /* Maximum number of tries exceeded - a very unlike event when many other
13057 threads are simultaneously touching allocations making it impossible to make
13058 lost at the same time as we try to allocate. */
13059 if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
13060 {
13061 return VK_ERROR_TOO_MANY_OBJECTS;
13062 }
13063 }
13064
13065 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
13066 }
13067
13068 void VmaBlockVector::Free(
13069 const VmaAllocation hAllocation)
13070 {
13071 VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
13072
13073 bool budgetExceeded = false;
13074 {
13075 const uint32_t heapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex);
13076 VmaBudget heapBudget = {};
13077 m_hAllocator->GetBudget(&heapBudget, heapIndex, 1);
13078 budgetExceeded = heapBudget.usage >= heapBudget.budget;
13079 }
13080
13081 // Scope for lock.
13082 {
13083 VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex);
13084
13085 VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
13086
13087 if(IsCorruptionDetectionEnabled())
13088 {
13089 VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
13090 VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
13091 }
13092
13093 if(hAllocation->IsPersistentMap())
13094 {
13095 pBlock->Unmap(m_hAllocator, 1);
13096 }
13097
13098 pBlock->m_pMetadata->Free(hAllocation);
13099 VMA_HEAVY_ASSERT(pBlock->Validate());
13100
13101 VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", m_MemoryTypeIndex);
13102
13103 const bool canDeleteBlock = m_Blocks.size() > m_MinBlockCount;
13104 // pBlock became empty after this deallocation.
13105 if(pBlock->m_pMetadata->IsEmpty())
13106 {
13107 // Already has empty block. We don't want to have two, so delete this one.
13108 if((m_HasEmptyBlock || budgetExceeded) && canDeleteBlock)
13109 {
13110 pBlockToDelete = pBlock;
13111 Remove(pBlock);
13112 }
13113 // else: We now have an empty block - leave it.
13114 }
13115 // pBlock didn't become empty, but we have another empty block - find and free that one.
13116 // (This is optional, heuristics.)
13117 else if(m_HasEmptyBlock && canDeleteBlock)
13118 {
13119 VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
13120 if(pLastBlock->m_pMetadata->IsEmpty())
13121 {
13122 pBlockToDelete = pLastBlock;
13123 m_Blocks.pop_back();
13124 }
13125 }
13126
13127 UpdateHasEmptyBlock();
13128 IncrementallySortBlocks();
13129 }
13130
13131 // Destruction of a free block. Deferred until this point, outside of mutex
13132 // lock, for performance reason.
13133 if(pBlockToDelete != VMA_NULL)
13134 {
13135 VMA_DEBUG_LOG(" Deleted empty block");
13136 pBlockToDelete->Destroy(m_hAllocator);
13137 vma_delete(m_hAllocator, pBlockToDelete);
13138 }
13139 }
13140
13141 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
13142 {
13143 VkDeviceSize result = 0;
13144 for(size_t i = m_Blocks.size(); i--; )
13145 {
13146 result = VMA_MAX(result, m_Blocks[i]->m_pMetadata->GetSize());
13147 if(result >= m_PreferredBlockSize)
13148 {
13149 break;
13150 }
13151 }
13152 return result;
13153 }
13154
13155 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
13156 {
13157 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
13158 {
13159 if(m_Blocks[blockIndex] == pBlock)
13160 {
13161 VmaVectorRemove(m_Blocks, blockIndex);
13162 return;
13163 }
13164 }
13165 VMA_ASSERT(0);
13166 }
13167
13168 void VmaBlockVector::IncrementallySortBlocks()
13169 {
13170 if(m_Algorithm != VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT)
13171 {
13172 // Bubble sort only until first swap.
13173 for(size_t i = 1; i < m_Blocks.size(); ++i)
13174 {
13175 if(m_Blocks[i - 1]->m_pMetadata->GetSumFreeSize() > m_Blocks[i]->m_pMetadata->GetSumFreeSize())
13176 {
13177 VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
13178 return;
13179 }
13180 }
13181 }
13182 }
13183
13184 VkResult VmaBlockVector::AllocateFromBlock(
13185 VmaDeviceMemoryBlock* pBlock,
13186 uint32_t currentFrameIndex,
13187 VkDeviceSize size,
13188 VkDeviceSize alignment,
13189 VmaAllocationCreateFlags allocFlags,
13190 void* pUserData,
13191 VmaSuballocationType suballocType,
13192 uint32_t strategy,
13193 VmaAllocation* pAllocation)
13194 {
13195 VMA_ASSERT((allocFlags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) == 0);
13196 const bool isUpperAddress = (allocFlags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0;
13197 const bool mapped = (allocFlags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
13198 const bool isUserDataString = (allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
13199
13200 VmaAllocationRequest currRequest = {};
13201 if(pBlock->m_pMetadata->CreateAllocationRequest(
13202 currentFrameIndex,
13203 m_FrameInUseCount,
13204 m_BufferImageGranularity,
13205 size,
13206 alignment,
13207 isUpperAddress,
13208 suballocType,
13209 false, // canMakeOtherLost
13210 strategy,
13211 &currRequest))
13212 {
13213 // Allocate from pCurrBlock.
13214 VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
13215
13216 if(mapped)
13217 {
13218 VkResult res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
13219 if(res != VK_SUCCESS)
13220 {
13221 return res;
13222 }
13223 }
13224
13225 *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(currentFrameIndex, isUserDataString);
13226 pBlock->m_pMetadata->Alloc(currRequest, suballocType, size, *pAllocation);
13227 UpdateHasEmptyBlock();
13228 (*pAllocation)->InitBlockAllocation(
13229 pBlock,
13230 currRequest.offset,
13231 alignment,
13232 size,
13233 m_MemoryTypeIndex,
13234 suballocType,
13235 mapped,
13236 (allocFlags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
13237 VMA_HEAVY_ASSERT(pBlock->Validate());
13238 (*pAllocation)->SetUserData(m_hAllocator, pUserData);
13239 m_hAllocator->m_Budget.AddAllocation(m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex), size);
13240 if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
13241 {
13242 m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
13243 }
13244 if(IsCorruptionDetectionEnabled())
13245 {
13246 VkResult res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
13247 VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
13248 }
13249 return VK_SUCCESS;
13250 }
13251 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
13252 }
13253
13254 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
13255 {
13256 VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
13257 allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
13258 allocInfo.allocationSize = blockSize;
13259
13260 #if VMA_BUFFER_DEVICE_ADDRESS
13261 // Every standalone block can potentially contain a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT - always enable the feature.
13262 VkMemoryAllocateFlagsInfoKHR allocFlagsInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO_KHR };
13263 if(m_hAllocator->m_UseKhrBufferDeviceAddress)
13264 {
13265 allocFlagsInfo.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR;
13266 VmaPnextChainPushFront(&allocInfo, &allocFlagsInfo);
13267 }
13268 #endif // #if VMA_BUFFER_DEVICE_ADDRESS
13269
13270 VkDeviceMemory mem = VK_NULL_HANDLE;
13271 VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
13272 if(res < 0)
13273 {
13274 return res;
13275 }
13276
13277 // New VkDeviceMemory successfully created.
13278
13279 // Create new Allocation for it.
13280 VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
13281 pBlock->Init(
13282 m_hAllocator,
13283 m_hParentPool,
13284 m_MemoryTypeIndex,
13285 mem,
13286 allocInfo.allocationSize,
13287 m_NextBlockId++,
13288 m_Algorithm);
13289
13290 m_Blocks.push_back(pBlock);
13291 if(pNewBlockIndex != VMA_NULL)
13292 {
13293 *pNewBlockIndex = m_Blocks.size() - 1;
13294 }
13295
13296 return VK_SUCCESS;
13297 }
13298
13299 void VmaBlockVector::ApplyDefragmentationMovesCpu(
13300 class VmaBlockVectorDefragmentationContext* pDefragCtx,
13301 const VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves)
13302 {
13303 const size_t blockCount = m_Blocks.size();
13304 const bool isNonCoherent = m_hAllocator->IsMemoryTypeNonCoherent(m_MemoryTypeIndex);
13305
13306 enum BLOCK_FLAG
13307 {
13308 BLOCK_FLAG_USED = 0x00000001,
13309 BLOCK_FLAG_MAPPED_FOR_DEFRAGMENTATION = 0x00000002,
13310 };
13311
13312 struct BlockInfo
13313 {
13314 uint32_t flags;
13315 void* pMappedData;
13316 };
13317 VmaVector< BlockInfo, VmaStlAllocator<BlockInfo> >
13318 blockInfo(blockCount, BlockInfo(), VmaStlAllocator<BlockInfo>(m_hAllocator->GetAllocationCallbacks()));
13319 memset(blockInfo.data(), 0, blockCount * sizeof(BlockInfo));
13320
13321 // Go over all moves. Mark blocks that are used with BLOCK_FLAG_USED.
13322 const size_t moveCount = moves.size();
13323 for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex)
13324 {
13325 const VmaDefragmentationMove& move = moves[moveIndex];
13326 blockInfo[move.srcBlockIndex].flags |= BLOCK_FLAG_USED;
13327 blockInfo[move.dstBlockIndex].flags |= BLOCK_FLAG_USED;
13328 }
13329
13330 VMA_ASSERT(pDefragCtx->res == VK_SUCCESS);
13331
13332 // Go over all blocks. Get mapped pointer or map if necessary.
13333 for(size_t blockIndex = 0; pDefragCtx->res == VK_SUCCESS && blockIndex < blockCount; ++blockIndex)
13334 {
13335 BlockInfo& currBlockInfo = blockInfo[blockIndex];
13336 VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
13337 if((currBlockInfo.flags & BLOCK_FLAG_USED) != 0)
13338 {
13339 currBlockInfo.pMappedData = pBlock->GetMappedData();
13340 // It is not originally mapped - map it.
13341 if(currBlockInfo.pMappedData == VMA_NULL)
13342 {
13343 pDefragCtx->res = pBlock->Map(m_hAllocator, 1, &currBlockInfo.pMappedData);
13344 if(pDefragCtx->res == VK_SUCCESS)
13345 {
13346 currBlockInfo.flags |= BLOCK_FLAG_MAPPED_FOR_DEFRAGMENTATION;
13347 }
13348 }
13349 }
13350 }
13351
13352 // Go over all moves. Do actual data transfer.
13353 if(pDefragCtx->res == VK_SUCCESS)
13354 {
13355 const VkDeviceSize nonCoherentAtomSize = m_hAllocator->m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
13356 VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
13357
13358 for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex)
13359 {
13360 const VmaDefragmentationMove& move = moves[moveIndex];
13361
13362 const BlockInfo& srcBlockInfo = blockInfo[move.srcBlockIndex];
13363 const BlockInfo& dstBlockInfo = blockInfo[move.dstBlockIndex];
13364
13365 VMA_ASSERT(srcBlockInfo.pMappedData && dstBlockInfo.pMappedData);
13366
13367 // Invalidate source.
13368 if(isNonCoherent)
13369 {
13370 VmaDeviceMemoryBlock* const pSrcBlock = m_Blocks[move.srcBlockIndex];
13371 memRange.memory = pSrcBlock->GetDeviceMemory();
13372 memRange.offset = VmaAlignDown(move.srcOffset, nonCoherentAtomSize);
13373 memRange.size = VMA_MIN(
13374 VmaAlignUp(move.size + (move.srcOffset - memRange.offset), nonCoherentAtomSize),
13375 pSrcBlock->m_pMetadata->GetSize() - memRange.offset);
13376 (*m_hAllocator->GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hAllocator->m_hDevice, 1, &memRange);
13377 }
13378
13379 // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
13380 memmove(
13381 reinterpret_cast<char*>(dstBlockInfo.pMappedData) + move.dstOffset,
13382 reinterpret_cast<char*>(srcBlockInfo.pMappedData) + move.srcOffset,
13383 static_cast<size_t>(move.size));
13384
13385 if(IsCorruptionDetectionEnabled())
13386 {
13387 VmaWriteMagicValue(dstBlockInfo.pMappedData, move.dstOffset - VMA_DEBUG_MARGIN);
13388 VmaWriteMagicValue(dstBlockInfo.pMappedData, move.dstOffset + move.size);
13389 }
13390
13391 // Flush destination.
13392 if(isNonCoherent)
13393 {
13394 VmaDeviceMemoryBlock* const pDstBlock = m_Blocks[move.dstBlockIndex];
13395 memRange.memory = pDstBlock->GetDeviceMemory();
13396 memRange.offset = VmaAlignDown(move.dstOffset, nonCoherentAtomSize);
13397 memRange.size = VMA_MIN(
13398 VmaAlignUp(move.size + (move.dstOffset - memRange.offset), nonCoherentAtomSize),
13399 pDstBlock->m_pMetadata->GetSize() - memRange.offset);
13400 (*m_hAllocator->GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hAllocator->m_hDevice, 1, &memRange);
13401 }
13402 }
13403 }
13404
13405 // Go over all blocks in reverse order. Unmap those that were mapped just for defragmentation.
13406 // Regardless of pCtx->res == VK_SUCCESS.
13407 for(size_t blockIndex = blockCount; blockIndex--; )
13408 {
13409 const BlockInfo& currBlockInfo = blockInfo[blockIndex];
13410 if((currBlockInfo.flags & BLOCK_FLAG_MAPPED_FOR_DEFRAGMENTATION) != 0)
13411 {
13412 VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
13413 pBlock->Unmap(m_hAllocator, 1);
13414 }
13415 }
13416 }
13417
13418 void VmaBlockVector::ApplyDefragmentationMovesGpu(
13419 class VmaBlockVectorDefragmentationContext* pDefragCtx,
13420 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
13421 VkCommandBuffer commandBuffer)
13422 {
13423 const size_t blockCount = m_Blocks.size();
13424
13425 pDefragCtx->blockContexts.resize(blockCount);
13426 memset(pDefragCtx->blockContexts.data(), 0, blockCount * sizeof(VmaBlockDefragmentationContext));
13427
13428 // Go over all moves. Mark blocks that are used with BLOCK_FLAG_USED.
13429 const size_t moveCount = moves.size();
13430 for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex)
13431 {
13432 const VmaDefragmentationMove& move = moves[moveIndex];
13433
13434 //if(move.type == VMA_ALLOCATION_TYPE_UNKNOWN)
13435 {
13436 // Old school move still require us to map the whole block
13437 pDefragCtx->blockContexts[move.srcBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED;
13438 pDefragCtx->blockContexts[move.dstBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED;
13439 }
13440 }
13441
13442 VMA_ASSERT(pDefragCtx->res == VK_SUCCESS);
13443
13444 // Go over all blocks. Create and bind buffer for whole block if necessary.
13445 {
13446 VkBufferCreateInfo bufCreateInfo;
13447 VmaFillGpuDefragmentationBufferCreateInfo(bufCreateInfo);
13448
13449 for(size_t blockIndex = 0; pDefragCtx->res == VK_SUCCESS && blockIndex < blockCount; ++blockIndex)
13450 {
13451 VmaBlockDefragmentationContext& currBlockCtx = pDefragCtx->blockContexts[blockIndex];
13452 VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
13453 if((currBlockCtx.flags & VmaBlockDefragmentationContext::BLOCK_FLAG_USED) != 0)
13454 {
13455 bufCreateInfo.size = pBlock->m_pMetadata->GetSize();
13456 pDefragCtx->res = (*m_hAllocator->GetVulkanFunctions().vkCreateBuffer)(
13457 m_hAllocator->m_hDevice, &bufCreateInfo, m_hAllocator->GetAllocationCallbacks(), &currBlockCtx.hBuffer);
13458 if(pDefragCtx->res == VK_SUCCESS)
13459 {
13460 pDefragCtx->res = (*m_hAllocator->GetVulkanFunctions().vkBindBufferMemory)(
13461 m_hAllocator->m_hDevice, currBlockCtx.hBuffer, pBlock->GetDeviceMemory(), 0);
13462 }
13463 }
13464 }
13465 }
13466
13467 // Go over all moves. Post data transfer commands to command buffer.
13468 if(pDefragCtx->res == VK_SUCCESS)
13469 {
13470 for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex)
13471 {
13472 const VmaDefragmentationMove& move = moves[moveIndex];
13473
13474 const VmaBlockDefragmentationContext& srcBlockCtx = pDefragCtx->blockContexts[move.srcBlockIndex];
13475 const VmaBlockDefragmentationContext& dstBlockCtx = pDefragCtx->blockContexts[move.dstBlockIndex];
13476
13477 VMA_ASSERT(srcBlockCtx.hBuffer && dstBlockCtx.hBuffer);
13478
13479 VkBufferCopy region = {
13480 move.srcOffset,
13481 move.dstOffset,
13482 move.size };
13483 (*m_hAllocator->GetVulkanFunctions().vkCmdCopyBuffer)(
13484 commandBuffer, srcBlockCtx.hBuffer, dstBlockCtx.hBuffer, 1, &region);
13485 }
13486 }
13487
13488 // Save buffers to defrag context for later destruction.
13489 if(pDefragCtx->res == VK_SUCCESS && moveCount > 0)
13490 {
13491 pDefragCtx->res = VK_NOT_READY;
13492 }
13493 }
13494
13495 void VmaBlockVector::FreeEmptyBlocks(VmaDefragmentationStats* pDefragmentationStats)
13496 {
13497 for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
13498 {
13499 VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
13500 if(pBlock->m_pMetadata->IsEmpty())
13501 {
13502 if(m_Blocks.size() > m_MinBlockCount)
13503 {
13504 if(pDefragmentationStats != VMA_NULL)
13505 {
13506 ++pDefragmentationStats->deviceMemoryBlocksFreed;
13507 pDefragmentationStats->bytesFreed += pBlock->m_pMetadata->GetSize();
13508 }
13509
13510 VmaVectorRemove(m_Blocks, blockIndex);
13511 pBlock->Destroy(m_hAllocator);
13512 vma_delete(m_hAllocator, pBlock);
13513 }
13514 else
13515 {
13516 break;
13517 }
13518 }
13519 }
13520 UpdateHasEmptyBlock();
13521 }
13522
13523 void VmaBlockVector::UpdateHasEmptyBlock()
13524 {
13525 m_HasEmptyBlock = false;
13526 for(size_t index = 0, count = m_Blocks.size(); index < count; ++index)
13527 {
13528 VmaDeviceMemoryBlock* const pBlock = m_Blocks[index];
13529 if(pBlock->m_pMetadata->IsEmpty())
13530 {
13531 m_HasEmptyBlock = true;
13532 break;
13533 }
13534 }
13535 }
13536
13537 #if VMA_STATS_STRING_ENABLED
13538
13539 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
13540 {
13541 VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex);
13542
13543 json.BeginObject();
13544
13545 if(IsCustomPool())
13546 {
13547 const char* poolName = m_hParentPool->GetName();
13548 if(poolName != VMA_NULL && poolName[0] != '\0')
13549 {
13550 json.WriteString("Name");
13551 json.WriteString(poolName);
13552 }
13553
13554 json.WriteString("MemoryTypeIndex");
13555 json.WriteNumber(m_MemoryTypeIndex);
13556
13557 json.WriteString("BlockSize");
13558 json.WriteNumber(m_PreferredBlockSize);
13559
13560 json.WriteString("BlockCount");
13561 json.BeginObject(true);
13562 if(m_MinBlockCount > 0)
13563 {
13564 json.WriteString("Min");
13565 json.WriteNumber((uint64_t)m_MinBlockCount);
13566 }
13567 if(m_MaxBlockCount < SIZE_MAX)
13568 {
13569 json.WriteString("Max");
13570 json.WriteNumber((uint64_t)m_MaxBlockCount);
13571 }
13572 json.WriteString("Cur");
13573 json.WriteNumber((uint64_t)m_Blocks.size());
13574 json.EndObject();
13575
13576 if(m_FrameInUseCount > 0)
13577 {
13578 json.WriteString("FrameInUseCount");
13579 json.WriteNumber(m_FrameInUseCount);
13580 }
13581
13582 if(m_Algorithm != 0)
13583 {
13584 json.WriteString("Algorithm");
13585 json.WriteString(VmaAlgorithmToStr(m_Algorithm));
13586 }
13587 }
13588 else
13589 {
13590 json.WriteString("PreferredBlockSize");
13591 json.WriteNumber(m_PreferredBlockSize);
13592 }
13593
13594 json.WriteString("Blocks");
13595 json.BeginObject();
13596 for(size_t i = 0; i < m_Blocks.size(); ++i)
13597 {
13598 json.BeginString();
13599 json.ContinueString(m_Blocks[i]->GetId());
13600 json.EndString();
13601
13602 m_Blocks[i]->m_pMetadata->PrintDetailedMap(json);
13603 }
13604 json.EndObject();
13605
13606 json.EndObject();
13607 }
13608
13609 #endif // #if VMA_STATS_STRING_ENABLED
13610
13611 void VmaBlockVector::Defragment(
13612 class VmaBlockVectorDefragmentationContext* pCtx,
13613 VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags,
13614 VkDeviceSize& maxCpuBytesToMove, uint32_t& maxCpuAllocationsToMove,
13615 VkDeviceSize& maxGpuBytesToMove, uint32_t& maxGpuAllocationsToMove,
13616 VkCommandBuffer commandBuffer)
13617 {
13618 pCtx->res = VK_SUCCESS;
13619
13620 const VkMemoryPropertyFlags memPropFlags =
13621 m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags;
13622 const bool isHostVisible = (memPropFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;
13623
13624 const bool canDefragmentOnCpu = maxCpuBytesToMove > 0 && maxCpuAllocationsToMove > 0 &&
13625 isHostVisible;
13626 const bool canDefragmentOnGpu = maxGpuBytesToMove > 0 && maxGpuAllocationsToMove > 0 &&
13627 !IsCorruptionDetectionEnabled() &&
13628 ((1u << m_MemoryTypeIndex) & m_hAllocator->GetGpuDefragmentationMemoryTypeBits()) != 0;
13629
13630 // There are options to defragment this memory type.
13631 if(canDefragmentOnCpu || canDefragmentOnGpu)
13632 {
13633 bool defragmentOnGpu;
13634 // There is only one option to defragment this memory type.
13635 if(canDefragmentOnGpu != canDefragmentOnCpu)
13636 {
13637 defragmentOnGpu = canDefragmentOnGpu;
13638 }
13639 // Both options are available: Heuristics to choose the best one.
13640 else
13641 {
13642 defragmentOnGpu = (memPropFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 ||
13643 m_hAllocator->IsIntegratedGpu();
13644 }
13645
13646 bool overlappingMoveSupported = !defragmentOnGpu;
13647
13648 if(m_hAllocator->m_UseMutex)
13649 {
13650 if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)
13651 {
13652 if(!m_Mutex.TryLockWrite())
13653 {
13654 pCtx->res = VK_ERROR_INITIALIZATION_FAILED;
13655 return;
13656 }
13657 }
13658 else
13659 {
13660 m_Mutex.LockWrite();
13661 pCtx->mutexLocked = true;
13662 }
13663 }
13664
13665 pCtx->Begin(overlappingMoveSupported, flags);
13666
13667 // Defragment.
13668
13669 const VkDeviceSize maxBytesToMove = defragmentOnGpu ? maxGpuBytesToMove : maxCpuBytesToMove;
13670 const uint32_t maxAllocationsToMove = defragmentOnGpu ? maxGpuAllocationsToMove : maxCpuAllocationsToMove;
13671 pCtx->res = pCtx->GetAlgorithm()->Defragment(pCtx->defragmentationMoves, maxBytesToMove, maxAllocationsToMove, flags);
13672
13673 // Accumulate statistics.
13674 if(pStats != VMA_NULL)
13675 {
13676 const VkDeviceSize bytesMoved = pCtx->GetAlgorithm()->GetBytesMoved();
13677 const uint32_t allocationsMoved = pCtx->GetAlgorithm()->GetAllocationsMoved();
13678 pStats->bytesMoved += bytesMoved;
13679 pStats->allocationsMoved += allocationsMoved;
13680 VMA_ASSERT(bytesMoved <= maxBytesToMove);
13681 VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
13682 if(defragmentOnGpu)
13683 {
13684 maxGpuBytesToMove -= bytesMoved;
13685 maxGpuAllocationsToMove -= allocationsMoved;
13686 }
13687 else
13688 {
13689 maxCpuBytesToMove -= bytesMoved;
13690 maxCpuAllocationsToMove -= allocationsMoved;
13691 }
13692 }
13693
13694 if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)
13695 {
13696 if(m_hAllocator->m_UseMutex)
13697 m_Mutex.UnlockWrite();
13698
13699 if(pCtx->res >= VK_SUCCESS && !pCtx->defragmentationMoves.empty())
13700 pCtx->res = VK_NOT_READY;
13701
13702 return;
13703 }
13704
13705 if(pCtx->res >= VK_SUCCESS)
13706 {
13707 if(defragmentOnGpu)
13708 {
13709 ApplyDefragmentationMovesGpu(pCtx, pCtx->defragmentationMoves, commandBuffer);
13710 }
13711 else
13712 {
13713 ApplyDefragmentationMovesCpu(pCtx, pCtx->defragmentationMoves);
13714 }
13715 }
13716 }
13717 }
13718
13719 void VmaBlockVector::DefragmentationEnd(
13720 class VmaBlockVectorDefragmentationContext* pCtx,
13721 uint32_t flags,
13722 VmaDefragmentationStats* pStats)
13723 {
13724 if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL && m_hAllocator->m_UseMutex)
13725 {
13726 VMA_ASSERT(pCtx->mutexLocked == false);
13727
13728 // Incremental defragmentation doesn't hold the lock, so when we enter here we don't actually have any
13729 // lock protecting us. Since we mutate state here, we have to take the lock out now
13730 m_Mutex.LockWrite();
13731 pCtx->mutexLocked = true;
13732 }
13733
13734 // If the mutex isn't locked we didn't do any work and there is nothing to delete.
13735 if(pCtx->mutexLocked || !m_hAllocator->m_UseMutex)
13736 {
13737 // Destroy buffers.
13738 for(size_t blockIndex = pCtx->blockContexts.size(); blockIndex--;)
13739 {
13740 VmaBlockDefragmentationContext &blockCtx = pCtx->blockContexts[blockIndex];
13741 if(blockCtx.hBuffer)
13742 {
13743 (*m_hAllocator->GetVulkanFunctions().vkDestroyBuffer)(m_hAllocator->m_hDevice, blockCtx.hBuffer, m_hAllocator->GetAllocationCallbacks());
13744 }
13745 }
13746
13747 if(pCtx->res >= VK_SUCCESS)
13748 {
13749 FreeEmptyBlocks(pStats);
13750 }
13751 }
13752
13753 if(pCtx->mutexLocked)
13754 {
13755 VMA_ASSERT(m_hAllocator->m_UseMutex);
13756 m_Mutex.UnlockWrite();
13757 }
13758 }
13759
13760 uint32_t VmaBlockVector::ProcessDefragmentations(
13761 class VmaBlockVectorDefragmentationContext *pCtx,
13762 VmaDefragmentationPassMoveInfo* pMove, uint32_t maxMoves)
13763 {
13764 VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex);
13765
13766 const uint32_t moveCount = std::min(uint32_t(pCtx->defragmentationMoves.size()) - pCtx->defragmentationMovesProcessed, maxMoves);
13767
13768 for(uint32_t i = 0; i < moveCount; ++ i)
13769 {
13770 VmaDefragmentationMove& move = pCtx->defragmentationMoves[pCtx->defragmentationMovesProcessed + i];
13771
13772 pMove->allocation = move.hAllocation;
13773 pMove->memory = move.pDstBlock->GetDeviceMemory();
13774 pMove->offset = move.dstOffset;
13775
13776 ++ pMove;
13777 }
13778
13779 pCtx->defragmentationMovesProcessed += moveCount;
13780
13781 return moveCount;
13782 }
13783
13784 void VmaBlockVector::CommitDefragmentations(
13785 class VmaBlockVectorDefragmentationContext *pCtx,
13786 VmaDefragmentationStats* pStats)
13787 {
13788 VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex);
13789
13790 for(uint32_t i = pCtx->defragmentationMovesCommitted; i < pCtx->defragmentationMovesProcessed; ++ i)
13791 {
13792 const VmaDefragmentationMove &move = pCtx->defragmentationMoves[i];
13793
13794 move.pSrcBlock->m_pMetadata->FreeAtOffset(move.srcOffset);
13795 move.hAllocation->ChangeBlockAllocation(m_hAllocator, move.pDstBlock, move.dstOffset);
13796 }
13797
13798 pCtx->defragmentationMovesCommitted = pCtx->defragmentationMovesProcessed;
13799 FreeEmptyBlocks(pStats);
13800 }
13801
13802 size_t VmaBlockVector::CalcAllocationCount() const
13803 {
13804 size_t result = 0;
13805 for(size_t i = 0; i < m_Blocks.size(); ++i)
13806 {
13807 result += m_Blocks[i]->m_pMetadata->GetAllocationCount();
13808 }
13809 return result;
13810 }
13811
13812 bool VmaBlockVector::IsBufferImageGranularityConflictPossible() const
13813 {
13814 if(m_BufferImageGranularity == 1)
13815 {
13816 return false;
13817 }
13818 VmaSuballocationType lastSuballocType = VMA_SUBALLOCATION_TYPE_FREE;
13819 for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
13820 {
13821 VmaDeviceMemoryBlock* const pBlock = m_Blocks[i];
13822 VMA_ASSERT(m_Algorithm == 0);
13823 VmaBlockMetadata_Generic* const pMetadata = (VmaBlockMetadata_Generic*)pBlock->m_pMetadata;
13824 if(pMetadata->IsBufferImageGranularityConflictPossible(m_BufferImageGranularity, lastSuballocType))
13825 {
13826 return true;
13827 }
13828 }
13829 return false;
13830 }
13831
13832 void VmaBlockVector::MakePoolAllocationsLost(
13833 uint32_t currentFrameIndex,
13834 size_t* pLostAllocationCount)
13835 {
13836 VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex);
13837 size_t lostAllocationCount = 0;
13838 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
13839 {
13840 VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
13841 VMA_ASSERT(pBlock);
13842 lostAllocationCount += pBlock->m_pMetadata->MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
13843 }
13844 if(pLostAllocationCount != VMA_NULL)
13845 {
13846 *pLostAllocationCount = lostAllocationCount;
13847 }
13848 }
13849
13850 VkResult VmaBlockVector::CheckCorruption()
13851 {
13852 if(!IsCorruptionDetectionEnabled())
13853 {
13854 return VK_ERROR_FEATURE_NOT_PRESENT;
13855 }
13856
13857 VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex);
13858 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
13859 {
13860 VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
13861 VMA_ASSERT(pBlock);
13862 VkResult res = pBlock->CheckCorruption(m_hAllocator);
13863 if(res != VK_SUCCESS)
13864 {
13865 return res;
13866 }
13867 }
13868 return VK_SUCCESS;
13869 }
13870
13871 void VmaBlockVector::AddStats(VmaStats* pStats)
13872 {
13873 const uint32_t memTypeIndex = m_MemoryTypeIndex;
13874 const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
13875
13876 VmaMutexLockRead lock(m_Mutex, m_hAllocator->m_UseMutex);
13877
13878 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
13879 {
13880 const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
13881 VMA_ASSERT(pBlock);
13882 VMA_HEAVY_ASSERT(pBlock->Validate());
13883 VmaStatInfo allocationStatInfo;
13884 pBlock->m_pMetadata->CalcAllocationStatInfo(allocationStatInfo);
13885 VmaAddStatInfo(pStats->total, allocationStatInfo);
13886 VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
13887 VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
13888 }
13889 }
13890
13891 ////////////////////////////////////////////////////////////////////////////////
13892 // VmaDefragmentationAlgorithm_Generic members definition
13893
13894 VmaDefragmentationAlgorithm_Generic::VmaDefragmentationAlgorithm_Generic(
13895 VmaAllocator hAllocator,
13896 VmaBlockVector* pBlockVector,
13897 uint32_t currentFrameIndex,
13898 bool overlappingMoveSupported) :
13899 VmaDefragmentationAlgorithm(hAllocator, pBlockVector, currentFrameIndex),
13900 m_AllocationCount(0),
13901 m_AllAllocations(false),
13902 m_BytesMoved(0),
13903 m_AllocationsMoved(0),
13904 m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
13905 {
13906 // Create block info for each block.
13907 const size_t blockCount = m_pBlockVector->m_Blocks.size();
13908 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
13909 {
13910 BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
13911 pBlockInfo->m_OriginalBlockIndex = blockIndex;
13912 pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
13913 m_Blocks.push_back(pBlockInfo);
13914 }
13915
13916 // Sort them by m_pBlock pointer value.
13917 VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
13918 }
13919
13920 VmaDefragmentationAlgorithm_Generic::~VmaDefragmentationAlgorithm_Generic()
13921 {
13922 for(size_t i = m_Blocks.size(); i--; )
13923 {
13924 vma_delete(m_hAllocator, m_Blocks[i]);
13925 }
13926 }
13927
13928 void VmaDefragmentationAlgorithm_Generic::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
13929 {
13930 // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
13931 if(hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
13932 {
13933 VmaDeviceMemoryBlock* pBlock = hAlloc->GetBlock();
13934 BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
13935 if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
13936 {
13937 AllocationInfo allocInfo = AllocationInfo(hAlloc, pChanged);
13938 (*it)->m_Allocations.push_back(allocInfo);
13939 }
13940 else
13941 {
13942 VMA_ASSERT(0);
13943 }
13944
13945 ++m_AllocationCount;
13946 }
13947 }
13948
13949 VkResult VmaDefragmentationAlgorithm_Generic::DefragmentRound(
13950 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
13951 VkDeviceSize maxBytesToMove,
13952 uint32_t maxAllocationsToMove,
13953 bool freeOldAllocations)
13954 {
13955 if(m_Blocks.empty())
13956 {
13957 return VK_SUCCESS;
13958 }
13959
13960 // This is a choice based on research.
13961 // Option 1:
13962 uint32_t strategy = VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT;
13963 // Option 2:
13964 //uint32_t strategy = VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT;
13965 // Option 3:
13966 //uint32_t strategy = VMA_ALLOCATION_CREATE_STRATEGY_MIN_FRAGMENTATION_BIT;
13967
13968 size_t srcBlockMinIndex = 0;
13969 // When FAST_ALGORITHM, move allocations from only last out of blocks that contain non-movable allocations.
13970 /*
13971 if(m_AlgorithmFlags & VMA_DEFRAGMENTATION_FAST_ALGORITHM_BIT)
13972 {
13973 const size_t blocksWithNonMovableCount = CalcBlocksWithNonMovableCount();
13974 if(blocksWithNonMovableCount > 0)
13975 {
13976 srcBlockMinIndex = blocksWithNonMovableCount - 1;
13977 }
13978 }
13979 */
13980
13981 size_t srcBlockIndex = m_Blocks.size() - 1;
13982 size_t srcAllocIndex = SIZE_MAX;
13983 for(;;)
13984 {
13985 // 1. Find next allocation to move.
13986 // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
13987 // 1.2. Then start from last to first m_Allocations.
13988 while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
13989 {
13990 if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
13991 {
13992 // Finished: no more allocations to process.
13993 if(srcBlockIndex == srcBlockMinIndex)
13994 {
13995 return VK_SUCCESS;
13996 }
13997 else
13998 {
13999 --srcBlockIndex;
14000 srcAllocIndex = SIZE_MAX;
14001 }
14002 }
14003 else
14004 {
14005 srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
14006 }
14007 }
14008
14009 BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
14010 AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
14011
14012 const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
14013 const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
14014 const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
14015 const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
14016
14017 // 2. Try to find new place for this allocation in preceding or current block.
14018 for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
14019 {
14020 BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
14021 VmaAllocationRequest dstAllocRequest;
14022 if(pDstBlockInfo->m_pBlock->m_pMetadata->CreateAllocationRequest(
14023 m_CurrentFrameIndex,
14024 m_pBlockVector->GetFrameInUseCount(),
14025 m_pBlockVector->GetBufferImageGranularity(),
14026 size,
14027 alignment,
14028 false, // upperAddress
14029 suballocType,
14030 false, // canMakeOtherLost
14031 strategy,
14032 &dstAllocRequest) &&
14033 MoveMakesSense(
14034 dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
14035 {
14036 VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
14037
14038 // Reached limit on number of allocations or bytes to move.
14039 if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
14040 (m_BytesMoved + size > maxBytesToMove))
14041 {
14042 return VK_SUCCESS;
14043 }
14044
14045 VmaDefragmentationMove move = {};
14046 move.srcBlockIndex = pSrcBlockInfo->m_OriginalBlockIndex;
14047 move.dstBlockIndex = pDstBlockInfo->m_OriginalBlockIndex;
14048 move.srcOffset = srcOffset;
14049 move.dstOffset = dstAllocRequest.offset;
14050 move.size = size;
14051 move.hAllocation = allocInfo.m_hAllocation;
14052 move.pSrcBlock = pSrcBlockInfo->m_pBlock;
14053 move.pDstBlock = pDstBlockInfo->m_pBlock;
14054
14055 moves.push_back(move);
14056
14057 pDstBlockInfo->m_pBlock->m_pMetadata->Alloc(
14058 dstAllocRequest,
14059 suballocType,
14060 size,
14061 allocInfo.m_hAllocation);
14062
14063 if(freeOldAllocations)
14064 {
14065 pSrcBlockInfo->m_pBlock->m_pMetadata->FreeAtOffset(srcOffset);
14066 allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
14067 }
14068
14069 if(allocInfo.m_pChanged != VMA_NULL)
14070 {
14071 *allocInfo.m_pChanged = VK_TRUE;
14072 }
14073
14074 ++m_AllocationsMoved;
14075 m_BytesMoved += size;
14076
14077 VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
14078
14079 break;
14080 }
14081 }
14082
14083 // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
14084
14085 if(srcAllocIndex > 0)
14086 {
14087 --srcAllocIndex;
14088 }
14089 else
14090 {
14091 if(srcBlockIndex > 0)
14092 {
14093 --srcBlockIndex;
14094 srcAllocIndex = SIZE_MAX;
14095 }
14096 else
14097 {
14098 return VK_SUCCESS;
14099 }
14100 }
14101 }
14102 }
14103
14104 size_t VmaDefragmentationAlgorithm_Generic::CalcBlocksWithNonMovableCount() const
14105 {
14106 size_t result = 0;
14107 for(size_t i = 0; i < m_Blocks.size(); ++i)
14108 {
14109 if(m_Blocks[i]->m_HasNonMovableAllocations)
14110 {
14111 ++result;
14112 }
14113 }
14114 return result;
14115 }
14116
14117 VkResult VmaDefragmentationAlgorithm_Generic::Defragment(
14118 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
14119 VkDeviceSize maxBytesToMove,
14120 uint32_t maxAllocationsToMove,
14121 VmaDefragmentationFlags flags)
14122 {
14123 if(!m_AllAllocations && m_AllocationCount == 0)
14124 {
14125 return VK_SUCCESS;
14126 }
14127
14128 const size_t blockCount = m_Blocks.size();
14129 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
14130 {
14131 BlockInfo* pBlockInfo = m_Blocks[blockIndex];
14132
14133 if(m_AllAllocations)
14134 {
14135 VmaBlockMetadata_Generic* pMetadata = (VmaBlockMetadata_Generic*)pBlockInfo->m_pBlock->m_pMetadata;
14136 for(VmaSuballocationList::const_iterator it = pMetadata->m_Suballocations.begin();
14137 it != pMetadata->m_Suballocations.end();
14138 ++it)
14139 {
14140 if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
14141 {
14142 AllocationInfo allocInfo = AllocationInfo(it->hAllocation, VMA_NULL);
14143 pBlockInfo->m_Allocations.push_back(allocInfo);
14144 }
14145 }
14146 }
14147
14148 pBlockInfo->CalcHasNonMovableAllocations();
14149
14150 // This is a choice based on research.
14151 // Option 1:
14152 pBlockInfo->SortAllocationsByOffsetDescending();
14153 // Option 2:
14154 //pBlockInfo->SortAllocationsBySizeDescending();
14155 }
14156
14157 // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
14158 VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
14159
14160 // This is a choice based on research.
14161 const uint32_t roundCount = 2;
14162
14163 // Execute defragmentation rounds (the main part).
14164 VkResult result = VK_SUCCESS;
14165 for(uint32_t round = 0; (round < roundCount) && (result == VK_SUCCESS); ++round)
14166 {
14167 result = DefragmentRound(moves, maxBytesToMove, maxAllocationsToMove, !(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL));
14168 }
14169
14170 return result;
14171 }
14172
14173 bool VmaDefragmentationAlgorithm_Generic::MoveMakesSense(
14174 size_t dstBlockIndex, VkDeviceSize dstOffset,
14175 size_t srcBlockIndex, VkDeviceSize srcOffset)
14176 {
14177 if(dstBlockIndex < srcBlockIndex)
14178 {
14179 return true;
14180 }
14181 if(dstBlockIndex > srcBlockIndex)
14182 {
14183 return false;
14184 }
14185 if(dstOffset < srcOffset)
14186 {
14187 return true;
14188 }
14189 return false;
14190 }
14191
14192 ////////////////////////////////////////////////////////////////////////////////
14193 // VmaDefragmentationAlgorithm_Fast
14194
14195 VmaDefragmentationAlgorithm_Fast::VmaDefragmentationAlgorithm_Fast(
14196 VmaAllocator hAllocator,
14197 VmaBlockVector* pBlockVector,
14198 uint32_t currentFrameIndex,
14199 bool overlappingMoveSupported) :
14200 VmaDefragmentationAlgorithm(hAllocator, pBlockVector, currentFrameIndex),
14201 m_OverlappingMoveSupported(overlappingMoveSupported),
14202 m_AllocationCount(0),
14203 m_AllAllocations(false),
14204 m_BytesMoved(0),
14205 m_AllocationsMoved(0),
14206 m_BlockInfos(VmaStlAllocator<BlockInfo>(hAllocator->GetAllocationCallbacks()))
14207 {
14208 VMA_ASSERT(VMA_DEBUG_MARGIN == 0);
14209
14210 }
14211
14212 VmaDefragmentationAlgorithm_Fast::~VmaDefragmentationAlgorithm_Fast()
14213 {
14214 }
14215
14216 VkResult VmaDefragmentationAlgorithm_Fast::Defragment(
14217 VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves,
14218 VkDeviceSize maxBytesToMove,
14219 uint32_t maxAllocationsToMove,
14220 VmaDefragmentationFlags flags)
14221 {
14222 VMA_ASSERT(m_AllAllocations || m_pBlockVector->CalcAllocationCount() == m_AllocationCount);
14223
14224 const size_t blockCount = m_pBlockVector->GetBlockCount();
14225 if(blockCount == 0 || maxBytesToMove == 0 || maxAllocationsToMove == 0)
14226 {
14227 return VK_SUCCESS;
14228 }
14229
14230 PreprocessMetadata();
14231
14232 // Sort blocks in order from most destination.
14233
14234 m_BlockInfos.resize(blockCount);
14235 for(size_t i = 0; i < blockCount; ++i)
14236 {
14237 m_BlockInfos[i].origBlockIndex = i;
14238 }
14239
14240 VMA_SORT(m_BlockInfos.begin(), m_BlockInfos.end(), [this](const BlockInfo& lhs, const BlockInfo& rhs) -> bool {
14241 return m_pBlockVector->GetBlock(lhs.origBlockIndex)->m_pMetadata->GetSumFreeSize() <
14242 m_pBlockVector->GetBlock(rhs.origBlockIndex)->m_pMetadata->GetSumFreeSize();
14243 });
14244
14245 // THE MAIN ALGORITHM
14246
14247 FreeSpaceDatabase freeSpaceDb;
14248
14249 size_t dstBlockInfoIndex = 0;
14250 size_t dstOrigBlockIndex = m_BlockInfos[dstBlockInfoIndex].origBlockIndex;
14251 VmaDeviceMemoryBlock* pDstBlock = m_pBlockVector->GetBlock(dstOrigBlockIndex);
14252 VmaBlockMetadata_Generic* pDstMetadata = (VmaBlockMetadata_Generic*)pDstBlock->m_pMetadata;
14253 VkDeviceSize dstBlockSize = pDstMetadata->GetSize();
14254 VkDeviceSize dstOffset = 0;
14255
14256 bool end = false;
14257 for(size_t srcBlockInfoIndex = 0; !end && srcBlockInfoIndex < blockCount; ++srcBlockInfoIndex)
14258 {
14259 const size_t srcOrigBlockIndex = m_BlockInfos[srcBlockInfoIndex].origBlockIndex;
14260 VmaDeviceMemoryBlock* const pSrcBlock = m_pBlockVector->GetBlock(srcOrigBlockIndex);
14261 VmaBlockMetadata_Generic* const pSrcMetadata = (VmaBlockMetadata_Generic*)pSrcBlock->m_pMetadata;
14262 for(VmaSuballocationList::iterator srcSuballocIt = pSrcMetadata->m_Suballocations.begin();
14263 !end && srcSuballocIt != pSrcMetadata->m_Suballocations.end(); )
14264 {
14265 VmaAllocation_T* const pAlloc = srcSuballocIt->hAllocation;
14266 const VkDeviceSize srcAllocAlignment = pAlloc->GetAlignment();
14267 const VkDeviceSize srcAllocSize = srcSuballocIt->size;
14268 if(m_AllocationsMoved == maxAllocationsToMove ||
14269 m_BytesMoved + srcAllocSize > maxBytesToMove)
14270 {
14271 end = true;
14272 break;
14273 }
14274 const VkDeviceSize srcAllocOffset = srcSuballocIt->offset;
14275
14276 VmaDefragmentationMove move = {};
14277 // Try to place it in one of free spaces from the database.
14278 size_t freeSpaceInfoIndex;
14279 VkDeviceSize dstAllocOffset;
14280 if(freeSpaceDb.Fetch(srcAllocAlignment, srcAllocSize,
14281 freeSpaceInfoIndex, dstAllocOffset))
14282 {
14283 size_t freeSpaceOrigBlockIndex = m_BlockInfos[freeSpaceInfoIndex].origBlockIndex;
14284 VmaDeviceMemoryBlock* pFreeSpaceBlock = m_pBlockVector->GetBlock(freeSpaceOrigBlockIndex);
14285 VmaBlockMetadata_Generic* pFreeSpaceMetadata = (VmaBlockMetadata_Generic*)pFreeSpaceBlock->m_pMetadata;
14286
14287 // Same block
14288 if(freeSpaceInfoIndex == srcBlockInfoIndex)
14289 {
14290 VMA_ASSERT(dstAllocOffset <= srcAllocOffset);
14291
14292 // MOVE OPTION 1: Move the allocation inside the same block by decreasing offset.
14293
14294 VmaSuballocation suballoc = *srcSuballocIt;
14295 suballoc.offset = dstAllocOffset;
14296 suballoc.hAllocation->ChangeOffset(dstAllocOffset);
14297 m_BytesMoved += srcAllocSize;
14298 ++m_AllocationsMoved;
14299
14300 VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt;
14301 ++nextSuballocIt;
14302 pSrcMetadata->m_Suballocations.erase(srcSuballocIt);
14303 srcSuballocIt = nextSuballocIt;
14304
14305 InsertSuballoc(pFreeSpaceMetadata, suballoc);
14306
14307 move.srcBlockIndex = srcOrigBlockIndex;
14308 move.dstBlockIndex = freeSpaceOrigBlockIndex;
14309 move.srcOffset = srcAllocOffset;
14310 move.dstOffset = dstAllocOffset;
14311 move.size = srcAllocSize;
14312
14313 moves.push_back(move);
14314 }
14315 // Different block
14316 else
14317 {
14318 // MOVE OPTION 2: Move the allocation to a different block.
14319
14320 VMA_ASSERT(freeSpaceInfoIndex < srcBlockInfoIndex);
14321
14322 VmaSuballocation suballoc = *srcSuballocIt;
14323 suballoc.offset = dstAllocOffset;
14324 suballoc.hAllocation->ChangeBlockAllocation(m_hAllocator, pFreeSpaceBlock, dstAllocOffset);
14325 m_BytesMoved += srcAllocSize;
14326 ++m_AllocationsMoved;
14327
14328 VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt;
14329 ++nextSuballocIt;
14330 pSrcMetadata->m_Suballocations.erase(srcSuballocIt);
14331 srcSuballocIt = nextSuballocIt;
14332
14333 InsertSuballoc(pFreeSpaceMetadata, suballoc);
14334
14335 move.srcBlockIndex = srcOrigBlockIndex;
14336 move.dstBlockIndex = freeSpaceOrigBlockIndex;
14337 move.srcOffset = srcAllocOffset;
14338 move.dstOffset = dstAllocOffset;
14339 move.size = srcAllocSize;
14340
14341 moves.push_back(move);
14342 }
14343 }
14344 else
14345 {
14346 dstAllocOffset = VmaAlignUp(dstOffset, srcAllocAlignment);
14347
14348 // If the allocation doesn't fit before the end of dstBlock, forward to next block.
14349 while(dstBlockInfoIndex < srcBlockInfoIndex &&
14350 dstAllocOffset + srcAllocSize > dstBlockSize)
14351 {
14352 // But before that, register remaining free space at the end of dst block.
14353 freeSpaceDb.Register(dstBlockInfoIndex, dstOffset, dstBlockSize - dstOffset);
14354
14355 ++dstBlockInfoIndex;
14356 dstOrigBlockIndex = m_BlockInfos[dstBlockInfoIndex].origBlockIndex;
14357 pDstBlock = m_pBlockVector->GetBlock(dstOrigBlockIndex);
14358 pDstMetadata = (VmaBlockMetadata_Generic*)pDstBlock->m_pMetadata;
14359 dstBlockSize = pDstMetadata->GetSize();
14360 dstOffset = 0;
14361 dstAllocOffset = 0;
14362 }
14363
14364 // Same block
14365 if(dstBlockInfoIndex == srcBlockInfoIndex)
14366 {
14367 VMA_ASSERT(dstAllocOffset <= srcAllocOffset);
14368
14369 const bool overlap = dstAllocOffset + srcAllocSize > srcAllocOffset;
14370
14371 bool skipOver = overlap;
14372 if(overlap && m_OverlappingMoveSupported && dstAllocOffset < srcAllocOffset)
14373 {
14374 // If destination and source place overlap, skip if it would move it
14375 // by only < 1/64 of its size.
14376 skipOver = (srcAllocOffset - dstAllocOffset) * 64 < srcAllocSize;
14377 }
14378
14379 if(skipOver)
14380 {
14381 freeSpaceDb.Register(dstBlockInfoIndex, dstOffset, srcAllocOffset - dstOffset);
14382
14383 dstOffset = srcAllocOffset + srcAllocSize;
14384 ++srcSuballocIt;
14385 }
14386 // MOVE OPTION 1: Move the allocation inside the same block by decreasing offset.
14387 else
14388 {
14389 srcSuballocIt->offset = dstAllocOffset;
14390 srcSuballocIt->hAllocation->ChangeOffset(dstAllocOffset);
14391 dstOffset = dstAllocOffset + srcAllocSize;
14392 m_BytesMoved += srcAllocSize;
14393 ++m_AllocationsMoved;
14394 ++srcSuballocIt;
14395
14396 move.srcBlockIndex = srcOrigBlockIndex;
14397 move.dstBlockIndex = dstOrigBlockIndex;
14398 move.srcOffset = srcAllocOffset;
14399 move.dstOffset = dstAllocOffset;
14400 move.size = srcAllocSize;
14401
14402 moves.push_back(move);
14403 }
14404 }
14405 // Different block
14406 else
14407 {
14408 // MOVE OPTION 2: Move the allocation to a different block.
14409
14410 VMA_ASSERT(dstBlockInfoIndex < srcBlockInfoIndex);
14411 VMA_ASSERT(dstAllocOffset + srcAllocSize <= dstBlockSize);
14412
14413 VmaSuballocation suballoc = *srcSuballocIt;
14414 suballoc.offset = dstAllocOffset;
14415 suballoc.hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlock, dstAllocOffset);
14416 dstOffset = dstAllocOffset + srcAllocSize;
14417 m_BytesMoved += srcAllocSize;
14418 ++m_AllocationsMoved;
14419
14420 VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt;
14421 ++nextSuballocIt;
14422 pSrcMetadata->m_Suballocations.erase(srcSuballocIt);
14423 srcSuballocIt = nextSuballocIt;
14424
14425 pDstMetadata->m_Suballocations.push_back(suballoc);
14426
14427 move.srcBlockIndex = srcOrigBlockIndex;
14428 move.dstBlockIndex = dstOrigBlockIndex;
14429 move.srcOffset = srcAllocOffset;
14430 move.dstOffset = dstAllocOffset;
14431 move.size = srcAllocSize;
14432
14433 moves.push_back(move);
14434 }
14435 }
14436 }
14437 }
14438
14439 m_BlockInfos.clear();
14440
14441 PostprocessMetadata();
14442
14443 return VK_SUCCESS;
14444 }
14445
14446 void VmaDefragmentationAlgorithm_Fast::PreprocessMetadata()
14447 {
14448 const size_t blockCount = m_pBlockVector->GetBlockCount();
14449 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
14450 {
14451 VmaBlockMetadata_Generic* const pMetadata =
14452 (VmaBlockMetadata_Generic*)m_pBlockVector->GetBlock(blockIndex)->m_pMetadata;
14453 pMetadata->m_FreeCount = 0;
14454 pMetadata->m_SumFreeSize = pMetadata->GetSize();
14455 pMetadata->m_FreeSuballocationsBySize.clear();
14456 for(VmaSuballocationList::iterator it = pMetadata->m_Suballocations.begin();
14457 it != pMetadata->m_Suballocations.end(); )
14458 {
14459 if(it->type == VMA_SUBALLOCATION_TYPE_FREE)
14460 {
14461 VmaSuballocationList::iterator nextIt = it;
14462 ++nextIt;
14463 pMetadata->m_Suballocations.erase(it);
14464 it = nextIt;
14465 }
14466 else
14467 {
14468 ++it;
14469 }
14470 }
14471 }
14472 }
14473
14474 void VmaDefragmentationAlgorithm_Fast::PostprocessMetadata()
14475 {
14476 const size_t blockCount = m_pBlockVector->GetBlockCount();
14477 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
14478 {
14479 VmaBlockMetadata_Generic* const pMetadata =
14480 (VmaBlockMetadata_Generic*)m_pBlockVector->GetBlock(blockIndex)->m_pMetadata;
14481 const VkDeviceSize blockSize = pMetadata->GetSize();
14482
14483 // No allocations in this block - entire area is free.
14484 if(pMetadata->m_Suballocations.empty())
14485 {
14486 pMetadata->m_FreeCount = 1;
14487 //pMetadata->m_SumFreeSize is already set to blockSize.
14488 VmaSuballocation suballoc = {
14489 0, // offset
14490 blockSize, // size
14491 VMA_NULL, // hAllocation
14492 VMA_SUBALLOCATION_TYPE_FREE };
14493 pMetadata->m_Suballocations.push_back(suballoc);
14494 pMetadata->RegisterFreeSuballocation(pMetadata->m_Suballocations.begin());
14495 }
14496 // There are some allocations in this block.
14497 else
14498 {
14499 VkDeviceSize offset = 0;
14500 VmaSuballocationList::iterator it;
14501 for(it = pMetadata->m_Suballocations.begin();
14502 it != pMetadata->m_Suballocations.end();
14503 ++it)
14504 {
14505 VMA_ASSERT(it->type != VMA_SUBALLOCATION_TYPE_FREE);
14506 VMA_ASSERT(it->offset >= offset);
14507
14508 // Need to insert preceding free space.
14509 if(it->offset > offset)
14510 {
14511 ++pMetadata->m_FreeCount;
14512 const VkDeviceSize freeSize = it->offset - offset;
14513 VmaSuballocation suballoc = {
14514 offset, // offset
14515 freeSize, // size
14516 VMA_NULL, // hAllocation
14517 VMA_SUBALLOCATION_TYPE_FREE };
14518 VmaSuballocationList::iterator precedingFreeIt = pMetadata->m_Suballocations.insert(it, suballoc);
14519 if(freeSize >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
14520 {
14521 pMetadata->m_FreeSuballocationsBySize.push_back(precedingFreeIt);
14522 }
14523 }
14524
14525 pMetadata->m_SumFreeSize -= it->size;
14526 offset = it->offset + it->size;
14527 }
14528
14529 // Need to insert trailing free space.
14530 if(offset < blockSize)
14531 {
14532 ++pMetadata->m_FreeCount;
14533 const VkDeviceSize freeSize = blockSize - offset;
14534 VmaSuballocation suballoc = {
14535 offset, // offset
14536 freeSize, // size
14537 VMA_NULL, // hAllocation
14538 VMA_SUBALLOCATION_TYPE_FREE };
14539 VMA_ASSERT(it == pMetadata->m_Suballocations.end());
14540 VmaSuballocationList::iterator trailingFreeIt = pMetadata->m_Suballocations.insert(it, suballoc);
14541 if(freeSize > VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
14542 {
14543 pMetadata->m_FreeSuballocationsBySize.push_back(trailingFreeIt);
14544 }
14545 }
14546
14547 VMA_SORT(
14548 pMetadata->m_FreeSuballocationsBySize.begin(),
14549 pMetadata->m_FreeSuballocationsBySize.end(),
14550 VmaSuballocationItemSizeLess());
14551 }
14552
14553 VMA_HEAVY_ASSERT(pMetadata->Validate());
14554 }
14555 }
14556
14557 void VmaDefragmentationAlgorithm_Fast::InsertSuballoc(VmaBlockMetadata_Generic* pMetadata, const VmaSuballocation& suballoc)
14558 {
14559 // TODO: Optimize somehow. Remember iterator instead of searching for it linearly.
14560 VmaSuballocationList::iterator it = pMetadata->m_Suballocations.begin();
14561 while(it != pMetadata->m_Suballocations.end())
14562 {
14563 if(it->offset < suballoc.offset)
14564 {
14565 ++it;
14566 }
14567 }
14568 pMetadata->m_Suballocations.insert(it, suballoc);
14569 }
14570
14571 ////////////////////////////////////////////////////////////////////////////////
14572 // VmaBlockVectorDefragmentationContext
14573
14574 VmaBlockVectorDefragmentationContext::VmaBlockVectorDefragmentationContext(
14575 VmaAllocator hAllocator,
14576 VmaPool hCustomPool,
14577 VmaBlockVector* pBlockVector,
14578 uint32_t currFrameIndex) :
14579 res(VK_SUCCESS),
14580 mutexLocked(false),
14581 blockContexts(VmaStlAllocator<VmaBlockDefragmentationContext>(hAllocator->GetAllocationCallbacks())),
14582 defragmentationMoves(VmaStlAllocator<VmaDefragmentationMove>(hAllocator->GetAllocationCallbacks())),
14583 defragmentationMovesProcessed(0),
14584 defragmentationMovesCommitted(0),
14585 hasDefragmentationPlan(0),
14586 m_hAllocator(hAllocator),
14587 m_hCustomPool(hCustomPool),
14588 m_pBlockVector(pBlockVector),
14589 m_CurrFrameIndex(currFrameIndex),
14590 m_pAlgorithm(VMA_NULL),
14591 m_Allocations(VmaStlAllocator<AllocInfo>(hAllocator->GetAllocationCallbacks())),
14592 m_AllAllocations(false)
14593 {
14594 }
14595
14596 VmaBlockVectorDefragmentationContext::~VmaBlockVectorDefragmentationContext()
14597 {
14598 vma_delete(m_hAllocator, m_pAlgorithm);
14599 }
14600
14601 void VmaBlockVectorDefragmentationContext::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
14602 {
14603 AllocInfo info = { hAlloc, pChanged };
14604 m_Allocations.push_back(info);
14605 }
14606
14607 void VmaBlockVectorDefragmentationContext::Begin(bool overlappingMoveSupported, VmaDefragmentationFlags flags)
14608 {
14609 const bool allAllocations = m_AllAllocations ||
14610 m_Allocations.size() == m_pBlockVector->CalcAllocationCount();
14611
14612 /********************************
14613 HERE IS THE CHOICE OF DEFRAGMENTATION ALGORITHM.
14614 ********************************/
14615
14616 /*
14617 Fast algorithm is supported only when certain criteria are met:
14618 - VMA_DEBUG_MARGIN is 0.
14619 - All allocations in this block vector are moveable.
14620 - There is no possibility of image/buffer granularity conflict.
14621 - The defragmentation is not incremental
14622 */
14623 if(VMA_DEBUG_MARGIN == 0 &&
14624 allAllocations &&
14625 !m_pBlockVector->IsBufferImageGranularityConflictPossible() &&
14626 !(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL))
14627 {
14628 m_pAlgorithm = vma_new(m_hAllocator, VmaDefragmentationAlgorithm_Fast)(
14629 m_hAllocator, m_pBlockVector, m_CurrFrameIndex, overlappingMoveSupported);
14630 }
14631 else
14632 {
14633 m_pAlgorithm = vma_new(m_hAllocator, VmaDefragmentationAlgorithm_Generic)(
14634 m_hAllocator, m_pBlockVector, m_CurrFrameIndex, overlappingMoveSupported);
14635 }
14636
14637 if(allAllocations)
14638 {
14639 m_pAlgorithm->AddAll();
14640 }
14641 else
14642 {
14643 for(size_t i = 0, count = m_Allocations.size(); i < count; ++i)
14644 {
14645 m_pAlgorithm->AddAllocation(m_Allocations[i].hAlloc, m_Allocations[i].pChanged);
14646 }
14647 }
14648 }
14649
14650 ////////////////////////////////////////////////////////////////////////////////
14651 // VmaDefragmentationContext
14652
14653 VmaDefragmentationContext_T::VmaDefragmentationContext_T(
14654 VmaAllocator hAllocator,
14655 uint32_t currFrameIndex,
14656 uint32_t flags,
14657 VmaDefragmentationStats* pStats) :
14658 m_hAllocator(hAllocator),
14659 m_CurrFrameIndex(currFrameIndex),
14660 m_Flags(flags),
14661 m_pStats(pStats),
14662 m_CustomPoolContexts(VmaStlAllocator<VmaBlockVectorDefragmentationContext*>(hAllocator->GetAllocationCallbacks()))
14663 {
14664 memset(m_DefaultPoolContexts, 0, sizeof(m_DefaultPoolContexts));
14665 }
14666
14667 VmaDefragmentationContext_T::~VmaDefragmentationContext_T()
14668 {
14669 for(size_t i = m_CustomPoolContexts.size(); i--; )
14670 {
14671 VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_CustomPoolContexts[i];
14672 pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_Flags, m_pStats);
14673 vma_delete(m_hAllocator, pBlockVectorCtx);
14674 }
14675 for(size_t i = m_hAllocator->m_MemProps.memoryTypeCount; i--; )
14676 {
14677 VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_DefaultPoolContexts[i];
14678 if(pBlockVectorCtx)
14679 {
14680 pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_Flags, m_pStats);
14681 vma_delete(m_hAllocator, pBlockVectorCtx);
14682 }
14683 }
14684 }
14685
14686 void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, const VmaPool* pPools)
14687 {
14688 for(uint32_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
14689 {
14690 VmaPool pool = pPools[poolIndex];
14691 VMA_ASSERT(pool);
14692 // Pools with algorithm other than default are not defragmented.
14693 if(pool->m_BlockVector.GetAlgorithm() == 0)
14694 {
14695 VmaBlockVectorDefragmentationContext* pBlockVectorDefragCtx = VMA_NULL;
14696
14697 for(size_t i = m_CustomPoolContexts.size(); i--; )
14698 {
14699 if(m_CustomPoolContexts[i]->GetCustomPool() == pool)
14700 {
14701 pBlockVectorDefragCtx = m_CustomPoolContexts[i];
14702 break;
14703 }
14704 }
14705
14706 if(!pBlockVectorDefragCtx)
14707 {
14708 pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)(
14709 m_hAllocator,
14710 pool,
14711 &pool->m_BlockVector,
14712 m_CurrFrameIndex);
14713 m_CustomPoolContexts.push_back(pBlockVectorDefragCtx);
14714 }
14715
14716 pBlockVectorDefragCtx->AddAll();
14717 }
14718 }
14719 }
14720
14721 void VmaDefragmentationContext_T::AddAllocations(
14722 uint32_t allocationCount,
14723 const VmaAllocation* pAllocations,
14724 VkBool32* pAllocationsChanged)
14725 {
14726 // Dispatch pAllocations among defragmentators. Create them when necessary.
14727 for(uint32_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
14728 {
14729 const VmaAllocation hAlloc = pAllocations[allocIndex];
14730 VMA_ASSERT(hAlloc);
14731 // DedicatedAlloc cannot be defragmented.
14732 if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
14733 // Lost allocation cannot be defragmented.
14734 (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
14735 {
14736 VmaBlockVectorDefragmentationContext* pBlockVectorDefragCtx = VMA_NULL;
14737
14738 const VmaPool hAllocPool = hAlloc->GetBlock()->GetParentPool();
14739 // This allocation belongs to custom pool.
14740 if(hAllocPool != VK_NULL_HANDLE)
14741 {
14742 // Pools with algorithm other than default are not defragmented.
14743 if(hAllocPool->m_BlockVector.GetAlgorithm() == 0)
14744 {
14745 for(size_t i = m_CustomPoolContexts.size(); i--; )
14746 {
14747 if(m_CustomPoolContexts[i]->GetCustomPool() == hAllocPool)
14748 {
14749 pBlockVectorDefragCtx = m_CustomPoolContexts[i];
14750 break;
14751 }
14752 }
14753 if(!pBlockVectorDefragCtx)
14754 {
14755 pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)(
14756 m_hAllocator,
14757 hAllocPool,
14758 &hAllocPool->m_BlockVector,
14759 m_CurrFrameIndex);
14760 m_CustomPoolContexts.push_back(pBlockVectorDefragCtx);
14761 }
14762 }
14763 }
14764 // This allocation belongs to default pool.
14765 else
14766 {
14767 const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
14768 pBlockVectorDefragCtx = m_DefaultPoolContexts[memTypeIndex];
14769 if(!pBlockVectorDefragCtx)
14770 {
14771 pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)(
14772 m_hAllocator,
14773 VMA_NULL, // hCustomPool
14774 m_hAllocator->m_pBlockVectors[memTypeIndex],
14775 m_CurrFrameIndex);
14776 m_DefaultPoolContexts[memTypeIndex] = pBlockVectorDefragCtx;
14777 }
14778 }
14779
14780 if(pBlockVectorDefragCtx)
14781 {
14782 VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
14783 &pAllocationsChanged[allocIndex] : VMA_NULL;
14784 pBlockVectorDefragCtx->AddAllocation(hAlloc, pChanged);
14785 }
14786 }
14787 }
14788 }
14789
14790 VkResult VmaDefragmentationContext_T::Defragment(
14791 VkDeviceSize maxCpuBytesToMove, uint32_t maxCpuAllocationsToMove,
14792 VkDeviceSize maxGpuBytesToMove, uint32_t maxGpuAllocationsToMove,
14793 VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags)
14794 {
14795 if(pStats)
14796 {
14797 memset(pStats, 0, sizeof(VmaDefragmentationStats));
14798 }
14799
14800 if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)
14801 {
14802 // For incremental defragmetnations, we just earmark how much we can move
14803 // The real meat is in the defragmentation steps
14804 m_MaxCpuBytesToMove = maxCpuBytesToMove;
14805 m_MaxCpuAllocationsToMove = maxCpuAllocationsToMove;
14806
14807 m_MaxGpuBytesToMove = maxGpuBytesToMove;
14808 m_MaxGpuAllocationsToMove = maxGpuAllocationsToMove;
14809
14810 if(m_MaxCpuBytesToMove == 0 && m_MaxCpuAllocationsToMove == 0 &&
14811 m_MaxGpuBytesToMove == 0 && m_MaxGpuAllocationsToMove == 0)
14812 return VK_SUCCESS;
14813
14814 return VK_NOT_READY;
14815 }
14816
14817 if(commandBuffer == VK_NULL_HANDLE)
14818 {
14819 maxGpuBytesToMove = 0;
14820 maxGpuAllocationsToMove = 0;
14821 }
14822
14823 VkResult res = VK_SUCCESS;
14824
14825 // Process default pools.
14826 for(uint32_t memTypeIndex = 0;
14827 memTypeIndex < m_hAllocator->GetMemoryTypeCount() && res >= VK_SUCCESS;
14828 ++memTypeIndex)
14829 {
14830 VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex];
14831 if(pBlockVectorCtx)
14832 {
14833 VMA_ASSERT(pBlockVectorCtx->GetBlockVector());
14834 pBlockVectorCtx->GetBlockVector()->Defragment(
14835 pBlockVectorCtx,
14836 pStats, flags,
14837 maxCpuBytesToMove, maxCpuAllocationsToMove,
14838 maxGpuBytesToMove, maxGpuAllocationsToMove,
14839 commandBuffer);
14840 if(pBlockVectorCtx->res != VK_SUCCESS)
14841 {
14842 res = pBlockVectorCtx->res;
14843 }
14844 }
14845 }
14846
14847 // Process custom pools.
14848 for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size();
14849 customCtxIndex < customCtxCount && res >= VK_SUCCESS;
14850 ++customCtxIndex)
14851 {
14852 VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex];
14853 VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector());
14854 pBlockVectorCtx->GetBlockVector()->Defragment(
14855 pBlockVectorCtx,
14856 pStats, flags,
14857 maxCpuBytesToMove, maxCpuAllocationsToMove,
14858 maxGpuBytesToMove, maxGpuAllocationsToMove,
14859 commandBuffer);
14860 if(pBlockVectorCtx->res != VK_SUCCESS)
14861 {
14862 res = pBlockVectorCtx->res;
14863 }
14864 }
14865
14866 return res;
14867 }
14868
14869 VkResult VmaDefragmentationContext_T::DefragmentPassBegin(VmaDefragmentationPassInfo* pInfo)
14870 {
14871 VmaDefragmentationPassMoveInfo* pCurrentMove = pInfo->pMoves;
14872 uint32_t movesLeft = pInfo->moveCount;
14873
14874 // Process default pools.
14875 for(uint32_t memTypeIndex = 0;
14876 memTypeIndex < m_hAllocator->GetMemoryTypeCount();
14877 ++memTypeIndex)
14878 {
14879 VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex];
14880 if(pBlockVectorCtx)
14881 {
14882 VMA_ASSERT(pBlockVectorCtx->GetBlockVector());
14883
14884 if(!pBlockVectorCtx->hasDefragmentationPlan)
14885 {
14886 pBlockVectorCtx->GetBlockVector()->Defragment(
14887 pBlockVectorCtx,
14888 m_pStats, m_Flags,
14889 m_MaxCpuBytesToMove, m_MaxCpuAllocationsToMove,
14890 m_MaxGpuBytesToMove, m_MaxGpuAllocationsToMove,
14891 VK_NULL_HANDLE);
14892
14893 if(pBlockVectorCtx->res < VK_SUCCESS)
14894 continue;
14895
14896 pBlockVectorCtx->hasDefragmentationPlan = true;
14897 }
14898
14899 const uint32_t processed = pBlockVectorCtx->GetBlockVector()->ProcessDefragmentations(
14900 pBlockVectorCtx,
14901 pCurrentMove, movesLeft);
14902
14903 movesLeft -= processed;
14904 pCurrentMove += processed;
14905 }
14906 }
14907
14908 // Process custom pools.
14909 for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size();
14910 customCtxIndex < customCtxCount;
14911 ++customCtxIndex)
14912 {
14913 VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex];
14914 VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector());
14915
14916 if(!pBlockVectorCtx->hasDefragmentationPlan)
14917 {
14918 pBlockVectorCtx->GetBlockVector()->Defragment(
14919 pBlockVectorCtx,
14920 m_pStats, m_Flags,
14921 m_MaxCpuBytesToMove, m_MaxCpuAllocationsToMove,
14922 m_MaxGpuBytesToMove, m_MaxGpuAllocationsToMove,
14923 VK_NULL_HANDLE);
14924
14925 if(pBlockVectorCtx->res < VK_SUCCESS)
14926 continue;
14927
14928 pBlockVectorCtx->hasDefragmentationPlan = true;
14929 }
14930
14931 const uint32_t processed = pBlockVectorCtx->GetBlockVector()->ProcessDefragmentations(
14932 pBlockVectorCtx,
14933 pCurrentMove, movesLeft);
14934
14935 movesLeft -= processed;
14936 pCurrentMove += processed;
14937 }
14938
14939 pInfo->moveCount = pInfo->moveCount - movesLeft;
14940
14941 return VK_SUCCESS;
14942 }
14943 VkResult VmaDefragmentationContext_T::DefragmentPassEnd()
14944 {
14945 VkResult res = VK_SUCCESS;
14946
14947 // Process default pools.
14948 for(uint32_t memTypeIndex = 0;
14949 memTypeIndex < m_hAllocator->GetMemoryTypeCount();
14950 ++memTypeIndex)
14951 {
14952 VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex];
14953 if(pBlockVectorCtx)
14954 {
14955 VMA_ASSERT(pBlockVectorCtx->GetBlockVector());
14956
14957 if(!pBlockVectorCtx->hasDefragmentationPlan)
14958 {
14959 res = VK_NOT_READY;
14960 continue;
14961 }
14962
14963 pBlockVectorCtx->GetBlockVector()->CommitDefragmentations(
14964 pBlockVectorCtx, m_pStats);
14965
14966 if(pBlockVectorCtx->defragmentationMoves.size() != pBlockVectorCtx->defragmentationMovesCommitted)
14967 res = VK_NOT_READY;
14968 }
14969 }
14970
14971 // Process custom pools.
14972 for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size();
14973 customCtxIndex < customCtxCount;
14974 ++customCtxIndex)
14975 {
14976 VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex];
14977 VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector());
14978
14979 if(!pBlockVectorCtx->hasDefragmentationPlan)
14980 {
14981 res = VK_NOT_READY;
14982 continue;
14983 }
14984
14985 pBlockVectorCtx->GetBlockVector()->CommitDefragmentations(
14986 pBlockVectorCtx, m_pStats);
14987
14988 if(pBlockVectorCtx->defragmentationMoves.size() != pBlockVectorCtx->defragmentationMovesCommitted)
14989 res = VK_NOT_READY;
14990 }
14991
14992 return res;
14993 }
14994
14995 ////////////////////////////////////////////////////////////////////////////////
14996 // VmaRecorder
14997
14998 #if VMA_RECORDING_ENABLED
14999
15000 VmaRecorder::VmaRecorder() :
15001 m_UseMutex(true),
15002 m_Flags(0),
15003 m_File(VMA_NULL),
15004 m_RecordingStartTime(std::chrono::high_resolution_clock::now())
15005 {
15006 }
15007
15008 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
15009 {
15010 m_UseMutex = useMutex;
15011 m_Flags = settings.flags;
15012
15013 #if defined(_WIN32)
15014 // Open file for writing.
15015 errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
15016
15017 if(err != 0)
15018 {
15019 return VK_ERROR_INITIALIZATION_FAILED;
15020 }
15021 #else
15022 // Open file for writing.
15023 m_File = fopen(settings.pFilePath, "wb");
15024
15025 if(m_File == 0)
15026 {
15027 return VK_ERROR_INITIALIZATION_FAILED;
15028 }
15029 #endif
15030
15031 // Write header.
15032 fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
15033 fprintf(m_File, "%s\n", "1,8");
15034
15035 return VK_SUCCESS;
15036 }
15037
15038 VmaRecorder::~VmaRecorder()
15039 {
15040 if(m_File != VMA_NULL)
15041 {
15042 fclose(m_File);
15043 }
15044 }
15045
15046 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
15047 {
15048 CallParams callParams;
15049 GetBasicParams(callParams);
15050
15051 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15052 fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
15053 Flush();
15054 }
15055
15056 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
15057 {
15058 CallParams callParams;
15059 GetBasicParams(callParams);
15060
15061 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15062 fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
15063 Flush();
15064 }
15065
15066 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
15067 {
15068 CallParams callParams;
15069 GetBasicParams(callParams);
15070
15071 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15072 fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
15073 createInfo.memoryTypeIndex,
15074 createInfo.flags,
15075 createInfo.blockSize,
15076 (uint64_t)createInfo.minBlockCount,
15077 (uint64_t)createInfo.maxBlockCount,
15078 createInfo.frameInUseCount,
15079 pool);
15080 Flush();
15081 }
15082
15083 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
15084 {
15085 CallParams callParams;
15086 GetBasicParams(callParams);
15087
15088 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15089 fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
15090 pool);
15091 Flush();
15092 }
15093
15094 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
15095 const VkMemoryRequirements& vkMemReq,
15096 const VmaAllocationCreateInfo& createInfo,
15097 VmaAllocation allocation)
15098 {
15099 CallParams callParams;
15100 GetBasicParams(callParams);
15101
15102 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15103 UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
15104 fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15105 vkMemReq.size,
15106 vkMemReq.alignment,
15107 vkMemReq.memoryTypeBits,
15108 createInfo.flags,
15109 createInfo.usage,
15110 createInfo.requiredFlags,
15111 createInfo.preferredFlags,
15112 createInfo.memoryTypeBits,
15113 createInfo.pool,
15114 allocation,
15115 userDataStr.GetString());
15116 Flush();
15117 }
15118
15119 void VmaRecorder::RecordAllocateMemoryPages(uint32_t frameIndex,
15120 const VkMemoryRequirements& vkMemReq,
15121 const VmaAllocationCreateInfo& createInfo,
15122 uint64_t allocationCount,
15123 const VmaAllocation* pAllocations)
15124 {
15125 CallParams callParams;
15126 GetBasicParams(callParams);
15127
15128 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15129 UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
15130 fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryPages,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,", callParams.threadId, callParams.time, frameIndex,
15131 vkMemReq.size,
15132 vkMemReq.alignment,
15133 vkMemReq.memoryTypeBits,
15134 createInfo.flags,
15135 createInfo.usage,
15136 createInfo.requiredFlags,
15137 createInfo.preferredFlags,
15138 createInfo.memoryTypeBits,
15139 createInfo.pool);
15140 PrintPointerList(allocationCount, pAllocations);
15141 fprintf(m_File, ",%s\n", userDataStr.GetString());
15142 Flush();
15143 }
15144
15145 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
15146 const VkMemoryRequirements& vkMemReq,
15147 bool requiresDedicatedAllocation,
15148 bool prefersDedicatedAllocation,
15149 const VmaAllocationCreateInfo& createInfo,
15150 VmaAllocation allocation)
15151 {
15152 CallParams callParams;
15153 GetBasicParams(callParams);
15154
15155 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15156 UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
15157 fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15158 vkMemReq.size,
15159 vkMemReq.alignment,
15160 vkMemReq.memoryTypeBits,
15161 requiresDedicatedAllocation ? 1 : 0,
15162 prefersDedicatedAllocation ? 1 : 0,
15163 createInfo.flags,
15164 createInfo.usage,
15165 createInfo.requiredFlags,
15166 createInfo.preferredFlags,
15167 createInfo.memoryTypeBits,
15168 createInfo.pool,
15169 allocation,
15170 userDataStr.GetString());
15171 Flush();
15172 }
15173
15174 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
15175 const VkMemoryRequirements& vkMemReq,
15176 bool requiresDedicatedAllocation,
15177 bool prefersDedicatedAllocation,
15178 const VmaAllocationCreateInfo& createInfo,
15179 VmaAllocation allocation)
15180 {
15181 CallParams callParams;
15182 GetBasicParams(callParams);
15183
15184 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15185 UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
15186 fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15187 vkMemReq.size,
15188 vkMemReq.alignment,
15189 vkMemReq.memoryTypeBits,
15190 requiresDedicatedAllocation ? 1 : 0,
15191 prefersDedicatedAllocation ? 1 : 0,
15192 createInfo.flags,
15193 createInfo.usage,
15194 createInfo.requiredFlags,
15195 createInfo.preferredFlags,
15196 createInfo.memoryTypeBits,
15197 createInfo.pool,
15198 allocation,
15199 userDataStr.GetString());
15200 Flush();
15201 }
15202
15203 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
15204 VmaAllocation allocation)
15205 {
15206 CallParams callParams;
15207 GetBasicParams(callParams);
15208
15209 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15210 fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
15211 allocation);
15212 Flush();
15213 }
15214
15215 void VmaRecorder::RecordFreeMemoryPages(uint32_t frameIndex,
15216 uint64_t allocationCount,
15217 const VmaAllocation* pAllocations)
15218 {
15219 CallParams callParams;
15220 GetBasicParams(callParams);
15221
15222 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15223 fprintf(m_File, "%u,%.3f,%u,vmaFreeMemoryPages,", callParams.threadId, callParams.time, frameIndex);
15224 PrintPointerList(allocationCount, pAllocations);
15225 fprintf(m_File, "\n");
15226 Flush();
15227 }
15228
15229 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
15230 VmaAllocation allocation,
15231 const void* pUserData)
15232 {
15233 CallParams callParams;
15234 GetBasicParams(callParams);
15235
15236 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15237 UserDataString userDataStr(
15238 allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
15239 pUserData);
15240 fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15241 allocation,
15242 userDataStr.GetString());
15243 Flush();
15244 }
15245
15246 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
15247 VmaAllocation allocation)
15248 {
15249 CallParams callParams;
15250 GetBasicParams(callParams);
15251
15252 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15253 fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
15254 allocation);
15255 Flush();
15256 }
15257
15258 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
15259 VmaAllocation allocation)
15260 {
15261 CallParams callParams;
15262 GetBasicParams(callParams);
15263
15264 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15265 fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
15266 allocation);
15267 Flush();
15268 }
15269
15270 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
15271 VmaAllocation allocation)
15272 {
15273 CallParams callParams;
15274 GetBasicParams(callParams);
15275
15276 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15277 fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
15278 allocation);
15279 Flush();
15280 }
15281
15282 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
15283 VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
15284 {
15285 CallParams callParams;
15286 GetBasicParams(callParams);
15287
15288 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15289 fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
15290 allocation,
15291 offset,
15292 size);
15293 Flush();
15294 }
15295
15296 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
15297 VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
15298 {
15299 CallParams callParams;
15300 GetBasicParams(callParams);
15301
15302 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15303 fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
15304 allocation,
15305 offset,
15306 size);
15307 Flush();
15308 }
15309
15310 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
15311 const VkBufferCreateInfo& bufCreateInfo,
15312 const VmaAllocationCreateInfo& allocCreateInfo,
15313 VmaAllocation allocation)
15314 {
15315 CallParams callParams;
15316 GetBasicParams(callParams);
15317
15318 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15319 UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
15320 fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15321 bufCreateInfo.flags,
15322 bufCreateInfo.size,
15323 bufCreateInfo.usage,
15324 bufCreateInfo.sharingMode,
15325 allocCreateInfo.flags,
15326 allocCreateInfo.usage,
15327 allocCreateInfo.requiredFlags,
15328 allocCreateInfo.preferredFlags,
15329 allocCreateInfo.memoryTypeBits,
15330 allocCreateInfo.pool,
15331 allocation,
15332 userDataStr.GetString());
15333 Flush();
15334 }
15335
15336 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
15337 const VkImageCreateInfo& imageCreateInfo,
15338 const VmaAllocationCreateInfo& allocCreateInfo,
15339 VmaAllocation allocation)
15340 {
15341 CallParams callParams;
15342 GetBasicParams(callParams);
15343
15344 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15345 UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
15346 fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15347 imageCreateInfo.flags,
15348 imageCreateInfo.imageType,
15349 imageCreateInfo.format,
15350 imageCreateInfo.extent.width,
15351 imageCreateInfo.extent.height,
15352 imageCreateInfo.extent.depth,
15353 imageCreateInfo.mipLevels,
15354 imageCreateInfo.arrayLayers,
15355 imageCreateInfo.samples,
15356 imageCreateInfo.tiling,
15357 imageCreateInfo.usage,
15358 imageCreateInfo.sharingMode,
15359 imageCreateInfo.initialLayout,
15360 allocCreateInfo.flags,
15361 allocCreateInfo.usage,
15362 allocCreateInfo.requiredFlags,
15363 allocCreateInfo.preferredFlags,
15364 allocCreateInfo.memoryTypeBits,
15365 allocCreateInfo.pool,
15366 allocation,
15367 userDataStr.GetString());
15368 Flush();
15369 }
15370
15371 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
15372 VmaAllocation allocation)
15373 {
15374 CallParams callParams;
15375 GetBasicParams(callParams);
15376
15377 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15378 fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
15379 allocation);
15380 Flush();
15381 }
15382
15383 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
15384 VmaAllocation allocation)
15385 {
15386 CallParams callParams;
15387 GetBasicParams(callParams);
15388
15389 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15390 fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
15391 allocation);
15392 Flush();
15393 }
15394
15395 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
15396 VmaAllocation allocation)
15397 {
15398 CallParams callParams;
15399 GetBasicParams(callParams);
15400
15401 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15402 fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
15403 allocation);
15404 Flush();
15405 }
15406
15407 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
15408 VmaAllocation allocation)
15409 {
15410 CallParams callParams;
15411 GetBasicParams(callParams);
15412
15413 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15414 fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
15415 allocation);
15416 Flush();
15417 }
15418
15419 void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex,
15420 VmaPool pool)
15421 {
15422 CallParams callParams;
15423 GetBasicParams(callParams);
15424
15425 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15426 fprintf(m_File, "%u,%.3f,%u,vmaMakePoolAllocationsLost,%p\n", callParams.threadId, callParams.time, frameIndex,
15427 pool);
15428 Flush();
15429 }
15430
15431 void VmaRecorder::RecordDefragmentationBegin(uint32_t frameIndex,
15432 const VmaDefragmentationInfo2& info,
15433 VmaDefragmentationContext ctx)
15434 {
15435 CallParams callParams;
15436 GetBasicParams(callParams);
15437
15438 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15439 fprintf(m_File, "%u,%.3f,%u,vmaDefragmentationBegin,%u,", callParams.threadId, callParams.time, frameIndex,
15440 info.flags);
15441 PrintPointerList(info.allocationCount, info.pAllocations);
15442 fprintf(m_File, ",");
15443 PrintPointerList(info.poolCount, info.pPools);
15444 fprintf(m_File, ",%llu,%u,%llu,%u,%p,%p\n",
15445 info.maxCpuBytesToMove,
15446 info.maxCpuAllocationsToMove,
15447 info.maxGpuBytesToMove,
15448 info.maxGpuAllocationsToMove,
15449 info.commandBuffer,
15450 ctx);
15451 Flush();
15452 }
15453
15454 void VmaRecorder::RecordDefragmentationEnd(uint32_t frameIndex,
15455 VmaDefragmentationContext ctx)
15456 {
15457 CallParams callParams;
15458 GetBasicParams(callParams);
15459
15460 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15461 fprintf(m_File, "%u,%.3f,%u,vmaDefragmentationEnd,%p\n", callParams.threadId, callParams.time, frameIndex,
15462 ctx);
15463 Flush();
15464 }
15465
15466 void VmaRecorder::RecordSetPoolName(uint32_t frameIndex,
15467 VmaPool pool,
15468 const char* name)
15469 {
15470 CallParams callParams;
15471 GetBasicParams(callParams);
15472
15473 VmaMutexLock lock(m_FileMutex, m_UseMutex);
15474 fprintf(m_File, "%u,%.3f,%u,vmaSetPoolName,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
15475 pool, name != VMA_NULL ? name : "");
15476 Flush();
15477 }
15478
15479 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
15480 {
15481 if(pUserData != VMA_NULL)
15482 {
15483 if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
15484 {
15485 m_Str = (const char*)pUserData;
15486 }
15487 else
15488 {
15489 // If VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is not specified, convert the string's memory address to a string and store it.
15490 snprintf(m_PtrStr, 17, "%p", pUserData);
15491 m_Str = m_PtrStr;
15492 }
15493 }
15494 else
15495 {
15496 m_Str = "";
15497 }
15498 }
15499
15500 void VmaRecorder::WriteConfiguration(
15501 const VkPhysicalDeviceProperties& devProps,
15502 const VkPhysicalDeviceMemoryProperties& memProps,
15503 uint32_t vulkanApiVersion,
15504 bool dedicatedAllocationExtensionEnabled,
15505 bool bindMemory2ExtensionEnabled,
15506 bool memoryBudgetExtensionEnabled,
15507 bool deviceCoherentMemoryExtensionEnabled)
15508 {
15509 fprintf(m_File, "Config,Begin\n");
15510
15511 fprintf(m_File, "VulkanApiVersion,%u,%u\n", VK_VERSION_MAJOR(vulkanApiVersion), VK_VERSION_MINOR(vulkanApiVersion));
15512
15513 fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion);
15514 fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion);
15515 fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID);
15516 fprintf(m_File, "PhysicalDevice,deviceID,%u\n", devProps.deviceID);
15517 fprintf(m_File, "PhysicalDevice,deviceType,%u\n", devProps.deviceType);
15518 fprintf(m_File, "PhysicalDevice,deviceName,%s\n", devProps.deviceName);
15519
15520 fprintf(m_File, "PhysicalDeviceLimits,maxMemoryAllocationCount,%u\n", devProps.limits.maxMemoryAllocationCount);
15521 fprintf(m_File, "PhysicalDeviceLimits,bufferImageGranularity,%llu\n", devProps.limits.bufferImageGranularity);
15522 fprintf(m_File, "PhysicalDeviceLimits,nonCoherentAtomSize,%llu\n", devProps.limits.nonCoherentAtomSize);
15523
15524 fprintf(m_File, "PhysicalDeviceMemory,HeapCount,%u\n", memProps.memoryHeapCount);
15525 for(uint32_t i = 0; i < memProps.memoryHeapCount; ++i)
15526 {
15527 fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,size,%llu\n", i, memProps.memoryHeaps[i].size);
15528 fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,flags,%u\n", i, memProps.memoryHeaps[i].flags);
15529 }
15530 fprintf(m_File, "PhysicalDeviceMemory,TypeCount,%u\n", memProps.memoryTypeCount);
15531 for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i)
15532 {
15533 fprintf(m_File, "PhysicalDeviceMemory,Type,%u,heapIndex,%u\n", i, memProps.memoryTypes[i].heapIndex);
15534 fprintf(m_File, "PhysicalDeviceMemory,Type,%u,propertyFlags,%u\n", i, memProps.memoryTypes[i].propertyFlags);
15535 }
15536
15537 fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0);
15538 fprintf(m_File, "Extension,VK_KHR_bind_memory2,%u\n", bindMemory2ExtensionEnabled ? 1 : 0);
15539 fprintf(m_File, "Extension,VK_EXT_memory_budget,%u\n", memoryBudgetExtensionEnabled ? 1 : 0);
15540 fprintf(m_File, "Extension,VK_AMD_device_coherent_memory,%u\n", deviceCoherentMemoryExtensionEnabled ? 1 : 0);
15541
15542 fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0);
15543 fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT);
15544 fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN);
15545 fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0);
15546 fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0);
15547 fprintf(m_File, "Macro,VMA_DEBUG_GLOBAL_MUTEX,%u\n", VMA_DEBUG_GLOBAL_MUTEX ? 1 : 0);
15548 fprintf(m_File, "Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,%llu\n", (VkDeviceSize)VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY);
15549 fprintf(m_File, "Macro,VMA_SMALL_HEAP_MAX_SIZE,%llu\n", (VkDeviceSize)VMA_SMALL_HEAP_MAX_SIZE);
15550 fprintf(m_File, "Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,%llu\n", (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
15551
15552 fprintf(m_File, "Config,End\n");
15553 }
15554
15555 void VmaRecorder::GetBasicParams(CallParams& outParams)
15556 {
15557 #if defined(_WIN32)
15558 outParams.threadId = GetCurrentThreadId();
15559 #else
15560 // Use C++11 features to get thread id and convert it to uint32_t.
15561 // There is room for optimization since sstream is quite slow.
15562 // Is there a better way to convert std::this_thread::get_id() to uint32_t?
15563 std::thread::id thread_id = std::this_thread::get_id();
15564 stringstream thread_id_to_string_converter;
15565 thread_id_to_string_converter << thread_id;
15566 string thread_id_as_string = thread_id_to_string_converter.str();
15567 outParams.threadId = static_cast<uint32_t>(std::stoi(thread_id_as_string.c_str()));
15568 #endif
15569
15570 auto current_time = std::chrono::high_resolution_clock::now();
15571
15572 outParams.time = std::chrono::duration<double, std::chrono::seconds::period>(current_time - m_RecordingStartTime).count();
15573 }
15574
15575 void VmaRecorder::PrintPointerList(uint64_t count, const VmaAllocation* pItems)
15576 {
15577 if(count)
15578 {
15579 fprintf(m_File, "%p", pItems[0]);
15580 for(uint64_t i = 1; i < count; ++i)
15581 {
15582 fprintf(m_File, " %p", pItems[i]);
15583 }
15584 }
15585 }
15586
15587 void VmaRecorder::Flush()
15588 {
15589 if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
15590 {
15591 fflush(m_File);
15592 }
15593 }
15594
15595 #endif // #if VMA_RECORDING_ENABLED
15596
15597 ////////////////////////////////////////////////////////////////////////////////
15598 // VmaAllocationObjectAllocator
15599
15600 VmaAllocationObjectAllocator::VmaAllocationObjectAllocator(const VkAllocationCallbacks* pAllocationCallbacks) :
15601 m_Allocator(pAllocationCallbacks, 1024)
15602 {
15603 }
15604
15605 template<typename... Types> VmaAllocation VmaAllocationObjectAllocator::Allocate(Types... args)
15606 {
15607 VmaMutexLock mutexLock(m_Mutex);
15608 return m_Allocator.Alloc<Types...>(std::forward<Types>(args)...);
15609 }
15610
15611 void VmaAllocationObjectAllocator::Free(VmaAllocation hAlloc)
15612 {
15613 VmaMutexLock mutexLock(m_Mutex);
15614 m_Allocator.Free(hAlloc);
15615 }
15616
15617 ////////////////////////////////////////////////////////////////////////////////
15618 // VmaAllocator_T
15619
15620 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
15621 m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
15622 m_VulkanApiVersion(pCreateInfo->vulkanApiVersion != 0 ? pCreateInfo->vulkanApiVersion : VK_API_VERSION_1_0),
15623 m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
15624 m_UseKhrBindMemory2((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT) != 0),
15625 m_UseExtMemoryBudget((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT) != 0),
15626 m_UseAmdDeviceCoherentMemory((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT) != 0),
15627 m_UseKhrBufferDeviceAddress((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT) != 0),
15628 m_hDevice(pCreateInfo->device),
15629 m_hInstance(pCreateInfo->instance),
15630 m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
15631 m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
15632 *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
15633 m_AllocationObjectAllocator(&m_AllocationCallbacks),
15634 m_HeapSizeLimitMask(0),
15635 m_PreferredLargeHeapBlockSize(0),
15636 m_PhysicalDevice(pCreateInfo->physicalDevice),
15637 m_CurrentFrameIndex(0),
15638 m_GpuDefragmentationMemoryTypeBits(UINT32_MAX),
15639 m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
15640 m_NextPoolId(0),
15641 m_GlobalMemoryTypeBits(UINT32_MAX)
15642 #if VMA_RECORDING_ENABLED
15643 ,m_pRecorder(VMA_NULL)
15644 #endif
15645 {
15646 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
15647 {
15648 m_UseKhrDedicatedAllocation = false;
15649 m_UseKhrBindMemory2 = false;
15650 }
15651
15652 if(VMA_DEBUG_DETECT_CORRUPTION)
15653 {
15654 // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
15655 VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
15656 }
15657
15658 VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device && pCreateInfo->instance);
15659
15660 if(m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0))
15661 {
15662 #if !(VMA_DEDICATED_ALLOCATION)
15663 if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
15664 {
15665 VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
15666 }
15667 #endif
15668 #if !(VMA_BIND_MEMORY2)
15669 if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT) != 0)
15670 {
15671 VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT set but required extension is disabled by preprocessor macros.");
15672 }
15673 #endif
15674 }
15675 #if !(VMA_MEMORY_BUDGET)
15676 if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT) != 0)
15677 {
15678 VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT set but required extension is disabled by preprocessor macros.");
15679 }
15680 #endif
15681 #if !(VMA_BUFFER_DEVICE_ADDRESS)
15682 if(m_UseKhrBufferDeviceAddress)
15683 {
15684 VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT is set but required extension or Vulkan 1.2 is not available in your Vulkan header or its support in VMA has been disabled by a preprocessor macro.");
15685 }
15686 #endif
15687 #if VMA_VULKAN_VERSION < 1002000
15688 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 2, 0))
15689 {
15690 VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_2 but required Vulkan version is disabled by preprocessor macros.");
15691 }
15692 #endif
15693 #if VMA_VULKAN_VERSION < 1001000
15694 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
15695 {
15696 VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_1 but required Vulkan version is disabled by preprocessor macros.");
15697 }
15698 #endif
15699
15700 memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
15701 memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
15702 memset(&m_MemProps, 0, sizeof(m_MemProps));
15703
15704 memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
15705 memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
15706 memset(&m_VulkanFunctions, 0, sizeof(m_VulkanFunctions));
15707
15708 if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
15709 {
15710 m_DeviceMemoryCallbacks.pUserData = pCreateInfo->pDeviceMemoryCallbacks->pUserData;
15711 m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
15712 m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
15713 }
15714
15715 ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
15716
15717 (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
15718 (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
15719
15720 VMA_ASSERT(VmaIsPow2(VMA_DEBUG_ALIGNMENT));
15721 VMA_ASSERT(VmaIsPow2(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY));
15722 VMA_ASSERT(VmaIsPow2(m_PhysicalDeviceProperties.limits.bufferImageGranularity));
15723 VMA_ASSERT(VmaIsPow2(m_PhysicalDeviceProperties.limits.nonCoherentAtomSize));
15724
15725 m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
15726 pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
15727
15728 m_GlobalMemoryTypeBits = CalculateGlobalMemoryTypeBits();
15729
15730 if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
15731 {
15732 for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
15733 {
15734 const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
15735 if(limit != VK_WHOLE_SIZE)
15736 {
15737 m_HeapSizeLimitMask |= 1u << heapIndex;
15738 if(limit < m_MemProps.memoryHeaps[heapIndex].size)
15739 {
15740 m_MemProps.memoryHeaps[heapIndex].size = limit;
15741 }
15742 }
15743 }
15744 }
15745
15746 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
15747 {
15748 const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
15749
15750 m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
15751 this,
15752 VK_NULL_HANDLE, // hParentPool
15753 memTypeIndex,
15754 preferredBlockSize,
15755 0,
15756 SIZE_MAX,
15757 GetBufferImageGranularity(),
15758 pCreateInfo->frameInUseCount,
15759 false, // explicitBlockSize
15760 false); // linearAlgorithm
15761 // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
15762 // becase minBlockCount is 0.
15763 m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
15764
15765 }
15766 }
15767
15768 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
15769 {
15770 VkResult res = VK_SUCCESS;
15771
15772 if(pCreateInfo->pRecordSettings != VMA_NULL &&
15773 !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
15774 {
15775 #if VMA_RECORDING_ENABLED
15776 m_pRecorder = vma_new(this, VmaRecorder)();
15777 res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
15778 if(res != VK_SUCCESS)
15779 {
15780 return res;
15781 }
15782 m_pRecorder->WriteConfiguration(
15783 m_PhysicalDeviceProperties,
15784 m_MemProps,
15785 m_VulkanApiVersion,
15786 m_UseKhrDedicatedAllocation,
15787 m_UseKhrBindMemory2,
15788 m_UseExtMemoryBudget,
15789 m_UseAmdDeviceCoherentMemory);
15790 m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
15791 #else
15792 VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
15793 return VK_ERROR_FEATURE_NOT_PRESENT;
15794 #endif
15795 }
15796
15797 #if VMA_MEMORY_BUDGET
15798 if(m_UseExtMemoryBudget)
15799 {
15800 UpdateVulkanBudget();
15801 }
15802 #endif // #if VMA_MEMORY_BUDGET
15803
15804 return res;
15805 }
15806
15807 VmaAllocator_T::~VmaAllocator_T()
15808 {
15809 #if VMA_RECORDING_ENABLED
15810 if(m_pRecorder != VMA_NULL)
15811 {
15812 m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
15813 vma_delete(this, m_pRecorder);
15814 }
15815 #endif
15816
15817 VMA_ASSERT(m_Pools.empty());
15818
15819 for(size_t i = GetMemoryTypeCount(); i--; )
15820 {
15821 if(m_pDedicatedAllocations[i] != VMA_NULL && !m_pDedicatedAllocations[i]->empty())
15822 {
15823 VMA_ASSERT(0 && "Unfreed dedicated allocations found.");
15824 }
15825
15826 vma_delete(this, m_pDedicatedAllocations[i]);
15827 vma_delete(this, m_pBlockVectors[i]);
15828 }
15829 }
15830
15831 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
15832 {
15833 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
15834 ImportVulkanFunctions_Static();
15835 #endif
15836
15837 if(pVulkanFunctions != VMA_NULL)
15838 {
15839 ImportVulkanFunctions_Custom(pVulkanFunctions);
15840 }
15841
15842 #if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1
15843 ImportVulkanFunctions_Dynamic();
15844 #endif
15845
15846 ValidateVulkanFunctions();
15847 }
15848
15849 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
15850
15851 void VmaAllocator_T::ImportVulkanFunctions_Static()
15852 {
15853 // Vulkan 1.0
15854 m_VulkanFunctions.vkGetPhysicalDeviceProperties = (PFN_vkGetPhysicalDeviceProperties)vkGetPhysicalDeviceProperties;
15855 m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = (PFN_vkGetPhysicalDeviceMemoryProperties)vkGetPhysicalDeviceMemoryProperties;
15856 m_VulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkAllocateMemory;
15857 m_VulkanFunctions.vkFreeMemory = (PFN_vkFreeMemory)vkFreeMemory;
15858 m_VulkanFunctions.vkMapMemory = (PFN_vkMapMemory)vkMapMemory;
15859 m_VulkanFunctions.vkUnmapMemory = (PFN_vkUnmapMemory)vkUnmapMemory;
15860 m_VulkanFunctions.vkFlushMappedMemoryRanges = (PFN_vkFlushMappedMemoryRanges)vkFlushMappedMemoryRanges;
15861 m_VulkanFunctions.vkInvalidateMappedMemoryRanges = (PFN_vkInvalidateMappedMemoryRanges)vkInvalidateMappedMemoryRanges;
15862 m_VulkanFunctions.vkBindBufferMemory = (PFN_vkBindBufferMemory)vkBindBufferMemory;
15863 m_VulkanFunctions.vkBindImageMemory = (PFN_vkBindImageMemory)vkBindImageMemory;
15864 m_VulkanFunctions.vkGetBufferMemoryRequirements = (PFN_vkGetBufferMemoryRequirements)vkGetBufferMemoryRequirements;
15865 m_VulkanFunctions.vkGetImageMemoryRequirements = (PFN_vkGetImageMemoryRequirements)vkGetImageMemoryRequirements;
15866 m_VulkanFunctions.vkCreateBuffer = (PFN_vkCreateBuffer)vkCreateBuffer;
15867 m_VulkanFunctions.vkDestroyBuffer = (PFN_vkDestroyBuffer)vkDestroyBuffer;
15868 m_VulkanFunctions.vkCreateImage = (PFN_vkCreateImage)vkCreateImage;
15869 m_VulkanFunctions.vkDestroyImage = (PFN_vkDestroyImage)vkDestroyImage;
15870 m_VulkanFunctions.vkCmdCopyBuffer = (PFN_vkCmdCopyBuffer)vkCmdCopyBuffer;
15871
15872 // Vulkan 1.1
15873 #if VMA_VULKAN_VERSION >= 1001000
15874 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
15875 {
15876 m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR = (PFN_vkGetBufferMemoryRequirements2)vkGetBufferMemoryRequirements2;
15877 m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = (PFN_vkGetImageMemoryRequirements2)vkGetImageMemoryRequirements2;
15878 m_VulkanFunctions.vkBindBufferMemory2KHR = (PFN_vkBindBufferMemory2)vkBindBufferMemory2;
15879 m_VulkanFunctions.vkBindImageMemory2KHR = (PFN_vkBindImageMemory2)vkBindImageMemory2;
15880 m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR = (PFN_vkGetPhysicalDeviceMemoryProperties2)vkGetPhysicalDeviceMemoryProperties2;
15881 }
15882 #endif
15883 }
15884
15885 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
15886
15887 void VmaAllocator_T::ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVulkanFunctions)
15888 {
15889 VMA_ASSERT(pVulkanFunctions != VMA_NULL);
15890
15891 #define VMA_COPY_IF_NOT_NULL(funcName) \
15892 if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
15893
15894 VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
15895 VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
15896 VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
15897 VMA_COPY_IF_NOT_NULL(vkFreeMemory);
15898 VMA_COPY_IF_NOT_NULL(vkMapMemory);
15899 VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
15900 VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
15901 VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
15902 VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
15903 VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
15904 VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
15905 VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
15906 VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
15907 VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
15908 VMA_COPY_IF_NOT_NULL(vkCreateImage);
15909 VMA_COPY_IF_NOT_NULL(vkDestroyImage);
15910 VMA_COPY_IF_NOT_NULL(vkCmdCopyBuffer);
15911
15912 #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
15913 VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
15914 VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
15915 #endif
15916
15917 #if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000
15918 VMA_COPY_IF_NOT_NULL(vkBindBufferMemory2KHR);
15919 VMA_COPY_IF_NOT_NULL(vkBindImageMemory2KHR);
15920 #endif
15921
15922 #if VMA_MEMORY_BUDGET
15923 VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties2KHR);
15924 #endif
15925
15926 #undef VMA_COPY_IF_NOT_NULL
15927 }
15928
15929 #if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1
15930
15931 void VmaAllocator_T::ImportVulkanFunctions_Dynamic()
15932 {
15933 #define VMA_FETCH_INSTANCE_FUNC(memberName, functionPointerType, functionNameString) \
15934 if(m_VulkanFunctions.memberName == VMA_NULL) \
15935 m_VulkanFunctions.memberName = \
15936 (functionPointerType)vkGetInstanceProcAddr(m_hInstance, functionNameString);
15937 #define VMA_FETCH_DEVICE_FUNC(memberName, functionPointerType, functionNameString) \
15938 if(m_VulkanFunctions.memberName == VMA_NULL) \
15939 m_VulkanFunctions.memberName = \
15940 (functionPointerType)vkGetDeviceProcAddr(m_hDevice, functionNameString);
15941
15942 VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceProperties, PFN_vkGetPhysicalDeviceProperties, "vkGetPhysicalDeviceProperties");
15943 VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties, PFN_vkGetPhysicalDeviceMemoryProperties, "vkGetPhysicalDeviceMemoryProperties");
15944 VMA_FETCH_DEVICE_FUNC(vkAllocateMemory, PFN_vkAllocateMemory, "vkAllocateMemory");
15945 VMA_FETCH_DEVICE_FUNC(vkFreeMemory, PFN_vkFreeMemory, "vkFreeMemory");
15946 VMA_FETCH_DEVICE_FUNC(vkMapMemory, PFN_vkMapMemory, "vkMapMemory");
15947 VMA_FETCH_DEVICE_FUNC(vkUnmapMemory, PFN_vkUnmapMemory, "vkUnmapMemory");
15948 VMA_FETCH_DEVICE_FUNC(vkFlushMappedMemoryRanges, PFN_vkFlushMappedMemoryRanges, "vkFlushMappedMemoryRanges");
15949 VMA_FETCH_DEVICE_FUNC(vkInvalidateMappedMemoryRanges, PFN_vkInvalidateMappedMemoryRanges, "vkInvalidateMappedMemoryRanges");
15950 VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory, PFN_vkBindBufferMemory, "vkBindBufferMemory");
15951 VMA_FETCH_DEVICE_FUNC(vkBindImageMemory, PFN_vkBindImageMemory, "vkBindImageMemory");
15952 VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements, PFN_vkGetBufferMemoryRequirements, "vkGetBufferMemoryRequirements");
15953 VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements, PFN_vkGetImageMemoryRequirements, "vkGetImageMemoryRequirements");
15954 VMA_FETCH_DEVICE_FUNC(vkCreateBuffer, PFN_vkCreateBuffer, "vkCreateBuffer");
15955 VMA_FETCH_DEVICE_FUNC(vkDestroyBuffer, PFN_vkDestroyBuffer, "vkDestroyBuffer");
15956 VMA_FETCH_DEVICE_FUNC(vkCreateImage, PFN_vkCreateImage, "vkCreateImage");
15957 VMA_FETCH_DEVICE_FUNC(vkDestroyImage, PFN_vkDestroyImage, "vkDestroyImage");
15958 VMA_FETCH_DEVICE_FUNC(vkCmdCopyBuffer, PFN_vkCmdCopyBuffer, "vkCmdCopyBuffer");
15959
15960 #if VMA_VULKAN_VERSION >= 1001000
15961 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
15962 {
15963 VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements2KHR, PFN_vkGetBufferMemoryRequirements2, "vkGetBufferMemoryRequirements2");
15964 VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2, "vkGetImageMemoryRequirements2");
15965 VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2, "vkBindBufferMemory2");
15966 VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2, "vkBindImageMemory2");
15967 VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2, "vkGetPhysicalDeviceMemoryProperties2");
15968 }
15969 #endif
15970
15971 #if VMA_DEDICATED_ALLOCATION
15972 if(m_UseKhrDedicatedAllocation)
15973 {
15974 VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements2KHR, PFN_vkGetBufferMemoryRequirements2KHR, "vkGetBufferMemoryRequirements2KHR");
15975 VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2KHR, "vkGetImageMemoryRequirements2KHR");
15976 }
15977 #endif
15978
15979 #if VMA_BIND_MEMORY2
15980 if(m_UseKhrBindMemory2)
15981 {
15982 VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2KHR, "vkBindBufferMemory2KHR");
15983 VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2KHR, "vkBindImageMemory2KHR");
15984 }
15985 #endif // #if VMA_BIND_MEMORY2
15986
15987 #if VMA_MEMORY_BUDGET
15988 if(m_UseExtMemoryBudget)
15989 {
15990 VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2KHR, "vkGetPhysicalDeviceMemoryProperties2KHR");
15991 }
15992 #endif // #if VMA_MEMORY_BUDGET
15993
15994 #undef VMA_FETCH_DEVICE_FUNC
15995 #undef VMA_FETCH_INSTANCE_FUNC
15996 }
15997
15998 #endif // #if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1
15999
16000 void VmaAllocator_T::ValidateVulkanFunctions()
16001 {
16002 VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
16003 VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
16004 VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
16005 VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
16006 VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
16007 VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
16008 VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
16009 VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
16010 VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
16011 VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
16012 VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
16013 VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
16014 VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
16015 VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
16016 VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
16017 VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
16018 VMA_ASSERT(m_VulkanFunctions.vkCmdCopyBuffer != VMA_NULL);
16019
16020 #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16021 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0) || m_UseKhrDedicatedAllocation)
16022 {
16023 VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
16024 VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
16025 }
16026 #endif
16027
16028 #if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000
16029 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0) || m_UseKhrBindMemory2)
16030 {
16031 VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory2KHR != VMA_NULL);
16032 VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory2KHR != VMA_NULL);
16033 }
16034 #endif
16035
16036 #if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000
16037 if(m_UseExtMemoryBudget || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
16038 {
16039 VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR != VMA_NULL);
16040 }
16041 #endif
16042 }
16043
16044 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
16045 {
16046 const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
16047 const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
16048 const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
16049 return VmaAlignUp(isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize, (VkDeviceSize)32);
16050 }
16051
16052 VkResult VmaAllocator_T::AllocateMemoryOfType(
16053 VkDeviceSize size,
16054 VkDeviceSize alignment,
16055 bool dedicatedAllocation,
16056 VkBuffer dedicatedBuffer,
16057 VkBufferUsageFlags dedicatedBufferUsage,
16058 VkImage dedicatedImage,
16059 const VmaAllocationCreateInfo& createInfo,
16060 uint32_t memTypeIndex,
16061 VmaSuballocationType suballocType,
16062 size_t allocationCount,
16063 VmaAllocation* pAllocations)
16064 {
16065 VMA_ASSERT(pAllocations != VMA_NULL);
16066 VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, AllocationCount=%zu, Size=%llu", memTypeIndex, allocationCount, size);
16067
16068 VmaAllocationCreateInfo finalCreateInfo = createInfo;
16069
16070 // If memory type is not HOST_VISIBLE, disable MAPPED.
16071 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
16072 (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
16073 {
16074 finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
16075 }
16076 // If memory is lazily allocated, it should be always dedicated.
16077 if(finalCreateInfo.usage == VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED)
16078 {
16079 finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
16080 }
16081
16082 VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
16083 VMA_ASSERT(blockVector);
16084
16085 const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
16086 bool preferDedicatedMemory =
16087 VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
16088 dedicatedAllocation ||
16089 // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
16090 size > preferredBlockSize / 2;
16091
16092 if(preferDedicatedMemory &&
16093 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
16094 finalCreateInfo.pool == VK_NULL_HANDLE)
16095 {
16096 finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
16097 }
16098
16099 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
16100 {
16101 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
16102 {
16103 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16104 }
16105 else
16106 {
16107 return AllocateDedicatedMemory(
16108 size,
16109 suballocType,
16110 memTypeIndex,
16111 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0,
16112 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
16113 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
16114 finalCreateInfo.pUserData,
16115 dedicatedBuffer,
16116 dedicatedBufferUsage,
16117 dedicatedImage,
16118 allocationCount,
16119 pAllocations);
16120 }
16121 }
16122 else
16123 {
16124 VkResult res = blockVector->Allocate(
16125 m_CurrentFrameIndex.load(),
16126 size,
16127 alignment,
16128 finalCreateInfo,
16129 suballocType,
16130 allocationCount,
16131 pAllocations);
16132 if(res == VK_SUCCESS)
16133 {
16134 return res;
16135 }
16136
16137 // 5. Try dedicated memory.
16138 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
16139 {
16140 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16141 }
16142 else
16143 {
16144 res = AllocateDedicatedMemory(
16145 size,
16146 suballocType,
16147 memTypeIndex,
16148 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0,
16149 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
16150 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
16151 finalCreateInfo.pUserData,
16152 dedicatedBuffer,
16153 dedicatedBufferUsage,
16154 dedicatedImage,
16155 allocationCount,
16156 pAllocations);
16157 if(res == VK_SUCCESS)
16158 {
16159 // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
16160 VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
16161 return VK_SUCCESS;
16162 }
16163 else
16164 {
16165 // Everything failed: Return error code.
16166 VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
16167 return res;
16168 }
16169 }
16170 }
16171 }
16172
16173 VkResult VmaAllocator_T::AllocateDedicatedMemory(
16174 VkDeviceSize size,
16175 VmaSuballocationType suballocType,
16176 uint32_t memTypeIndex,
16177 bool withinBudget,
16178 bool map,
16179 bool isUserDataString,
16180 void* pUserData,
16181 VkBuffer dedicatedBuffer,
16182 VkBufferUsageFlags dedicatedBufferUsage,
16183 VkImage dedicatedImage,
16184 size_t allocationCount,
16185 VmaAllocation* pAllocations)
16186 {
16187 VMA_ASSERT(allocationCount > 0 && pAllocations);
16188
16189 if(withinBudget)
16190 {
16191 const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
16192 VmaBudget heapBudget = {};
16193 GetBudget(&heapBudget, heapIndex, 1);
16194 if(heapBudget.usage + size * allocationCount > heapBudget.budget)
16195 {
16196 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16197 }
16198 }
16199
16200 VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
16201 allocInfo.memoryTypeIndex = memTypeIndex;
16202 allocInfo.allocationSize = size;
16203
16204 #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16205 VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
16206 if(m_UseKhrDedicatedAllocation || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
16207 {
16208 if(dedicatedBuffer != VK_NULL_HANDLE)
16209 {
16210 VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
16211 dedicatedAllocInfo.buffer = dedicatedBuffer;
16212 VmaPnextChainPushFront(&allocInfo, &dedicatedAllocInfo);
16213 }
16214 else if(dedicatedImage != VK_NULL_HANDLE)
16215 {
16216 dedicatedAllocInfo.image = dedicatedImage;
16217 VmaPnextChainPushFront(&allocInfo, &dedicatedAllocInfo);
16218 }
16219 }
16220 #endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16221
16222 #if VMA_BUFFER_DEVICE_ADDRESS
16223 VkMemoryAllocateFlagsInfoKHR allocFlagsInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO_KHR };
16224 if(m_UseKhrBufferDeviceAddress)
16225 {
16226 bool canContainBufferWithDeviceAddress = true;
16227 if(dedicatedBuffer != VK_NULL_HANDLE)
16228 {
16229 canContainBufferWithDeviceAddress = dedicatedBufferUsage == UINT32_MAX || // Usage flags unknown
16230 (dedicatedBufferUsage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_EXT) != 0;
16231 }
16232 else if(dedicatedImage != VK_NULL_HANDLE)
16233 {
16234 canContainBufferWithDeviceAddress = false;
16235 }
16236 if(canContainBufferWithDeviceAddress)
16237 {
16238 allocFlagsInfo.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR;
16239 VmaPnextChainPushFront(&allocInfo, &allocFlagsInfo);
16240 }
16241 }
16242 #endif // #if VMA_BUFFER_DEVICE_ADDRESS
16243
16244 size_t allocIndex;
16245 VkResult res = VK_SUCCESS;
16246 for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
16247 {
16248 res = AllocateDedicatedMemoryPage(
16249 size,
16250 suballocType,
16251 memTypeIndex,
16252 allocInfo,
16253 map,
16254 isUserDataString,
16255 pUserData,
16256 pAllocations + allocIndex);
16257 if(res != VK_SUCCESS)
16258 {
16259 break;
16260 }
16261 }
16262
16263 if(res == VK_SUCCESS)
16264 {
16265 // Register them in m_pDedicatedAllocations.
16266 {
16267 VmaMutexLockWrite lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
16268 AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
16269 VMA_ASSERT(pDedicatedAllocations);
16270 for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
16271 {
16272 VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, pAllocations[allocIndex]);
16273 }
16274 }
16275
16276 VMA_DEBUG_LOG(" Allocated DedicatedMemory Count=%zu, MemoryTypeIndex=#%u", allocationCount, memTypeIndex);
16277 }
16278 else
16279 {
16280 // Free all already created allocations.
16281 while(allocIndex--)
16282 {
16283 VmaAllocation currAlloc = pAllocations[allocIndex];
16284 VkDeviceMemory hMemory = currAlloc->GetMemory();
16285
16286 /*
16287 There is no need to call this, because Vulkan spec allows to skip vkUnmapMemory
16288 before vkFreeMemory.
16289
16290 if(currAlloc->GetMappedData() != VMA_NULL)
16291 {
16292 (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
16293 }
16294 */
16295
16296 FreeVulkanMemory(memTypeIndex, currAlloc->GetSize(), hMemory);
16297 m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(memTypeIndex), currAlloc->GetSize());
16298 currAlloc->SetUserData(this, VMA_NULL);
16299 m_AllocationObjectAllocator.Free(currAlloc);
16300 }
16301
16302 memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount);
16303 }
16304
16305 return res;
16306 }
16307
16308 VkResult VmaAllocator_T::AllocateDedicatedMemoryPage(
16309 VkDeviceSize size,
16310 VmaSuballocationType suballocType,
16311 uint32_t memTypeIndex,
16312 const VkMemoryAllocateInfo& allocInfo,
16313 bool map,
16314 bool isUserDataString,
16315 void* pUserData,
16316 VmaAllocation* pAllocation)
16317 {
16318 VkDeviceMemory hMemory = VK_NULL_HANDLE;
16319 VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
16320 if(res < 0)
16321 {
16322 VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
16323 return res;
16324 }
16325
16326 void* pMappedData = VMA_NULL;
16327 if(map)
16328 {
16329 res = (*m_VulkanFunctions.vkMapMemory)(
16330 m_hDevice,
16331 hMemory,
16332 0,
16333 VK_WHOLE_SIZE,
16334 0,
16335 &pMappedData);
16336 if(res < 0)
16337 {
16338 VMA_DEBUG_LOG(" vkMapMemory FAILED");
16339 FreeVulkanMemory(memTypeIndex, size, hMemory);
16340 return res;
16341 }
16342 }
16343
16344 *pAllocation = m_AllocationObjectAllocator.Allocate(m_CurrentFrameIndex.load(), isUserDataString);
16345 (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
16346 (*pAllocation)->SetUserData(this, pUserData);
16347 m_Budget.AddAllocation(MemoryTypeIndexToHeapIndex(memTypeIndex), size);
16348 if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
16349 {
16350 FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
16351 }
16352
16353 return VK_SUCCESS;
16354 }
16355
16356 void VmaAllocator_T::GetBufferMemoryRequirements(
16357 VkBuffer hBuffer,
16358 VkMemoryRequirements& memReq,
16359 bool& requiresDedicatedAllocation,
16360 bool& prefersDedicatedAllocation) const
16361 {
16362 #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16363 if(m_UseKhrDedicatedAllocation || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
16364 {
16365 VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
16366 memReqInfo.buffer = hBuffer;
16367
16368 VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
16369
16370 VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
16371 VmaPnextChainPushFront(&memReq2, &memDedicatedReq);
16372
16373 (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
16374
16375 memReq = memReq2.memoryRequirements;
16376 requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
16377 prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
16378 }
16379 else
16380 #endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16381 {
16382 (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
16383 requiresDedicatedAllocation = false;
16384 prefersDedicatedAllocation = false;
16385 }
16386 }
16387
16388 void VmaAllocator_T::GetImageMemoryRequirements(
16389 VkImage hImage,
16390 VkMemoryRequirements& memReq,
16391 bool& requiresDedicatedAllocation,
16392 bool& prefersDedicatedAllocation) const
16393 {
16394 #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16395 if(m_UseKhrDedicatedAllocation || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0))
16396 {
16397 VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
16398 memReqInfo.image = hImage;
16399
16400 VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
16401
16402 VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
16403 VmaPnextChainPushFront(&memReq2, &memDedicatedReq);
16404
16405 (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
16406
16407 memReq = memReq2.memoryRequirements;
16408 requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
16409 prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
16410 }
16411 else
16412 #endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000
16413 {
16414 (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
16415 requiresDedicatedAllocation = false;
16416 prefersDedicatedAllocation = false;
16417 }
16418 }
16419
16420 VkResult VmaAllocator_T::AllocateMemory(
16421 const VkMemoryRequirements& vkMemReq,
16422 bool requiresDedicatedAllocation,
16423 bool prefersDedicatedAllocation,
16424 VkBuffer dedicatedBuffer,
16425 VkBufferUsageFlags dedicatedBufferUsage,
16426 VkImage dedicatedImage,
16427 const VmaAllocationCreateInfo& createInfo,
16428 VmaSuballocationType suballocType,
16429 size_t allocationCount,
16430 VmaAllocation* pAllocations)
16431 {
16432 memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount);
16433
16434 VMA_ASSERT(VmaIsPow2(vkMemReq.alignment));
16435
16436 if(vkMemReq.size == 0)
16437 {
16438 return VK_ERROR_VALIDATION_FAILED_EXT;
16439 }
16440 if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
16441 (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
16442 {
16443 VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
16444 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16445 }
16446 if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
16447 (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
16448 {
16449 VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
16450 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16451 }
16452 if(requiresDedicatedAllocation)
16453 {
16454 if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
16455 {
16456 VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
16457 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16458 }
16459 if(createInfo.pool != VK_NULL_HANDLE)
16460 {
16461 VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
16462 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16463 }
16464 }
16465 if((createInfo.pool != VK_NULL_HANDLE) &&
16466 ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
16467 {
16468 VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
16469 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16470 }
16471
16472 if(createInfo.pool != VK_NULL_HANDLE)
16473 {
16474 const VkDeviceSize alignmentForPool = VMA_MAX(
16475 vkMemReq.alignment,
16476 GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
16477
16478 VmaAllocationCreateInfo createInfoForPool = createInfo;
16479 // If memory type is not HOST_VISIBLE, disable MAPPED.
16480 if((createInfoForPool.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
16481 (m_MemProps.memoryTypes[createInfo.pool->m_BlockVector.GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
16482 {
16483 createInfoForPool.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
16484 }
16485
16486 return createInfo.pool->m_BlockVector.Allocate(
16487 m_CurrentFrameIndex.load(),
16488 vkMemReq.size,
16489 alignmentForPool,
16490 createInfoForPool,
16491 suballocType,
16492 allocationCount,
16493 pAllocations);
16494 }
16495 else
16496 {
16497 // Bit mask of memory Vulkan types acceptable for this allocation.
16498 uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
16499 uint32_t memTypeIndex = UINT32_MAX;
16500 VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
16501 if(res == VK_SUCCESS)
16502 {
16503 VkDeviceSize alignmentForMemType = VMA_MAX(
16504 vkMemReq.alignment,
16505 GetMemoryTypeMinAlignment(memTypeIndex));
16506
16507 res = AllocateMemoryOfType(
16508 vkMemReq.size,
16509 alignmentForMemType,
16510 requiresDedicatedAllocation || prefersDedicatedAllocation,
16511 dedicatedBuffer,
16512 dedicatedBufferUsage,
16513 dedicatedImage,
16514 createInfo,
16515 memTypeIndex,
16516 suballocType,
16517 allocationCount,
16518 pAllocations);
16519 // Succeeded on first try.
16520 if(res == VK_SUCCESS)
16521 {
16522 return res;
16523 }
16524 // Allocation from this memory type failed. Try other compatible memory types.
16525 else
16526 {
16527 for(;;)
16528 {
16529 // Remove old memTypeIndex from list of possibilities.
16530 memoryTypeBits &= ~(1u << memTypeIndex);
16531 // Find alternative memTypeIndex.
16532 res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
16533 if(res == VK_SUCCESS)
16534 {
16535 alignmentForMemType = VMA_MAX(
16536 vkMemReq.alignment,
16537 GetMemoryTypeMinAlignment(memTypeIndex));
16538
16539 res = AllocateMemoryOfType(
16540 vkMemReq.size,
16541 alignmentForMemType,
16542 requiresDedicatedAllocation || prefersDedicatedAllocation,
16543 dedicatedBuffer,
16544 dedicatedBufferUsage,
16545 dedicatedImage,
16546 createInfo,
16547 memTypeIndex,
16548 suballocType,
16549 allocationCount,
16550 pAllocations);
16551 // Allocation from this alternative memory type succeeded.
16552 if(res == VK_SUCCESS)
16553 {
16554 return res;
16555 }
16556 // else: Allocation from this memory type failed. Try next one - next loop iteration.
16557 }
16558 // No other matching memory type index could be found.
16559 else
16560 {
16561 // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
16562 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
16563 }
16564 }
16565 }
16566 }
16567 // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
16568 else
16569 return res;
16570 }
16571 }
16572
16573 void VmaAllocator_T::FreeMemory(
16574 size_t allocationCount,
16575 const VmaAllocation* pAllocations)
16576 {
16577 VMA_ASSERT(pAllocations);
16578
16579 for(size_t allocIndex = allocationCount; allocIndex--; )
16580 {
16581 VmaAllocation allocation = pAllocations[allocIndex];
16582
16583 if(allocation != VK_NULL_HANDLE)
16584 {
16585 if(TouchAllocation(allocation))
16586 {
16587 if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
16588 {
16589 FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
16590 }
16591
16592 switch(allocation->GetType())
16593 {
16594 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
16595 {
16596 VmaBlockVector* pBlockVector = VMA_NULL;
16597 VmaPool hPool = allocation->GetBlock()->GetParentPool();
16598 if(hPool != VK_NULL_HANDLE)
16599 {
16600 pBlockVector = &hPool->m_BlockVector;
16601 }
16602 else
16603 {
16604 const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
16605 pBlockVector = m_pBlockVectors[memTypeIndex];
16606 }
16607 pBlockVector->Free(allocation);
16608 }
16609 break;
16610 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
16611 FreeDedicatedMemory(allocation);
16612 break;
16613 default:
16614 VMA_ASSERT(0);
16615 }
16616 }
16617
16618 // Do this regardless of whether the allocation is lost. Lost allocations still account to Budget.AllocationBytes.
16619 m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(allocation->GetMemoryTypeIndex()), allocation->GetSize());
16620 allocation->SetUserData(this, VMA_NULL);
16621 m_AllocationObjectAllocator.Free(allocation);
16622 }
16623 }
16624 }
16625
16626 VkResult VmaAllocator_T::ResizeAllocation(
16627 const VmaAllocation alloc,
16628 VkDeviceSize newSize)
16629 {
16630 // This function is deprecated and so it does nothing. It's left for backward compatibility.
16631 if(newSize == 0 || alloc->GetLastUseFrameIndex() == VMA_FRAME_INDEX_LOST)
16632 {
16633 return VK_ERROR_VALIDATION_FAILED_EXT;
16634 }
16635 if(newSize == alloc->GetSize())
16636 {
16637 return VK_SUCCESS;
16638 }
16639 return VK_ERROR_OUT_OF_POOL_MEMORY;
16640 }
16641
16642 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
16643 {
16644 // Initialize.
16645 InitStatInfo(pStats->total);
16646 for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
16647 InitStatInfo(pStats->memoryType[i]);
16648 for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
16649 InitStatInfo(pStats->memoryHeap[i]);
16650
16651 // Process default pools.
16652 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
16653 {
16654 VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
16655 VMA_ASSERT(pBlockVector);
16656 pBlockVector->AddStats(pStats);
16657 }
16658
16659 // Process custom pools.
16660 {
16661 VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex);
16662 for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
16663 {
16664 m_Pools[poolIndex]->m_BlockVector.AddStats(pStats);
16665 }
16666 }
16667
16668 // Process dedicated allocations.
16669 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
16670 {
16671 const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
16672 VmaMutexLockRead dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
16673 AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
16674 VMA_ASSERT(pDedicatedAllocVector);
16675 for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
16676 {
16677 VmaStatInfo allocationStatInfo;
16678 (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
16679 VmaAddStatInfo(pStats->total, allocationStatInfo);
16680 VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
16681 VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
16682 }
16683 }
16684
16685 // Postprocess.
16686 VmaPostprocessCalcStatInfo(pStats->total);
16687 for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
16688 VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
16689 for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
16690 VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
16691 }
16692
16693 void VmaAllocator_T::GetBudget(VmaBudget* outBudget, uint32_t firstHeap, uint32_t heapCount)
16694 {
16695 #if VMA_MEMORY_BUDGET
16696 if(m_UseExtMemoryBudget)
16697 {
16698 if(m_Budget.m_OperationsSinceBudgetFetch < 30)
16699 {
16700 VmaMutexLockRead lockRead(m_Budget.m_BudgetMutex, m_UseMutex);
16701 for(uint32_t i = 0; i < heapCount; ++i, ++outBudget)
16702 {
16703 const uint32_t heapIndex = firstHeap + i;
16704
16705 outBudget->blockBytes = m_Budget.m_BlockBytes[heapIndex];
16706 outBudget->allocationBytes = m_Budget.m_AllocationBytes[heapIndex];
16707
16708 if(m_Budget.m_VulkanUsage[heapIndex] + outBudget->blockBytes > m_Budget.m_BlockBytesAtBudgetFetch[heapIndex])
16709 {
16710 outBudget->usage = m_Budget.m_VulkanUsage[heapIndex] +
16711 outBudget->blockBytes - m_Budget.m_BlockBytesAtBudgetFetch[heapIndex];
16712 }
16713 else
16714 {
16715 outBudget->usage = 0;
16716 }
16717
16718 // Have to take MIN with heap size because explicit HeapSizeLimit is included in it.
16719 outBudget->budget = VMA_MIN(
16720 m_Budget.m_VulkanBudget[heapIndex], m_MemProps.memoryHeaps[heapIndex].size);
16721 }
16722 }
16723 else
16724 {
16725 UpdateVulkanBudget(); // Outside of mutex lock
16726 GetBudget(outBudget, firstHeap, heapCount); // Recursion
16727 }
16728 }
16729 else
16730 #endif
16731 {
16732 for(uint32_t i = 0; i < heapCount; ++i, ++outBudget)
16733 {
16734 const uint32_t heapIndex = firstHeap + i;
16735
16736 outBudget->blockBytes = m_Budget.m_BlockBytes[heapIndex];
16737 outBudget->allocationBytes = m_Budget.m_AllocationBytes[heapIndex];
16738
16739 outBudget->usage = outBudget->blockBytes;
16740 outBudget->budget = m_MemProps.memoryHeaps[heapIndex].size * 8 / 10; // 80% heuristics.
16741 }
16742 }
16743 }
16744
16745 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
16746
16747 VkResult VmaAllocator_T::DefragmentationBegin(
16748 const VmaDefragmentationInfo2& info,
16749 VmaDefragmentationStats* pStats,
16750 VmaDefragmentationContext* pContext)
16751 {
16752 if(info.pAllocationsChanged != VMA_NULL)
16753 {
16754 memset(info.pAllocationsChanged, 0, info.allocationCount * sizeof(VkBool32));
16755 }
16756
16757 *pContext = vma_new(this, VmaDefragmentationContext_T)(
16758 this, m_CurrentFrameIndex.load(), info.flags, pStats);
16759
16760 (*pContext)->AddPools(info.poolCount, info.pPools);
16761 (*pContext)->AddAllocations(
16762 info.allocationCount, info.pAllocations, info.pAllocationsChanged);
16763
16764 VkResult res = (*pContext)->Defragment(
16765 info.maxCpuBytesToMove, info.maxCpuAllocationsToMove,
16766 info.maxGpuBytesToMove, info.maxGpuAllocationsToMove,
16767 info.commandBuffer, pStats, info.flags);
16768
16769 if(res != VK_NOT_READY)
16770 {
16771 vma_delete(this, *pContext);
16772 *pContext = VMA_NULL;
16773 }
16774
16775 return res;
16776 }
16777
16778 VkResult VmaAllocator_T::DefragmentationEnd(
16779 VmaDefragmentationContext context)
16780 {
16781 vma_delete(this, context);
16782 return VK_SUCCESS;
16783 }
16784
16785 VkResult VmaAllocator_T::DefragmentationPassBegin(
16786 VmaDefragmentationPassInfo* pInfo,
16787 VmaDefragmentationContext context)
16788 {
16789 return context->DefragmentPassBegin(pInfo);
16790 }
16791 VkResult VmaAllocator_T::DefragmentationPassEnd(
16792 VmaDefragmentationContext context)
16793 {
16794 return context->DefragmentPassEnd();
16795
16796 }
16797
16798 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
16799 {
16800 if(hAllocation->CanBecomeLost())
16801 {
16802 /*
16803 Warning: This is a carefully designed algorithm.
16804 Do not modify unless you really know what you're doing :)
16805 */
16806 const uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
16807 uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
16808 for(;;)
16809 {
16810 if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
16811 {
16812 pAllocationInfo->memoryType = UINT32_MAX;
16813 pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
16814 pAllocationInfo->offset = 0;
16815 pAllocationInfo->size = hAllocation->GetSize();
16816 pAllocationInfo->pMappedData = VMA_NULL;
16817 pAllocationInfo->pUserData = hAllocation->GetUserData();
16818 return;
16819 }
16820 else if(localLastUseFrameIndex == localCurrFrameIndex)
16821 {
16822 pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
16823 pAllocationInfo->deviceMemory = hAllocation->GetMemory();
16824 pAllocationInfo->offset = hAllocation->GetOffset();
16825 pAllocationInfo->size = hAllocation->GetSize();
16826 pAllocationInfo->pMappedData = VMA_NULL;
16827 pAllocationInfo->pUserData = hAllocation->GetUserData();
16828 return;
16829 }
16830 else // Last use time earlier than current time.
16831 {
16832 if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
16833 {
16834 localLastUseFrameIndex = localCurrFrameIndex;
16835 }
16836 }
16837 }
16838 }
16839 else
16840 {
16841 #if VMA_STATS_STRING_ENABLED
16842 uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
16843 uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
16844 for(;;)
16845 {
16846 VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
16847 if(localLastUseFrameIndex == localCurrFrameIndex)
16848 {
16849 break;
16850 }
16851 else // Last use time earlier than current time.
16852 {
16853 if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
16854 {
16855 localLastUseFrameIndex = localCurrFrameIndex;
16856 }
16857 }
16858 }
16859 #endif
16860
16861 pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
16862 pAllocationInfo->deviceMemory = hAllocation->GetMemory();
16863 pAllocationInfo->offset = hAllocation->GetOffset();
16864 pAllocationInfo->size = hAllocation->GetSize();
16865 pAllocationInfo->pMappedData = hAllocation->GetMappedData();
16866 pAllocationInfo->pUserData = hAllocation->GetUserData();
16867 }
16868 }
16869
16870 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
16871 {
16872 // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
16873 if(hAllocation->CanBecomeLost())
16874 {
16875 uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
16876 uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
16877 for(;;)
16878 {
16879 if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
16880 {
16881 return false;
16882 }
16883 else if(localLastUseFrameIndex == localCurrFrameIndex)
16884 {
16885 return true;
16886 }
16887 else // Last use time earlier than current time.
16888 {
16889 if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
16890 {
16891 localLastUseFrameIndex = localCurrFrameIndex;
16892 }
16893 }
16894 }
16895 }
16896 else
16897 {
16898 #if VMA_STATS_STRING_ENABLED
16899 uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
16900 uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
16901 for(;;)
16902 {
16903 VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
16904 if(localLastUseFrameIndex == localCurrFrameIndex)
16905 {
16906 break;
16907 }
16908 else // Last use time earlier than current time.
16909 {
16910 if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
16911 {
16912 localLastUseFrameIndex = localCurrFrameIndex;
16913 }
16914 }
16915 }
16916 #endif
16917
16918 return true;
16919 }
16920 }
16921
16922 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
16923 {
16924 VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u, flags=%u", pCreateInfo->memoryTypeIndex, pCreateInfo->flags);
16925
16926 VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
16927
16928 if(newCreateInfo.maxBlockCount == 0)
16929 {
16930 newCreateInfo.maxBlockCount = SIZE_MAX;
16931 }
16932 if(newCreateInfo.minBlockCount > newCreateInfo.maxBlockCount)
16933 {
16934 return VK_ERROR_INITIALIZATION_FAILED;
16935 }
16936 // Memory type index out of range or forbidden.
16937 if(pCreateInfo->memoryTypeIndex >= GetMemoryTypeCount() ||
16938 ((1u << pCreateInfo->memoryTypeIndex) & m_GlobalMemoryTypeBits) == 0)
16939 {
16940 return VK_ERROR_FEATURE_NOT_PRESENT;
16941 }
16942
16943 const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
16944
16945 *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo, preferredBlockSize);
16946
16947 VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
16948 if(res != VK_SUCCESS)
16949 {
16950 vma_delete(this, *pPool);
16951 *pPool = VMA_NULL;
16952 return res;
16953 }
16954
16955 // Add to m_Pools.
16956 {
16957 VmaMutexLockWrite lock(m_PoolsMutex, m_UseMutex);
16958 (*pPool)->SetId(m_NextPoolId++);
16959 VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
16960 }
16961
16962 return VK_SUCCESS;
16963 }
16964
16965 void VmaAllocator_T::DestroyPool(VmaPool pool)
16966 {
16967 // Remove from m_Pools.
16968 {
16969 VmaMutexLockWrite lock(m_PoolsMutex, m_UseMutex);
16970 bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
16971 VMA_ASSERT(success && "Pool not found in Allocator.");
16972 }
16973
16974 vma_delete(this, pool);
16975 }
16976
16977 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
16978 {
16979 pool->m_BlockVector.GetPoolStats(pPoolStats);
16980 }
16981
16982 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
16983 {
16984 m_CurrentFrameIndex.store(frameIndex);
16985
16986 #if VMA_MEMORY_BUDGET
16987 if(m_UseExtMemoryBudget)
16988 {
16989 UpdateVulkanBudget();
16990 }
16991 #endif // #if VMA_MEMORY_BUDGET
16992 }
16993
16994 void VmaAllocator_T::MakePoolAllocationsLost(
16995 VmaPool hPool,
16996 size_t* pLostAllocationCount)
16997 {
16998 hPool->m_BlockVector.MakePoolAllocationsLost(
16999 m_CurrentFrameIndex.load(),
17000 pLostAllocationCount);
17001 }
17002
17003 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
17004 {
17005 return hPool->m_BlockVector.CheckCorruption();
17006 }
17007
17008 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
17009 {
17010 VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
17011
17012 // Process default pools.
17013 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
17014 {
17015 if(((1u << memTypeIndex) & memoryTypeBits) != 0)
17016 {
17017 VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
17018 VMA_ASSERT(pBlockVector);
17019 VkResult localRes = pBlockVector->CheckCorruption();
17020 switch(localRes)
17021 {
17022 case VK_ERROR_FEATURE_NOT_PRESENT:
17023 break;
17024 case VK_SUCCESS:
17025 finalRes = VK_SUCCESS;
17026 break;
17027 default:
17028 return localRes;
17029 }
17030 }
17031 }
17032
17033 // Process custom pools.
17034 {
17035 VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex);
17036 for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
17037 {
17038 if(((1u << m_Pools[poolIndex]->m_BlockVector.GetMemoryTypeIndex()) & memoryTypeBits) != 0)
17039 {
17040 VkResult localRes = m_Pools[poolIndex]->m_BlockVector.CheckCorruption();
17041 switch(localRes)
17042 {
17043 case VK_ERROR_FEATURE_NOT_PRESENT:
17044 break;
17045 case VK_SUCCESS:
17046 finalRes = VK_SUCCESS;
17047 break;
17048 default:
17049 return localRes;
17050 }
17051 }
17052 }
17053 }
17054
17055 return finalRes;
17056 }
17057
17058 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
17059 {
17060 *pAllocation = m_AllocationObjectAllocator.Allocate(VMA_FRAME_INDEX_LOST, false);
17061 (*pAllocation)->InitLost();
17062 }
17063
17064 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
17065 {
17066 const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
17067
17068 // HeapSizeLimit is in effect for this heap.
17069 if((m_HeapSizeLimitMask & (1u << heapIndex)) != 0)
17070 {
17071 const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
17072 VkDeviceSize blockBytes = m_Budget.m_BlockBytes[heapIndex];
17073 for(;;)
17074 {
17075 const VkDeviceSize blockBytesAfterAllocation = blockBytes + pAllocateInfo->allocationSize;
17076 if(blockBytesAfterAllocation > heapSize)
17077 {
17078 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
17079 }
17080 if(m_Budget.m_BlockBytes[heapIndex].compare_exchange_strong(blockBytes, blockBytesAfterAllocation))
17081 {
17082 break;
17083 }
17084 }
17085 }
17086 else
17087 {
17088 m_Budget.m_BlockBytes[heapIndex] += pAllocateInfo->allocationSize;
17089 }
17090
17091 // VULKAN CALL vkAllocateMemory.
17092 VkResult res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
17093
17094 if(res == VK_SUCCESS)
17095 {
17096 #if VMA_MEMORY_BUDGET
17097 ++m_Budget.m_OperationsSinceBudgetFetch;
17098 #endif
17099
17100 // Informative callback.
17101 if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
17102 {
17103 (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize, m_DeviceMemoryCallbacks.pUserData);
17104 }
17105 }
17106 else
17107 {
17108 m_Budget.m_BlockBytes[heapIndex] -= pAllocateInfo->allocationSize;
17109 }
17110
17111 return res;
17112 }
17113
17114 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
17115 {
17116 // Informative callback.
17117 if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
17118 {
17119 (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size, m_DeviceMemoryCallbacks.pUserData);
17120 }
17121
17122 // VULKAN CALL vkFreeMemory.
17123 (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
17124
17125 m_Budget.m_BlockBytes[MemoryTypeIndexToHeapIndex(memoryType)] -= size;
17126 }
17127
17128 VkResult VmaAllocator_T::BindVulkanBuffer(
17129 VkDeviceMemory memory,
17130 VkDeviceSize memoryOffset,
17131 VkBuffer buffer,
17132 const void* pNext)
17133 {
17134 if(pNext != VMA_NULL)
17135 {
17136 #if VMA_VULKAN_VERSION >= 1001000 || VMA_BIND_MEMORY2
17137 if((m_UseKhrBindMemory2 || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) &&
17138 m_VulkanFunctions.vkBindBufferMemory2KHR != VMA_NULL)
17139 {
17140 VkBindBufferMemoryInfoKHR bindBufferMemoryInfo = { VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO_KHR };
17141 bindBufferMemoryInfo.pNext = pNext;
17142 bindBufferMemoryInfo.buffer = buffer;
17143 bindBufferMemoryInfo.memory = memory;
17144 bindBufferMemoryInfo.memoryOffset = memoryOffset;
17145 return (*m_VulkanFunctions.vkBindBufferMemory2KHR)(m_hDevice, 1, &bindBufferMemoryInfo);
17146 }
17147 else
17148 #endif // #if VMA_VULKAN_VERSION >= 1001000 || VMA_BIND_MEMORY2
17149 {
17150 return VK_ERROR_EXTENSION_NOT_PRESENT;
17151 }
17152 }
17153 else
17154 {
17155 return (*m_VulkanFunctions.vkBindBufferMemory)(m_hDevice, buffer, memory, memoryOffset);
17156 }
17157 }
17158
17159 VkResult VmaAllocator_T::BindVulkanImage(
17160 VkDeviceMemory memory,
17161 VkDeviceSize memoryOffset,
17162 VkImage image,
17163 const void* pNext)
17164 {
17165 if(pNext != VMA_NULL)
17166 {
17167 #if VMA_VULKAN_VERSION >= 1001000 || VMA_BIND_MEMORY2
17168 if((m_UseKhrBindMemory2 || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) &&
17169 m_VulkanFunctions.vkBindImageMemory2KHR != VMA_NULL)
17170 {
17171 VkBindImageMemoryInfoKHR bindBufferMemoryInfo = { VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO_KHR };
17172 bindBufferMemoryInfo.pNext = pNext;
17173 bindBufferMemoryInfo.image = image;
17174 bindBufferMemoryInfo.memory = memory;
17175 bindBufferMemoryInfo.memoryOffset = memoryOffset;
17176 return (*m_VulkanFunctions.vkBindImageMemory2KHR)(m_hDevice, 1, &bindBufferMemoryInfo);
17177 }
17178 else
17179 #endif // #if VMA_BIND_MEMORY2
17180 {
17181 return VK_ERROR_EXTENSION_NOT_PRESENT;
17182 }
17183 }
17184 else
17185 {
17186 return (*m_VulkanFunctions.vkBindImageMemory)(m_hDevice, image, memory, memoryOffset);
17187 }
17188 }
17189
17190 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
17191 {
17192 if(hAllocation->CanBecomeLost())
17193 {
17194 return VK_ERROR_MEMORY_MAP_FAILED;
17195 }
17196
17197 switch(hAllocation->GetType())
17198 {
17199 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
17200 {
17201 VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
17202 char *pBytes = VMA_NULL;
17203 VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
17204 if(res == VK_SUCCESS)
17205 {
17206 *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
17207 hAllocation->BlockAllocMap();
17208 }
17209 return res;
17210 }
17211 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
17212 return hAllocation->DedicatedAllocMap(this, ppData);
17213 default:
17214 VMA_ASSERT(0);
17215 return VK_ERROR_MEMORY_MAP_FAILED;
17216 }
17217 }
17218
17219 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
17220 {
17221 switch(hAllocation->GetType())
17222 {
17223 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
17224 {
17225 VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
17226 hAllocation->BlockAllocUnmap();
17227 pBlock->Unmap(this, 1);
17228 }
17229 break;
17230 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
17231 hAllocation->DedicatedAllocUnmap(this);
17232 break;
17233 default:
17234 VMA_ASSERT(0);
17235 }
17236 }
17237
17238 VkResult VmaAllocator_T::BindBufferMemory(
17239 VmaAllocation hAllocation,
17240 VkDeviceSize allocationLocalOffset,
17241 VkBuffer hBuffer,
17242 const void* pNext)
17243 {
17244 VkResult res = VK_SUCCESS;
17245 switch(hAllocation->GetType())
17246 {
17247 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
17248 res = BindVulkanBuffer(hAllocation->GetMemory(), allocationLocalOffset, hBuffer, pNext);
17249 break;
17250 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
17251 {
17252 VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
17253 VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
17254 res = pBlock->BindBufferMemory(this, hAllocation, allocationLocalOffset, hBuffer, pNext);
17255 break;
17256 }
17257 default:
17258 VMA_ASSERT(0);
17259 }
17260 return res;
17261 }
17262
17263 VkResult VmaAllocator_T::BindImageMemory(
17264 VmaAllocation hAllocation,
17265 VkDeviceSize allocationLocalOffset,
17266 VkImage hImage,
17267 const void* pNext)
17268 {
17269 VkResult res = VK_SUCCESS;
17270 switch(hAllocation->GetType())
17271 {
17272 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
17273 res = BindVulkanImage(hAllocation->GetMemory(), allocationLocalOffset, hImage, pNext);
17274 break;
17275 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
17276 {
17277 VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
17278 VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
17279 res = pBlock->BindImageMemory(this, hAllocation, allocationLocalOffset, hImage, pNext);
17280 break;
17281 }
17282 default:
17283 VMA_ASSERT(0);
17284 }
17285 return res;
17286 }
17287
17288 VkResult VmaAllocator_T::FlushOrInvalidateAllocation(
17289 VmaAllocation hAllocation,
17290 VkDeviceSize offset, VkDeviceSize size,
17291 VMA_CACHE_OPERATION op)
17292 {
17293 VkResult res = VK_SUCCESS;
17294
17295 VkMappedMemoryRange memRange = {};
17296 if(GetFlushOrInvalidateRange(hAllocation, offset, size, memRange))
17297 {
17298 switch(op)
17299 {
17300 case VMA_CACHE_FLUSH:
17301 res = (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
17302 break;
17303 case VMA_CACHE_INVALIDATE:
17304 res = (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
17305 break;
17306 default:
17307 VMA_ASSERT(0);
17308 }
17309 }
17310 // else: Just ignore this call.
17311 return res;
17312 }
17313
17314 VkResult VmaAllocator_T::FlushOrInvalidateAllocations(
17315 uint32_t allocationCount,
17316 const VmaAllocation* allocations,
17317 const VkDeviceSize* offsets, const VkDeviceSize* sizes,
17318 VMA_CACHE_OPERATION op)
17319 {
17320 typedef VmaStlAllocator<VkMappedMemoryRange> RangeAllocator;
17321 typedef VmaSmallVector<VkMappedMemoryRange, RangeAllocator, 16> RangeVector;
17322 RangeVector ranges = RangeVector(RangeAllocator(GetAllocationCallbacks()));
17323
17324 for(uint32_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
17325 {
17326 const VmaAllocation alloc = allocations[allocIndex];
17327 const VkDeviceSize offset = offsets != VMA_NULL ? offsets[allocIndex] : 0;
17328 const VkDeviceSize size = sizes != VMA_NULL ? sizes[allocIndex] : VK_WHOLE_SIZE;
17329 VkMappedMemoryRange newRange;
17330 if(GetFlushOrInvalidateRange(alloc, offset, size, newRange))
17331 {
17332 ranges.push_back(newRange);
17333 }
17334 }
17335
17336 VkResult res = VK_SUCCESS;
17337 if(!ranges.empty())
17338 {
17339 switch(op)
17340 {
17341 case VMA_CACHE_FLUSH:
17342 res = (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, (uint32_t)ranges.size(), ranges.data());
17343 break;
17344 case VMA_CACHE_INVALIDATE:
17345 res = (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, (uint32_t)ranges.size(), ranges.data());
17346 break;
17347 default:
17348 VMA_ASSERT(0);
17349 }
17350 }
17351 // else: Just ignore this call.
17352 return res;
17353 }
17354
17355 void VmaAllocator_T::FreeDedicatedMemory(const VmaAllocation allocation)
17356 {
17357 VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
17358
17359 const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
17360 {
17361 VmaMutexLockWrite lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
17362 AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
17363 VMA_ASSERT(pDedicatedAllocations);
17364 bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
17365 VMA_ASSERT(success);
17366 }
17367
17368 VkDeviceMemory hMemory = allocation->GetMemory();
17369
17370 /*
17371 There is no need to call this, because Vulkan spec allows to skip vkUnmapMemory
17372 before vkFreeMemory.
17373
17374 if(allocation->GetMappedData() != VMA_NULL)
17375 {
17376 (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
17377 }
17378 */
17379
17380 FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
17381
17382 VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
17383 }
17384
17385 uint32_t VmaAllocator_T::CalculateGpuDefragmentationMemoryTypeBits() const
17386 {
17387 VkBufferCreateInfo dummyBufCreateInfo;
17388 VmaFillGpuDefragmentationBufferCreateInfo(dummyBufCreateInfo);
17389
17390 uint32_t memoryTypeBits = 0;
17391
17392 // Create buffer.
17393 VkBuffer buf = VK_NULL_HANDLE;
17394 VkResult res = (*GetVulkanFunctions().vkCreateBuffer)(
17395 m_hDevice, &dummyBufCreateInfo, GetAllocationCallbacks(), &buf);
17396 if(res == VK_SUCCESS)
17397 {
17398 // Query for supported memory types.
17399 VkMemoryRequirements memReq;
17400 (*GetVulkanFunctions().vkGetBufferMemoryRequirements)(m_hDevice, buf, &memReq);
17401 memoryTypeBits = memReq.memoryTypeBits;
17402
17403 // Destroy buffer.
17404 (*GetVulkanFunctions().vkDestroyBuffer)(m_hDevice, buf, GetAllocationCallbacks());
17405 }
17406
17407 return memoryTypeBits;
17408 }
17409
17410 uint32_t VmaAllocator_T::CalculateGlobalMemoryTypeBits() const
17411 {
17412 // Make sure memory information is already fetched.
17413 VMA_ASSERT(GetMemoryTypeCount() > 0);
17414
17415 uint32_t memoryTypeBits = UINT32_MAX;
17416
17417 if(!m_UseAmdDeviceCoherentMemory)
17418 {
17419 // Exclude memory types that have VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD.
17420 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
17421 {
17422 if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY) != 0)
17423 {
17424 memoryTypeBits &= ~(1u << memTypeIndex);
17425 }
17426 }
17427 }
17428
17429 return memoryTypeBits;
17430 }
17431
17432 bool VmaAllocator_T::GetFlushOrInvalidateRange(
17433 VmaAllocation allocation,
17434 VkDeviceSize offset, VkDeviceSize size,
17435 VkMappedMemoryRange& outRange) const
17436 {
17437 const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
17438 if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
17439 {
17440 const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
17441 const VkDeviceSize allocationSize = allocation->GetSize();
17442 VMA_ASSERT(offset <= allocationSize);
17443
17444 outRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
17445 outRange.pNext = VMA_NULL;
17446 outRange.memory = allocation->GetMemory();
17447
17448 switch(allocation->GetType())
17449 {
17450 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
17451 outRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
17452 if(size == VK_WHOLE_SIZE)
17453 {
17454 outRange.size = allocationSize - outRange.offset;
17455 }
17456 else
17457 {
17458 VMA_ASSERT(offset + size <= allocationSize);
17459 outRange.size = VMA_MIN(
17460 VmaAlignUp(size + (offset - outRange.offset), nonCoherentAtomSize),
17461 allocationSize - outRange.offset);
17462 }
17463 break;
17464 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
17465 {
17466 // 1. Still within this allocation.
17467 outRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
17468 if(size == VK_WHOLE_SIZE)
17469 {
17470 size = allocationSize - offset;
17471 }
17472 else
17473 {
17474 VMA_ASSERT(offset + size <= allocationSize);
17475 }
17476 outRange.size = VmaAlignUp(size + (offset - outRange.offset), nonCoherentAtomSize);
17477
17478 // 2. Adjust to whole block.
17479 const VkDeviceSize allocationOffset = allocation->GetOffset();
17480 VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
17481 const VkDeviceSize blockSize = allocation->GetBlock()->m_pMetadata->GetSize();
17482 outRange.offset += allocationOffset;
17483 outRange.size = VMA_MIN(outRange.size, blockSize - outRange.offset);
17484
17485 break;
17486 }
17487 default:
17488 VMA_ASSERT(0);
17489 }
17490 return true;
17491 }
17492 return false;
17493 }
17494
17495 #if VMA_MEMORY_BUDGET
17496
17497 void VmaAllocator_T::UpdateVulkanBudget()
17498 {
17499 VMA_ASSERT(m_UseExtMemoryBudget);
17500
17501 VkPhysicalDeviceMemoryProperties2KHR memProps = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2_KHR };
17502
17503 VkPhysicalDeviceMemoryBudgetPropertiesEXT budgetProps = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT };
17504 VmaPnextChainPushFront(&memProps, &budgetProps);
17505
17506 GetVulkanFunctions().vkGetPhysicalDeviceMemoryProperties2KHR(m_PhysicalDevice, &memProps);
17507
17508 {
17509 VmaMutexLockWrite lockWrite(m_Budget.m_BudgetMutex, m_UseMutex);
17510
17511 for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
17512 {
17513 m_Budget.m_VulkanUsage[heapIndex] = budgetProps.heapUsage[heapIndex];
17514 m_Budget.m_VulkanBudget[heapIndex] = budgetProps.heapBudget[heapIndex];
17515 m_Budget.m_BlockBytesAtBudgetFetch[heapIndex] = m_Budget.m_BlockBytes[heapIndex].load();
17516
17517 // Some bugged drivers return the budget incorrectly, e.g. 0 or much bigger than heap size.
17518 if(m_Budget.m_VulkanBudget[heapIndex] == 0)
17519 {
17520 m_Budget.m_VulkanBudget[heapIndex] = m_MemProps.memoryHeaps[heapIndex].size * 8 / 10; // 80% heuristics.
17521 }
17522 else if(m_Budget.m_VulkanBudget[heapIndex] > m_MemProps.memoryHeaps[heapIndex].size)
17523 {
17524 m_Budget.m_VulkanBudget[heapIndex] = m_MemProps.memoryHeaps[heapIndex].size;
17525 }
17526 if(m_Budget.m_VulkanUsage[heapIndex] == 0 && m_Budget.m_BlockBytesAtBudgetFetch[heapIndex] > 0)
17527 {
17528 m_Budget.m_VulkanUsage[heapIndex] = m_Budget.m_BlockBytesAtBudgetFetch[heapIndex];
17529 }
17530 }
17531 m_Budget.m_OperationsSinceBudgetFetch = 0;
17532 }
17533 }
17534
17535 #endif // #if VMA_MEMORY_BUDGET
17536
17537 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
17538 {
17539 if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
17540 !hAllocation->CanBecomeLost() &&
17541 (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
17542 {
17543 void* pData = VMA_NULL;
17544 VkResult res = Map(hAllocation, &pData);
17545 if(res == VK_SUCCESS)
17546 {
17547 memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
17548 FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
17549 Unmap(hAllocation);
17550 }
17551 else
17552 {
17553 VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
17554 }
17555 }
17556 }
17557
17558 uint32_t VmaAllocator_T::GetGpuDefragmentationMemoryTypeBits()
17559 {
17560 uint32_t memoryTypeBits = m_GpuDefragmentationMemoryTypeBits.load();
17561 if(memoryTypeBits == UINT32_MAX)
17562 {
17563 memoryTypeBits = CalculateGpuDefragmentationMemoryTypeBits();
17564 m_GpuDefragmentationMemoryTypeBits.store(memoryTypeBits);
17565 }
17566 return memoryTypeBits;
17567 }
17568
17569 #if VMA_STATS_STRING_ENABLED
17570
17571 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
17572 {
17573 bool dedicatedAllocationsStarted = false;
17574 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
17575 {
17576 VmaMutexLockRead dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
17577 AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
17578 VMA_ASSERT(pDedicatedAllocVector);
17579 if(pDedicatedAllocVector->empty() == false)
17580 {
17581 if(dedicatedAllocationsStarted == false)
17582 {
17583 dedicatedAllocationsStarted = true;
17584 json.WriteString("DedicatedAllocations");
17585 json.BeginObject();
17586 }
17587
17588 json.BeginString("Type ");
17589 json.ContinueString(memTypeIndex);
17590 json.EndString();
17591
17592 json.BeginArray();
17593
17594 for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
17595 {
17596 json.BeginObject(true);
17597 const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
17598 hAlloc->PrintParameters(json);
17599 json.EndObject();
17600 }
17601
17602 json.EndArray();
17603 }
17604 }
17605 if(dedicatedAllocationsStarted)
17606 {
17607 json.EndObject();
17608 }
17609
17610 {
17611 bool allocationsStarted = false;
17612 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
17613 {
17614 if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
17615 {
17616 if(allocationsStarted == false)
17617 {
17618 allocationsStarted = true;
17619 json.WriteString("DefaultPools");
17620 json.BeginObject();
17621 }
17622
17623 json.BeginString("Type ");
17624 json.ContinueString(memTypeIndex);
17625 json.EndString();
17626
17627 m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
17628 }
17629 }
17630 if(allocationsStarted)
17631 {
17632 json.EndObject();
17633 }
17634 }
17635
17636 // Custom pools
17637 {
17638 VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex);
17639 const size_t poolCount = m_Pools.size();
17640 if(poolCount > 0)
17641 {
17642 json.WriteString("Pools");
17643 json.BeginObject();
17644 for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
17645 {
17646 json.BeginString();
17647 json.ContinueString(m_Pools[poolIndex]->GetId());
17648 json.EndString();
17649
17650 m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
17651 }
17652 json.EndObject();
17653 }
17654 }
17655 }
17656
17657 #endif // #if VMA_STATS_STRING_ENABLED
17658
17659 ////////////////////////////////////////////////////////////////////////////////
17660 // Public interface
17661
17662 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAllocator(
17663 const VmaAllocatorCreateInfo* pCreateInfo,
17664 VmaAllocator* pAllocator)
17665 {
17666 VMA_ASSERT(pCreateInfo && pAllocator);
17667 VMA_ASSERT(pCreateInfo->vulkanApiVersion == 0 ||
17668 (VK_VERSION_MAJOR(pCreateInfo->vulkanApiVersion) == 1 && VK_VERSION_MINOR(pCreateInfo->vulkanApiVersion) <= 2));
17669 VMA_DEBUG_LOG("vmaCreateAllocator");
17670 *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
17671 return (*pAllocator)->Init(pCreateInfo);
17672 }
17673
17674 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyAllocator(
17675 VmaAllocator allocator)
17676 {
17677 if(allocator != VK_NULL_HANDLE)
17678 {
17679 VMA_DEBUG_LOG("vmaDestroyAllocator");
17680 VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
17681 vma_delete(&allocationCallbacks, allocator);
17682 }
17683 }
17684
17685 VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocatorInfo(VmaAllocator allocator, VmaAllocatorInfo* pAllocatorInfo)
17686 {
17687 VMA_ASSERT(allocator && pAllocatorInfo);
17688 pAllocatorInfo->instance = allocator->m_hInstance;
17689 pAllocatorInfo->physicalDevice = allocator->GetPhysicalDevice();
17690 pAllocatorInfo->device = allocator->m_hDevice;
17691 }
17692
17693 VMA_CALL_PRE void VMA_CALL_POST vmaGetPhysicalDeviceProperties(
17694 VmaAllocator allocator,
17695 const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
17696 {
17697 VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
17698 *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
17699 }
17700
17701 VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryProperties(
17702 VmaAllocator allocator,
17703 const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
17704 {
17705 VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
17706 *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
17707 }
17708
17709 VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryTypeProperties(
17710 VmaAllocator allocator,
17711 uint32_t memoryTypeIndex,
17712 VkMemoryPropertyFlags* pFlags)
17713 {
17714 VMA_ASSERT(allocator && pFlags);
17715 VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
17716 *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
17717 }
17718
17719 VMA_CALL_PRE void VMA_CALL_POST vmaSetCurrentFrameIndex(
17720 VmaAllocator allocator,
17721 uint32_t frameIndex)
17722 {
17723 VMA_ASSERT(allocator);
17724 VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
17725
17726 VMA_DEBUG_GLOBAL_MUTEX_LOCK
17727
17728 allocator->SetCurrentFrameIndex(frameIndex);
17729 }
17730
17731 VMA_CALL_PRE void VMA_CALL_POST vmaCalculateStats(
17732 VmaAllocator allocator,
17733 VmaStats* pStats)
17734 {
17735 VMA_ASSERT(allocator && pStats);
17736 VMA_DEBUG_GLOBAL_MUTEX_LOCK
17737 allocator->CalculateStats(pStats);
17738 }
17739
17740 VMA_CALL_PRE void VMA_CALL_POST vmaGetBudget(
17741 VmaAllocator allocator,
17742 VmaBudget* pBudget)
17743 {
17744 VMA_ASSERT(allocator && pBudget);
17745 VMA_DEBUG_GLOBAL_MUTEX_LOCK
17746 allocator->GetBudget(pBudget, 0, allocator->GetMemoryHeapCount());
17747 }
17748
17749 #if VMA_STATS_STRING_ENABLED
17750
17751 VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString(
17752 VmaAllocator allocator,
17753 char** ppStatsString,
17754 VkBool32 detailedMap)
17755 {
17756 VMA_ASSERT(allocator && ppStatsString);
17757 VMA_DEBUG_GLOBAL_MUTEX_LOCK
17758
17759 VmaStringBuilder sb(allocator);
17760 {
17761 VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
17762 json.BeginObject();
17763
17764 VmaBudget budget[VK_MAX_MEMORY_HEAPS];
17765 allocator->GetBudget(budget, 0, allocator->GetMemoryHeapCount());
17766
17767 VmaStats stats;
17768 allocator->CalculateStats(&stats);
17769
17770 json.WriteString("Total");
17771 VmaPrintStatInfo(json, stats.total);
17772
17773 for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
17774 {
17775 json.BeginString("Heap ");
17776 json.ContinueString(heapIndex);
17777 json.EndString();
17778 json.BeginObject();
17779
17780 json.WriteString("Size");
17781 json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
17782
17783 json.WriteString("Flags");
17784 json.BeginArray(true);
17785 if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
17786 {
17787 json.WriteString("DEVICE_LOCAL");
17788 }
17789 json.EndArray();
17790
17791 json.WriteString("Budget");
17792 json.BeginObject();
17793 {
17794 json.WriteString("BlockBytes");
17795 json.WriteNumber(budget[heapIndex].blockBytes);
17796 json.WriteString("AllocationBytes");
17797 json.WriteNumber(budget[heapIndex].allocationBytes);
17798 json.WriteString("Usage");
17799 json.WriteNumber(budget[heapIndex].usage);
17800 json.WriteString("Budget");
17801 json.WriteNumber(budget[heapIndex].budget);
17802 }
17803 json.EndObject();
17804
17805 if(stats.memoryHeap[heapIndex].blockCount > 0)
17806 {
17807 json.WriteString("Stats");
17808 VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
17809 }
17810
17811 for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
17812 {
17813 if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
17814 {
17815 json.BeginString("Type ");
17816 json.ContinueString(typeIndex);
17817 json.EndString();
17818
17819 json.BeginObject();
17820
17821 json.WriteString("Flags");
17822 json.BeginArray(true);
17823 VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
17824 if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
17825 {
17826 json.WriteString("DEVICE_LOCAL");
17827 }
17828 if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
17829 {
17830 json.WriteString("HOST_VISIBLE");
17831 }
17832 if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
17833 {
17834 json.WriteString("HOST_COHERENT");
17835 }
17836 if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
17837 {
17838 json.WriteString("HOST_CACHED");
17839 }
17840 if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
17841 {
17842 json.WriteString("LAZILY_ALLOCATED");
17843 }
17844 if((flags & VK_MEMORY_PROPERTY_PROTECTED_BIT) != 0)
17845 {
17846 json.WriteString(" PROTECTED");
17847 }
17848 if((flags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY) != 0)
17849 {
17850 json.WriteString(" DEVICE_COHERENT");
17851 }
17852 if((flags & VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY) != 0)
17853 {
17854 json.WriteString(" DEVICE_UNCACHED");
17855 }
17856 json.EndArray();
17857
17858 if(stats.memoryType[typeIndex].blockCount > 0)
17859 {
17860 json.WriteString("Stats");
17861 VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
17862 }
17863
17864 json.EndObject();
17865 }
17866 }
17867
17868 json.EndObject();
17869 }
17870 if(detailedMap == VK_TRUE)
17871 {
17872 allocator->PrintDetailedMap(json);
17873 }
17874
17875 json.EndObject();
17876 }
17877
17878 const size_t len = sb.GetLength();
17879 char* const pChars = vma_new_array(allocator, char, len + 1);
17880 if(len > 0)
17881 {
17882 memcpy(pChars, sb.GetData(), len);
17883 }
17884 pChars[len] = '\0';
17885 *ppStatsString = pChars;
17886 }
17887
17888 VMA_CALL_PRE void VMA_CALL_POST vmaFreeStatsString(
17889 VmaAllocator allocator,
17890 char* pStatsString)
17891 {
17892 if(pStatsString != VMA_NULL)
17893 {
17894 VMA_ASSERT(allocator);
17895 size_t len = strlen(pStatsString);
17896 vma_delete_array(allocator, pStatsString, len + 1);
17897 }
17898 }
17899
17900 #endif // #if VMA_STATS_STRING_ENABLED
17901
17902 /*
17903 This function is not protected by any mutex because it just reads immutable data.
17904 */
17905 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex(
17906 VmaAllocator allocator,
17907 uint32_t memoryTypeBits,
17908 const VmaAllocationCreateInfo* pAllocationCreateInfo,
17909 uint32_t* pMemoryTypeIndex)
17910 {
17911 VMA_ASSERT(allocator != VK_NULL_HANDLE);
17912 VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
17913 VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
17914
17915 memoryTypeBits &= allocator->GetGlobalMemoryTypeBits();
17916
17917 if(pAllocationCreateInfo->memoryTypeBits != 0)
17918 {
17919 memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
17920 }
17921
17922 uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
17923 uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
17924 uint32_t notPreferredFlags = 0;
17925
17926 // Convert usage to requiredFlags and preferredFlags.
17927 switch(pAllocationCreateInfo->usage)
17928 {
17929 case VMA_MEMORY_USAGE_UNKNOWN:
17930 break;
17931 case VMA_MEMORY_USAGE_GPU_ONLY:
17932 if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
17933 {
17934 preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
17935 }
17936 break;
17937 case VMA_MEMORY_USAGE_CPU_ONLY:
17938 requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
17939 break;
17940 case VMA_MEMORY_USAGE_CPU_TO_GPU:
17941 requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
17942 if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
17943 {
17944 preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
17945 }
17946 break;
17947 case VMA_MEMORY_USAGE_GPU_TO_CPU:
17948 requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
17949 preferredFlags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
17950 break;
17951 case VMA_MEMORY_USAGE_CPU_COPY:
17952 notPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
17953 break;
17954 case VMA_MEMORY_USAGE_GPU_LAZILY_ALLOCATED:
17955 requiredFlags |= VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT;
17956 break;
17957 default:
17958 VMA_ASSERT(0);
17959 break;
17960 }
17961
17962 // Avoid DEVICE_COHERENT unless explicitly requested.
17963 if(((pAllocationCreateInfo->requiredFlags | pAllocationCreateInfo->preferredFlags) &
17964 (VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY | VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY)) == 0)
17965 {
17966 notPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY;
17967 }
17968
17969 *pMemoryTypeIndex = UINT32_MAX;
17970 uint32_t minCost = UINT32_MAX;
17971 for(uint32_t memTypeIndex = 0, memTypeBit = 1;
17972 memTypeIndex < allocator->GetMemoryTypeCount();
17973 ++memTypeIndex, memTypeBit <<= 1)
17974 {
17975 // This memory type is acceptable according to memoryTypeBits bitmask.
17976 if((memTypeBit & memoryTypeBits) != 0)
17977 {
17978 const VkMemoryPropertyFlags currFlags =
17979 allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
17980 // This memory type contains requiredFlags.
17981 if((requiredFlags & ~currFlags) == 0)
17982 {
17983 // Calculate cost as number of bits from preferredFlags not present in this memory type.
17984 uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags) +
17985 VmaCountBitsSet(currFlags & notPreferredFlags);
17986 // Remember memory type with lowest cost.
17987 if(currCost < minCost)
17988 {
17989 *pMemoryTypeIndex = memTypeIndex;
17990 if(currCost == 0)
17991 {
17992 return VK_SUCCESS;
17993 }
17994 minCost = currCost;
17995 }
17996 }
17997 }
17998 }
17999 return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
18000 }
18001
18002 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForBufferInfo(
18003 VmaAllocator allocator,
18004 const VkBufferCreateInfo* pBufferCreateInfo,
18005 const VmaAllocationCreateInfo* pAllocationCreateInfo,
18006 uint32_t* pMemoryTypeIndex)
18007 {
18008 VMA_ASSERT(allocator != VK_NULL_HANDLE);
18009 VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
18010 VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
18011 VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
18012
18013 const VkDevice hDev = allocator->m_hDevice;
18014 VkBuffer hBuffer = VK_NULL_HANDLE;
18015 VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
18016 hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
18017 if(res == VK_SUCCESS)
18018 {
18019 VkMemoryRequirements memReq = {};
18020 allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
18021 hDev, hBuffer, &memReq);
18022
18023 res = vmaFindMemoryTypeIndex(
18024 allocator,
18025 memReq.memoryTypeBits,
18026 pAllocationCreateInfo,
18027 pMemoryTypeIndex);
18028
18029 allocator->GetVulkanFunctions().vkDestroyBuffer(
18030 hDev, hBuffer, allocator->GetAllocationCallbacks());
18031 }
18032 return res;
18033 }
18034
18035 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForImageInfo(
18036 VmaAllocator allocator,
18037 const VkImageCreateInfo* pImageCreateInfo,
18038 const VmaAllocationCreateInfo* pAllocationCreateInfo,
18039 uint32_t* pMemoryTypeIndex)
18040 {
18041 VMA_ASSERT(allocator != VK_NULL_HANDLE);
18042 VMA_ASSERT(pImageCreateInfo != VMA_NULL);
18043 VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
18044 VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
18045
18046 const VkDevice hDev = allocator->m_hDevice;
18047 VkImage hImage = VK_NULL_HANDLE;
18048 VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
18049 hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
18050 if(res == VK_SUCCESS)
18051 {
18052 VkMemoryRequirements memReq = {};
18053 allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
18054 hDev, hImage, &memReq);
18055
18056 res = vmaFindMemoryTypeIndex(
18057 allocator,
18058 memReq.memoryTypeBits,
18059 pAllocationCreateInfo,
18060 pMemoryTypeIndex);
18061
18062 allocator->GetVulkanFunctions().vkDestroyImage(
18063 hDev, hImage, allocator->GetAllocationCallbacks());
18064 }
18065 return res;
18066 }
18067
18068 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreatePool(
18069 VmaAllocator allocator,
18070 const VmaPoolCreateInfo* pCreateInfo,
18071 VmaPool* pPool)
18072 {
18073 VMA_ASSERT(allocator && pCreateInfo && pPool);
18074
18075 VMA_DEBUG_LOG("vmaCreatePool");
18076
18077 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18078
18079 VkResult res = allocator->CreatePool(pCreateInfo, pPool);
18080
18081 #if VMA_RECORDING_ENABLED
18082 if(allocator->GetRecorder() != VMA_NULL)
18083 {
18084 allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
18085 }
18086 #endif
18087
18088 return res;
18089 }
18090
18091 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool(
18092 VmaAllocator allocator,
18093 VmaPool pool)
18094 {
18095 VMA_ASSERT(allocator);
18096
18097 if(pool == VK_NULL_HANDLE)
18098 {
18099 return;
18100 }
18101
18102 VMA_DEBUG_LOG("vmaDestroyPool");
18103
18104 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18105
18106 #if VMA_RECORDING_ENABLED
18107 if(allocator->GetRecorder() != VMA_NULL)
18108 {
18109 allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
18110 }
18111 #endif
18112
18113 allocator->DestroyPool(pool);
18114 }
18115
18116 VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolStats(
18117 VmaAllocator allocator,
18118 VmaPool pool,
18119 VmaPoolStats* pPoolStats)
18120 {
18121 VMA_ASSERT(allocator && pool && pPoolStats);
18122
18123 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18124
18125 allocator->GetPoolStats(pool, pPoolStats);
18126 }
18127
18128 VMA_CALL_PRE void VMA_CALL_POST vmaMakePoolAllocationsLost(
18129 VmaAllocator allocator,
18130 VmaPool pool,
18131 size_t* pLostAllocationCount)
18132 {
18133 VMA_ASSERT(allocator && pool);
18134
18135 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18136
18137 #if VMA_RECORDING_ENABLED
18138 if(allocator->GetRecorder() != VMA_NULL)
18139 {
18140 allocator->GetRecorder()->RecordMakePoolAllocationsLost(allocator->GetCurrentFrameIndex(), pool);
18141 }
18142 #endif
18143
18144 allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
18145 }
18146
18147 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
18148 {
18149 VMA_ASSERT(allocator && pool);
18150
18151 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18152
18153 VMA_DEBUG_LOG("vmaCheckPoolCorruption");
18154
18155 return allocator->CheckPoolCorruption(pool);
18156 }
18157
18158 VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName(
18159 VmaAllocator allocator,
18160 VmaPool pool,
18161 const char** ppName)
18162 {
18163 VMA_ASSERT(allocator && pool && ppName);
18164
18165 VMA_DEBUG_LOG("vmaGetPoolName");
18166
18167 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18168
18169 *ppName = pool->GetName();
18170 }
18171
18172 VMA_CALL_PRE void VMA_CALL_POST vmaSetPoolName(
18173 VmaAllocator allocator,
18174 VmaPool pool,
18175 const char* pName)
18176 {
18177 VMA_ASSERT(allocator && pool);
18178
18179 VMA_DEBUG_LOG("vmaSetPoolName");
18180
18181 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18182
18183 pool->SetName(pName);
18184
18185 #if VMA_RECORDING_ENABLED
18186 if(allocator->GetRecorder() != VMA_NULL)
18187 {
18188 allocator->GetRecorder()->RecordSetPoolName(allocator->GetCurrentFrameIndex(), pool, pName);
18189 }
18190 #endif
18191 }
18192
18193 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory(
18194 VmaAllocator allocator,
18195 const VkMemoryRequirements* pVkMemoryRequirements,
18196 const VmaAllocationCreateInfo* pCreateInfo,
18197 VmaAllocation* pAllocation,
18198 VmaAllocationInfo* pAllocationInfo)
18199 {
18200 VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
18201
18202 VMA_DEBUG_LOG("vmaAllocateMemory");
18203
18204 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18205
18206 VkResult result = allocator->AllocateMemory(
18207 *pVkMemoryRequirements,
18208 false, // requiresDedicatedAllocation
18209 false, // prefersDedicatedAllocation
18210 VK_NULL_HANDLE, // dedicatedBuffer
18211 UINT32_MAX, // dedicatedBufferUsage
18212 VK_NULL_HANDLE, // dedicatedImage
18213 *pCreateInfo,
18214 VMA_SUBALLOCATION_TYPE_UNKNOWN,
18215 1, // allocationCount
18216 pAllocation);
18217
18218 #if VMA_RECORDING_ENABLED
18219 if(allocator->GetRecorder() != VMA_NULL)
18220 {
18221 allocator->GetRecorder()->RecordAllocateMemory(
18222 allocator->GetCurrentFrameIndex(),
18223 *pVkMemoryRequirements,
18224 *pCreateInfo,
18225 *pAllocation);
18226 }
18227 #endif
18228
18229 if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
18230 {
18231 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
18232 }
18233
18234 return result;
18235 }
18236
18237 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages(
18238 VmaAllocator allocator,
18239 const VkMemoryRequirements* pVkMemoryRequirements,
18240 const VmaAllocationCreateInfo* pCreateInfo,
18241 size_t allocationCount,
18242 VmaAllocation* pAllocations,
18243 VmaAllocationInfo* pAllocationInfo)
18244 {
18245 if(allocationCount == 0)
18246 {
18247 return VK_SUCCESS;
18248 }
18249
18250 VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocations);
18251
18252 VMA_DEBUG_LOG("vmaAllocateMemoryPages");
18253
18254 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18255
18256 VkResult result = allocator->AllocateMemory(
18257 *pVkMemoryRequirements,
18258 false, // requiresDedicatedAllocation
18259 false, // prefersDedicatedAllocation
18260 VK_NULL_HANDLE, // dedicatedBuffer
18261 UINT32_MAX, // dedicatedBufferUsage
18262 VK_NULL_HANDLE, // dedicatedImage
18263 *pCreateInfo,
18264 VMA_SUBALLOCATION_TYPE_UNKNOWN,
18265 allocationCount,
18266 pAllocations);
18267
18268 #if VMA_RECORDING_ENABLED
18269 if(allocator->GetRecorder() != VMA_NULL)
18270 {
18271 allocator->GetRecorder()->RecordAllocateMemoryPages(
18272 allocator->GetCurrentFrameIndex(),
18273 *pVkMemoryRequirements,
18274 *pCreateInfo,
18275 (uint64_t)allocationCount,
18276 pAllocations);
18277 }
18278 #endif
18279
18280 if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
18281 {
18282 for(size_t i = 0; i < allocationCount; ++i)
18283 {
18284 allocator->GetAllocationInfo(pAllocations[i], pAllocationInfo + i);
18285 }
18286 }
18287
18288 return result;
18289 }
18290
18291 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer(
18292 VmaAllocator allocator,
18293 VkBuffer buffer,
18294 const VmaAllocationCreateInfo* pCreateInfo,
18295 VmaAllocation* pAllocation,
18296 VmaAllocationInfo* pAllocationInfo)
18297 {
18298 VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
18299
18300 VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
18301
18302 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18303
18304 VkMemoryRequirements vkMemReq = {};
18305 bool requiresDedicatedAllocation = false;
18306 bool prefersDedicatedAllocation = false;
18307 allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
18308 requiresDedicatedAllocation,
18309 prefersDedicatedAllocation);
18310
18311 VkResult result = allocator->AllocateMemory(
18312 vkMemReq,
18313 requiresDedicatedAllocation,
18314 prefersDedicatedAllocation,
18315 buffer, // dedicatedBuffer
18316 UINT32_MAX, // dedicatedBufferUsage
18317 VK_NULL_HANDLE, // dedicatedImage
18318 *pCreateInfo,
18319 VMA_SUBALLOCATION_TYPE_BUFFER,
18320 1, // allocationCount
18321 pAllocation);
18322
18323 #if VMA_RECORDING_ENABLED
18324 if(allocator->GetRecorder() != VMA_NULL)
18325 {
18326 allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
18327 allocator->GetCurrentFrameIndex(),
18328 vkMemReq,
18329 requiresDedicatedAllocation,
18330 prefersDedicatedAllocation,
18331 *pCreateInfo,
18332 *pAllocation);
18333 }
18334 #endif
18335
18336 if(pAllocationInfo && result == VK_SUCCESS)
18337 {
18338 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
18339 }
18340
18341 return result;
18342 }
18343
18344 VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage(
18345 VmaAllocator allocator,
18346 VkImage image,
18347 const VmaAllocationCreateInfo* pCreateInfo,
18348 VmaAllocation* pAllocation,
18349 VmaAllocationInfo* pAllocationInfo)
18350 {
18351 VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
18352
18353 VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
18354
18355 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18356
18357 VkMemoryRequirements vkMemReq = {};
18358 bool requiresDedicatedAllocation = false;
18359 bool prefersDedicatedAllocation = false;
18360 allocator->GetImageMemoryRequirements(image, vkMemReq,
18361 requiresDedicatedAllocation, prefersDedicatedAllocation);
18362
18363 VkResult result = allocator->AllocateMemory(
18364 vkMemReq,
18365 requiresDedicatedAllocation,
18366 prefersDedicatedAllocation,
18367 VK_NULL_HANDLE, // dedicatedBuffer
18368 UINT32_MAX, // dedicatedBufferUsage
18369 image, // dedicatedImage
18370 *pCreateInfo,
18371 VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
18372 1, // allocationCount
18373 pAllocation);
18374
18375 #if VMA_RECORDING_ENABLED
18376 if(allocator->GetRecorder() != VMA_NULL)
18377 {
18378 allocator->GetRecorder()->RecordAllocateMemoryForImage(
18379 allocator->GetCurrentFrameIndex(),
18380 vkMemReq,
18381 requiresDedicatedAllocation,
18382 prefersDedicatedAllocation,
18383 *pCreateInfo,
18384 *pAllocation);
18385 }
18386 #endif
18387
18388 if(pAllocationInfo && result == VK_SUCCESS)
18389 {
18390 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
18391 }
18392
18393 return result;
18394 }
18395
18396 VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory(
18397 VmaAllocator allocator,
18398 VmaAllocation allocation)
18399 {
18400 VMA_ASSERT(allocator);
18401
18402 if(allocation == VK_NULL_HANDLE)
18403 {
18404 return;
18405 }
18406
18407 VMA_DEBUG_LOG("vmaFreeMemory");
18408
18409 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18410
18411 #if VMA_RECORDING_ENABLED
18412 if(allocator->GetRecorder() != VMA_NULL)
18413 {
18414 allocator->GetRecorder()->RecordFreeMemory(
18415 allocator->GetCurrentFrameIndex(),
18416 allocation);
18417 }
18418 #endif
18419
18420 allocator->FreeMemory(
18421 1, // allocationCount
18422 &allocation);
18423 }
18424
18425 VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages(
18426 VmaAllocator allocator,
18427 size_t allocationCount,
18428 const VmaAllocation* pAllocations)
18429 {
18430 if(allocationCount == 0)
18431 {
18432 return;
18433 }
18434
18435 VMA_ASSERT(allocator);
18436
18437 VMA_DEBUG_LOG("vmaFreeMemoryPages");
18438
18439 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18440
18441 #if VMA_RECORDING_ENABLED
18442 if(allocator->GetRecorder() != VMA_NULL)
18443 {
18444 allocator->GetRecorder()->RecordFreeMemoryPages(
18445 allocator->GetCurrentFrameIndex(),
18446 (uint64_t)allocationCount,
18447 pAllocations);
18448 }
18449 #endif
18450
18451 allocator->FreeMemory(allocationCount, pAllocations);
18452 }
18453
18454 VMA_CALL_PRE VkResult VMA_CALL_POST vmaResizeAllocation(
18455 VmaAllocator allocator,
18456 VmaAllocation allocation,
18457 VkDeviceSize newSize)
18458 {
18459 VMA_ASSERT(allocator && allocation);
18460
18461 VMA_DEBUG_LOG("vmaResizeAllocation");
18462
18463 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18464
18465 return allocator->ResizeAllocation(allocation, newSize);
18466 }
18467
18468 VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocationInfo(
18469 VmaAllocator allocator,
18470 VmaAllocation allocation,
18471 VmaAllocationInfo* pAllocationInfo)
18472 {
18473 VMA_ASSERT(allocator && allocation && pAllocationInfo);
18474
18475 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18476
18477 #if VMA_RECORDING_ENABLED
18478 if(allocator->GetRecorder() != VMA_NULL)
18479 {
18480 allocator->GetRecorder()->RecordGetAllocationInfo(
18481 allocator->GetCurrentFrameIndex(),
18482 allocation);
18483 }
18484 #endif
18485
18486 allocator->GetAllocationInfo(allocation, pAllocationInfo);
18487 }
18488
18489 VMA_CALL_PRE VkBool32 VMA_CALL_POST vmaTouchAllocation(
18490 VmaAllocator allocator,
18491 VmaAllocation allocation)
18492 {
18493 VMA_ASSERT(allocator && allocation);
18494
18495 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18496
18497 #if VMA_RECORDING_ENABLED
18498 if(allocator->GetRecorder() != VMA_NULL)
18499 {
18500 allocator->GetRecorder()->RecordTouchAllocation(
18501 allocator->GetCurrentFrameIndex(),
18502 allocation);
18503 }
18504 #endif
18505
18506 return allocator->TouchAllocation(allocation);
18507 }
18508
18509 VMA_CALL_PRE void VMA_CALL_POST vmaSetAllocationUserData(
18510 VmaAllocator allocator,
18511 VmaAllocation allocation,
18512 void* pUserData)
18513 {
18514 VMA_ASSERT(allocator && allocation);
18515
18516 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18517
18518 allocation->SetUserData(allocator, pUserData);
18519
18520 #if VMA_RECORDING_ENABLED
18521 if(allocator->GetRecorder() != VMA_NULL)
18522 {
18523 allocator->GetRecorder()->RecordSetAllocationUserData(
18524 allocator->GetCurrentFrameIndex(),
18525 allocation,
18526 pUserData);
18527 }
18528 #endif
18529 }
18530
18531 VMA_CALL_PRE void VMA_CALL_POST vmaCreateLostAllocation(
18532 VmaAllocator allocator,
18533 VmaAllocation* pAllocation)
18534 {
18535 VMA_ASSERT(allocator && pAllocation);
18536
18537 VMA_DEBUG_GLOBAL_MUTEX_LOCK;
18538
18539 allocator->CreateLostAllocation(pAllocation);
18540
18541 #if VMA_RECORDING_ENABLED
18542 if(allocator->GetRecorder() != VMA_NULL)
18543 {
18544 allocator->GetRecorder()->RecordCreateLostAllocation(
18545 allocator->GetCurrentFrameIndex(),
18546 *pAllocation);
18547 }
18548 #endif
18549 }
18550
18551 VMA_CALL_PRE VkResult VMA_CALL_POST vmaMapMemory(
18552 VmaAllocator allocator,
18553 VmaAllocation allocation,
18554 void** ppData)
18555 {
18556 VMA_ASSERT(allocator && allocation && ppData);
18557
18558 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18559
18560 VkResult res = allocator->Map(allocation, ppData);
18561
18562 #if VMA_RECORDING_ENABLED
18563 if(allocator->GetRecorder() != VMA_NULL)
18564 {
18565 allocator->GetRecorder()->RecordMapMemory(
18566 allocator->GetCurrentFrameIndex(),
18567 allocation);
18568 }
18569 #endif
18570
18571 return res;
18572 }
18573
18574 VMA_CALL_PRE void VMA_CALL_POST vmaUnmapMemory(
18575 VmaAllocator allocator,
18576 VmaAllocation allocation)
18577 {
18578 VMA_ASSERT(allocator && allocation);
18579
18580 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18581
18582 #if VMA_RECORDING_ENABLED
18583 if(allocator->GetRecorder() != VMA_NULL)
18584 {
18585 allocator->GetRecorder()->RecordUnmapMemory(
18586 allocator->GetCurrentFrameIndex(),
18587 allocation);
18588 }
18589 #endif
18590
18591 allocator->Unmap(allocation);
18592 }
18593
18594 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
18595 {
18596 VMA_ASSERT(allocator && allocation);
18597
18598 VMA_DEBUG_LOG("vmaFlushAllocation");
18599
18600 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18601
18602 const VkResult res = allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
18603
18604 #if VMA_RECORDING_ENABLED
18605 if(allocator->GetRecorder() != VMA_NULL)
18606 {
18607 allocator->GetRecorder()->RecordFlushAllocation(
18608 allocator->GetCurrentFrameIndex(),
18609 allocation, offset, size);
18610 }
18611 #endif
18612
18613 return res;
18614 }
18615
18616 VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
18617 {
18618 VMA_ASSERT(allocator && allocation);
18619
18620 VMA_DEBUG_LOG("vmaInvalidateAllocation");
18621
18622 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18623
18624 const VkResult res = allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
18625
18626 #if VMA_RECORDING_ENABLED
18627 if(allocator->GetRecorder() != VMA_NULL)
18628 {
18629 allocator->GetRecorder()->RecordInvalidateAllocation(
18630 allocator->GetCurrentFrameIndex(),
18631 allocation, offset, size);
18632 }
18633 #endif
18634
18635 return res;
18636 }
18637
18638 VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocations(
18639 VmaAllocator allocator,
18640 uint32_t allocationCount,
18641 const VmaAllocation* allocations,
18642 const VkDeviceSize* offsets,
18643 const VkDeviceSize* sizes)
18644 {
18645 VMA_ASSERT(allocator);
18646
18647 if(allocationCount == 0)
18648 {
18649 return VK_SUCCESS;
18650 }
18651
18652 VMA_ASSERT(allocations);
18653
18654 VMA_DEBUG_LOG("vmaFlushAllocations");
18655
18656 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18657
18658 const VkResult res = allocator->FlushOrInvalidateAllocations(allocationCount, allocations, offsets, sizes, VMA_CACHE_FLUSH);
18659
18660 #if VMA_RECORDING_ENABLED
18661 if(allocator->GetRecorder() != VMA_NULL)
18662 {
18663 //TODO
18664 }
18665 #endif
18666
18667 return res;
18668 }
18669
18670 VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocations(
18671 VmaAllocator allocator,
18672 uint32_t allocationCount,
18673 const VmaAllocation* allocations,
18674 const VkDeviceSize* offsets,
18675 const VkDeviceSize* sizes)
18676 {
18677 VMA_ASSERT(allocator);
18678
18679 if(allocationCount == 0)
18680 {
18681 return VK_SUCCESS;
18682 }
18683
18684 VMA_ASSERT(allocations);
18685
18686 VMA_DEBUG_LOG("vmaInvalidateAllocations");
18687
18688 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18689
18690 const VkResult res = allocator->FlushOrInvalidateAllocations(allocationCount, allocations, offsets, sizes, VMA_CACHE_INVALIDATE);
18691
18692 #if VMA_RECORDING_ENABLED
18693 if(allocator->GetRecorder() != VMA_NULL)
18694 {
18695 //TODO
18696 }
18697 #endif
18698
18699 return res;
18700 }
18701
18702 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
18703 {
18704 VMA_ASSERT(allocator);
18705
18706 VMA_DEBUG_LOG("vmaCheckCorruption");
18707
18708 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18709
18710 return allocator->CheckCorruption(memoryTypeBits);
18711 }
18712
18713 VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragment(
18714 VmaAllocator allocator,
18715 const VmaAllocation* pAllocations,
18716 size_t allocationCount,
18717 VkBool32* pAllocationsChanged,
18718 const VmaDefragmentationInfo *pDefragmentationInfo,
18719 VmaDefragmentationStats* pDefragmentationStats)
18720 {
18721 // Deprecated interface, reimplemented using new one.
18722
18723 VmaDefragmentationInfo2 info2 = {};
18724 info2.allocationCount = (uint32_t)allocationCount;
18725 info2.pAllocations = pAllocations;
18726 info2.pAllocationsChanged = pAllocationsChanged;
18727 if(pDefragmentationInfo != VMA_NULL)
18728 {
18729 info2.maxCpuAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
18730 info2.maxCpuBytesToMove = pDefragmentationInfo->maxBytesToMove;
18731 }
18732 else
18733 {
18734 info2.maxCpuAllocationsToMove = UINT32_MAX;
18735 info2.maxCpuBytesToMove = VK_WHOLE_SIZE;
18736 }
18737 // info2.flags, maxGpuAllocationsToMove, maxGpuBytesToMove, commandBuffer deliberately left zero.
18738
18739 VmaDefragmentationContext ctx;
18740 VkResult res = vmaDefragmentationBegin(allocator, &info2, pDefragmentationStats, &ctx);
18741 if(res == VK_NOT_READY)
18742 {
18743 res = vmaDefragmentationEnd( allocator, ctx);
18744 }
18745 return res;
18746 }
18747
18748 VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationBegin(
18749 VmaAllocator allocator,
18750 const VmaDefragmentationInfo2* pInfo,
18751 VmaDefragmentationStats* pStats,
18752 VmaDefragmentationContext *pContext)
18753 {
18754 VMA_ASSERT(allocator && pInfo && pContext);
18755
18756 // Degenerate case: Nothing to defragment.
18757 if(pInfo->allocationCount == 0 && pInfo->poolCount == 0)
18758 {
18759 return VK_SUCCESS;
18760 }
18761
18762 VMA_ASSERT(pInfo->allocationCount == 0 || pInfo->pAllocations != VMA_NULL);
18763 VMA_ASSERT(pInfo->poolCount == 0 || pInfo->pPools != VMA_NULL);
18764 VMA_HEAVY_ASSERT(VmaValidatePointerArray(pInfo->allocationCount, pInfo->pAllocations));
18765 VMA_HEAVY_ASSERT(VmaValidatePointerArray(pInfo->poolCount, pInfo->pPools));
18766
18767 VMA_DEBUG_LOG("vmaDefragmentationBegin");
18768
18769 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18770
18771 VkResult res = allocator->DefragmentationBegin(*pInfo, pStats, pContext);
18772
18773 #if VMA_RECORDING_ENABLED
18774 if(allocator->GetRecorder() != VMA_NULL)
18775 {
18776 allocator->GetRecorder()->RecordDefragmentationBegin(
18777 allocator->GetCurrentFrameIndex(), *pInfo, *pContext);
18778 }
18779 #endif
18780
18781 return res;
18782 }
18783
18784 VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationEnd(
18785 VmaAllocator allocator,
18786 VmaDefragmentationContext context)
18787 {
18788 VMA_ASSERT(allocator);
18789
18790 VMA_DEBUG_LOG("vmaDefragmentationEnd");
18791
18792 if(context != VK_NULL_HANDLE)
18793 {
18794 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18795
18796 #if VMA_RECORDING_ENABLED
18797 if(allocator->GetRecorder() != VMA_NULL)
18798 {
18799 allocator->GetRecorder()->RecordDefragmentationEnd(
18800 allocator->GetCurrentFrameIndex(), context);
18801 }
18802 #endif
18803
18804 return allocator->DefragmentationEnd(context);
18805 }
18806 else
18807 {
18808 return VK_SUCCESS;
18809 }
18810 }
18811
18812 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass(
18813 VmaAllocator allocator,
18814 VmaDefragmentationContext context,
18815 VmaDefragmentationPassInfo* pInfo
18816 )
18817 {
18818 VMA_ASSERT(allocator);
18819 VMA_ASSERT(pInfo);
18820
18821 VMA_DEBUG_LOG("vmaBeginDefragmentationPass");
18822
18823 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18824
18825 if(context == VK_NULL_HANDLE)
18826 {
18827 pInfo->moveCount = 0;
18828 return VK_SUCCESS;
18829 }
18830
18831 return allocator->DefragmentationPassBegin(pInfo, context);
18832 }
18833 VMA_CALL_PRE VkResult VMA_CALL_POST vmaEndDefragmentationPass(
18834 VmaAllocator allocator,
18835 VmaDefragmentationContext context)
18836 {
18837 VMA_ASSERT(allocator);
18838
18839 VMA_DEBUG_LOG("vmaEndDefragmentationPass");
18840 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18841
18842 if(context == VK_NULL_HANDLE)
18843 return VK_SUCCESS;
18844
18845 return allocator->DefragmentationPassEnd(context);
18846 }
18847
18848 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory(
18849 VmaAllocator allocator,
18850 VmaAllocation allocation,
18851 VkBuffer buffer)
18852 {
18853 VMA_ASSERT(allocator && allocation && buffer);
18854
18855 VMA_DEBUG_LOG("vmaBindBufferMemory");
18856
18857 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18858
18859 return allocator->BindBufferMemory(allocation, 0, buffer, VMA_NULL);
18860 }
18861
18862 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory2(
18863 VmaAllocator allocator,
18864 VmaAllocation allocation,
18865 VkDeviceSize allocationLocalOffset,
18866 VkBuffer buffer,
18867 const void* pNext)
18868 {
18869 VMA_ASSERT(allocator && allocation && buffer);
18870
18871 VMA_DEBUG_LOG("vmaBindBufferMemory2");
18872
18873 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18874
18875 return allocator->BindBufferMemory(allocation, allocationLocalOffset, buffer, pNext);
18876 }
18877
18878 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory(
18879 VmaAllocator allocator,
18880 VmaAllocation allocation,
18881 VkImage image)
18882 {
18883 VMA_ASSERT(allocator && allocation && image);
18884
18885 VMA_DEBUG_LOG("vmaBindImageMemory");
18886
18887 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18888
18889 return allocator->BindImageMemory(allocation, 0, image, VMA_NULL);
18890 }
18891
18892 VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory2(
18893 VmaAllocator allocator,
18894 VmaAllocation allocation,
18895 VkDeviceSize allocationLocalOffset,
18896 VkImage image,
18897 const void* pNext)
18898 {
18899 VMA_ASSERT(allocator && allocation && image);
18900
18901 VMA_DEBUG_LOG("vmaBindImageMemory2");
18902
18903 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18904
18905 return allocator->BindImageMemory(allocation, allocationLocalOffset, image, pNext);
18906 }
18907
18908 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer(
18909 VmaAllocator allocator,
18910 const VkBufferCreateInfo* pBufferCreateInfo,
18911 const VmaAllocationCreateInfo* pAllocationCreateInfo,
18912 VkBuffer* pBuffer,
18913 VmaAllocation* pAllocation,
18914 VmaAllocationInfo* pAllocationInfo)
18915 {
18916 VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
18917
18918 if(pBufferCreateInfo->size == 0)
18919 {
18920 return VK_ERROR_VALIDATION_FAILED_EXT;
18921 }
18922 if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_COPY) != 0 &&
18923 !allocator->m_UseKhrBufferDeviceAddress)
18924 {
18925 VMA_ASSERT(0 && "Creating a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT is not valid if VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT was not used.");
18926 return VK_ERROR_VALIDATION_FAILED_EXT;
18927 }
18928
18929 VMA_DEBUG_LOG("vmaCreateBuffer");
18930
18931 VMA_DEBUG_GLOBAL_MUTEX_LOCK
18932
18933 *pBuffer = VK_NULL_HANDLE;
18934 *pAllocation = VK_NULL_HANDLE;
18935
18936 // 1. Create VkBuffer.
18937 VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
18938 allocator->m_hDevice,
18939 pBufferCreateInfo,
18940 allocator->GetAllocationCallbacks(),
18941 pBuffer);
18942 if(res >= 0)
18943 {
18944 // 2. vkGetBufferMemoryRequirements.
18945 VkMemoryRequirements vkMemReq = {};
18946 bool requiresDedicatedAllocation = false;
18947 bool prefersDedicatedAllocation = false;
18948 allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
18949 requiresDedicatedAllocation, prefersDedicatedAllocation);
18950
18951 // 3. Allocate memory using allocator.
18952 res = allocator->AllocateMemory(
18953 vkMemReq,
18954 requiresDedicatedAllocation,
18955 prefersDedicatedAllocation,
18956 *pBuffer, // dedicatedBuffer
18957 pBufferCreateInfo->usage, // dedicatedBufferUsage
18958 VK_NULL_HANDLE, // dedicatedImage
18959 *pAllocationCreateInfo,
18960 VMA_SUBALLOCATION_TYPE_BUFFER,
18961 1, // allocationCount
18962 pAllocation);
18963
18964 #if VMA_RECORDING_ENABLED
18965 if(allocator->GetRecorder() != VMA_NULL)
18966 {
18967 allocator->GetRecorder()->RecordCreateBuffer(
18968 allocator->GetCurrentFrameIndex(),
18969 *pBufferCreateInfo,
18970 *pAllocationCreateInfo,
18971 *pAllocation);
18972 }
18973 #endif
18974
18975 if(res >= 0)
18976 {
18977 // 3. Bind buffer with memory.
18978 if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_DONT_BIND_BIT) == 0)
18979 {
18980 res = allocator->BindBufferMemory(*pAllocation, 0, *pBuffer, VMA_NULL);
18981 }
18982 if(res >= 0)
18983 {
18984 // All steps succeeded.
18985 #if VMA_STATS_STRING_ENABLED
18986 (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
18987 #endif
18988 if(pAllocationInfo != VMA_NULL)
18989 {
18990 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
18991 }
18992
18993 return VK_SUCCESS;
18994 }
18995 allocator->FreeMemory(
18996 1, // allocationCount
18997 pAllocation);
18998 *pAllocation = VK_NULL_HANDLE;
18999 (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
19000 *pBuffer = VK_NULL_HANDLE;
19001 return res;
19002 }
19003 (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
19004 *pBuffer = VK_NULL_HANDLE;
19005 return res;
19006 }
19007 return res;
19008 }
19009
19010 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyBuffer(
19011 VmaAllocator allocator,
19012 VkBuffer buffer,
19013 VmaAllocation allocation)
19014 {
19015 VMA_ASSERT(allocator);
19016
19017 if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
19018 {
19019 return;
19020 }
19021
19022 VMA_DEBUG_LOG("vmaDestroyBuffer");
19023
19024 VMA_DEBUG_GLOBAL_MUTEX_LOCK
19025
19026 #if VMA_RECORDING_ENABLED
19027 if(allocator->GetRecorder() != VMA_NULL)
19028 {
19029 allocator->GetRecorder()->RecordDestroyBuffer(
19030 allocator->GetCurrentFrameIndex(),
19031 allocation);
19032 }
19033 #endif
19034
19035 if(buffer != VK_NULL_HANDLE)
19036 {
19037 (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
19038 }
19039
19040 if(allocation != VK_NULL_HANDLE)
19041 {
19042 allocator->FreeMemory(
19043 1, // allocationCount
19044 &allocation);
19045 }
19046 }
19047
19048 VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage(
19049 VmaAllocator allocator,
19050 const VkImageCreateInfo* pImageCreateInfo,
19051 const VmaAllocationCreateInfo* pAllocationCreateInfo,
19052 VkImage* pImage,
19053 VmaAllocation* pAllocation,
19054 VmaAllocationInfo* pAllocationInfo)
19055 {
19056 VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
19057
19058 if(pImageCreateInfo->extent.width == 0 ||
19059 pImageCreateInfo->extent.height == 0 ||
19060 pImageCreateInfo->extent.depth == 0 ||
19061 pImageCreateInfo->mipLevels == 0 ||
19062 pImageCreateInfo->arrayLayers == 0)
19063 {
19064 return VK_ERROR_VALIDATION_FAILED_EXT;
19065 }
19066
19067 VMA_DEBUG_LOG("vmaCreateImage");
19068
19069 VMA_DEBUG_GLOBAL_MUTEX_LOCK
19070
19071 *pImage = VK_NULL_HANDLE;
19072 *pAllocation = VK_NULL_HANDLE;
19073
19074 // 1. Create VkImage.
19075 VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
19076 allocator->m_hDevice,
19077 pImageCreateInfo,
19078 allocator->GetAllocationCallbacks(),
19079 pImage);
19080 if(res >= 0)
19081 {
19082 VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
19083 VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
19084 VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
19085
19086 // 2. Allocate memory using allocator.
19087 VkMemoryRequirements vkMemReq = {};
19088 bool requiresDedicatedAllocation = false;
19089 bool prefersDedicatedAllocation = false;
19090 allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
19091 requiresDedicatedAllocation, prefersDedicatedAllocation);
19092
19093 res = allocator->AllocateMemory(
19094 vkMemReq,
19095 requiresDedicatedAllocation,
19096 prefersDedicatedAllocation,
19097 VK_NULL_HANDLE, // dedicatedBuffer
19098 UINT32_MAX, // dedicatedBufferUsage
19099 *pImage, // dedicatedImage
19100 *pAllocationCreateInfo,
19101 suballocType,
19102 1, // allocationCount
19103 pAllocation);
19104
19105 #if VMA_RECORDING_ENABLED
19106 if(allocator->GetRecorder() != VMA_NULL)
19107 {
19108 allocator->GetRecorder()->RecordCreateImage(
19109 allocator->GetCurrentFrameIndex(),
19110 *pImageCreateInfo,
19111 *pAllocationCreateInfo,
19112 *pAllocation);
19113 }
19114 #endif
19115
19116 if(res >= 0)
19117 {
19118 // 3. Bind image with memory.
19119 if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_DONT_BIND_BIT) == 0)
19120 {
19121 res = allocator->BindImageMemory(*pAllocation, 0, *pImage, VMA_NULL);
19122 }
19123 if(res >= 0)
19124 {
19125 // All steps succeeded.
19126 #if VMA_STATS_STRING_ENABLED
19127 (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
19128 #endif
19129 if(pAllocationInfo != VMA_NULL)
19130 {
19131 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
19132 }
19133
19134 return VK_SUCCESS;
19135 }
19136 allocator->FreeMemory(
19137 1, // allocationCount
19138 pAllocation);
19139 *pAllocation = VK_NULL_HANDLE;
19140 (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
19141 *pImage = VK_NULL_HANDLE;
19142 return res;
19143 }
19144 (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
19145 *pImage = VK_NULL_HANDLE;
19146 return res;
19147 }
19148 return res;
19149 }
19150
19151 VMA_CALL_PRE void VMA_CALL_POST vmaDestroyImage(
19152 VmaAllocator allocator,
19153 VkImage image,
19154 VmaAllocation allocation)
19155 {
19156 VMA_ASSERT(allocator);
19157
19158 if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
19159 {
19160 return;
19161 }
19162
19163 VMA_DEBUG_LOG("vmaDestroyImage");
19164
19165 VMA_DEBUG_GLOBAL_MUTEX_LOCK
19166
19167 #if VMA_RECORDING_ENABLED
19168 if(allocator->GetRecorder() != VMA_NULL)
19169 {
19170 allocator->GetRecorder()->RecordDestroyImage(
19171 allocator->GetCurrentFrameIndex(),
19172 allocation);
19173 }
19174 #endif
19175
19176 if(image != VK_NULL_HANDLE)
19177 {
19178 (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
19179 }
19180 if(allocation != VK_NULL_HANDLE)
19181 {
19182 allocator->FreeMemory(
19183 1, // allocationCount
19184 &allocation);
19185 }
19186 }
19187
19188 #endif // #ifdef VMA_IMPLEMENTATION