| 1 |
/* |
| 2 |
* Copyright (c) 2015-2017, Intel Corporation |
| 3 |
* |
| 4 |
* Permission is hereby granted, free of charge, to any person obtaining a |
| 5 |
* copy of this software and associated documentation files (the "Software"), |
| 6 |
* to deal in the Software without restriction, including without limitation |
| 7 |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 |
* and/or sell copies of the Software, and to permit persons to whom the |
| 9 |
* Software is furnished to do so, subject to the following conditions: |
| 10 |
* |
| 11 |
* The above copyright notice and this permission notice shall be included |
| 12 |
* in all copies or substantial portions of the Software. |
| 13 |
* |
| 14 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
| 15 |
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 16 |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 17 |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 18 |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 19 |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 20 |
* OTHER DEALINGS IN THE SOFTWARE. |
| 21 |
*/ |
| 22 |
//! |
| 23 |
//! \file mhw_block_manager.cpp |
| 24 |
//! \brief This modules implements memory block management functions as part of MHW dynamic state heap implementation |
| 25 |
//! |
| 26 |
#include "mhw_block_manager.h" |
| 27 |
#include "mhw_utilities.h" |
| 28 |
#include "mos_os_specific.h" |
| 29 |
|
| 30 |
MHW_BLOCK_MANAGER_PARAMS MhwBlockManagerParams_default = |
| 31 |
{ |
| 32 |
64, // Initial number of block objects in pool |
| 33 |
1024, // Maximum number of block objects in pool |
| 34 |
64, // Block pool increment size |
| 35 |
0x00080000, // Initial heap size (512k) |
| 36 |
0x00080000, // Heap size increment (512k) |
| 37 |
0x01000000, // Maximum overall heap size (16M) |
| 38 |
32, // Maximum number of heaps (32) (512k x 32 = 16M) |
| 39 |
0x0800, // Block granularity = 2k (also represents min block alignment) |
| 40 |
0x0800 // Min free block size = 2k (to reduce fragmentation) |
| 41 |
}; |
| 42 |
|
| 43 |
const char *szListName[MHW_BLOCK_STATE_COUNT] = { |
| 44 |
"POOL", |
| 45 |
"FREE", |
| 46 |
"ALLOCATED", |
| 47 |
"SUBMITTED", |
| 48 |
"DELETED" |
| 49 |
}; |
| 50 |
|
| 51 |
void Mhw_BlockManager_ReverseMergeSort_With_Index(const uint32_t *pdwSizes, int32_t iCount, uint8_t *pSortedIndex) |
| 52 |
{ |
| 53 |
uint8_t i, n; |
| 54 |
uint8_t *pSrc, *pDst; // Source and Destination groups (alternate) |
| 55 |
uint8_t Index1[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted indexes 1 |
| 56 |
uint8_t Index2[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted indexes 2 (alternating with Index1) |
| 57 |
uint8_t *s1, *s2; // Merge source groups 1 and 2 |
| 58 |
uint8_t n1, n2; // Merge sizes groups 1 and 2 |
| 59 |
uint8_t *d; // Merge destination |
| 60 |
|
| 61 |
// Very simple cases |
| 62 |
if (iCount <= 1) |
| 63 |
{ |
| 64 |
pSortedIndex[0] = 0; |
| 65 |
return; |
| 66 |
} |
| 67 |
else if (iCount == 2) |
| 68 |
{ |
| 69 |
pSortedIndex[0] = (pdwSizes[0] < pdwSizes[1]) ? 1 : 0; |
| 70 |
pSortedIndex[1] = 1 - pSortedIndex[0]; |
| 71 |
return; |
| 72 |
} |
| 73 |
|
| 74 |
// Initialize sorted index table |
| 75 |
for (i = 0; i < iCount; i++) |
| 76 |
{ |
| 77 |
Index1[i] = i; |
| 78 |
} |
| 79 |
|
| 80 |
// Start alternating buffers (last will be the actual output) |
| 81 |
pSrc = Index1; |
| 82 |
pDst = Index2; |
| 83 |
|
| 84 |
// Merge sort algorithm: |
| 85 |
// Sort will perform O(log n) passes; first pass sorts (iCount/2) groups of 2, then (iCount/4) groups of 4, and so on. |
| 86 |
// Each pass requires traversal of the entire list - O(n) |
| 87 |
// Algorithm is expected to be O(n * log n) |
| 88 |
for (n = 1; n < iCount; n *= 2) |
| 89 |
{ |
| 90 |
// Setup sorted target output |
| 91 |
if (n*2 < iCount) |
| 92 |
{ |
| 93 |
d = pDst; |
| 94 |
} |
| 95 |
else |
| 96 |
{ // Last pass, output goes to caller |
| 97 |
d = pSortedIndex; |
| 98 |
} |
| 99 |
|
| 100 |
// Group selection and merge - O(n) |
| 101 |
s1 = pSrc - n; // First group |
| 102 |
s2 = pSrc; // Second group |
| 103 |
for (i = n; i < iCount; i += 2*n) // i is the offset of the 2nd group |
| 104 |
{ |
| 105 |
s1 += n; |
| 106 |
n1 = n; |
| 107 |
s2 += n; |
| 108 |
n2 = n; |
| 109 |
|
| 110 |
// Limit size of last group |
| 111 |
if (i + n > iCount) |
| 112 |
{ |
| 113 |
n2 = iCount - i; |
| 114 |
} |
| 115 |
|
| 116 |
// Merge groups |
| 117 |
while (n1 > 0 && n2 > 0) |
| 118 |
{ |
| 119 |
if (pdwSizes[*s1] >= pdwSizes[*s2]) |
| 120 |
{ |
| 121 |
*(d++) = *(s1++); |
| 122 |
n1--; |
| 123 |
} |
| 124 |
else |
| 125 |
{ |
| 126 |
*(d++) = *(s2++); |
| 127 |
n2--; |
| 128 |
} |
| 129 |
} |
| 130 |
|
| 131 |
// Merge remaining items |
| 132 |
while (n1 > 0) |
| 133 |
{ |
| 134 |
*(d++) = *s1++; |
| 135 |
n1--; |
| 136 |
} |
| 137 |
while (n2 > 0) |
| 138 |
{ |
| 139 |
*(d++) = *s2++; |
| 140 |
n2--; |
| 141 |
} |
| 142 |
} |
| 143 |
|
| 144 |
// Copy the last group (not merge-sorted) |
| 145 |
for (i = i - n; i < iCount; i++) |
| 146 |
{ |
| 147 |
*(d++) = *(s2++); |
| 148 |
} |
| 149 |
|
| 150 |
// New pass, switch Src/Dst sorted index buffers |
| 151 |
d = pDst; |
| 152 |
pDst = pSrc; |
| 153 |
pSrc = d; |
| 154 |
} |
| 155 |
} |
| 156 |
|
| 157 |
void Mhw_BlockManager_ReverseMergeSort(uint32_t *pdwSizes, int32_t iCount) |
| 158 |
{ |
| 159 |
uint8_t i, n; |
| 160 |
uint32_t *pSrc, *pDst; // Source and Destination groups (alternate) |
| 161 |
uint32_t Buffer1[2 * MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted buffer 1 |
| 162 |
uint32_t Buffer2[2 * MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted buffer 1 |
| 163 |
uint32_t *s1, *s2; // Merge source groups 1 and 2 |
| 164 |
uint8_t n1, n2; // Merge sizes groups 1 and 2 |
| 165 |
uint32_t *d; // Merge destination |
| 166 |
|
| 167 |
// Very simple cases |
| 168 |
if (iCount <= 1) |
| 169 |
{ |
| 170 |
return; |
| 171 |
} |
| 172 |
else if (iCount == 2) |
| 173 |
{ |
| 174 |
if (pdwSizes[0] < pdwSizes[1]) |
| 175 |
{ |
| 176 |
uint32_t tmp = pdwSizes[1]; |
| 177 |
pdwSizes[1] = pdwSizes[0]; |
| 178 |
pdwSizes[0] = tmp; |
| 179 |
} |
| 180 |
return; |
| 181 |
} |
| 182 |
else if (iCount > 2 * MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY) |
| 183 |
{ |
| 184 |
return; |
| 185 |
} |
| 186 |
|
| 187 |
// Merge sort algorithm: |
| 188 |
// Sort will perform O(log n) passes; first pass sorts (iCount/2) groups of 2, then (iCount/4) groups of 4, and so on. |
| 189 |
// Each pass requires traversal of the entire list - O(n) |
| 190 |
// Algorithm is expected to be O(n * log n) |
| 191 |
pSrc = Buffer1; |
| 192 |
pDst = Buffer2; |
| 193 |
for (n = 1; n < iCount; n *= 2) |
| 194 |
{ |
| 195 |
if (n == 1) |
| 196 |
{ |
| 197 |
s1 = pdwSizes - 1; |
| 198 |
s2 = pdwSizes; |
| 199 |
} |
| 200 |
else |
| 201 |
{ |
| 202 |
s1 = pSrc - n; |
| 203 |
s2 = pSrc; |
| 204 |
} |
| 205 |
|
| 206 |
// Setup sorted target output |
| 207 |
if (n*2 < iCount) |
| 208 |
{ |
| 209 |
d = pDst; |
| 210 |
} |
| 211 |
else |
| 212 |
{ // Last pass, output goes to caller |
| 213 |
d = pdwSizes; |
| 214 |
} |
| 215 |
|
| 216 |
// Group selection and merge - O(n) |
| 217 |
for (i = n; i < iCount; i += 2*n) // i is the offset of the 2nd group |
| 218 |
{ |
| 219 |
s1 += n; |
| 220 |
n1 = n; |
| 221 |
s2 += n; |
| 222 |
n2 = n; |
| 223 |
|
| 224 |
// Limit size of last group |
| 225 |
if (i + n > iCount) |
| 226 |
{ |
| 227 |
n2 = iCount - i; |
| 228 |
} |
| 229 |
|
| 230 |
// Merge groups |
| 231 |
while (n1 > 0 && n2 > 0) |
| 232 |
{ |
| 233 |
if (*s1 >= *s2) |
| 234 |
{ |
| 235 |
*(d++) = *(s1++); |
| 236 |
n1--; |
| 237 |
} |
| 238 |
else |
| 239 |
{ |
| 240 |
*(d++) = *(s2++); |
| 241 |
n2--; |
| 242 |
} |
| 243 |
} |
| 244 |
|
| 245 |
// Merge remaining items |
| 246 |
while (n1 > 0) |
| 247 |
{ |
| 248 |
*(d++) = *s1++; |
| 249 |
n1--; |
| 250 |
} |
| 251 |
while (n2 > 0) |
| 252 |
{ |
| 253 |
*(d++) = *s2++; |
| 254 |
n2--; |
| 255 |
} |
| 256 |
} |
| 257 |
|
| 258 |
// Copy the last group (not merge-sorted) |
| 259 |
for (i = i - n; i < iCount; i++) |
| 260 |
{ |
| 261 |
*(d++) = *(s2++); |
| 262 |
} |
| 263 |
|
| 264 |
// New pass, switch Src/Dst |
| 265 |
d = pDst; |
| 266 |
pDst = pSrc; |
| 267 |
pSrc = d; |
| 268 |
} |
| 269 |
} |
| 270 |
|
| 271 |
MHW_BLOCK_MANAGER::MHW_BLOCK_MANAGER(PMHW_BLOCK_MANAGER_PARAMS pParams): |
| 272 |
m_MemoryPool(sizeof(MHW_STATE_HEAP_MEMORY_BLOCK), sizeof(void *)), |
| 273 |
m_pStateHeap(nullptr) |
| 274 |
{ |
| 275 |
//Init Parameters |
| 276 |
if(pParams != nullptr) |
| 277 |
{ |
| 278 |
m_Params = *pParams; |
| 279 |
} |
| 280 |
else |
| 281 |
{ |
| 282 |
m_Params = MhwBlockManagerParams_default; |
| 283 |
} |
| 284 |
|
| 285 |
//Init Memory block list |
| 286 |
for (int32_t i = (int32_t)MHW_BLOCK_STATE_POOL; i < MHW_BLOCK_STATE_COUNT; i++) |
| 287 |
{ |
| 288 |
MOS_ZeroMemory(&m_BlockList[i], sizeof(MHW_BLOCK_LIST)); |
| 289 |
m_BlockList[i].BlockState = (MHW_BLOCK_STATE) i; |
| 290 |
m_BlockList[i].pBlockManager = this; |
| 291 |
MOS_SecureStrcpy(m_BlockList[i].szListName, 16, szListName[i]); |
| 292 |
} |
| 293 |
|
| 294 |
//Extend Pool |
| 295 |
ExtendPool(m_Params.dwPoolInitialCount); |
| 296 |
} |
| 297 |
|
| 298 |
MHW_BLOCK_MANAGER::~MHW_BLOCK_MANAGER() |
| 299 |
{ |
| 300 |
|
| 301 |
} |
| 302 |
|
| 303 |
void MHW_BLOCK_MANAGER::SetStateHeap(PMHW_STATE_HEAP pStateHeap) |
| 304 |
{ |
| 305 |
if (pStateHeap) |
| 306 |
{ |
| 307 |
m_pStateHeap = pStateHeap; |
| 308 |
} |
| 309 |
return; |
| 310 |
} |
| 311 |
|
| 312 |
MOS_STATUS MHW_BLOCK_MANAGER::RegisterStateHeap( |
| 313 |
PMHW_STATE_HEAP pStateHeap) |
| 314 |
{ |
| 315 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 316 |
|
| 317 |
// Setup state heap associated with the memory block manager |
| 318 |
if (!m_pStateHeap) |
| 319 |
{ |
| 320 |
m_pStateHeap = pStateHeap; |
| 321 |
} |
| 322 |
|
| 323 |
// Indicates that state heap is now being managed by this block manager object |
| 324 |
pStateHeap->pBlockManager = this; |
| 325 |
|
| 326 |
// Get memory block object to represent the state heap memory (free) |
| 327 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = GetBlockFromPool(); |
| 328 |
if (!pBlock) |
| 329 |
{ |
| 330 |
return MOS_STATUS_NO_SPACE; |
| 331 |
} |
| 332 |
|
| 333 |
// Setup block parameters |
| 334 |
pBlock->pStateHeap = pStateHeap; // Points to state heap |
| 335 |
pBlock->pHeapPrev = nullptr; // First Block in current heap |
| 336 |
pBlock->pHeapNext = nullptr; // Last Block in current heap |
| 337 |
pBlock->dwOffsetInStateHeap = 0; // Offset is 0 = base of state heap |
| 338 |
pBlock->dwBlockSize = pStateHeap->dwSize; // Size of the entire state heap |
| 339 |
FrameTrackerTokenFlat_Validate(&pBlock->trackerToken); |
| 340 |
pBlock->bStatic = 0; |
| 341 |
|
| 342 |
// Set first/last memory block in state heap |
| 343 |
pStateHeap->pMemoryHead = pBlock; |
| 344 |
pStateHeap->pMemoryTail = pBlock; |
| 345 |
|
| 346 |
// Reset special state heap controls |
| 347 |
pStateHeap->pDebugKernel = nullptr; // Debug kernel loaded in this heap (if ISH) |
| 348 |
pStateHeap->pScratchSpace = nullptr; // Active scratch space in this heap (if GSH) |
| 349 |
pStateHeap->dwScratchSpace = 0; // Current scratch space size in this heap |
| 350 |
|
| 351 |
// Insert block at the end of the free memory pool |
| 352 |
AttachBlock(MHW_BLOCK_STATE_FREE, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 353 |
|
| 354 |
return eStatus; |
| 355 |
} |
| 356 |
|
| 357 |
MOS_STATUS MHW_BLOCK_MANAGER::UnregisterStateHeap( |
| 358 |
PMHW_STATE_HEAP pStateHeap) |
| 359 |
{ |
| 360 |
MHW_STATE_HEAP_MEMORY_BLOCK *pBlock; |
| 361 |
|
| 362 |
bool bReleaseHeap = true; |
| 363 |
|
| 364 |
// Verification loop - check if heap can be freed |
| 365 |
for (pBlock = pStateHeap->pMemoryHead; pBlock != nullptr; pBlock = pBlock->pHeapNext) |
| 366 |
{ |
| 367 |
// Move blocks not in use to deleted queue - so they cannot be reused |
| 368 |
// NOTE: Blocks in SUBMITTED state need to wait for completion before releasing |
| 369 |
if (pBlock->BlockState == MHW_BLOCK_STATE_FREE || |
| 370 |
pBlock->BlockState == MHW_BLOCK_STATE_ALLOCATED) |
| 371 |
{ |
| 372 |
// Update heap usage |
| 373 |
if (pBlock->BlockState == MHW_BLOCK_STATE_FREE) |
| 374 |
{ |
| 375 |
pStateHeap->dwFree -= pBlock->dwBlockSize; |
| 376 |
} |
| 377 |
else |
| 378 |
{ |
| 379 |
pStateHeap->dwUsed -= pBlock->dwBlockSize; |
| 380 |
} |
| 381 |
|
| 382 |
DetachBlock(pBlock->BlockState, pBlock); |
| 383 |
AttachBlock(MHW_BLOCK_STATE_DELETED, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 384 |
} |
| 385 |
// Block is allocated or submitted - mark block for deletion when released |
| 386 |
else if (pBlock->BlockState != MHW_BLOCK_STATE_DELETED) |
| 387 |
{ |
| 388 |
pBlock->bStatic = false; // Unlock block |
| 389 |
pBlock->bDelete = true; // Mark block for deletion when done |
| 390 |
bReleaseHeap = false; // Heap cannot be released at this moment |
| 391 |
} |
| 392 |
} |
| 393 |
|
| 394 |
// All blocks have been sucessfully deleted -> move all blocks to pool, unregister state heap |
| 395 |
if (bReleaseHeap) |
| 396 |
{ |
| 397 |
// Return deleted blocks back to pool |
| 398 |
for (pBlock = pStateHeap->pMemoryHead; pBlock != nullptr; pBlock = pBlock->pHeapNext) |
| 399 |
{ |
| 400 |
// Sanity check - all blocks must be in this state! |
| 401 |
if (pBlock->BlockState == MHW_BLOCK_STATE_DELETED) |
| 402 |
{ |
| 403 |
DetachBlock(MHW_BLOCK_STATE_DELETED, pBlock); |
| 404 |
ReturnBlockToPool(pBlock); |
| 405 |
} |
| 406 |
else |
| 407 |
{ |
| 408 |
// NEVER SUPPOSED TO HAPPEN DUE TO PREVIOUS LOOP |
| 409 |
MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_UnregisterStateHeap: Invalid state, heap blocks are supposed to be all deleted by now"); |
| 410 |
} |
| 411 |
} |
| 412 |
return MOS_STATUS_SUCCESS; |
| 413 |
} |
| 414 |
else |
| 415 |
{ // State heap cannot be unregistered because some blocks are still in use |
| 416 |
return MOS_STATUS_UNKNOWN; |
| 417 |
} |
| 418 |
} |
| 419 |
|
| 420 |
PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::GetBlockFromPool() |
| 421 |
{ |
| 422 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr; |
| 423 |
|
| 424 |
// Ran out of memory blocks... extend pool of memory block objects |
| 425 |
if (m_BlockList[MHW_BLOCK_STATE_POOL].iCount == 0) |
| 426 |
{ |
| 427 |
ExtendPool(m_Params.dwPoolIncrement); |
| 428 |
} |
| 429 |
|
| 430 |
// Retrieve block object from head of the pool |
| 431 |
pBlock = DetachBlock(MHW_BLOCK_STATE_POOL, MHW_BLOCK_POSITION_HEAD); |
| 432 |
|
| 433 |
return pBlock; |
| 434 |
} |
| 435 |
|
| 436 |
void MHW_BLOCK_MANAGER::ExtendPool(uint32_t dwCount) |
| 437 |
{ |
| 438 |
uint32_t dwBlockID = m_MemoryPool.m_dwObjCount; // Block ID starts from the current block count |
| 439 |
|
| 440 |
// Limits the number of memory blocks |
| 441 |
if (m_MemoryPool.m_dwCount + dwCount > m_Params.dwPoolMaxCount) |
| 442 |
{ |
| 443 |
dwCount = m_Params.dwPoolMaxCount - m_MemoryPool.m_dwCount; |
| 444 |
} |
| 445 |
|
| 446 |
// Extend pool of block objects |
| 447 |
if (dwCount > 0) |
| 448 |
{ |
| 449 |
// Allocate array of block objects into the pool |
| 450 |
MHW_STATE_HEAP_MEMORY_BLOCK *pBlockArray = (MHW_STATE_HEAP_MEMORY_BLOCK *) (m_MemoryPool.Allocate(dwCount)); |
| 451 |
if (pBlockArray) |
| 452 |
{ |
| 453 |
// Insert newly created block objects into the linked list of pool objects available to the memory block manager |
| 454 |
for (; dwCount > 0; dwCount--, pBlockArray++) |
| 455 |
{ |
| 456 |
pBlockArray->dwBlockSize = 0; |
| 457 |
pBlockArray->pPrev = pBlockArray->pNext = nullptr; |
| 458 |
pBlockArray->Reserved = dwBlockID++; |
| 459 |
AttachBlock(MHW_BLOCK_STATE_POOL, pBlockArray, MHW_BLOCK_POSITION_TAIL); |
| 460 |
} |
| 461 |
} |
| 462 |
} |
| 463 |
} |
| 464 |
|
| 465 |
MOS_STATUS MHW_BLOCK_MANAGER::AttachBlock( |
| 466 |
MHW_BLOCK_STATE BlockState, |
| 467 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, |
| 468 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos) |
| 469 |
{ |
| 470 |
PMHW_BLOCK_LIST pList; |
| 471 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 472 |
|
| 473 |
// Verify parameters - objects cannot be null, block state must be valid |
| 474 |
if (pBlock == nullptr || |
| 475 |
BlockState < MHW_BLOCK_STATE_POOL || |
| 476 |
BlockState >= MHW_BLOCK_STATE_COUNT) |
| 477 |
{ |
| 478 |
return MOS_STATUS_INVALID_PARAMETER; |
| 479 |
} |
| 480 |
|
| 481 |
// Fails if block is still attached to a list |
| 482 |
if (pBlock->pPrev || pBlock->pNext) |
| 483 |
{ |
| 484 |
return MOS_STATUS_INVALID_PARAMETER; |
| 485 |
} |
| 486 |
|
| 487 |
// Get list associated with block state; move block to the list |
| 488 |
pList = &m_BlockList[BlockState]; |
| 489 |
BLOCK_MANAGER_CHK_STATUS(AttachBlockInternal(pList, BlockState, pBlock, pBlockPos)); |
| 490 |
|
| 491 |
return MOS_STATUS_SUCCESS; |
| 492 |
} |
| 493 |
|
| 494 |
MOS_STATUS MHW_BLOCK_MANAGER::AttachBlockInternal( |
| 495 |
PMHW_BLOCK_LIST pList, |
| 496 |
MHW_BLOCK_STATE BlockState, |
| 497 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, |
| 498 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos) |
| 499 |
{ |
| 500 |
|
| 501 |
// Check if this is the correct list! |
| 502 |
if (pList->BlockState != BlockState) |
| 503 |
{ |
| 504 |
MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_AttachBlock_Internal: Block state doesn't match the list provided"); |
| 505 |
return MOS_STATUS_INVALID_PARAMETER; |
| 506 |
} |
| 507 |
|
| 508 |
// Setup the block state |
| 509 |
pBlock->BlockState = BlockState; |
| 510 |
|
| 511 |
// Attaches to the head of the list |
| 512 |
if (pBlockPos == MHW_BLOCK_POSITION_TAIL) |
| 513 |
{ |
| 514 |
pBlock->pPrev = pList->pTail; |
| 515 |
pBlock->pNext = nullptr; |
| 516 |
} |
| 517 |
// Attaches to the tail of the list |
| 518 |
else if (pBlockPos == MHW_BLOCK_POSITION_HEAD) |
| 519 |
{ |
| 520 |
pBlock->pPrev = nullptr; |
| 521 |
pBlock->pNext = pList->pHead; |
| 522 |
} |
| 523 |
// Insert after block provided - ensures that it belongs to the same list |
| 524 |
else if (pBlockPos->BlockState == BlockState) |
| 525 |
{ |
| 526 |
pBlock->pPrev = pBlockPos; |
| 527 |
pBlock->pNext = pBlockPos->pNext; |
| 528 |
} |
| 529 |
// Insertion point does not belong to the same list |
| 530 |
else |
| 531 |
{ |
| 532 |
MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_AttachBlock_Internal: Reference block does not belong to the list provided"); |
| 533 |
return MOS_STATUS_INVALID_PARAMETER; |
| 534 |
} |
| 535 |
|
| 536 |
// Modify previous block or head of the list |
| 537 |
if (pBlock->pPrev) |
| 538 |
{ |
| 539 |
pBlock->pPrev->pNext = pBlock; |
| 540 |
} |
| 541 |
else |
| 542 |
{ |
| 543 |
pList->pHead = pBlock; |
| 544 |
} |
| 545 |
|
| 546 |
// Modify next block or tail of the list |
| 547 |
if (pBlock->pNext) |
| 548 |
{ |
| 549 |
pBlock->pNext->pPrev = pBlock; |
| 550 |
} |
| 551 |
else |
| 552 |
{ |
| 553 |
pList->pTail = pBlock; |
| 554 |
} |
| 555 |
|
| 556 |
// Track size and number of blocks in the list |
| 557 |
pList->dwSize += pBlock->dwBlockSize; |
| 558 |
pList->iCount++; |
| 559 |
|
| 560 |
return MOS_STATUS_SUCCESS; |
| 561 |
} |
| 562 |
|
| 563 |
PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::DetachBlock( |
| 564 |
MHW_BLOCK_STATE BlockState, |
| 565 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos) |
| 566 |
{ |
| 567 |
PMHW_BLOCK_LIST pList; |
| 568 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr; |
| 569 |
|
| 570 |
// Verify parameters - object cannot be null, block state must be valid |
| 571 |
if (BlockState < MHW_BLOCK_STATE_POOL || |
| 572 |
BlockState >= MHW_BLOCK_STATE_COUNT) |
| 573 |
{ |
| 574 |
return nullptr; |
| 575 |
} |
| 576 |
|
| 577 |
// Get list associated with block state |
| 578 |
pList = &m_BlockList[BlockState]; |
| 579 |
|
| 580 |
// Remove block from list, performing sanity check (check if block is in correct list) |
| 581 |
pBlock = DetachBlockInternal(pList, pBlockPos); |
| 582 |
|
| 583 |
return pBlock; |
| 584 |
} |
| 585 |
|
| 586 |
PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::DetachBlockInternal( |
| 587 |
PMHW_BLOCK_LIST pList, |
| 588 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock) |
| 589 |
{ |
| 590 |
|
| 591 |
if(pList == nullptr) |
| 592 |
{ |
| 593 |
return nullptr; |
| 594 |
} |
| 595 |
|
| 596 |
#ifdef MHW_DYNAMIC_STATE_HEAP_LOGGING |
| 597 |
const char *szPos = "REF "; |
| 598 |
if (pBlock == MHW_BLOCK_POSITION_HEAD) |
| 599 |
szPos = "HEAD"; |
| 600 |
else if (pBlock == MHW_BLOCK_POSITION_TAIL) |
| 601 |
szPos = "TAIL"; |
| 602 |
#endif |
| 603 |
|
| 604 |
// Get block from the head of the list |
| 605 |
if (pBlock == MHW_BLOCK_POSITION_HEAD) |
| 606 |
{ |
| 607 |
pBlock = pList->pHead; |
| 608 |
} |
| 609 |
// Get block from the tail of the list |
| 610 |
else if (pBlock == MHW_BLOCK_POSITION_TAIL) |
| 611 |
{ |
| 612 |
pBlock = pList->pTail; |
| 613 |
} |
| 614 |
// Block does not belong to correct list - ASSERT and get block from head of the list |
| 615 |
else if (pBlock->BlockState != pList->BlockState) |
| 616 |
{ |
| 617 |
MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_DetachBlock_Internal: Block provided does not belong to the list provided"); |
| 618 |
pBlock = nullptr; |
| 619 |
} |
| 620 |
|
| 621 |
if (!pBlock) |
| 622 |
{ |
| 623 |
return nullptr; |
| 624 |
} |
| 625 |
|
| 626 |
// Detach block from previous; if first, update head of the list |
| 627 |
if (pBlock->pPrev) |
| 628 |
{ |
| 629 |
pBlock->pPrev->pNext = pBlock->pNext; |
| 630 |
} |
| 631 |
else |
| 632 |
{ |
| 633 |
pList->pHead = pBlock->pNext; |
| 634 |
} |
| 635 |
|
| 636 |
// Detach block from next; if last, update tail of the list |
| 637 |
if (pBlock->pNext) |
| 638 |
{ |
| 639 |
pBlock->pNext->pPrev = pBlock->pPrev; |
| 640 |
} |
| 641 |
else |
| 642 |
{ |
| 643 |
pList->pTail = pBlock->pPrev; |
| 644 |
} |
| 645 |
|
| 646 |
// reset pointers - block is detached |
| 647 |
pBlock->pNext = pBlock->pPrev = nullptr; |
| 648 |
|
| 649 |
// track size and number of block in the list |
| 650 |
pList->dwSize -= pBlock->dwBlockSize; |
| 651 |
pList->iCount--; |
| 652 |
|
| 653 |
return pBlock; |
| 654 |
} |
| 655 |
|
| 656 |
MOS_STATUS MHW_BLOCK_MANAGER::MoveBlock( |
| 657 |
PMHW_BLOCK_LIST pSrcList, // Source list |
| 658 |
PMHW_BLOCK_LIST pDstList, // Destination list |
| 659 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, // Block to be moved (or HEAD/TAIL of source list) |
| 660 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos) // Position to insert (or HEAD/TAIL of target list) |
| 661 |
{ |
| 662 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 663 |
|
| 664 |
BLOCK_MANAGER_CHK_NULL(pSrcList); |
| 665 |
BLOCK_MANAGER_CHK_NULL(pDstList); |
| 666 |
|
| 667 |
pBlock = DetachBlockInternal(pSrcList, pBlock); |
| 668 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 669 |
|
| 670 |
eStatus = AttachBlockInternal(pDstList, pDstList->BlockState, pBlock, pBlockPos); |
| 671 |
|
| 672 |
if (eStatus != MOS_STATUS_SUCCESS) |
| 673 |
{ |
| 674 |
MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_MoveBlock_Internal: Failed to move block"); |
| 675 |
AttachBlockInternal(pDstList, pDstList->BlockState, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 676 |
} |
| 677 |
|
| 678 |
return eStatus; |
| 679 |
} |
| 680 |
|
| 681 |
void MHW_BLOCK_MANAGER::ReturnBlockToPool(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock) |
| 682 |
{ |
| 683 |
pBlock->dwBlockSize = 0; |
| 684 |
pBlock->pPrev = pBlock->pNext = nullptr; |
| 685 |
|
| 686 |
AttachBlock(MHW_BLOCK_STATE_POOL, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 687 |
|
| 688 |
return; |
| 689 |
} |
| 690 |
|
| 691 |
MOS_STATUS MHW_BLOCK_MANAGER::Refresh() |
| 692 |
{ |
| 693 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, pNext; |
| 694 |
PMHW_BLOCK_LIST pList; |
| 695 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 696 |
|
| 697 |
// Refresh status of SUBMITTED blocks |
| 698 |
pList = &m_BlockList[MHW_BLOCK_STATE_SUBMITTED]; |
| 699 |
pNext = nullptr; |
| 700 |
for (pBlock = pList->pHead; pBlock != nullptr; pBlock = pNext) |
| 701 |
{ |
| 702 |
// check of block may be released - if not, continue loop |
| 703 |
// NOTE - blocks are to be inserted in the sequence of execution, |
| 704 |
// so we may finish the search as soon as we find the first |
| 705 |
// block still "in use" (block tag > current sync tag) |
| 706 |
// For now we're doing an exhaustive search, but it may not be needed - once |
| 707 |
// we find the first block still in execution, we can probably stop the search |
| 708 |
|
| 709 |
// Save next pointer before moving the block to another queue |
| 710 |
pNext = pBlock->pNext; |
| 711 |
|
| 712 |
// Check if block is still in use, if so, continue search |
| 713 |
// NOTE - the following expression avoids sync tag wrapping around MAX_INT -> 0 |
| 714 |
if (!FrameTrackerTokenFlat_IsExpired(&pBlock->trackerToken)) |
| 715 |
{ |
| 716 |
continue; |
| 717 |
} |
| 718 |
|
| 719 |
// Block is flagged for deletion |
| 720 |
if (pBlock->bDelete) |
| 721 |
{ |
| 722 |
BLOCK_MANAGER_CHK_STATUS(FreeBlock(pBlock)); |
| 723 |
} |
| 724 |
// Block is static -> move back to allocated list |
| 725 |
else if (pBlock->bStatic) |
| 726 |
{ |
| 727 |
BLOCK_MANAGER_CHK_STATUS(MoveBlock(pList, &m_BlockList[MHW_BLOCK_STATE_ALLOCATED], |
| 728 |
pBlock, MHW_BLOCK_POSITION_TAIL)); |
| 729 |
} |
| 730 |
else |
| 731 |
// Block is non-static -> free block |
| 732 |
{ |
| 733 |
FreeBlock(pBlock); |
| 734 |
} |
| 735 |
} |
| 736 |
|
| 737 |
return eStatus; |
| 738 |
} |
| 739 |
|
| 740 |
void MHW_BLOCK_MANAGER::ConsolidateBlock( |
| 741 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock) |
| 742 |
{ |
| 743 |
PMHW_STATE_HEAP_MEMORY_BLOCK pAux; |
| 744 |
|
| 745 |
// Check input parameters |
| 746 |
if (!pBlock || pBlock->BlockState != MHW_BLOCK_STATE_FREE) |
| 747 |
{ |
| 748 |
return; |
| 749 |
} |
| 750 |
|
| 751 |
// Consolidate pBlock with previous blocks |
| 752 |
PMHW_BLOCK_LIST pFree = &m_BlockList[MHW_BLOCK_STATE_FREE]; |
| 753 |
for (pAux = pBlock->pHeapPrev; (pAux != nullptr) && (pAux->BlockState == MHW_BLOCK_STATE_FREE); pAux = pBlock->pHeapPrev) |
| 754 |
{ |
| 755 |
// Remove block from free block list |
| 756 |
DetachBlock(MHW_BLOCK_STATE_FREE, pAux); |
| 757 |
|
| 758 |
// Absorb free space into original block by adjusting offset/size |
| 759 |
pBlock->dwOffsetInStateHeap -= pAux->dwBlockSize; |
| 760 |
pBlock->dwBlockSize += pAux->dwBlockSize; |
| 761 |
pFree->dwSize += pAux->dwBlockSize; |
| 762 |
|
| 763 |
// Detach memory block from sequential memory list |
| 764 |
pBlock->pHeapPrev = pAux->pHeapPrev; |
| 765 |
if (pBlock->pHeapPrev) |
| 766 |
{ |
| 767 |
pBlock->pHeapPrev->pHeapNext = pBlock; |
| 768 |
} |
| 769 |
else |
| 770 |
{ |
| 771 |
pBlock->pStateHeap->pMemoryHead = pBlock; |
| 772 |
} |
| 773 |
|
| 774 |
// Memory block object no longer needed - return to pool after consolidation |
| 775 |
ReturnBlockToPool(pAux); |
| 776 |
} |
| 777 |
|
| 778 |
// Consolidate pBlock with following blocks |
| 779 |
for (pAux = pBlock->pHeapNext; (pAux != nullptr) && (pAux->BlockState == MHW_BLOCK_STATE_FREE); pAux = pBlock->pHeapNext) |
| 780 |
{ |
| 781 |
// Remove block from free block list |
| 782 |
DetachBlock(MHW_BLOCK_STATE_FREE, pAux); |
| 783 |
|
| 784 |
// Absorb free space into original block by adjusting block size (offset remains the same) |
| 785 |
pBlock->dwBlockSize += pAux->dwBlockSize; |
| 786 |
pFree->dwSize += pAux->dwBlockSize; |
| 787 |
|
| 788 |
// Detach memory block from sequential memory list |
| 789 |
pBlock->pHeapNext = pAux->pHeapNext; |
| 790 |
if (pBlock->pHeapNext) |
| 791 |
{ |
| 792 |
pBlock->pHeapNext->pHeapPrev = pBlock; |
| 793 |
} |
| 794 |
else |
| 795 |
{ |
| 796 |
pBlock->pStateHeap->pMemoryTail = pBlock; |
| 797 |
} |
| 798 |
|
| 799 |
// Memory block object no longer needed - return to pool after consolidation |
| 800 |
ReturnBlockToPool(pAux); |
| 801 |
} |
| 802 |
} |
| 803 |
|
| 804 |
MOS_STATUS MHW_BLOCK_MANAGER::AllocateBlockInternal( |
| 805 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, |
| 806 |
uint32_t dwAlignment) |
| 807 |
{ |
| 808 |
PMHW_STATE_HEAP pHeap; |
| 809 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 810 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 811 |
|
| 812 |
// Remove block from free list - if block is not in free list, this operation will fail and return nullptr |
| 813 |
pBlock = DetachBlock( MHW_BLOCK_STATE_FREE, pBlock); |
| 814 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 815 |
|
| 816 |
// Initialize block data structures |
| 817 |
pBlock->bDelete = false; |
| 818 |
|
| 819 |
pBlock->dwDataOffset = MOS_ALIGN_CEIL(pBlock->dwOffsetInStateHeap, dwAlignment); |
| 820 |
pBlock->dwAlignment = pBlock->dwDataOffset - pBlock->dwOffsetInStateHeap; |
| 821 |
pBlock->dwDataSize = pBlock->dwBlockSize - pBlock->dwAlignment; |
| 822 |
pBlock->pDataPtr = (uint8_t*)pBlock->pStateHeap->pvLockedHeap + pBlock->dwDataOffset; |
| 823 |
|
| 824 |
// Move block to allocated list |
| 825 |
AttachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 826 |
|
| 827 |
// Update available space in heap |
| 828 |
pHeap = pBlock->pStateHeap; |
| 829 |
pHeap->dwFree -= pBlock->dwBlockSize; |
| 830 |
pHeap->dwUsed += pBlock->dwBlockSize; |
| 831 |
|
| 832 |
return MOS_STATUS_SUCCESS; |
| 833 |
} |
| 834 |
|
| 835 |
MOS_STATUS MHW_BLOCK_MANAGER::SplitBlockInternal( |
| 836 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, |
| 837 |
uint32_t dwSplitSize, |
| 838 |
uint32_t dwAlignment, |
| 839 |
bool bBackward) |
| 840 |
{ |
| 841 |
uint32_t dwSplitOffset = 0; |
| 842 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 843 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockL, pBlockH; |
| 844 |
|
| 845 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 846 |
|
| 847 |
// Split cannot be less than min block size allowed |
| 848 |
dwSplitSize = MOS_MAX(dwSplitSize, m_Params.dwHeapBlockMinSize); |
| 849 |
if (pBlock->dwBlockSize < dwSplitSize) |
| 850 |
{ |
| 851 |
return MOS_STATUS_UNKNOWN; |
| 852 |
} |
| 853 |
|
| 854 |
// Verify block state |
| 855 |
if (pBlock->BlockState <= MHW_BLOCK_STATE_POOL || // Cannot split a block object from pool (contains invalid data) |
| 856 |
pBlock->BlockState >= MHW_BLOCK_STATE_DELETED) // Cannot split a block being deleted |
| 857 |
{ |
| 858 |
return MOS_STATUS_INVALID_PARAMETER; |
| 859 |
|
| 860 |
} |
| 861 |
|
| 862 |
// Select list |
| 863 |
PMHW_BLOCK_LIST pList = &(m_BlockList[pBlock->BlockState]); |
| 864 |
|
| 865 |
if (bBackward) |
| 866 |
{ |
| 867 |
// Split FROM THE END of the block (higher offset) |
| 868 |
dwSplitOffset = MOS_ALIGN_FLOOR(pBlock->dwOffsetInStateHeap + pBlock->dwBlockSize - dwSplitSize, dwAlignment); |
| 869 |
dwSplitOffset = MOS_ALIGN_FLOOR(dwSplitOffset, m_Params.dwHeapGranularity); |
| 870 |
} |
| 871 |
else |
| 872 |
{ |
| 873 |
// Split FROM THE BEGINNING of the block (lower offset) |
| 874 |
dwSplitOffset = MOS_ALIGN_CEIL(pBlock->dwOffsetInStateHeap, dwAlignment); |
| 875 |
dwSplitOffset = MOS_ALIGN_CEIL(dwSplitOffset + dwSplitSize, m_Params.dwHeapGranularity); |
| 876 |
} |
| 877 |
|
| 878 |
// Fail if block cannot be split |
| 879 |
if (dwSplitOffset < pBlock->dwOffsetInStateHeap + m_Params.dwHeapBlockMinSize || // First fragment is too small |
| 880 |
pBlock->dwOffsetInStateHeap + pBlock->dwBlockSize < dwSplitOffset + m_Params.dwHeapBlockMinSize) // Second fragment is too small |
| 881 |
{ |
| 882 |
return MOS_STATUS_UNKNOWN; |
| 883 |
} |
| 884 |
|
| 885 |
if (bBackward) |
| 886 |
{ |
| 887 |
pBlockH = pBlock; // We'll keep the high end of the block |
| 888 |
pBlockL = GetBlockFromPool(); |
| 889 |
BLOCK_MANAGER_CHK_NULL(pBlockL); |
| 890 |
|
| 891 |
uint32_t reserved = pBlockL->Reserved; |
| 892 |
*pBlockL = *pBlock; |
| 893 |
pBlockL->Reserved = reserved; |
| 894 |
|
| 895 |
if (pBlock->pPrev) |
| 896 |
{ |
| 897 |
pBlock->pPrev->pNext = pBlockL; |
| 898 |
} |
| 899 |
else |
| 900 |
{ |
| 901 |
pList->pHead = pBlockL; |
| 902 |
} |
| 903 |
|
| 904 |
if (pBlock->pHeapPrev) |
| 905 |
{ |
| 906 |
pBlock->pHeapPrev->pHeapNext = pBlockL; |
| 907 |
} |
| 908 |
else |
| 909 |
{ |
| 910 |
pBlock->pStateHeap->pMemoryHead = pBlockL; |
| 911 |
} |
| 912 |
} |
| 913 |
else |
| 914 |
{ |
| 915 |
pBlockL = pBlock; // We'll keep the low end of the block |
| 916 |
pBlockH = GetBlockFromPool(); |
| 917 |
BLOCK_MANAGER_CHK_NULL(pBlockH); |
| 918 |
|
| 919 |
uint32_t reserved = pBlockH->Reserved; |
| 920 |
*pBlockH = *pBlock; |
| 921 |
pBlockH->Reserved = reserved; |
| 922 |
|
| 923 |
if (pBlock->pNext) |
| 924 |
{ |
| 925 |
pBlock->pNext->pPrev = pBlockH; |
| 926 |
} |
| 927 |
else |
| 928 |
{ |
| 929 |
pList->pTail = pBlockH; |
| 930 |
} |
| 931 |
|
| 932 |
if (pBlock->pHeapNext) |
| 933 |
{ |
| 934 |
pBlock->pHeapNext->pHeapPrev = pBlockH; |
| 935 |
} |
| 936 |
else |
| 937 |
{ |
| 938 |
pBlock->pStateHeap->pMemoryTail = pBlockH; |
| 939 |
} |
| 940 |
} |
| 941 |
|
| 942 |
// Update block adjacency list |
| 943 |
pBlockL->pHeapNext = pBlockH; |
| 944 |
pBlockH->pHeapPrev = pBlockL; |
| 945 |
|
| 946 |
// Ensures that the new block is tracked in the same list as the parent block, update block count |
| 947 |
pList->iCount++; |
| 948 |
pBlockL->pNext = pBlockH; |
| 949 |
pBlockH->pPrev = pBlockL; |
| 950 |
|
| 951 |
// Adjust Block sizes |
| 952 |
pBlockL->dwBlockSize = dwSplitOffset - pBlockL->dwOffsetInStateHeap; // Updates L block size based on split offset |
| 953 |
pBlockH->dwOffsetInStateHeap = dwSplitOffset; // Sets 2nd block offset |
| 954 |
pBlockH->dwBlockSize -= pBlockL->dwBlockSize; // Updates H block size by subtracting L block size |
| 955 |
|
| 956 |
// Adjust Block data related pointers/sizes only if block is not free |
| 957 |
if (pBlockL->BlockState != MHW_BLOCK_STATE_FREE) |
| 958 |
{ |
| 959 |
pBlockL->dwDataSize -= pBlockH->dwBlockSize; // Removes size of new block from amount of data available |
| 960 |
pBlockH->dwDataOffset = MOS_ALIGN_CEIL(dwSplitOffset, dwAlignment); // Adjust offset to data (accounting for alignment) |
| 961 |
pBlockH->dwAlignment = pBlockH->dwDataOffset - dwSplitOffset; // Calculate alignment shift |
| 962 |
pBlockH->dwDataSize = pBlockH->dwBlockSize - dwAlignment; // Adjust amount of data available |
| 963 |
pBlockH->pDataPtr = (uint8_t*)pBlockH->pStateHeap->pvLockedHeap + pBlockH->dwDataOffset; // Setup pointer to data (the heap is locked) |
| 964 |
} |
| 965 |
|
| 966 |
return eStatus; |
| 967 |
} |
| 968 |
|
| 969 |
MOS_STATUS MHW_BLOCK_MANAGER::MergeBlocksInternal( |
| 970 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockL, // block in lower memory |
| 971 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockH, // block in higher memory |
| 972 |
uint32_t dwAlignment, // final block alignment |
| 973 |
bool bBackward) // true if pBlockL (free) is merged into pBlockH; false if pBlockH (free) is merged into pBlockL |
| 974 |
{ |
| 975 |
PMHW_BLOCK_LIST pList; |
| 976 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 977 |
|
| 978 |
BLOCK_MANAGER_CHK_NULL(pBlockL); |
| 979 |
BLOCK_MANAGER_CHK_NULL(pBlockH); |
| 980 |
|
| 981 |
// Blocks must be contiguous in memory |
| 982 |
if (pBlockL->pHeapNext != pBlockH || |
| 983 |
pBlockH->pHeapPrev != pBlockL) |
| 984 |
{ |
| 985 |
return MOS_STATUS_INVALID_PARAMETER; |
| 986 |
} |
| 987 |
|
| 988 |
// 1st block (L) is merged into 2nd (H) |
| 989 |
if (bBackward) |
| 990 |
{ |
| 991 |
if (pBlockL->BlockState != MHW_BLOCK_STATE_FREE || // 1st block must be free |
| 992 |
pBlockH->BlockState < MHW_BLOCK_STATE_FREE || // 2nd block cannot be in pool |
| 993 |
pBlockH->BlockState > MHW_BLOCK_STATE_SUBMITTED) // or deleted |
| 994 |
{ |
| 995 |
return MOS_STATUS_INVALID_PARAMETER; |
| 996 |
} |
| 997 |
|
| 998 |
// Merge blocks |
| 999 |
pBlockL = DetachBlock(MHW_BLOCK_STATE_FREE, pBlockL); |
| 1000 |
BLOCK_MANAGER_CHK_NULL(pBlockL); |
| 1001 |
|
| 1002 |
pBlockH->dwOffsetInStateHeap = pBlockL->dwOffsetInStateHeap; |
| 1003 |
pBlockH->dwBlockSize += pBlockL->dwBlockSize; |
| 1004 |
|
| 1005 |
// Add size to the target block list |
| 1006 |
pList = &(m_BlockList[pBlockH->BlockState]); |
| 1007 |
pList->dwSize += pBlockL->dwBlockSize; |
| 1008 |
|
| 1009 |
// If block allocated or submitted, adjust data references (don't care if block is free) |
| 1010 |
if (pBlockH->BlockState != MHW_BLOCK_STATE_FREE) |
| 1011 |
{ |
| 1012 |
pBlockH->dwDataOffset = MOS_ALIGN_CEIL(pBlockH->dwOffsetInStateHeap, dwAlignment); |
| 1013 |
pBlockH->dwAlignment = pBlockH->dwDataOffset - pBlockH->dwOffsetInStateHeap; |
| 1014 |
pBlockH->dwDataSize = pBlockH->dwBlockSize - pBlockH->dwAlignment; |
| 1015 |
pBlockH->pDataPtr = (uint8_t*)pBlockH->pStateHeap->pvLockedHeap + pBlockH->dwDataOffset; |
| 1016 |
|
| 1017 |
// Free block is now in use - track heap usage |
| 1018 |
pBlockH->pStateHeap->dwFree -= pBlockL->dwBlockSize; |
| 1019 |
pBlockH->pStateHeap->dwUsed += pBlockL->dwBlockSize; |
| 1020 |
} |
| 1021 |
|
| 1022 |
// Return block object to the pool |
| 1023 |
ReturnBlockToPool(pBlockL); |
| 1024 |
} |
| 1025 |
else |
| 1026 |
// 2nd block (H) is merged into 1st (L) |
| 1027 |
{ |
| 1028 |
if (pBlockH->BlockState != MHW_BLOCK_STATE_FREE || // 2nd block must be free |
| 1029 |
pBlockL->BlockState < MHW_BLOCK_STATE_FREE || // 1nd block must not be in pool |
| 1030 |
pBlockL->BlockState > MHW_BLOCK_STATE_SUBMITTED) // or deleted |
| 1031 |
{ |
| 1032 |
return MOS_STATUS_INVALID_PARAMETER; |
| 1033 |
} |
| 1034 |
|
| 1035 |
// Merge blocks |
| 1036 |
pBlockH = DetachBlock(MHW_BLOCK_STATE_FREE, pBlockH); |
| 1037 |
BLOCK_MANAGER_CHK_NULL(pBlockH); |
| 1038 |
|
| 1039 |
pBlockL->dwBlockSize += pBlockH->dwBlockSize; |
| 1040 |
if (pBlockL->BlockState != MHW_BLOCK_STATE_FREE) |
| 1041 |
{ |
| 1042 |
pBlockL->dwDataSize += pBlockH->dwBlockSize; |
| 1043 |
pBlockL->pStateHeap->dwFree -= pBlockL->dwBlockSize; |
| 1044 |
pBlockL->pStateHeap->dwUsed += pBlockL->dwBlockSize; |
| 1045 |
} |
| 1046 |
|
| 1047 |
// Add size to the target block list |
| 1048 |
pList = &(m_BlockList[pBlockL->BlockState]); |
| 1049 |
pList->dwSize += pBlockH->dwBlockSize; |
| 1050 |
|
| 1051 |
// Return block object to the pool |
| 1052 |
ReturnBlockToPool(pBlockH); |
| 1053 |
} |
| 1054 |
|
| 1055 |
return eStatus; |
| 1056 |
} |
| 1057 |
|
| 1058 |
MOS_STATUS MHW_BLOCK_MANAGER::ResizeBlock( |
| 1059 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, |
| 1060 |
uint32_t dwNewSize, |
| 1061 |
uint32_t dwAlignment, |
| 1062 |
bool bBackward) // false => Always grow/shrink forward; true => allow block to grow forward/backwards (moving its start offset) |
| 1063 |
{ |
| 1064 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 1065 |
PMHW_STATE_HEAP_MEMORY_BLOCK pNewBlock; |
| 1066 |
|
| 1067 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 1068 |
MHW_ASSERT(dwNewSize > 0); |
| 1069 |
|
| 1070 |
// Verify block state |
| 1071 |
if (pBlock->BlockState <= MHW_BLOCK_STATE_POOL || // Cannot touch a block object from pool - invalid data |
| 1072 |
pBlock->BlockState >= MHW_BLOCK_STATE_DELETED) // Cannot touch a block being deleted |
| 1073 |
{ |
| 1074 |
return MOS_STATUS_INVALID_PARAMETER; |
| 1075 |
} |
| 1076 |
|
| 1077 |
// Shrinking current block |
| 1078 |
if (dwNewSize < pBlock->dwBlockSize) |
| 1079 |
{ |
| 1080 |
// Split block into 2 - (bBackwards -> shrink the block by keeping the 2nd half (anchor end of the block) |
| 1081 |
eStatus = SplitBlockInternal(pBlock, dwNewSize, dwAlignment, bBackward); |
| 1082 |
if (eStatus != MOS_STATUS_SUCCESS) |
| 1083 |
{ |
| 1084 |
// This error just means that the block cannot shrink - not an actual issue here |
| 1085 |
if (eStatus == MOS_STATUS_UNKNOWN) |
| 1086 |
{ |
| 1087 |
eStatus = MOS_STATUS_SUCCESS; |
| 1088 |
return eStatus; |
| 1089 |
} |
| 1090 |
} |
| 1091 |
|
| 1092 |
// Select block to be freed |
| 1093 |
pBlock = (bBackward) ? pBlock->pPrev : pBlock->pNext; |
| 1094 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 1095 |
|
| 1096 |
if (pBlock->BlockState == MHW_BLOCK_STATE_SUBMITTED) |
| 1097 |
{ |
| 1098 |
// mark block for release as soon it is no longer in use by GPU |
| 1099 |
pBlock->bStatic = false; |
| 1100 |
} |
| 1101 |
else |
| 1102 |
{ |
| 1103 |
// Free block - block is in allocated state |
| 1104 |
FreeBlock(pBlock); |
| 1105 |
} |
| 1106 |
|
| 1107 |
// Success! |
| 1108 |
return MOS_STATUS_SUCCESS; |
| 1109 |
} |
| 1110 |
|
| 1111 |
// Check for contiguous available space in forward direction first |
| 1112 |
uint32_t dwAvailable = pBlock->dwDataSize; |
| 1113 |
for (pNewBlock = pBlock->pHeapNext; |
| 1114 |
(dwAvailable < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE); |
| 1115 |
pNewBlock = pNewBlock->pHeapNext) |
| 1116 |
{ |
| 1117 |
dwAvailable += pNewBlock->dwBlockSize; |
| 1118 |
} |
| 1119 |
|
| 1120 |
// Check for contiguous available space in backward direction |
| 1121 |
if (bBackward) |
| 1122 |
{ |
| 1123 |
// Update available space to account for block alignment |
| 1124 |
dwAvailable += pBlock->dwAlignment - dwAlignment; |
| 1125 |
for (pNewBlock = pBlock->pHeapPrev; |
| 1126 |
(dwAvailable < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE); |
| 1127 |
pNewBlock = pNewBlock->pHeapPrev) |
| 1128 |
{ |
| 1129 |
dwAvailable += pNewBlock->dwBlockSize; |
| 1130 |
} |
| 1131 |
} |
| 1132 |
|
| 1133 |
// Check if block can be resized |
| 1134 |
if (dwAvailable < dwNewSize) |
| 1135 |
{ |
| 1136 |
return MOS_STATUS_UNKNOWN; |
| 1137 |
} |
| 1138 |
|
| 1139 |
// Start block expansion forward |
| 1140 |
for (pNewBlock = pBlock->pHeapNext; |
| 1141 |
(pBlock->dwDataSize < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE); |
| 1142 |
pNewBlock = pBlock->pHeapNext) |
| 1143 |
{ |
| 1144 |
// Next block is too large - split the block before merging |
| 1145 |
if (pBlock->dwDataSize + pNewBlock->dwBlockSize > dwNewSize) |
| 1146 |
{ |
| 1147 |
SplitBlockInternal(pNewBlock, dwNewSize - pBlock->dwDataSize, dwAlignment, false); |
| 1148 |
} |
| 1149 |
|
| 1150 |
// Merge block with next |
| 1151 |
MergeBlocksInternal(pBlock, pNewBlock, dwAlignment, false); |
| 1152 |
} |
| 1153 |
|
| 1154 |
// Continue expanding backward |
| 1155 |
if (bBackward) |
| 1156 |
{ |
| 1157 |
for (pNewBlock = pBlock->pHeapPrev; |
| 1158 |
(dwAvailable < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE); |
| 1159 |
pNewBlock = pBlock->pHeapPrev) |
| 1160 |
{ |
| 1161 |
// Prev block is too large - split the block before merging |
| 1162 |
uint32_t dwAdjust = MOS_ALIGN_CEIL(pNewBlock->dwOffsetInStateHeap, dwAlignment) - pNewBlock->dwOffsetInStateHeap; |
| 1163 |
if (pBlock->dwBlockSize + pNewBlock->dwBlockSize - dwAdjust > dwNewSize) |
| 1164 |
{ |
| 1165 |
SplitBlockInternal(pNewBlock, dwNewSize - pBlock->dwBlockSize, dwAlignment, true); |
| 1166 |
} |
| 1167 |
|
| 1168 |
// Merge block with previous |
| 1169 |
MergeBlocksInternal(pNewBlock, pBlock, dwAlignment, true); |
| 1170 |
} |
| 1171 |
} |
| 1172 |
|
| 1173 |
return eStatus; |
| 1174 |
} |
| 1175 |
|
| 1176 |
PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::AllocateWithScratchSpace( |
| 1177 |
uint32_t dwSize, |
| 1178 |
uint32_t dwAlignment, |
| 1179 |
uint32_t dwScratchSpace) |
| 1180 |
{ |
| 1181 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 1182 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr; |
| 1183 |
PMHW_STATE_HEAP_MEMORY_BLOCK pScratch = nullptr; |
| 1184 |
|
| 1185 |
// Fix alignment - must be a power of 2 |
| 1186 |
if (dwAlignment < m_Params.dwHeapGranularity) |
| 1187 |
{ |
| 1188 |
// Blocks are already aligned |
| 1189 |
dwAlignment = 1; |
| 1190 |
} |
| 1191 |
else |
| 1192 |
{ |
| 1193 |
dwAlignment--; |
| 1194 |
dwAlignment |= dwAlignment >> 1; |
| 1195 |
dwAlignment |= dwAlignment >> 2; |
| 1196 |
dwAlignment |= dwAlignment >> 4; |
| 1197 |
dwAlignment |= dwAlignment >> 8; |
| 1198 |
dwAlignment |= dwAlignment >> 16; |
| 1199 |
dwAlignment++; |
| 1200 |
} |
| 1201 |
|
| 1202 |
// Try to search search state heap with large enough scratch space |
| 1203 |
// and with enough free space |
| 1204 |
PMHW_STATE_HEAP pNextStateHeap; |
| 1205 |
for (PMHW_STATE_HEAP pStateHeap = m_pStateHeap; (pStateHeap); pStateHeap = pNextStateHeap) |
| 1206 |
{ |
| 1207 |
// Save next state heap |
| 1208 |
pNextStateHeap = pStateHeap->pNext; |
| 1209 |
|
| 1210 |
// Space needed for block (accounting for alignment and for scratch space) |
| 1211 |
uint32_t dwBlockNeeded = MOS_ALIGN_CEIL(dwSize + dwAlignment - 1, m_Params.dwHeapGranularity); |
| 1212 |
|
| 1213 |
// Scratch space needed = space requested + room for alignment - space already allocated |
| 1214 |
uint32_t dwScratchNeeded = 0; |
| 1215 |
if (dwScratchSpace > 0 && pStateHeap->dwScratchSpace < dwScratchSpace) |
| 1216 |
{ |
| 1217 |
dwScratchNeeded = dwScratchSpace; |
| 1218 |
if (m_Params.dwHeapGranularity < MHW_SCRATCH_SPACE_ALIGN) |
| 1219 |
{ |
| 1220 |
dwScratchNeeded += MHW_SCRATCH_SPACE_ALIGN - m_Params.dwHeapGranularity; |
| 1221 |
} |
| 1222 |
if (pStateHeap->pScratchSpace) |
| 1223 |
{ |
| 1224 |
dwScratchNeeded -= pStateHeap->pScratchSpace->dwBlockSize; |
| 1225 |
} |
| 1226 |
} |
| 1227 |
|
| 1228 |
if (pStateHeap->dwSize < dwScratchNeeded) |
| 1229 |
{ |
| 1230 |
// Heap is too small for current scratch space size - mark for deletion |
| 1231 |
pStateHeap->pMhwStateHeapInterface->ReleaseStateHeapDyn(pStateHeap); |
| 1232 |
continue; |
| 1233 |
} |
| 1234 |
else if (pStateHeap->dwFree < (dwScratchNeeded + dwBlockNeeded)) |
| 1235 |
{ |
| 1236 |
// Heap can still be used, but currently full, try next heap |
| 1237 |
continue; |
| 1238 |
} |
| 1239 |
|
| 1240 |
// Allocate scratch space first |
| 1241 |
if (dwScratchNeeded) |
| 1242 |
{ |
| 1243 |
pScratch = nullptr; |
| 1244 |
|
| 1245 |
// Already present - EXPAND SCRATCH SPACE |
| 1246 |
if (pStateHeap->pScratchSpace) |
| 1247 |
{ |
| 1248 |
// Resize existing scratch space trying to use all free space towards the end of the heap and then growing towards the center. |
| 1249 |
eStatus = ResizeBlock(pStateHeap->pScratchSpace, dwScratchSpace, MHW_SCRATCH_SPACE_ALIGN, true); |
| 1250 |
if (eStatus == MOS_STATUS_SUCCESS) |
| 1251 |
{ |
| 1252 |
pScratch = pStateHeap->pScratchSpace; // Indicates success |
| 1253 |
pStateHeap->dwScratchSpace = pScratch->dwDataSize; // Available scratch space size (aligned) |
| 1254 |
} |
| 1255 |
} |
| 1256 |
|
| 1257 |
// Scratch space not present or failed to expand - find a new scratch space |
| 1258 |
if (!pScratch) |
| 1259 |
{ |
| 1260 |
// Search for scratch space at the end of the heap towards the beginning |
| 1261 |
// This model allows for better growth without collision with media state heaps |
| 1262 |
for (pScratch = pStateHeap->pMemoryTail; pScratch != pBlock; pScratch = pScratch->pHeapPrev) |
| 1263 |
{ |
| 1264 |
if (pScratch->BlockState == MHW_BLOCK_STATE_FREE && |
| 1265 |
pScratch->dwBlockSize >= dwScratchNeeded) |
| 1266 |
{ |
| 1267 |
// Found scratch space large enough |
| 1268 |
break; |
| 1269 |
} |
| 1270 |
} |
| 1271 |
|
| 1272 |
// Not enough contiguous space for scratch in heap - try next heap |
| 1273 |
if (!pScratch) |
| 1274 |
{ |
| 1275 |
continue; |
| 1276 |
} |
| 1277 |
|
| 1278 |
// CREATE SCRATCH SPACE |
| 1279 |
|
| 1280 |
// Split block to the necessary size, using the higher portion (closer to end of the heap) |
| 1281 |
// NOTE: Block select could be much larger than needed, even the size of the entire state heap - that's why it needs to be split |
| 1282 |
eStatus = SplitBlockInternal(pScratch, dwScratchNeeded, MHW_SCRATCH_SPACE_ALIGN, true); |
| 1283 |
if (eStatus == MOS_STATUS_UNKNOWN) eStatus = MOS_STATUS_SUCCESS; // Don't care if block could not be split |
| 1284 |
if (eStatus != MOS_STATUS_SUCCESS) |
| 1285 |
{ |
| 1286 |
continue; |
| 1287 |
} |
| 1288 |
|
| 1289 |
// Move block to allocated list, mark it as static (do not release upon completion) |
| 1290 |
AllocateBlockInternal(pScratch, MHW_SCRATCH_SPACE_ALIGN); |
| 1291 |
pScratch->bStatic = true; |
| 1292 |
|
| 1293 |
// Free the old scratch space |
| 1294 |
// NOTE: scratch spaces are also tracked by Sync Tags and maintained in submitted/allocated lists, |
| 1295 |
// If in use, it just clears the bStatic flag, so it will be freed when no longer in use. |
| 1296 |
if (pStateHeap->pScratchSpace) |
| 1297 |
{ |
| 1298 |
FreeBlock(pStateHeap->pScratchSpace); |
| 1299 |
} |
| 1300 |
|
| 1301 |
// Setup new heap scratch space and size |
| 1302 |
pStateHeap->pScratchSpace = pScratch; |
| 1303 |
pStateHeap->dwScratchSpace = pScratch->dwDataSize; |
| 1304 |
} |
| 1305 |
} |
| 1306 |
|
| 1307 |
// Try to allocate block in the same heap as the scratch space |
| 1308 |
pBlock = AllocateBlock(dwSize, dwAlignment, pStateHeap); |
| 1309 |
if (pBlock) |
| 1310 |
{ |
| 1311 |
break; |
| 1312 |
} |
| 1313 |
} |
| 1314 |
|
| 1315 |
return pBlock; |
| 1316 |
} |
| 1317 |
|
| 1318 |
PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::AllocateBlock( |
| 1319 |
uint32_t dwSize, |
| 1320 |
uint32_t dwAlignment, |
| 1321 |
PMHW_STATE_HEAP pHeapAffinity) |
| 1322 |
{ |
| 1323 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr; |
| 1324 |
PMHW_BLOCK_LIST pFree = &m_BlockList[MHW_BLOCK_STATE_FREE]; |
| 1325 |
uint32_t dwAdjust; // Offset adjustment for alignment purposes |
| 1326 |
uint32_t dwAllocSize; // Actual allocation size accounting for alignment and other restrictions |
| 1327 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 1328 |
|
| 1329 |
// Fix alignment - must be a power of 2 (minimum 1) |
| 1330 |
if (dwAlignment) dwAlignment--; |
| 1331 |
dwAlignment |= dwAlignment >> 1; |
| 1332 |
dwAlignment |= dwAlignment >> 2; |
| 1333 |
dwAlignment |= dwAlignment >> 4; |
| 1334 |
dwAlignment |= dwAlignment >> 8; |
| 1335 |
dwAlignment |= dwAlignment >> 16; |
| 1336 |
dwAlignment++; |
| 1337 |
|
| 1338 |
// Search must include space for block granularity |
| 1339 |
if (dwAlignment <= m_Params.dwHeapGranularity) |
| 1340 |
{ |
| 1341 |
// Alignment should be fulfilled by the heap granularity |
| 1342 |
dwAllocSize = dwSize; |
| 1343 |
} |
| 1344 |
else |
| 1345 |
{ |
| 1346 |
// Worst case scenario - original implementation was checking alignment of |
| 1347 |
// each free block, but it was overkill - so now we just consider the worst case |
| 1348 |
dwAllocSize = dwSize + dwAlignment - m_Params.dwHeapGranularity; |
| 1349 |
} |
| 1350 |
|
| 1351 |
// Enforce min block size |
| 1352 |
dwAllocSize = MOS_MAX(m_Params.dwHeapBlockMinSize, dwAllocSize); |
| 1353 |
|
| 1354 |
// Search list of free blocks for the first match |
| 1355 |
for (pBlock = pFree->pHead; pBlock != nullptr; pBlock = pBlock->pNext) |
| 1356 |
{ |
| 1357 |
// Skip this heap if we are looking for allocation in a specific heap |
| 1358 |
if (pHeapAffinity && pBlock->pStateHeap != pHeapAffinity) |
| 1359 |
{ |
| 1360 |
continue; |
| 1361 |
} |
| 1362 |
|
| 1363 |
// Check if aligned block fits the request -> break with a successful block |
| 1364 |
if (pBlock->dwBlockSize >= dwAllocSize) |
| 1365 |
{ |
| 1366 |
break; |
| 1367 |
} |
| 1368 |
} |
| 1369 |
|
| 1370 |
// No block was found - fail search |
| 1371 |
if (!pBlock) |
| 1372 |
{ |
| 1373 |
return nullptr; |
| 1374 |
} |
| 1375 |
|
| 1376 |
// Block was found, adjust the allocation size to account for |
| 1377 |
// heap granularity and block alignment |
| 1378 |
dwAdjust = MOS_ALIGN_OFFSET(pBlock->dwOffsetInStateHeap, dwAlignment); // Increase in size to align data |
| 1379 |
dwAllocSize = MOS_ALIGN_CEIL(dwSize + dwAdjust, m_Params.dwHeapGranularity); // Account for heap granularity (avoid odd addresses in heap) |
| 1380 |
dwAllocSize = MOS_MAX(dwAllocSize, m_Params.dwHeapBlockMinSize); |
| 1381 |
|
| 1382 |
// Just a precaution - sanity check - in case of last block in heap, and total heap size is not a multiple of granularity |
| 1383 |
if (pBlock->dwBlockSize < dwAllocSize) |
| 1384 |
{ |
| 1385 |
// This should never happend because it is part of the search condition! |
| 1386 |
MHW_ASSERT(pBlock->dwBlockSize >= (dwAdjust + dwSize)); |
| 1387 |
|
| 1388 |
dwAllocSize = pBlock->dwBlockSize; |
| 1389 |
} |
| 1390 |
|
| 1391 |
// Split block, move to allocated list |
| 1392 |
if (pBlock->dwBlockSize > dwAllocSize) |
| 1393 |
{ |
| 1394 |
// Split free block in 2, keep the first part (lower offset) |
| 1395 |
eStatus = SplitBlockInternal(pBlock, dwAllocSize, dwAlignment, false); |
| 1396 |
if (eStatus != MOS_STATUS_SUCCESS && |
| 1397 |
eStatus != MOS_STATUS_UNKNOWN) |
| 1398 |
{ |
| 1399 |
MHW_ASSERTMESSAGE("ERROR: AllocateBlock: Failed to allocate block"); |
| 1400 |
return nullptr; |
| 1401 |
} |
| 1402 |
} |
| 1403 |
|
| 1404 |
// Move block from free to allocated queue |
| 1405 |
DetachBlock(MHW_BLOCK_STATE_FREE, pBlock); |
| 1406 |
AttachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 1407 |
pBlock->pStateHeap->dwUsed += pBlock->dwBlockSize; |
| 1408 |
pBlock->pStateHeap->dwFree -= pBlock->dwBlockSize; |
| 1409 |
|
| 1410 |
// Reset some fields |
| 1411 |
pBlock->bDelete = false; |
| 1412 |
FrameTrackerTokenFlat_Validate(&pBlock->trackerToken); |
| 1413 |
|
| 1414 |
// Setup aligned offset, size and data |
| 1415 |
pBlock->dwDataOffset = MOS_ALIGN_CEIL(pBlock->dwOffsetInStateHeap, dwAlignment); |
| 1416 |
pBlock->dwAlignment = pBlock->dwDataOffset - pBlock->dwOffsetInStateHeap; |
| 1417 |
pBlock->dwDataSize = pBlock->dwBlockSize - pBlock->dwAlignment; |
| 1418 |
pBlock->pDataPtr = (uint8_t*)pBlock->pStateHeap->pvLockedHeap + pBlock->dwDataOffset; |
| 1419 |
|
| 1420 |
// return block to client |
| 1421 |
return pBlock; |
| 1422 |
} |
| 1423 |
|
| 1424 |
MOS_STATUS MHW_BLOCK_MANAGER::FreeBlock( |
| 1425 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock) |
| 1426 |
{ |
| 1427 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 1428 |
|
| 1429 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 1430 |
|
| 1431 |
// Block still in use - mark for auto-release when complete |
| 1432 |
if (pBlock->BlockState == MHW_BLOCK_STATE_SUBMITTED) |
| 1433 |
{ |
| 1434 |
// sync tag not provided or block still in use - flag it for automatic release when complete |
| 1435 |
if (!FrameTrackerTokenFlat_IsExpired(&pBlock->trackerToken)) |
| 1436 |
{ |
| 1437 |
pBlock->bStatic = false; |
| 1438 |
return eStatus; |
| 1439 |
} |
| 1440 |
} |
| 1441 |
else if (pBlock->BlockState != MHW_BLOCK_STATE_ALLOCATED) |
| 1442 |
{ |
| 1443 |
return MOS_STATUS_INVALID_PARAMETER; |
| 1444 |
} |
| 1445 |
|
| 1446 |
// Remove block from its current list |
| 1447 |
DetachBlock(pBlock->BlockState, pBlock); |
| 1448 |
|
| 1449 |
// If block is marked for deletion - move to deleted list so it cannot be reallocated |
| 1450 |
if (pBlock->bDelete) |
| 1451 |
{ |
| 1452 |
MHW_STATE_HEAP *pStateHeap = pBlock->pStateHeap; |
| 1453 |
pStateHeap->dwUsed -= pBlock->dwBlockSize; |
| 1454 |
|
| 1455 |
AttachBlock(MHW_BLOCK_STATE_DELETED, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 1456 |
|
| 1457 |
// Last block was removed from StateHeap -> state heap may be unregistered and deleted |
| 1458 |
if (pStateHeap->dwUsed == 0) |
| 1459 |
{ |
| 1460 |
pStateHeap->pMhwStateHeapInterface->ReleaseStateHeapDyn(pStateHeap); |
| 1461 |
} |
| 1462 |
} |
| 1463 |
else |
| 1464 |
{ |
| 1465 |
// Update state heap usage |
| 1466 |
pBlock->pStateHeap->dwUsed -= pBlock->dwBlockSize; |
| 1467 |
pBlock->pStateHeap->dwFree += pBlock->dwBlockSize; |
| 1468 |
|
| 1469 |
// Blocks are freed and placed at the beginning of the free block list |
| 1470 |
AttachBlock(MHW_BLOCK_STATE_FREE, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 1471 |
|
| 1472 |
// Consolidate memory immediately after release - so free block are ALWAYS merged |
| 1473 |
ConsolidateBlock(pBlock); |
| 1474 |
} |
| 1475 |
|
| 1476 |
return eStatus; |
| 1477 |
} |
| 1478 |
|
| 1479 |
uint32_t MHW_BLOCK_MANAGER::CalculateSpaceNeeded( |
| 1480 |
const uint32_t *pdwSizes, |
| 1481 |
int32_t iCount, |
| 1482 |
uint32_t dwAlignment, |
| 1483 |
bool bHeapAffinity, |
| 1484 |
PMHW_STATE_HEAP pHeapAffinity) |
| 1485 |
{ |
| 1486 |
uint32_t dwNeeded = 0; |
| 1487 |
uint8_t SortedIndex[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; |
| 1488 |
uint32_t FreeBlockSizes[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY * 2]; |
| 1489 |
uint32_t dwBlockOverhead = 0; // Block size overhead to ensure alignment |
| 1490 |
uint32_t dwAllocSize; |
| 1491 |
|
| 1492 |
// Check parameters |
| 1493 |
if (iCount <= 0 || iCount > MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY) |
| 1494 |
{ |
| 1495 |
return dwNeeded; |
| 1496 |
} |
| 1497 |
|
| 1498 |
// This is the minimum block |
| 1499 |
uint32_t dwBlockGranularity = m_Params.dwHeapGranularity; |
| 1500 |
uint32_t dwBlockMinSize = m_Params.dwHeapBlockMinSize; |
| 1501 |
if (dwAlignment > dwBlockGranularity) |
| 1502 |
{ |
| 1503 |
dwBlockOverhead = dwAlignment - dwBlockGranularity; |
| 1504 |
} |
| 1505 |
|
| 1506 |
// Very simple case - single block search |
| 1507 |
if (iCount == 1) |
| 1508 |
{ |
| 1509 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = m_BlockList[MHW_BLOCK_STATE_FREE].pHead; |
| 1510 |
dwNeeded = pdwSizes[0]; |
| 1511 |
for ( ; pBlock != nullptr && dwNeeded > 0; pBlock = pBlock->pNext) |
| 1512 |
{ |
| 1513 |
if (bHeapAffinity && pHeapAffinity != pBlock->pStateHeap) |
| 1514 |
{ |
| 1515 |
continue; |
| 1516 |
} |
| 1517 |
|
| 1518 |
if (dwNeeded < pBlock->dwBlockSize) |
| 1519 |
{ |
| 1520 |
dwNeeded = 0; |
| 1521 |
} |
| 1522 |
} |
| 1523 |
|
| 1524 |
return dwNeeded; |
| 1525 |
} |
| 1526 |
|
| 1527 |
// Sort input block sizes (with index, to avoid modifying the input array) |
| 1528 |
Mhw_BlockManager_ReverseMergeSort_With_Index(pdwSizes, iCount, SortedIndex); |
| 1529 |
|
| 1530 |
// Read all available blocks, keep the largest blocks |
| 1531 |
int iIndex = 0; |
| 1532 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = m_BlockList[MHW_BLOCK_STATE_FREE].pHead; |
| 1533 |
for ( ; pBlock != nullptr; pBlock = pBlock->pNext) |
| 1534 |
{ |
| 1535 |
// Select free blocks that fit the request |
| 1536 |
if (bHeapAffinity && pHeapAffinity != pBlock->pStateHeap) continue; |
| 1537 |
FreeBlockSizes[iIndex++] = pBlock->dwBlockSize; |
| 1538 |
|
| 1539 |
// If buffer is full, sort it, only take the largest blocks (overwrite remaining blocks) |
| 1540 |
if (iIndex == MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY * 2) |
| 1541 |
{ |
| 1542 |
Mhw_BlockManager_ReverseMergeSort(FreeBlockSizes, iIndex); |
| 1543 |
iIndex = iCount; |
| 1544 |
} |
| 1545 |
} |
| 1546 |
|
| 1547 |
// Final sort after all block are in |
| 1548 |
Mhw_BlockManager_ReverseMergeSort(FreeBlockSizes, iIndex); |
| 1549 |
FreeBlockSizes[iIndex] = 0; // Null termination simplifies the algorithm that follows |
| 1550 |
|
| 1551 |
// Start process of fitting requested blocks with available blocks |
| 1552 |
uint32_t *pFreeBlock = &FreeBlockSizes[0]; |
| 1553 |
for (iIndex = 0; iIndex < iCount; iIndex++) |
| 1554 |
{ |
| 1555 |
dwAllocSize = MOS_ALIGN_CEIL(pdwSizes[SortedIndex[iIndex]] + dwBlockOverhead, dwBlockGranularity); |
| 1556 |
dwAllocSize = MOS_MAX(dwAllocSize, dwBlockMinSize); |
| 1557 |
|
| 1558 |
// Block doesn't fit - add to space needed |
| 1559 |
if (dwAllocSize > *pFreeBlock) |
| 1560 |
{ |
| 1561 |
dwNeeded += dwAllocSize; |
| 1562 |
} |
| 1563 |
else |
| 1564 |
{ |
| 1565 |
// Allocate block |
| 1566 |
uint32_t dwRemaining = *pFreeBlock = *pFreeBlock - dwAllocSize; |
| 1567 |
uint32_t *pAux; |
| 1568 |
|
| 1569 |
// Remaining is out of order, keep list sorted using insertion sort |
| 1570 |
if (dwRemaining < pFreeBlock[1]) |
| 1571 |
{ |
| 1572 |
for (pAux = pFreeBlock; dwRemaining < pAux[1]; pAux++) pAux[0] = pAux[1]; |
| 1573 |
pAux[0] = dwRemaining; |
| 1574 |
} |
| 1575 |
} |
| 1576 |
} |
| 1577 |
|
| 1578 |
return dwNeeded; |
| 1579 |
} |
| 1580 |
|
| 1581 |
MOS_STATUS MHW_BLOCK_MANAGER::SubmitBlock( |
| 1582 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, |
| 1583 |
const FrameTrackerTokenFlat *trackerToken) |
| 1584 |
{ |
| 1585 |
MOS_STATUS eStatus = MOS_STATUS_SUCCESS; |
| 1586 |
|
| 1587 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 1588 |
|
| 1589 |
if (pBlock->BlockState == MHW_BLOCK_STATE_POOL || // Block cannot be submitted directly from pool |
| 1590 |
pBlock->BlockState == MHW_BLOCK_STATE_FREE || // or from free (must be allocated first) |
| 1591 |
pBlock->BlockState == MHW_BLOCK_STATE_DELETED) // or from deleted (the block is being DELETED for crying out loud!) |
| 1592 |
{ |
| 1593 |
MHW_ASSERTMESSAGE("ERROR: SubmitBlock: Block in invalid state, cannot be enqueued"); |
| 1594 |
return MOS_STATUS_UNKNOWN; |
| 1595 |
} |
| 1596 |
|
| 1597 |
// Detach block from whatever list it's in - could be in SUBMITTED state (block is being reused - i.e., kernel or scratch space) |
| 1598 |
pBlock = DetachBlock(pBlock->BlockState, pBlock); |
| 1599 |
BLOCK_MANAGER_CHK_NULL(pBlock); |
| 1600 |
|
| 1601 |
// Set block sync tag - used for refreshing the block status |
| 1602 |
FrameTrackerTokenFlat_Merge(&pBlock->trackerToken, trackerToken); |
| 1603 |
|
| 1604 |
BLOCK_MANAGER_CHK_STATUS(AttachBlock(MHW_BLOCK_STATE_SUBMITTED, pBlock, MHW_BLOCK_POSITION_TAIL)); |
| 1605 |
|
| 1606 |
return eStatus; |
| 1607 |
} |
| 1608 |
|
| 1609 |
PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::AllocateMultiple( |
| 1610 |
uint32_t *pdwSizes, |
| 1611 |
int32_t iCount, |
| 1612 |
uint32_t dwAlignment, |
| 1613 |
bool bHeapAffinity, |
| 1614 |
PMHW_STATE_HEAP pHeapAffinity) |
| 1615 |
{ |
| 1616 |
PMHW_STATE_HEAP pStateHeap; |
| 1617 |
uint32_t dwTotalSize = 0; |
| 1618 |
uint8_t SortedIndex[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // uint8_t is only used because the array size is <256 |
| 1619 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlockArray[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // This could be allocated dynamically, same for the above |
| 1620 |
PMHW_STATE_HEAP_MEMORY_BLOCK pBlock; |
| 1621 |
|
| 1622 |
// Clear the result |
| 1623 |
pBlockArray[0] = nullptr; |
| 1624 |
|
| 1625 |
if (iCount <= 0 || iCount > MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY) |
| 1626 |
{ |
| 1627 |
return nullptr; |
| 1628 |
} |
| 1629 |
|
| 1630 |
// Get total allocation size |
| 1631 |
for (int i = 0; i < iCount; i++) |
| 1632 |
{ |
| 1633 |
dwTotalSize += pdwSizes[i]; |
| 1634 |
} |
| 1635 |
|
| 1636 |
Mhw_BlockManager_ReverseMergeSort_With_Index(pdwSizes, iCount, SortedIndex); |
| 1637 |
|
| 1638 |
if (bHeapAffinity) |
| 1639 |
{ |
| 1640 |
if (pHeapAffinity) |
| 1641 |
{ |
| 1642 |
// Set heap affinity - all blocks will be allocated from this one particular heap |
| 1643 |
pStateHeap = pHeapAffinity; |
| 1644 |
} |
| 1645 |
else |
| 1646 |
{ |
| 1647 |
// Heap affinity is not set - start from the current active heap (most recently allocated heap) |
| 1648 |
pStateHeap = m_pStateHeap; |
| 1649 |
} |
| 1650 |
} |
| 1651 |
else |
| 1652 |
{ // Don't care about heap affinity, blocks can be spread across all available state heaps |
| 1653 |
// Just calculate total space available to make sure we have it |
| 1654 |
uint32_t dwTotalHeapSpace = 0; |
| 1655 |
for (pStateHeap = m_pStateHeap; pStateHeap != nullptr; pStateHeap = pStateHeap->pNext) |
| 1656 |
{ |
| 1657 |
// Calculate total free space in all available heaps |
| 1658 |
// NOTE: Heap being deleted as free space set to 0 |
| 1659 |
// and all Free blocks are now Deleted (in deleted queue) |
| 1660 |
dwTotalHeapSpace += pStateHeap->dwFree; |
| 1661 |
} |
| 1662 |
|
| 1663 |
// Not enough space |
| 1664 |
if (dwTotalHeapSpace < dwTotalSize) |
| 1665 |
{ |
| 1666 |
return pBlockArray[0]; |
| 1667 |
} |
| 1668 |
} |
| 1669 |
|
| 1670 |
// Loop to try loading all kernels into the same heap |
| 1671 |
do { |
| 1672 |
if ( (!bHeapAffinity) || // no affinity set -> blocks can be spread across multiple heaps |
| 1673 |
pStateHeap->dwFree >= dwTotalSize) // this heap has enough free space |
| 1674 |
{ |
| 1675 |
int32_t i, j; |
| 1676 |
|
| 1677 |
// Allocate array according to size |
| 1678 |
for (i = 0, pBlock = nullptr; i < iCount; i++) |
| 1679 |
{ // NOTE - don't care about scratch space, already checked |
| 1680 |
pBlock = pBlockArray[SortedIndex[i]] = AllocateBlock(pdwSizes[SortedIndex[i]], dwAlignment, pStateHeap); |
| 1681 |
if (!pBlock) |
| 1682 |
{ |
| 1683 |
break; |
| 1684 |
} |
| 1685 |
} |
| 1686 |
|
| 1687 |
// Allocations all sucessfull, quit search |
| 1688 |
if (pBlock) |
| 1689 |
{ |
| 1690 |
break; |
| 1691 |
} |
| 1692 |
|
| 1693 |
// One of the block allocations failed - free all blocks already allocated (try another heap) |
| 1694 |
for (j = 0; j < i; j++) |
| 1695 |
{ |
| 1696 |
FreeBlock(pBlockArray[SortedIndex[j]]); |
| 1697 |
} |
| 1698 |
|
| 1699 |
// Clear the output |
| 1700 |
pBlockArray[0] = nullptr; |
| 1701 |
} |
| 1702 |
|
| 1703 |
// Try another heap if bHeapAffinity is set |
| 1704 |
if (bHeapAffinity) |
| 1705 |
{ |
| 1706 |
pStateHeap = (pHeapAffinity) ? nullptr : pStateHeap->pNext; |
| 1707 |
} |
| 1708 |
} while (pStateHeap); |
| 1709 |
|
| 1710 |
// Allocation successful, reorder blocks according to original request |
| 1711 |
if (pBlockArray[0]) |
| 1712 |
{ |
| 1713 |
for (int32_t i = 0; i < iCount; i++) |
| 1714 |
{ |
| 1715 |
pBlock = DetachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlockArray[i]); |
| 1716 |
AttachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlock, MHW_BLOCK_POSITION_TAIL); |
| 1717 |
} |
| 1718 |
} |
| 1719 |
|
| 1720 |
return pBlockArray[0]; |
| 1721 |
} |