| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* aset.c |
| 4 |
* Allocation set definitions. |
| 5 |
* |
| 6 |
* AllocSet is our standard implementation of the abstract MemoryContext |
| 7 |
* type. |
| 8 |
* |
| 9 |
* |
| 10 |
* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group |
| 11 |
* Portions Copyright (c) 1994, Regents of the University of California |
| 12 |
* |
| 13 |
* IDENTIFICATION |
| 14 |
* src/backend/utils/mmgr/aset.c |
| 15 |
* |
| 16 |
* NOTE: |
| 17 |
* This is a new (Feb. 05, 1999) implementation of the allocation set |
| 18 |
* routines. AllocSet...() does not use OrderedSet...() any more. |
| 19 |
* Instead it manages allocations in a block pool by itself, combining |
| 20 |
* many small allocations in a few bigger blocks. AllocSetFree() normally |
| 21 |
* doesn't free() memory really. It just add's the free'd area to some |
| 22 |
* list for later reuse by AllocSetAlloc(). All memory blocks are free()'d |
| 23 |
* at once on AllocSetReset(), which happens when the memory context gets |
| 24 |
* destroyed. |
| 25 |
* Jan Wieck |
| 26 |
* |
| 27 |
* Performance improvement from Tom Lane, 8/99: for extremely large request |
| 28 |
* sizes, we do want to be able to give the memory back to free() as soon |
| 29 |
* as it is pfree()'d. Otherwise we risk tying up a lot of memory in |
| 30 |
* freelist entries that might never be usable. This is specially needed |
| 31 |
* when the caller is repeatedly repalloc()'ing a block bigger and bigger; |
| 32 |
* the previous instances of the block were guaranteed to be wasted until |
| 33 |
* AllocSetReset() under the old way. |
| 34 |
* |
| 35 |
* Further improvement 12/00: as the code stood, request sizes in the |
| 36 |
* midrange between "small" and "large" were handled very inefficiently, |
| 37 |
* because any sufficiently large free chunk would be used to satisfy a |
| 38 |
* request, even if it was much larger than necessary. This led to more |
| 39 |
* and more wasted space in allocated chunks over time. To fix, get rid |
| 40 |
* of the midrange behavior: we now handle only "small" power-of-2-size |
| 41 |
* chunks as chunks. Anything "large" is passed off to malloc(). Change |
| 42 |
* the number of freelists to change the small/large boundary. |
| 43 |
* |
| 44 |
*------------------------------------------------------------------------- |
| 45 |
*/ |
| 46 |
|
| 47 |
#include "postgres.h" |
| 48 |
|
| 49 |
#include "port/pg_bitutils.h" |
| 50 |
#include "utils/memdebug.h" |
| 51 |
#include "utils/memutils.h" |
| 52 |
#include "utils/memutils_internal.h" |
| 53 |
#include "utils/memutils_memorychunk.h" |
| 54 |
|
| 55 |
/*-------------------- |
| 56 |
* Chunk freelist k holds chunks of size 1 << (k + ALLOC_MINBITS), |
| 57 |
* for k = 0 .. ALLOCSET_NUM_FREELISTS-1. |
| 58 |
* |
| 59 |
* Note that all chunks in the freelists have power-of-2 sizes. This |
| 60 |
* improves recyclability: we may waste some space, but the wasted space |
| 61 |
* should stay pretty constant as requests are made and released. |
| 62 |
* |
| 63 |
* A request too large for the last freelist is handled by allocating a |
| 64 |
* dedicated block from malloc(). The block still has a block header and |
| 65 |
* chunk header, but when the chunk is freed we'll return the whole block |
| 66 |
* to malloc(), not put it on our freelists. |
| 67 |
* |
| 68 |
* CAUTION: ALLOC_MINBITS must be large enough so that |
| 69 |
* 1<<ALLOC_MINBITS is at least MAXALIGN, |
| 70 |
* or we may fail to align the smallest chunks adequately. |
| 71 |
* 8-byte alignment is enough on all currently known machines. This 8-byte |
| 72 |
* minimum also allows us to store a pointer to the next freelist item within |
| 73 |
* the chunk of memory itself. |
| 74 |
* |
| 75 |
* With the current parameters, request sizes up to 8K are treated as chunks, |
| 76 |
* larger requests go into dedicated blocks. Change ALLOCSET_NUM_FREELISTS |
| 77 |
* to adjust the boundary point; and adjust ALLOCSET_SEPARATE_THRESHOLD in |
| 78 |
* memutils.h to agree. (Note: in contexts with small maxBlockSize, we may |
| 79 |
* set the allocChunkLimit to less than 8K, so as to avoid space wastage.) |
| 80 |
*-------------------- |
| 81 |
*/ |
| 82 |
|
| 83 |
#define ALLOC_MINBITS 3 /* smallest chunk size is 8 bytes */ |
| 84 |
#define ALLOCSET_NUM_FREELISTS 11 |
| 85 |
#define ALLOC_CHUNK_LIMIT (1 << (ALLOCSET_NUM_FREELISTS-1+ALLOC_MINBITS)) |
| 86 |
/* Size of largest chunk that we use a fixed size for */ |
| 87 |
#define ALLOC_CHUNK_FRACTION 4 |
| 88 |
/* We allow chunks to be at most 1/4 of maxBlockSize (less overhead) */ |
| 89 |
|
| 90 |
/*-------------------- |
| 91 |
* The first block allocated for an allocset has size initBlockSize. |
| 92 |
* Each time we have to allocate another block, we double the block size |
| 93 |
* (if possible, and without exceeding maxBlockSize), so as to reduce |
| 94 |
* the bookkeeping load on malloc(). |
| 95 |
* |
| 96 |
* Blocks allocated to hold oversize chunks do not follow this rule, however; |
| 97 |
* they are just however big they need to be to hold that single chunk. |
| 98 |
* |
| 99 |
* Also, if a minContextSize is specified, the first block has that size, |
| 100 |
* and then initBlockSize is used for the next one. |
| 101 |
*-------------------- |
| 102 |
*/ |
| 103 |
|
| 104 |
#define ALLOC_BLOCKHDRSZ MAXALIGN(sizeof(AllocBlockData)) |
| 105 |
#define ALLOC_CHUNKHDRSZ sizeof(MemoryChunk) |
| 106 |
|
| 107 |
typedef struct AllocBlockData *AllocBlock; /* forward reference */ |
| 108 |
|
| 109 |
/* |
| 110 |
* AllocPointer |
| 111 |
* Aligned pointer which may be a member of an allocation set. |
| 112 |
*/ |
| 113 |
typedef void *AllocPointer; |
| 114 |
|
| 115 |
/* |
| 116 |
* AllocFreeListLink |
| 117 |
* When pfreeing memory, if we maintain a freelist for the given chunk's |
| 118 |
* size then we use a AllocFreeListLink to point to the current item in |
| 119 |
* the AllocSetContext's freelist and then set the given freelist element |
| 120 |
* to point to the chunk being freed. |
| 121 |
*/ |
| 122 |
typedef struct AllocFreeListLink |
| 123 |
{ |
| 124 |
MemoryChunk *next; |
| 125 |
} AllocFreeListLink; |
| 126 |
|
| 127 |
/* |
| 128 |
* Obtain a AllocFreeListLink for the given chunk. Allocation sizes are |
| 129 |
* always at least sizeof(AllocFreeListLink), so we reuse the pointer's memory |
| 130 |
* itself to store the freelist link. |
| 131 |
*/ |
| 132 |
#define GetFreeListLink(chkptr) \ |
| 133 |
(AllocFreeListLink *) ((char *) (chkptr) + ALLOC_CHUNKHDRSZ) |
| 134 |
|
| 135 |
/* Validate a freelist index retrieved from a chunk header */ |
| 136 |
#define FreeListIdxIsValid(fidx) \ |
| 137 |
((fidx) >= 0 && (fidx) < ALLOCSET_NUM_FREELISTS) |
| 138 |
|
| 139 |
/* Determine the size of the chunk based on the freelist index */ |
| 140 |
#define GetChunkSizeFromFreeListIdx(fidx) \ |
| 141 |
((((Size) 1) << ALLOC_MINBITS) << (fidx)) |
| 142 |
|
| 143 |
/* |
| 144 |
* AllocSetContext is our standard implementation of MemoryContext. |
| 145 |
* |
| 146 |
* Note: header.isReset means there is nothing for AllocSetReset to do. |
| 147 |
* This is different from the aset being physically empty (empty blocks list) |
| 148 |
* because we will still have a keeper block. It's also different from the set |
| 149 |
* being logically empty, because we don't attempt to detect pfree'ing the |
| 150 |
* last active chunk. |
| 151 |
*/ |
| 152 |
typedef struct AllocSetContext |
| 153 |
{ |
| 154 |
MemoryContextData header; /* Standard memory-context fields */ |
| 155 |
/* Info about storage allocated in this context: */ |
| 156 |
AllocBlock blocks; /* head of list of blocks in this set */ |
| 157 |
MemoryChunk *freelist[ALLOCSET_NUM_FREELISTS]; /* free chunk lists */ |
| 158 |
/* Allocation parameters for this context: */ |
| 159 |
uint32 initBlockSize; /* initial block size */ |
| 160 |
uint32 maxBlockSize; /* maximum block size */ |
| 161 |
uint32 nextBlockSize; /* next block size to allocate */ |
| 162 |
uint32 allocChunkLimit; /* effective chunk size limit */ |
| 163 |
/* freelist this context could be put in, or -1 if not a candidate: */ |
| 164 |
int freeListIndex; /* index in context_freelists[], or -1 */ |
| 165 |
} AllocSetContext; |
| 166 |
|
| 167 |
typedef AllocSetContext *AllocSet; |
| 168 |
|
| 169 |
/* |
| 170 |
* AllocBlock |
| 171 |
* An AllocBlock is the unit of memory that is obtained by aset.c |
| 172 |
* from malloc(). It contains one or more MemoryChunks, which are |
| 173 |
* the units requested by palloc() and freed by pfree(). MemoryChunks |
| 174 |
* cannot be returned to malloc() individually, instead they are put |
| 175 |
* on freelists by pfree() and re-used by the next palloc() that has |
| 176 |
* a matching request size. |
| 177 |
* |
| 178 |
* AllocBlockData is the header data for a block --- the usable space |
| 179 |
* within the block begins at the next alignment boundary. |
| 180 |
*/ |
| 181 |
typedef struct AllocBlockData |
| 182 |
{ |
| 183 |
AllocSet aset; /* aset that owns this block */ |
| 184 |
AllocBlock prev; /* prev block in aset's blocks list, if any */ |
| 185 |
AllocBlock next; /* next block in aset's blocks list, if any */ |
| 186 |
char *freeptr; /* start of free space in this block */ |
| 187 |
char *endptr; /* end of space in this block */ |
| 188 |
} AllocBlockData; |
| 189 |
|
| 190 |
/* |
| 191 |
* AllocPointerIsValid |
| 192 |
* True iff pointer is valid allocation pointer. |
| 193 |
*/ |
| 194 |
#define AllocPointerIsValid(pointer) PointerIsValid(pointer) |
| 195 |
|
| 196 |
/* |
| 197 |
* AllocSetIsValid |
| 198 |
* True iff set is valid allocation set. |
| 199 |
*/ |
| 200 |
#define AllocSetIsValid(set) \ |
| 201 |
(PointerIsValid(set) && IsA(set, AllocSetContext)) |
| 202 |
|
| 203 |
/* |
| 204 |
* AllocBlockIsValid |
| 205 |
* True iff block is valid block of allocation set. |
| 206 |
*/ |
| 207 |
#define AllocBlockIsValid(block) \ |
| 208 |
(PointerIsValid(block) && AllocSetIsValid((block)->aset)) |
| 209 |
|
| 210 |
/* |
| 211 |
* We always store external chunks on a dedicated block. This makes fetching |
| 212 |
* the block from an external chunk easy since it's always the first and only |
| 213 |
* chunk on the block. |
| 214 |
*/ |
| 215 |
#define ExternalChunkGetBlock(chunk) \ |
| 216 |
(AllocBlock) ((char *) chunk - ALLOC_BLOCKHDRSZ) |
| 217 |
|
| 218 |
/* |
| 219 |
* Rather than repeatedly creating and deleting memory contexts, we keep some |
| 220 |
* freed contexts in freelists so that we can hand them out again with little |
| 221 |
* work. Before putting a context in a freelist, we reset it so that it has |
| 222 |
* only its initial malloc chunk and no others. To be a candidate for a |
| 223 |
* freelist, a context must have the same minContextSize/initBlockSize as |
| 224 |
* other contexts in the list; but its maxBlockSize is irrelevant since that |
| 225 |
* doesn't affect the size of the initial chunk. |
| 226 |
* |
| 227 |
* We currently provide one freelist for ALLOCSET_DEFAULT_SIZES contexts |
| 228 |
* and one for ALLOCSET_SMALL_SIZES contexts; the latter works for |
| 229 |
* ALLOCSET_START_SMALL_SIZES too, since only the maxBlockSize differs. |
| 230 |
* |
| 231 |
* Ordinarily, we re-use freelist contexts in last-in-first-out order, in |
| 232 |
* hopes of improving locality of reference. But if there get to be too |
| 233 |
* many contexts in the list, we'd prefer to drop the most-recently-created |
| 234 |
* contexts in hopes of keeping the process memory map compact. |
| 235 |
* We approximate that by simply deleting all existing entries when the list |
| 236 |
* overflows, on the assumption that queries that allocate a lot of contexts |
| 237 |
* will probably free them in more or less reverse order of allocation. |
| 238 |
* |
| 239 |
* Contexts in a freelist are chained via their nextchild pointers. |
| 240 |
*/ |
| 241 |
#define MAX_FREE_CONTEXTS 100 /* arbitrary limit on freelist length */ |
| 242 |
|
| 243 |
/* Obtain the keeper block for an allocation set */ |
| 244 |
#define KeeperBlock(set) \ |
| 245 |
((AllocBlock) (((char *) set) + MAXALIGN(sizeof(AllocSetContext)))) |
| 246 |
|
| 247 |
/* Check if the block is the keeper block of the given allocation set */ |
| 248 |
#define IsKeeperBlock(set, block) ((block) == (KeeperBlock(set))) |
| 249 |
|
| 250 |
typedef struct AllocSetFreeList |
| 251 |
{ |
| 252 |
int num_free; /* current list length */ |
| 253 |
AllocSetContext *first_free; /* list header */ |
| 254 |
} AllocSetFreeList; |
| 255 |
|
| 256 |
/* context_freelists[0] is for default params, [1] for small params */ |
| 257 |
static AllocSetFreeList context_freelists[2] = |
| 258 |
{ |
| 259 |
{ |
| 260 |
0, NULL |
| 261 |
}, |
| 262 |
{ |
| 263 |
0, NULL |
| 264 |
} |
| 265 |
}; |
| 266 |
|
| 267 |
|
| 268 |
/* ---------- |
| 269 |
* AllocSetFreeIndex - |
| 270 |
* |
| 271 |
* Depending on the size of an allocation compute which freechunk |
| 272 |
* list of the alloc set it belongs to. Caller must have verified |
| 273 |
* that size <= ALLOC_CHUNK_LIMIT. |
| 274 |
* ---------- |
| 275 |
*/ |
| 276 |
static inline int |
| 277 |
AllocSetFreeIndex(Size size) |
| 278 |
{ |
| 279 |
int idx; |
| 280 |
|
| 281 |
if (size > (1 << ALLOC_MINBITS)) |
| 282 |
{ |
| 283 |
/*---------- |
| 284 |
* At this point we must compute ceil(log2(size >> ALLOC_MINBITS)). |
| 285 |
* This is the same as |
| 286 |
* pg_leftmost_one_pos32((size - 1) >> ALLOC_MINBITS) + 1 |
| 287 |
* or equivalently |
| 288 |
* pg_leftmost_one_pos32(size - 1) - ALLOC_MINBITS + 1 |
| 289 |
* |
| 290 |
* However, for platforms without intrinsic support, we duplicate the |
| 291 |
* logic here, allowing an additional optimization. It's reasonable |
| 292 |
* to assume that ALLOC_CHUNK_LIMIT fits in 16 bits, so we can unroll |
| 293 |
* the byte-at-a-time loop in pg_leftmost_one_pos32 and just handle |
| 294 |
* the last two bytes. |
| 295 |
* |
| 296 |
* Yes, this function is enough of a hot-spot to make it worth this |
| 297 |
* much trouble. |
| 298 |
*---------- |
| 299 |
*/ |
| 300 |
#ifdef HAVE_BITSCAN_REVERSE |
| 301 |
idx = pg_leftmost_one_pos32((uint32) size - 1) - ALLOC_MINBITS + 1; |
| 302 |
#else |
| 303 |
uint32 t, |
| 304 |
tsize; |
| 305 |
|
| 306 |
/* Statically assert that we only have a 16-bit input value. */ |
| 307 |
StaticAssertDecl(ALLOC_CHUNK_LIMIT < (1 << 16), |
| 308 |
"ALLOC_CHUNK_LIMIT must be less than 64kB"); |
| 309 |
|
| 310 |
tsize = size - 1; |
| 311 |
t = tsize >> 8; |
| 312 |
idx = t ? pg_leftmost_one_pos[t] + 8 : pg_leftmost_one_pos[tsize]; |
| 313 |
idx -= ALLOC_MINBITS - 1; |
| 314 |
#endif |
| 315 |
|
| 316 |
Assert(idx < ALLOCSET_NUM_FREELISTS); |
| 317 |
} |
| 318 |
else |
| 319 |
idx = 0; |
| 320 |
|
| 321 |
return idx; |
| 322 |
} |
| 323 |
|
| 324 |
|
| 325 |
/* |
| 326 |
* Public routines |
| 327 |
*/ |
| 328 |
|
| 329 |
|
| 330 |
/* |
| 331 |
* AllocSetContextCreateInternal |
| 332 |
* Create a new AllocSet context. |
| 333 |
* |
| 334 |
* parent: parent context, or NULL if top-level context |
| 335 |
* name: name of context (must be statically allocated) |
| 336 |
* minContextSize: minimum context size |
| 337 |
* initBlockSize: initial allocation block size |
| 338 |
* maxBlockSize: maximum allocation block size |
| 339 |
* |
| 340 |
* Most callers should abstract the context size parameters using a macro |
| 341 |
* such as ALLOCSET_DEFAULT_SIZES. |
| 342 |
* |
| 343 |
* Note: don't call this directly; go through the wrapper macro |
| 344 |
* AllocSetContextCreate. |
| 345 |
*/ |
| 346 |
MemoryContext |
| 347 |
AllocSetContextCreateInternal(MemoryContext parent, |
| 348 |
const char *name, |
| 349 |
Size minContextSize, |
| 350 |
Size initBlockSize, |
| 351 |
Size maxBlockSize) |
| 352 |
{ |
| 353 |
int freeListIndex; |
| 354 |
Size firstBlockSize; |
| 355 |
AllocSet set; |
| 356 |
AllocBlock block; |
| 357 |
|
| 358 |
/* ensure MemoryChunk's size is properly maxaligned */ |
| 359 |
StaticAssertDecl(ALLOC_CHUNKHDRSZ == MAXALIGN(ALLOC_CHUNKHDRSZ), |
| 360 |
"sizeof(MemoryChunk) is not maxaligned"); |
| 361 |
/* check we have enough space to store the freelist link */ |
| 362 |
StaticAssertDecl(sizeof(AllocFreeListLink) <= (1 << ALLOC_MINBITS), |
| 363 |
"sizeof(AllocFreeListLink) larger than minimum allocation size"); |
| 364 |
|
| 365 |
/* |
| 366 |
* First, validate allocation parameters. Once these were regular runtime |
| 367 |
* tests and elog's, but in practice Asserts seem sufficient because |
| 368 |
* nobody varies their parameters at runtime. We somewhat arbitrarily |
| 369 |
* enforce a minimum 1K block size. We restrict the maximum block size to |
| 370 |
* MEMORYCHUNK_MAX_BLOCKOFFSET as MemoryChunks are limited to this in |
| 371 |
* regards to addressing the offset between the chunk and the block that |
| 372 |
* the chunk is stored on. We would be unable to store the offset between |
| 373 |
* the chunk and block for any chunks that were beyond |
| 374 |
* MEMORYCHUNK_MAX_BLOCKOFFSET bytes into the block if the block was to be |
| 375 |
* larger than this. |
| 376 |
*/ |
| 377 |
Assert(initBlockSize == MAXALIGN(initBlockSize) && |
| 378 |
initBlockSize >= 1024); |
| 379 |
Assert(maxBlockSize == MAXALIGN(maxBlockSize) && |
| 380 |
maxBlockSize >= initBlockSize && |
| 381 |
AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */ |
| 382 |
Assert(minContextSize == 0 || |
| 383 |
(minContextSize == MAXALIGN(minContextSize) && |
| 384 |
minContextSize >= 1024 && |
| 385 |
minContextSize <= maxBlockSize)); |
| 386 |
Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET); |
| 387 |
|
| 388 |
/* |
| 389 |
* Check whether the parameters match either available freelist. We do |
| 390 |
* not need to demand a match of maxBlockSize. |
| 391 |
*/ |
| 392 |
if (minContextSize == ALLOCSET_DEFAULT_MINSIZE && |
| 393 |
initBlockSize == ALLOCSET_DEFAULT_INITSIZE) |
| 394 |
freeListIndex = 0; |
| 395 |
else if (minContextSize == ALLOCSET_SMALL_MINSIZE && |
| 396 |
initBlockSize == ALLOCSET_SMALL_INITSIZE) |
| 397 |
freeListIndex = 1; |
| 398 |
else |
| 399 |
freeListIndex = -1; |
| 400 |
|
| 401 |
/* |
| 402 |
* If a suitable freelist entry exists, just recycle that context. |
| 403 |
*/ |
| 404 |
if (freeListIndex >= 0) |
| 405 |
{ |
| 406 |
AllocSetFreeList *freelist = &context_freelists[freeListIndex]; |
| 407 |
|
| 408 |
if (freelist->first_free != NULL) |
| 409 |
{ |
| 410 |
/* Remove entry from freelist */ |
| 411 |
set = freelist->first_free; |
| 412 |
freelist->first_free = (AllocSet) set->header.nextchild; |
| 413 |
freelist->num_free--; |
| 414 |
|
| 415 |
/* Update its maxBlockSize; everything else should be OK */ |
| 416 |
set->maxBlockSize = maxBlockSize; |
| 417 |
|
| 418 |
/* Reinitialize its header, installing correct name and parent */ |
| 419 |
MemoryContextCreate((MemoryContext) set, |
| 420 |
T_AllocSetContext, |
| 421 |
MCTX_ASET_ID, |
| 422 |
parent, |
| 423 |
name); |
| 424 |
|
| 425 |
((MemoryContext) set)->mem_allocated = |
| 426 |
KeeperBlock(set)->endptr - ((char *) set); |
| 427 |
|
| 428 |
return (MemoryContext) set; |
| 429 |
} |
| 430 |
} |
| 431 |
|
| 432 |
/* Determine size of initial block */ |
| 433 |
firstBlockSize = MAXALIGN(sizeof(AllocSetContext)) + |
| 434 |
ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; |
| 435 |
if (minContextSize != 0) |
| 436 |
firstBlockSize = Max(firstBlockSize, minContextSize); |
| 437 |
else |
| 438 |
firstBlockSize = Max(firstBlockSize, initBlockSize); |
| 439 |
|
| 440 |
/* |
| 441 |
* Allocate the initial block. Unlike other aset.c blocks, it starts with |
| 442 |
* the context header and its block header follows that. |
| 443 |
*/ |
| 444 |
set = (AllocSet) malloc(firstBlockSize); |
| 445 |
if (set == NULL) |
| 446 |
{ |
| 447 |
if (TopMemoryContext) |
| 448 |
MemoryContextStats(TopMemoryContext); |
| 449 |
ereport(ERROR, |
| 450 |
(errcode(ERRCODE_OUT_OF_MEMORY), |
| 451 |
errmsg("out of memory"), |
| 452 |
errdetail("Failed while creating memory context \"%s\".", |
| 453 |
name))); |
| 454 |
} |
| 455 |
|
| 456 |
/* |
| 457 |
* Avoid writing code that can fail between here and MemoryContextCreate; |
| 458 |
* we'd leak the header/initial block if we ereport in this stretch. |
| 459 |
*/ |
| 460 |
|
| 461 |
/* Fill in the initial block's block header */ |
| 462 |
block = KeeperBlock(set); |
| 463 |
block->aset = set; |
| 464 |
block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; |
| 465 |
block->endptr = ((char *) set) + firstBlockSize; |
| 466 |
block->prev = NULL; |
| 467 |
block->next = NULL; |
| 468 |
|
| 469 |
/* Mark unallocated space NOACCESS; leave the block header alone. */ |
| 470 |
VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, block->endptr - block->freeptr); |
| 471 |
|
| 472 |
/* Remember block as part of block list */ |
| 473 |
set->blocks = block; |
| 474 |
|
| 475 |
/* Finish filling in aset-specific parts of the context header */ |
| 476 |
MemSetAligned(set->freelist, 0, sizeof(set->freelist)); |
| 477 |
|
| 478 |
set->initBlockSize = (uint32) initBlockSize; |
| 479 |
set->maxBlockSize = (uint32) maxBlockSize; |
| 480 |
set->nextBlockSize = (uint32) initBlockSize; |
| 481 |
set->freeListIndex = freeListIndex; |
| 482 |
|
| 483 |
/* |
| 484 |
* Compute the allocation chunk size limit for this context. It can't be |
| 485 |
* more than ALLOC_CHUNK_LIMIT because of the fixed number of freelists. |
| 486 |
* If maxBlockSize is small then requests exceeding the maxBlockSize, or |
| 487 |
* even a significant fraction of it, should be treated as large chunks |
| 488 |
* too. For the typical case of maxBlockSize a power of 2, the chunk size |
| 489 |
* limit will be at most 1/8th maxBlockSize, so that given a stream of |
| 490 |
* requests that are all the maximum chunk size we will waste at most |
| 491 |
* 1/8th of the allocated space. |
| 492 |
* |
| 493 |
* Also, allocChunkLimit must not exceed ALLOCSET_SEPARATE_THRESHOLD. |
| 494 |
*/ |
| 495 |
StaticAssertStmt(ALLOC_CHUNK_LIMIT == ALLOCSET_SEPARATE_THRESHOLD, |
| 496 |
"ALLOC_CHUNK_LIMIT != ALLOCSET_SEPARATE_THRESHOLD"); |
| 497 |
|
| 498 |
/* |
| 499 |
* Determine the maximum size that a chunk can be before we allocate an |
| 500 |
* entire AllocBlock dedicated for that chunk. We set the absolute limit |
| 501 |
* of that size as ALLOC_CHUNK_LIMIT but we reduce it further so that we |
| 502 |
* can fit about ALLOC_CHUNK_FRACTION chunks this size on a maximally |
| 503 |
* sized block. (We opt to keep allocChunkLimit a power-of-2 value |
| 504 |
* primarily for legacy reasons rather than calculating it so that exactly |
| 505 |
* ALLOC_CHUNK_FRACTION chunks fit on a maximally sized block.) |
| 506 |
*/ |
| 507 |
set->allocChunkLimit = ALLOC_CHUNK_LIMIT; |
| 508 |
while ((Size) (set->allocChunkLimit + ALLOC_CHUNKHDRSZ) > |
| 509 |
(Size) ((maxBlockSize - ALLOC_BLOCKHDRSZ) / ALLOC_CHUNK_FRACTION)) |
| 510 |
set->allocChunkLimit >>= 1; |
| 511 |
|
| 512 |
/* Finally, do the type-independent part of context creation */ |
| 513 |
MemoryContextCreate((MemoryContext) set, |
| 514 |
T_AllocSetContext, |
| 515 |
MCTX_ASET_ID, |
| 516 |
parent, |
| 517 |
name); |
| 518 |
|
| 519 |
((MemoryContext) set)->mem_allocated = firstBlockSize; |
| 520 |
|
| 521 |
return (MemoryContext) set; |
| 522 |
} |
| 523 |
|
| 524 |
/* |
| 525 |
* AllocSetReset |
| 526 |
* Frees all memory which is allocated in the given set. |
| 527 |
* |
| 528 |
* Actually, this routine has some discretion about what to do. |
| 529 |
* It should mark all allocated chunks freed, but it need not necessarily |
| 530 |
* give back all the resources the set owns. Our actual implementation is |
| 531 |
* that we give back all but the "keeper" block (which we must keep, since |
| 532 |
* it shares a malloc chunk with the context header). In this way, we don't |
| 533 |
* thrash malloc() when a context is repeatedly reset after small allocations, |
| 534 |
* which is typical behavior for per-tuple contexts. |
| 535 |
*/ |
| 536 |
void |
| 537 |
AllocSetReset(MemoryContext context) |
| 538 |
{ |
| 539 |
AllocSet set = (AllocSet) context; |
| 540 |
AllocBlock block; |
| 541 |
Size keepersize PG_USED_FOR_ASSERTS_ONLY; |
| 542 |
|
| 543 |
Assert(AllocSetIsValid(set)); |
| 544 |
|
| 545 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 546 |
/* Check for corruption and leaks before freeing */ |
| 547 |
AllocSetCheck(context); |
| 548 |
#endif |
| 549 |
|
| 550 |
/* Remember keeper block size for Assert below */ |
| 551 |
keepersize = KeeperBlock(set)->endptr - ((char *) set); |
| 552 |
|
| 553 |
/* Clear chunk freelists */ |
| 554 |
MemSetAligned(set->freelist, 0, sizeof(set->freelist)); |
| 555 |
|
| 556 |
block = set->blocks; |
| 557 |
|
| 558 |
/* New blocks list will be just the keeper block */ |
| 559 |
set->blocks = KeeperBlock(set); |
| 560 |
|
| 561 |
while (block != NULL) |
| 562 |
{ |
| 563 |
AllocBlock next = block->next; |
| 564 |
|
| 565 |
if (IsKeeperBlock(set, block)) |
| 566 |
{ |
| 567 |
/* Reset the block, but don't return it to malloc */ |
| 568 |
char *datastart = ((char *) block) + ALLOC_BLOCKHDRSZ; |
| 569 |
|
| 570 |
#ifdef CLOBBER_FREED_MEMORY |
| 571 |
wipe_mem(datastart, block->freeptr - datastart); |
| 572 |
#else |
| 573 |
/* wipe_mem() would have done this */ |
| 574 |
VALGRIND_MAKE_MEM_NOACCESS(datastart, block->freeptr - datastart); |
| 575 |
#endif |
| 576 |
block->freeptr = datastart; |
| 577 |
block->prev = NULL; |
| 578 |
block->next = NULL; |
| 579 |
} |
| 580 |
else |
| 581 |
{ |
| 582 |
/* Normal case, release the block */ |
| 583 |
context->mem_allocated -= block->endptr - ((char *) block); |
| 584 |
|
| 585 |
#ifdef CLOBBER_FREED_MEMORY |
| 586 |
wipe_mem(block, block->freeptr - ((char *) block)); |
| 587 |
#endif |
| 588 |
free(block); |
| 589 |
} |
| 590 |
block = next; |
| 591 |
} |
| 592 |
|
| 593 |
Assert(context->mem_allocated == keepersize); |
| 594 |
|
| 595 |
/* Reset block size allocation sequence, too */ |
| 596 |
set->nextBlockSize = set->initBlockSize; |
| 597 |
} |
| 598 |
|
| 599 |
/* |
| 600 |
* AllocSetDelete |
| 601 |
* Frees all memory which is allocated in the given set, |
| 602 |
* in preparation for deletion of the set. |
| 603 |
* |
| 604 |
* Unlike AllocSetReset, this *must* free all resources of the set. |
| 605 |
*/ |
| 606 |
void |
| 607 |
AllocSetDelete(MemoryContext context) |
| 608 |
{ |
| 609 |
AllocSet set = (AllocSet) context; |
| 610 |
AllocBlock block = set->blocks; |
| 611 |
Size keepersize PG_USED_FOR_ASSERTS_ONLY; |
| 612 |
|
| 613 |
Assert(AllocSetIsValid(set)); |
| 614 |
|
| 615 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 616 |
/* Check for corruption and leaks before freeing */ |
| 617 |
AllocSetCheck(context); |
| 618 |
#endif |
| 619 |
|
| 620 |
/* Remember keeper block size for Assert below */ |
| 621 |
keepersize = KeeperBlock(set)->endptr - ((char *) set); |
| 622 |
|
| 623 |
/* |
| 624 |
* If the context is a candidate for a freelist, put it into that freelist |
| 625 |
* instead of destroying it. |
| 626 |
*/ |
| 627 |
if (set->freeListIndex >= 0) |
| 628 |
{ |
| 629 |
AllocSetFreeList *freelist = &context_freelists[set->freeListIndex]; |
| 630 |
|
| 631 |
/* |
| 632 |
* Reset the context, if it needs it, so that we aren't hanging on to |
| 633 |
* more than the initial malloc chunk. |
| 634 |
*/ |
| 635 |
if (!context->isReset) |
| 636 |
MemoryContextResetOnly(context); |
| 637 |
|
| 638 |
/* |
| 639 |
* If the freelist is full, just discard what's already in it. See |
| 640 |
* comments with context_freelists[]. |
| 641 |
*/ |
| 642 |
if (freelist->num_free >= MAX_FREE_CONTEXTS) |
| 643 |
{ |
| 644 |
while (freelist->first_free != NULL) |
| 645 |
{ |
| 646 |
AllocSetContext *oldset = freelist->first_free; |
| 647 |
|
| 648 |
freelist->first_free = (AllocSetContext *) oldset->header.nextchild; |
| 649 |
freelist->num_free--; |
| 650 |
|
| 651 |
/* All that remains is to free the header/initial block */ |
| 652 |
free(oldset); |
| 653 |
} |
| 654 |
Assert(freelist->num_free == 0); |
| 655 |
} |
| 656 |
|
| 657 |
/* Now add the just-deleted context to the freelist. */ |
| 658 |
set->header.nextchild = (MemoryContext) freelist->first_free; |
| 659 |
freelist->first_free = set; |
| 660 |
freelist->num_free++; |
| 661 |
|
| 662 |
return; |
| 663 |
} |
| 664 |
|
| 665 |
/* Free all blocks, except the keeper which is part of context header */ |
| 666 |
while (block != NULL) |
| 667 |
{ |
| 668 |
AllocBlock next = block->next; |
| 669 |
|
| 670 |
if (!IsKeeperBlock(set, block)) |
| 671 |
context->mem_allocated -= block->endptr - ((char *) block); |
| 672 |
|
| 673 |
#ifdef CLOBBER_FREED_MEMORY |
| 674 |
wipe_mem(block, block->freeptr - ((char *) block)); |
| 675 |
#endif |
| 676 |
|
| 677 |
if (!IsKeeperBlock(set, block)) |
| 678 |
free(block); |
| 679 |
|
| 680 |
block = next; |
| 681 |
} |
| 682 |
|
| 683 |
Assert(context->mem_allocated == keepersize); |
| 684 |
|
| 685 |
/* Finally, free the context header, including the keeper block */ |
| 686 |
free(set); |
| 687 |
} |
| 688 |
|
| 689 |
/* |
| 690 |
* Helper for AllocSetAlloc() that allocates an entire block for the chunk. |
| 691 |
* |
| 692 |
* AllocSetAlloc()'s comment explains why this is separate. |
| 693 |
*/ |
| 694 |
pg_noinline |
| 695 |
static void * |
| 696 |
AllocSetAllocLarge(MemoryContext context, Size size, int flags) |
| 697 |
{ |
| 698 |
AllocSet set = (AllocSet) context; |
| 699 |
AllocBlock block; |
| 700 |
MemoryChunk *chunk; |
| 701 |
Size chunk_size; |
| 702 |
Size blksize; |
| 703 |
|
| 704 |
/* validate 'size' is within the limits for the given 'flags' */ |
| 705 |
MemoryContextCheckSize(context, size, flags); |
| 706 |
|
| 707 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 708 |
/* ensure there's always space for the sentinel byte */ |
| 709 |
chunk_size = MAXALIGN(size + 1); |
| 710 |
#else |
| 711 |
chunk_size = MAXALIGN(size); |
| 712 |
#endif |
| 713 |
|
| 714 |
blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; |
| 715 |
block = (AllocBlock) malloc(blksize); |
| 716 |
if (block == NULL) |
| 717 |
return MemoryContextAllocationFailure(context, size, flags); |
| 718 |
|
| 719 |
context->mem_allocated += blksize; |
| 720 |
|
| 721 |
block->aset = set; |
| 722 |
block->freeptr = block->endptr = ((char *) block) + blksize; |
| 723 |
|
| 724 |
chunk = (MemoryChunk *) (((char *) block) + ALLOC_BLOCKHDRSZ); |
| 725 |
|
| 726 |
/* mark the MemoryChunk as externally managed */ |
| 727 |
MemoryChunkSetHdrMaskExternal(chunk, MCTX_ASET_ID); |
| 728 |
|
| 729 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 730 |
chunk->requested_size = size; |
| 731 |
/* set mark to catch clobber of "unused" space */ |
| 732 |
Assert(size < chunk_size); |
| 733 |
set_sentinel(MemoryChunkGetPointer(chunk), size); |
| 734 |
#endif |
| 735 |
#ifdef RANDOMIZE_ALLOCATED_MEMORY |
| 736 |
/* fill the allocated space with junk */ |
| 737 |
randomize_mem((char *) MemoryChunkGetPointer(chunk), size); |
| 738 |
#endif |
| 739 |
|
| 740 |
/* |
| 741 |
* Stick the new block underneath the active allocation block, if any, so |
| 742 |
* that we don't lose the use of the space remaining therein. |
| 743 |
*/ |
| 744 |
if (set->blocks != NULL) |
| 745 |
{ |
| 746 |
block->prev = set->blocks; |
| 747 |
block->next = set->blocks->next; |
| 748 |
if (block->next) |
| 749 |
block->next->prev = block; |
| 750 |
set->blocks->next = block; |
| 751 |
} |
| 752 |
else |
| 753 |
{ |
| 754 |
block->prev = NULL; |
| 755 |
block->next = NULL; |
| 756 |
set->blocks = block; |
| 757 |
} |
| 758 |
|
| 759 |
/* Ensure any padding bytes are marked NOACCESS. */ |
| 760 |
VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, |
| 761 |
chunk_size - size); |
| 762 |
|
| 763 |
/* Disallow access to the chunk header. */ |
| 764 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 765 |
|
| 766 |
return MemoryChunkGetPointer(chunk); |
| 767 |
} |
| 768 |
|
| 769 |
/* |
| 770 |
* Small helper for allocating a new chunk from a chunk, to avoid duplicating |
| 771 |
* the code between AllocSetAlloc() and AllocSetAllocFromNewBlock(). |
| 772 |
*/ |
| 773 |
static inline void * |
| 774 |
AllocSetAllocChunkFromBlock(MemoryContext context, AllocBlock block, |
| 775 |
Size size, Size chunk_size, int fidx) |
| 776 |
{ |
| 777 |
MemoryChunk *chunk; |
| 778 |
|
| 779 |
chunk = (MemoryChunk *) (block->freeptr); |
| 780 |
|
| 781 |
/* Prepare to initialize the chunk header. */ |
| 782 |
VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 783 |
|
| 784 |
block->freeptr += (chunk_size + ALLOC_CHUNKHDRSZ); |
| 785 |
Assert(block->freeptr <= block->endptr); |
| 786 |
|
| 787 |
/* store the free list index in the value field */ |
| 788 |
MemoryChunkSetHdrMask(chunk, block, fidx, MCTX_ASET_ID); |
| 789 |
|
| 790 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 791 |
chunk->requested_size = size; |
| 792 |
/* set mark to catch clobber of "unused" space */ |
| 793 |
if (size < chunk_size) |
| 794 |
set_sentinel(MemoryChunkGetPointer(chunk), size); |
| 795 |
#endif |
| 796 |
#ifdef RANDOMIZE_ALLOCATED_MEMORY |
| 797 |
/* fill the allocated space with junk */ |
| 798 |
randomize_mem((char *) MemoryChunkGetPointer(chunk), size); |
| 799 |
#endif |
| 800 |
|
| 801 |
/* Ensure any padding bytes are marked NOACCESS. */ |
| 802 |
VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, |
| 803 |
chunk_size - size); |
| 804 |
|
| 805 |
/* Disallow access to the chunk header. */ |
| 806 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 807 |
|
| 808 |
return MemoryChunkGetPointer(chunk); |
| 809 |
} |
| 810 |
|
| 811 |
/* |
| 812 |
* Helper for AllocSetAlloc() that allocates a new block and returns a chunk |
| 813 |
* allocated from it. |
| 814 |
* |
| 815 |
* AllocSetAlloc()'s comment explains why this is separate. |
| 816 |
*/ |
| 817 |
pg_noinline |
| 818 |
static void * |
| 819 |
AllocSetAllocFromNewBlock(MemoryContext context, Size size, int flags, |
| 820 |
int fidx) |
| 821 |
{ |
| 822 |
AllocSet set = (AllocSet) context; |
| 823 |
AllocBlock block; |
| 824 |
Size availspace; |
| 825 |
Size blksize; |
| 826 |
Size required_size; |
| 827 |
Size chunk_size; |
| 828 |
|
| 829 |
/* due to the keeper block set->blocks should always be valid */ |
| 830 |
Assert(set->blocks != NULL); |
| 831 |
block = set->blocks; |
| 832 |
availspace = block->endptr - block->freeptr; |
| 833 |
|
| 834 |
/* |
| 835 |
* The existing active (top) block does not have enough room for the |
| 836 |
* requested allocation, but it might still have a useful amount of space |
| 837 |
* in it. Once we push it down in the block list, we'll never try to |
| 838 |
* allocate more space from it. So, before we do that, carve up its free |
| 839 |
* space into chunks that we can put on the set's freelists. |
| 840 |
* |
| 841 |
* Because we can only get here when there's less than ALLOC_CHUNK_LIMIT |
| 842 |
* left in the block, this loop cannot iterate more than |
| 843 |
* ALLOCSET_NUM_FREELISTS-1 times. |
| 844 |
*/ |
| 845 |
while (availspace >= ((1 << ALLOC_MINBITS) + ALLOC_CHUNKHDRSZ)) |
| 846 |
{ |
| 847 |
AllocFreeListLink *link; |
| 848 |
MemoryChunk *chunk; |
| 849 |
Size availchunk = availspace - ALLOC_CHUNKHDRSZ; |
| 850 |
int a_fidx = AllocSetFreeIndex(availchunk); |
| 851 |
|
| 852 |
/* |
| 853 |
* In most cases, we'll get back the index of the next larger freelist |
| 854 |
* than the one we need to put this chunk on. The exception is when |
| 855 |
* availchunk is exactly a power of 2. |
| 856 |
*/ |
| 857 |
if (availchunk != GetChunkSizeFromFreeListIdx(a_fidx)) |
| 858 |
{ |
| 859 |
a_fidx--; |
| 860 |
Assert(a_fidx >= 0); |
| 861 |
availchunk = GetChunkSizeFromFreeListIdx(a_fidx); |
| 862 |
} |
| 863 |
|
| 864 |
chunk = (MemoryChunk *) (block->freeptr); |
| 865 |
|
| 866 |
/* Prepare to initialize the chunk header. */ |
| 867 |
VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 868 |
block->freeptr += (availchunk + ALLOC_CHUNKHDRSZ); |
| 869 |
availspace -= (availchunk + ALLOC_CHUNKHDRSZ); |
| 870 |
|
| 871 |
/* store the freelist index in the value field */ |
| 872 |
MemoryChunkSetHdrMask(chunk, block, a_fidx, MCTX_ASET_ID); |
| 873 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 874 |
chunk->requested_size = InvalidAllocSize; /* mark it free */ |
| 875 |
#endif |
| 876 |
/* push this chunk onto the free list */ |
| 877 |
link = GetFreeListLink(chunk); |
| 878 |
|
| 879 |
VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); |
| 880 |
link->next = set->freelist[a_fidx]; |
| 881 |
VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); |
| 882 |
|
| 883 |
set->freelist[a_fidx] = chunk; |
| 884 |
} |
| 885 |
|
| 886 |
/* |
| 887 |
* The first such block has size initBlockSize, and we double the space in |
| 888 |
* each succeeding block, but not more than maxBlockSize. |
| 889 |
*/ |
| 890 |
blksize = set->nextBlockSize; |
| 891 |
set->nextBlockSize <<= 1; |
| 892 |
if (set->nextBlockSize > set->maxBlockSize) |
| 893 |
set->nextBlockSize = set->maxBlockSize; |
| 894 |
|
| 895 |
/* Choose the actual chunk size to allocate */ |
| 896 |
chunk_size = GetChunkSizeFromFreeListIdx(fidx); |
| 897 |
Assert(chunk_size >= size); |
| 898 |
|
| 899 |
/* |
| 900 |
* If initBlockSize is less than ALLOC_CHUNK_LIMIT, we could need more |
| 901 |
* space... but try to keep it a power of 2. |
| 902 |
*/ |
| 903 |
required_size = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; |
| 904 |
while (blksize < required_size) |
| 905 |
blksize <<= 1; |
| 906 |
|
| 907 |
/* Try to allocate it */ |
| 908 |
block = (AllocBlock) malloc(blksize); |
| 909 |
|
| 910 |
/* |
| 911 |
* We could be asking for pretty big blocks here, so cope if malloc fails. |
| 912 |
* But give up if there's less than 1 MB or so available... |
| 913 |
*/ |
| 914 |
while (block == NULL && blksize > 1024 * 1024) |
| 915 |
{ |
| 916 |
blksize >>= 1; |
| 917 |
if (blksize < required_size) |
| 918 |
break; |
| 919 |
block = (AllocBlock) malloc(blksize); |
| 920 |
} |
| 921 |
|
| 922 |
if (block == NULL) |
| 923 |
return MemoryContextAllocationFailure(context, size, flags); |
| 924 |
|
| 925 |
context->mem_allocated += blksize; |
| 926 |
|
| 927 |
block->aset = set; |
| 928 |
block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; |
| 929 |
block->endptr = ((char *) block) + blksize; |
| 930 |
|
| 931 |
/* Mark unallocated space NOACCESS. */ |
| 932 |
VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, |
| 933 |
blksize - ALLOC_BLOCKHDRSZ); |
| 934 |
|
| 935 |
block->prev = NULL; |
| 936 |
block->next = set->blocks; |
| 937 |
if (block->next) |
| 938 |
block->next->prev = block; |
| 939 |
set->blocks = block; |
| 940 |
|
| 941 |
return AllocSetAllocChunkFromBlock(context, block, size, chunk_size, fidx); |
| 942 |
} |
| 943 |
|
| 944 |
/* |
| 945 |
* AllocSetAlloc |
| 946 |
* Returns a pointer to allocated memory of given size or raises an ERROR |
| 947 |
* on allocation failure, or returns NULL when flags contains |
| 948 |
* MCXT_ALLOC_NO_OOM. |
| 949 |
* |
| 950 |
* No request may exceed: |
| 951 |
* MAXALIGN_DOWN(SIZE_MAX) - ALLOC_BLOCKHDRSZ - ALLOC_CHUNKHDRSZ |
| 952 |
* All callers use a much-lower limit. |
| 953 |
* |
| 954 |
* Note: when using valgrind, it doesn't matter how the returned allocation |
| 955 |
* is marked, as mcxt.c will set it to UNDEFINED. In some paths we will |
| 956 |
* return space that is marked NOACCESS - AllocSetRealloc has to beware! |
| 957 |
* |
| 958 |
* This function should only contain the most common code paths. Everything |
| 959 |
* else should be in pg_noinline helper functions, thus avoiding the overhead |
| 960 |
* of creating a stack frame for the common cases. Allocating memory is often |
| 961 |
* a bottleneck in many workloads, so avoiding stack frame setup is |
| 962 |
* worthwhile. Helper functions should always directly return the newly |
| 963 |
* allocated memory so that we can just return that address directly as a tail |
| 964 |
* call. |
| 965 |
*/ |
| 966 |
void * |
| 967 |
AllocSetAlloc(MemoryContext context, Size size, int flags) |
| 968 |
{ |
| 969 |
AllocSet set = (AllocSet) context; |
| 970 |
AllocBlock block; |
| 971 |
MemoryChunk *chunk; |
| 972 |
int fidx; |
| 973 |
Size chunk_size; |
| 974 |
Size availspace; |
| 975 |
|
| 976 |
Assert(AllocSetIsValid(set)); |
| 977 |
|
| 978 |
/* due to the keeper block set->blocks should never be NULL */ |
| 979 |
Assert(set->blocks != NULL); |
| 980 |
|
| 981 |
/* |
| 982 |
* If requested size exceeds maximum for chunks we hand the request off to |
| 983 |
* AllocSetAllocLarge(). |
| 984 |
*/ |
| 985 |
if (size > set->allocChunkLimit) |
| 986 |
return AllocSetAllocLarge(context, size, flags); |
| 987 |
|
| 988 |
/* |
| 989 |
* Request is small enough to be treated as a chunk. Look in the |
| 990 |
* corresponding free list to see if there is a free chunk we could reuse. |
| 991 |
* If one is found, remove it from the free list, make it again a member |
| 992 |
* of the alloc set and return its data address. |
| 993 |
* |
| 994 |
* Note that we don't attempt to ensure there's space for the sentinel |
| 995 |
* byte here. We expect a large proportion of allocations to be for sizes |
| 996 |
* which are already a power of 2. If we were to always make space for a |
| 997 |
* sentinel byte in MEMORY_CONTEXT_CHECKING builds, then we'd end up |
| 998 |
* doubling the memory requirements for such allocations. |
| 999 |
*/ |
| 1000 |
fidx = AllocSetFreeIndex(size); |
| 1001 |
chunk = set->freelist[fidx]; |
| 1002 |
if (chunk != NULL) |
| 1003 |
{ |
| 1004 |
AllocFreeListLink *link = GetFreeListLink(chunk); |
| 1005 |
|
| 1006 |
/* Allow access to the chunk header. */ |
| 1007 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1008 |
|
| 1009 |
Assert(fidx == MemoryChunkGetValue(chunk)); |
| 1010 |
|
| 1011 |
/* pop this chunk off the freelist */ |
| 1012 |
VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); |
| 1013 |
set->freelist[fidx] = link->next; |
| 1014 |
VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); |
| 1015 |
|
| 1016 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1017 |
chunk->requested_size = size; |
| 1018 |
/* set mark to catch clobber of "unused" space */ |
| 1019 |
if (size < GetChunkSizeFromFreeListIdx(fidx)) |
| 1020 |
set_sentinel(MemoryChunkGetPointer(chunk), size); |
| 1021 |
#endif |
| 1022 |
#ifdef RANDOMIZE_ALLOCATED_MEMORY |
| 1023 |
/* fill the allocated space with junk */ |
| 1024 |
randomize_mem((char *) MemoryChunkGetPointer(chunk), size); |
| 1025 |
#endif |
| 1026 |
|
| 1027 |
/* Ensure any padding bytes are marked NOACCESS. */ |
| 1028 |
VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, |
| 1029 |
GetChunkSizeFromFreeListIdx(fidx) - size); |
| 1030 |
|
| 1031 |
/* Disallow access to the chunk header. */ |
| 1032 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1033 |
|
| 1034 |
return MemoryChunkGetPointer(chunk); |
| 1035 |
} |
| 1036 |
|
| 1037 |
/* |
| 1038 |
* Choose the actual chunk size to allocate. |
| 1039 |
*/ |
| 1040 |
chunk_size = GetChunkSizeFromFreeListIdx(fidx); |
| 1041 |
Assert(chunk_size >= size); |
| 1042 |
|
| 1043 |
block = set->blocks; |
| 1044 |
availspace = block->endptr - block->freeptr; |
| 1045 |
|
| 1046 |
/* |
| 1047 |
* If there is enough room in the active allocation block, we will put the |
| 1048 |
* chunk into that block. Else must start a new one. |
| 1049 |
*/ |
| 1050 |
if (unlikely(availspace < (chunk_size + ALLOC_CHUNKHDRSZ))) |
| 1051 |
return AllocSetAllocFromNewBlock(context, size, flags, fidx); |
| 1052 |
|
| 1053 |
/* There's enough space on the current block, so allocate from that */ |
| 1054 |
return AllocSetAllocChunkFromBlock(context, block, size, chunk_size, fidx); |
| 1055 |
} |
| 1056 |
|
| 1057 |
/* |
| 1058 |
* AllocSetFree |
| 1059 |
* Frees allocated memory; memory is removed from the set. |
| 1060 |
*/ |
| 1061 |
void |
| 1062 |
AllocSetFree(void *pointer) |
| 1063 |
{ |
| 1064 |
AllocSet set; |
| 1065 |
MemoryChunk *chunk = PointerGetMemoryChunk(pointer); |
| 1066 |
|
| 1067 |
/* Allow access to the chunk header. */ |
| 1068 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1069 |
|
| 1070 |
if (MemoryChunkIsExternal(chunk)) |
| 1071 |
{ |
| 1072 |
/* Release single-chunk block. */ |
| 1073 |
AllocBlock block = ExternalChunkGetBlock(chunk); |
| 1074 |
|
| 1075 |
/* |
| 1076 |
* Try to verify that we have a sane block pointer: the block header |
| 1077 |
* should reference an aset and the freeptr should match the endptr. |
| 1078 |
*/ |
| 1079 |
if (!AllocBlockIsValid(block) || block->freeptr != block->endptr) |
| 1080 |
elog(ERROR, "could not find block containing chunk %p", chunk); |
| 1081 |
|
| 1082 |
set = block->aset; |
| 1083 |
|
| 1084 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1085 |
{ |
| 1086 |
/* Test for someone scribbling on unused space in chunk */ |
| 1087 |
Assert(chunk->requested_size < (block->endptr - (char *) pointer)); |
| 1088 |
if (!sentinel_ok(pointer, chunk->requested_size)) |
| 1089 |
elog(WARNING, "detected write past chunk end in %s %p", |
| 1090 |
set->header.name, chunk); |
| 1091 |
} |
| 1092 |
#endif |
| 1093 |
|
| 1094 |
/* OK, remove block from aset's list and free it */ |
| 1095 |
if (block->prev) |
| 1096 |
block->prev->next = block->next; |
| 1097 |
else |
| 1098 |
set->blocks = block->next; |
| 1099 |
if (block->next) |
| 1100 |
block->next->prev = block->prev; |
| 1101 |
|
| 1102 |
set->header.mem_allocated -= block->endptr - ((char *) block); |
| 1103 |
|
| 1104 |
#ifdef CLOBBER_FREED_MEMORY |
| 1105 |
wipe_mem(block, block->freeptr - ((char *) block)); |
| 1106 |
#endif |
| 1107 |
free(block); |
| 1108 |
} |
| 1109 |
else |
| 1110 |
{ |
| 1111 |
AllocBlock block = MemoryChunkGetBlock(chunk); |
| 1112 |
int fidx; |
| 1113 |
AllocFreeListLink *link; |
| 1114 |
|
| 1115 |
/* |
| 1116 |
* In this path, for speed reasons we just Assert that the referenced |
| 1117 |
* block is good. We can also Assert that the value field is sane. |
| 1118 |
* Future field experience may show that these Asserts had better |
| 1119 |
* become regular runtime test-and-elog checks. |
| 1120 |
*/ |
| 1121 |
Assert(AllocBlockIsValid(block)); |
| 1122 |
set = block->aset; |
| 1123 |
|
| 1124 |
fidx = MemoryChunkGetValue(chunk); |
| 1125 |
Assert(FreeListIdxIsValid(fidx)); |
| 1126 |
link = GetFreeListLink(chunk); |
| 1127 |
|
| 1128 |
/* |
| 1129 |
* It might seem odd that we use elevel ERROR for double-pfree but |
| 1130 |
* only WARNING for write-past-chunk-end. But the two conditions are |
| 1131 |
* not very comparable. In the double-pfree case we can prevent |
| 1132 |
* corruption before it happens; while if we let it go through, the |
| 1133 |
* result would be a corrupted freelist that allows this chunk to get |
| 1134 |
* re-allocated twice. Thus the original bug could cascade into |
| 1135 |
* hard-to-understand misbehavior that might manifest far away from |
| 1136 |
* the actual source of the problem. On the other hand, a write past |
| 1137 |
* chunk end can be relatively benign if just a few bytes too many |
| 1138 |
* were written: often, only padding or unused space gets affected. |
| 1139 |
* Moreover, whatever damage was done is already done, and we're just |
| 1140 |
* reporting after the fact with no ability to clean it up. So just |
| 1141 |
* warn, like AllocSetCheck would do if the chunk didn't get freed. |
| 1142 |
*/ |
| 1143 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1144 |
/* Test for previously-freed chunk */ |
| 1145 |
if (unlikely(chunk->requested_size == InvalidAllocSize)) |
| 1146 |
elog(ERROR, "detected double pfree in %s %p", |
| 1147 |
set->header.name, chunk); |
| 1148 |
/* Test for someone scribbling on unused space in chunk */ |
| 1149 |
if (chunk->requested_size < GetChunkSizeFromFreeListIdx(fidx)) |
| 1150 |
if (!sentinel_ok(pointer, chunk->requested_size)) |
| 1151 |
elog(WARNING, "detected write past chunk end in %s %p", |
| 1152 |
set->header.name, chunk); |
| 1153 |
#endif |
| 1154 |
|
| 1155 |
#ifdef CLOBBER_FREED_MEMORY |
| 1156 |
wipe_mem(pointer, GetChunkSizeFromFreeListIdx(fidx)); |
| 1157 |
#endif |
| 1158 |
/* push this chunk onto the top of the free list */ |
| 1159 |
VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); |
| 1160 |
link->next = set->freelist[fidx]; |
| 1161 |
VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); |
| 1162 |
set->freelist[fidx] = chunk; |
| 1163 |
|
| 1164 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1165 |
|
| 1166 |
/* |
| 1167 |
* Reset requested_size to InvalidAllocSize in chunks that are on free |
| 1168 |
* list. |
| 1169 |
*/ |
| 1170 |
chunk->requested_size = InvalidAllocSize; |
| 1171 |
#endif |
| 1172 |
} |
| 1173 |
} |
| 1174 |
|
| 1175 |
/* |
| 1176 |
* AllocSetRealloc |
| 1177 |
* Returns new pointer to allocated memory of given size or NULL if |
| 1178 |
* request could not be completed; this memory is added to the set. |
| 1179 |
* Memory associated with given pointer is copied into the new memory, |
| 1180 |
* and the old memory is freed. |
| 1181 |
* |
| 1182 |
* Without MEMORY_CONTEXT_CHECKING, we don't know the old request size. This |
| 1183 |
* makes our Valgrind client requests less-precise, hazarding false negatives. |
| 1184 |
* (In principle, we could use VALGRIND_GET_VBITS() to rediscover the old |
| 1185 |
* request size.) |
| 1186 |
*/ |
| 1187 |
void * |
| 1188 |
AllocSetRealloc(void *pointer, Size size, int flags) |
| 1189 |
{ |
| 1190 |
AllocBlock block; |
| 1191 |
AllocSet set; |
| 1192 |
MemoryChunk *chunk = PointerGetMemoryChunk(pointer); |
| 1193 |
Size oldchksize; |
| 1194 |
int fidx; |
| 1195 |
|
| 1196 |
/* Allow access to the chunk header. */ |
| 1197 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1198 |
|
| 1199 |
if (MemoryChunkIsExternal(chunk)) |
| 1200 |
{ |
| 1201 |
/* |
| 1202 |
* The chunk must have been allocated as a single-chunk block. Use |
| 1203 |
* realloc() to make the containing block bigger, or smaller, with |
| 1204 |
* minimum space wastage. |
| 1205 |
*/ |
| 1206 |
Size chksize; |
| 1207 |
Size blksize; |
| 1208 |
Size oldblksize; |
| 1209 |
|
| 1210 |
block = ExternalChunkGetBlock(chunk); |
| 1211 |
|
| 1212 |
/* |
| 1213 |
* Try to verify that we have a sane block pointer: the block header |
| 1214 |
* should reference an aset and the freeptr should match the endptr. |
| 1215 |
*/ |
| 1216 |
if (!AllocBlockIsValid(block) || block->freeptr != block->endptr) |
| 1217 |
elog(ERROR, "could not find block containing chunk %p", chunk); |
| 1218 |
|
| 1219 |
set = block->aset; |
| 1220 |
|
| 1221 |
/* only check size in paths where the limits could be hit */ |
| 1222 |
MemoryContextCheckSize((MemoryContext) set, size, flags); |
| 1223 |
|
| 1224 |
oldchksize = block->endptr - (char *) pointer; |
| 1225 |
|
| 1226 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1227 |
/* Test for someone scribbling on unused space in chunk */ |
| 1228 |
Assert(chunk->requested_size < oldchksize); |
| 1229 |
if (!sentinel_ok(pointer, chunk->requested_size)) |
| 1230 |
elog(WARNING, "detected write past chunk end in %s %p", |
| 1231 |
set->header.name, chunk); |
| 1232 |
#endif |
| 1233 |
|
| 1234 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1235 |
/* ensure there's always space for the sentinel byte */ |
| 1236 |
chksize = MAXALIGN(size + 1); |
| 1237 |
#else |
| 1238 |
chksize = MAXALIGN(size); |
| 1239 |
#endif |
| 1240 |
|
| 1241 |
/* Do the realloc */ |
| 1242 |
blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; |
| 1243 |
oldblksize = block->endptr - ((char *) block); |
| 1244 |
|
| 1245 |
block = (AllocBlock) realloc(block, blksize); |
| 1246 |
if (block == NULL) |
| 1247 |
{ |
| 1248 |
/* Disallow access to the chunk header. */ |
| 1249 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1250 |
return MemoryContextAllocationFailure(&set->header, size, flags); |
| 1251 |
} |
| 1252 |
|
| 1253 |
/* updated separately, not to underflow when (oldblksize > blksize) */ |
| 1254 |
set->header.mem_allocated -= oldblksize; |
| 1255 |
set->header.mem_allocated += blksize; |
| 1256 |
|
| 1257 |
block->freeptr = block->endptr = ((char *) block) + blksize; |
| 1258 |
|
| 1259 |
/* Update pointers since block has likely been moved */ |
| 1260 |
chunk = (MemoryChunk *) (((char *) block) + ALLOC_BLOCKHDRSZ); |
| 1261 |
pointer = MemoryChunkGetPointer(chunk); |
| 1262 |
if (block->prev) |
| 1263 |
block->prev->next = block; |
| 1264 |
else |
| 1265 |
set->blocks = block; |
| 1266 |
if (block->next) |
| 1267 |
block->next->prev = block; |
| 1268 |
|
| 1269 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1270 |
#ifdef RANDOMIZE_ALLOCATED_MEMORY |
| 1271 |
|
| 1272 |
/* |
| 1273 |
* We can only randomize the extra space if we know the prior request. |
| 1274 |
* When using Valgrind, randomize_mem() also marks memory UNDEFINED. |
| 1275 |
*/ |
| 1276 |
if (size > chunk->requested_size) |
| 1277 |
randomize_mem((char *) pointer + chunk->requested_size, |
| 1278 |
size - chunk->requested_size); |
| 1279 |
#else |
| 1280 |
|
| 1281 |
/* |
| 1282 |
* If this is an increase, realloc() will have marked any |
| 1283 |
* newly-allocated part (from oldchksize to chksize) UNDEFINED, but we |
| 1284 |
* also need to adjust trailing bytes from the old allocation (from |
| 1285 |
* chunk->requested_size to oldchksize) as they are marked NOACCESS. |
| 1286 |
* Make sure not to mark too many bytes in case chunk->requested_size |
| 1287 |
* < size < oldchksize. |
| 1288 |
*/ |
| 1289 |
#ifdef USE_VALGRIND |
| 1290 |
if (Min(size, oldchksize) > chunk->requested_size) |
| 1291 |
VALGRIND_MAKE_MEM_UNDEFINED((char *) pointer + chunk->requested_size, |
| 1292 |
Min(size, oldchksize) - chunk->requested_size); |
| 1293 |
#endif |
| 1294 |
#endif |
| 1295 |
|
| 1296 |
chunk->requested_size = size; |
| 1297 |
/* set mark to catch clobber of "unused" space */ |
| 1298 |
Assert(size < chksize); |
| 1299 |
set_sentinel(pointer, size); |
| 1300 |
#else /* !MEMORY_CONTEXT_CHECKING */ |
| 1301 |
|
| 1302 |
/* |
| 1303 |
* We may need to adjust marking of bytes from the old allocation as |
| 1304 |
* some of them may be marked NOACCESS. We don't know how much of the |
| 1305 |
* old chunk size was the requested size; it could have been as small |
| 1306 |
* as one byte. We have to be conservative and just mark the entire |
| 1307 |
* old portion DEFINED. Make sure not to mark memory beyond the new |
| 1308 |
* allocation in case it's smaller than the old one. |
| 1309 |
*/ |
| 1310 |
VALGRIND_MAKE_MEM_DEFINED(pointer, Min(size, oldchksize)); |
| 1311 |
#endif |
| 1312 |
|
| 1313 |
/* Ensure any padding bytes are marked NOACCESS. */ |
| 1314 |
VALGRIND_MAKE_MEM_NOACCESS((char *) pointer + size, chksize - size); |
| 1315 |
|
| 1316 |
/* Disallow access to the chunk header . */ |
| 1317 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1318 |
|
| 1319 |
return pointer; |
| 1320 |
} |
| 1321 |
|
| 1322 |
block = MemoryChunkGetBlock(chunk); |
| 1323 |
|
| 1324 |
/* |
| 1325 |
* In this path, for speed reasons we just Assert that the referenced |
| 1326 |
* block is good. We can also Assert that the value field is sane. Future |
| 1327 |
* field experience may show that these Asserts had better become regular |
| 1328 |
* runtime test-and-elog checks. |
| 1329 |
*/ |
| 1330 |
Assert(AllocBlockIsValid(block)); |
| 1331 |
set = block->aset; |
| 1332 |
|
| 1333 |
fidx = MemoryChunkGetValue(chunk); |
| 1334 |
Assert(FreeListIdxIsValid(fidx)); |
| 1335 |
oldchksize = GetChunkSizeFromFreeListIdx(fidx); |
| 1336 |
|
| 1337 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1338 |
/* See comments in AllocSetFree about uses of ERROR and WARNING here */ |
| 1339 |
/* Test for previously-freed chunk */ |
| 1340 |
if (unlikely(chunk->requested_size == InvalidAllocSize)) |
| 1341 |
elog(ERROR, "detected realloc of freed chunk in %s %p", |
| 1342 |
set->header.name, chunk); |
| 1343 |
/* Test for someone scribbling on unused space in chunk */ |
| 1344 |
if (chunk->requested_size < oldchksize) |
| 1345 |
if (!sentinel_ok(pointer, chunk->requested_size)) |
| 1346 |
elog(WARNING, "detected write past chunk end in %s %p", |
| 1347 |
set->header.name, chunk); |
| 1348 |
#endif |
| 1349 |
|
| 1350 |
/* |
| 1351 |
* Chunk sizes are aligned to power of 2 in AllocSetAlloc(). Maybe the |
| 1352 |
* allocated area already is >= the new size. (In particular, we will |
| 1353 |
* fall out here if the requested size is a decrease.) |
| 1354 |
*/ |
| 1355 |
if (oldchksize >= size) |
| 1356 |
{ |
| 1357 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1358 |
Size oldrequest = chunk->requested_size; |
| 1359 |
|
| 1360 |
#ifdef RANDOMIZE_ALLOCATED_MEMORY |
| 1361 |
/* We can only fill the extra space if we know the prior request */ |
| 1362 |
if (size > oldrequest) |
| 1363 |
randomize_mem((char *) pointer + oldrequest, |
| 1364 |
size - oldrequest); |
| 1365 |
#endif |
| 1366 |
|
| 1367 |
chunk->requested_size = size; |
| 1368 |
|
| 1369 |
/* |
| 1370 |
* If this is an increase, mark any newly-available part UNDEFINED. |
| 1371 |
* Otherwise, mark the obsolete part NOACCESS. |
| 1372 |
*/ |
| 1373 |
if (size > oldrequest) |
| 1374 |
VALGRIND_MAKE_MEM_UNDEFINED((char *) pointer + oldrequest, |
| 1375 |
size - oldrequest); |
| 1376 |
else |
| 1377 |
VALGRIND_MAKE_MEM_NOACCESS((char *) pointer + size, |
| 1378 |
oldchksize - size); |
| 1379 |
|
| 1380 |
/* set mark to catch clobber of "unused" space */ |
| 1381 |
if (size < oldchksize) |
| 1382 |
set_sentinel(pointer, size); |
| 1383 |
#else /* !MEMORY_CONTEXT_CHECKING */ |
| 1384 |
|
| 1385 |
/* |
| 1386 |
* We don't have the information to determine whether we're growing |
| 1387 |
* the old request or shrinking it, so we conservatively mark the |
| 1388 |
* entire new allocation DEFINED. |
| 1389 |
*/ |
| 1390 |
VALGRIND_MAKE_MEM_NOACCESS(pointer, oldchksize); |
| 1391 |
VALGRIND_MAKE_MEM_DEFINED(pointer, size); |
| 1392 |
#endif |
| 1393 |
|
| 1394 |
/* Disallow access to the chunk header. */ |
| 1395 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1396 |
|
| 1397 |
return pointer; |
| 1398 |
} |
| 1399 |
else |
| 1400 |
{ |
| 1401 |
/* |
| 1402 |
* Enlarge-a-small-chunk case. We just do this by brute force, ie, |
| 1403 |
* allocate a new chunk and copy the data. Since we know the existing |
| 1404 |
* data isn't huge, this won't involve any great memcpy expense, so |
| 1405 |
* it's not worth being smarter. (At one time we tried to avoid |
| 1406 |
* memcpy when it was possible to enlarge the chunk in-place, but that |
| 1407 |
* turns out to misbehave unpleasantly for repeated cycles of |
| 1408 |
* palloc/repalloc/pfree: the eventually freed chunks go into the |
| 1409 |
* wrong freelist for the next initial palloc request, and so we leak |
| 1410 |
* memory indefinitely. See pgsql-hackers archives for 2007-08-11.) |
| 1411 |
*/ |
| 1412 |
AllocPointer newPointer; |
| 1413 |
Size oldsize; |
| 1414 |
|
| 1415 |
/* allocate new chunk (this also checks size is valid) */ |
| 1416 |
newPointer = AllocSetAlloc((MemoryContext) set, size, flags); |
| 1417 |
|
| 1418 |
/* leave immediately if request was not completed */ |
| 1419 |
if (newPointer == NULL) |
| 1420 |
{ |
| 1421 |
/* Disallow access to the chunk header. */ |
| 1422 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1423 |
return MemoryContextAllocationFailure((MemoryContext) set, size, flags); |
| 1424 |
} |
| 1425 |
|
| 1426 |
/* |
| 1427 |
* AllocSetAlloc() may have returned a region that is still NOACCESS. |
| 1428 |
* Change it to UNDEFINED for the moment; memcpy() will then transfer |
| 1429 |
* definedness from the old allocation to the new. If we know the old |
| 1430 |
* allocation, copy just that much. Otherwise, make the entire old |
| 1431 |
* chunk defined to avoid errors as we copy the currently-NOACCESS |
| 1432 |
* trailing bytes. |
| 1433 |
*/ |
| 1434 |
VALGRIND_MAKE_MEM_UNDEFINED(newPointer, size); |
| 1435 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1436 |
oldsize = chunk->requested_size; |
| 1437 |
#else |
| 1438 |
oldsize = oldchksize; |
| 1439 |
VALGRIND_MAKE_MEM_DEFINED(pointer, oldsize); |
| 1440 |
#endif |
| 1441 |
|
| 1442 |
/* transfer existing data (certain to fit) */ |
| 1443 |
memcpy(newPointer, pointer, oldsize); |
| 1444 |
|
| 1445 |
/* free old chunk */ |
| 1446 |
AllocSetFree(pointer); |
| 1447 |
|
| 1448 |
return newPointer; |
| 1449 |
} |
| 1450 |
} |
| 1451 |
|
| 1452 |
/* |
| 1453 |
* AllocSetGetChunkContext |
| 1454 |
* Return the MemoryContext that 'pointer' belongs to. |
| 1455 |
*/ |
| 1456 |
MemoryContext |
| 1457 |
AllocSetGetChunkContext(void *pointer) |
| 1458 |
{ |
| 1459 |
MemoryChunk *chunk = PointerGetMemoryChunk(pointer); |
| 1460 |
AllocBlock block; |
| 1461 |
AllocSet set; |
| 1462 |
|
| 1463 |
/* Allow access to the chunk header. */ |
| 1464 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1465 |
|
| 1466 |
if (MemoryChunkIsExternal(chunk)) |
| 1467 |
block = ExternalChunkGetBlock(chunk); |
| 1468 |
else |
| 1469 |
block = (AllocBlock) MemoryChunkGetBlock(chunk); |
| 1470 |
|
| 1471 |
/* Disallow access to the chunk header. */ |
| 1472 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1473 |
|
| 1474 |
Assert(AllocBlockIsValid(block)); |
| 1475 |
set = block->aset; |
| 1476 |
|
| 1477 |
return &set->header; |
| 1478 |
} |
| 1479 |
|
| 1480 |
/* |
| 1481 |
* AllocSetGetChunkSpace |
| 1482 |
* Given a currently-allocated chunk, determine the total space |
| 1483 |
* it occupies (including all memory-allocation overhead). |
| 1484 |
*/ |
| 1485 |
Size |
| 1486 |
AllocSetGetChunkSpace(void *pointer) |
| 1487 |
{ |
| 1488 |
MemoryChunk *chunk = PointerGetMemoryChunk(pointer); |
| 1489 |
int fidx; |
| 1490 |
|
| 1491 |
/* Allow access to the chunk header. */ |
| 1492 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1493 |
|
| 1494 |
if (MemoryChunkIsExternal(chunk)) |
| 1495 |
{ |
| 1496 |
AllocBlock block = ExternalChunkGetBlock(chunk); |
| 1497 |
|
| 1498 |
/* Disallow access to the chunk header. */ |
| 1499 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1500 |
|
| 1501 |
Assert(AllocBlockIsValid(block)); |
| 1502 |
|
| 1503 |
return block->endptr - (char *) chunk; |
| 1504 |
} |
| 1505 |
|
| 1506 |
fidx = MemoryChunkGetValue(chunk); |
| 1507 |
Assert(FreeListIdxIsValid(fidx)); |
| 1508 |
|
| 1509 |
/* Disallow access to the chunk header. */ |
| 1510 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1511 |
|
| 1512 |
return GetChunkSizeFromFreeListIdx(fidx) + ALLOC_CHUNKHDRSZ; |
| 1513 |
} |
| 1514 |
|
| 1515 |
/* |
| 1516 |
* AllocSetIsEmpty |
| 1517 |
* Is an allocset empty of any allocated space? |
| 1518 |
*/ |
| 1519 |
bool |
| 1520 |
AllocSetIsEmpty(MemoryContext context) |
| 1521 |
{ |
| 1522 |
Assert(AllocSetIsValid(context)); |
| 1523 |
|
| 1524 |
/* |
| 1525 |
* For now, we say "empty" only if the context is new or just reset. We |
| 1526 |
* could examine the freelists to determine if all space has been freed, |
| 1527 |
* but it's not really worth the trouble for present uses of this |
| 1528 |
* functionality. |
| 1529 |
*/ |
| 1530 |
if (context->isReset) |
| 1531 |
return true; |
| 1532 |
return false; |
| 1533 |
} |
| 1534 |
|
| 1535 |
/* |
| 1536 |
* AllocSetStats |
| 1537 |
* Compute stats about memory consumption of an allocset. |
| 1538 |
* |
| 1539 |
* printfunc: if not NULL, pass a human-readable stats string to this. |
| 1540 |
* passthru: pass this pointer through to printfunc. |
| 1541 |
* totals: if not NULL, add stats about this context into *totals. |
| 1542 |
* print_to_stderr: print stats to stderr if true, elog otherwise. |
| 1543 |
*/ |
| 1544 |
void |
| 1545 |
AllocSetStats(MemoryContext context, |
| 1546 |
MemoryStatsPrintFunc printfunc, void *passthru, |
| 1547 |
MemoryContextCounters *totals, bool print_to_stderr) |
| 1548 |
{ |
| 1549 |
AllocSet set = (AllocSet) context; |
| 1550 |
Size nblocks = 0; |
| 1551 |
Size freechunks = 0; |
| 1552 |
Size totalspace; |
| 1553 |
Size freespace = 0; |
| 1554 |
AllocBlock block; |
| 1555 |
int fidx; |
| 1556 |
|
| 1557 |
Assert(AllocSetIsValid(set)); |
| 1558 |
|
| 1559 |
/* Include context header in totalspace */ |
| 1560 |
totalspace = MAXALIGN(sizeof(AllocSetContext)); |
| 1561 |
|
| 1562 |
for (block = set->blocks; block != NULL; block = block->next) |
| 1563 |
{ |
| 1564 |
nblocks++; |
| 1565 |
totalspace += block->endptr - ((char *) block); |
| 1566 |
freespace += block->endptr - block->freeptr; |
| 1567 |
} |
| 1568 |
for (fidx = 0; fidx < ALLOCSET_NUM_FREELISTS; fidx++) |
| 1569 |
{ |
| 1570 |
Size chksz = GetChunkSizeFromFreeListIdx(fidx); |
| 1571 |
MemoryChunk *chunk = set->freelist[fidx]; |
| 1572 |
|
| 1573 |
while (chunk != NULL) |
| 1574 |
{ |
| 1575 |
AllocFreeListLink *link = GetFreeListLink(chunk); |
| 1576 |
|
| 1577 |
/* Allow access to the chunk header. */ |
| 1578 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1579 |
Assert(MemoryChunkGetValue(chunk) == fidx); |
| 1580 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1581 |
|
| 1582 |
freechunks++; |
| 1583 |
freespace += chksz + ALLOC_CHUNKHDRSZ; |
| 1584 |
|
| 1585 |
VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); |
| 1586 |
chunk = link->next; |
| 1587 |
VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); |
| 1588 |
} |
| 1589 |
} |
| 1590 |
|
| 1591 |
if (printfunc) |
| 1592 |
{ |
| 1593 |
char stats_string[200]; |
| 1594 |
|
| 1595 |
snprintf(stats_string, sizeof(stats_string), |
| 1596 |
"%zu total in %zu blocks; %zu free (%zu chunks); %zu used", |
| 1597 |
totalspace, nblocks, freespace, freechunks, |
| 1598 |
totalspace - freespace); |
| 1599 |
printfunc(context, passthru, stats_string, print_to_stderr); |
| 1600 |
} |
| 1601 |
|
| 1602 |
if (totals) |
| 1603 |
{ |
| 1604 |
totals->nblocks += nblocks; |
| 1605 |
totals->freechunks += freechunks; |
| 1606 |
totals->totalspace += totalspace; |
| 1607 |
totals->freespace += freespace; |
| 1608 |
} |
| 1609 |
} |
| 1610 |
|
| 1611 |
|
| 1612 |
#ifdef MEMORY_CONTEXT_CHECKING |
| 1613 |
|
| 1614 |
/* |
| 1615 |
* AllocSetCheck |
| 1616 |
* Walk through chunks and check consistency of memory. |
| 1617 |
* |
| 1618 |
* NOTE: report errors as WARNING, *not* ERROR or FATAL. Otherwise you'll |
| 1619 |
* find yourself in an infinite loop when trouble occurs, because this |
| 1620 |
* routine will be entered again when elog cleanup tries to release memory! |
| 1621 |
*/ |
| 1622 |
void |
| 1623 |
AllocSetCheck(MemoryContext context) |
| 1624 |
{ |
| 1625 |
AllocSet set = (AllocSet) context; |
| 1626 |
const char *name = set->header.name; |
| 1627 |
AllocBlock prevblock; |
| 1628 |
AllocBlock block; |
| 1629 |
Size total_allocated = 0; |
| 1630 |
|
| 1631 |
for (prevblock = NULL, block = set->blocks; |
| 1632 |
block != NULL; |
| 1633 |
prevblock = block, block = block->next) |
| 1634 |
{ |
| 1635 |
char *bpoz = ((char *) block) + ALLOC_BLOCKHDRSZ; |
| 1636 |
Size blk_used = block->freeptr - bpoz; |
| 1637 |
Size blk_data = 0; |
| 1638 |
Size nchunks = 0; |
| 1639 |
bool has_external_chunk = false; |
| 1640 |
|
| 1641 |
if (IsKeeperBlock(set, block)) |
| 1642 |
total_allocated += block->endptr - ((char *) set); |
| 1643 |
else |
| 1644 |
total_allocated += block->endptr - ((char *) block); |
| 1645 |
|
| 1646 |
/* |
| 1647 |
* Empty block - empty can be keeper-block only |
| 1648 |
*/ |
| 1649 |
if (!blk_used) |
| 1650 |
{ |
| 1651 |
if (!IsKeeperBlock(set, block)) |
| 1652 |
elog(WARNING, "problem in alloc set %s: empty block %p", |
| 1653 |
name, block); |
| 1654 |
} |
| 1655 |
|
| 1656 |
/* |
| 1657 |
* Check block header fields |
| 1658 |
*/ |
| 1659 |
if (block->aset != set || |
| 1660 |
block->prev != prevblock || |
| 1661 |
block->freeptr < bpoz || |
| 1662 |
block->freeptr > block->endptr) |
| 1663 |
elog(WARNING, "problem in alloc set %s: corrupt header in block %p", |
| 1664 |
name, block); |
| 1665 |
|
| 1666 |
/* |
| 1667 |
* Chunk walker |
| 1668 |
*/ |
| 1669 |
while (bpoz < block->freeptr) |
| 1670 |
{ |
| 1671 |
MemoryChunk *chunk = (MemoryChunk *) bpoz; |
| 1672 |
Size chsize, |
| 1673 |
dsize; |
| 1674 |
|
| 1675 |
/* Allow access to the chunk header. */ |
| 1676 |
VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ); |
| 1677 |
|
| 1678 |
if (MemoryChunkIsExternal(chunk)) |
| 1679 |
{ |
| 1680 |
chsize = block->endptr - (char *) MemoryChunkGetPointer(chunk); /* aligned chunk size */ |
| 1681 |
has_external_chunk = true; |
| 1682 |
|
| 1683 |
/* make sure this chunk consumes the entire block */ |
| 1684 |
if (chsize + ALLOC_CHUNKHDRSZ != blk_used) |
| 1685 |
elog(WARNING, "problem in alloc set %s: bad single-chunk %p in block %p", |
| 1686 |
name, chunk, block); |
| 1687 |
} |
| 1688 |
else |
| 1689 |
{ |
| 1690 |
int fidx = MemoryChunkGetValue(chunk); |
| 1691 |
|
| 1692 |
if (!FreeListIdxIsValid(fidx)) |
| 1693 |
elog(WARNING, "problem in alloc set %s: bad chunk size for chunk %p in block %p", |
| 1694 |
name, chunk, block); |
| 1695 |
|
| 1696 |
chsize = GetChunkSizeFromFreeListIdx(fidx); /* aligned chunk size */ |
| 1697 |
|
| 1698 |
/* |
| 1699 |
* Check the stored block offset correctly references this |
| 1700 |
* block. |
| 1701 |
*/ |
| 1702 |
if (block != MemoryChunkGetBlock(chunk)) |
| 1703 |
elog(WARNING, "problem in alloc set %s: bad block offset for chunk %p in block %p", |
| 1704 |
name, chunk, block); |
| 1705 |
} |
| 1706 |
dsize = chunk->requested_size; /* real data */ |
| 1707 |
|
| 1708 |
/* an allocated chunk's requested size must be <= the chsize */ |
| 1709 |
if (dsize != InvalidAllocSize && dsize > chsize) |
| 1710 |
elog(WARNING, "problem in alloc set %s: req size > alloc size for chunk %p in block %p", |
| 1711 |
name, chunk, block); |
| 1712 |
|
| 1713 |
/* chsize must not be smaller than the first freelist's size */ |
| 1714 |
if (chsize < (1 << ALLOC_MINBITS)) |
| 1715 |
elog(WARNING, "problem in alloc set %s: bad size %zu for chunk %p in block %p", |
| 1716 |
name, chsize, chunk, block); |
| 1717 |
|
| 1718 |
/* |
| 1719 |
* Check for overwrite of padding space in an allocated chunk. |
| 1720 |
*/ |
| 1721 |
if (dsize != InvalidAllocSize && dsize < chsize && |
| 1722 |
!sentinel_ok(chunk, ALLOC_CHUNKHDRSZ + dsize)) |
| 1723 |
elog(WARNING, "problem in alloc set %s: detected write past chunk end in block %p, chunk %p", |
| 1724 |
name, block, chunk); |
| 1725 |
|
| 1726 |
/* if chunk is allocated, disallow access to the chunk header */ |
| 1727 |
if (dsize != InvalidAllocSize) |
| 1728 |
VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ); |
| 1729 |
|
| 1730 |
blk_data += chsize; |
| 1731 |
nchunks++; |
| 1732 |
|
| 1733 |
bpoz += ALLOC_CHUNKHDRSZ + chsize; |
| 1734 |
} |
| 1735 |
|
| 1736 |
if ((blk_data + (nchunks * ALLOC_CHUNKHDRSZ)) != blk_used) |
| 1737 |
elog(WARNING, "problem in alloc set %s: found inconsistent memory block %p", |
| 1738 |
name, block); |
| 1739 |
|
| 1740 |
if (has_external_chunk && nchunks > 1) |
| 1741 |
elog(WARNING, "problem in alloc set %s: external chunk on non-dedicated block %p", |
| 1742 |
name, block); |
| 1743 |
} |
| 1744 |
|
| 1745 |
Assert(total_allocated == context->mem_allocated); |
| 1746 |
} |
| 1747 |
|
| 1748 |
#endif /* MEMORY_CONTEXT_CHECKING */ |