| 1 |
#define _GNU_SOURCE |
| 2 |
#include <stdlib.h> |
| 3 |
#include <string.h> |
| 4 |
#include <limits.h> |
| 5 |
#include <stdint.h> |
| 6 |
#include <errno.h> |
| 7 |
#include <sys/mman.h> |
| 8 |
#include "libc.h" |
| 9 |
#include "atomic.h" |
| 10 |
#include "pthread_impl.h" |
| 11 |
#include "malloc_impl.h" |
| 12 |
#include "fork_impl.h" |
| 13 |
|
| 14 |
#define malloc __libc_malloc_impl |
| 15 |
#define realloc __libc_realloc |
| 16 |
#define free __libc_free |
| 17 |
|
| 18 |
#if defined(__GNUC__) && defined(__PIC__) |
| 19 |
#define inline inline __attribute__((always_inline)) |
| 20 |
#endif |
| 21 |
|
| 22 |
static struct { |
| 23 |
volatile uint64_t binmap; |
| 24 |
struct bin bins[64]; |
| 25 |
volatile int split_merge_lock[2]; |
| 26 |
} mal; |
| 27 |
|
| 28 |
/* Synchronization tools */ |
| 29 |
|
| 30 |
static inline void lock(volatile int *lk) |
| 31 |
{ |
| 32 |
int need_locks = libc.need_locks; |
| 33 |
if (need_locks) { |
| 34 |
while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); |
| 35 |
if (need_locks < 0) libc.need_locks = 0; |
| 36 |
} |
| 37 |
} |
| 38 |
|
| 39 |
static inline void unlock(volatile int *lk) |
| 40 |
{ |
| 41 |
if (lk[0]) { |
| 42 |
a_store(lk, 0); |
| 43 |
if (lk[1]) __wake(lk, 1, 1); |
| 44 |
} |
| 45 |
} |
| 46 |
|
| 47 |
static inline void lock_bin(int i) |
| 48 |
{ |
| 49 |
lock(mal.bins[i].lock); |
| 50 |
if (!mal.bins[i].head) |
| 51 |
mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i); |
| 52 |
} |
| 53 |
|
| 54 |
static inline void unlock_bin(int i) |
| 55 |
{ |
| 56 |
unlock(mal.bins[i].lock); |
| 57 |
} |
| 58 |
|
| 59 |
static int first_set(uint64_t x) |
| 60 |
{ |
| 61 |
#if 1 |
| 62 |
return a_ctz_64(x); |
| 63 |
#else |
| 64 |
static const char debruijn64[64] = { |
| 65 |
0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, |
| 66 |
62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, |
| 67 |
63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, |
| 68 |
51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 |
| 69 |
}; |
| 70 |
static const char debruijn32[32] = { |
| 71 |
0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, |
| 72 |
31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 |
| 73 |
}; |
| 74 |
if (sizeof(long) < 8) { |
| 75 |
uint32_t y = x; |
| 76 |
if (!y) { |
| 77 |
y = x>>32; |
| 78 |
return 32 + debruijn32[(y&-y)*0x076be629 >> 27]; |
| 79 |
} |
| 80 |
return debruijn32[(y&-y)*0x076be629 >> 27]; |
| 81 |
} |
| 82 |
return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; |
| 83 |
#endif |
| 84 |
} |
| 85 |
|
| 86 |
static const unsigned char bin_tab[60] = { |
| 87 |
32,33,34,35,36,36,37,37,38,38,39,39, |
| 88 |
40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43, |
| 89 |
44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45, |
| 90 |
46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47, |
| 91 |
}; |
| 92 |
|
| 93 |
static int bin_index(size_t x) |
| 94 |
{ |
| 95 |
x = x / SIZE_ALIGN - 1; |
| 96 |
if (x <= 32) return x; |
| 97 |
if (x < 512) return bin_tab[x/8-4]; |
| 98 |
if (x > 0x1c00) return 63; |
| 99 |
return bin_tab[x/128-4] + 16; |
| 100 |
} |
| 101 |
|
| 102 |
static int bin_index_up(size_t x) |
| 103 |
{ |
| 104 |
x = x / SIZE_ALIGN - 1; |
| 105 |
if (x <= 32) return x; |
| 106 |
x--; |
| 107 |
if (x < 512) return bin_tab[x/8-4] + 1; |
| 108 |
return bin_tab[x/128-4] + 17; |
| 109 |
} |
| 110 |
|
| 111 |
#if 0 |
| 112 |
void __dump_heap(int x) |
| 113 |
{ |
| 114 |
struct chunk *c; |
| 115 |
int i; |
| 116 |
for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c)) |
| 117 |
fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n", |
| 118 |
c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)), |
| 119 |
c->csize & 15, |
| 120 |
NEXT_CHUNK(c)->psize & 15); |
| 121 |
for (i=0; i<64; i++) { |
| 122 |
if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) { |
| 123 |
fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head); |
| 124 |
if (!(mal.binmap & 1ULL<<i)) |
| 125 |
fprintf(stderr, "missing from binmap!\n"); |
| 126 |
} else if (mal.binmap & 1ULL<<i) |
| 127 |
fprintf(stderr, "binmap wrongly contains %d!\n", i); |
| 128 |
} |
| 129 |
} |
| 130 |
#endif |
| 131 |
|
| 132 |
/* This function returns true if the interval [old,new] |
| 133 |
* intersects the 'len'-sized interval below &libc.auxv |
| 134 |
* (interpreted as the main-thread stack) or below &b |
| 135 |
* (the current stack). It is used to defend against |
| 136 |
* buggy brk implementations that can cross the stack. */ |
| 137 |
|
| 138 |
static int traverses_stack_p(uintptr_t old, uintptr_t new) |
| 139 |
{ |
| 140 |
const uintptr_t len = 8<<20; |
| 141 |
uintptr_t a, b; |
| 142 |
|
| 143 |
b = (uintptr_t)libc.auxv; |
| 144 |
a = b > len ? b-len : 0; |
| 145 |
if (new>a && old<b) return 1; |
| 146 |
|
| 147 |
b = (uintptr_t)&b; |
| 148 |
a = b > len ? b-len : 0; |
| 149 |
if (new>a && old<b) return 1; |
| 150 |
|
| 151 |
return 0; |
| 152 |
} |
| 153 |
|
| 154 |
/* Expand the heap in-place if brk can be used, or otherwise via mmap, |
| 155 |
* using an exponential lower bound on growth by mmap to make |
| 156 |
* fragmentation asymptotically irrelevant. The size argument is both |
| 157 |
* an input and an output, since the caller needs to know the size |
| 158 |
* allocated, which will be larger than requested due to page alignment |
| 159 |
* and mmap minimum size rules. The caller is responsible for locking |
| 160 |
* to prevent concurrent calls. */ |
| 161 |
|
| 162 |
static void *__expand_heap(size_t *pn) |
| 163 |
{ |
| 164 |
static uintptr_t brk; |
| 165 |
static unsigned mmap_step; |
| 166 |
size_t n = *pn; |
| 167 |
|
| 168 |
if (n > SIZE_MAX/2 - PAGE_SIZE) { |
| 169 |
errno = ENOMEM; |
| 170 |
return 0; |
| 171 |
} |
| 172 |
n += -n & PAGE_SIZE-1; |
| 173 |
|
| 174 |
if (!brk) { |
| 175 |
brk = __syscall(SYS_brk, 0); |
| 176 |
brk += -brk & PAGE_SIZE-1; |
| 177 |
} |
| 178 |
|
| 179 |
if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) |
| 180 |
&& __syscall(SYS_brk, brk+n)==brk+n) { |
| 181 |
*pn = n; |
| 182 |
brk += n; |
| 183 |
return (void *)(brk-n); |
| 184 |
} |
| 185 |
|
| 186 |
size_t min = (size_t)PAGE_SIZE << mmap_step/2; |
| 187 |
if (n < min) n = min; |
| 188 |
void *area = __mmap(0, n, PROT_READ|PROT_WRITE, |
| 189 |
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); |
| 190 |
if (area == MAP_FAILED) return 0; |
| 191 |
*pn = n; |
| 192 |
mmap_step++; |
| 193 |
return area; |
| 194 |
} |
| 195 |
|
| 196 |
static struct chunk *expand_heap(size_t n) |
| 197 |
{ |
| 198 |
static void *end; |
| 199 |
void *p; |
| 200 |
struct chunk *w; |
| 201 |
|
| 202 |
/* The argument n already accounts for the caller's chunk |
| 203 |
* overhead needs, but if the heap can't be extended in-place, |
| 204 |
* we need room for an extra zero-sized sentinel chunk. */ |
| 205 |
n += SIZE_ALIGN; |
| 206 |
|
| 207 |
p = __expand_heap(&n); |
| 208 |
if (!p) return 0; |
| 209 |
|
| 210 |
/* If not just expanding existing space, we need to make a |
| 211 |
* new sentinel chunk below the allocated space. */ |
| 212 |
if (p != end) { |
| 213 |
/* Valid/safe because of the prologue increment. */ |
| 214 |
n -= SIZE_ALIGN; |
| 215 |
p = (char *)p + SIZE_ALIGN; |
| 216 |
w = MEM_TO_CHUNK(p); |
| 217 |
w->psize = 0 | C_INUSE; |
| 218 |
} |
| 219 |
|
| 220 |
/* Record new heap end and fill in footer. */ |
| 221 |
end = (char *)p + n; |
| 222 |
w = MEM_TO_CHUNK(end); |
| 223 |
w->psize = n | C_INUSE; |
| 224 |
w->csize = 0 | C_INUSE; |
| 225 |
|
| 226 |
/* Fill in header, which may be new or may be replacing a |
| 227 |
* zero-size sentinel header at the old end-of-heap. */ |
| 228 |
w = MEM_TO_CHUNK(p); |
| 229 |
w->csize = n | C_INUSE; |
| 230 |
|
| 231 |
return w; |
| 232 |
} |
| 233 |
|
| 234 |
static int adjust_size(size_t *n) |
| 235 |
{ |
| 236 |
/* Result of pointer difference must fit in ptrdiff_t. */ |
| 237 |
if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) { |
| 238 |
if (*n) { |
| 239 |
errno = ENOMEM; |
| 240 |
return -1; |
| 241 |
} else { |
| 242 |
*n = SIZE_ALIGN; |
| 243 |
return 0; |
| 244 |
} |
| 245 |
} |
| 246 |
*n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK; |
| 247 |
return 0; |
| 248 |
} |
| 249 |
|
| 250 |
static void unbin(struct chunk *c, int i) |
| 251 |
{ |
| 252 |
if (c->prev == c->next) |
| 253 |
a_and_64(&mal.binmap, ~(1ULL<<i)); |
| 254 |
c->prev->next = c->next; |
| 255 |
c->next->prev = c->prev; |
| 256 |
c->csize |= C_INUSE; |
| 257 |
NEXT_CHUNK(c)->psize |= C_INUSE; |
| 258 |
} |
| 259 |
|
| 260 |
static void bin_chunk(struct chunk *self, int i) |
| 261 |
{ |
| 262 |
self->next = BIN_TO_CHUNK(i); |
| 263 |
self->prev = mal.bins[i].tail; |
| 264 |
self->next->prev = self; |
| 265 |
self->prev->next = self; |
| 266 |
if (self->prev == BIN_TO_CHUNK(i)) |
| 267 |
a_or_64(&mal.binmap, 1ULL<<i); |
| 268 |
} |
| 269 |
|
| 270 |
static void trim(struct chunk *self, size_t n) |
| 271 |
{ |
| 272 |
size_t n1 = CHUNK_SIZE(self); |
| 273 |
struct chunk *next, *split; |
| 274 |
|
| 275 |
if (n >= n1 - DONTCARE) return; |
| 276 |
|
| 277 |
next = NEXT_CHUNK(self); |
| 278 |
split = (void *)((char *)self + n); |
| 279 |
|
| 280 |
split->psize = n | C_INUSE; |
| 281 |
split->csize = n1-n; |
| 282 |
next->psize = n1-n; |
| 283 |
self->csize = n | C_INUSE; |
| 284 |
|
| 285 |
int i = bin_index(n1-n); |
| 286 |
lock_bin(i); |
| 287 |
|
| 288 |
bin_chunk(split, i); |
| 289 |
|
| 290 |
unlock_bin(i); |
| 291 |
} |
| 292 |
|
| 293 |
void *malloc(size_t n) |
| 294 |
{ |
| 295 |
struct chunk *c; |
| 296 |
int i, j; |
| 297 |
uint64_t mask; |
| 298 |
|
| 299 |
if (adjust_size(&n) < 0) return 0; |
| 300 |
|
| 301 |
if (n > MMAP_THRESHOLD) { |
| 302 |
size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE; |
| 303 |
char *base = __mmap(0, len, PROT_READ|PROT_WRITE, |
| 304 |
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); |
| 305 |
if (base == (void *)-1) return 0; |
| 306 |
c = (void *)(base + SIZE_ALIGN - OVERHEAD); |
| 307 |
c->csize = len - (SIZE_ALIGN - OVERHEAD); |
| 308 |
c->psize = SIZE_ALIGN - OVERHEAD; |
| 309 |
return CHUNK_TO_MEM(c); |
| 310 |
} |
| 311 |
|
| 312 |
i = bin_index_up(n); |
| 313 |
if (i<63 && (mal.binmap & (1ULL<<i))) { |
| 314 |
lock_bin(i); |
| 315 |
c = mal.bins[i].head; |
| 316 |
if (c != BIN_TO_CHUNK(i) && CHUNK_SIZE(c)-n <= DONTCARE) { |
| 317 |
unbin(c, i); |
| 318 |
unlock_bin(i); |
| 319 |
return CHUNK_TO_MEM(c); |
| 320 |
} |
| 321 |
unlock_bin(i); |
| 322 |
} |
| 323 |
lock(mal.split_merge_lock); |
| 324 |
for (mask = mal.binmap & -(1ULL<<i); mask; mask -= (mask&-mask)) { |
| 325 |
j = first_set(mask); |
| 326 |
lock_bin(j); |
| 327 |
c = mal.bins[j].head; |
| 328 |
if (c != BIN_TO_CHUNK(j)) { |
| 329 |
unbin(c, j); |
| 330 |
unlock_bin(j); |
| 331 |
break; |
| 332 |
} |
| 333 |
unlock_bin(j); |
| 334 |
} |
| 335 |
if (!mask) { |
| 336 |
c = expand_heap(n); |
| 337 |
if (!c) { |
| 338 |
unlock(mal.split_merge_lock); |
| 339 |
return 0; |
| 340 |
} |
| 341 |
} |
| 342 |
trim(c, n); |
| 343 |
unlock(mal.split_merge_lock); |
| 344 |
return CHUNK_TO_MEM(c); |
| 345 |
} |
| 346 |
|
| 347 |
int __malloc_allzerop(void *p) |
| 348 |
{ |
| 349 |
return IS_MMAPPED(MEM_TO_CHUNK(p)); |
| 350 |
} |
| 351 |
|
| 352 |
void *realloc(void *p, size_t n) |
| 353 |
{ |
| 354 |
struct chunk *self, *next; |
| 355 |
size_t n0, n1; |
| 356 |
void *new; |
| 357 |
|
| 358 |
if (!p) return malloc(n); |
| 359 |
|
| 360 |
if (adjust_size(&n) < 0) return 0; |
| 361 |
|
| 362 |
self = MEM_TO_CHUNK(p); |
| 363 |
n1 = n0 = CHUNK_SIZE(self); |
| 364 |
|
| 365 |
if (n<=n0 && n0-n<=DONTCARE) return p; |
| 366 |
|
| 367 |
if (IS_MMAPPED(self)) { |
| 368 |
size_t extra = self->psize; |
| 369 |
char *base = (char *)self - extra; |
| 370 |
size_t oldlen = n0 + extra; |
| 371 |
size_t newlen = n + extra; |
| 372 |
/* Crash on realloc of freed chunk */ |
| 373 |
if (extra & 1) a_crash(); |
| 374 |
if (newlen < PAGE_SIZE && (new = malloc(n-OVERHEAD))) { |
| 375 |
n0 = n; |
| 376 |
goto copy_free_ret; |
| 377 |
} |
| 378 |
newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE; |
| 379 |
if (oldlen == newlen) return p; |
| 380 |
base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE); |
| 381 |
if (base == (void *)-1) |
| 382 |
goto copy_realloc; |
| 383 |
self = (void *)(base + extra); |
| 384 |
self->csize = newlen - extra; |
| 385 |
return CHUNK_TO_MEM(self); |
| 386 |
} |
| 387 |
|
| 388 |
next = NEXT_CHUNK(self); |
| 389 |
|
| 390 |
/* Crash on corrupted footer (likely from buffer overflow) */ |
| 391 |
if (next->psize != self->csize) a_crash(); |
| 392 |
|
| 393 |
if (n < n0) { |
| 394 |
int i = bin_index_up(n); |
| 395 |
int j = bin_index(n0); |
| 396 |
if (i<j && (mal.binmap & (1ULL << i))) |
| 397 |
goto copy_realloc; |
| 398 |
struct chunk *split = (void *)((char *)self + n); |
| 399 |
self->csize = split->psize = n | C_INUSE; |
| 400 |
split->csize = next->psize = n0-n | C_INUSE; |
| 401 |
__bin_chunk(split); |
| 402 |
return CHUNK_TO_MEM(self); |
| 403 |
} |
| 404 |
|
| 405 |
lock(mal.split_merge_lock); |
| 406 |
|
| 407 |
size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); |
| 408 |
if (n0+nsize >= n) { |
| 409 |
int i = bin_index(nsize); |
| 410 |
lock_bin(i); |
| 411 |
if (!(next->csize & C_INUSE)) { |
| 412 |
unbin(next, i); |
| 413 |
unlock_bin(i); |
| 414 |
next = NEXT_CHUNK(next); |
| 415 |
self->csize = next->psize = n0+nsize | C_INUSE; |
| 416 |
trim(self, n); |
| 417 |
unlock(mal.split_merge_lock); |
| 418 |
return CHUNK_TO_MEM(self); |
| 419 |
} |
| 420 |
unlock_bin(i); |
| 421 |
} |
| 422 |
unlock(mal.split_merge_lock); |
| 423 |
|
| 424 |
copy_realloc: |
| 425 |
/* As a last resort, allocate a new chunk and copy to it. */ |
| 426 |
new = malloc(n-OVERHEAD); |
| 427 |
if (!new) return 0; |
| 428 |
copy_free_ret: |
| 429 |
memcpy(new, p, (n<n0 ? n : n0) - OVERHEAD); |
| 430 |
free(CHUNK_TO_MEM(self)); |
| 431 |
return new; |
| 432 |
} |
| 433 |
|
| 434 |
void __bin_chunk(struct chunk *self) |
| 435 |
{ |
| 436 |
struct chunk *next = NEXT_CHUNK(self); |
| 437 |
|
| 438 |
/* Crash on corrupted footer (likely from buffer overflow) */ |
| 439 |
if (next->psize != self->csize) a_crash(); |
| 440 |
|
| 441 |
lock(mal.split_merge_lock); |
| 442 |
|
| 443 |
size_t osize = CHUNK_SIZE(self), size = osize; |
| 444 |
|
| 445 |
/* Since we hold split_merge_lock, only transition from free to |
| 446 |
* in-use can race; in-use to free is impossible */ |
| 447 |
size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self); |
| 448 |
size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); |
| 449 |
|
| 450 |
if (psize) { |
| 451 |
int i = bin_index(psize); |
| 452 |
lock_bin(i); |
| 453 |
if (!(self->psize & C_INUSE)) { |
| 454 |
struct chunk *prev = PREV_CHUNK(self); |
| 455 |
unbin(prev, i); |
| 456 |
self = prev; |
| 457 |
size += psize; |
| 458 |
} |
| 459 |
unlock_bin(i); |
| 460 |
} |
| 461 |
if (nsize) { |
| 462 |
int i = bin_index(nsize); |
| 463 |
lock_bin(i); |
| 464 |
if (!(next->csize & C_INUSE)) { |
| 465 |
unbin(next, i); |
| 466 |
next = NEXT_CHUNK(next); |
| 467 |
size += nsize; |
| 468 |
} |
| 469 |
unlock_bin(i); |
| 470 |
} |
| 471 |
|
| 472 |
int i = bin_index(size); |
| 473 |
lock_bin(i); |
| 474 |
|
| 475 |
self->csize = size; |
| 476 |
next->psize = size; |
| 477 |
bin_chunk(self, i); |
| 478 |
unlock(mal.split_merge_lock); |
| 479 |
|
| 480 |
/* Replace middle of large chunks with fresh zero pages */ |
| 481 |
if (size > RECLAIM && (size^(size-osize)) > size-osize) { |
| 482 |
uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE; |
| 483 |
uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE; |
| 484 |
int e = errno; |
| 485 |
#if 1 |
| 486 |
__madvise((void *)a, b-a, MADV_DONTNEED); |
| 487 |
#else |
| 488 |
__mmap((void *)a, b-a, PROT_READ|PROT_WRITE, |
| 489 |
MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0); |
| 490 |
#endif |
| 491 |
errno = e; |
| 492 |
} |
| 493 |
|
| 494 |
unlock_bin(i); |
| 495 |
} |
| 496 |
|
| 497 |
static void unmap_chunk(struct chunk *self) |
| 498 |
{ |
| 499 |
size_t extra = self->psize; |
| 500 |
char *base = (char *)self - extra; |
| 501 |
size_t len = CHUNK_SIZE(self) + extra; |
| 502 |
/* Crash on double free */ |
| 503 |
if (extra & 1) a_crash(); |
| 504 |
int e = errno; |
| 505 |
__munmap(base, len); |
| 506 |
errno = e; |
| 507 |
} |
| 508 |
|
| 509 |
void free(void *p) |
| 510 |
{ |
| 511 |
if (!p) return; |
| 512 |
|
| 513 |
struct chunk *self = MEM_TO_CHUNK(p); |
| 514 |
|
| 515 |
if (IS_MMAPPED(self)) |
| 516 |
unmap_chunk(self); |
| 517 |
else |
| 518 |
__bin_chunk(self); |
| 519 |
} |
| 520 |
|
| 521 |
void __malloc_donate(char *start, char *end) |
| 522 |
{ |
| 523 |
size_t align_start_up = (SIZE_ALIGN-1) & (-(uintptr_t)start - OVERHEAD); |
| 524 |
size_t align_end_down = (SIZE_ALIGN-1) & (uintptr_t)end; |
| 525 |
|
| 526 |
/* Getting past this condition ensures that the padding for alignment |
| 527 |
* and header overhead will not overflow and will leave a nonzero |
| 528 |
* multiple of SIZE_ALIGN bytes between start and end. */ |
| 529 |
if (end - start <= OVERHEAD + align_start_up + align_end_down) |
| 530 |
return; |
| 531 |
start += align_start_up + OVERHEAD; |
| 532 |
end -= align_end_down; |
| 533 |
|
| 534 |
struct chunk *c = MEM_TO_CHUNK(start), *n = MEM_TO_CHUNK(end); |
| 535 |
c->psize = n->csize = C_INUSE; |
| 536 |
c->csize = n->psize = C_INUSE | (end-start); |
| 537 |
__bin_chunk(c); |
| 538 |
} |
| 539 |
|
| 540 |
void __malloc_atfork(int who) |
| 541 |
{ |
| 542 |
if (who<0) { |
| 543 |
lock(mal.split_merge_lock); |
| 544 |
for (int i=0; i<64; i++) |
| 545 |
lock(mal.bins[i].lock); |
| 546 |
} else if (!who) { |
| 547 |
for (int i=0; i<64; i++) |
| 548 |
unlock(mal.bins[i].lock); |
| 549 |
unlock(mal.split_merge_lock); |
| 550 |
} else { |
| 551 |
for (int i=0; i<64; i++) |
| 552 |
mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0; |
| 553 |
mal.split_merge_lock[1] = 0; |
| 554 |
mal.split_merge_lock[0] = 0; |
| 555 |
} |
| 556 |
} |