| 1 |
/* ---------------------------------------------------------------------------- |
| 2 |
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen |
| 3 |
This is free software; you can redistribute it and/or modify it under the |
| 4 |
terms of the MIT license. A copy of the license can be found in the file |
| 5 |
"LICENSE" at the root of this distribution. |
| 6 |
-----------------------------------------------------------------------------*/ |
| 7 |
|
| 8 |
#include "mimalloc.h" |
| 9 |
#include "mimalloc/internal.h" |
| 10 |
|
| 11 |
#if defined(MI_MALLOC_OVERRIDE) |
| 12 |
|
| 13 |
#if !defined(__APPLE__) |
| 14 |
#error "this file should only be included on macOS" |
| 15 |
#endif |
| 16 |
|
| 17 |
/* ------------------------------------------------------ |
| 18 |
Override system malloc on macOS |
| 19 |
This is done through the malloc zone interface. |
| 20 |
It seems to be most robust in combination with interposing |
| 21 |
though or otherwise we may get zone errors as there are could |
| 22 |
be allocations done by the time we take over the |
| 23 |
zone. |
| 24 |
------------------------------------------------------ */ |
| 25 |
|
| 26 |
#include <AvailabilityMacros.h> |
| 27 |
#include <malloc/malloc.h> |
| 28 |
#include <string.h> // memset |
| 29 |
#include <stdlib.h> |
| 30 |
|
| 31 |
#ifdef __cplusplus |
| 32 |
extern "C" { |
| 33 |
#endif |
| 34 |
|
| 35 |
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) |
| 36 |
// only available from OSX 10.6 |
| 37 |
extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); |
| 38 |
#endif |
| 39 |
|
| 40 |
/* ------------------------------------------------------ |
| 41 |
malloc zone members |
| 42 |
------------------------------------------------------ */ |
| 43 |
|
| 44 |
static size_t zone_size(malloc_zone_t* zone, const void* p) { |
| 45 |
MI_UNUSED(zone); |
| 46 |
if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out |
| 47 |
return mi_usable_size(p); |
| 48 |
} |
| 49 |
|
| 50 |
static void* zone_malloc(malloc_zone_t* zone, size_t size) { |
| 51 |
MI_UNUSED(zone); |
| 52 |
return mi_malloc(size); |
| 53 |
} |
| 54 |
|
| 55 |
static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { |
| 56 |
MI_UNUSED(zone); |
| 57 |
return mi_calloc(count, size); |
| 58 |
} |
| 59 |
|
| 60 |
static void* zone_valloc(malloc_zone_t* zone, size_t size) { |
| 61 |
MI_UNUSED(zone); |
| 62 |
return mi_malloc_aligned(size, _mi_os_page_size()); |
| 63 |
} |
| 64 |
|
| 65 |
static void zone_free(malloc_zone_t* zone, void* p) { |
| 66 |
MI_UNUSED(zone); |
| 67 |
mi_cfree(p); |
| 68 |
} |
| 69 |
|
| 70 |
static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { |
| 71 |
MI_UNUSED(zone); |
| 72 |
return mi_realloc(p, newsize); |
| 73 |
} |
| 74 |
|
| 75 |
static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { |
| 76 |
MI_UNUSED(zone); |
| 77 |
return mi_malloc_aligned(size,alignment); |
| 78 |
} |
| 79 |
|
| 80 |
static void zone_destroy(malloc_zone_t* zone) { |
| 81 |
MI_UNUSED(zone); |
| 82 |
// todo: ignore for now? |
| 83 |
} |
| 84 |
|
| 85 |
static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) { |
| 86 |
size_t i; |
| 87 |
for (i = 0; i < count; i++) { |
| 88 |
ps[i] = zone_malloc(zone, size); |
| 89 |
if (ps[i] == NULL) break; |
| 90 |
} |
| 91 |
return i; |
| 92 |
} |
| 93 |
|
| 94 |
static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { |
| 95 |
for(size_t i = 0; i < count; i++) { |
| 96 |
zone_free(zone, ps[i]); |
| 97 |
ps[i] = NULL; |
| 98 |
} |
| 99 |
} |
| 100 |
|
| 101 |
static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { |
| 102 |
MI_UNUSED(zone); MI_UNUSED(size); |
| 103 |
mi_collect(false); |
| 104 |
return 0; |
| 105 |
} |
| 106 |
|
| 107 |
static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { |
| 108 |
MI_UNUSED(size); |
| 109 |
zone_free(zone,p); |
| 110 |
} |
| 111 |
|
| 112 |
static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { |
| 113 |
MI_UNUSED(zone); |
| 114 |
return mi_is_in_heap_region(p); |
| 115 |
} |
| 116 |
|
| 117 |
|
| 118 |
/* ------------------------------------------------------ |
| 119 |
Introspection members |
| 120 |
------------------------------------------------------ */ |
| 121 |
|
| 122 |
static kern_return_t intro_enumerator(task_t task, void* p, |
| 123 |
unsigned type_mask, vm_address_t zone_address, |
| 124 |
memory_reader_t reader, |
| 125 |
vm_range_recorder_t recorder) |
| 126 |
{ |
| 127 |
// todo: enumerate all memory |
| 128 |
MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address); |
| 129 |
MI_UNUSED(reader); MI_UNUSED(recorder); |
| 130 |
return KERN_SUCCESS; |
| 131 |
} |
| 132 |
|
| 133 |
static size_t intro_good_size(malloc_zone_t* zone, size_t size) { |
| 134 |
MI_UNUSED(zone); |
| 135 |
return mi_good_size(size); |
| 136 |
} |
| 137 |
|
| 138 |
static boolean_t intro_check(malloc_zone_t* zone) { |
| 139 |
MI_UNUSED(zone); |
| 140 |
return true; |
| 141 |
} |
| 142 |
|
| 143 |
static void intro_print(malloc_zone_t* zone, boolean_t verbose) { |
| 144 |
MI_UNUSED(zone); MI_UNUSED(verbose); |
| 145 |
mi_stats_print(NULL); |
| 146 |
} |
| 147 |
|
| 148 |
static void intro_log(malloc_zone_t* zone, void* p) { |
| 149 |
MI_UNUSED(zone); MI_UNUSED(p); |
| 150 |
// todo? |
| 151 |
} |
| 152 |
|
| 153 |
static void intro_force_lock(malloc_zone_t* zone) { |
| 154 |
MI_UNUSED(zone); |
| 155 |
// todo? |
| 156 |
} |
| 157 |
|
| 158 |
static void intro_force_unlock(malloc_zone_t* zone) { |
| 159 |
MI_UNUSED(zone); |
| 160 |
// todo? |
| 161 |
} |
| 162 |
|
| 163 |
static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { |
| 164 |
MI_UNUSED(zone); |
| 165 |
// todo... |
| 166 |
stats->blocks_in_use = 0; |
| 167 |
stats->size_in_use = 0; |
| 168 |
stats->max_size_in_use = 0; |
| 169 |
stats->size_allocated = 0; |
| 170 |
} |
| 171 |
|
| 172 |
static boolean_t intro_zone_locked(malloc_zone_t* zone) { |
| 173 |
MI_UNUSED(zone); |
| 174 |
return false; |
| 175 |
} |
| 176 |
|
| 177 |
|
| 178 |
/* ------------------------------------------------------ |
| 179 |
At process start, override the default allocator |
| 180 |
------------------------------------------------------ */ |
| 181 |
|
| 182 |
#if defined(__GNUC__) && !defined(__clang__) |
| 183 |
#pragma GCC diagnostic ignored "-Wmissing-field-initializers" |
| 184 |
#endif |
| 185 |
|
| 186 |
#if defined(__clang__) |
| 187 |
#pragma clang diagnostic ignored "-Wc99-extensions" |
| 188 |
#endif |
| 189 |
|
| 190 |
static malloc_introspection_t mi_introspect = { |
| 191 |
.enumerator = &intro_enumerator, |
| 192 |
.good_size = &intro_good_size, |
| 193 |
.check = &intro_check, |
| 194 |
.print = &intro_print, |
| 195 |
.log = &intro_log, |
| 196 |
.force_lock = &intro_force_lock, |
| 197 |
.force_unlock = &intro_force_unlock, |
| 198 |
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) |
| 199 |
.statistics = &intro_statistics, |
| 200 |
.zone_locked = &intro_zone_locked, |
| 201 |
#endif |
| 202 |
}; |
| 203 |
|
| 204 |
static malloc_zone_t mi_malloc_zone = { |
| 205 |
// note: even with designators, the order is important for C++ compilation |
| 206 |
//.reserved1 = NULL, |
| 207 |
//.reserved2 = NULL, |
| 208 |
.size = &zone_size, |
| 209 |
.malloc = &zone_malloc, |
| 210 |
.calloc = &zone_calloc, |
| 211 |
.valloc = &zone_valloc, |
| 212 |
.free = &zone_free, |
| 213 |
.realloc = &zone_realloc, |
| 214 |
.destroy = &zone_destroy, |
| 215 |
.zone_name = "mimalloc", |
| 216 |
.batch_malloc = &zone_batch_malloc, |
| 217 |
.batch_free = &zone_batch_free, |
| 218 |
.introspect = &mi_introspect, |
| 219 |
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) |
| 220 |
#if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) |
| 221 |
.version = 10, |
| 222 |
#else |
| 223 |
.version = 9, |
| 224 |
#endif |
| 225 |
// switch to version 9+ on OSX 10.6 to support memalign. |
| 226 |
.memalign = &zone_memalign, |
| 227 |
.free_definite_size = &zone_free_definite_size, |
| 228 |
.pressure_relief = &zone_pressure_relief, |
| 229 |
#if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) |
| 230 |
.claimed_address = &zone_claimed_address, |
| 231 |
#endif |
| 232 |
#else |
| 233 |
.version = 4, |
| 234 |
#endif |
| 235 |
}; |
| 236 |
|
| 237 |
#ifdef __cplusplus |
| 238 |
} |
| 239 |
#endif |
| 240 |
|
| 241 |
|
| 242 |
#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT) |
| 243 |
|
| 244 |
// ------------------------------------------------------ |
| 245 |
// Override malloc_xxx and malloc_zone_xxx api's to use only |
| 246 |
// our mimalloc zone. Since even the loader uses malloc |
| 247 |
// on macOS, this ensures that all allocations go through |
| 248 |
// mimalloc (as all calls are interposed). |
| 249 |
// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, |
| 250 |
// Here, we also override macOS specific API's like |
| 251 |
// `malloc_zone_calloc` etc. see <https://github.com/aosm/libmalloc/blob/master/man/malloc_zone_malloc.3> |
| 252 |
// ------------------------------------------------------ |
| 253 |
|
| 254 |
static inline malloc_zone_t* mi_get_default_zone(void) |
| 255 |
{ |
| 256 |
static bool init; |
| 257 |
if mi_unlikely(!init) { |
| 258 |
init = true; |
| 259 |
malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>) |
| 260 |
} |
| 261 |
return &mi_malloc_zone; |
| 262 |
} |
| 263 |
|
| 264 |
mi_decl_externc int malloc_jumpstart(uintptr_t cookie); |
| 265 |
mi_decl_externc void _malloc_fork_prepare(void); |
| 266 |
mi_decl_externc void _malloc_fork_parent(void); |
| 267 |
mi_decl_externc void _malloc_fork_child(void); |
| 268 |
|
| 269 |
|
| 270 |
static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { |
| 271 |
MI_UNUSED(size); MI_UNUSED(flags); |
| 272 |
return mi_get_default_zone(); |
| 273 |
} |
| 274 |
|
| 275 |
static malloc_zone_t* mi_malloc_default_zone (void) { |
| 276 |
return mi_get_default_zone(); |
| 277 |
} |
| 278 |
|
| 279 |
static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { |
| 280 |
return mi_get_default_zone(); |
| 281 |
} |
| 282 |
|
| 283 |
static void mi_malloc_destroy_zone(malloc_zone_t* zone) { |
| 284 |
MI_UNUSED(zone); |
| 285 |
// nothing. |
| 286 |
} |
| 287 |
|
| 288 |
static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { |
| 289 |
MI_UNUSED(task); MI_UNUSED(mr); |
| 290 |
if (addresses != NULL) *addresses = NULL; |
| 291 |
if (count != NULL) *count = 0; |
| 292 |
return KERN_SUCCESS; |
| 293 |
} |
| 294 |
|
| 295 |
static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { |
| 296 |
return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name); |
| 297 |
} |
| 298 |
|
| 299 |
static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { |
| 300 |
MI_UNUSED(zone); MI_UNUSED(name); |
| 301 |
} |
| 302 |
|
| 303 |
static int mi_malloc_jumpstart(uintptr_t cookie) { |
| 304 |
MI_UNUSED(cookie); |
| 305 |
return 1; // or 0 for no error? |
| 306 |
} |
| 307 |
|
| 308 |
static void mi__malloc_fork_prepare(void) { |
| 309 |
// nothing |
| 310 |
} |
| 311 |
static void mi__malloc_fork_parent(void) { |
| 312 |
// nothing |
| 313 |
} |
| 314 |
static void mi__malloc_fork_child(void) { |
| 315 |
// nothing |
| 316 |
} |
| 317 |
|
| 318 |
static void mi_malloc_printf(const char* fmt, ...) { |
| 319 |
MI_UNUSED(fmt); |
| 320 |
} |
| 321 |
|
| 322 |
static bool zone_check(malloc_zone_t* zone) { |
| 323 |
MI_UNUSED(zone); |
| 324 |
return true; |
| 325 |
} |
| 326 |
|
| 327 |
static malloc_zone_t* zone_from_ptr(const void* p) { |
| 328 |
MI_UNUSED(p); |
| 329 |
return mi_get_default_zone(); |
| 330 |
} |
| 331 |
|
| 332 |
static void zone_log(malloc_zone_t* zone, void* p) { |
| 333 |
MI_UNUSED(zone); MI_UNUSED(p); |
| 334 |
} |
| 335 |
|
| 336 |
static void zone_print(malloc_zone_t* zone, bool b) { |
| 337 |
MI_UNUSED(zone); MI_UNUSED(b); |
| 338 |
} |
| 339 |
|
| 340 |
static void zone_print_ptr_info(void* p) { |
| 341 |
MI_UNUSED(p); |
| 342 |
} |
| 343 |
|
| 344 |
static void zone_register(malloc_zone_t* zone) { |
| 345 |
MI_UNUSED(zone); |
| 346 |
} |
| 347 |
|
| 348 |
static void zone_unregister(malloc_zone_t* zone) { |
| 349 |
MI_UNUSED(zone); |
| 350 |
} |
| 351 |
|
| 352 |
// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` |
| 353 |
// See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73> |
| 354 |
struct mi_interpose_s { |
| 355 |
const void* replacement; |
| 356 |
const void* target; |
| 357 |
}; |
| 358 |
#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } |
| 359 |
#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) |
| 360 |
#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) |
| 361 |
__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = |
| 362 |
{ |
| 363 |
|
| 364 |
MI_INTERPOSE_MI(malloc_create_zone), |
| 365 |
MI_INTERPOSE_MI(malloc_default_purgeable_zone), |
| 366 |
MI_INTERPOSE_MI(malloc_default_zone), |
| 367 |
MI_INTERPOSE_MI(malloc_destroy_zone), |
| 368 |
MI_INTERPOSE_MI(malloc_get_all_zones), |
| 369 |
MI_INTERPOSE_MI(malloc_get_zone_name), |
| 370 |
MI_INTERPOSE_MI(malloc_jumpstart), |
| 371 |
MI_INTERPOSE_MI(malloc_printf), |
| 372 |
MI_INTERPOSE_MI(malloc_set_zone_name), |
| 373 |
MI_INTERPOSE_MI(_malloc_fork_child), |
| 374 |
MI_INTERPOSE_MI(_malloc_fork_parent), |
| 375 |
MI_INTERPOSE_MI(_malloc_fork_prepare), |
| 376 |
|
| 377 |
MI_INTERPOSE_ZONE(zone_batch_free), |
| 378 |
MI_INTERPOSE_ZONE(zone_batch_malloc), |
| 379 |
MI_INTERPOSE_ZONE(zone_calloc), |
| 380 |
MI_INTERPOSE_ZONE(zone_check), |
| 381 |
MI_INTERPOSE_ZONE(zone_free), |
| 382 |
MI_INTERPOSE_ZONE(zone_from_ptr), |
| 383 |
MI_INTERPOSE_ZONE(zone_log), |
| 384 |
MI_INTERPOSE_ZONE(zone_malloc), |
| 385 |
MI_INTERPOSE_ZONE(zone_memalign), |
| 386 |
MI_INTERPOSE_ZONE(zone_print), |
| 387 |
MI_INTERPOSE_ZONE(zone_print_ptr_info), |
| 388 |
MI_INTERPOSE_ZONE(zone_realloc), |
| 389 |
MI_INTERPOSE_ZONE(zone_register), |
| 390 |
MI_INTERPOSE_ZONE(zone_unregister), |
| 391 |
MI_INTERPOSE_ZONE(zone_valloc) |
| 392 |
}; |
| 393 |
|
| 394 |
|
| 395 |
#else |
| 396 |
|
| 397 |
// ------------------------------------------------------ |
| 398 |
// hook into the zone api's without interposing |
| 399 |
// This is the official way of adding an allocator but |
| 400 |
// it seems less robust than using interpose. |
| 401 |
// ------------------------------------------------------ |
| 402 |
|
| 403 |
static inline malloc_zone_t* mi_get_default_zone(void) |
| 404 |
{ |
| 405 |
// The first returned zone is the real default |
| 406 |
malloc_zone_t** zones = NULL; |
| 407 |
unsigned count = 0; |
| 408 |
kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); |
| 409 |
if (ret == KERN_SUCCESS && count > 0) { |
| 410 |
return zones[0]; |
| 411 |
} |
| 412 |
else { |
| 413 |
// fallback |
| 414 |
return malloc_default_zone(); |
| 415 |
} |
| 416 |
} |
| 417 |
|
| 418 |
#if defined(__clang__) |
| 419 |
__attribute__((constructor(0))) |
| 420 |
#else |
| 421 |
__attribute__((constructor)) // seems not supported by g++-11 on the M1 |
| 422 |
#endif |
| 423 |
static void _mi_macos_override_malloc(void) { |
| 424 |
malloc_zone_t* purgeable_zone = NULL; |
| 425 |
|
| 426 |
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) |
| 427 |
// force the purgeable zone to exist to avoid strange bugs |
| 428 |
if (malloc_default_purgeable_zone) { |
| 429 |
purgeable_zone = malloc_default_purgeable_zone(); |
| 430 |
} |
| 431 |
#endif |
| 432 |
|
| 433 |
// Register our zone. |
| 434 |
// thomcc: I think this is still needed to put us in the zone list. |
| 435 |
malloc_zone_register(&mi_malloc_zone); |
| 436 |
// Unregister the default zone, this makes our zone the new default |
| 437 |
// as that was the last registered. |
| 438 |
malloc_zone_t *default_zone = mi_get_default_zone(); |
| 439 |
// thomcc: Unsure if the next test is *always* false or just false in the |
| 440 |
// cases I've tried. I'm also unsure if the code inside is needed. at all |
| 441 |
if (default_zone != &mi_malloc_zone) { |
| 442 |
malloc_zone_unregister(default_zone); |
| 443 |
|
| 444 |
// Reregister the default zone so free and realloc in that zone keep working. |
| 445 |
malloc_zone_register(default_zone); |
| 446 |
} |
| 447 |
|
| 448 |
// Unregister, and re-register the purgeable_zone to avoid bugs if it occurs |
| 449 |
// earlier than the default zone. |
| 450 |
if (purgeable_zone != NULL) { |
| 451 |
malloc_zone_unregister(purgeable_zone); |
| 452 |
malloc_zone_register(purgeable_zone); |
| 453 |
} |
| 454 |
|
| 455 |
} |
| 456 |
#endif // MI_OSX_INTERPOSE |
| 457 |
|
| 458 |
#endif // MI_MALLOC_OVERRIDE |