Debian Codegraph @

src/prim/osx/alloc-override-zone.c

raw
1 /* ----------------------------------------------------------------------------
2 Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
3 This is free software; you can redistribute it and/or modify it under the
4 terms of the MIT license. A copy of the license can be found in the file
5 "LICENSE" at the root of this distribution.
6 -----------------------------------------------------------------------------*/
7
8 #include "mimalloc.h"
9 #include "mimalloc/internal.h"
10
11 #if defined(MI_MALLOC_OVERRIDE)
12
13 #if !defined(__APPLE__)
14 #error "this file should only be included on macOS"
15 #endif
16
17 /* ------------------------------------------------------
18 Override system malloc on macOS
19 This is done through the malloc zone interface.
20 It seems to be most robust in combination with interposing
21 though or otherwise we may get zone errors as there are could
22 be allocations done by the time we take over the
23 zone.
24 ------------------------------------------------------ */
25
26 #include <AvailabilityMacros.h>
27 #include <malloc/malloc.h>
28 #include <string.h> // memset
29 #include <stdlib.h>
30
31 #ifdef __cplusplus
32 extern "C" {
33 #endif
34
35 #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
36 // only available from OSX 10.6
37 extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));
38 #endif
39
40 /* ------------------------------------------------------
41 malloc zone members
42 ------------------------------------------------------ */
43
44 static bool is_mimalloc_zone( malloc_zone_t* zone );
45
46 static size_t zone_size(malloc_zone_t* zone, const void* p) {
47 if (mi_any_heap_contains(p)) {
48 return mi_usable_size(p);
49 }
50 else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
51 return zone->size(zone,p);
52 }
53 else {
54 return 0;
55 }
56 }
57
58 static void* zone_malloc(malloc_zone_t* zone, size_t size) {
59 MI_UNUSED(zone);
60 return mi_malloc(size);
61 }
62
63 static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) {
64 MI_UNUSED(zone);
65 return mi_calloc(count, size);
66 }
67
68 static void* zone_valloc(malloc_zone_t* zone, size_t size) {
69 MI_UNUSED(zone);
70 return mi_malloc_aligned(size, _mi_os_page_size());
71 }
72
73 static void zone_free(malloc_zone_t* zone, void* p) {
74 if (mi_any_heap_contains(p)) {
75 mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free
76 }
77 else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
78 zone->free(zone,p);
79 }
80 }
81
82 static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
83 if (p == NULL || mi_any_heap_contains(p)) {
84 return mi_realloc(p, newsize);
85 }
86 else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
87 return zone->realloc(zone,p,newsize);
88 }
89 else {
90 return NULL;
91 }
92 }
93
94 static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) {
95 MI_UNUSED(zone);
96 return mi_malloc_aligned(size,alignment);
97 }
98
99 static void zone_destroy(malloc_zone_t* zone) {
100 if (!is_mimalloc_zone(zone)) {
101 zone->destroy(zone);
102 }
103 }
104
105 static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) {
106 unsigned i;
107 for (i = 0; i < count; i++) {
108 ps[i] = zone_malloc(zone, size);
109 if (ps[i] == NULL) break;
110 }
111 return i;
112 }
113
114 static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) {
115 for(size_t i = 0; i < count; i++) {
116 zone_free(zone, ps[i]);
117 ps[i] = NULL;
118 }
119 }
120
121 static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) {
122 MI_UNUSED(zone); MI_UNUSED(size);
123 mi_collect(false);
124 return 0;
125 }
126
127 static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) {
128 MI_UNUSED(size);
129 zone_free(zone,p);
130 }
131
132 static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {
133 MI_UNUSED(zone);
134 return mi_is_in_heap_region(p);
135 }
136
137 /* ------------------------------------------------------
138 Introspection members
139 ------------------------------------------------------ */
140
141 static kern_return_t intro_enumerator(task_t task, void* p,
142 unsigned type_mask, vm_address_t zone_address,
143 memory_reader_t reader,
144 vm_range_recorder_t recorder)
145 {
146 // todo: enumerate all memory
147 MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address);
148 MI_UNUSED(reader); MI_UNUSED(recorder);
149 return KERN_SUCCESS;
150 }
151
152 static size_t intro_good_size(malloc_zone_t* zone, size_t size) {
153 MI_UNUSED(zone);
154 return mi_good_size(size);
155 }
156
157 static boolean_t intro_check(malloc_zone_t* zone) {
158 MI_UNUSED(zone);
159 return true;
160 }
161
162 static void intro_print(malloc_zone_t* zone, boolean_t verbose) {
163 MI_UNUSED(zone); MI_UNUSED(verbose);
164 mi_stats_print(NULL);
165 }
166
167 static void intro_log(malloc_zone_t* zone, void* p) {
168 MI_UNUSED(zone); MI_UNUSED(p);
169 // todo?
170 }
171
172 static void intro_force_lock(malloc_zone_t* zone) {
173 MI_UNUSED(zone);
174 // todo?
175 }
176
177 static void intro_force_unlock(malloc_zone_t* zone) {
178 MI_UNUSED(zone);
179 // todo?
180 }
181
182 static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) {
183 MI_UNUSED(zone);
184 // todo...
185 stats->blocks_in_use = 0;
186 stats->size_in_use = 0;
187 stats->max_size_in_use = 0;
188 stats->size_allocated = 0;
189 }
190
191 static boolean_t intro_zone_locked(malloc_zone_t* zone) {
192 MI_UNUSED(zone);
193 return false;
194 }
195
196 // Required whenever the zone advertises version >= 9: macOS calls this from the
197 // atfork_child handler (_malloc_fork_child) without a NULL check. mimalloc keeps
198 // no zone-level locks that need reinitializing after fork, so a no-op is safe.
199 // Leaving it NULL makes the forked child jump to address 0 and crash in fork().
200 static void intro_reinit_lock(malloc_zone_t* zone) {
201 MI_UNUSED(zone);
202 }
203
204
205 /* ------------------------------------------------------
206 At process start, override the default allocator
207 ------------------------------------------------------ */
208
209 #if defined(__GNUC__) && !defined(__clang__)
210 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
211 #endif
212
213 #if defined(__clang__)
214 #pragma clang diagnostic ignored "-Wc99-extensions"
215 #endif
216
217 static malloc_introspection_t mi_introspect = {
218 .enumerator = &intro_enumerator,
219 .good_size = &intro_good_size,
220 .check = &intro_check,
221 .print = &intro_print,
222 .log = &intro_log,
223 .force_lock = &intro_force_lock,
224 .force_unlock = &intro_force_unlock,
225 #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
226 .statistics = &intro_statistics,
227 .zone_locked = &intro_zone_locked,
228 .reinit_lock = &intro_reinit_lock,
229 #endif
230 };
231
232 static malloc_zone_t mi_malloc_zone = {
233 // note: even with designators, the order is important for C++ compilation
234 //.reserved1 = NULL,
235 //.reserved2 = NULL,
236 .size = &zone_size,
237 .malloc = &zone_malloc,
238 .calloc = &zone_calloc,
239 .valloc = &zone_valloc,
240 .free = &zone_free,
241 .realloc = &zone_realloc,
242 .destroy = &zone_destroy,
243 .zone_name = "mimalloc",
244 .batch_malloc = &zone_batch_malloc,
245 .batch_free = &zone_batch_free,
246 .introspect = &mi_introspect,
247 #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
248 #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
249 .version = 10,
250 #else
251 .version = 9,
252 #endif
253 // switch to version 9+ on OSX 10.6 to support memalign.
254 .memalign = &zone_memalign,
255 .free_definite_size = &zone_free_definite_size,
256 #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
257 .pressure_relief = &zone_pressure_relief,
258 #endif
259 #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)
260 .claimed_address = &zone_claimed_address,
261 #endif
262 #else
263 .version = 4,
264 #endif
265 };
266
267 #ifdef __cplusplus
268 }
269 #endif
270
271 static bool is_mimalloc_zone( malloc_zone_t* zone ) {
272 return (zone==NULL || zone==&mi_malloc_zone);
273 }
274
275 #if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT)
276
277 // ------------------------------------------------------
278 // Override malloc_xxx and malloc_zone_xxx api's to use only
279 // our mimalloc zone. Since even the loader uses malloc
280 // on macOS, this ensures that all allocations go through
281 // mimalloc (as all calls are interposed).
282 // The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`,
283 // Here, we also override macOS specific API's like
284 // `malloc_zone_calloc` etc. see <https://github.com/aosm/libmalloc/blob/master/man/malloc_zone_malloc.3>
285 // ------------------------------------------------------
286
287 static inline malloc_zone_t* mi_get_default_zone(void) {
288 mi_atomic_do_once {
289 malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>)
290 }
291 return &mi_malloc_zone;
292 }
293
294 mi_decl_externc int malloc_jumpstart(uintptr_t cookie);
295 mi_decl_externc void _malloc_fork_prepare(void);
296 mi_decl_externc void _malloc_fork_parent(void);
297 mi_decl_externc void _malloc_fork_child(void);
298
299
300 static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) {
301 MI_UNUSED(size); MI_UNUSED(flags);
302 return mi_get_default_zone();
303 }
304
305 static malloc_zone_t* mi_malloc_default_zone (void) {
306 return mi_get_default_zone();
307 }
308
309 static malloc_zone_t* mi_malloc_default_purgeable_zone(void) {
310 return mi_get_default_zone();
311 }
312
313 static void mi_malloc_destroy_zone(malloc_zone_t* zone) {
314 MI_UNUSED(zone);
315 // nothing.
316 }
317
318 static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) {
319 MI_UNUSED(task); MI_UNUSED(mr);
320 if (addresses != NULL) *addresses = NULL;
321 if (count != NULL) *count = 0;
322 return KERN_SUCCESS;
323 }
324
325 static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) {
326 return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name);
327 }
328
329 static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) {
330 MI_UNUSED(zone); MI_UNUSED(name);
331 }
332
333 static int mi_malloc_jumpstart(uintptr_t cookie) {
334 MI_UNUSED(cookie);
335 return 1; // or 0 for no error?
336 }
337
338 static void mi__malloc_fork_prepare(void) {
339 // nothing
340 }
341 static void mi__malloc_fork_parent(void) {
342 // nothing
343 }
344 static void mi__malloc_fork_child(void) {
345 // nothing
346 }
347
348 static void mi_malloc_printf(const char* fmt, ...) {
349 MI_UNUSED(fmt);
350 }
351
352 static bool zone_check(malloc_zone_t* zone) {
353 MI_UNUSED(zone);
354 return true;
355 }
356
357 static malloc_zone_t* zone_from_ptr(const void* p) {
358 MI_UNUSED(p);
359 return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL);
360 }
361
362 static void zone_log(malloc_zone_t* zone, void* p) {
363 MI_UNUSED(zone); MI_UNUSED(p);
364 }
365
366 static void zone_print(malloc_zone_t* zone, bool b) {
367 MI_UNUSED(zone); MI_UNUSED(b);
368 }
369
370 static void zone_print_ptr_info(void* p) {
371 MI_UNUSED(p);
372 }
373
374 static void zone_register(malloc_zone_t* zone) {
375 MI_UNUSED(zone);
376 }
377
378 static void zone_unregister(malloc_zone_t* zone) {
379 MI_UNUSED(zone);
380 }
381
382 // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`
383 // See: <https://books.google.com/books?id=K8vUkpOXhN4C&pg=PA73>
384 struct mi_interpose_s {
385 const void* replacement;
386 const void* target;
387 };
388 #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }
389 #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)
390 #define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun)
391 __attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) =
392 {
393
394 MI_INTERPOSE_MI(malloc_create_zone),
395 MI_INTERPOSE_MI(malloc_default_purgeable_zone),
396 MI_INTERPOSE_MI(malloc_default_zone),
397 MI_INTERPOSE_MI(malloc_destroy_zone),
398 MI_INTERPOSE_MI(malloc_get_all_zones),
399 MI_INTERPOSE_MI(malloc_get_zone_name),
400 MI_INTERPOSE_MI(malloc_jumpstart),
401 MI_INTERPOSE_MI(malloc_printf),
402 MI_INTERPOSE_MI(malloc_set_zone_name),
403 MI_INTERPOSE_MI(_malloc_fork_child),
404 MI_INTERPOSE_MI(_malloc_fork_parent),
405 MI_INTERPOSE_MI(_malloc_fork_prepare),
406
407 MI_INTERPOSE_ZONE(zone_batch_free),
408 MI_INTERPOSE_ZONE(zone_batch_malloc),
409 MI_INTERPOSE_ZONE(zone_calloc),
410 MI_INTERPOSE_ZONE(zone_check),
411 MI_INTERPOSE_ZONE(zone_free),
412 MI_INTERPOSE_ZONE(zone_from_ptr),
413 MI_INTERPOSE_ZONE(zone_log),
414 MI_INTERPOSE_ZONE(zone_malloc),
415 MI_INTERPOSE_ZONE(zone_memalign),
416 MI_INTERPOSE_ZONE(zone_print),
417 MI_INTERPOSE_ZONE(zone_print_ptr_info),
418 MI_INTERPOSE_ZONE(zone_realloc),
419 MI_INTERPOSE_ZONE(zone_register),
420 MI_INTERPOSE_ZONE(zone_unregister),
421 MI_INTERPOSE_ZONE(zone_valloc)
422 };
423
424
425 #else
426
427 // ------------------------------------------------------
428 // hook into the zone api's without interposing
429 // This is the official way of adding an allocator but
430 // it seems less robust than using interpose.
431 // ------------------------------------------------------
432
433 static inline malloc_zone_t* mi_get_default_zone(void)
434 {
435 // The first returned zone is the real default
436 malloc_zone_t** zones = NULL;
437 unsigned count = 0;
438 kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count);
439 if (ret == KERN_SUCCESS && count > 0) {
440 return zones[0];
441 }
442 else {
443 // fallback
444 return malloc_default_zone();
445 }
446 }
447
448 #if defined(__clang__)
449 __attribute__((constructor(101))) // highest priority
450 #else
451 __attribute__((constructor)) // priority level is not supported by gcc
452 #endif
453 __attribute__((used))
454 static void _mi_macos_override_malloc(void) {
455 malloc_zone_t* purgeable_zone = NULL;
456
457 #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
458 // force the purgeable zone to exist to avoid strange bugs
459 if (malloc_default_purgeable_zone) {
460 purgeable_zone = malloc_default_purgeable_zone();
461 }
462 #endif
463
464 // Register our zone.
465 // thomcc: I think this is still needed to put us in the zone list.
466 malloc_zone_register(&mi_malloc_zone);
467 // Unregister the default zone, this makes our zone the new default
468 // as that was the last registered.
469 malloc_zone_t *default_zone = mi_get_default_zone();
470 // thomcc: Unsure if the next test is *always* false or just false in the
471 // cases I've tried. I'm also unsure if the code inside is needed. at all
472 if (default_zone != &mi_malloc_zone) {
473 malloc_zone_unregister(default_zone);
474
475 // Reregister the default zone so free and realloc in that zone keep working.
476 malloc_zone_register(default_zone);
477 }
478
479 // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs
480 // earlier than the default zone.
481 if (purgeable_zone != NULL) {
482 malloc_zone_unregister(purgeable_zone);
483 malloc_zone_register(purgeable_zone);
484 }
485
486 }
487 #endif // MI_OSX_INTERPOSE
488
489 #endif // MI_MALLOC_OVERRIDE