| 1 |
/* ---------------------------------------------------------------------------- |
| 2 |
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen |
| 3 |
This is free software; you can redistribute it and/or modify it under the |
| 4 |
terms of the MIT license. A copy of the license can be found in the file |
| 5 |
"LICENSE" at the root of this distribution. |
| 6 |
-----------------------------------------------------------------------------*/ |
| 7 |
#include "mimalloc.h" |
| 8 |
#include "mimalloc/internal.h" |
| 9 |
#include "mimalloc/prim.h" |
| 10 |
|
| 11 |
#include <string.h> // memcpy, memset |
| 12 |
#include <stdlib.h> // atexit |
| 13 |
|
| 14 |
|
| 15 |
// Empty page used to initialize the small free pages array |
| 16 |
const mi_page_t _mi_page_empty; |
| 17 |
|
| 18 |
#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) |
| 19 |
|
| 20 |
#if (MI_SMALL_WSIZE_MAX==128) |
| 21 |
#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) |
| 22 |
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } |
| 23 |
#elif (MI_PADDING>0) |
| 24 |
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } |
| 25 |
#else |
| 26 |
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } |
| 27 |
#endif |
| 28 |
#else |
| 29 |
#error "define right initialization sizes corresponding to MI_SMALL_WSIZE_MAX" |
| 30 |
#endif |
| 31 |
|
| 32 |
// Empty page queues for every bin |
| 33 |
#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } |
| 34 |
#define MI_PAGE_QUEUES_EMPTY \ |
| 35 |
{ QNULL(1), \ |
| 36 |
QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ |
| 37 |
QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \ |
| 38 |
QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \ |
| 39 |
QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \ |
| 40 |
QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \ |
| 41 |
QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \ |
| 42 |
QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ |
| 43 |
QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ |
| 44 |
QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ |
| 45 |
QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ |
| 46 |
QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 2) /* Full queue */ } |
| 47 |
|
| 48 |
#define MI_STAT_COUNT_NULL() {0,0,0,0} |
| 49 |
|
| 50 |
// Empty statistics |
| 51 |
#if MI_STAT>1 |
| 52 |
#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } |
| 53 |
#else |
| 54 |
#define MI_STAT_COUNT_END_NULL() |
| 55 |
#endif |
| 56 |
|
| 57 |
#define MI_STATS_NULL \ |
| 58 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 59 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 60 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 61 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 62 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 63 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 64 |
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ |
| 65 |
MI_STAT_COUNT_NULL(), \ |
| 66 |
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ |
| 67 |
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ |
| 68 |
MI_STAT_COUNT_END_NULL() |
| 69 |
|
| 70 |
|
| 71 |
// Empty slice span queues for every bin |
| 72 |
#define SQNULL(sz) { NULL, NULL, sz } |
| 73 |
#define MI_SEGMENT_SPAN_QUEUES_EMPTY \ |
| 74 |
{ SQNULL(1), \ |
| 75 |
SQNULL( 1), SQNULL( 2), SQNULL( 3), SQNULL( 4), SQNULL( 5), SQNULL( 6), SQNULL( 7), SQNULL( 10), /* 8 */ \ |
| 76 |
SQNULL( 12), SQNULL( 14), SQNULL( 16), SQNULL( 20), SQNULL( 24), SQNULL( 28), SQNULL( 32), SQNULL( 40), /* 16 */ \ |
| 77 |
SQNULL( 48), SQNULL( 56), SQNULL( 64), SQNULL( 80), SQNULL( 96), SQNULL( 112), SQNULL( 128), SQNULL( 160), /* 24 */ \ |
| 78 |
SQNULL( 192), SQNULL( 224), SQNULL( 256), SQNULL( 320), SQNULL( 384), SQNULL( 448), SQNULL( 512), SQNULL( 640), /* 32 */ \ |
| 79 |
SQNULL( 768), SQNULL( 896), SQNULL( 1024) /* 35 */ } |
| 80 |
|
| 81 |
|
| 82 |
// -------------------------------------------------------- |
| 83 |
// Statically allocate an empty heap as the initial |
| 84 |
// thread local value for the default heap, |
| 85 |
// and statically allocate the backing heap for the main |
| 86 |
// thread so it can function without doing any allocation |
| 87 |
// itself (as accessing a thread local for the first time |
| 88 |
// may lead to allocation itself on some platforms) |
| 89 |
// -------------------------------------------------------- |
| 90 |
|
| 91 |
mi_decl_cache_align const mi_heap_t _mi_heap_empty = { |
| 92 |
NULL, |
| 93 |
MI_SMALL_PAGES_EMPTY, |
| 94 |
MI_PAGE_QUEUES_EMPTY, |
| 95 |
MI_ATOMIC_VAR_INIT(NULL), |
| 96 |
0, // tid |
| 97 |
0, // cookie |
| 98 |
0, // arena id |
| 99 |
{ 0, 0 }, // keys |
| 100 |
{ {0}, {0}, 0, true }, // random |
| 101 |
0, // page count |
| 102 |
MI_BIN_FULL, 0, // page retired min/max |
| 103 |
NULL, // next |
| 104 |
false, |
| 105 |
0, |
| 106 |
0 |
| 107 |
}; |
| 108 |
|
| 109 |
#define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats))) |
| 110 |
#define tld_empty_os ((mi_os_tld_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,os))) |
| 111 |
|
| 112 |
mi_decl_cache_align static const mi_tld_t tld_empty = { |
| 113 |
0, |
| 114 |
false, |
| 115 |
NULL, NULL, |
| 116 |
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, tld_empty_stats, tld_empty_os, &_mi_abandoned_default }, // segments |
| 117 |
{ 0, tld_empty_stats }, // os |
| 118 |
{ MI_STATS_NULL } // stats |
| 119 |
}; |
| 120 |
|
| 121 |
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { |
| 122 |
return _mi_prim_thread_id(); |
| 123 |
} |
| 124 |
|
| 125 |
// the thread-local default heap for allocation |
| 126 |
mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; |
| 127 |
|
| 128 |
extern mi_heap_t _mi_heap_main; |
| 129 |
|
| 130 |
static mi_tld_t tld_main = { |
| 131 |
0, false, |
| 132 |
&_mi_heap_main, & _mi_heap_main, |
| 133 |
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, &tld_main.stats, &tld_main.os, &_mi_abandoned_default }, // segments |
| 134 |
{ 0, &tld_main.stats }, // os |
| 135 |
{ MI_STATS_NULL } // stats |
| 136 |
}; |
| 137 |
|
| 138 |
mi_heap_t _mi_heap_main = { |
| 139 |
&tld_main, |
| 140 |
MI_SMALL_PAGES_EMPTY, |
| 141 |
MI_PAGE_QUEUES_EMPTY, |
| 142 |
MI_ATOMIC_VAR_INIT(NULL), |
| 143 |
0, // thread id |
| 144 |
0, // initial cookie |
| 145 |
0, // arena id |
| 146 |
{ 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!) |
| 147 |
{ {0x846ca68b}, {0}, 0, true }, // random |
| 148 |
0, // page count |
| 149 |
MI_BIN_FULL, 0, // page retired min/max |
| 150 |
NULL, // next heap |
| 151 |
false // can reclaim |
| 152 |
}; |
| 153 |
|
| 154 |
bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. |
| 155 |
|
| 156 |
mi_stats_t _mi_stats_main = { MI_STATS_NULL }; |
| 157 |
|
| 158 |
|
| 159 |
static void mi_heap_main_init(void) { |
| 160 |
if (_mi_heap_main.cookie == 0) { |
| 161 |
_mi_heap_main.thread_id = _mi_thread_id(); |
| 162 |
_mi_heap_main.cookie = 1; |
| 163 |
#if defined(_WIN32) && !defined(MI_SHARED_LIB) |
| 164 |
_mi_random_init_weak(&_mi_heap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking |
| 165 |
#else |
| 166 |
_mi_random_init(&_mi_heap_main.random); |
| 167 |
#endif |
| 168 |
_mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main); |
| 169 |
_mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); |
| 170 |
_mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); |
| 171 |
} |
| 172 |
} |
| 173 |
|
| 174 |
mi_heap_t* _mi_heap_main_get(void) { |
| 175 |
mi_heap_main_init(); |
| 176 |
return &_mi_heap_main; |
| 177 |
} |
| 178 |
|
| 179 |
|
| 180 |
/* ----------------------------------------------------------- |
| 181 |
Initialization and freeing of the thread local heaps |
| 182 |
----------------------------------------------------------- */ |
| 183 |
|
| 184 |
// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). |
| 185 |
typedef struct mi_thread_data_s { |
| 186 |
mi_heap_t heap; // must come first due to cast in `_mi_heap_done` |
| 187 |
mi_tld_t tld; |
| 188 |
mi_memid_t memid; // must come last due to zero'ing |
| 189 |
} mi_thread_data_t; |
| 190 |
|
| 191 |
|
| 192 |
// Thread meta-data is allocated directly from the OS. For |
| 193 |
// some programs that do not use thread pools and allocate and |
| 194 |
// destroy many OS threads, this may causes too much overhead |
| 195 |
// per thread so we maintain a small cache of recently freed metadata. |
| 196 |
|
| 197 |
#define TD_CACHE_SIZE (16) |
| 198 |
static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; |
| 199 |
|
| 200 |
static mi_thread_data_t* mi_thread_data_zalloc(void) { |
| 201 |
// try to find thread metadata in the cache |
| 202 |
bool is_zero = false; |
| 203 |
mi_thread_data_t* td = NULL; |
| 204 |
for (int i = 0; i < TD_CACHE_SIZE; i++) { |
| 205 |
td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); |
| 206 |
if (td != NULL) { |
| 207 |
// found cached allocation, try use it |
| 208 |
td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); |
| 209 |
if (td != NULL) { |
| 210 |
break; |
| 211 |
} |
| 212 |
} |
| 213 |
} |
| 214 |
|
| 215 |
// if that fails, allocate as meta data |
| 216 |
if (td == NULL) { |
| 217 |
mi_memid_t memid; |
| 218 |
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); |
| 219 |
if (td == NULL) { |
| 220 |
// if this fails, try once more. (issue #257) |
| 221 |
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); |
| 222 |
if (td == NULL) { |
| 223 |
// really out of memory |
| 224 |
_mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); |
| 225 |
} |
| 226 |
} |
| 227 |
if (td != NULL) { |
| 228 |
td->memid = memid; |
| 229 |
is_zero = memid.initially_zero; |
| 230 |
} |
| 231 |
} |
| 232 |
|
| 233 |
if (td != NULL && !is_zero) { |
| 234 |
_mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid)); |
| 235 |
} |
| 236 |
return td; |
| 237 |
} |
| 238 |
|
| 239 |
static void mi_thread_data_free( mi_thread_data_t* tdfree ) { |
| 240 |
// try to add the thread metadata to the cache |
| 241 |
for (int i = 0; i < TD_CACHE_SIZE; i++) { |
| 242 |
mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); |
| 243 |
if (td == NULL) { |
| 244 |
mi_thread_data_t* expected = NULL; |
| 245 |
if (mi_atomic_cas_ptr_weak_acq_rel(mi_thread_data_t, &td_cache[i], &expected, tdfree)) { |
| 246 |
return; |
| 247 |
} |
| 248 |
} |
| 249 |
} |
| 250 |
// if that fails, just free it directly |
| 251 |
_mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid, &_mi_stats_main); |
| 252 |
} |
| 253 |
|
| 254 |
void _mi_thread_data_collect(void) { |
| 255 |
// free all thread metadata from the cache |
| 256 |
for (int i = 0; i < TD_CACHE_SIZE; i++) { |
| 257 |
mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); |
| 258 |
if (td != NULL) { |
| 259 |
td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); |
| 260 |
if (td != NULL) { |
| 261 |
_mi_os_free(td, sizeof(mi_thread_data_t), td->memid, &_mi_stats_main); |
| 262 |
} |
| 263 |
} |
| 264 |
} |
| 265 |
} |
| 266 |
|
| 267 |
// Initialize the thread local default heap, called from `mi_thread_init` |
| 268 |
static bool _mi_heap_init(void) { |
| 269 |
if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; |
| 270 |
if (_mi_is_main_thread()) { |
| 271 |
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization |
| 272 |
// the main heap is statically allocated |
| 273 |
mi_heap_main_init(); |
| 274 |
_mi_heap_set_default_direct(&_mi_heap_main); |
| 275 |
//mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap()); |
| 276 |
} |
| 277 |
else { |
| 278 |
// use `_mi_os_alloc` to allocate directly from the OS |
| 279 |
mi_thread_data_t* td = mi_thread_data_zalloc(); |
| 280 |
if (td == NULL) return false; |
| 281 |
|
| 282 |
_mi_tld_init(&td->tld, &td->heap); |
| 283 |
_mi_heap_init_ex(&td->heap, &td->tld, _mi_arena_id_none(), false, 0); |
| 284 |
_mi_heap_set_default_direct(&td->heap); |
| 285 |
} |
| 286 |
return false; |
| 287 |
} |
| 288 |
|
| 289 |
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { |
| 290 |
_mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld)); |
| 291 |
tld->segments.stats = &tld->stats; |
| 292 |
tld->segments.os = &tld->os; |
| 293 |
tld->segments.abandoned = &_mi_abandoned_default; |
| 294 |
tld->os.stats = &tld->stats; |
| 295 |
tld->heap_backing = bheap; |
| 296 |
} |
| 297 |
|
| 298 |
// Free the thread local default heap (called from `mi_thread_done`) |
| 299 |
static bool _mi_heap_done(mi_heap_t* heap) { |
| 300 |
if (!mi_heap_is_initialized(heap)) return true; |
| 301 |
|
| 302 |
// reset default heap |
| 303 |
_mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); |
| 304 |
|
| 305 |
// switch to backing heap |
| 306 |
heap = heap->tld->heap_backing; |
| 307 |
if (!mi_heap_is_initialized(heap)) return false; |
| 308 |
|
| 309 |
// delete all non-backing heaps in this thread |
| 310 |
mi_heap_t* curr = heap->tld->heaps; |
| 311 |
while (curr != NULL) { |
| 312 |
mi_heap_t* next = curr->next; // save `next` as `curr` will be freed |
| 313 |
if (curr != heap) { |
| 314 |
mi_assert_internal(!mi_heap_is_backing(curr)); |
| 315 |
mi_heap_delete(curr); |
| 316 |
} |
| 317 |
curr = next; |
| 318 |
} |
| 319 |
mi_assert_internal(heap->tld->heaps == heap && heap->next == NULL); |
| 320 |
mi_assert_internal(mi_heap_is_backing(heap)); |
| 321 |
|
| 322 |
// collect if not the main thread |
| 323 |
if (heap != &_mi_heap_main) { |
| 324 |
_mi_heap_collect_abandon(heap); |
| 325 |
} |
| 326 |
|
| 327 |
// merge stats |
| 328 |
_mi_stats_done(&heap->tld->stats); |
| 329 |
|
| 330 |
// free if not the main thread |
| 331 |
if (heap != &_mi_heap_main) { |
| 332 |
// the following assertion does not always hold for huge segments as those are always treated |
| 333 |
// as abondened: one may allocate it in one thread, but deallocate in another in which case |
| 334 |
// the count can be too large or negative. todo: perhaps not count huge segments? see issue #363 |
| 335 |
// mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); |
| 336 |
mi_thread_data_free((mi_thread_data_t*)heap); |
| 337 |
} |
| 338 |
else { |
| 339 |
#if 0 |
| 340 |
// never free the main thread even in debug mode; if a dll is linked statically with mimalloc, |
| 341 |
// there may still be delete/free calls after the mi_fls_done is called. Issue #207 |
| 342 |
_mi_heap_destroy_pages(heap); |
| 343 |
mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main); |
| 344 |
#endif |
| 345 |
} |
| 346 |
return false; |
| 347 |
} |
| 348 |
|
| 349 |
|
| 350 |
|
| 351 |
// -------------------------------------------------------- |
| 352 |
// Try to run `mi_thread_done()` automatically so any memory |
| 353 |
// owned by the thread but not yet released can be abandoned |
| 354 |
// and re-owned by another thread. |
| 355 |
// |
| 356 |
// 1. windows dynamic library: |
| 357 |
// call from DllMain on DLL_THREAD_DETACH |
| 358 |
// 2. windows static library: |
| 359 |
// use `FlsAlloc` to call a destructor when the thread is done |
| 360 |
// 3. unix, pthreads: |
| 361 |
// use a pthread key to call a destructor when a pthread is done |
| 362 |
// |
| 363 |
// In the last two cases we also need to call `mi_process_init` |
| 364 |
// to set up the thread local keys. |
| 365 |
// -------------------------------------------------------- |
| 366 |
|
| 367 |
// Set up handlers so `mi_thread_done` is called automatically |
| 368 |
static void mi_process_setup_auto_thread_done(void) { |
| 369 |
static bool tls_initialized = false; // fine if it races |
| 370 |
if (tls_initialized) return; |
| 371 |
tls_initialized = true; |
| 372 |
_mi_prim_thread_init_auto_done(); |
| 373 |
_mi_heap_set_default_direct(&_mi_heap_main); |
| 374 |
} |
| 375 |
|
| 376 |
|
| 377 |
bool _mi_is_main_thread(void) { |
| 378 |
return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); |
| 379 |
} |
| 380 |
|
| 381 |
static _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1); |
| 382 |
|
| 383 |
size_t _mi_current_thread_count(void) { |
| 384 |
return mi_atomic_load_relaxed(&thread_count); |
| 385 |
} |
| 386 |
|
| 387 |
// This is called from the `mi_malloc_generic` |
| 388 |
void mi_thread_init(void) mi_attr_noexcept |
| 389 |
{ |
| 390 |
// ensure our process has started already |
| 391 |
mi_process_init(); |
| 392 |
|
| 393 |
// initialize the thread local default heap |
| 394 |
// (this will call `_mi_heap_set_default_direct` and thus set the |
| 395 |
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) |
| 396 |
if (_mi_heap_init()) return; // returns true if already initialized |
| 397 |
|
| 398 |
_mi_stat_increase(&_mi_stats_main.threads, 1); |
| 399 |
mi_atomic_increment_relaxed(&thread_count); |
| 400 |
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); |
| 401 |
} |
| 402 |
|
| 403 |
void mi_thread_done(void) mi_attr_noexcept { |
| 404 |
_mi_thread_done(NULL); |
| 405 |
} |
| 406 |
|
| 407 |
void _mi_thread_done(mi_heap_t* heap) |
| 408 |
{ |
| 409 |
// calling with NULL implies using the default heap |
| 410 |
if (heap == NULL) { |
| 411 |
heap = mi_prim_get_default_heap(); |
| 412 |
if (heap == NULL) return; |
| 413 |
} |
| 414 |
|
| 415 |
// prevent re-entrancy through heap_done/heap_set_default_direct (issue #699) |
| 416 |
if (!mi_heap_is_initialized(heap)) { |
| 417 |
return; |
| 418 |
} |
| 419 |
|
| 420 |
// adjust stats |
| 421 |
mi_atomic_decrement_relaxed(&thread_count); |
| 422 |
_mi_stat_decrease(&_mi_stats_main.threads, 1); |
| 423 |
|
| 424 |
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... |
| 425 |
if (heap->thread_id != _mi_thread_id()) return; |
| 426 |
|
| 427 |
// abandon the thread local heap |
| 428 |
if (_mi_heap_done(heap)) return; // returns true if already ran |
| 429 |
} |
| 430 |
|
| 431 |
void _mi_heap_set_default_direct(mi_heap_t* heap) { |
| 432 |
mi_assert_internal(heap != NULL); |
| 433 |
#if defined(MI_TLS_SLOT) |
| 434 |
mi_prim_tls_slot_set(MI_TLS_SLOT,heap); |
| 435 |
#elif defined(MI_TLS_PTHREAD_SLOT_OFS) |
| 436 |
*mi_tls_pthread_heap_slot() = heap; |
| 437 |
#elif defined(MI_TLS_PTHREAD) |
| 438 |
// we use _mi_heap_default_key |
| 439 |
#else |
| 440 |
_mi_heap_default = heap; |
| 441 |
#endif |
| 442 |
|
| 443 |
// ensure the default heap is passed to `_mi_thread_done` |
| 444 |
// setting to a non-NULL value also ensures `mi_thread_done` is called. |
| 445 |
_mi_prim_thread_associate_default_heap(heap); |
| 446 |
} |
| 447 |
|
| 448 |
|
| 449 |
// -------------------------------------------------------- |
| 450 |
// Run functions on process init/done, and thread init/done |
| 451 |
// -------------------------------------------------------- |
| 452 |
static void mi_cdecl mi_process_done(void); |
| 453 |
|
| 454 |
static bool os_preloading = true; // true until this module is initialized |
| 455 |
static bool mi_redirected = false; // true if malloc redirects to mi_malloc |
| 456 |
|
| 457 |
// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. |
| 458 |
bool mi_decl_noinline _mi_preloading(void) { |
| 459 |
return os_preloading; |
| 460 |
} |
| 461 |
|
| 462 |
mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { |
| 463 |
return mi_redirected; |
| 464 |
} |
| 465 |
|
| 466 |
// Communicate with the redirection module on Windows |
| 467 |
#if defined(_WIN32) && defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) |
| 468 |
#ifdef __cplusplus |
| 469 |
extern "C" { |
| 470 |
#endif |
| 471 |
mi_decl_export void _mi_redirect_entry(DWORD reason) { |
| 472 |
// called on redirection; careful as this may be called before DllMain |
| 473 |
if (reason == DLL_PROCESS_ATTACH) { |
| 474 |
mi_redirected = true; |
| 475 |
} |
| 476 |
else if (reason == DLL_PROCESS_DETACH) { |
| 477 |
mi_redirected = false; |
| 478 |
} |
| 479 |
else if (reason == DLL_THREAD_DETACH) { |
| 480 |
mi_thread_done(); |
| 481 |
} |
| 482 |
} |
| 483 |
__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); |
| 484 |
__declspec(dllimport) void mi_cdecl mi_allocator_done(void); |
| 485 |
#ifdef __cplusplus |
| 486 |
} |
| 487 |
#endif |
| 488 |
#else |
| 489 |
static bool mi_allocator_init(const char** message) { |
| 490 |
if (message != NULL) *message = NULL; |
| 491 |
return true; |
| 492 |
} |
| 493 |
static void mi_allocator_done(void) { |
| 494 |
// nothing to do |
| 495 |
} |
| 496 |
#endif |
| 497 |
|
| 498 |
// Called once by the process loader |
| 499 |
static void mi_process_load(void) { |
| 500 |
mi_heap_main_init(); |
| 501 |
#if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) |
| 502 |
volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; |
| 503 |
if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697) |
| 504 |
#endif |
| 505 |
os_preloading = false; |
| 506 |
mi_assert_internal(_mi_is_main_thread()); |
| 507 |
#if !(defined(_WIN32) && defined(MI_SHARED_LIB)) // use Dll process detach (see below) instead of atexit (issue #521) |
| 508 |
atexit(&mi_process_done); |
| 509 |
#endif |
| 510 |
_mi_options_init(); |
| 511 |
mi_process_setup_auto_thread_done(); |
| 512 |
mi_process_init(); |
| 513 |
if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); |
| 514 |
|
| 515 |
// show message from the redirector (if present) |
| 516 |
const char* msg = NULL; |
| 517 |
mi_allocator_init(&msg); |
| 518 |
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { |
| 519 |
_mi_fputs(NULL,NULL,NULL,msg); |
| 520 |
} |
| 521 |
|
| 522 |
// reseed random |
| 523 |
_mi_random_reinit_if_weak(&_mi_heap_main.random); |
| 524 |
} |
| 525 |
|
| 526 |
#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) |
| 527 |
#include <intrin.h> |
| 528 |
mi_decl_cache_align bool _mi_cpu_has_fsrm = false; |
| 529 |
|
| 530 |
static void mi_detect_cpu_features(void) { |
| 531 |
// FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) |
| 532 |
int32_t cpu_info[4]; |
| 533 |
__cpuid(cpu_info, 7); |
| 534 |
_mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features> |
| 535 |
} |
| 536 |
#else |
| 537 |
static void mi_detect_cpu_features(void) { |
| 538 |
// nothing |
| 539 |
} |
| 540 |
#endif |
| 541 |
|
| 542 |
// Initialize the process; called by thread_init or the process loader |
| 543 |
void mi_process_init(void) mi_attr_noexcept { |
| 544 |
// ensure we are called once |
| 545 |
static mi_atomic_once_t process_init; |
| 546 |
#if _MSC_VER < 1920 |
| 547 |
mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main |
| 548 |
#endif |
| 549 |
if (!mi_atomic_once(&process_init)) return; |
| 550 |
_mi_process_is_initialized = true; |
| 551 |
_mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); |
| 552 |
mi_process_setup_auto_thread_done(); |
| 553 |
|
| 554 |
mi_detect_cpu_features(); |
| 555 |
_mi_os_init(); |
| 556 |
mi_heap_main_init(); |
| 557 |
#if MI_DEBUG |
| 558 |
_mi_verbose_message("debug level : %d\n", MI_DEBUG); |
| 559 |
#endif |
| 560 |
_mi_verbose_message("secure level: %d\n", MI_SECURE); |
| 561 |
_mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); |
| 562 |
#if MI_TSAN |
| 563 |
_mi_verbose_message("thread sanitizer enabled\n"); |
| 564 |
#endif |
| 565 |
mi_thread_init(); |
| 566 |
|
| 567 |
#if defined(_WIN32) |
| 568 |
// On windows, when building as a static lib the FLS cleanup happens to early for the main thread. |
| 569 |
// To avoid this, set the FLS value for the main thread to NULL so the fls cleanup |
| 570 |
// will not call _mi_thread_done on the (still executing) main thread. See issue #508. |
| 571 |
_mi_prim_thread_associate_default_heap(NULL); |
| 572 |
#endif |
| 573 |
|
| 574 |
mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) |
| 575 |
mi_track_init(); |
| 576 |
|
| 577 |
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { |
| 578 |
size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); |
| 579 |
long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at); |
| 580 |
if (reserve_at != -1) { |
| 581 |
mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); |
| 582 |
} else { |
| 583 |
mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); |
| 584 |
} |
| 585 |
} |
| 586 |
if (mi_option_is_enabled(mi_option_reserve_os_memory)) { |
| 587 |
long ksize = mi_option_get(mi_option_reserve_os_memory); |
| 588 |
if (ksize > 0) { |
| 589 |
mi_reserve_os_memory((size_t)ksize*MI_KiB, true /* commit? */, true /* allow large pages? */); |
| 590 |
} |
| 591 |
} |
| 592 |
} |
| 593 |
|
| 594 |
// Called when the process is done (through `at_exit`) |
| 595 |
static void mi_cdecl mi_process_done(void) { |
| 596 |
// only shutdown if we were initialized |
| 597 |
if (!_mi_process_is_initialized) return; |
| 598 |
// ensure we are called once |
| 599 |
static bool process_done = false; |
| 600 |
if (process_done) return; |
| 601 |
process_done = true; |
| 602 |
|
| 603 |
// release any thread specific resources and ensure _mi_thread_done is called on all but the main thread |
| 604 |
_mi_prim_thread_done_auto_done(); |
| 605 |
|
| 606 |
#ifndef MI_SKIP_COLLECT_ON_EXIT |
| 607 |
#if (MI_DEBUG || !defined(MI_SHARED_LIB)) |
| 608 |
// free all memory if possible on process exit. This is not needed for a stand-alone process |
| 609 |
// but should be done if mimalloc is statically linked into another shared library which |
| 610 |
// is repeatedly loaded/unloaded, see issue #281. |
| 611 |
mi_collect(true /* force */ ); |
| 612 |
#endif |
| 613 |
#endif |
| 614 |
|
| 615 |
// Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free |
| 616 |
// since after process_done there might still be other code running that calls `free` (like at_exit routines, |
| 617 |
// or C-runtime termination code. |
| 618 |
if (mi_option_is_enabled(mi_option_destroy_on_exit)) { |
| 619 |
mi_collect(true /* force */); |
| 620 |
_mi_heap_unsafe_destroy_all(); // forcefully release all memory held by all heaps (of this thread only!) |
| 621 |
_mi_arena_unsafe_destroy_all(& _mi_heap_main_get()->tld->stats); |
| 622 |
} |
| 623 |
|
| 624 |
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { |
| 625 |
mi_stats_print(NULL); |
| 626 |
} |
| 627 |
mi_allocator_done(); |
| 628 |
_mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); |
| 629 |
os_preloading = true; // don't call the C runtime anymore |
| 630 |
} |
| 631 |
|
| 632 |
|
| 633 |
|
| 634 |
#if defined(_WIN32) && defined(MI_SHARED_LIB) |
| 635 |
// Windows DLL: easy to hook into process_init and thread_done |
| 636 |
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { |
| 637 |
MI_UNUSED(reserved); |
| 638 |
MI_UNUSED(inst); |
| 639 |
if (reason==DLL_PROCESS_ATTACH) { |
| 640 |
mi_process_load(); |
| 641 |
} |
| 642 |
else if (reason==DLL_PROCESS_DETACH) { |
| 643 |
mi_process_done(); |
| 644 |
} |
| 645 |
else if (reason==DLL_THREAD_DETACH) { |
| 646 |
if (!mi_is_redirected()) { |
| 647 |
mi_thread_done(); |
| 648 |
} |
| 649 |
} |
| 650 |
return TRUE; |
| 651 |
} |
| 652 |
|
| 653 |
#elif defined(_MSC_VER) |
| 654 |
// MSVC: use data section magic for static libraries |
| 655 |
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm> |
| 656 |
static int _mi_process_init(void) { |
| 657 |
mi_process_load(); |
| 658 |
return 0; |
| 659 |
} |
| 660 |
typedef int(*_mi_crt_callback_t)(void); |
| 661 |
#if defined(_M_X64) || defined(_M_ARM64) |
| 662 |
__pragma(comment(linker, "/include:" "_mi_msvc_initu")) |
| 663 |
#pragma section(".CRT$XIU", long, read) |
| 664 |
#else |
| 665 |
__pragma(comment(linker, "/include:" "__mi_msvc_initu")) |
| 666 |
#endif |
| 667 |
#pragma data_seg(".CRT$XIU") |
| 668 |
mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init }; |
| 669 |
#pragma data_seg() |
| 670 |
|
| 671 |
#elif defined(__cplusplus) |
| 672 |
// C++: use static initialization to detect process start |
| 673 |
static bool _mi_process_init(void) { |
| 674 |
mi_process_load(); |
| 675 |
return (_mi_heap_main.thread_id != 0); |
| 676 |
} |
| 677 |
static bool mi_initialized = _mi_process_init(); |
| 678 |
|
| 679 |
#elif defined(__GNUC__) || defined(__clang__) |
| 680 |
// GCC,Clang: use the constructor attribute |
| 681 |
static void __attribute__((constructor)) _mi_process_init(void) { |
| 682 |
mi_process_load(); |
| 683 |
} |
| 684 |
|
| 685 |
#else |
| 686 |
#pragma message("define a way to call mi_process_load on your platform") |
| 687 |
#endif |