Debian Codegraph @

Include/internal/mimalloc/mimalloc.h

raw
1 /* ----------------------------------------------------------------------------
2 Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
3 This is free software; you can redistribute it and/or modify it under the
4 terms of the MIT license. A copy of the license can be found in the file
5 "LICENSE" at the root of this distribution.
6 -----------------------------------------------------------------------------*/
7 #pragma once
8 #ifndef MIMALLOC_H
9 #define MIMALLOC_H
10
11 #define MI_MALLOC_VERSION 212 // major + 2 digits minor
12
13 // ------------------------------------------------------
14 // Compiler specific attributes
15 // ------------------------------------------------------
16
17 #ifdef __cplusplus
18 #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
19 #define mi_attr_noexcept noexcept
20 #else
21 #define mi_attr_noexcept throw()
22 #endif
23 #else
24 #define mi_attr_noexcept
25 #endif
26
27 #if defined(__cplusplus) && (__cplusplus >= 201703)
28 #define mi_decl_nodiscard [[nodiscard]]
29 #elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl
30 #define mi_decl_nodiscard __attribute__((warn_unused_result))
31 #elif defined(_HAS_NODISCARD)
32 #define mi_decl_nodiscard _NODISCARD
33 #elif (_MSC_VER >= 1700)
34 #define mi_decl_nodiscard _Check_return_
35 #else
36 #define mi_decl_nodiscard
37 #endif
38
39 #if defined(_MSC_VER) || defined(__MINGW32__)
40 #if !defined(MI_SHARED_LIB)
41 #define mi_decl_export
42 #elif defined(MI_SHARED_LIB_EXPORT)
43 #define mi_decl_export __declspec(dllexport)
44 #else
45 #define mi_decl_export __declspec(dllimport)
46 #endif
47 #if defined(__MINGW32__)
48 #define mi_decl_restrict
49 #define mi_attr_malloc __attribute__((malloc))
50 #else
51 #if (_MSC_VER >= 1900) && !defined(__EDG__)
52 #define mi_decl_restrict __declspec(allocator) __declspec(restrict)
53 #else
54 #define mi_decl_restrict __declspec(restrict)
55 #endif
56 #define mi_attr_malloc
57 #endif
58 #define mi_cdecl __cdecl
59 #define mi_attr_alloc_size(s)
60 #define mi_attr_alloc_size2(s1,s2)
61 #define mi_attr_alloc_align(p)
62 #elif defined(__GNUC__) // includes clang and icc
63 #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
64 #define mi_decl_export __attribute__((visibility("default")))
65 #else
66 #define mi_decl_export
67 #endif
68 #define mi_cdecl // leads to warnings... __attribute__((cdecl))
69 #define mi_decl_restrict
70 #define mi_attr_malloc __attribute__((malloc))
71 #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
72 #define mi_attr_alloc_size(s)
73 #define mi_attr_alloc_size2(s1,s2)
74 #define mi_attr_alloc_align(p)
75 #elif defined(__INTEL_COMPILER)
76 #define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
77 #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
78 #define mi_attr_alloc_align(p)
79 #else
80 #define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
81 #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
82 #define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))
83 #endif
84 #else
85 #define mi_cdecl
86 #define mi_decl_export
87 #define mi_decl_restrict
88 #define mi_attr_malloc
89 #define mi_attr_alloc_size(s)
90 #define mi_attr_alloc_size2(s1,s2)
91 #define mi_attr_alloc_align(p)
92 #endif
93
94 // ------------------------------------------------------
95 // Includes
96 // ------------------------------------------------------
97
98 #include <stddef.h> // size_t
99 #include <stdbool.h> // bool
100 #include <stdint.h> // INTPTR_MAX
101
102 #ifdef __cplusplus
103 extern "C" {
104 #endif
105
106 // ------------------------------------------------------
107 // Standard malloc interface
108 // ------------------------------------------------------
109
110 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
111 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
112 mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
113 mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
114
115 mi_decl_export void mi_free(void* p) mi_attr_noexcept;
116 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
117 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
118 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc;
119
120 // ------------------------------------------------------
121 // Extended functionality
122 // ------------------------------------------------------
123 #define MI_SMALL_WSIZE_MAX (128)
124 #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))
125
126 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
127 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
128 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
129
130 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
131 mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
132 mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
133
134 mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
135 mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
136
137
138 // ------------------------------------------------------
139 // Internals
140 // ------------------------------------------------------
141
142 typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
143 mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
144
145 typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);
146 mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
147
148 typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
149 mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
150
151 mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
152 mi_decl_export int mi_version(void) mi_attr_noexcept;
153 mi_decl_export void mi_stats_reset(void) mi_attr_noexcept;
154 mi_decl_export void mi_stats_merge(void) mi_attr_noexcept;
155 mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL
156 mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
157
158 mi_decl_export void mi_process_init(void) mi_attr_noexcept;
159 mi_decl_export void mi_thread_init(void) mi_attr_noexcept;
160 mi_decl_export void mi_thread_done(void) mi_attr_noexcept;
161 mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
162
163 mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,
164 size_t* current_rss, size_t* peak_rss,
165 size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;
166
167 // -------------------------------------------------------------------------------------
168 // Aligned allocation
169 // Note that `alignment` always follows `size` for consistency with unaligned
170 // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
171 // -------------------------------------------------------------------------------------
172
173 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
174 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
175 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
176 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
177 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3);
178 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
179 mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
180 mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
181
182
183 // -------------------------------------------------------------------------------------
184 // Heaps: first-class, but can only allocate from the same thread that created it.
185 // -------------------------------------------------------------------------------------
186
187 struct mi_heap_s;
188 typedef struct mi_heap_s mi_heap_t;
189
190 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);
191 mi_decl_export void mi_heap_delete(mi_heap_t* heap);
192 mi_decl_export void mi_heap_destroy(mi_heap_t* heap);
193 mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
194 mi_decl_export mi_heap_t* mi_heap_get_default(void);
195 mi_decl_export mi_heap_t* mi_heap_get_backing(void);
196 mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept;
197
198 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
199 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
200 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
201 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
202 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
203
204 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
205 mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4);
206 mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
207
208 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc;
209 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
210 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc;
211
212 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
213 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
214 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
215 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
216 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
217 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
218 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
219 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
220
221
222 // --------------------------------------------------------------------------------
223 // Zero initialized re-allocation.
224 // Only valid on memory that was originally allocated with zero initialization too.
225 // e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
226 // see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
227 // --------------------------------------------------------------------------------
228
229 mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
230 mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
231
232 mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
233 mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
234 mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);
235 mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
236
237 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
238 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4);
239
240 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
241 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
242 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5);
243 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4);
244
245
246 // ------------------------------------------------------
247 // Analysis
248 // ------------------------------------------------------
249
250 mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
251 mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
252 mi_decl_export bool mi_check_owned(const void* p);
253
254 // An area of heap space contains blocks of a single size.
255 typedef struct mi_heap_area_s {
256 void* blocks; // start of the area containing heap blocks
257 size_t reserved; // bytes reserved for this area (virtual)
258 size_t committed; // current available bytes for this area
259 size_t used; // number of allocated blocks
260 size_t block_size; // size in bytes of each block
261 size_t full_block_size; // size in bytes of a full block including padding and metadata.
262 } mi_heap_area_t;
263
264 typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
265
266 mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
267
268 // Experimental
269 mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
270 mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
271
272 mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
273 mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
274
275 mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
276 mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
277
278 mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;
279
280 // Experimental: heaps associated with specific memory arena's
281 typedef int mi_arena_id_t;
282 mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
283 mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
284 mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
285 mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
286
287 #if MI_MALLOC_VERSION >= 182
288 // Create a heap that only allocates in the specified arena
289 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
290 #endif
291
292 // deprecated
293 mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
294
295
296 // ------------------------------------------------------
297 // Convenience
298 // ------------------------------------------------------
299
300 #define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
301 #define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
302 #define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))
303 #define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))
304 #define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))
305 #define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))
306
307 #define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
308 #define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
309 #define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
310 #define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
311 #define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
312 #define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
313
314
315 // ------------------------------------------------------
316 // Options
317 // ------------------------------------------------------
318
319 typedef enum mi_option_e {
320 // stable options
321 mi_option_show_errors, // print error messages
322 mi_option_show_stats, // print statistics on termination
323 mi_option_verbose, // print verbose messages
324 // the following options are experimental (see src/options.h)
325 mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1)
326 mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2)
327 mi_option_purge_decommits, // should a memory purge decommit (or only reset) (=1)
328 mi_option_allow_large_os_pages, // allow large (2MiB) OS pages, implies eager commit
329 mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB/page) at startup
330 mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
331 mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup
332 mi_option_deprecated_segment_cache,
333 mi_option_deprecated_page_reset,
334 mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination
335 mi_option_deprecated_segment_reset,
336 mi_option_eager_commit_delay,
337 mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all.
338 mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes.
339 mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)
340 mi_option_os_tag, // tag used for OS logging (macOS only for now)
341 mi_option_max_errors, // issue at most N error messages
342 mi_option_max_warnings, // issue at most N warning messages
343 mi_option_max_segment_reclaim,
344 mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe.
345 mi_option_arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit)
346 mi_option_arena_purge_mult,
347 mi_option_purge_extend_delay,
348 _mi_option_last,
349 // legacy option names
350 mi_option_large_os_pages = mi_option_allow_large_os_pages,
351 mi_option_eager_region_commit = mi_option_arena_eager_commit,
352 mi_option_reset_decommits = mi_option_purge_decommits,
353 mi_option_reset_delay = mi_option_purge_delay,
354 mi_option_abandoned_page_reset = mi_option_abandoned_page_purge
355 } mi_option_t;
356
357
358 mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
359 mi_decl_export void mi_option_enable(mi_option_t option);
360 mi_decl_export void mi_option_disable(mi_option_t option);
361 mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
362 mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
363
364 mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
365 mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);
366 mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);
367 mi_decl_export void mi_option_set(mi_option_t option, long value);
368 mi_decl_export void mi_option_set_default(mi_option_t option, long value);
369
370
371 // -------------------------------------------------------------------------------------------------------
372 // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
373 // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
374 // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing.
375 // -------------------------------------------------------------------------------------------------------
376
377 mi_decl_export void mi_cfree(void* p) mi_attr_noexcept;
378 mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
379 mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;
380 mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;
381 mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
382
383 mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;
384 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
385 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
386 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
387 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
388
389 mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
390 mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept;
391 mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
392 mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
393
394 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc;
395 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;
396 mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;
397 mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept;
398
399 mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
400 mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
401 mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;
402
403 // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
404 // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
405 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);
406 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
407 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
408 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
409 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);
410 mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);
411 mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
412
413 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);
414 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
415
416 #ifdef __cplusplus
417 }
418 #endif
419
420 // ---------------------------------------------------------------------------------------------
421 // Implement the C++ std::allocator interface for use in STL containers.
422 // (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)
423 // ---------------------------------------------------------------------------------------------
424 #ifdef __cplusplus
425
426 #include <cstddef> // std::size_t
427 #include <cstdint> // PTRDIFF_MAX
428 #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
429 #include <type_traits> // std::true_type
430 #include <utility> // std::forward
431 #endif
432
433 template<class T> struct _mi_stl_allocator_common {
434 typedef T value_type;
435 typedef std::size_t size_type;
436 typedef std::ptrdiff_t difference_type;
437 typedef value_type& reference;
438 typedef value_type const& const_reference;
439 typedef value_type* pointer;
440 typedef value_type const* const_pointer;
441
442 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
443 using propagate_on_container_copy_assignment = std::true_type;
444 using propagate_on_container_move_assignment = std::true_type;
445 using propagate_on_container_swap = std::true_type;
446 template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
447 template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
448 #else
449 void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
450 void destroy(pointer p) { p->~value_type(); }
451 #endif
452
453 size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
454 pointer address(reference x) const { return &x; }
455 const_pointer address(const_reference x) const { return &x; }
456 };
457
458 template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> {
459 using typename _mi_stl_allocator_common<T>::size_type;
460 using typename _mi_stl_allocator_common<T>::value_type;
461 using typename _mi_stl_allocator_common<T>::pointer;
462 template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
463
464 mi_stl_allocator() mi_attr_noexcept = default;
465 mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default;
466 template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
467 mi_stl_allocator select_on_container_copy_construction() const { return *this; }
468 void deallocate(T* p, size_type) { mi_free(p); }
469
470 #if (__cplusplus >= 201703L) // C++17
471 mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
472 mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
473 #else
474 mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
475 #endif
476
477 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
478 using is_always_equal = std::true_type;
479 #endif
480 };
481
482 template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
483 template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
484
485
486 #if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11
487 #define MI_HAS_HEAP_STL_ALLOCATOR 1
488
489 #include <memory> // std::shared_ptr
490
491 // Common base class for STL allocators in a specific heap
492 template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> {
493 using typename _mi_stl_allocator_common<T>::size_type;
494 using typename _mi_stl_allocator_common<T>::value_type;
495 using typename _mi_stl_allocator_common<T>::pointer;
496
497 _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp) { } /* will not delete nor destroy the passed in heap */
498
499 #if (__cplusplus >= 201703L) // C++17
500 mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }
501 mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }
502 #else
503 mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }
504 #endif
505
506 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
507 using is_always_equal = std::false_type;
508 #endif
509
510 void collect(bool force) { mi_heap_collect(this->heap.get(), force); }
511 template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); }
512
513 protected:
514 std::shared_ptr<mi_heap_t> heap;
515 template<class U, bool D> friend struct _mi_heap_stl_allocator_common;
516
517 _mi_heap_stl_allocator_common() {
518 mi_heap_t* hp = mi_heap_new();
519 this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */
520 }
521 _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }
522 template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { }
523
524 private:
525 static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } }
526 static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }
527 };
528
529 // STL allocator allocation in a specific heap
530 template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> {
531 using typename _mi_heap_stl_allocator_common<T, false>::size_type;
532 mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called
533 mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap
534 template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { }
535
536 mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }
537 void deallocate(T* p, size_type) { mi_free(p); }
538 template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; };
539 };
540
541 template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
542 template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
543
544
545 // STL allocator allocation in a specific heap, where `free` does nothing and
546 // the heap is destroyed in one go on destruction -- use with care!
547 template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> {
548 using typename _mi_heap_stl_allocator_common<T, true>::size_type;
549 mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called
550 mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap
551 template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { }
552
553 mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }
554 void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }
555 template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; };
556 };
557
558 template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); }
559 template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); }
560
561 #endif // C++11
562
563 #endif // __cplusplus
564
565 #endif