| 1 |
/* |
| 2 |
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers |
| 3 |
* Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved. |
| 4 |
* Copyright (c) 1998 by Silicon Graphics. All rights reserved. |
| 5 |
* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. |
| 6 |
* Copyright (c) 2008-2021 Ivan Maidanski |
| 7 |
* |
| 8 |
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED |
| 9 |
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK. |
| 10 |
* |
| 11 |
* Permission is hereby granted to use or copy this program |
| 12 |
* for any purpose, provided the above notices are retained on all copies. |
| 13 |
* Permission to modify the code and to distribute modified code is granted, |
| 14 |
* provided the above notices are retained, and a notice that the code was |
| 15 |
* modified is included with the above copyright notice. |
| 16 |
* |
| 17 |
*/ |
| 18 |
|
| 19 |
#include "private/gc_priv.h" |
| 20 |
|
| 21 |
#include <stdio.h> |
| 22 |
#if !defined(MACOS) && !defined(MSWINCE) |
| 23 |
# include <signal.h> |
| 24 |
# if !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2) \ |
| 25 |
&& !defined(__CC_ARM) |
| 26 |
# include <sys/types.h> |
| 27 |
# endif |
| 28 |
#endif |
| 29 |
|
| 30 |
/* |
| 31 |
* Separate free lists are maintained for different sized objects |
| 32 |
* up to MAXOBJBYTES. |
| 33 |
* The call GC_allocobj(i,k) ensures that the freelist for |
| 34 |
* kind k objects of size i points to a non-empty |
| 35 |
* free list. It returns a pointer to the first entry on the free list. |
| 36 |
* In a single-threaded world, GC_allocobj may be called to allocate |
| 37 |
* an object of small size lb (and NORMAL kind) as follows |
| 38 |
* (GC_generic_malloc_inner is a wrapper over GC_allocobj which also |
| 39 |
* fills in GC_size_map if needed): |
| 40 |
* |
| 41 |
* lg = GC_size_map[lb]; |
| 42 |
* op = GC_objfreelist[lg]; |
| 43 |
* if (NULL == op) { |
| 44 |
* op = GC_generic_malloc_inner(lb, NORMAL); |
| 45 |
* } else { |
| 46 |
* GC_objfreelist[lg] = obj_link(op); |
| 47 |
* GC_bytes_allocd += GRANULES_TO_BYTES((word)lg); |
| 48 |
* } |
| 49 |
* |
| 50 |
* Note that this is very fast if the free list is non-empty; it should |
| 51 |
* only involve the execution of 4 or 5 simple instructions. |
| 52 |
* All composite objects on freelists are cleared, except for |
| 53 |
* their first word. |
| 54 |
*/ |
| 55 |
|
| 56 |
/* |
| 57 |
* The allocator uses GC_allochblk to allocate large chunks of objects. |
| 58 |
* These chunks all start on addresses which are multiples of |
| 59 |
* HBLKSZ. Each allocated chunk has an associated header, |
| 60 |
* which can be located quickly based on the address of the chunk. |
| 61 |
* (See headers.c for details.) |
| 62 |
* This makes it possible to check quickly whether an |
| 63 |
* arbitrary address corresponds to an object administered by the |
| 64 |
* allocator. |
| 65 |
*/ |
| 66 |
|
| 67 |
word GC_non_gc_bytes = 0; /* Number of bytes not intended to be collected */ |
| 68 |
|
| 69 |
word GC_gc_no = 0; |
| 70 |
|
| 71 |
#ifndef NO_CLOCK |
| 72 |
static unsigned long full_gc_total_time = 0; /* in ms, may wrap */ |
| 73 |
static unsigned full_gc_total_ns_frac = 0; /* fraction of 1 ms */ |
| 74 |
static GC_bool measure_performance = FALSE; |
| 75 |
/* Do performance measurements if set to true (e.g., */ |
| 76 |
/* accumulation of the total time of full collections). */ |
| 77 |
|
| 78 |
GC_API void GC_CALL GC_start_performance_measurement(void) |
| 79 |
{ |
| 80 |
measure_performance = TRUE; |
| 81 |
} |
| 82 |
|
| 83 |
GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void) |
| 84 |
{ |
| 85 |
return full_gc_total_time; |
| 86 |
} |
| 87 |
#endif /* !NO_CLOCK */ |
| 88 |
|
| 89 |
#ifndef GC_DISABLE_INCREMENTAL |
| 90 |
GC_INNER GC_bool GC_incremental = FALSE; /* By default, stop the world. */ |
| 91 |
STATIC GC_bool GC_should_start_incremental_collection = FALSE; |
| 92 |
#endif |
| 93 |
|
| 94 |
GC_API int GC_CALL GC_is_incremental_mode(void) |
| 95 |
{ |
| 96 |
return (int)GC_incremental; |
| 97 |
} |
| 98 |
|
| 99 |
#ifdef THREADS |
| 100 |
int GC_parallel = FALSE; /* By default, parallel GC is off. */ |
| 101 |
#endif |
| 102 |
|
| 103 |
#if defined(GC_FULL_FREQ) && !defined(CPPCHECK) |
| 104 |
int GC_full_freq = GC_FULL_FREQ; |
| 105 |
#else |
| 106 |
int GC_full_freq = 19; /* Every 20th collection is a full */ |
| 107 |
/* collection, whether we need it */ |
| 108 |
/* or not. */ |
| 109 |
#endif |
| 110 |
|
| 111 |
STATIC GC_bool GC_need_full_gc = FALSE; |
| 112 |
/* Need full GC due to heap growth. */ |
| 113 |
|
| 114 |
#ifdef THREAD_LOCAL_ALLOC |
| 115 |
GC_INNER GC_bool GC_world_stopped = FALSE; |
| 116 |
#endif |
| 117 |
|
| 118 |
STATIC GC_bool GC_disable_automatic_collection = FALSE; |
| 119 |
|
| 120 |
GC_API void GC_CALL GC_set_disable_automatic_collection(int value) |
| 121 |
{ |
| 122 |
DCL_LOCK_STATE; |
| 123 |
|
| 124 |
LOCK(); |
| 125 |
GC_disable_automatic_collection = (GC_bool)value; |
| 126 |
UNLOCK(); |
| 127 |
} |
| 128 |
|
| 129 |
GC_API int GC_CALL GC_get_disable_automatic_collection(void) |
| 130 |
{ |
| 131 |
int value; |
| 132 |
DCL_LOCK_STATE; |
| 133 |
|
| 134 |
LOCK(); |
| 135 |
value = (int)GC_disable_automatic_collection; |
| 136 |
UNLOCK(); |
| 137 |
return value; |
| 138 |
} |
| 139 |
|
| 140 |
STATIC word GC_used_heap_size_after_full = 0; |
| 141 |
|
| 142 |
/* GC_copyright symbol is externally visible. */ |
| 143 |
EXTERN_C_BEGIN |
| 144 |
extern const char * const GC_copyright[]; |
| 145 |
EXTERN_C_END |
| 146 |
const char * const GC_copyright[] = |
| 147 |
{"Copyright 1988, 1989 Hans-J. Boehm and Alan J. Demers ", |
| 148 |
"Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. ", |
| 149 |
"Copyright (c) 1996-1998 by Silicon Graphics. All rights reserved. ", |
| 150 |
"Copyright (c) 1999-2009 by Hewlett-Packard Company. All rights reserved. ", |
| 151 |
"Copyright (c) 2008-2021 Ivan Maidanski ", |
| 152 |
"THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY", |
| 153 |
" EXPRESSED OR IMPLIED. ANY USE IS AT YOUR OWN RISK.", |
| 154 |
"See source code for details." }; |
| 155 |
|
| 156 |
/* Version macros are now defined in gc_version.h, which is included by */ |
| 157 |
/* gc.h, which is included by gc_priv.h. */ |
| 158 |
#ifndef GC_NO_VERSION_VAR |
| 159 |
EXTERN_C_BEGIN |
| 160 |
extern const unsigned GC_version; |
| 161 |
EXTERN_C_END |
| 162 |
const unsigned GC_version = ((GC_VERSION_MAJOR << 16) | |
| 163 |
(GC_VERSION_MINOR << 8) | GC_VERSION_MICRO); |
| 164 |
#endif |
| 165 |
|
| 166 |
GC_API unsigned GC_CALL GC_get_version(void) |
| 167 |
{ |
| 168 |
return (GC_VERSION_MAJOR << 16) | (GC_VERSION_MINOR << 8) | |
| 169 |
GC_VERSION_MICRO; |
| 170 |
} |
| 171 |
|
| 172 |
/* some more variables */ |
| 173 |
|
| 174 |
#ifdef GC_DONT_EXPAND |
| 175 |
int GC_dont_expand = TRUE; |
| 176 |
#else |
| 177 |
int GC_dont_expand = FALSE; |
| 178 |
#endif |
| 179 |
|
| 180 |
#if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK) |
| 181 |
word GC_free_space_divisor = GC_FREE_SPACE_DIVISOR; /* must be > 0 */ |
| 182 |
#else |
| 183 |
word GC_free_space_divisor = 3; |
| 184 |
#endif |
| 185 |
|
| 186 |
GC_INNER int GC_CALLBACK GC_never_stop_func(void) |
| 187 |
{ |
| 188 |
return(0); |
| 189 |
} |
| 190 |
|
| 191 |
#if defined(GC_TIME_LIMIT) && !defined(CPPCHECK) |
| 192 |
unsigned long GC_time_limit = GC_TIME_LIMIT; |
| 193 |
/* We try to keep pause times from exceeding */ |
| 194 |
/* this by much. In milliseconds. */ |
| 195 |
#elif defined(PARALLEL_MARK) |
| 196 |
unsigned long GC_time_limit = GC_TIME_UNLIMITED; |
| 197 |
/* The parallel marker cannot be interrupted for */ |
| 198 |
/* now, so the time limit is absent by default. */ |
| 199 |
#else |
| 200 |
unsigned long GC_time_limit = 50; |
| 201 |
#endif |
| 202 |
|
| 203 |
#ifndef NO_CLOCK |
| 204 |
STATIC unsigned long GC_time_lim_nsec = 0; |
| 205 |
/* The nanoseconds add-on to GC_time_limit */ |
| 206 |
/* value. Not updated by GC_set_time_limit(). */ |
| 207 |
/* Ignored if the value of GC_time_limit is */ |
| 208 |
/* GC_TIME_UNLIMITED. */ |
| 209 |
|
| 210 |
# define TV_NSEC_LIMIT (1000UL * 1000) /* amount of nanoseconds in 1 ms */ |
| 211 |
|
| 212 |
GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s tv) |
| 213 |
{ |
| 214 |
GC_ASSERT(tv.tv_ms <= GC_TIME_UNLIMITED); |
| 215 |
GC_ASSERT(tv.tv_nsec < TV_NSEC_LIMIT); |
| 216 |
GC_time_limit = tv.tv_ms; |
| 217 |
GC_time_lim_nsec = tv.tv_nsec; |
| 218 |
} |
| 219 |
|
| 220 |
GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void) |
| 221 |
{ |
| 222 |
struct GC_timeval_s tv; |
| 223 |
|
| 224 |
tv.tv_ms = GC_time_limit; |
| 225 |
tv.tv_nsec = GC_time_lim_nsec; |
| 226 |
return tv; |
| 227 |
} |
| 228 |
|
| 229 |
STATIC CLOCK_TYPE GC_start_time = CLOCK_TYPE_INITIALIZER; |
| 230 |
/* Time at which we stopped world. */ |
| 231 |
/* used only in GC_timeout_stop_func. */ |
| 232 |
#endif /* !NO_CLOCK */ |
| 233 |
|
| 234 |
STATIC int GC_n_attempts = 0; /* Number of attempts at finishing */ |
| 235 |
/* collection within GC_time_limit. */ |
| 236 |
|
| 237 |
STATIC GC_stop_func GC_default_stop_func = GC_never_stop_func; |
| 238 |
/* accessed holding the lock. */ |
| 239 |
|
| 240 |
GC_API void GC_CALL GC_set_stop_func(GC_stop_func stop_func) |
| 241 |
{ |
| 242 |
DCL_LOCK_STATE; |
| 243 |
GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func)); |
| 244 |
LOCK(); |
| 245 |
GC_default_stop_func = stop_func; |
| 246 |
UNLOCK(); |
| 247 |
} |
| 248 |
|
| 249 |
GC_API GC_stop_func GC_CALL GC_get_stop_func(void) |
| 250 |
{ |
| 251 |
GC_stop_func stop_func; |
| 252 |
DCL_LOCK_STATE; |
| 253 |
LOCK(); |
| 254 |
stop_func = GC_default_stop_func; |
| 255 |
UNLOCK(); |
| 256 |
return stop_func; |
| 257 |
} |
| 258 |
|
| 259 |
#if defined(GC_DISABLE_INCREMENTAL) || defined(NO_CLOCK) |
| 260 |
# define GC_timeout_stop_func GC_default_stop_func |
| 261 |
#else |
| 262 |
STATIC int GC_CALLBACK GC_timeout_stop_func (void) |
| 263 |
{ |
| 264 |
CLOCK_TYPE current_time; |
| 265 |
static unsigned count = 0; |
| 266 |
unsigned long time_diff, nsec_diff; |
| 267 |
|
| 268 |
if ((*GC_default_stop_func)()) |
| 269 |
return(1); |
| 270 |
|
| 271 |
if ((count++ & 3) != 0) return(0); |
| 272 |
GET_TIME(current_time); |
| 273 |
time_diff = MS_TIME_DIFF(current_time,GC_start_time); |
| 274 |
nsec_diff = NS_FRAC_TIME_DIFF(current_time, GC_start_time); |
| 275 |
# if defined(CPPCHECK) |
| 276 |
GC_noop1((word)&nsec_diff); |
| 277 |
# endif |
| 278 |
if (time_diff >= GC_time_limit |
| 279 |
&& (time_diff > GC_time_limit || nsec_diff >= GC_time_lim_nsec)) { |
| 280 |
GC_COND_LOG_PRINTF("Abandoning stopped marking after %lu ms %lu ns" |
| 281 |
" (attempt %d)\n", |
| 282 |
time_diff, nsec_diff, GC_n_attempts); |
| 283 |
return 1; |
| 284 |
} |
| 285 |
return(0); |
| 286 |
} |
| 287 |
#endif /* !GC_DISABLE_INCREMENTAL */ |
| 288 |
|
| 289 |
#ifdef THREADS |
| 290 |
GC_INNER word GC_total_stacksize = 0; /* updated on every push_all_stacks */ |
| 291 |
#endif |
| 292 |
|
| 293 |
static size_t min_bytes_allocd_minimum = 1; |
| 294 |
/* The lowest value returned by min_bytes_allocd(). */ |
| 295 |
|
| 296 |
GC_API void GC_CALL GC_set_min_bytes_allocd(size_t value) |
| 297 |
{ |
| 298 |
GC_ASSERT(value > 0); |
| 299 |
min_bytes_allocd_minimum = value; |
| 300 |
} |
| 301 |
|
| 302 |
GC_API size_t GC_CALL GC_get_min_bytes_allocd(void) |
| 303 |
{ |
| 304 |
return min_bytes_allocd_minimum; |
| 305 |
} |
| 306 |
|
| 307 |
/* Return the minimum number of bytes that must be allocated between */ |
| 308 |
/* collections to amortize the collection cost. Should be non-zero. */ |
| 309 |
static word min_bytes_allocd(void) |
| 310 |
{ |
| 311 |
word result; |
| 312 |
word stack_size; |
| 313 |
word total_root_size; /* includes double stack size, */ |
| 314 |
/* since the stack is expensive */ |
| 315 |
/* to scan. */ |
| 316 |
word scan_size; /* Estimate of memory to be scanned */ |
| 317 |
/* during normal GC. */ |
| 318 |
|
| 319 |
# ifdef THREADS |
| 320 |
if (GC_need_to_lock) { |
| 321 |
/* We are multi-threaded... */ |
| 322 |
stack_size = GC_total_stacksize; |
| 323 |
/* For now, we just use the value computed during the latest GC. */ |
| 324 |
# ifdef DEBUG_THREADS |
| 325 |
GC_log_printf("Total stacks size: %lu\n", |
| 326 |
(unsigned long)stack_size); |
| 327 |
# endif |
| 328 |
} else |
| 329 |
# endif |
| 330 |
/* else*/ { |
| 331 |
# ifdef STACK_NOT_SCANNED |
| 332 |
stack_size = 0; |
| 333 |
# elif defined(STACK_GROWS_UP) |
| 334 |
stack_size = GC_approx_sp() - GC_stackbottom; |
| 335 |
# else |
| 336 |
stack_size = GC_stackbottom - GC_approx_sp(); |
| 337 |
# endif |
| 338 |
} |
| 339 |
|
| 340 |
total_root_size = 2 * stack_size + GC_root_size; |
| 341 |
scan_size = 2 * GC_composite_in_use + GC_atomic_in_use / 4 |
| 342 |
+ total_root_size; |
| 343 |
result = scan_size / GC_free_space_divisor; |
| 344 |
if (GC_incremental) { |
| 345 |
result /= 2; |
| 346 |
} |
| 347 |
return result > min_bytes_allocd_minimum |
| 348 |
? result : min_bytes_allocd_minimum; |
| 349 |
} |
| 350 |
|
| 351 |
STATIC word GC_non_gc_bytes_at_gc = 0; |
| 352 |
/* Number of explicitly managed bytes of storage */ |
| 353 |
/* at last collection. */ |
| 354 |
|
| 355 |
/* Return the number of bytes allocated, adjusted for explicit storage */ |
| 356 |
/* management, etc.. This number is used in deciding when to trigger */ |
| 357 |
/* collections. */ |
| 358 |
STATIC word GC_adj_bytes_allocd(void) |
| 359 |
{ |
| 360 |
signed_word result; |
| 361 |
signed_word expl_managed = (signed_word)GC_non_gc_bytes |
| 362 |
- (signed_word)GC_non_gc_bytes_at_gc; |
| 363 |
|
| 364 |
/* Don't count what was explicitly freed, or newly allocated for */ |
| 365 |
/* explicit management. Note that deallocating an explicitly */ |
| 366 |
/* managed object should not alter result, assuming the client */ |
| 367 |
/* is playing by the rules. */ |
| 368 |
result = (signed_word)GC_bytes_allocd |
| 369 |
+ (signed_word)GC_bytes_dropped |
| 370 |
- (signed_word)GC_bytes_freed |
| 371 |
+ (signed_word)GC_finalizer_bytes_freed |
| 372 |
- expl_managed; |
| 373 |
if (result > (signed_word)GC_bytes_allocd) { |
| 374 |
result = GC_bytes_allocd; |
| 375 |
/* probably client bug or unfortunate scheduling */ |
| 376 |
} |
| 377 |
result += GC_bytes_finalized; |
| 378 |
/* We count objects enqueued for finalization as though they */ |
| 379 |
/* had been reallocated this round. Finalization is user */ |
| 380 |
/* visible progress. And if we don't count this, we have */ |
| 381 |
/* stability problems for programs that finalize all objects. */ |
| 382 |
if (result < (signed_word)(GC_bytes_allocd >> 3)) { |
| 383 |
/* Always count at least 1/8 of the allocations. We don't want */ |
| 384 |
/* to collect too infrequently, since that would inhibit */ |
| 385 |
/* coalescing of free storage blocks. */ |
| 386 |
/* This also makes us partially robust against client bugs. */ |
| 387 |
return(GC_bytes_allocd >> 3); |
| 388 |
} else { |
| 389 |
return(result); |
| 390 |
} |
| 391 |
} |
| 392 |
|
| 393 |
|
| 394 |
/* Clear up a few frames worth of garbage left at the top of the stack. */ |
| 395 |
/* This is used to prevent us from accidentally treating garbage left */ |
| 396 |
/* on the stack by other parts of the collector as roots. This */ |
| 397 |
/* differs from the code in misc.c, which actually tries to keep the */ |
| 398 |
/* stack clear of long-lived, client-generated garbage. */ |
| 399 |
STATIC void GC_clear_a_few_frames(void) |
| 400 |
{ |
| 401 |
# ifndef CLEAR_NWORDS |
| 402 |
# define CLEAR_NWORDS 64 |
| 403 |
# endif |
| 404 |
volatile word frames[CLEAR_NWORDS]; |
| 405 |
BZERO((word *)frames, CLEAR_NWORDS * sizeof(word)); |
| 406 |
} |
| 407 |
|
| 408 |
/* Heap size at which we need a collection to avoid expanding past */ |
| 409 |
/* limits used by blacklisting. */ |
| 410 |
STATIC word GC_collect_at_heapsize = GC_WORD_MAX; |
| 411 |
|
| 412 |
GC_API void GC_CALL GC_start_incremental_collection(void) |
| 413 |
{ |
| 414 |
# ifndef GC_DISABLE_INCREMENTAL |
| 415 |
DCL_LOCK_STATE; |
| 416 |
|
| 417 |
if (!GC_incremental) return; |
| 418 |
LOCK(); |
| 419 |
GC_should_start_incremental_collection = TRUE; |
| 420 |
if (!GC_dont_gc) { |
| 421 |
ENTER_GC(); |
| 422 |
GC_collect_a_little_inner(1); |
| 423 |
EXIT_GC(); |
| 424 |
} |
| 425 |
UNLOCK(); |
| 426 |
# endif |
| 427 |
} |
| 428 |
|
| 429 |
/* Have we allocated enough to amortize a collection? */ |
| 430 |
GC_INNER GC_bool GC_should_collect(void) |
| 431 |
{ |
| 432 |
static word last_min_bytes_allocd; |
| 433 |
static word last_gc_no; |
| 434 |
|
| 435 |
GC_ASSERT(I_HOLD_LOCK()); |
| 436 |
if (last_gc_no != GC_gc_no) { |
| 437 |
last_min_bytes_allocd = min_bytes_allocd(); |
| 438 |
last_gc_no = GC_gc_no; |
| 439 |
} |
| 440 |
# ifndef GC_DISABLE_INCREMENTAL |
| 441 |
if (GC_should_start_incremental_collection) { |
| 442 |
GC_should_start_incremental_collection = FALSE; |
| 443 |
return TRUE; |
| 444 |
} |
| 445 |
# endif |
| 446 |
if (GC_disable_automatic_collection) return FALSE; |
| 447 |
|
| 448 |
return(GC_adj_bytes_allocd() >= last_min_bytes_allocd |
| 449 |
|| GC_heapsize >= GC_collect_at_heapsize); |
| 450 |
} |
| 451 |
|
| 452 |
/* STATIC */ GC_start_callback_proc GC_start_call_back = 0; |
| 453 |
/* Called at start of full collections. */ |
| 454 |
/* Not called if 0. Called with the allocation */ |
| 455 |
/* lock held. Not used by GC itself. */ |
| 456 |
|
| 457 |
GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc fn) |
| 458 |
{ |
| 459 |
DCL_LOCK_STATE; |
| 460 |
LOCK(); |
| 461 |
GC_start_call_back = fn; |
| 462 |
UNLOCK(); |
| 463 |
} |
| 464 |
|
| 465 |
GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void) |
| 466 |
{ |
| 467 |
GC_start_callback_proc fn; |
| 468 |
DCL_LOCK_STATE; |
| 469 |
LOCK(); |
| 470 |
fn = GC_start_call_back; |
| 471 |
UNLOCK(); |
| 472 |
return fn; |
| 473 |
} |
| 474 |
|
| 475 |
GC_INLINE void GC_notify_full_gc(void) |
| 476 |
{ |
| 477 |
if (GC_start_call_back != 0) { |
| 478 |
(*GC_start_call_back)(); |
| 479 |
} |
| 480 |
} |
| 481 |
|
| 482 |
STATIC GC_bool GC_is_full_gc = FALSE; |
| 483 |
|
| 484 |
STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func); |
| 485 |
STATIC void GC_finish_collection(void); |
| 486 |
|
| 487 |
/* |
| 488 |
* Initiate a garbage collection if appropriate. |
| 489 |
* Choose judiciously |
| 490 |
* between partial, full, and stop-world collections. |
| 491 |
*/ |
| 492 |
STATIC void GC_maybe_gc(void) |
| 493 |
{ |
| 494 |
GC_ASSERT(I_HOLD_LOCK()); |
| 495 |
ASSERT_CANCEL_DISABLED(); |
| 496 |
if (GC_should_collect()) { |
| 497 |
static int n_partial_gcs = 0; |
| 498 |
|
| 499 |
if (!GC_incremental) { |
| 500 |
/* TODO: If possible, GC_default_stop_func should be used here */ |
| 501 |
GC_try_to_collect_inner(GC_never_stop_func); |
| 502 |
n_partial_gcs = 0; |
| 503 |
return; |
| 504 |
} else { |
| 505 |
# ifdef PARALLEL_MARK |
| 506 |
if (GC_parallel) |
| 507 |
GC_wait_for_reclaim(); |
| 508 |
# endif |
| 509 |
if (GC_need_full_gc || n_partial_gcs >= GC_full_freq) { |
| 510 |
GC_COND_LOG_PRINTF( |
| 511 |
"***>Full mark for collection #%lu after %lu allocd bytes\n", |
| 512 |
(unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd); |
| 513 |
GC_promote_black_lists(); |
| 514 |
(void)GC_reclaim_all((GC_stop_func)0, TRUE); |
| 515 |
GC_notify_full_gc(); |
| 516 |
GC_clear_marks(); |
| 517 |
n_partial_gcs = 0; |
| 518 |
GC_is_full_gc = TRUE; |
| 519 |
} else { |
| 520 |
n_partial_gcs++; |
| 521 |
} |
| 522 |
} |
| 523 |
/* We try to mark with the world stopped. */ |
| 524 |
/* If we run out of time, this turns into */ |
| 525 |
/* incremental marking. */ |
| 526 |
# ifndef NO_CLOCK |
| 527 |
if (GC_time_limit != GC_TIME_UNLIMITED) { GET_TIME(GC_start_time); } |
| 528 |
# endif |
| 529 |
/* TODO: If possible, GC_default_stop_func should be */ |
| 530 |
/* used instead of GC_never_stop_func here. */ |
| 531 |
if (GC_stopped_mark(GC_time_limit == GC_TIME_UNLIMITED? |
| 532 |
GC_never_stop_func : GC_timeout_stop_func)) { |
| 533 |
SAVE_CALLERS_TO_LAST_STACK(); |
| 534 |
GC_finish_collection(); |
| 535 |
} else { |
| 536 |
if (!GC_is_full_gc) { |
| 537 |
/* Count this as the first attempt */ |
| 538 |
GC_n_attempts++; |
| 539 |
} |
| 540 |
} |
| 541 |
} |
| 542 |
} |
| 543 |
|
| 544 |
STATIC GC_on_collection_event_proc GC_on_collection_event = 0; |
| 545 |
|
| 546 |
GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc fn) |
| 547 |
{ |
| 548 |
/* fn may be 0 (means no event notifier). */ |
| 549 |
DCL_LOCK_STATE; |
| 550 |
LOCK(); |
| 551 |
GC_on_collection_event = fn; |
| 552 |
UNLOCK(); |
| 553 |
} |
| 554 |
|
| 555 |
GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void) |
| 556 |
{ |
| 557 |
GC_on_collection_event_proc fn; |
| 558 |
DCL_LOCK_STATE; |
| 559 |
LOCK(); |
| 560 |
fn = GC_on_collection_event; |
| 561 |
UNLOCK(); |
| 562 |
return fn; |
| 563 |
} |
| 564 |
|
| 565 |
/* Stop the world garbage collection. If stop_func is not */ |
| 566 |
/* GC_never_stop_func then abort if stop_func returns TRUE. */ |
| 567 |
/* Return TRUE if we successfully completed the collection. */ |
| 568 |
GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func) |
| 569 |
{ |
| 570 |
# ifndef NO_CLOCK |
| 571 |
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; |
| 572 |
GC_bool start_time_valid; |
| 573 |
# endif |
| 574 |
|
| 575 |
ASSERT_CANCEL_DISABLED(); |
| 576 |
GC_ASSERT(I_HOLD_LOCK()); |
| 577 |
if (GC_dont_gc || (*stop_func)()) return FALSE; |
| 578 |
if (GC_on_collection_event) |
| 579 |
GC_on_collection_event(GC_EVENT_START); |
| 580 |
if (GC_incremental && GC_collection_in_progress()) { |
| 581 |
GC_COND_LOG_PRINTF( |
| 582 |
"GC_try_to_collect_inner: finishing collection in progress\n"); |
| 583 |
/* Just finish collection already in progress. */ |
| 584 |
while(GC_collection_in_progress()) { |
| 585 |
if ((*stop_func)()) { |
| 586 |
/* TODO: Notify GC_EVENT_ABANDON */ |
| 587 |
return(FALSE); |
| 588 |
} |
| 589 |
ENTER_GC(); |
| 590 |
GC_collect_a_little_inner(1); |
| 591 |
EXIT_GC(); |
| 592 |
} |
| 593 |
} |
| 594 |
GC_notify_full_gc(); |
| 595 |
# ifndef NO_CLOCK |
| 596 |
start_time_valid = FALSE; |
| 597 |
if ((GC_print_stats | (int)measure_performance) != 0) { |
| 598 |
if (GC_print_stats) |
| 599 |
GC_log_printf("Initiating full world-stop collection!\n"); |
| 600 |
start_time_valid = TRUE; |
| 601 |
GET_TIME(start_time); |
| 602 |
} |
| 603 |
# endif |
| 604 |
GC_promote_black_lists(); |
| 605 |
/* Make sure all blocks have been reclaimed, so sweep routines */ |
| 606 |
/* don't see cleared mark bits. */ |
| 607 |
/* If we're guaranteed to finish, then this is unnecessary. */ |
| 608 |
/* In the find_leak case, we have to finish to guarantee that */ |
| 609 |
/* previously unmarked objects are not reported as leaks. */ |
| 610 |
# ifdef PARALLEL_MARK |
| 611 |
if (GC_parallel) |
| 612 |
GC_wait_for_reclaim(); |
| 613 |
# endif |
| 614 |
if ((GC_find_leak || stop_func != GC_never_stop_func) |
| 615 |
&& !GC_reclaim_all(stop_func, FALSE)) { |
| 616 |
/* Aborted. So far everything is still consistent. */ |
| 617 |
/* TODO: Notify GC_EVENT_ABANDON */ |
| 618 |
return(FALSE); |
| 619 |
} |
| 620 |
GC_invalidate_mark_state(); /* Flush mark stack. */ |
| 621 |
GC_clear_marks(); |
| 622 |
SAVE_CALLERS_TO_LAST_STACK(); |
| 623 |
GC_is_full_gc = TRUE; |
| 624 |
if (!GC_stopped_mark(stop_func)) { |
| 625 |
if (!GC_incremental) { |
| 626 |
/* We're partially done and have no way to complete or use */ |
| 627 |
/* current work. Reestablish invariants as cheaply as */ |
| 628 |
/* possible. */ |
| 629 |
GC_invalidate_mark_state(); |
| 630 |
GC_unpromote_black_lists(); |
| 631 |
} /* else we claim the world is already still consistent. We'll */ |
| 632 |
/* finish incrementally. */ |
| 633 |
/* TODO: Notify GC_EVENT_ABANDON */ |
| 634 |
return(FALSE); |
| 635 |
} |
| 636 |
GC_finish_collection(); |
| 637 |
# ifndef NO_CLOCK |
| 638 |
if (start_time_valid) { |
| 639 |
CLOCK_TYPE current_time; |
| 640 |
unsigned long time_diff, ns_frac_diff; |
| 641 |
|
| 642 |
GET_TIME(current_time); |
| 643 |
time_diff = MS_TIME_DIFF(current_time, start_time); |
| 644 |
ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time); |
| 645 |
if (measure_performance) { |
| 646 |
full_gc_total_time += time_diff; /* may wrap */ |
| 647 |
full_gc_total_ns_frac += (unsigned)ns_frac_diff; |
| 648 |
if (full_gc_total_ns_frac >= 1000000U) { |
| 649 |
/* Overflow of the nanoseconds part. */ |
| 650 |
full_gc_total_ns_frac -= 1000000U; |
| 651 |
full_gc_total_time++; |
| 652 |
} |
| 653 |
} |
| 654 |
if (GC_print_stats) |
| 655 |
GC_log_printf("Complete collection took %lu ms %lu ns\n", |
| 656 |
time_diff, ns_frac_diff); |
| 657 |
} |
| 658 |
# endif |
| 659 |
if (GC_on_collection_event) |
| 660 |
GC_on_collection_event(GC_EVENT_END); |
| 661 |
return(TRUE); |
| 662 |
} |
| 663 |
|
| 664 |
/* The number of extra calls to GC_mark_some that we have made. */ |
| 665 |
STATIC int GC_deficit = 0; |
| 666 |
|
| 667 |
/* The default value of GC_rate. */ |
| 668 |
#ifndef GC_RATE |
| 669 |
# define GC_RATE 10 |
| 670 |
#endif |
| 671 |
|
| 672 |
/* When GC_collect_a_little_inner() performs n units of GC work, a unit */ |
| 673 |
/* is intended to touch roughly GC_rate pages. (But, every once in */ |
| 674 |
/* a while, we do more than that.) This needs to be a fairly large */ |
| 675 |
/* number with our current incremental GC strategy, since otherwise we */ |
| 676 |
/* allocate too much during GC, and the cleanup gets expensive. */ |
| 677 |
STATIC int GC_rate = GC_RATE; |
| 678 |
|
| 679 |
GC_API void GC_CALL GC_set_rate(int value) |
| 680 |
{ |
| 681 |
GC_ASSERT(value > 0); |
| 682 |
GC_rate = value; |
| 683 |
} |
| 684 |
|
| 685 |
GC_API int GC_CALL GC_get_rate(void) |
| 686 |
{ |
| 687 |
return GC_rate; |
| 688 |
} |
| 689 |
|
| 690 |
/* The default maximum number of prior attempts at world stop marking. */ |
| 691 |
#ifndef MAX_PRIOR_ATTEMPTS |
| 692 |
# define MAX_PRIOR_ATTEMPTS 1 |
| 693 |
#endif |
| 694 |
|
| 695 |
/* The maximum number of prior attempts at world stop marking. */ |
| 696 |
/* A value of 1 means that we finish the second time, no matter how */ |
| 697 |
/* long it takes. Does not count the initial root scan for a full GC. */ |
| 698 |
static int max_prior_attempts = MAX_PRIOR_ATTEMPTS; |
| 699 |
|
| 700 |
GC_API void GC_CALL GC_set_max_prior_attempts(int value) |
| 701 |
{ |
| 702 |
GC_ASSERT(value >= 0); |
| 703 |
max_prior_attempts = value; |
| 704 |
} |
| 705 |
|
| 706 |
GC_API int GC_CALL GC_get_max_prior_attempts(void) |
| 707 |
{ |
| 708 |
return max_prior_attempts; |
| 709 |
} |
| 710 |
|
| 711 |
GC_INNER void GC_collect_a_little_inner(int n) |
| 712 |
{ |
| 713 |
IF_CANCEL(int cancel_state;) |
| 714 |
|
| 715 |
GC_ASSERT(I_HOLD_LOCK()); |
| 716 |
DISABLE_CANCEL(cancel_state); |
| 717 |
if (GC_incremental && GC_collection_in_progress()) { |
| 718 |
int i; |
| 719 |
int max_deficit = GC_rate * n; |
| 720 |
|
| 721 |
# ifdef PARALLEL_MARK |
| 722 |
if (GC_time_limit != GC_TIME_UNLIMITED) |
| 723 |
GC_parallel_mark_disabled = TRUE; |
| 724 |
# endif |
| 725 |
for (i = GC_deficit; i < max_deficit; i++) { |
| 726 |
if (GC_mark_some(NULL)) |
| 727 |
break; |
| 728 |
} |
| 729 |
# ifdef PARALLEL_MARK |
| 730 |
GC_parallel_mark_disabled = FALSE; |
| 731 |
# endif |
| 732 |
|
| 733 |
if (i < max_deficit && !GC_dont_gc) { |
| 734 |
/* Need to finish a collection. */ |
| 735 |
SAVE_CALLERS_TO_LAST_STACK(); |
| 736 |
# ifdef PARALLEL_MARK |
| 737 |
if (GC_parallel) |
| 738 |
GC_wait_for_reclaim(); |
| 739 |
# endif |
| 740 |
if (GC_n_attempts < max_prior_attempts |
| 741 |
&& GC_time_limit != GC_TIME_UNLIMITED) { |
| 742 |
# ifndef NO_CLOCK |
| 743 |
GET_TIME(GC_start_time); |
| 744 |
# endif |
| 745 |
if (GC_stopped_mark(GC_timeout_stop_func)) { |
| 746 |
GC_finish_collection(); |
| 747 |
} else { |
| 748 |
GC_n_attempts++; |
| 749 |
} |
| 750 |
} else { |
| 751 |
/* TODO: If possible, GC_default_stop_func should be */ |
| 752 |
/* used here. */ |
| 753 |
(void)GC_stopped_mark(GC_never_stop_func); |
| 754 |
GC_finish_collection(); |
| 755 |
} |
| 756 |
} |
| 757 |
if (GC_deficit > 0) { |
| 758 |
GC_deficit -= max_deficit; |
| 759 |
if (GC_deficit < 0) |
| 760 |
GC_deficit = 0; |
| 761 |
} |
| 762 |
} else if (!GC_dont_gc) { |
| 763 |
GC_maybe_gc(); |
| 764 |
} |
| 765 |
RESTORE_CANCEL(cancel_state); |
| 766 |
} |
| 767 |
|
| 768 |
GC_INNER void (*GC_check_heap)(void) = 0; |
| 769 |
GC_INNER void (*GC_print_all_smashed)(void) = 0; |
| 770 |
|
| 771 |
GC_API int GC_CALL GC_collect_a_little(void) |
| 772 |
{ |
| 773 |
int result; |
| 774 |
DCL_LOCK_STATE; |
| 775 |
|
| 776 |
LOCK(); |
| 777 |
if (!GC_dont_gc) { |
| 778 |
ENTER_GC(); |
| 779 |
GC_collect_a_little_inner(1); |
| 780 |
EXIT_GC(); |
| 781 |
} |
| 782 |
result = (int)GC_collection_in_progress(); |
| 783 |
UNLOCK(); |
| 784 |
if (!result && GC_debugging_started) GC_print_all_smashed(); |
| 785 |
return(result); |
| 786 |
} |
| 787 |
|
| 788 |
#ifndef NO_CLOCK |
| 789 |
/* Variables for world-stop average delay time statistic computation. */ |
| 790 |
/* "divisor" is incremented every world-stop and halved when reached */ |
| 791 |
/* its maximum (or upon "total_time" overflow). */ |
| 792 |
static unsigned world_stopped_total_time = 0; |
| 793 |
static unsigned world_stopped_total_divisor = 0; |
| 794 |
# ifndef MAX_TOTAL_TIME_DIVISOR |
| 795 |
/* We shall not use big values here (so "outdated" delay time */ |
| 796 |
/* values would have less impact on "average" delay time value than */ |
| 797 |
/* newer ones). */ |
| 798 |
# define MAX_TOTAL_TIME_DIVISOR 1000 |
| 799 |
# endif |
| 800 |
#endif /* !NO_CLOCK */ |
| 801 |
|
| 802 |
#ifdef USE_MUNMAP |
| 803 |
# define IF_USE_MUNMAP(x) x |
| 804 |
# define COMMA_IF_USE_MUNMAP(x) /* comma */, x |
| 805 |
#else |
| 806 |
# define IF_USE_MUNMAP(x) /* empty */ |
| 807 |
# define COMMA_IF_USE_MUNMAP(x) /* empty */ |
| 808 |
#endif |
| 809 |
|
| 810 |
/* |
| 811 |
* We stop the world and mark from all roots. |
| 812 |
* If stop_func() ever returns TRUE, we may fail and return FALSE. |
| 813 |
* Increment GC_gc_no if we succeed. |
| 814 |
*/ |
| 815 |
STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func) |
| 816 |
{ |
| 817 |
int i; |
| 818 |
# ifndef NO_CLOCK |
| 819 |
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; |
| 820 |
# endif |
| 821 |
|
| 822 |
GC_ASSERT(I_HOLD_LOCK()); |
| 823 |
# if !defined(REDIRECT_MALLOC) && defined(USE_WINALLOC) |
| 824 |
GC_add_current_malloc_heap(); |
| 825 |
# endif |
| 826 |
# if defined(REGISTER_LIBRARIES_EARLY) |
| 827 |
GC_cond_register_dynamic_libraries(); |
| 828 |
# endif |
| 829 |
|
| 830 |
# ifndef NO_CLOCK |
| 831 |
if (GC_PRINT_STATS_FLAG) |
| 832 |
GET_TIME(start_time); |
| 833 |
# endif |
| 834 |
|
| 835 |
# if !defined(GC_NO_FINALIZATION) && !defined(GC_TOGGLE_REFS_NOT_NEEDED) |
| 836 |
GC_process_togglerefs(); |
| 837 |
# endif |
| 838 |
# ifdef THREADS |
| 839 |
if (GC_on_collection_event) |
| 840 |
GC_on_collection_event(GC_EVENT_PRE_STOP_WORLD); |
| 841 |
# endif |
| 842 |
STOP_WORLD(); |
| 843 |
# ifdef THREADS |
| 844 |
if (GC_on_collection_event) |
| 845 |
GC_on_collection_event(GC_EVENT_POST_STOP_WORLD); |
| 846 |
# endif |
| 847 |
|
| 848 |
# ifdef THREAD_LOCAL_ALLOC |
| 849 |
GC_world_stopped = TRUE; |
| 850 |
# endif |
| 851 |
/* Output blank line for convenience here */ |
| 852 |
GC_COND_LOG_PRINTF( |
| 853 |
"\n--> Marking for collection #%lu after %lu allocated bytes\n", |
| 854 |
(unsigned long)GC_gc_no + 1, (unsigned long) GC_bytes_allocd); |
| 855 |
# ifdef MAKE_BACK_GRAPH |
| 856 |
if (GC_print_back_height) { |
| 857 |
GC_build_back_graph(); |
| 858 |
} |
| 859 |
# endif |
| 860 |
|
| 861 |
/* Mark from all roots. */ |
| 862 |
if (GC_on_collection_event) |
| 863 |
GC_on_collection_event(GC_EVENT_MARK_START); |
| 864 |
|
| 865 |
/* Minimize junk left in my registers and on the stack */ |
| 866 |
GC_clear_a_few_frames(); |
| 867 |
GC_noop6(0,0,0,0,0,0); |
| 868 |
|
| 869 |
GC_initiate_gc(); |
| 870 |
# ifdef PARALLEL_MARK |
| 871 |
if (stop_func != GC_never_stop_func) |
| 872 |
GC_parallel_mark_disabled = TRUE; |
| 873 |
# endif |
| 874 |
for (i = 0; !(*stop_func)(); i++) { |
| 875 |
if (GC_mark_some(GC_approx_sp())) { |
| 876 |
# ifdef PARALLEL_MARK |
| 877 |
if (GC_parallel && GC_parallel_mark_disabled) { |
| 878 |
GC_COND_LOG_PRINTF("Stopped marking done after %d iterations" |
| 879 |
" with disabled parallel marker\n", i); |
| 880 |
} |
| 881 |
# endif |
| 882 |
i = -1; |
| 883 |
break; |
| 884 |
} |
| 885 |
} |
| 886 |
# ifdef PARALLEL_MARK |
| 887 |
GC_parallel_mark_disabled = FALSE; |
| 888 |
# endif |
| 889 |
|
| 890 |
if (i >= 0) { |
| 891 |
GC_COND_LOG_PRINTF("Abandoned stopped marking after" |
| 892 |
" %d iterations\n", i); |
| 893 |
GC_deficit = i; /* Give the mutator a chance. */ |
| 894 |
# ifdef THREAD_LOCAL_ALLOC |
| 895 |
GC_world_stopped = FALSE; |
| 896 |
# endif |
| 897 |
|
| 898 |
# ifdef THREADS |
| 899 |
if (GC_on_collection_event) |
| 900 |
GC_on_collection_event(GC_EVENT_PRE_START_WORLD); |
| 901 |
# endif |
| 902 |
|
| 903 |
START_WORLD(); |
| 904 |
|
| 905 |
# ifdef THREADS |
| 906 |
if (GC_on_collection_event) |
| 907 |
GC_on_collection_event(GC_EVENT_POST_START_WORLD); |
| 908 |
# endif |
| 909 |
|
| 910 |
/* TODO: Notify GC_EVENT_MARK_ABANDON */ |
| 911 |
return FALSE; |
| 912 |
} |
| 913 |
|
| 914 |
GC_gc_no++; |
| 915 |
# ifdef USE_MUNMAP |
| 916 |
GC_ASSERT(GC_heapsize >= GC_unmapped_bytes); |
| 917 |
# endif |
| 918 |
GC_ASSERT(GC_our_mem_bytes >= GC_heapsize); |
| 919 |
GC_DBGLOG_PRINTF("GC #%lu freed %ld bytes, heap %lu KiB (" |
| 920 |
IF_USE_MUNMAP("+ %lu KiB unmapped ") |
| 921 |
"+ %lu KiB internal)\n", |
| 922 |
(unsigned long)GC_gc_no, (long)GC_bytes_found, |
| 923 |
TO_KiB_UL(GC_heapsize - GC_unmapped_bytes) /*, */ |
| 924 |
COMMA_IF_USE_MUNMAP(TO_KiB_UL(GC_unmapped_bytes)), |
| 925 |
TO_KiB_UL(GC_our_mem_bytes - GC_heapsize)); |
| 926 |
|
| 927 |
/* Check all debugged objects for consistency */ |
| 928 |
if (GC_debugging_started) { |
| 929 |
(*GC_check_heap)(); |
| 930 |
} |
| 931 |
if (GC_on_collection_event) { |
| 932 |
GC_on_collection_event(GC_EVENT_MARK_END); |
| 933 |
# ifdef THREADS |
| 934 |
GC_on_collection_event(GC_EVENT_PRE_START_WORLD); |
| 935 |
# endif |
| 936 |
} |
| 937 |
# ifdef THREAD_LOCAL_ALLOC |
| 938 |
GC_world_stopped = FALSE; |
| 939 |
# endif |
| 940 |
|
| 941 |
START_WORLD(); |
| 942 |
|
| 943 |
# ifdef THREADS |
| 944 |
if (GC_on_collection_event) |
| 945 |
GC_on_collection_event(GC_EVENT_POST_START_WORLD); |
| 946 |
# endif |
| 947 |
|
| 948 |
# ifndef NO_CLOCK |
| 949 |
if (GC_PRINT_STATS_FLAG) { |
| 950 |
unsigned long time_diff; |
| 951 |
unsigned total_time, divisor; |
| 952 |
CLOCK_TYPE current_time; |
| 953 |
|
| 954 |
GET_TIME(current_time); |
| 955 |
time_diff = MS_TIME_DIFF(current_time,start_time); |
| 956 |
|
| 957 |
/* Compute new world-stop delay total time */ |
| 958 |
total_time = world_stopped_total_time; |
| 959 |
divisor = world_stopped_total_divisor; |
| 960 |
if ((int)total_time < 0 || divisor >= MAX_TOTAL_TIME_DIVISOR) { |
| 961 |
/* Halve values if overflow occurs */ |
| 962 |
total_time >>= 1; |
| 963 |
divisor >>= 1; |
| 964 |
} |
| 965 |
total_time += time_diff < (((unsigned)-1) >> 1) ? |
| 966 |
(unsigned)time_diff : ((unsigned)-1) >> 1; |
| 967 |
/* Update old world_stopped_total_time and its divisor */ |
| 968 |
world_stopped_total_time = total_time; |
| 969 |
world_stopped_total_divisor = ++divisor; |
| 970 |
|
| 971 |
GC_ASSERT(divisor != 0); |
| 972 |
GC_log_printf("World-stopped marking took %lu ms %lu ns" |
| 973 |
" (%u ms in average)\n", |
| 974 |
time_diff, NS_FRAC_TIME_DIFF(current_time, start_time), |
| 975 |
total_time / divisor); |
| 976 |
} |
| 977 |
# endif |
| 978 |
return(TRUE); |
| 979 |
} |
| 980 |
|
| 981 |
/* Set all mark bits for the free list whose first entry is q */ |
| 982 |
GC_INNER void GC_set_fl_marks(ptr_t q) |
| 983 |
{ |
| 984 |
if (q /* != NULL */) { /* CPPCHECK */ |
| 985 |
struct hblk *h = HBLKPTR(q); |
| 986 |
struct hblk *last_h = h; |
| 987 |
hdr *hhdr = HDR(h); |
| 988 |
IF_PER_OBJ(word sz = hhdr->hb_sz;) |
| 989 |
|
| 990 |
for (;;) { |
| 991 |
word bit_no = MARK_BIT_NO((ptr_t)q - (ptr_t)h, sz); |
| 992 |
|
| 993 |
if (!mark_bit_from_hdr(hhdr, bit_no)) { |
| 994 |
set_mark_bit_from_hdr(hhdr, bit_no); |
| 995 |
++hhdr -> hb_n_marks; |
| 996 |
} |
| 997 |
|
| 998 |
q = (ptr_t)obj_link(q); |
| 999 |
if (q == NULL) |
| 1000 |
break; |
| 1001 |
|
| 1002 |
h = HBLKPTR(q); |
| 1003 |
if (h != last_h) { |
| 1004 |
last_h = h; |
| 1005 |
hhdr = HDR(h); |
| 1006 |
IF_PER_OBJ(sz = hhdr->hb_sz;) |
| 1007 |
} |
| 1008 |
} |
| 1009 |
} |
| 1010 |
} |
| 1011 |
|
| 1012 |
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) |
| 1013 |
/* Check that all mark bits for the free list whose first entry is */ |
| 1014 |
/* (*pfreelist) are set. Check skipped if points to a special value. */ |
| 1015 |
void GC_check_fl_marks(void **pfreelist) |
| 1016 |
{ |
| 1017 |
/* TODO: There is a data race with GC_FAST_MALLOC_GRANS (which does */ |
| 1018 |
/* not do atomic updates to the free-list). The race seems to be */ |
| 1019 |
/* harmless, and for now we just skip this check in case of TSan. */ |
| 1020 |
# if defined(AO_HAVE_load_acquire_read) && !defined(THREAD_SANITIZER) |
| 1021 |
AO_t *list = (AO_t *)AO_load_acquire_read((AO_t *)pfreelist); |
| 1022 |
/* Atomic operations are used because the world is running. */ |
| 1023 |
AO_t *prev; |
| 1024 |
AO_t *p; |
| 1025 |
|
| 1026 |
if ((word)list <= HBLKSIZE) return; |
| 1027 |
|
| 1028 |
prev = (AO_t *)pfreelist; |
| 1029 |
for (p = list; p != NULL;) { |
| 1030 |
AO_t *next; |
| 1031 |
|
| 1032 |
if (!GC_is_marked(p)) { |
| 1033 |
ABORT_ARG2("Unmarked local free list entry", |
| 1034 |
": object %p on list %p", (void *)p, (void *)list); |
| 1035 |
} |
| 1036 |
|
| 1037 |
/* While traversing the free-list, it re-reads the pointer to */ |
| 1038 |
/* the current node before accepting its next pointer and */ |
| 1039 |
/* bails out if the latter has changed. That way, it won't */ |
| 1040 |
/* try to follow the pointer which might be been modified */ |
| 1041 |
/* after the object was returned to the client. It might */ |
| 1042 |
/* perform the mark-check on the just allocated object but */ |
| 1043 |
/* that should be harmless. */ |
| 1044 |
next = (AO_t *)AO_load_acquire_read(p); |
| 1045 |
if (AO_load(prev) != (AO_t)p) |
| 1046 |
break; |
| 1047 |
prev = p; |
| 1048 |
p = next; |
| 1049 |
} |
| 1050 |
# else |
| 1051 |
/* FIXME: Not implemented (just skipped). */ |
| 1052 |
(void)pfreelist; |
| 1053 |
# endif |
| 1054 |
} |
| 1055 |
#endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */ |
| 1056 |
|
| 1057 |
/* Clear all mark bits for the free list whose first entry is q */ |
| 1058 |
/* Decrement GC_bytes_found by number of bytes on free list. */ |
| 1059 |
STATIC void GC_clear_fl_marks(ptr_t q) |
| 1060 |
{ |
| 1061 |
struct hblk *h = HBLKPTR(q); |
| 1062 |
struct hblk *last_h = h; |
| 1063 |
hdr *hhdr = HDR(h); |
| 1064 |
word sz = hhdr->hb_sz; /* Normally set only once. */ |
| 1065 |
|
| 1066 |
for (;;) { |
| 1067 |
word bit_no = MARK_BIT_NO((ptr_t)q - (ptr_t)h, sz); |
| 1068 |
|
| 1069 |
if (mark_bit_from_hdr(hhdr, bit_no)) { |
| 1070 |
size_t n_marks = hhdr -> hb_n_marks; |
| 1071 |
|
| 1072 |
GC_ASSERT(n_marks != 0); |
| 1073 |
clear_mark_bit_from_hdr(hhdr, bit_no); |
| 1074 |
n_marks--; |
| 1075 |
# ifdef PARALLEL_MARK |
| 1076 |
/* Appr. count, don't decrement to zero! */ |
| 1077 |
if (0 != n_marks || !GC_parallel) { |
| 1078 |
hhdr -> hb_n_marks = n_marks; |
| 1079 |
} |
| 1080 |
# else |
| 1081 |
hhdr -> hb_n_marks = n_marks; |
| 1082 |
# endif |
| 1083 |
} |
| 1084 |
GC_bytes_found -= sz; |
| 1085 |
|
| 1086 |
q = (ptr_t)obj_link(q); |
| 1087 |
if (q == NULL) |
| 1088 |
break; |
| 1089 |
|
| 1090 |
h = HBLKPTR(q); |
| 1091 |
if (h != last_h) { |
| 1092 |
last_h = h; |
| 1093 |
hhdr = HDR(h); |
| 1094 |
sz = hhdr->hb_sz; |
| 1095 |
} |
| 1096 |
} |
| 1097 |
} |
| 1098 |
|
| 1099 |
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) |
| 1100 |
void GC_check_tls(void); |
| 1101 |
#endif |
| 1102 |
|
| 1103 |
GC_on_heap_resize_proc GC_on_heap_resize = 0; |
| 1104 |
|
| 1105 |
/* Used for logging only. */ |
| 1106 |
GC_INLINE int GC_compute_heap_usage_percent(void) |
| 1107 |
{ |
| 1108 |
word used = GC_composite_in_use + GC_atomic_in_use; |
| 1109 |
word heap_sz = GC_heapsize - GC_unmapped_bytes; |
| 1110 |
# if defined(CPPCHECK) |
| 1111 |
word limit = (GC_WORD_MAX >> 1) / 50; /* to avoid a false positive */ |
| 1112 |
# else |
| 1113 |
const word limit = GC_WORD_MAX / 100; |
| 1114 |
# endif |
| 1115 |
|
| 1116 |
return used >= heap_sz ? 0 : used < limit ? |
| 1117 |
(int)((used * 100) / heap_sz) : (int)(used / (heap_sz / 100)); |
| 1118 |
} |
| 1119 |
|
| 1120 |
/* Finish up a collection. Assumes mark bits are consistent, lock is */ |
| 1121 |
/* held, but the world is otherwise running. */ |
| 1122 |
STATIC void GC_finish_collection(void) |
| 1123 |
{ |
| 1124 |
# ifndef NO_CLOCK |
| 1125 |
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; |
| 1126 |
CLOCK_TYPE finalize_time = CLOCK_TYPE_INITIALIZER; |
| 1127 |
# endif |
| 1128 |
|
| 1129 |
GC_ASSERT(I_HOLD_LOCK()); |
| 1130 |
# if defined(GC_ASSERTIONS) \ |
| 1131 |
&& defined(THREAD_LOCAL_ALLOC) && !defined(DBG_HDRS_ALL) |
| 1132 |
/* Check that we marked some of our own data. */ |
| 1133 |
/* TODO: Add more checks. */ |
| 1134 |
GC_check_tls(); |
| 1135 |
# endif |
| 1136 |
|
| 1137 |
# ifndef NO_CLOCK |
| 1138 |
if (GC_print_stats) |
| 1139 |
GET_TIME(start_time); |
| 1140 |
# endif |
| 1141 |
if (GC_on_collection_event) |
| 1142 |
GC_on_collection_event(GC_EVENT_RECLAIM_START); |
| 1143 |
|
| 1144 |
# ifndef GC_GET_HEAP_USAGE_NOT_NEEDED |
| 1145 |
if (GC_bytes_found > 0) |
| 1146 |
GC_reclaimed_bytes_before_gc += (word)GC_bytes_found; |
| 1147 |
# endif |
| 1148 |
GC_bytes_found = 0; |
| 1149 |
# if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG) |
| 1150 |
if (GETENV("GC_PRINT_ADDRESS_MAP") != 0) { |
| 1151 |
GC_print_address_map(); |
| 1152 |
} |
| 1153 |
# endif |
| 1154 |
COND_DUMP; |
| 1155 |
if (GC_find_leak) { |
| 1156 |
/* Mark all objects on the free list. All objects should be */ |
| 1157 |
/* marked when we're done. */ |
| 1158 |
word size; /* current object size */ |
| 1159 |
unsigned kind; |
| 1160 |
ptr_t q; |
| 1161 |
|
| 1162 |
for (kind = 0; kind < GC_n_kinds; kind++) { |
| 1163 |
for (size = 1; size <= MAXOBJGRANULES; size++) { |
| 1164 |
q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size]; |
| 1165 |
if (q != NULL) |
| 1166 |
GC_set_fl_marks(q); |
| 1167 |
} |
| 1168 |
} |
| 1169 |
GC_start_reclaim(TRUE); |
| 1170 |
/* The above just checks; it doesn't really reclaim anything. */ |
| 1171 |
} |
| 1172 |
|
| 1173 |
# ifndef GC_NO_FINALIZATION |
| 1174 |
GC_finalize(); |
| 1175 |
# endif |
| 1176 |
# ifndef NO_CLOCK |
| 1177 |
if (GC_print_stats) |
| 1178 |
GET_TIME(finalize_time); |
| 1179 |
# endif |
| 1180 |
|
| 1181 |
if (GC_print_back_height) { |
| 1182 |
# ifdef MAKE_BACK_GRAPH |
| 1183 |
GC_traverse_back_graph(); |
| 1184 |
# elif !defined(SMALL_CONFIG) |
| 1185 |
GC_err_printf("Back height not available: " |
| 1186 |
"Rebuild collector with -DMAKE_BACK_GRAPH\n"); |
| 1187 |
# endif |
| 1188 |
} |
| 1189 |
|
| 1190 |
/* Clear free list mark bits, in case they got accidentally marked */ |
| 1191 |
/* (or GC_find_leak is set and they were intentionally marked). */ |
| 1192 |
/* Also subtract memory remaining from GC_bytes_found count. */ |
| 1193 |
/* Note that composite objects on free list are cleared. */ |
| 1194 |
/* Thus accidentally marking a free list is not a problem; only */ |
| 1195 |
/* objects on the list itself will be marked, and that's fixed here. */ |
| 1196 |
{ |
| 1197 |
word size; /* current object size */ |
| 1198 |
ptr_t q; /* pointer to current object */ |
| 1199 |
unsigned kind; |
| 1200 |
|
| 1201 |
for (kind = 0; kind < GC_n_kinds; kind++) { |
| 1202 |
for (size = 1; size <= MAXOBJGRANULES; size++) { |
| 1203 |
q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size]; |
| 1204 |
if (q != NULL) |
| 1205 |
GC_clear_fl_marks(q); |
| 1206 |
} |
| 1207 |
} |
| 1208 |
} |
| 1209 |
|
| 1210 |
GC_VERBOSE_LOG_PRINTF("Bytes recovered before sweep - f.l. count = %ld\n", |
| 1211 |
(long)GC_bytes_found); |
| 1212 |
|
| 1213 |
/* Reconstruct free lists to contain everything not marked */ |
| 1214 |
GC_start_reclaim(FALSE); |
| 1215 |
GC_DBGLOG_PRINTF("In-use heap: %d%% (%lu KiB pointers + %lu KiB other)\n", |
| 1216 |
GC_compute_heap_usage_percent(), |
| 1217 |
TO_KiB_UL(GC_composite_in_use), |
| 1218 |
TO_KiB_UL(GC_atomic_in_use)); |
| 1219 |
if (GC_is_full_gc) { |
| 1220 |
GC_used_heap_size_after_full = USED_HEAP_SIZE; |
| 1221 |
GC_need_full_gc = FALSE; |
| 1222 |
} else { |
| 1223 |
GC_need_full_gc = USED_HEAP_SIZE - GC_used_heap_size_after_full |
| 1224 |
> min_bytes_allocd(); |
| 1225 |
} |
| 1226 |
|
| 1227 |
GC_VERBOSE_LOG_PRINTF("Immediately reclaimed %ld bytes, heapsize:" |
| 1228 |
" %lu bytes" IF_USE_MUNMAP(" (%lu unmapped)") "\n", |
| 1229 |
(long)GC_bytes_found, |
| 1230 |
(unsigned long)GC_heapsize /*, */ |
| 1231 |
COMMA_IF_USE_MUNMAP((unsigned long) |
| 1232 |
GC_unmapped_bytes)); |
| 1233 |
|
| 1234 |
/* Reset or increment counters for next cycle */ |
| 1235 |
GC_n_attempts = 0; |
| 1236 |
GC_is_full_gc = FALSE; |
| 1237 |
GC_bytes_allocd_before_gc += GC_bytes_allocd; |
| 1238 |
GC_non_gc_bytes_at_gc = GC_non_gc_bytes; |
| 1239 |
GC_bytes_allocd = 0; |
| 1240 |
GC_bytes_dropped = 0; |
| 1241 |
GC_bytes_freed = 0; |
| 1242 |
GC_finalizer_bytes_freed = 0; |
| 1243 |
|
| 1244 |
IF_USE_MUNMAP(GC_unmap_old()); |
| 1245 |
|
| 1246 |
if (GC_on_collection_event) |
| 1247 |
GC_on_collection_event(GC_EVENT_RECLAIM_END); |
| 1248 |
# ifndef NO_CLOCK |
| 1249 |
if (GC_print_stats) { |
| 1250 |
CLOCK_TYPE done_time; |
| 1251 |
|
| 1252 |
GET_TIME(done_time); |
| 1253 |
# if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION) |
| 1254 |
/* A convenient place to output finalization statistics. */ |
| 1255 |
GC_print_finalization_stats(); |
| 1256 |
# endif |
| 1257 |
GC_log_printf("Finalize and initiate sweep took %lu ms %lu ns" |
| 1258 |
" + %lu ms %lu ns\n", |
| 1259 |
MS_TIME_DIFF(finalize_time, start_time), |
| 1260 |
NS_FRAC_TIME_DIFF(finalize_time, start_time), |
| 1261 |
MS_TIME_DIFF(done_time, finalize_time), |
| 1262 |
NS_FRAC_TIME_DIFF(done_time, finalize_time)); |
| 1263 |
} |
| 1264 |
# elif !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION) |
| 1265 |
if (GC_print_stats) |
| 1266 |
GC_print_finalization_stats(); |
| 1267 |
# endif |
| 1268 |
} |
| 1269 |
|
| 1270 |
STATIC word GC_heapsize_at_forced_unmap = 0; |
| 1271 |
/* accessed with the allocation lock held */ |
| 1272 |
|
| 1273 |
/* If stop_func == 0 then GC_default_stop_func is used instead. */ |
| 1274 |
STATIC GC_bool GC_try_to_collect_general(GC_stop_func stop_func, |
| 1275 |
GC_bool force_unmap) |
| 1276 |
{ |
| 1277 |
GC_bool result; |
| 1278 |
IF_USE_MUNMAP(int old_unmap_threshold;) |
| 1279 |
IF_CANCEL(int cancel_state;) |
| 1280 |
DCL_LOCK_STATE; |
| 1281 |
|
| 1282 |
if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); |
| 1283 |
if (GC_debugging_started) GC_print_all_smashed(); |
| 1284 |
GC_INVOKE_FINALIZERS(); |
| 1285 |
LOCK(); |
| 1286 |
if (force_unmap) { |
| 1287 |
/* Record current heap size to make heap growth more conservative */ |
| 1288 |
/* afterwards (as if the heap is growing from zero size again). */ |
| 1289 |
GC_heapsize_at_forced_unmap = GC_heapsize; |
| 1290 |
} |
| 1291 |
DISABLE_CANCEL(cancel_state); |
| 1292 |
# ifdef USE_MUNMAP |
| 1293 |
old_unmap_threshold = GC_unmap_threshold; |
| 1294 |
if (force_unmap || |
| 1295 |
(GC_force_unmap_on_gcollect && old_unmap_threshold > 0)) |
| 1296 |
GC_unmap_threshold = 1; /* unmap as much as possible */ |
| 1297 |
# endif |
| 1298 |
ENTER_GC(); |
| 1299 |
/* Minimize junk left in my registers */ |
| 1300 |
GC_noop6(0,0,0,0,0,0); |
| 1301 |
result = GC_try_to_collect_inner(stop_func != 0 ? stop_func : |
| 1302 |
GC_default_stop_func); |
| 1303 |
EXIT_GC(); |
| 1304 |
IF_USE_MUNMAP(GC_unmap_threshold = old_unmap_threshold); /* restore */ |
| 1305 |
RESTORE_CANCEL(cancel_state); |
| 1306 |
UNLOCK(); |
| 1307 |
if (result) { |
| 1308 |
if (GC_debugging_started) GC_print_all_smashed(); |
| 1309 |
GC_INVOKE_FINALIZERS(); |
| 1310 |
} |
| 1311 |
return(result); |
| 1312 |
} |
| 1313 |
|
| 1314 |
/* Externally callable routines to invoke full, stop-the-world collection. */ |
| 1315 |
|
| 1316 |
GC_API int GC_CALL GC_try_to_collect(GC_stop_func stop_func) |
| 1317 |
{ |
| 1318 |
GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func)); |
| 1319 |
return (int)GC_try_to_collect_general(stop_func, FALSE); |
| 1320 |
} |
| 1321 |
|
| 1322 |
GC_API void GC_CALL GC_gcollect(void) |
| 1323 |
{ |
| 1324 |
/* 0 is passed as stop_func to get GC_default_stop_func value */ |
| 1325 |
/* while holding the allocation lock (to prevent data races). */ |
| 1326 |
(void)GC_try_to_collect_general(0, FALSE); |
| 1327 |
if (get_have_errors()) |
| 1328 |
GC_print_all_errors(); |
| 1329 |
} |
| 1330 |
|
| 1331 |
GC_API void GC_CALL GC_gcollect_and_unmap(void) |
| 1332 |
{ |
| 1333 |
/* Collect and force memory unmapping to OS. */ |
| 1334 |
(void)GC_try_to_collect_general(GC_never_stop_func, TRUE); |
| 1335 |
} |
| 1336 |
|
| 1337 |
#ifdef USE_PROC_FOR_LIBRARIES |
| 1338 |
/* Add HBLKSIZE aligned, GET_MEM-generated block to GC_our_memory. */ |
| 1339 |
GC_INNER void GC_add_to_our_memory(ptr_t p, size_t bytes) |
| 1340 |
{ |
| 1341 |
GC_ASSERT(p != NULL); |
| 1342 |
if (GC_n_memory >= MAX_HEAP_SECTS) |
| 1343 |
ABORT("Too many GC-allocated memory sections: Increase MAX_HEAP_SECTS"); |
| 1344 |
GC_our_memory[GC_n_memory].hs_start = p; |
| 1345 |
GC_our_memory[GC_n_memory].hs_bytes = bytes; |
| 1346 |
GC_n_memory++; |
| 1347 |
GC_our_mem_bytes += bytes; |
| 1348 |
} |
| 1349 |
#endif |
| 1350 |
|
| 1351 |
/* Use the chunk of memory starting at p of size bytes as part of the heap. */ |
| 1352 |
/* Assumes p is HBLKSIZE aligned, bytes argument is a multiple of HBLKSIZE. */ |
| 1353 |
STATIC void GC_add_to_heap(struct hblk *p, size_t bytes) |
| 1354 |
{ |
| 1355 |
hdr * phdr; |
| 1356 |
word endp; |
| 1357 |
size_t old_capacity = 0; |
| 1358 |
void *old_heap_sects = NULL; |
| 1359 |
# ifdef GC_ASSERTIONS |
| 1360 |
unsigned i; |
| 1361 |
# endif |
| 1362 |
|
| 1363 |
GC_ASSERT((word)p % HBLKSIZE == 0); |
| 1364 |
GC_ASSERT(bytes % HBLKSIZE == 0); |
| 1365 |
GC_ASSERT(bytes > 0); |
| 1366 |
GC_ASSERT(GC_all_nils != NULL); |
| 1367 |
|
| 1368 |
if (GC_n_heap_sects == GC_capacity_heap_sects) { |
| 1369 |
/* Allocate new GC_heap_sects with sufficient capacity. */ |
| 1370 |
# ifndef INITIAL_HEAP_SECTS |
| 1371 |
# define INITIAL_HEAP_SECTS 32 |
| 1372 |
# endif |
| 1373 |
size_t new_capacity = GC_n_heap_sects > 0 ? |
| 1374 |
(size_t)GC_n_heap_sects * 2 : INITIAL_HEAP_SECTS; |
| 1375 |
void *new_heap_sects = |
| 1376 |
GC_scratch_alloc(new_capacity * sizeof(struct HeapSect)); |
| 1377 |
|
| 1378 |
if (EXPECT(NULL == new_heap_sects, FALSE)) { |
| 1379 |
/* Retry with smaller yet sufficient capacity. */ |
| 1380 |
new_capacity = (size_t)GC_n_heap_sects + INITIAL_HEAP_SECTS; |
| 1381 |
new_heap_sects = |
| 1382 |
GC_scratch_alloc(new_capacity * sizeof(struct HeapSect)); |
| 1383 |
if (NULL == new_heap_sects) |
| 1384 |
ABORT("Insufficient memory for heap sections"); |
| 1385 |
} |
| 1386 |
old_capacity = GC_capacity_heap_sects; |
| 1387 |
old_heap_sects = GC_heap_sects; |
| 1388 |
/* Transfer GC_heap_sects contents to the newly allocated array. */ |
| 1389 |
if (GC_n_heap_sects > 0) |
| 1390 |
BCOPY(old_heap_sects, new_heap_sects, |
| 1391 |
GC_n_heap_sects * sizeof(struct HeapSect)); |
| 1392 |
GC_capacity_heap_sects = new_capacity; |
| 1393 |
GC_heap_sects = (struct HeapSect *)new_heap_sects; |
| 1394 |
GC_COND_LOG_PRINTF("Grew heap sections array to %lu elements\n", |
| 1395 |
(unsigned long)new_capacity); |
| 1396 |
} |
| 1397 |
|
| 1398 |
while ((word)p <= HBLKSIZE) { |
| 1399 |
/* Can't handle memory near address zero. */ |
| 1400 |
++p; |
| 1401 |
bytes -= HBLKSIZE; |
| 1402 |
if (0 == bytes) return; |
| 1403 |
} |
| 1404 |
endp = (word)p + bytes; |
| 1405 |
while (endp <= (word)p) { |
| 1406 |
/* Address wrapped. */ |
| 1407 |
bytes -= HBLKSIZE; |
| 1408 |
if (0 == bytes) return; |
| 1409 |
endp -= HBLKSIZE; |
| 1410 |
} |
| 1411 |
phdr = GC_install_header(p); |
| 1412 |
if (0 == phdr) { |
| 1413 |
/* This is extremely unlikely. Can't add it. This will */ |
| 1414 |
/* almost certainly result in a 0 return from the allocator, */ |
| 1415 |
/* which is entirely appropriate. */ |
| 1416 |
return; |
| 1417 |
} |
| 1418 |
GC_ASSERT(endp > (word)p && endp == (word)p + bytes); |
| 1419 |
# ifdef GC_ASSERTIONS |
| 1420 |
/* Ensure no intersection between sections. */ |
| 1421 |
for (i = 0; i < GC_n_heap_sects; i++) { |
| 1422 |
word hs_start = (word)GC_heap_sects[i].hs_start; |
| 1423 |
word hs_end = hs_start + GC_heap_sects[i].hs_bytes; |
| 1424 |
word p_e = (word)p + bytes; |
| 1425 |
|
| 1426 |
GC_ASSERT(!((hs_start <= (word)p && (word)p < hs_end) |
| 1427 |
|| (hs_start < p_e && p_e <= hs_end) |
| 1428 |
|| ((word)p < hs_start && hs_end < p_e))); |
| 1429 |
} |
| 1430 |
# endif |
| 1431 |
GC_heap_sects[GC_n_heap_sects].hs_start = (ptr_t)p; |
| 1432 |
GC_heap_sects[GC_n_heap_sects].hs_bytes = bytes; |
| 1433 |
GC_n_heap_sects++; |
| 1434 |
phdr -> hb_sz = bytes; |
| 1435 |
phdr -> hb_flags = 0; |
| 1436 |
GC_freehblk(p); |
| 1437 |
GC_heapsize += bytes; |
| 1438 |
|
| 1439 |
/* Normally the caller calculates a new GC_collect_at_heapsize, |
| 1440 |
* but this is also called directly from GC_scratch_recycle_inner, so |
| 1441 |
* adjust here. It will be recalculated when called from |
| 1442 |
* GC_expand_hp_inner. |
| 1443 |
*/ |
| 1444 |
GC_collect_at_heapsize += bytes; |
| 1445 |
if (GC_collect_at_heapsize < GC_heapsize /* wrapped */) |
| 1446 |
GC_collect_at_heapsize = GC_WORD_MAX; |
| 1447 |
|
| 1448 |
if ((word)p <= (word)GC_least_plausible_heap_addr |
| 1449 |
|| GC_least_plausible_heap_addr == 0) { |
| 1450 |
GC_least_plausible_heap_addr = (void *)((ptr_t)p - sizeof(word)); |
| 1451 |
/* Making it a little smaller than necessary prevents */ |
| 1452 |
/* us from getting a false hit from the variable */ |
| 1453 |
/* itself. There's some unintentional reflection */ |
| 1454 |
/* here. */ |
| 1455 |
} |
| 1456 |
if ((word)p + bytes >= (word)GC_greatest_plausible_heap_addr) { |
| 1457 |
GC_greatest_plausible_heap_addr = (void *)endp; |
| 1458 |
} |
| 1459 |
# ifdef SET_REAL_HEAP_BOUNDS |
| 1460 |
if ((word)p < GC_least_real_heap_addr |
| 1461 |
|| EXPECT(0 == GC_least_real_heap_addr, FALSE)) |
| 1462 |
GC_least_real_heap_addr = (word)p - sizeof(word); |
| 1463 |
if (endp > GC_greatest_real_heap_addr) { |
| 1464 |
# ifdef INCLUDE_LINUX_THREAD_DESCR |
| 1465 |
/* Avoid heap intersection with the static data roots. */ |
| 1466 |
GC_exclude_static_roots_inner((void *)p, (void *)endp); |
| 1467 |
# endif |
| 1468 |
GC_greatest_real_heap_addr = endp; |
| 1469 |
} |
| 1470 |
# endif |
| 1471 |
GC_handle_protected_regions_limit(); |
| 1472 |
if (old_capacity > 0) { |
| 1473 |
# ifndef GWW_VDB |
| 1474 |
/* Recycling may call GC_add_to_heap() again but should not */ |
| 1475 |
/* cause resizing of GC_heap_sects. */ |
| 1476 |
GC_scratch_recycle_no_gww(old_heap_sects, |
| 1477 |
old_capacity * sizeof(struct HeapSect)); |
| 1478 |
# else |
| 1479 |
/* TODO: implement GWW-aware recycling as in alloc_mark_stack */ |
| 1480 |
GC_noop1((word)old_heap_sects); |
| 1481 |
# endif |
| 1482 |
} |
| 1483 |
} |
| 1484 |
|
| 1485 |
#if !defined(NO_DEBUGGING) |
| 1486 |
void GC_print_heap_sects(void) |
| 1487 |
{ |
| 1488 |
unsigned i; |
| 1489 |
|
| 1490 |
GC_printf("Total heap size: %lu" IF_USE_MUNMAP(" (%lu unmapped)") "\n", |
| 1491 |
(unsigned long)GC_heapsize /*, */ |
| 1492 |
COMMA_IF_USE_MUNMAP((unsigned long)GC_unmapped_bytes)); |
| 1493 |
|
| 1494 |
for (i = 0; i < GC_n_heap_sects; i++) { |
| 1495 |
ptr_t start = GC_heap_sects[i].hs_start; |
| 1496 |
size_t len = GC_heap_sects[i].hs_bytes; |
| 1497 |
struct hblk *h; |
| 1498 |
unsigned nbl = 0; |
| 1499 |
|
| 1500 |
for (h = (struct hblk *)start; (word)h < (word)(start + len); h++) { |
| 1501 |
if (GC_is_black_listed(h, HBLKSIZE)) nbl++; |
| 1502 |
} |
| 1503 |
GC_printf("Section %d from %p to %p %u/%lu blacklisted\n", |
| 1504 |
i, (void *)start, (void *)&start[len], |
| 1505 |
nbl, (unsigned long)divHBLKSZ(len)); |
| 1506 |
} |
| 1507 |
} |
| 1508 |
#endif |
| 1509 |
|
| 1510 |
void * GC_least_plausible_heap_addr = (void *)GC_WORD_MAX; |
| 1511 |
void * GC_greatest_plausible_heap_addr = 0; |
| 1512 |
|
| 1513 |
GC_INLINE word GC_max(word x, word y) |
| 1514 |
{ |
| 1515 |
return(x > y? x : y); |
| 1516 |
} |
| 1517 |
|
| 1518 |
GC_INLINE word GC_min(word x, word y) |
| 1519 |
{ |
| 1520 |
return(x < y? x : y); |
| 1521 |
} |
| 1522 |
|
| 1523 |
STATIC word GC_max_heapsize = 0; |
| 1524 |
|
| 1525 |
GC_API void GC_CALL GC_set_max_heap_size(GC_word n) |
| 1526 |
{ |
| 1527 |
GC_max_heapsize = n; |
| 1528 |
} |
| 1529 |
|
| 1530 |
GC_word GC_max_retries = 0; |
| 1531 |
|
| 1532 |
GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t bytes) |
| 1533 |
{ |
| 1534 |
size_t page_offset; |
| 1535 |
size_t displ = 0; |
| 1536 |
size_t recycled_bytes; |
| 1537 |
|
| 1538 |
if (NULL == ptr) return; |
| 1539 |
|
| 1540 |
GC_ASSERT(bytes != 0); |
| 1541 |
GC_ASSERT(GC_page_size != 0); |
| 1542 |
/* TODO: Assert correct memory flags if GWW_VDB */ |
| 1543 |
page_offset = (word)ptr & (GC_page_size - 1); |
| 1544 |
if (page_offset != 0) |
| 1545 |
displ = GC_page_size - page_offset; |
| 1546 |
recycled_bytes = bytes > displ ? (bytes - displ) & ~(GC_page_size - 1) : 0; |
| 1547 |
GC_COND_LOG_PRINTF("Recycle %lu/%lu scratch-allocated bytes at %p\n", |
| 1548 |
(unsigned long)recycled_bytes, (unsigned long)bytes, ptr); |
| 1549 |
if (recycled_bytes > 0) |
| 1550 |
GC_add_to_heap((struct hblk *)((word)ptr + displ), recycled_bytes); |
| 1551 |
} |
| 1552 |
|
| 1553 |
/* This explicitly increases the size of the heap. It is used */ |
| 1554 |
/* internally, but may also be invoked from GC_expand_hp by the user. */ |
| 1555 |
/* The argument is in units of HBLKSIZE (tiny values are rounded up). */ |
| 1556 |
/* Returns FALSE on failure. */ |
| 1557 |
GC_INNER GC_bool GC_expand_hp_inner(word n) |
| 1558 |
{ |
| 1559 |
size_t bytes; |
| 1560 |
struct hblk * space; |
| 1561 |
word expansion_slop; /* Number of bytes by which we expect */ |
| 1562 |
/* the heap to expand soon. */ |
| 1563 |
|
| 1564 |
GC_ASSERT(I_HOLD_LOCK()); |
| 1565 |
GC_ASSERT(GC_page_size != 0); |
| 1566 |
if (n < MINHINCR) n = MINHINCR; |
| 1567 |
bytes = ROUNDUP_PAGESIZE((size_t)n * HBLKSIZE); |
| 1568 |
if (GC_max_heapsize != 0 |
| 1569 |
&& (GC_max_heapsize < (word)bytes |
| 1570 |
|| GC_heapsize > GC_max_heapsize - (word)bytes)) { |
| 1571 |
/* Exceeded self-imposed limit */ |
| 1572 |
return(FALSE); |
| 1573 |
} |
| 1574 |
space = GET_MEM(bytes); |
| 1575 |
if (EXPECT(NULL == space, FALSE)) { |
| 1576 |
WARN("Failed to expand heap by %" WARN_PRIuPTR " KiB\n", bytes >> 10); |
| 1577 |
return(FALSE); |
| 1578 |
} |
| 1579 |
GC_add_to_our_memory((ptr_t)space, bytes); |
| 1580 |
GC_INFOLOG_PRINTF("Grow heap to %lu KiB after %lu bytes allocated\n", |
| 1581 |
TO_KiB_UL(GC_heapsize + bytes), |
| 1582 |
(unsigned long)GC_bytes_allocd); |
| 1583 |
|
| 1584 |
/* Adjust heap limits generously for blacklisting to work better. */ |
| 1585 |
/* GC_add_to_heap performs minimal adjustment needed for */ |
| 1586 |
/* correctness. */ |
| 1587 |
expansion_slop = min_bytes_allocd() + 4 * MAXHINCR * HBLKSIZE; |
| 1588 |
if ((GC_last_heap_addr == 0 && !((word)space & SIGNB)) |
| 1589 |
|| (GC_last_heap_addr != 0 |
| 1590 |
&& (word)GC_last_heap_addr < (word)space)) { |
| 1591 |
/* Assume the heap is growing up. */ |
| 1592 |
word new_limit = (word)space + (word)bytes + expansion_slop; |
| 1593 |
if (new_limit > (word)space) { |
| 1594 |
GC_greatest_plausible_heap_addr = |
| 1595 |
(void *)GC_max((word)GC_greatest_plausible_heap_addr, |
| 1596 |
(word)new_limit); |
| 1597 |
} |
| 1598 |
} else { |
| 1599 |
/* Heap is growing down. */ |
| 1600 |
word new_limit = (word)space - expansion_slop; |
| 1601 |
if (new_limit < (word)space) { |
| 1602 |
GC_least_plausible_heap_addr = |
| 1603 |
(void *)GC_min((word)GC_least_plausible_heap_addr, |
| 1604 |
(word)space - expansion_slop); |
| 1605 |
} |
| 1606 |
} |
| 1607 |
GC_last_heap_addr = (ptr_t)space; |
| 1608 |
|
| 1609 |
GC_add_to_heap(space, bytes); |
| 1610 |
|
| 1611 |
/* Force GC before we are likely to allocate past expansion_slop. */ |
| 1612 |
GC_collect_at_heapsize = |
| 1613 |
GC_heapsize + expansion_slop - 2 * MAXHINCR * HBLKSIZE; |
| 1614 |
if (GC_collect_at_heapsize < GC_heapsize /* wrapped */) |
| 1615 |
GC_collect_at_heapsize = GC_WORD_MAX; |
| 1616 |
if (GC_on_heap_resize) |
| 1617 |
(*GC_on_heap_resize)(GC_heapsize); |
| 1618 |
|
| 1619 |
return(TRUE); |
| 1620 |
} |
| 1621 |
|
| 1622 |
/* Really returns a bool, but it's externally visible, so that's clumsy. */ |
| 1623 |
/* The argument is in bytes. Includes GC_init() call. */ |
| 1624 |
GC_API int GC_CALL GC_expand_hp(size_t bytes) |
| 1625 |
{ |
| 1626 |
int result; |
| 1627 |
DCL_LOCK_STATE; |
| 1628 |
|
| 1629 |
if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); |
| 1630 |
LOCK(); |
| 1631 |
result = (int)GC_expand_hp_inner(divHBLKSZ((word)bytes)); |
| 1632 |
if (result) GC_requested_heapsize += bytes; |
| 1633 |
UNLOCK(); |
| 1634 |
return(result); |
| 1635 |
} |
| 1636 |
|
| 1637 |
GC_INNER unsigned GC_fail_count = 0; |
| 1638 |
/* How many consecutive GC/expansion failures? */ |
| 1639 |
/* Reset by GC_allochblk. */ |
| 1640 |
|
| 1641 |
/* The minimum value of the ratio of allocated bytes since the latest */ |
| 1642 |
/* GC to the amount of finalizers created since that GC which triggers */ |
| 1643 |
/* the collection instead heap expansion. Has no effect in the */ |
| 1644 |
/* incremental mode. */ |
| 1645 |
#if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK) |
| 1646 |
STATIC word GC_allocd_bytes_per_finalizer = GC_ALLOCD_BYTES_PER_FINALIZER; |
| 1647 |
#else |
| 1648 |
STATIC word GC_allocd_bytes_per_finalizer = 10000; |
| 1649 |
#endif |
| 1650 |
|
| 1651 |
GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word value) |
| 1652 |
{ |
| 1653 |
GC_allocd_bytes_per_finalizer = value; |
| 1654 |
} |
| 1655 |
|
| 1656 |
GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void) |
| 1657 |
{ |
| 1658 |
return GC_allocd_bytes_per_finalizer; |
| 1659 |
} |
| 1660 |
|
| 1661 |
static word last_fo_entries = 0; |
| 1662 |
static word last_bytes_finalized = 0; |
| 1663 |
|
| 1664 |
/* Collect or expand heap in an attempt make the indicated number of */ |
| 1665 |
/* free blocks available. Should be called until the blocks are */ |
| 1666 |
/* available (setting retry value to TRUE unless this is the first call */ |
| 1667 |
/* in a loop) or until it fails by returning FALSE. */ |
| 1668 |
GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, |
| 1669 |
GC_bool ignore_off_page, |
| 1670 |
GC_bool retry) |
| 1671 |
{ |
| 1672 |
GC_bool gc_not_stopped = TRUE; |
| 1673 |
word blocks_to_get; |
| 1674 |
IF_CANCEL(int cancel_state;) |
| 1675 |
|
| 1676 |
GC_ASSERT(I_HOLD_LOCK()); |
| 1677 |
DISABLE_CANCEL(cancel_state); |
| 1678 |
if (!GC_incremental && !GC_dont_gc && |
| 1679 |
((GC_dont_expand && GC_bytes_allocd > 0) |
| 1680 |
|| (GC_fo_entries > last_fo_entries |
| 1681 |
&& (last_bytes_finalized | GC_bytes_finalized) != 0 |
| 1682 |
&& (GC_fo_entries - last_fo_entries) |
| 1683 |
* GC_allocd_bytes_per_finalizer > GC_bytes_allocd) |
| 1684 |
|| GC_should_collect())) { |
| 1685 |
/* Try to do a full collection using 'default' stop_func (unless */ |
| 1686 |
/* nothing has been allocated since the latest collection or heap */ |
| 1687 |
/* expansion is disabled). */ |
| 1688 |
gc_not_stopped = GC_try_to_collect_inner( |
| 1689 |
GC_bytes_allocd > 0 && (!GC_dont_expand || !retry) ? |
| 1690 |
GC_default_stop_func : GC_never_stop_func); |
| 1691 |
if (gc_not_stopped == TRUE || !retry) { |
| 1692 |
/* Either the collection hasn't been aborted or this is the */ |
| 1693 |
/* first attempt (in a loop). */ |
| 1694 |
last_fo_entries = GC_fo_entries; |
| 1695 |
last_bytes_finalized = GC_bytes_finalized; |
| 1696 |
RESTORE_CANCEL(cancel_state); |
| 1697 |
return(TRUE); |
| 1698 |
} |
| 1699 |
} |
| 1700 |
|
| 1701 |
blocks_to_get = (GC_heapsize - GC_heapsize_at_forced_unmap) |
| 1702 |
/ (HBLKSIZE * GC_free_space_divisor) |
| 1703 |
+ needed_blocks; |
| 1704 |
if (blocks_to_get > MAXHINCR) { |
| 1705 |
word slop; |
| 1706 |
|
| 1707 |
/* Get the minimum required to make it likely that we can satisfy */ |
| 1708 |
/* the current request in the presence of black-listing. */ |
| 1709 |
/* This will probably be more than MAXHINCR. */ |
| 1710 |
if (ignore_off_page) { |
| 1711 |
slop = 4; |
| 1712 |
} else { |
| 1713 |
slop = 2 * divHBLKSZ(BL_LIMIT); |
| 1714 |
if (slop > needed_blocks) slop = needed_blocks; |
| 1715 |
} |
| 1716 |
if (needed_blocks + slop > MAXHINCR) { |
| 1717 |
blocks_to_get = needed_blocks + slop; |
| 1718 |
} else { |
| 1719 |
blocks_to_get = MAXHINCR; |
| 1720 |
} |
| 1721 |
if (blocks_to_get > divHBLKSZ(GC_WORD_MAX)) |
| 1722 |
blocks_to_get = divHBLKSZ(GC_WORD_MAX); |
| 1723 |
} |
| 1724 |
|
| 1725 |
if (!GC_expand_hp_inner(blocks_to_get) |
| 1726 |
&& (blocks_to_get == needed_blocks |
| 1727 |
|| !GC_expand_hp_inner(needed_blocks))) { |
| 1728 |
if (gc_not_stopped == FALSE) { |
| 1729 |
/* Don't increment GC_fail_count here (and no warning). */ |
| 1730 |
GC_gcollect_inner(); |
| 1731 |
GC_ASSERT(GC_bytes_allocd == 0); |
| 1732 |
} else if (GC_fail_count++ < GC_max_retries) { |
| 1733 |
WARN("Out of Memory! Trying to continue...\n", 0); |
| 1734 |
GC_gcollect_inner(); |
| 1735 |
} else { |
| 1736 |
# if !defined(AMIGA) || !defined(GC_AMIGA_FASTALLOC) |
| 1737 |
# ifdef USE_MUNMAP |
| 1738 |
GC_ASSERT(GC_heapsize >= GC_unmapped_bytes); |
| 1739 |
# endif |
| 1740 |
WARN("Out of Memory! Heap size: %" WARN_PRIuPTR " MiB." |
| 1741 |
" Returning NULL!\n", (GC_heapsize - GC_unmapped_bytes) >> 20); |
| 1742 |
# endif |
| 1743 |
RESTORE_CANCEL(cancel_state); |
| 1744 |
return(FALSE); |
| 1745 |
} |
| 1746 |
} else if (GC_fail_count) { |
| 1747 |
GC_COND_LOG_PRINTF("Memory available again...\n"); |
| 1748 |
} |
| 1749 |
RESTORE_CANCEL(cancel_state); |
| 1750 |
return(TRUE); |
| 1751 |
} |
| 1752 |
|
| 1753 |
/* |
| 1754 |
* Make sure the object free list for size gran (in granules) is not empty. |
| 1755 |
* Return a pointer to the first object on the free list. |
| 1756 |
* The object MUST BE REMOVED FROM THE FREE LIST BY THE CALLER. |
| 1757 |
*/ |
| 1758 |
GC_INNER ptr_t GC_allocobj(size_t gran, int kind) |
| 1759 |
{ |
| 1760 |
void ** flh = &(GC_obj_kinds[kind].ok_freelist[gran]); |
| 1761 |
GC_bool tried_minor = FALSE; |
| 1762 |
GC_bool retry = FALSE; |
| 1763 |
|
| 1764 |
GC_ASSERT(I_HOLD_LOCK()); |
| 1765 |
if (gran == 0) return(0); |
| 1766 |
|
| 1767 |
while (*flh == 0) { |
| 1768 |
ENTER_GC(); |
| 1769 |
# ifndef GC_DISABLE_INCREMENTAL |
| 1770 |
if (GC_incremental && GC_time_limit != GC_TIME_UNLIMITED |
| 1771 |
&& !GC_dont_gc) { |
| 1772 |
/* True incremental mode, not just generational. */ |
| 1773 |
/* Do our share of marking work. */ |
| 1774 |
GC_collect_a_little_inner(1); |
| 1775 |
} |
| 1776 |
# endif |
| 1777 |
/* Sweep blocks for objects of this size */ |
| 1778 |
GC_ASSERT(!GC_is_full_gc |
| 1779 |
|| NULL == GC_obj_kinds[kind].ok_reclaim_list |
| 1780 |
|| NULL == GC_obj_kinds[kind].ok_reclaim_list[gran]); |
| 1781 |
GC_continue_reclaim(gran, kind); |
| 1782 |
EXIT_GC(); |
| 1783 |
# if defined(CPPCHECK) |
| 1784 |
GC_noop1((word)&flh); |
| 1785 |
# endif |
| 1786 |
if (NULL == *flh) { |
| 1787 |
GC_new_hblk(gran, kind); |
| 1788 |
# if defined(CPPCHECK) |
| 1789 |
GC_noop1((word)&flh); |
| 1790 |
# endif |
| 1791 |
if (NULL == *flh) { |
| 1792 |
ENTER_GC(); |
| 1793 |
if (GC_incremental && GC_time_limit == GC_TIME_UNLIMITED |
| 1794 |
&& !tried_minor && !GC_dont_gc) { |
| 1795 |
GC_collect_a_little_inner(1); |
| 1796 |
tried_minor = TRUE; |
| 1797 |
} else { |
| 1798 |
if (!GC_collect_or_expand(1, FALSE, retry)) { |
| 1799 |
EXIT_GC(); |
| 1800 |
return(0); |
| 1801 |
} |
| 1802 |
retry = TRUE; |
| 1803 |
} |
| 1804 |
EXIT_GC(); |
| 1805 |
} |
| 1806 |
} |
| 1807 |
} |
| 1808 |
/* Successful allocation; reset failure count. */ |
| 1809 |
GC_fail_count = 0; |
| 1810 |
|
| 1811 |
return (ptr_t)(*flh); |
| 1812 |
} |