Debian Codegraph @

gc/reclaim.c

raw
1 /*
2 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
3 * Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved.
4 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
5 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
6 * Copyright (c) 2009-2021 Ivan Maidanski
7 *
8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 *
11 * Permission is hereby granted to use or copy this program
12 * for any purpose, provided the above notices are retained on all copies.
13 * Permission to modify the code and to distribute modified code is granted,
14 * provided the above notices are retained, and a notice that the code was
15 * modified is included with the above copyright notice.
16 */
17
18 #include "private/gc_priv.h"
19
20 #ifdef ENABLE_DISCLAIM
21 # include "gc_disclaim.h"
22 #endif
23
24 #include <stdio.h>
25
26 GC_INNER signed_word GC_bytes_found = 0;
27 /* Number of bytes of memory reclaimed */
28 /* minus the number of bytes originally */
29 /* on free lists which we had to drop. */
30
31 #if defined(PARALLEL_MARK)
32 GC_INNER signed_word GC_fl_builder_count = 0;
33 /* Number of threads currently building free lists without */
34 /* holding GC lock. It is not safe to collect if this is */
35 /* nonzero. Also, together with the mark lock, it is used as */
36 /* a semaphore during marker threads startup. */
37 #endif /* PARALLEL_MARK */
38
39 /* We defer printing of leaked objects until we're done with the GC */
40 /* cycle, since the routine for printing objects needs to run outside */
41 /* the collector, e.g. without the allocation lock. */
42 #ifndef MAX_LEAKED
43 # define MAX_LEAKED 40
44 #endif
45 STATIC ptr_t GC_leaked[MAX_LEAKED] = { NULL };
46 STATIC unsigned GC_n_leaked = 0;
47
48 #ifdef AO_HAVE_store
49 GC_INNER volatile AO_t GC_have_errors = 0;
50 #else
51 GC_INNER GC_bool GC_have_errors = FALSE;
52 #endif
53
54 #if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
55 STATIC void GC_reclaim_unconditionally_marked(void);
56 #endif
57
58 GC_INLINE void GC_add_leaked(ptr_t leaked)
59 {
60 # ifndef SHORT_DBG_HDRS
61 if (GC_findleak_delay_free && !GC_check_leaked(leaked))
62 return;
63 # endif
64
65 GC_SET_HAVE_ERRORS();
66 if (GC_n_leaked < MAX_LEAKED) {
67 GC_leaked[GC_n_leaked++] = leaked;
68 /* Make sure it's not reclaimed this cycle */
69 GC_set_mark_bit(leaked);
70 }
71 }
72
73 /* Print all objects on the list after printing any smashed objects. */
74 /* Clear both lists. Called without the allocation lock held. */
75 GC_INNER void GC_print_all_errors(void)
76 {
77 static GC_bool printing_errors = FALSE;
78 GC_bool have_errors;
79 unsigned i, n_leaked;
80 ptr_t leaked[MAX_LEAKED];
81 DCL_LOCK_STATE;
82
83 LOCK();
84 if (printing_errors) {
85 UNLOCK();
86 return;
87 }
88 have_errors = get_have_errors();
89 printing_errors = TRUE;
90 n_leaked = GC_n_leaked;
91 if (n_leaked > 0) {
92 GC_ASSERT(n_leaked <= MAX_LEAKED);
93 BCOPY(GC_leaked, leaked, n_leaked * sizeof(ptr_t));
94 GC_n_leaked = 0;
95 BZERO(GC_leaked, n_leaked * sizeof(ptr_t));
96 }
97 UNLOCK();
98
99 if (GC_debugging_started) {
100 GC_print_all_smashed();
101 } else {
102 have_errors = FALSE;
103 }
104
105 if (n_leaked > 0) {
106 GC_err_printf("Found %u leaked objects:\n", n_leaked);
107 have_errors = TRUE;
108 }
109 for (i = 0; i < n_leaked; i++) {
110 ptr_t p = leaked[i];
111 # ifndef SKIP_LEAKED_OBJECTS_PRINTING
112 GC_print_heap_obj(p);
113 # endif
114 GC_free(p);
115 }
116
117 if (have_errors
118 # ifndef GC_ABORT_ON_LEAK
119 && GETENV("GC_ABORT_ON_LEAK") != NULL
120 # endif
121 ) {
122 ABORT("Leaked or smashed objects encountered");
123 }
124
125 LOCK();
126 printing_errors = FALSE;
127 UNLOCK();
128 }
129
130
131 /*
132 * reclaim phase
133 *
134 */
135
136 /* Test whether a block is completely empty, i.e. contains no marked */
137 /* objects. This does not require the block to be in physical memory. */
138 GC_INNER GC_bool GC_block_empty(hdr *hhdr)
139 {
140 return (hhdr -> hb_n_marks == 0);
141 }
142
143 STATIC GC_bool GC_block_nearly_full(hdr *hhdr, word sz)
144 {
145 return hhdr -> hb_n_marks > HBLK_OBJS(sz) * 7 / 8;
146 }
147
148 /* TODO: This should perhaps again be specialized for USE_MARK_BYTES */
149 /* and USE_MARK_BITS cases. */
150
151 GC_INLINE word *GC_clear_block(word *p, word sz, signed_word *count)
152 {
153 word *q = (word *)((ptr_t)p + sz);
154
155 /* Clear object, advance p to next object in the process. */
156 # ifdef USE_MARK_BYTES
157 GC_ASSERT((sz & 1) == 0);
158 GC_ASSERT(((word)p & (2 * sizeof(word) - 1)) == 0);
159 p[1] = 0;
160 p += 2;
161 while ((word)p < (word)q) {
162 CLEAR_DOUBLE(p);
163 p += 2;
164 }
165 # else
166 p++; /* Skip link field */
167 while ((word)p < (word)q) {
168 *p++ = 0;
169 }
170 # endif
171 *count += sz;
172 return p;
173 }
174
175 /*
176 * Restore unmarked small objects in h of size sz to the object
177 * free list. Returns the new list.
178 * Clears unmarked objects. Sz is in bytes.
179 */
180 STATIC ptr_t GC_reclaim_clear(struct hblk *hbp, hdr *hhdr, word sz,
181 ptr_t list, signed_word *count)
182 {
183 word bit_no = 0;
184 ptr_t p, plim;
185
186 GC_ASSERT(hhdr == GC_find_header((ptr_t)hbp));
187 # ifndef THREADS
188 GC_ASSERT(sz == hhdr -> hb_sz);
189 # else
190 /* Skip the assertion because of a potential race with GC_realloc. */
191 # endif
192 GC_ASSERT((sz & (BYTES_PER_WORD-1)) == 0);
193 p = hbp->hb_body;
194 plim = p + HBLKSIZE - sz;
195
196 /* go through all words in block */
197 while ((word)p <= (word)plim) {
198 if (mark_bit_from_hdr(hhdr, bit_no)) {
199 p += sz;
200 } else {
201 /* Object is available - put it on list. */
202 obj_link(p) = list;
203 list = p;
204
205 p = (ptr_t)GC_clear_block((word *)p, sz, count);
206 }
207 bit_no += MARK_BIT_OFFSET(sz);
208 }
209 return list;
210 }
211
212 /* The same thing, but don't clear objects: */
213 STATIC ptr_t GC_reclaim_uninit(struct hblk *hbp, hdr *hhdr, word sz,
214 ptr_t list, signed_word *count)
215 {
216 word bit_no = 0;
217 word *p, *plim;
218 signed_word n_bytes_found = 0;
219
220 # ifndef THREADS
221 GC_ASSERT(sz == hhdr -> hb_sz);
222 # endif
223 p = (word *)(hbp->hb_body);
224 plim = (word *)((ptr_t)hbp + HBLKSIZE - sz);
225
226 /* go through all words in block */
227 while ((word)p <= (word)plim) {
228 if (!mark_bit_from_hdr(hhdr, bit_no)) {
229 n_bytes_found += sz;
230 /* object is available - put on list */
231 obj_link(p) = list;
232 list = ((ptr_t)p);
233 }
234 p = (word *)((ptr_t)p + sz);
235 bit_no += MARK_BIT_OFFSET(sz);
236 }
237 *count += n_bytes_found;
238 return(list);
239 }
240
241 #ifdef ENABLE_DISCLAIM
242 /* Call reclaim notifier for block's kind on each unmarked object in */
243 /* block, all within a pair of corresponding enter/leave callbacks. */
244 STATIC ptr_t GC_disclaim_and_reclaim(struct hblk *hbp, hdr *hhdr, word sz,
245 ptr_t list, signed_word *count)
246 {
247 word bit_no = 0;
248 ptr_t p, plim;
249 struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind];
250 int (GC_CALLBACK *disclaim)(void *) = ok->ok_disclaim_proc;
251
252 GC_ASSERT(disclaim != 0);
253 # ifndef THREADS
254 GC_ASSERT(sz == hhdr -> hb_sz);
255 # endif
256 p = hbp->hb_body;
257 plim = p + HBLKSIZE - sz;
258
259 for (; (word)p <= (word)plim; bit_no += MARK_BIT_OFFSET(sz)) {
260 if (mark_bit_from_hdr(hhdr, bit_no)) {
261 p += sz;
262 } else if ((*disclaim)(p)) {
263 set_mark_bit_from_hdr(hhdr, bit_no);
264 hhdr -> hb_n_marks++;
265 p += sz;
266 } else {
267 obj_link(p) = list;
268 list = p;
269 p = (ptr_t)GC_clear_block((word *)p, sz, count);
270 }
271 }
272 return list;
273 }
274 #endif /* ENABLE_DISCLAIM */
275
276 /* Don't really reclaim objects, just check for unmarked ones: */
277 STATIC void GC_reclaim_check(struct hblk *hbp, hdr *hhdr, word sz)
278 {
279 word bit_no;
280 ptr_t p, plim;
281
282 # ifndef THREADS
283 GC_ASSERT(sz == hhdr -> hb_sz);
284 # endif
285 /* go through all words in block */
286 p = hbp->hb_body;
287 plim = p + HBLKSIZE - sz;
288 for (bit_no = 0; (word)p <= (word)plim;
289 p += sz, bit_no += MARK_BIT_OFFSET(sz)) {
290 if (!mark_bit_from_hdr(hhdr, bit_no)) {
291 GC_add_leaked(p);
292 }
293 }
294 }
295
296 /* Is a pointer-free block? Same as IS_PTRFREE macro (in os_dep.c) but */
297 /* uses unordered atomic access to avoid racing with GC_realloc. */
298 #ifdef AO_HAVE_load
299 # define IS_PTRFREE_SAFE(hhdr) \
300 (AO_load((volatile AO_t *)&(hhdr)->hb_descr) == 0)
301 #else
302 /* No race as GC_realloc holds the lock while updating hb_descr. */
303 # define IS_PTRFREE_SAFE(hhdr) ((hhdr)->hb_descr == 0)
304 #endif
305
306 /*
307 * Generic procedure to rebuild a free list in hbp.
308 * Also called directly from GC_malloc_many.
309 * Sz is now in bytes.
310 */
311 GC_INNER ptr_t GC_reclaim_generic(struct hblk * hbp, hdr *hhdr, size_t sz,
312 GC_bool init, ptr_t list,
313 signed_word *count)
314 {
315 ptr_t result;
316
317 GC_ASSERT(GC_find_header((ptr_t)hbp) == hhdr);
318 # ifndef GC_DISABLE_INCREMENTAL
319 GC_remove_protection(hbp, 1, IS_PTRFREE_SAFE(hhdr));
320 # endif
321 # ifdef ENABLE_DISCLAIM
322 if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
323 result = GC_disclaim_and_reclaim(hbp, hhdr, sz, list, count);
324 } else
325 # endif
326 /* else */ if (init || GC_debugging_started) {
327 result = GC_reclaim_clear(hbp, hhdr, sz, list, count);
328 } else {
329 GC_ASSERT(IS_PTRFREE_SAFE(hhdr));
330 result = GC_reclaim_uninit(hbp, hhdr, sz, list, count);
331 }
332 if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) GC_set_hdr_marks(hhdr);
333 return result;
334 }
335
336 /*
337 * Restore unmarked small objects in the block pointed to by hbp
338 * to the appropriate object free list.
339 * If entirely empty blocks are to be completely deallocated, then
340 * caller should perform that check.
341 */
342 STATIC void GC_reclaim_small_nonempty_block(struct hblk *hbp, word sz,
343 GC_bool report_if_found)
344 {
345 hdr *hhdr = HDR(hbp);
346 struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
347 void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);
348
349 hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
350
351 if (report_if_found) {
352 GC_reclaim_check(hbp, hhdr, sz);
353 } else {
354 *flh = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init,
355 (ptr_t)(*flh), &GC_bytes_found);
356 }
357 }
358
359 #ifdef ENABLE_DISCLAIM
360 STATIC void GC_disclaim_and_reclaim_or_free_small_block(struct hblk *hbp)
361 {
362 hdr *hhdr = HDR(hbp);
363 word sz = hhdr -> hb_sz;
364 struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
365 void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);
366 void *flh_next;
367
368 hhdr -> hb_last_reclaimed = (unsigned short) GC_gc_no;
369 flh_next = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init,
370 (ptr_t)(*flh), &GC_bytes_found);
371 if (hhdr -> hb_n_marks)
372 *flh = flh_next;
373 else {
374 GC_bytes_found += HBLKSIZE;
375 GC_freehblk(hbp);
376 }
377 }
378 #endif /* ENABLE_DISCLAIM */
379
380 /*
381 * Restore an unmarked large object or an entirely empty blocks of small objects
382 * to the heap block free list.
383 * Otherwise enqueue the block for later processing
384 * by GC_reclaim_small_nonempty_block.
385 * If report_if_found is TRUE, then process any block immediately, and
386 * simply report free objects; do not actually reclaim them.
387 */
388 STATIC void GC_reclaim_block(struct hblk *hbp, word report_if_found)
389 {
390 hdr * hhdr = HDR(hbp);
391 word sz; /* size of objects in current block */
392 struct obj_kind * ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
393
394 # ifdef AO_HAVE_load
395 /* Atomic access is used to avoid racing with GC_realloc. */
396 sz = (word)AO_load((volatile AO_t *)&hhdr->hb_sz);
397 # else
398 /* No race as GC_realloc holds the lock while updating hb_sz. */
399 sz = hhdr -> hb_sz;
400 # endif
401 if( sz > MAXOBJBYTES ) { /* 1 big object */
402 if( !mark_bit_from_hdr(hhdr, 0) ) {
403 if (report_if_found) {
404 GC_add_leaked((ptr_t)hbp);
405 } else {
406 word blocks;
407
408 # ifdef ENABLE_DISCLAIM
409 if (EXPECT(hhdr->hb_flags & HAS_DISCLAIM, 0)) {
410 if ((*ok->ok_disclaim_proc)(hbp)) {
411 /* Not disclaimed => resurrect the object. */
412 set_mark_bit_from_hdr(hhdr, 0);
413 goto in_use;
414 }
415 }
416 # endif
417 blocks = OBJ_SZ_TO_BLOCKS(sz);
418 # if defined(CPPCHECK)
419 GC_noop1((word)&blocks);
420 # endif
421 if (blocks > 1) {
422 GC_large_allocd_bytes -= blocks * HBLKSIZE;
423 }
424 GC_bytes_found += sz;
425 GC_freehblk(hbp);
426 }
427 } else {
428 # ifdef ENABLE_DISCLAIM
429 in_use:
430 # endif
431 if (IS_PTRFREE_SAFE(hhdr)) {
432 GC_atomic_in_use += sz;
433 } else {
434 GC_composite_in_use += sz;
435 }
436 }
437 } else {
438 GC_bool empty = GC_block_empty(hhdr);
439 # ifdef PARALLEL_MARK
440 /* Count can be low or one too high because we sometimes */
441 /* have to ignore decrements. Objects can also potentially */
442 /* be repeatedly marked by each marker. */
443 /* Here we assume 3 markers at most, but this is extremely */
444 /* unlikely to fail spuriously with more. And if it does, it */
445 /* should be looked at. */
446 GC_ASSERT(sz != 0 && (GC_markers_m1 > 1 ? 3 : GC_markers_m1 + 1)
447 * (HBLKSIZE/sz + 1) + 16 >= hhdr->hb_n_marks);
448 # else
449 GC_ASSERT(sz * hhdr -> hb_n_marks <= HBLKSIZE);
450 # endif
451 if (report_if_found) {
452 GC_reclaim_small_nonempty_block(hbp, sz,
453 TRUE /* report_if_found */);
454 } else if (empty) {
455 # ifdef ENABLE_DISCLAIM
456 if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
457 GC_disclaim_and_reclaim_or_free_small_block(hbp);
458 } else
459 # endif
460 /* else */ {
461 GC_bytes_found += HBLKSIZE;
462 GC_freehblk(hbp);
463 }
464 } else if (GC_find_leak || !GC_block_nearly_full(hhdr, sz)) {
465 /* group of smaller objects, enqueue the real work */
466 struct hblk **rlh = ok -> ok_reclaim_list;
467
468 if (rlh != NULL) {
469 rlh += BYTES_TO_GRANULES(sz);
470 hhdr -> hb_next = *rlh;
471 *rlh = hbp;
472 }
473 } /* else not worth salvaging. */
474 /* We used to do the nearly_full check later, but we */
475 /* already have the right cache context here. Also */
476 /* doing it here avoids some silly lock contention in */
477 /* GC_malloc_many. */
478 if (IS_PTRFREE_SAFE(hhdr)) {
479 GC_atomic_in_use += sz * hhdr -> hb_n_marks;
480 } else {
481 GC_composite_in_use += sz * hhdr -> hb_n_marks;
482 }
483 }
484 }
485
486 #if !defined(NO_DEBUGGING)
487 /* Routines to gather and print heap block info */
488 /* intended for debugging. Otherwise should be called */
489 /* with lock. */
490
491 struct Print_stats
492 {
493 size_t number_of_blocks;
494 size_t total_bytes;
495 };
496
497 #ifdef USE_MARK_BYTES
498
499 /* Return the number of set mark bits in the given header. */
500 /* Remains externally visible as used by GNU GCJ currently. */
501 unsigned GC_n_set_marks(hdr *hhdr)
502 {
503 unsigned result = 0;
504 word i;
505 word sz = hhdr -> hb_sz;
506 word offset = MARK_BIT_OFFSET(sz);
507 word limit = FINAL_MARK_BIT(sz);
508
509 for (i = 0; i < limit; i += offset) {
510 result += hhdr -> hb_marks[i];
511 }
512 GC_ASSERT(hhdr -> hb_marks[limit]);
513 return(result);
514 }
515
516 #else
517
518 /* Number of set bits in a word. Not performance critical. */
519 static unsigned set_bits(word n)
520 {
521 word m = n;
522 unsigned result = 0;
523
524 while (m > 0) {
525 if (m & 1) result++;
526 m >>= 1;
527 }
528 return(result);
529 }
530
531 unsigned GC_n_set_marks(hdr *hhdr)
532 {
533 unsigned result = 0;
534 word i;
535 word n_mark_words;
536 # ifdef MARK_BIT_PER_OBJ
537 word n_objs = HBLK_OBJS(hhdr -> hb_sz);
538
539 if (0 == n_objs) n_objs = 1;
540 n_mark_words = divWORDSZ(n_objs + WORDSZ - 1);
541 # else /* MARK_BIT_PER_GRANULE */
542 n_mark_words = MARK_BITS_SZ;
543 # endif
544 for (i = 0; i < n_mark_words - 1; i++) {
545 result += set_bits(hhdr -> hb_marks[i]);
546 }
547 # ifdef MARK_BIT_PER_OBJ
548 result += set_bits((hhdr -> hb_marks[n_mark_words - 1])
549 << (n_mark_words * WORDSZ - n_objs));
550 # else
551 result += set_bits(hhdr -> hb_marks[n_mark_words - 1]);
552 # endif
553 return result; /* the number of set bits excluding the one past the end */
554 }
555
556 #endif /* !USE_MARK_BYTES */
557
558 STATIC void GC_print_block_descr(struct hblk *h,
559 word /* struct PrintStats */ raw_ps)
560 {
561 hdr * hhdr = HDR(h);
562 size_t bytes = hhdr -> hb_sz;
563 struct Print_stats *ps;
564 unsigned n_marks = GC_n_set_marks(hhdr);
565 unsigned n_objs = (unsigned)HBLK_OBJS(bytes);
566
567 if (0 == n_objs) n_objs = 1;
568 if (hhdr -> hb_n_marks != n_marks) {
569 GC_printf("%u,%u,%u!=%u,%u\n", hhdr->hb_obj_kind, (unsigned)bytes,
570 (unsigned)hhdr->hb_n_marks, n_marks, n_objs);
571 } else {
572 GC_printf("%u,%u,%u,%u\n", hhdr->hb_obj_kind, (unsigned)bytes,
573 n_marks, n_objs);
574 }
575
576 ps = (struct Print_stats *)raw_ps;
577 ps->total_bytes +=
578 (bytes + HBLKSIZE-1) & ~(word)(HBLKSIZE-1); /* round up */
579 ps->number_of_blocks++;
580 }
581
582 void GC_print_block_list(void)
583 {
584 struct Print_stats pstats;
585
586 GC_printf("kind(0=ptrfree,1=normal,2=unc.),"
587 "size_in_bytes,#_marks_set,#objs\n");
588 pstats.number_of_blocks = 0;
589 pstats.total_bytes = 0;
590 GC_apply_to_all_blocks(GC_print_block_descr, (word)&pstats);
591 GC_printf("blocks= %lu, bytes= %lu\n",
592 (unsigned long)pstats.number_of_blocks,
593 (unsigned long)pstats.total_bytes);
594 }
595
596 #include "gc_inline.h" /* for GC_print_free_list prototype */
597
598 /* Currently for debugger use only: */
599 GC_API void GC_CALL GC_print_free_list(int kind, size_t sz_in_granules)
600 {
601 void *flh_next;
602 int n;
603
604 GC_ASSERT(kind < MAXOBJKINDS);
605 GC_ASSERT(sz_in_granules <= MAXOBJGRANULES);
606 flh_next = GC_obj_kinds[kind].ok_freelist[sz_in_granules];
607 for (n = 0; flh_next; n++) {
608 GC_printf("Free object in heap block %p [%d]: %p\n",
609 (void *)HBLKPTR(flh_next), n, flh_next);
610 flh_next = obj_link(flh_next);
611 }
612 }
613
614 #endif /* !NO_DEBUGGING */
615
616 /*
617 * Clear all obj_link pointers in the list of free objects *flp.
618 * Clear *flp.
619 * This must be done before dropping a list of free gcj-style objects,
620 * since may otherwise end up with dangling "descriptor" pointers.
621 * It may help for other pointer-containing objects.
622 */
623 STATIC void GC_clear_fl_links(void **flp)
624 {
625 void *next = *flp;
626
627 while (0 != next) {
628 *flp = 0;
629 flp = &(obj_link(next));
630 next = *flp;
631 }
632 }
633
634 /*
635 * Perform GC_reclaim_block on the entire heap, after first clearing
636 * small object free lists (if we are not just looking for leaks).
637 */
638 GC_INNER void GC_start_reclaim(GC_bool report_if_found)
639 {
640 unsigned kind;
641
642 # if defined(PARALLEL_MARK)
643 GC_ASSERT(0 == GC_fl_builder_count);
644 # endif
645 /* Reset in use counters. GC_reclaim_block recomputes them. */
646 GC_composite_in_use = 0;
647 GC_atomic_in_use = 0;
648 /* Clear reclaim- and free-lists */
649 for (kind = 0; kind < GC_n_kinds; kind++) {
650 struct hblk ** rlist = GC_obj_kinds[kind].ok_reclaim_list;
651 GC_bool should_clobber = (GC_obj_kinds[kind].ok_descriptor != 0);
652
653 if (rlist == 0) continue; /* This kind not used. */
654 if (!report_if_found) {
655 void **fop;
656 void **lim = &(GC_obj_kinds[kind].ok_freelist[MAXOBJGRANULES+1]);
657
658 for (fop = GC_obj_kinds[kind].ok_freelist;
659 (word)fop < (word)lim; (*(word **)&fop)++) {
660 if (*fop != 0) {
661 if (should_clobber) {
662 GC_clear_fl_links(fop);
663 } else {
664 *fop = 0;
665 }
666 }
667 }
668 } /* otherwise free list objects are marked, */
669 /* and it's safe to leave them. */
670 BZERO(rlist, (MAXOBJGRANULES + 1) * sizeof(void *));
671 }
672
673
674 /* Go through all heap blocks (in hblklist) and reclaim unmarked objects */
675 /* or enqueue the block for later processing. */
676 GC_apply_to_all_blocks(GC_reclaim_block, (word)report_if_found);
677
678 # ifdef EAGER_SWEEP
679 /* This is a very stupid thing to do. We make it possible anyway, */
680 /* so that you can convince yourself that it really is very stupid. */
681 GC_reclaim_all((GC_stop_func)0, FALSE);
682 # elif defined(ENABLE_DISCLAIM)
683 /* However, make sure to clear reclaimable objects of kinds with */
684 /* unconditional marking enabled before we do any significant */
685 /* marking work. */
686 GC_reclaim_unconditionally_marked();
687 # endif
688 # if defined(PARALLEL_MARK)
689 GC_ASSERT(0 == GC_fl_builder_count);
690 # endif
691
692 }
693
694 /*
695 * Sweep blocks of the indicated object size and kind until either the
696 * appropriate free list is nonempty, or there are no more blocks to
697 * sweep.
698 */
699 GC_INNER void GC_continue_reclaim(word sz /* granules */, int kind)
700 {
701 hdr * hhdr;
702 struct hblk * hbp;
703 struct obj_kind * ok = &(GC_obj_kinds[kind]);
704 struct hblk ** rlh = ok -> ok_reclaim_list;
705 void **flh = &(ok -> ok_freelist[sz]);
706
707 if (NULL == rlh)
708 return; /* No blocks of this kind. */
709
710 for (rlh += sz; (hbp = *rlh) != NULL; ) {
711 hhdr = HDR(hbp);
712 *rlh = hhdr -> hb_next;
713 GC_reclaim_small_nonempty_block(hbp, hhdr -> hb_sz, FALSE);
714 if (*flh != 0)
715 break;
716 }
717 }
718
719 /*
720 * Reclaim all small blocks waiting to be reclaimed.
721 * Abort and return FALSE when/if (*stop_func)() returns TRUE.
722 * If this returns TRUE, then it's safe to restart the world
723 * with incorrectly cleared mark bits.
724 * If ignore_old is TRUE, then reclaim only blocks that have been
725 * recently reclaimed, and discard the rest.
726 * Stop_func may be 0.
727 */
728 GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old)
729 {
730 word sz;
731 unsigned kind;
732 hdr * hhdr;
733 struct hblk * hbp;
734 struct obj_kind * ok;
735 struct hblk ** rlp;
736 struct hblk ** rlh;
737 # ifndef NO_CLOCK
738 CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
739
740 if (GC_print_stats == VERBOSE)
741 GET_TIME(start_time);
742 # endif
743
744 for (kind = 0; kind < GC_n_kinds; kind++) {
745 ok = &(GC_obj_kinds[kind]);
746 rlp = ok -> ok_reclaim_list;
747 if (rlp == 0) continue;
748 for (sz = 1; sz <= MAXOBJGRANULES; sz++) {
749 for (rlh = rlp + sz; (hbp = *rlh) != NULL; ) {
750 if (stop_func != (GC_stop_func)0 && (*stop_func)()) {
751 return(FALSE);
752 }
753 hhdr = HDR(hbp);
754 *rlh = hhdr -> hb_next;
755 if (!ignore_old
756 || (word)hhdr->hb_last_reclaimed == GC_gc_no - 1) {
757 /* It's likely we'll need it this time, too */
758 /* It's been touched recently, so this */
759 /* shouldn't trigger paging. */
760 GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
761 }
762 }
763 }
764 }
765 # ifndef NO_CLOCK
766 if (GC_print_stats == VERBOSE) {
767 CLOCK_TYPE done_time;
768
769 GET_TIME(done_time);
770 GC_verbose_log_printf(
771 "Disposing of reclaim lists took %lu ms %lu ns\n",
772 MS_TIME_DIFF(done_time, start_time),
773 NS_FRAC_TIME_DIFF(done_time, start_time));
774 }
775 # endif
776 return(TRUE);
777 }
778
779 #if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
780 /* We do an eager sweep on heap blocks where unconditional marking has */
781 /* been enabled, so that any reclaimable objects have been reclaimed */
782 /* before we start marking. This is a simplified GC_reclaim_all */
783 /* restricted to kinds where ok_mark_unconditionally is true. */
784 STATIC void GC_reclaim_unconditionally_marked(void)
785 {
786 word sz;
787 unsigned kind;
788 hdr * hhdr;
789 struct hblk * hbp;
790 struct obj_kind * ok;
791 struct hblk ** rlp;
792 struct hblk ** rlh;
793
794 for (kind = 0; kind < GC_n_kinds; kind++) {
795 ok = &(GC_obj_kinds[kind]);
796 if (!ok->ok_mark_unconditionally)
797 continue;
798 rlp = ok->ok_reclaim_list;
799 if (rlp == 0)
800 continue;
801 for (sz = 1; sz <= MAXOBJGRANULES; sz++) {
802 rlh = rlp + sz;
803 while ((hbp = *rlh) != 0) {
804 hhdr = HDR(hbp);
805 *rlh = hhdr->hb_next;
806 GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
807 }
808 }
809 }
810 }
811 #endif /* !EAGER_SWEEP && ENABLE_DISCLAIM */
812
813 struct enumerate_reachable_s {
814 GC_reachable_object_proc proc;
815 void *client_data;
816 };
817
818 STATIC void GC_do_enumerate_reachable_objects(struct hblk *hbp, word ped)
819 {
820 struct hblkhdr *hhdr = HDR(hbp);
821 size_t sz = (size_t)hhdr->hb_sz;
822 size_t bit_no;
823 char *p, *plim;
824
825 if (GC_block_empty(hhdr)) {
826 return;
827 }
828
829 p = hbp->hb_body;
830 if (sz > MAXOBJBYTES) { /* one big object */
831 plim = p;
832 } else {
833 plim = hbp->hb_body + HBLKSIZE - sz;
834 }
835 /* Go through all words in block. */
836 for (bit_no = 0; (word)p <= (word)plim;
837 bit_no += MARK_BIT_OFFSET(sz), p += sz) {
838 if (mark_bit_from_hdr(hhdr, bit_no)) {
839 ((struct enumerate_reachable_s *)ped)->proc(p, sz,
840 ((struct enumerate_reachable_s *)ped)->client_data);
841 }
842 }
843 }
844
845 GC_API void GC_CALL GC_enumerate_reachable_objects_inner(
846 GC_reachable_object_proc proc,
847 void *client_data)
848 {
849 struct enumerate_reachable_s ed;
850
851 GC_ASSERT(I_HOLD_LOCK());
852 ed.proc = proc;
853 ed.client_data = client_data;
854 GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects, (word)&ed);
855 }