Debian Codegraph @

gc/os_dep.c

raw
1 /*
2 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
3 * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved.
4 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
5 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
6 * Copyright (c) 2008-2021 Ivan Maidanski
7 *
8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 *
11 * Permission is hereby granted to use or copy this program
12 * for any purpose, provided the above notices are retained on all copies.
13 * Permission to modify the code and to distribute modified code is granted,
14 * provided the above notices are retained, and a notice that the code was
15 * modified is included with the above copyright notice.
16 */
17
18 #include "private/gc_priv.h"
19
20 #if !defined(OS2) && !defined(PCR) && !defined(AMIGA) && !defined(MACOS) \
21 && !defined(MSWINCE) && !defined(SN_TARGET_ORBIS) \
22 && !defined(SN_TARGET_PSP2) && !defined(__CC_ARM)
23 # include <sys/types.h>
24 # if !defined(MSWIN32) && !defined(MSWIN_XBOX1)
25 # include <unistd.h>
26 # endif
27 #endif
28
29 #include <stdio.h>
30 #if defined(MSWINCE) || defined(SN_TARGET_PS3)
31 # define SIGSEGV 0 /* value is irrelevant */
32 #else
33 # include <signal.h>
34 #endif
35
36 #if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(NACL) \
37 || defined(SYMBIAN)
38 # include <fcntl.h>
39 #endif
40
41 #if defined(LINUX) || defined(LINUX_STACKBOTTOM)
42 # include <ctype.h>
43 #endif
44
45 /* Blatantly OS dependent routines, except for those that are related */
46 /* to dynamic loading. */
47
48 #ifdef AMIGA
49 # define GC_AMIGA_DEF
50 # include "extra/AmigaOS.c"
51 # undef GC_AMIGA_DEF
52 #endif
53
54 #ifdef MACOS
55 # include <Processes.h>
56 #endif
57
58 #ifdef IRIX5
59 # include <sys/uio.h>
60 # include <malloc.h> /* for locking */
61 #endif
62
63 #if defined(MMAP_SUPPORTED) || defined(ADD_HEAP_GUARD_PAGES)
64 # if defined(USE_MUNMAP) && !defined(USE_MMAP) && !defined(CPPCHECK)
65 # error Invalid config: USE_MUNMAP requires USE_MMAP
66 # endif
67 # include <sys/types.h>
68 # include <sys/mman.h>
69 # include <sys/stat.h>
70 #endif
71
72 #if defined(ADD_HEAP_GUARD_PAGES) || defined(LINUX_STACKBOTTOM) \
73 || defined(MMAP_SUPPORTED) || defined(NEED_PROC_MAPS)
74 # include <errno.h>
75 #endif
76
77 #ifdef DARWIN
78 /* for get_etext and friends */
79 # include <mach-o/getsect.h>
80 #endif
81
82 #ifdef DJGPP
83 /* Apparently necessary for djgpp 2.01. May cause problems with */
84 /* other versions. */
85 typedef long unsigned int caddr_t;
86 #endif
87
88 #ifdef PCR
89 # include "il/PCR_IL.h"
90 # include "th/PCR_ThCtl.h"
91 # include "mm/PCR_MM.h"
92 #endif
93
94 #if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB)
95 /* Declare GC_mprotect_stop and GC_mprotect_resume as extern "C". */
96 # include "private/darwin_stop_world.h"
97 #endif
98
99 #if !defined(NO_EXECUTE_PERMISSION)
100 STATIC GC_bool GC_pages_executable = TRUE;
101 #else
102 STATIC GC_bool GC_pages_executable = FALSE;
103 #endif
104 #define IGNORE_PAGES_EXECUTABLE 1
105 /* Undefined on GC_pages_executable real use. */
106
107 #if ((defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS) \
108 || defined(PROC_VDB) || defined(SOFT_VDB)) && !defined(PROC_READ)) \
109 || defined(CPPCHECK)
110 # define PROC_READ read
111 /* Should probably call the real read, if read is wrapped. */
112 #endif
113
114 #if defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS)
115 /* Repeatedly perform a read call until the buffer is filled */
116 /* up, or we encounter EOF or an error. */
117 STATIC ssize_t GC_repeat_read(int fd, char *buf, size_t count)
118 {
119 size_t num_read = 0;
120
121 ASSERT_CANCEL_DISABLED();
122 while (num_read < count) {
123 ssize_t result = PROC_READ(fd, buf + num_read, count - num_read);
124
125 if (result < 0) return result;
126 if (result == 0) break;
127 num_read += result;
128 }
129 return num_read;
130 }
131 #endif /* LINUX_STACKBOTTOM || NEED_PROC_MAPS */
132
133 #ifdef NEED_PROC_MAPS
134 /* We need to parse /proc/self/maps, either to find dynamic libraries, */
135 /* and/or to find the register backing store base (IA64). Do it once */
136 /* here. */
137
138 #ifdef THREADS
139 /* Determine the length of a file by incrementally reading it into a */
140 /* buffer. This would be silly to use it on a file supporting lseek, */
141 /* but Linux /proc files usually do not. */
142 /* As of Linux 4.15.0, lseek(SEEK_END) fails for /proc/self/maps. */
143 STATIC size_t GC_get_file_len(int f)
144 {
145 size_t total = 0;
146 ssize_t result;
147 # define GET_FILE_LEN_BUF_SZ 500
148 char buf[GET_FILE_LEN_BUF_SZ];
149
150 do {
151 result = PROC_READ(f, buf, sizeof(buf));
152 if (result == -1) return 0;
153 total += result;
154 } while (result > 0);
155 return total;
156 }
157
158 STATIC size_t GC_get_maps_len(void)
159 {
160 int f = open("/proc/self/maps", O_RDONLY);
161 size_t result;
162 if (f < 0) return 0; /* treat missing file as empty */
163 result = GC_get_file_len(f);
164 close(f);
165 return result;
166 }
167 #endif /* THREADS */
168
169 /* Copy the contents of /proc/self/maps to a buffer in our address */
170 /* space. Return the address of the buffer. */
171 GC_INNER const char * GC_get_maps(void)
172 {
173 ssize_t result;
174 static char *maps_buf = NULL;
175 static size_t maps_buf_sz = 1;
176 size_t maps_size;
177 # ifdef THREADS
178 size_t old_maps_size = 0;
179 # endif
180
181 /* The buffer is essentially static, so there must be a single client. */
182 GC_ASSERT(I_HOLD_LOCK());
183
184 /* Note that in the presence of threads, the maps file can */
185 /* essentially shrink asynchronously and unexpectedly as */
186 /* threads that we already think of as dead release their */
187 /* stacks. And there is no easy way to read the entire */
188 /* file atomically. This is arguably a misfeature of the */
189 /* /proc/self/maps interface. */
190 /* Since we expect the file can grow asynchronously in rare */
191 /* cases, it should suffice to first determine */
192 /* the size (using read), and then to reread the file. */
193 /* If the size is inconsistent we have to retry. */
194 /* This only matters with threads enabled, and if we use */
195 /* this to locate roots (not the default). */
196
197 # ifdef THREADS
198 /* Determine the initial size of /proc/self/maps. */
199 maps_size = GC_get_maps_len();
200 if (0 == maps_size)
201 ABORT("Cannot determine length of /proc/self/maps");
202 # else
203 maps_size = 4000; /* Guess */
204 # endif
205
206 /* Read /proc/self/maps, growing maps_buf as necessary. */
207 /* Note that we may not allocate conventionally, and */
208 /* thus can't use stdio. */
209 do {
210 int f;
211
212 while (maps_size >= maps_buf_sz) {
213 # ifdef LINT2
214 /* Workaround passing tainted maps_buf to a tainted sink. */
215 GC_noop1((word)maps_buf);
216 # else
217 GC_scratch_recycle_no_gww(maps_buf, maps_buf_sz);
218 # endif
219 /* Grow only by powers of 2, since we leak "too small" buffers.*/
220 while (maps_size >= maps_buf_sz) maps_buf_sz *= 2;
221 maps_buf = GC_scratch_alloc(maps_buf_sz);
222 if (NULL == maps_buf)
223 ABORT_ARG1("Insufficient space for /proc/self/maps buffer",
224 ", %lu bytes requested", (unsigned long)maps_buf_sz);
225 # ifdef THREADS
226 /* Recompute initial length, since we allocated. */
227 /* This can only happen a few times per program */
228 /* execution. */
229 maps_size = GC_get_maps_len();
230 if (0 == maps_size)
231 ABORT("Cannot determine length of /proc/self/maps");
232 # endif
233 }
234 GC_ASSERT(maps_buf_sz >= maps_size + 1);
235 f = open("/proc/self/maps", O_RDONLY);
236 if (-1 == f)
237 ABORT_ARG1("Cannot open /proc/self/maps",
238 ": errno= %d", errno);
239 # ifdef THREADS
240 old_maps_size = maps_size;
241 # endif
242 maps_size = 0;
243 do {
244 result = GC_repeat_read(f, maps_buf, maps_buf_sz-1);
245 if (result < 0) {
246 ABORT_ARG1("Failed to read /proc/self/maps",
247 ": errno= %d", errno);
248 }
249 maps_size += result;
250 } while ((size_t)result == maps_buf_sz-1);
251 close(f);
252 if (0 == maps_size)
253 ABORT("Empty /proc/self/maps");
254 # ifdef THREADS
255 if (maps_size > old_maps_size) {
256 /* This might be caused by e.g. thread creation. */
257 WARN("Unexpected asynchronous /proc/self/maps growth"
258 " (to %" WARN_PRIuPTR " bytes)\n", maps_size);
259 }
260 # endif
261 } while (maps_size >= maps_buf_sz
262 # ifdef THREADS
263 || maps_size < old_maps_size
264 # endif
265 );
266 maps_buf[maps_size] = '\0';
267 return maps_buf;
268 }
269
270 /*
271 * GC_parse_map_entry parses an entry from /proc/self/maps so we can
272 * locate all writable data segments that belong to shared libraries.
273 * The format of one of these entries and the fields we care about
274 * is as follows:
275 * XXXXXXXX-XXXXXXXX r-xp 00000000 30:05 260537 name of mapping...\n
276 * ^^^^^^^^ ^^^^^^^^ ^^^^ ^^
277 * start end prot maj_dev
278 *
279 * Note that since about august 2003 kernels, the columns no longer have
280 * fixed offsets on 64-bit kernels. Hence we no longer rely on fixed offsets
281 * anywhere, which is safer anyway.
282 */
283
284 /* Assign various fields of the first line in maps_ptr to (*start), */
285 /* (*end), (*prot), (*maj_dev) and (*mapping_name). mapping_name may */
286 /* be NULL. (*prot) and (*mapping_name) are assigned pointers into the */
287 /* original buffer. */
288 #if (defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES)) \
289 || defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
290 || (defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS))
291 GC_INNER const char *GC_parse_map_entry(const char *maps_ptr,
292 ptr_t *start, ptr_t *end,
293 const char **prot, unsigned *maj_dev,
294 const char **mapping_name)
295 {
296 const unsigned char *start_start, *end_start, *maj_dev_start;
297 const unsigned char *p; /* unsigned for isspace, isxdigit */
298
299 if (maps_ptr == NULL || *maps_ptr == '\0') {
300 return NULL;
301 }
302
303 p = (const unsigned char *)maps_ptr;
304 while (isspace(*p)) ++p;
305 start_start = p;
306 GC_ASSERT(isxdigit(*start_start));
307 *start = (ptr_t)strtoul((const char *)start_start, (char **)&p, 16);
308 GC_ASSERT(*p=='-');
309
310 ++p;
311 end_start = p;
312 GC_ASSERT(isxdigit(*end_start));
313 *end = (ptr_t)strtoul((const char *)end_start, (char **)&p, 16);
314 GC_ASSERT(isspace(*p));
315
316 while (isspace(*p)) ++p;
317 GC_ASSERT(*p == 'r' || *p == '-');
318 *prot = (const char *)p;
319 /* Skip past protection field to offset field */
320 while (!isspace(*p)) ++p;
321 while (isspace(*p)) p++;
322 GC_ASSERT(isxdigit(*p));
323 /* Skip past offset field, which we ignore */
324 while (!isspace(*p)) ++p;
325 while (isspace(*p)) p++;
326 maj_dev_start = p;
327 GC_ASSERT(isxdigit(*maj_dev_start));
328 *maj_dev = strtoul((const char *)maj_dev_start, NULL, 16);
329
330 if (mapping_name != NULL) {
331 while (*p && *p != '\n' && *p != '/' && *p != '[') p++;
332 *mapping_name = (const char *)p;
333 }
334 while (*p && *p++ != '\n');
335 return (const char *)p;
336 }
337 #endif /* REDIRECT_MALLOC || DYNAMIC_LOADING || IA64 || ... */
338
339 #if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR)
340 /* Try to read the backing store base from /proc/self/maps. */
341 /* Return the bounds of the writable mapping with a 0 major device, */
342 /* which includes the address passed as data. */
343 /* Return FALSE if there is no such mapping. */
344 GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr, ptr_t *startp,
345 ptr_t *endp)
346 {
347 const char *prot;
348 ptr_t my_start, my_end;
349 unsigned int maj_dev;
350 const char *maps_ptr = GC_get_maps();
351
352 for (;;) {
353 maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end,
354 &prot, &maj_dev, 0);
355 if (NULL == maps_ptr) break;
356
357 if (prot[1] == 'w' && maj_dev == 0
358 && (word)my_end > (word)addr && (word)my_start <= (word)addr) {
359 *startp = my_start;
360 *endp = my_end;
361 return TRUE;
362 }
363 }
364 return FALSE;
365 }
366 #endif /* IA64 || INCLUDE_LINUX_THREAD_DESCR */
367
368 #if defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS)
369 /* Find the text(code) mapping for the library whose name, after */
370 /* stripping the directory part, starts with nm. */
371 GC_INNER GC_bool GC_text_mapping(char *nm, ptr_t *startp, ptr_t *endp)
372 {
373 size_t nm_len = strlen(nm);
374 const char *prot, *map_path;
375 ptr_t my_start, my_end;
376 unsigned int maj_dev;
377 const char *maps_ptr = GC_get_maps();
378
379 for (;;) {
380 maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end,
381 &prot, &maj_dev, &map_path);
382 if (NULL == maps_ptr) break;
383
384 if (prot[0] == 'r' && prot[1] == '-' && prot[2] == 'x') {
385 const char *p = map_path;
386
387 /* Set p to point just past last slash, if any. */
388 while (*p != '\0' && *p != '\n' && *p != ' ' && *p != '\t') ++p;
389 while ((word)p >= (word)map_path && *p != '/') --p;
390 ++p;
391 if (strncmp(nm, p, nm_len) == 0) {
392 *startp = my_start;
393 *endp = my_end;
394 return TRUE;
395 }
396 }
397 }
398 return FALSE;
399 }
400 #endif /* REDIRECT_MALLOC */
401
402 #ifdef IA64
403 static ptr_t backing_store_base_from_proc(void)
404 {
405 ptr_t my_start, my_end;
406 if (!GC_enclosing_mapping(GC_save_regs_in_stack(), &my_start, &my_end)) {
407 GC_COND_LOG_PRINTF("Failed to find backing store base from /proc\n");
408 return 0;
409 }
410 return my_start;
411 }
412 #endif
413
414 #endif /* NEED_PROC_MAPS */
415
416 #if defined(SEARCH_FOR_DATA_START)
417 /* The x86 case can be handled without a search. The Alpha case */
418 /* used to be handled differently as well, but the rules changed */
419 /* for recent Linux versions. This seems to be the easiest way to */
420 /* cover all versions. */
421
422 # if defined(LINUX) || defined(HURD)
423 /* Some Linux distributions arrange to define __data_start. Some */
424 /* define data_start as a weak symbol. The latter is technically */
425 /* broken, since the user program may define data_start, in which */
426 /* case we lose. Nonetheless, we try both, preferring __data_start.*/
427 /* We assume gcc-compatible pragmas. */
428 EXTERN_C_BEGIN
429 # pragma weak __data_start
430 # pragma weak data_start
431 extern int __data_start[], data_start[];
432 EXTERN_C_END
433 # endif /* LINUX */
434
435 ptr_t GC_data_start = NULL;
436
437 GC_INNER void GC_init_linux_data_start(void)
438 {
439 ptr_t data_end = DATAEND;
440
441 # if (defined(LINUX) || defined(HURD)) && defined(USE_PROG_DATA_START)
442 /* Try the easy approaches first: */
443 /* However, this may lead to wrong data start value if libgc */
444 /* code is put into a shared library (directly or indirectly) */
445 /* which is linked with -Bsymbolic-functions option. Thus, */
446 /* the following is not used by default. */
447 if (COVERT_DATAFLOW(__data_start) != 0) {
448 GC_data_start = (ptr_t)(__data_start);
449 } else {
450 GC_data_start = (ptr_t)(data_start);
451 }
452 if (COVERT_DATAFLOW(GC_data_start) != 0) {
453 if ((word)GC_data_start > (word)data_end)
454 ABORT_ARG2("Wrong __data_start/_end pair",
455 ": %p .. %p", (void *)GC_data_start, (void *)data_end);
456 return;
457 }
458 # ifdef DEBUG_ADD_DEL_ROOTS
459 GC_log_printf("__data_start not provided\n");
460 # endif
461 # endif /* LINUX */
462
463 if (GC_no_dls) {
464 /* Not needed, avoids the SIGSEGV caused by */
465 /* GC_find_limit which complicates debugging. */
466 GC_data_start = data_end; /* set data root size to 0 */
467 return;
468 }
469
470 GC_data_start = (ptr_t)GC_find_limit(data_end, FALSE);
471 }
472 #endif /* SEARCH_FOR_DATA_START */
473
474 #ifdef ECOS
475
476 # ifndef ECOS_GC_MEMORY_SIZE
477 # define ECOS_GC_MEMORY_SIZE (448 * 1024)
478 # endif /* ECOS_GC_MEMORY_SIZE */
479
480 /* TODO: This is a simple way of allocating memory which is */
481 /* compatible with ECOS early releases. Later releases use a more */
482 /* sophisticated means of allocating memory than this simple static */
483 /* allocator, but this method is at least bound to work. */
484 static char ecos_gc_memory[ECOS_GC_MEMORY_SIZE];
485 static char *ecos_gc_brk = ecos_gc_memory;
486
487 static void *tiny_sbrk(ptrdiff_t increment)
488 {
489 void *p = ecos_gc_brk;
490 ecos_gc_brk += increment;
491 if ((word)ecos_gc_brk > (word)(ecos_gc_memory + sizeof(ecos_gc_memory))) {
492 ecos_gc_brk -= increment;
493 return NULL;
494 }
495 return p;
496 }
497 # define sbrk tiny_sbrk
498 #endif /* ECOS */
499
500 #if defined(NETBSD) && defined(__ELF__)
501 ptr_t GC_data_start = NULL;
502
503 EXTERN_C_BEGIN
504 extern char **environ;
505 EXTERN_C_END
506
507 GC_INNER void GC_init_netbsd_elf(void)
508 {
509 /* This may need to be environ, without the underscore, for */
510 /* some versions. */
511 GC_data_start = (ptr_t)GC_find_limit(&environ, FALSE);
512 }
513 #endif /* NETBSD */
514
515 #if defined(ADDRESS_SANITIZER) && (defined(UNIX_LIKE) \
516 || defined(NEED_FIND_LIMIT) || defined(MPROTECT_VDB)) \
517 && !defined(CUSTOM_ASAN_DEF_OPTIONS)
518 EXTERN_C_BEGIN
519 GC_API const char *__asan_default_options(void);
520 EXTERN_C_END
521
522 /* To tell ASan to allow GC to use its own SIGBUS/SEGV handlers. */
523 /* The function is exported just to be visible to ASan library. */
524 GC_API const char *__asan_default_options(void)
525 {
526 return "allow_user_segv_handler=1";
527 }
528 #endif
529
530 #ifdef OPENBSD
531 static struct sigaction old_segv_act;
532 STATIC JMP_BUF GC_jmp_buf_openbsd;
533
534 STATIC void GC_fault_handler_openbsd(int sig GC_ATTR_UNUSED)
535 {
536 LONGJMP(GC_jmp_buf_openbsd, 1);
537 }
538
539 # ifdef GC_OPENBSD_UTHREADS
540 # include <sys/syscall.h>
541 EXTERN_C_BEGIN
542 extern sigset_t __syscall(quad_t, ...);
543 EXTERN_C_END
544
545 /* Don't use GC_find_limit() because siglongjmp() outside of the */
546 /* signal handler by-passes our userland pthreads lib, leaving */
547 /* SIGSEGV and SIGPROF masked. Instead, use this custom one that */
548 /* works-around the issues. */
549
550 /* Return the first non-addressable location > p or bound. */
551 /* Requires the allocation lock. */
552 STATIC ptr_t GC_find_limit_openbsd(ptr_t p, ptr_t bound)
553 {
554 static volatile ptr_t result;
555 /* Safer if static, since otherwise it may not be */
556 /* preserved across the longjmp. Can safely be */
557 /* static since it's only called with the */
558 /* allocation lock held. */
559
560 struct sigaction act;
561 word pgsz = (word)sysconf(_SC_PAGESIZE);
562
563 GC_ASSERT((word)bound >= pgsz);
564 GC_ASSERT(I_HOLD_LOCK());
565
566 act.sa_handler = GC_fault_handler_openbsd;
567 sigemptyset(&act.sa_mask);
568 act.sa_flags = SA_NODEFER | SA_RESTART;
569 /* act.sa_restorer is deprecated and should not be initialized. */
570 sigaction(SIGSEGV, &act, &old_segv_act);
571
572 if (SETJMP(GC_jmp_buf_openbsd) == 0) {
573 result = (ptr_t)((word)p & ~(pgsz-1));
574 for (;;) {
575 if ((word)result >= (word)bound - pgsz) {
576 result = bound;
577 break;
578 }
579 result += pgsz; /* no overflow expected */
580 GC_noop1((word)(*result));
581 }
582 }
583
584 # ifdef THREADS
585 /* Due to the siglongjump we need to manually unmask SIGPROF. */
586 __syscall(SYS_sigprocmask, SIG_UNBLOCK, sigmask(SIGPROF));
587 # endif
588
589 sigaction(SIGSEGV, &old_segv_act, 0);
590 return(result);
591 }
592 # endif /* GC_OPENBSD_UTHREADS */
593
594 static volatile int firstpass;
595
596 /* Return first addressable location > p or bound. */
597 /* Requires the allocation lock. */
598 STATIC ptr_t GC_skip_hole_openbsd(ptr_t p, ptr_t bound)
599 {
600 static volatile ptr_t result;
601
602 struct sigaction act;
603 word pgsz = (word)sysconf(_SC_PAGESIZE);
604
605 GC_ASSERT((word)bound >= pgsz);
606 GC_ASSERT(I_HOLD_LOCK());
607
608 act.sa_handler = GC_fault_handler_openbsd;
609 sigemptyset(&act.sa_mask);
610 act.sa_flags = SA_NODEFER | SA_RESTART;
611 /* act.sa_restorer is deprecated and should not be initialized. */
612 sigaction(SIGSEGV, &act, &old_segv_act);
613
614 firstpass = 1;
615 result = (ptr_t)((word)p & ~(pgsz-1));
616 if (SETJMP(GC_jmp_buf_openbsd) != 0 || firstpass) {
617 firstpass = 0;
618 if ((word)result >= (word)bound - pgsz) {
619 result = bound;
620 } else {
621 result += pgsz; /* no overflow expected */
622 GC_noop1((word)(unsigned char)(*result));
623 }
624 }
625
626 sigaction(SIGSEGV, &old_segv_act, 0);
627 return(result);
628 }
629 #endif /* OPENBSD */
630
631 # ifdef OS2
632
633 # include <stddef.h>
634
635 # if !defined(__IBMC__) && !defined(__WATCOMC__) /* e.g. EMX */
636
637 struct exe_hdr {
638 unsigned short magic_number;
639 unsigned short padding[29];
640 long new_exe_offset;
641 };
642
643 #define E_MAGIC(x) (x).magic_number
644 #define EMAGIC 0x5A4D
645 #define E_LFANEW(x) (x).new_exe_offset
646
647 struct e32_exe {
648 unsigned char magic_number[2];
649 unsigned char byte_order;
650 unsigned char word_order;
651 unsigned long exe_format_level;
652 unsigned short cpu;
653 unsigned short os;
654 unsigned long padding1[13];
655 unsigned long object_table_offset;
656 unsigned long object_count;
657 unsigned long padding2[31];
658 };
659
660 #define E32_MAGIC1(x) (x).magic_number[0]
661 #define E32MAGIC1 'L'
662 #define E32_MAGIC2(x) (x).magic_number[1]
663 #define E32MAGIC2 'X'
664 #define E32_BORDER(x) (x).byte_order
665 #define E32LEBO 0
666 #define E32_WORDER(x) (x).word_order
667 #define E32LEWO 0
668 #define E32_CPU(x) (x).cpu
669 #define E32CPU286 1
670 #define E32_OBJTAB(x) (x).object_table_offset
671 #define E32_OBJCNT(x) (x).object_count
672
673 struct o32_obj {
674 unsigned long size;
675 unsigned long base;
676 unsigned long flags;
677 unsigned long pagemap;
678 unsigned long mapsize;
679 unsigned long reserved;
680 };
681
682 #define O32_FLAGS(x) (x).flags
683 #define OBJREAD 0x0001L
684 #define OBJWRITE 0x0002L
685 #define OBJINVALID 0x0080L
686 #define O32_SIZE(x) (x).size
687 #define O32_BASE(x) (x).base
688
689 # else /* IBM's compiler */
690
691 /* A kludge to get around what appears to be a header file bug */
692 # ifndef WORD
693 # define WORD unsigned short
694 # endif
695 # ifndef DWORD
696 # define DWORD unsigned long
697 # endif
698
699 # define EXE386 1
700 # include <newexe.h>
701 # include <exe386.h>
702
703 # endif /* __IBMC__ */
704
705 # define INCL_DOSEXCEPTIONS
706 # define INCL_DOSPROCESS
707 # define INCL_DOSERRORS
708 # define INCL_DOSMODULEMGR
709 # define INCL_DOSMEMMGR
710 # include <os2.h>
711
712 # endif /* OS/2 */
713
714 /* Find the page size */
715 GC_INNER size_t GC_page_size = 0;
716
717 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
718 # ifndef VER_PLATFORM_WIN32_CE
719 # define VER_PLATFORM_WIN32_CE 3
720 # endif
721
722 # if defined(MSWINCE) && defined(THREADS)
723 GC_INNER GC_bool GC_dont_query_stack_min = FALSE;
724 # endif
725
726 GC_INNER SYSTEM_INFO GC_sysinfo;
727
728 GC_INNER void GC_setpagesize(void)
729 {
730 GetSystemInfo(&GC_sysinfo);
731 # if defined(CYGWIN32) && (defined(MPROTECT_VDB) || defined(USE_MUNMAP))
732 /* Allocations made with mmap() are aligned to the allocation */
733 /* granularity, which (at least on Win64) is not the */
734 /* same as the page size. Probably a separate variable could */
735 /* be added to distinguish the allocation granularity from the */
736 /* actual page size, but in practice there is no good reason to */
737 /* make allocations smaller than dwAllocationGranularity, so we */
738 /* just use it instead of the actual page size here (as Cygwin */
739 /* itself does in many cases). */
740 GC_page_size = (size_t)GC_sysinfo.dwAllocationGranularity;
741 GC_ASSERT(GC_page_size >= (size_t)GC_sysinfo.dwPageSize);
742 # else
743 GC_page_size = (size_t)GC_sysinfo.dwPageSize;
744 # endif
745 # if defined(MSWINCE) && !defined(_WIN32_WCE_EMULATION)
746 {
747 OSVERSIONINFO verInfo;
748 /* Check the current WinCE version. */
749 verInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
750 if (!GetVersionEx(&verInfo))
751 ABORT("GetVersionEx failed");
752 if (verInfo.dwPlatformId == VER_PLATFORM_WIN32_CE &&
753 verInfo.dwMajorVersion < 6) {
754 /* Only the first 32 MB of address space belongs to the */
755 /* current process (unless WinCE 6.0+ or emulation). */
756 GC_sysinfo.lpMaximumApplicationAddress = (LPVOID)((word)32 << 20);
757 # ifdef THREADS
758 /* On some old WinCE versions, it's observed that */
759 /* VirtualQuery calls don't work properly when used to */
760 /* get thread current stack committed minimum. */
761 if (verInfo.dwMajorVersion < 5)
762 GC_dont_query_stack_min = TRUE;
763 # endif
764 }
765 }
766 # endif
767 }
768
769 # ifndef CYGWIN32
770 # define is_writable(prot) ((prot) == PAGE_READWRITE \
771 || (prot) == PAGE_WRITECOPY \
772 || (prot) == PAGE_EXECUTE_READWRITE \
773 || (prot) == PAGE_EXECUTE_WRITECOPY)
774 /* Return the number of bytes that are writable starting at p. */
775 /* The pointer p is assumed to be page aligned. */
776 /* If base is not 0, *base becomes the beginning of the */
777 /* allocation region containing p. */
778 STATIC word GC_get_writable_length(ptr_t p, ptr_t *base)
779 {
780 MEMORY_BASIC_INFORMATION buf;
781 word result;
782 word protect;
783
784 result = VirtualQuery(p, &buf, sizeof(buf));
785 if (result != sizeof(buf)) ABORT("Weird VirtualQuery result");
786 if (base != 0) *base = (ptr_t)(buf.AllocationBase);
787 protect = buf.Protect & ~(word)(PAGE_GUARD | PAGE_NOCACHE);
788 if (!is_writable(protect)) {
789 return(0);
790 }
791 if (buf.State != MEM_COMMIT) return(0);
792 return(buf.RegionSize);
793 }
794
795 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
796 {
797 ptr_t trunc_sp;
798 word size;
799
800 /* Set page size if it is not ready (so client can use this */
801 /* function even before GC is initialized). */
802 if (!GC_page_size) GC_setpagesize();
803
804 trunc_sp = (ptr_t)((word)GC_approx_sp() & ~(word)(GC_page_size-1));
805 /* FIXME: This won't work if called from a deeply recursive */
806 /* client code (and the committed stack space has grown). */
807 size = GC_get_writable_length(trunc_sp, 0);
808 GC_ASSERT(size != 0);
809 sb -> mem_base = trunc_sp + size;
810 return GC_SUCCESS;
811 }
812 # else /* CYGWIN32 */
813 /* An alternate version for Cygwin (adapted from Dave Korn's */
814 /* gcc version of boehm-gc). */
815 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
816 {
817 # ifdef X86_64
818 sb -> mem_base = ((NT_TIB*)NtCurrentTeb())->StackBase;
819 # else
820 void * _tlsbase;
821
822 __asm__ ("movl %%fs:4, %0"
823 : "=r" (_tlsbase));
824 sb -> mem_base = _tlsbase;
825 # endif
826 return GC_SUCCESS;
827 }
828 # endif /* CYGWIN32 */
829 # define HAVE_GET_STACK_BASE
830
831 #else /* !MSWIN32 */
832 GC_INNER void GC_setpagesize(void)
833 {
834 # if defined(MPROTECT_VDB) || defined(PROC_VDB) || defined(SOFT_VDB) \
835 || defined(USE_MMAP)
836 GC_page_size = (size_t)GETPAGESIZE();
837 # if !defined(CPPCHECK)
838 if (0 == GC_page_size)
839 ABORT("getpagesize failed");
840 # endif
841 # else
842 /* It's acceptable to fake it. */
843 GC_page_size = HBLKSIZE;
844 # endif
845 }
846 #endif /* !MSWIN32 */
847
848 #ifdef HAIKU
849 # include <kernel/OS.h>
850
851 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
852 {
853 thread_info th;
854 get_thread_info(find_thread(NULL),&th);
855 sb->mem_base = th.stack_end;
856 return GC_SUCCESS;
857 }
858 # define HAVE_GET_STACK_BASE
859 #endif /* HAIKU */
860
861 #ifdef OS2
862 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
863 {
864 PTIB ptib; /* thread information block */
865 PPIB ppib;
866 if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
867 WARN("DosGetInfoBlocks failed\n", 0);
868 return GC_UNIMPLEMENTED;
869 }
870 sb->mem_base = ptib->tib_pstacklimit;
871 return GC_SUCCESS;
872 }
873 # define HAVE_GET_STACK_BASE
874 #endif /* OS2 */
875
876 # ifdef AMIGA
877 # define GC_AMIGA_SB
878 # include "extra/AmigaOS.c"
879 # undef GC_AMIGA_SB
880 # define GET_MAIN_STACKBASE_SPECIAL
881 # endif /* AMIGA */
882
883 # if defined(NEED_FIND_LIMIT) \
884 || (defined(UNIX_LIKE) && !defined(NO_DEBUGGING)) \
885 || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
886 || (defined(WRAP_MARK_SOME) && defined(__GNUC__))
887
888 typedef void (*GC_fault_handler_t)(int);
889
890 # if defined(SUNOS5SIGS) || defined(IRIX5) || defined(OSF1) \
891 || defined(HAIKU) || defined(HURD) || defined(FREEBSD) \
892 || defined(NETBSD)
893 static struct sigaction old_segv_act;
894 # if defined(_sigargs) /* !Irix6.x */ || defined(HPUX) \
895 || defined(HURD) || defined(NETBSD) || defined(FREEBSD)
896 static struct sigaction old_bus_act;
897 # endif
898 # elif !defined(OPENBSD)
899 static GC_fault_handler_t old_segv_handler;
900 # ifdef HAVE_SIGBUS
901 static GC_fault_handler_t old_bus_handler;
902 # endif
903 # endif
904
905 GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t h)
906 {
907 # if defined(SUNOS5SIGS) || defined(IRIX5) || defined(OSF1) \
908 || defined(HAIKU) || defined(HURD) || defined(FREEBSD) \
909 || defined(NETBSD) || defined(OPENBSD)
910 struct sigaction act;
911
912 act.sa_handler = h;
913 # ifdef SIGACTION_FLAGS_NODEFER_HACK
914 /* Was necessary for Solaris 2.3 and very temporary */
915 /* NetBSD bugs. */
916 act.sa_flags = SA_RESTART | SA_NODEFER;
917 # else
918 act.sa_flags = SA_RESTART;
919 # endif
920
921 (void) sigemptyset(&act.sa_mask);
922 /* act.sa_restorer is deprecated and should not be initialized. */
923 # ifdef GC_IRIX_THREADS
924 /* Older versions have a bug related to retrieving and */
925 /* and setting a handler at the same time. */
926 (void) sigaction(SIGSEGV, 0, &old_segv_act);
927 (void) sigaction(SIGSEGV, &act, 0);
928 # else
929 (void) sigaction(SIGSEGV, &act, &old_segv_act);
930 # if defined(IRIX5) && defined(_sigargs) /* Irix 5.x, not 6.x */ \
931 || defined(HPUX) || defined(HURD) || defined(NETBSD) \
932 || defined(FREEBSD)
933 /* Under Irix 5.x or HP/UX, we may get SIGBUS. */
934 /* Pthreads doesn't exist under Irix 5.x, so we */
935 /* don't have to worry in the threads case. */
936 (void) sigaction(SIGBUS, &act, &old_bus_act);
937 # endif
938 # endif /* !GC_IRIX_THREADS */
939 # else
940 old_segv_handler = signal(SIGSEGV, h);
941 # ifdef HAVE_SIGBUS
942 old_bus_handler = signal(SIGBUS, h);
943 # endif
944 # endif
945 # if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
946 GC_noop1((word)&__asan_default_options);
947 # endif
948 }
949 # endif /* NEED_FIND_LIMIT || UNIX_LIKE */
950
951 # if defined(NEED_FIND_LIMIT) \
952 || (defined(WRAP_MARK_SOME) && defined(__GNUC__)) \
953 || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))
954 /* Some tools to implement HEURISTIC2 */
955 # define MIN_PAGE_SIZE 256 /* Smallest conceivable page size, bytes */
956
957 GC_INNER JMP_BUF GC_jmp_buf;
958
959 STATIC void GC_fault_handler(int sig GC_ATTR_UNUSED)
960 {
961 LONGJMP(GC_jmp_buf, 1);
962 }
963
964 GC_INNER void GC_setup_temporary_fault_handler(void)
965 {
966 /* Handler is process-wide, so this should only happen in */
967 /* one thread at a time. */
968 GC_ASSERT(I_HOLD_LOCK());
969 GC_set_and_save_fault_handler(GC_fault_handler);
970 }
971
972 GC_INNER void GC_reset_fault_handler(void)
973 {
974 # if defined(SUNOS5SIGS) || defined(IRIX5) || defined(OSF1) \
975 || defined(HAIKU) || defined(HURD) || defined(FREEBSD) \
976 || defined(NETBSD) || defined(OPENBSD)
977 (void) sigaction(SIGSEGV, &old_segv_act, 0);
978 # if defined(IRIX5) && defined(_sigargs) /* Irix 5.x, not 6.x */ \
979 || defined(HPUX) || defined(HURD) || defined(NETBSD) \
980 || defined(FREEBSD)
981 (void) sigaction(SIGBUS, &old_bus_act, 0);
982 # endif
983 # else
984 (void) signal(SIGSEGV, old_segv_handler);
985 # ifdef HAVE_SIGBUS
986 (void) signal(SIGBUS, old_bus_handler);
987 # endif
988 # endif
989 }
990
991 /* Return the first non-addressable location > p (up) or */
992 /* the smallest location q s.t. [q,p) is addressable (!up). */
993 /* We assume that p (up) or p-1 (!up) is addressable. */
994 /* Requires allocation lock. */
995 GC_ATTR_NO_SANITIZE_ADDR
996 STATIC ptr_t GC_find_limit_with_bound(ptr_t p, GC_bool up, ptr_t bound)
997 {
998 static volatile ptr_t result;
999 /* Safer if static, since otherwise it may not be */
1000 /* preserved across the longjmp. Can safely be */
1001 /* static since it's only called with the */
1002 /* allocation lock held. */
1003
1004 GC_ASSERT(up ? (word)bound >= MIN_PAGE_SIZE
1005 : (word)bound <= ~(word)MIN_PAGE_SIZE);
1006 GC_ASSERT(I_HOLD_LOCK());
1007 GC_setup_temporary_fault_handler();
1008 if (SETJMP(GC_jmp_buf) == 0) {
1009 result = (ptr_t)((word)p & ~(word)(MIN_PAGE_SIZE-1));
1010 for (;;) {
1011 if (up) {
1012 if ((word)result >= (word)bound - MIN_PAGE_SIZE) {
1013 result = bound;
1014 break;
1015 }
1016 result += MIN_PAGE_SIZE; /* no overflow expected */
1017 } else {
1018 if ((word)result <= (word)bound + MIN_PAGE_SIZE) {
1019 result = bound - MIN_PAGE_SIZE;
1020 /* This is to compensate */
1021 /* further result increment (we */
1022 /* do not modify "up" variable */
1023 /* since it might be clobbered */
1024 /* by setjmp otherwise). */
1025 break;
1026 }
1027 result -= MIN_PAGE_SIZE; /* no underflow expected */
1028 }
1029 GC_noop1((word)(unsigned char)(*result));
1030 }
1031 }
1032 GC_reset_fault_handler();
1033 if (!up) {
1034 result += MIN_PAGE_SIZE;
1035 }
1036 return(result);
1037 }
1038
1039 void * GC_find_limit(void * p, int up)
1040 {
1041 return GC_find_limit_with_bound((ptr_t)p, (GC_bool)up,
1042 up ? (ptr_t)GC_WORD_MAX : 0);
1043 }
1044 # endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES */
1045
1046 #ifdef HPUX_MAIN_STACKBOTTOM
1047 # include <sys/param.h>
1048 # include <sys/pstat.h>
1049
1050 STATIC ptr_t GC_hpux_main_stack_base(void)
1051 {
1052 struct pst_vm_status vm_status;
1053 int i = 0;
1054
1055 while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
1056 if (vm_status.pst_type == PS_STACK)
1057 return (ptr_t)vm_status.pst_vaddr;
1058 }
1059
1060 /* Old way to get the stack bottom. */
1061 # ifdef STACK_GROWS_UP
1062 return (ptr_t)GC_find_limit(GC_approx_sp(), /* up= */ FALSE);
1063 # else /* not HP_PA */
1064 return (ptr_t)GC_find_limit(GC_approx_sp(), TRUE);
1065 # endif
1066 }
1067 #endif /* HPUX_MAIN_STACKBOTTOM */
1068
1069 #ifdef HPUX_STACKBOTTOM
1070
1071 #include <sys/param.h>
1072 #include <sys/pstat.h>
1073
1074 GC_INNER ptr_t GC_get_register_stack_base(void)
1075 {
1076 struct pst_vm_status vm_status;
1077
1078 int i = 0;
1079 while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
1080 if (vm_status.pst_type == PS_RSESTACK) {
1081 return (ptr_t) vm_status.pst_vaddr;
1082 }
1083 }
1084
1085 /* old way to get the register stackbottom */
1086 return (ptr_t)(((word)GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1)
1087 & ~(word)(BACKING_STORE_ALIGNMENT-1));
1088 }
1089
1090 #endif /* HPUX_STACK_BOTTOM */
1091
1092 #ifdef LINUX_STACKBOTTOM
1093
1094 # include <sys/types.h>
1095 # include <sys/stat.h>
1096
1097 # define STAT_SKIP 27 /* Number of fields preceding startstack */
1098 /* field in /proc/self/stat */
1099
1100 # ifdef USE_LIBC_PRIVATES
1101 EXTERN_C_BEGIN
1102 # pragma weak __libc_stack_end
1103 extern ptr_t __libc_stack_end;
1104 # ifdef IA64
1105 # pragma weak __libc_ia64_register_backing_store_base
1106 extern ptr_t __libc_ia64_register_backing_store_base;
1107 # endif
1108 EXTERN_C_END
1109 # endif
1110
1111 # ifdef IA64
1112 GC_INNER ptr_t GC_get_register_stack_base(void)
1113 {
1114 ptr_t result;
1115
1116 # ifdef USE_LIBC_PRIVATES
1117 if (0 != &__libc_ia64_register_backing_store_base
1118 && 0 != __libc_ia64_register_backing_store_base) {
1119 /* Glibc 2.2.4 has a bug such that for dynamically linked */
1120 /* executables __libc_ia64_register_backing_store_base is */
1121 /* defined but uninitialized during constructor calls. */
1122 /* Hence we check for both nonzero address and value. */
1123 return __libc_ia64_register_backing_store_base;
1124 }
1125 # endif
1126 result = backing_store_base_from_proc();
1127 if (0 == result) {
1128 result = (ptr_t)GC_find_limit(GC_save_regs_in_stack(), FALSE);
1129 /* This works better than a constant displacement heuristic. */
1130 }
1131 return result;
1132 }
1133 # endif /* IA64 */
1134
1135 STATIC ptr_t GC_linux_main_stack_base(void)
1136 {
1137 /* We read the stack bottom value from /proc/self/stat. We do this */
1138 /* using direct I/O system calls in order to avoid calling malloc */
1139 /* in case REDIRECT_MALLOC is defined. */
1140 # define STAT_BUF_SIZE 4096
1141 char stat_buf[STAT_BUF_SIZE];
1142 int f;
1143 word result;
1144 ssize_t i, buf_offset = 0, len;
1145
1146 /* First try the easy way. This should work for glibc 2.2 */
1147 /* This fails in a prelinked ("prelink" command) executable */
1148 /* since the correct value of __libc_stack_end never */
1149 /* becomes visible to us. The second test works around */
1150 /* this. */
1151 # ifdef USE_LIBC_PRIVATES
1152 if (0 != &__libc_stack_end && 0 != __libc_stack_end ) {
1153 # if defined(IA64)
1154 /* Some versions of glibc set the address 16 bytes too */
1155 /* low while the initialization code is running. */
1156 if (((word)__libc_stack_end & 0xfff) + 0x10 < 0x1000) {
1157 return __libc_stack_end + 0x10;
1158 } /* Otherwise it's not safe to add 16 bytes and we fall */
1159 /* back to using /proc. */
1160 # elif defined(SPARC)
1161 /* Older versions of glibc for 64-bit SPARC do not set this */
1162 /* variable correctly, it gets set to either zero or one. */
1163 if (__libc_stack_end != (ptr_t) (unsigned long)0x1)
1164 return __libc_stack_end;
1165 # else
1166 return __libc_stack_end;
1167 # endif
1168 }
1169 # endif
1170
1171 f = open("/proc/self/stat", O_RDONLY);
1172 if (-1 == f)
1173 ABORT_ARG1("Could not open /proc/self/stat", ": errno= %d", errno);
1174 len = GC_repeat_read(f, stat_buf, sizeof(stat_buf));
1175 if (len < 0)
1176 ABORT_ARG1("Failed to read /proc/self/stat",
1177 ": errno= %d", errno);
1178 close(f);
1179
1180 /* Skip the required number of fields. This number is hopefully */
1181 /* constant across all Linux implementations. */
1182 for (i = 0; i < STAT_SKIP; ++i) {
1183 while (buf_offset < len && isspace(stat_buf[buf_offset++])) {
1184 /* empty */
1185 }
1186 while (buf_offset < len && !isspace(stat_buf[buf_offset++])) {
1187 /* empty */
1188 }
1189 }
1190 /* Skip spaces. */
1191 while (buf_offset < len && isspace(stat_buf[buf_offset])) {
1192 buf_offset++;
1193 }
1194 /* Find the end of the number and cut the buffer there. */
1195 for (i = 0; buf_offset + i < len; i++) {
1196 if (!isdigit(stat_buf[buf_offset + i])) break;
1197 }
1198 if (buf_offset + i >= len) ABORT("Could not parse /proc/self/stat");
1199 stat_buf[buf_offset + i] = '\0';
1200
1201 result = (word)STRTOULL(&stat_buf[buf_offset], NULL, 10);
1202 if (result < 0x100000 || (result & (sizeof(word) - 1)) != 0)
1203 ABORT_ARG1("Absurd stack bottom value",
1204 ": 0x%lx", (unsigned long)result);
1205 return (ptr_t)result;
1206 }
1207 #endif /* LINUX_STACKBOTTOM */
1208
1209 #ifdef FREEBSD_STACKBOTTOM
1210 /* This uses an undocumented sysctl call, but at least one expert */
1211 /* believes it will stay. */
1212
1213 # include <unistd.h>
1214 # include <sys/types.h>
1215 # include <sys/sysctl.h>
1216
1217 STATIC ptr_t GC_freebsd_main_stack_base(void)
1218 {
1219 int nm[2] = {CTL_KERN, KERN_USRSTACK};
1220 ptr_t base;
1221 size_t len = sizeof(ptr_t);
1222 int r = sysctl(nm, 2, &base, &len, NULL, 0);
1223 if (r) ABORT("Error getting main stack base");
1224 return base;
1225 }
1226 #endif /* FREEBSD_STACKBOTTOM */
1227
1228 #if defined(ECOS) || defined(NOSYS)
1229 ptr_t GC_get_main_stack_base(void)
1230 {
1231 return STACKBOTTOM;
1232 }
1233 # define GET_MAIN_STACKBASE_SPECIAL
1234 #elif defined(SYMBIAN)
1235 EXTERN_C_BEGIN
1236 extern int GC_get_main_symbian_stack_base(void);
1237 EXTERN_C_END
1238
1239 ptr_t GC_get_main_stack_base(void)
1240 {
1241 return (ptr_t)GC_get_main_symbian_stack_base();
1242 }
1243 # define GET_MAIN_STACKBASE_SPECIAL
1244 #elif defined(EMSCRIPTEN)
1245
1246 # if defined(USE_EMSCRIPTEN_SCAN_STACK) && defined(EMSCRIPTEN_ASYNCIFY)
1247 /* According to the documentation, emscripten_scan_stack() is only */
1248 /* guaranteed to be available when building with ASYNCIFY. */
1249 # include <emscripten.h>
1250
1251 static void *emscripten_stack_base;
1252
1253 static void scan_stack_cb(void *begin, void *end)
1254 {
1255 (void)begin;
1256 emscripten_stack_base = end;
1257 }
1258 # else
1259 # include <emscripten/stack.h>
1260 # endif
1261
1262 ptr_t GC_get_main_stack_base(void)
1263 {
1264 # if defined(USE_EMSCRIPTEN_SCAN_STACK) && defined(EMSCRIPTEN_ASYNCIFY)
1265 emscripten_scan_stack(scan_stack_cb);
1266 return (ptr_t)emscripten_stack_base;
1267 # else
1268 return (ptr_t)emscripten_stack_get_base();
1269 # endif
1270 }
1271 # define GET_MAIN_STACKBASE_SPECIAL
1272 #elif !defined(AMIGA) && !defined(HAIKU) && !defined(OS2) \
1273 && !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) \
1274 && !defined(GC_OPENBSD_THREADS) \
1275 && (!defined(GC_SOLARIS_THREADS) || defined(_STRICT_STDC))
1276
1277 # if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
1278 && (defined(THREADS) || defined(USE_GET_STACKBASE_FOR_MAIN))
1279 # include <pthread.h>
1280 # ifdef HAVE_PTHREAD_NP_H
1281 # include <pthread_np.h> /* for pthread_attr_get_np() */
1282 # endif
1283 # elif defined(DARWIN) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
1284 /* We could use pthread_get_stackaddr_np even in case of a */
1285 /* single-threaded gclib (there is no -lpthread on Darwin). */
1286 # include <pthread.h>
1287 # undef STACKBOTTOM
1288 # define STACKBOTTOM (ptr_t)pthread_get_stackaddr_np(pthread_self())
1289 # endif
1290
1291 ptr_t GC_get_main_stack_base(void)
1292 {
1293 ptr_t result;
1294 # if (defined(HAVE_PTHREAD_ATTR_GET_NP) \
1295 || defined(HAVE_PTHREAD_GETATTR_NP)) \
1296 && (defined(USE_GET_STACKBASE_FOR_MAIN) \
1297 || (defined(THREADS) && !defined(REDIRECT_MALLOC)))
1298 pthread_attr_t attr;
1299 void *stackaddr;
1300 size_t size;
1301
1302 # ifdef HAVE_PTHREAD_ATTR_GET_NP
1303 if (pthread_attr_init(&attr) == 0
1304 && (pthread_attr_get_np(pthread_self(), &attr) == 0
1305 ? TRUE : (pthread_attr_destroy(&attr), FALSE)))
1306 # else /* HAVE_PTHREAD_GETATTR_NP */
1307 if (pthread_getattr_np(pthread_self(), &attr) == 0)
1308 # endif
1309 {
1310 if (pthread_attr_getstack(&attr, &stackaddr, &size) == 0
1311 && stackaddr != NULL) {
1312 (void)pthread_attr_destroy(&attr);
1313 # ifdef STACK_GROWS_DOWN
1314 stackaddr = (char *)stackaddr + size;
1315 # endif
1316 return (ptr_t)stackaddr;
1317 }
1318 (void)pthread_attr_destroy(&attr);
1319 }
1320 WARN("pthread_getattr_np or pthread_attr_getstack failed"
1321 " for main thread\n", 0);
1322 # endif
1323 # ifdef STACKBOTTOM
1324 result = STACKBOTTOM;
1325 # else
1326 # ifdef HEURISTIC1
1327 # define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1)
1328 # ifdef STACK_GROWS_DOWN
1329 result = (ptr_t)(((word)GC_approx_sp() + STACKBOTTOM_ALIGNMENT_M1)
1330 & ~(word)STACKBOTTOM_ALIGNMENT_M1);
1331 # else
1332 result = (ptr_t)((word)GC_approx_sp()
1333 & ~(word)STACKBOTTOM_ALIGNMENT_M1);
1334 # endif
1335 # elif defined(HPUX_MAIN_STACKBOTTOM)
1336 result = GC_hpux_main_stack_base();
1337 # elif defined(LINUX_STACKBOTTOM)
1338 result = GC_linux_main_stack_base();
1339 # elif defined(FREEBSD_STACKBOTTOM)
1340 result = GC_freebsd_main_stack_base();
1341 # elif defined(HEURISTIC2)
1342 {
1343 ptr_t sp = GC_approx_sp();
1344 # ifdef STACK_GROWS_DOWN
1345 result = (ptr_t)GC_find_limit(sp, TRUE);
1346 # if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
1347 if ((word)result > (word)HEURISTIC2_LIMIT
1348 && (word)sp < (word)HEURISTIC2_LIMIT) {
1349 result = HEURISTIC2_LIMIT;
1350 }
1351 # endif
1352 # else
1353 result = (ptr_t)GC_find_limit(sp, FALSE);
1354 # if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
1355 if ((word)result < (word)HEURISTIC2_LIMIT
1356 && (word)sp > (word)HEURISTIC2_LIMIT) {
1357 result = HEURISTIC2_LIMIT;
1358 }
1359 # endif
1360 # endif
1361 }
1362 # elif defined(STACK_NOT_SCANNED) || defined(CPPCHECK)
1363 result = NULL;
1364 # else
1365 # error None of HEURISTIC* and *STACKBOTTOM defined!
1366 # endif
1367 # if defined(STACK_GROWS_DOWN) && !defined(CPPCHECK)
1368 if (result == 0)
1369 result = (ptr_t)(signed_word)(-sizeof(ptr_t));
1370 # endif
1371 # endif
1372 # if !defined(CPPCHECK)
1373 GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)result);
1374 # endif
1375 return(result);
1376 }
1377 # define GET_MAIN_STACKBASE_SPECIAL
1378 #endif /* !AMIGA, !HAIKU, !OPENBSD, !OS2, !Windows */
1379
1380 #if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
1381 && defined(THREADS) && !defined(HAVE_GET_STACK_BASE)
1382 # include <pthread.h>
1383 # ifdef HAVE_PTHREAD_NP_H
1384 # include <pthread_np.h>
1385 # endif
1386
1387 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
1388 {
1389 pthread_attr_t attr;
1390 size_t size;
1391 # ifdef IA64
1392 DCL_LOCK_STATE;
1393 # endif
1394
1395 # ifdef HAVE_PTHREAD_ATTR_GET_NP
1396 if (pthread_attr_init(&attr) != 0)
1397 ABORT("pthread_attr_init failed");
1398 if (pthread_attr_get_np(pthread_self(), &attr) != 0) {
1399 WARN("pthread_attr_get_np failed\n", 0);
1400 (void)pthread_attr_destroy(&attr);
1401 return GC_UNIMPLEMENTED;
1402 }
1403 # else /* HAVE_PTHREAD_GETATTR_NP */
1404 if (pthread_getattr_np(pthread_self(), &attr) != 0) {
1405 WARN("pthread_getattr_np failed\n", 0);
1406 return GC_UNIMPLEMENTED;
1407 }
1408 # endif
1409 if (pthread_attr_getstack(&attr, &(b -> mem_base), &size) != 0) {
1410 ABORT("pthread_attr_getstack failed");
1411 }
1412 (void)pthread_attr_destroy(&attr);
1413 # ifdef STACK_GROWS_DOWN
1414 b -> mem_base = (char *)(b -> mem_base) + size;
1415 # endif
1416 # ifdef IA64
1417 /* We could try backing_store_base_from_proc, but that's safe */
1418 /* only if no mappings are being asynchronously created. */
1419 /* Subtracting the size from the stack base doesn't work for at */
1420 /* least the main thread. */
1421 LOCK();
1422 {
1423 IF_CANCEL(int cancel_state;)
1424 ptr_t bsp;
1425 ptr_t next_stack;
1426
1427 DISABLE_CANCEL(cancel_state);
1428 bsp = GC_save_regs_in_stack();
1429 next_stack = GC_greatest_stack_base_below(bsp);
1430 if (0 == next_stack) {
1431 b -> reg_base = GC_find_limit(bsp, FALSE);
1432 } else {
1433 /* Avoid walking backwards into preceding memory stack and */
1434 /* growing it. */
1435 b -> reg_base = GC_find_limit_with_bound(bsp, FALSE, next_stack);
1436 }
1437 RESTORE_CANCEL(cancel_state);
1438 }
1439 UNLOCK();
1440 # elif defined(E2K)
1441 b -> reg_base = NULL;
1442 # endif
1443 return GC_SUCCESS;
1444 }
1445 # define HAVE_GET_STACK_BASE
1446 #endif /* THREADS && (HAVE_PTHREAD_ATTR_GET_NP || HAVE_PTHREAD_GETATTR_NP) */
1447
1448 #if defined(GC_DARWIN_THREADS) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
1449 # include <pthread.h>
1450
1451 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
1452 {
1453 /* pthread_get_stackaddr_np() should return stack bottom (highest */
1454 /* stack address plus 1). */
1455 b->mem_base = pthread_get_stackaddr_np(pthread_self());
1456 GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)b->mem_base);
1457 return GC_SUCCESS;
1458 }
1459 # define HAVE_GET_STACK_BASE
1460 #endif /* GC_DARWIN_THREADS */
1461
1462 #ifdef GC_OPENBSD_THREADS
1463 # include <sys/signal.h>
1464 # include <pthread.h>
1465 # include <pthread_np.h>
1466
1467 /* Find the stack using pthread_stackseg_np(). */
1468 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
1469 {
1470 stack_t stack;
1471 if (pthread_stackseg_np(pthread_self(), &stack))
1472 ABORT("pthread_stackseg_np(self) failed");
1473 sb->mem_base = stack.ss_sp;
1474 return GC_SUCCESS;
1475 }
1476 # define HAVE_GET_STACK_BASE
1477 #endif /* GC_OPENBSD_THREADS */
1478
1479 #if defined(GC_SOLARIS_THREADS) && !defined(_STRICT_STDC)
1480
1481 # include <thread.h>
1482 # include <signal.h>
1483 # include <pthread.h>
1484
1485 /* These variables are used to cache ss_sp value for the primordial */
1486 /* thread (it's better not to call thr_stksegment() twice for this */
1487 /* thread - see JDK bug #4352906). */
1488 static pthread_t stackbase_main_self = 0;
1489 /* 0 means stackbase_main_ss_sp value is unset. */
1490 static void *stackbase_main_ss_sp = NULL;
1491
1492 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
1493 {
1494 stack_t s;
1495 pthread_t self = pthread_self();
1496
1497 if (self == stackbase_main_self)
1498 {
1499 /* If the client calls GC_get_stack_base() from the main thread */
1500 /* then just return the cached value. */
1501 b -> mem_base = stackbase_main_ss_sp;
1502 GC_ASSERT(b -> mem_base != NULL);
1503 return GC_SUCCESS;
1504 }
1505
1506 if (thr_stksegment(&s)) {
1507 /* According to the manual, the only failure error code returned */
1508 /* is EAGAIN meaning "the information is not available due to the */
1509 /* thread is not yet completely initialized or it is an internal */
1510 /* thread" - this shouldn't happen here. */
1511 ABORT("thr_stksegment failed");
1512 }
1513 /* s.ss_sp holds the pointer to the stack bottom. */
1514 GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)s.ss_sp);
1515
1516 if (!stackbase_main_self && thr_main() != 0)
1517 {
1518 /* Cache the stack bottom pointer for the primordial thread */
1519 /* (this is done during GC_init, so there is no race). */
1520 stackbase_main_ss_sp = s.ss_sp;
1521 stackbase_main_self = self;
1522 }
1523
1524 b -> mem_base = s.ss_sp;
1525 return GC_SUCCESS;
1526 }
1527 # define HAVE_GET_STACK_BASE
1528 #endif /* GC_SOLARIS_THREADS */
1529
1530 #ifdef GC_RTEMS_PTHREADS
1531 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
1532 {
1533 sb->mem_base = rtems_get_stack_bottom();
1534 return GC_SUCCESS;
1535 }
1536 # define HAVE_GET_STACK_BASE
1537 #endif /* GC_RTEMS_PTHREADS */
1538
1539 #ifndef HAVE_GET_STACK_BASE
1540 # ifdef NEED_FIND_LIMIT
1541 /* Retrieve the stack bottom. */
1542 /* Using the GC_find_limit version is risky. */
1543 /* On IA64, for example, there is no guard page between the */
1544 /* stack of one thread and the register backing store of the */
1545 /* next. Thus this is likely to identify way too large a */
1546 /* "stack" and thus at least result in disastrous performance. */
1547 /* TODO: Implement better strategies here. */
1548 GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
1549 {
1550 IF_CANCEL(int cancel_state;)
1551 DCL_LOCK_STATE;
1552
1553 LOCK();
1554 DISABLE_CANCEL(cancel_state); /* May be unnecessary? */
1555 # ifdef STACK_GROWS_DOWN
1556 b -> mem_base = GC_find_limit(GC_approx_sp(), TRUE);
1557 # else
1558 b -> mem_base = GC_find_limit(GC_approx_sp(), FALSE);
1559 # endif
1560 # ifdef IA64
1561 b -> reg_base = GC_find_limit(GC_save_regs_in_stack(), FALSE);
1562 # elif defined(E2K)
1563 b -> reg_base = NULL;
1564 # endif
1565 RESTORE_CANCEL(cancel_state);
1566 UNLOCK();
1567 return GC_SUCCESS;
1568 }
1569 # else
1570 GC_API int GC_CALL GC_get_stack_base(
1571 struct GC_stack_base *b GC_ATTR_UNUSED)
1572 {
1573 # if defined(GET_MAIN_STACKBASE_SPECIAL) && !defined(THREADS) \
1574 && !defined(IA64)
1575 b->mem_base = GC_get_main_stack_base();
1576 return GC_SUCCESS;
1577 # else
1578 return GC_UNIMPLEMENTED;
1579 # endif
1580 }
1581 # endif /* !NEED_FIND_LIMIT */
1582 #endif /* !HAVE_GET_STACK_BASE */
1583
1584 #ifndef GET_MAIN_STACKBASE_SPECIAL
1585 /* This is always called from the main thread. Default implementation. */
1586 ptr_t GC_get_main_stack_base(void)
1587 {
1588 struct GC_stack_base sb;
1589
1590 if (GC_get_stack_base(&sb) != GC_SUCCESS)
1591 ABORT("GC_get_stack_base failed");
1592 GC_ASSERT((word)GC_approx_sp() HOTTER_THAN (word)sb.mem_base);
1593 return (ptr_t)sb.mem_base;
1594 }
1595 #endif /* !GET_MAIN_STACKBASE_SPECIAL */
1596
1597 /* Register static data segment(s) as roots. If more data segments are */
1598 /* added later then they need to be registered at that point (as we do */
1599 /* with SunOS dynamic loading), or GC_mark_roots needs to check for */
1600 /* them (as we do with PCR). Called with allocator lock held. */
1601 # ifdef OS2
1602
1603 void GC_register_data_segments(void)
1604 {
1605 PTIB ptib;
1606 PPIB ppib;
1607 HMODULE module_handle;
1608 # define PBUFSIZ 512
1609 UCHAR path[PBUFSIZ];
1610 FILE * myexefile;
1611 struct exe_hdr hdrdos; /* MSDOS header. */
1612 struct e32_exe hdr386; /* Real header for my executable */
1613 struct o32_obj seg; /* Current segment */
1614 int nsegs;
1615
1616 # if defined(CPPCHECK)
1617 hdrdos.padding[0] = 0; /* to prevent "field unused" warnings */
1618 hdr386.exe_format_level = 0;
1619 hdr386.os = 0;
1620 hdr386.padding1[0] = 0;
1621 hdr386.padding2[0] = 0;
1622 seg.pagemap = 0;
1623 seg.mapsize = 0;
1624 seg.reserved = 0;
1625 # endif
1626 if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
1627 ABORT("DosGetInfoBlocks failed");
1628 }
1629 module_handle = ppib -> pib_hmte;
1630 if (DosQueryModuleName(module_handle, PBUFSIZ, path) != NO_ERROR) {
1631 ABORT("DosQueryModuleName failed");
1632 }
1633 myexefile = fopen(path, "rb");
1634 if (myexefile == 0) {
1635 ABORT_ARG1("Failed to open executable", ": %s", path);
1636 }
1637 if (fread((char *)(&hdrdos), 1, sizeof(hdrdos), myexefile)
1638 < sizeof(hdrdos)) {
1639 ABORT_ARG1("Could not read MSDOS header", " from: %s", path);
1640 }
1641 if (E_MAGIC(hdrdos) != EMAGIC) {
1642 ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
1643 }
1644 if (fseek(myexefile, E_LFANEW(hdrdos), SEEK_SET) != 0) {
1645 ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
1646 }
1647 if (fread((char *)(&hdr386), 1, sizeof(hdr386), myexefile)
1648 < sizeof(hdr386)) {
1649 ABORT_ARG1("Could not read OS/2 header", " from: %s", path);
1650 }
1651 if (E32_MAGIC1(hdr386) != E32MAGIC1 || E32_MAGIC2(hdr386) != E32MAGIC2) {
1652 ABORT_ARG1("Bad OS/2 magic number", " in file: %s", path);
1653 }
1654 if (E32_BORDER(hdr386) != E32LEBO || E32_WORDER(hdr386) != E32LEWO) {
1655 ABORT_ARG1("Bad byte order in executable", " file: %s", path);
1656 }
1657 if (E32_CPU(hdr386) == E32CPU286) {
1658 ABORT_ARG1("GC cannot handle 80286 executables", ": %s", path);
1659 }
1660 if (fseek(myexefile, E_LFANEW(hdrdos) + E32_OBJTAB(hdr386),
1661 SEEK_SET) != 0) {
1662 ABORT_ARG1("Seek to object table failed", " in file: %s", path);
1663 }
1664 for (nsegs = E32_OBJCNT(hdr386); nsegs > 0; nsegs--) {
1665 int flags;
1666 if (fread((char *)(&seg), 1, sizeof(seg), myexefile) < sizeof(seg)) {
1667 ABORT_ARG1("Could not read obj table entry", " from file: %s", path);
1668 }
1669 flags = O32_FLAGS(seg);
1670 if (!(flags & OBJWRITE)) continue;
1671 if (!(flags & OBJREAD)) continue;
1672 if (flags & OBJINVALID) {
1673 GC_err_printf("Object with invalid pages?\n");
1674 continue;
1675 }
1676 GC_add_roots_inner((ptr_t)O32_BASE(seg),
1677 (ptr_t)(O32_BASE(seg)+O32_SIZE(seg)), FALSE);
1678 }
1679 (void)fclose(myexefile);
1680 }
1681
1682 # else /* !OS2 */
1683
1684 # if defined(GWW_VDB)
1685 # ifndef MEM_WRITE_WATCH
1686 # define MEM_WRITE_WATCH 0x200000
1687 # endif
1688 # ifndef WRITE_WATCH_FLAG_RESET
1689 # define WRITE_WATCH_FLAG_RESET 1
1690 # endif
1691
1692 /* Since we can't easily check whether ULONG_PTR and SIZE_T are */
1693 /* defined in Win32 basetsd.h, we define own ULONG_PTR. */
1694 # define GC_ULONG_PTR word
1695
1696 typedef UINT (WINAPI * GetWriteWatch_type)(
1697 DWORD, PVOID, GC_ULONG_PTR /* SIZE_T */,
1698 PVOID *, GC_ULONG_PTR *, PULONG);
1699 static FARPROC GetWriteWatch_func;
1700 static DWORD GetWriteWatch_alloc_flag;
1701
1702 # define GC_GWW_AVAILABLE() (GetWriteWatch_func != 0)
1703
1704 static void detect_GetWriteWatch(void)
1705 {
1706 static GC_bool done;
1707 HMODULE hK32;
1708 if (done)
1709 return;
1710
1711 # if defined(MPROTECT_VDB)
1712 {
1713 char * str = GETENV("GC_USE_GETWRITEWATCH");
1714 # if defined(GC_PREFER_MPROTECT_VDB)
1715 if (str == NULL || (*str == '0' && *(str + 1) == '\0')) {
1716 /* GC_USE_GETWRITEWATCH is unset or set to "0". */
1717 done = TRUE; /* falling back to MPROTECT_VDB strategy. */
1718 /* This should work as if GWW_VDB is undefined. */
1719 return;
1720 }
1721 # else
1722 if (str != NULL && *str == '0' && *(str + 1) == '\0') {
1723 /* GC_USE_GETWRITEWATCH is set "0". */
1724 done = TRUE; /* falling back to MPROTECT_VDB strategy. */
1725 return;
1726 }
1727 # endif
1728 }
1729 # endif
1730
1731 # ifdef MSWINRT_FLAVOR
1732 {
1733 MEMORY_BASIC_INFORMATION memInfo;
1734 SIZE_T result = VirtualQuery((void*)(word)GetProcAddress,
1735 &memInfo, sizeof(memInfo));
1736 if (result != sizeof(memInfo))
1737 ABORT("Weird VirtualQuery result");
1738 hK32 = (HMODULE)memInfo.AllocationBase;
1739 }
1740 # else
1741 hK32 = GetModuleHandle(TEXT("kernel32.dll"));
1742 # endif
1743 if (hK32 != (HMODULE)0 &&
1744 (GetWriteWatch_func = GetProcAddress(hK32, "GetWriteWatch")) != 0) {
1745 /* Also check whether VirtualAlloc accepts MEM_WRITE_WATCH, */
1746 /* as some versions of kernel32.dll have one but not the */
1747 /* other, making the feature completely broken. */
1748 void * page;
1749
1750 GC_ASSERT(GC_page_size != 0);
1751 page = VirtualAlloc(NULL, GC_page_size, MEM_WRITE_WATCH | MEM_RESERVE,
1752 PAGE_READWRITE);
1753 if (page != NULL) {
1754 PVOID pages[16];
1755 GC_ULONG_PTR count = 16;
1756 DWORD page_size;
1757 /* Check that it actually works. In spite of some */
1758 /* documentation it actually seems to exist on Win2K. */
1759 /* This test may be unnecessary, but ... */
1760 if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)(
1761 WRITE_WATCH_FLAG_RESET, page,
1762 GC_page_size, pages, &count,
1763 &page_size) != 0) {
1764 /* GetWriteWatch always fails. */
1765 GetWriteWatch_func = 0;
1766 } else {
1767 GetWriteWatch_alloc_flag = MEM_WRITE_WATCH;
1768 }
1769 VirtualFree(page, 0 /* dwSize */, MEM_RELEASE);
1770 } else {
1771 /* GetWriteWatch will be useless. */
1772 GetWriteWatch_func = 0;
1773 }
1774 }
1775 done = TRUE;
1776 }
1777
1778 # else
1779 # define GetWriteWatch_alloc_flag 0
1780 # endif /* !GWW_VDB */
1781
1782 # if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)
1783
1784 # ifdef MSWIN32
1785 /* Unfortunately, we have to handle win32s very differently from NT, */
1786 /* Since VirtualQuery has very different semantics. In particular, */
1787 /* under win32s a VirtualQuery call on an unmapped page returns an */
1788 /* invalid result. Under NT, GC_register_data_segments is a no-op */
1789 /* and all real work is done by GC_register_dynamic_libraries. Under */
1790 /* win32s, we cannot find the data segments associated with dll's. */
1791 /* We register the main data segment here. */
1792 GC_INNER GC_bool GC_no_win32_dlls = FALSE;
1793 /* This used to be set for gcc, to avoid dealing with */
1794 /* the structured exception handling issues. But we now have */
1795 /* assembly code to do that right. */
1796
1797 GC_INNER GC_bool GC_wnt = FALSE;
1798 /* This is a Windows NT derivative, i.e. NT, Win2K, XP or later. */
1799
1800 GC_INNER void GC_init_win32(void)
1801 {
1802 # if defined(_WIN64) || (defined(_MSC_VER) && _MSC_VER >= 1800)
1803 /* MS Visual Studio 2013 deprecates GetVersion, but on the other */
1804 /* hand it cannot be used to target pre-Win2K. */
1805 GC_wnt = TRUE;
1806 # else
1807 /* Set GC_wnt. If we're running under win32s, assume that no */
1808 /* DLLs will be loaded. I doubt anyone still runs win32s, but... */
1809 DWORD v = GetVersion();
1810
1811 GC_wnt = !(v & 0x80000000);
1812 GC_no_win32_dlls |= ((!GC_wnt) && (v & 0xff) <= 3);
1813 # endif
1814 # ifdef USE_MUNMAP
1815 if (GC_no_win32_dlls) {
1816 /* Turn off unmapping for safety (since may not work well with */
1817 /* GlobalAlloc). */
1818 GC_unmap_threshold = 0;
1819 }
1820 # endif
1821 }
1822
1823 /* Return the smallest address a such that VirtualQuery */
1824 /* returns correct results for all addresses between a and start. */
1825 /* Assumes VirtualQuery returns correct information for start. */
1826 STATIC ptr_t GC_least_described_address(ptr_t start)
1827 {
1828 MEMORY_BASIC_INFORMATION buf;
1829 LPVOID limit = GC_sysinfo.lpMinimumApplicationAddress;
1830 ptr_t p = (ptr_t)((word)start & ~(word)(GC_page_size-1));
1831
1832 GC_ASSERT(GC_page_size != 0);
1833 for (;;) {
1834 size_t result;
1835 LPVOID q = (LPVOID)(p - GC_page_size);
1836
1837 if ((word)q > (word)p /* underflow */ || (word)q < (word)limit) break;
1838 result = VirtualQuery(q, &buf, sizeof(buf));
1839 if (result != sizeof(buf) || buf.AllocationBase == 0) break;
1840 p = (ptr_t)(buf.AllocationBase);
1841 }
1842 return p;
1843 }
1844 # endif /* MSWIN32 */
1845
1846 # if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
1847 /* We maintain a linked list of AllocationBase values that we know */
1848 /* correspond to malloc heap sections. Currently this is only called */
1849 /* during a GC. But there is some hope that for long running */
1850 /* programs we will eventually see most heap sections. */
1851
1852 /* In the long run, it would be more reliable to occasionally walk */
1853 /* the malloc heap with HeapWalk on the default heap. But that */
1854 /* apparently works only for NT-based Windows. */
1855
1856 STATIC size_t GC_max_root_size = 100000; /* Appr. largest root size. */
1857
1858 /* In the long run, a better data structure would also be nice ... */
1859 STATIC struct GC_malloc_heap_list {
1860 void * allocation_base;
1861 struct GC_malloc_heap_list *next;
1862 } *GC_malloc_heap_l = 0;
1863
1864 /* Is p the base of one of the malloc heap sections we already know */
1865 /* about? */
1866 STATIC GC_bool GC_is_malloc_heap_base(void *p)
1867 {
1868 struct GC_malloc_heap_list *q = GC_malloc_heap_l;
1869
1870 while (0 != q) {
1871 if (q -> allocation_base == p) return TRUE;
1872 q = q -> next;
1873 }
1874 return FALSE;
1875 }
1876
1877 STATIC void *GC_get_allocation_base(void *p)
1878 {
1879 MEMORY_BASIC_INFORMATION buf;
1880 size_t result = VirtualQuery(p, &buf, sizeof(buf));
1881 if (result != sizeof(buf)) {
1882 ABORT("Weird VirtualQuery result");
1883 }
1884 return buf.AllocationBase;
1885 }
1886
1887 GC_INNER void GC_add_current_malloc_heap(void)
1888 {
1889 struct GC_malloc_heap_list *new_l = (struct GC_malloc_heap_list *)
1890 malloc(sizeof(struct GC_malloc_heap_list));
1891 void *candidate;
1892
1893 if (NULL == new_l) return;
1894 new_l -> allocation_base = NULL;
1895 /* to suppress maybe-uninitialized gcc warning */
1896
1897 candidate = GC_get_allocation_base(new_l);
1898 if (GC_is_malloc_heap_base(candidate)) {
1899 /* Try a little harder to find malloc heap. */
1900 size_t req_size = 10000;
1901 do {
1902 void *p = malloc(req_size);
1903 if (0 == p) {
1904 free(new_l);
1905 return;
1906 }
1907 candidate = GC_get_allocation_base(p);
1908 free(p);
1909 req_size *= 2;
1910 } while (GC_is_malloc_heap_base(candidate)
1911 && req_size < GC_max_root_size/10 && req_size < 500000);
1912 if (GC_is_malloc_heap_base(candidate)) {
1913 free(new_l);
1914 return;
1915 }
1916 }
1917 GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n",
1918 candidate);
1919 new_l -> allocation_base = candidate;
1920 new_l -> next = GC_malloc_heap_l;
1921 GC_malloc_heap_l = new_l;
1922 }
1923
1924 /* Free all the linked list nodes. Could be invoked at process exit */
1925 /* to avoid memory leak complains of a dynamic code analysis tool. */
1926 STATIC void GC_free_malloc_heap_list(void)
1927 {
1928 struct GC_malloc_heap_list *q = GC_malloc_heap_l;
1929
1930 GC_malloc_heap_l = NULL;
1931 while (q != NULL) {
1932 struct GC_malloc_heap_list *next = q -> next;
1933 free(q);
1934 q = next;
1935 }
1936 }
1937 # endif /* USE_WINALLOC && !REDIRECT_MALLOC */
1938
1939 /* Is p the start of either the malloc heap, or of one of our */
1940 /* heap sections? */
1941 GC_INNER GC_bool GC_is_heap_base(void *p)
1942 {
1943 int i;
1944
1945 # if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
1946 if (GC_root_size > GC_max_root_size)
1947 GC_max_root_size = GC_root_size;
1948 if (GC_is_malloc_heap_base(p))
1949 return TRUE;
1950 # endif
1951 for (i = 0; i < (int)GC_n_heap_bases; i++) {
1952 if (GC_heap_bases[i] == p) return TRUE;
1953 }
1954 return FALSE;
1955 }
1956
1957 #ifdef MSWIN32
1958 STATIC void GC_register_root_section(ptr_t static_root)
1959 {
1960 MEMORY_BASIC_INFORMATION buf;
1961 LPVOID p;
1962 char * base;
1963 char * limit;
1964
1965 if (!GC_no_win32_dlls) return;
1966 p = base = limit = GC_least_described_address(static_root);
1967 while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress) {
1968 size_t result = VirtualQuery(p, &buf, sizeof(buf));
1969 DWORD protect;
1970
1971 if (result != sizeof(buf) || buf.AllocationBase == 0
1972 || GC_is_heap_base(buf.AllocationBase)) break;
1973 if ((word)p > GC_WORD_MAX - buf.RegionSize) break; /* overflow */
1974 protect = buf.Protect;
1975 if (buf.State == MEM_COMMIT
1976 && is_writable(protect)) {
1977 if ((char *)p != limit) {
1978 if (base != limit) GC_add_roots_inner(base, limit, FALSE);
1979 base = (char *)p;
1980 }
1981 limit = (char *)p + buf.RegionSize;
1982 }
1983 p = (char *)p + buf.RegionSize;
1984 }
1985 if (base != limit) GC_add_roots_inner(base, limit, FALSE);
1986 }
1987 #endif /* MSWIN32 */
1988
1989 void GC_register_data_segments(void)
1990 {
1991 # ifdef MSWIN32
1992 GC_register_root_section((ptr_t)&GC_pages_executable);
1993 /* any other GC global variable would fit too. */
1994 # endif
1995 }
1996
1997 # else /* !OS2 && !Windows */
1998
1999 # if (defined(SVR4) || defined(AIX) || defined(DGUX) \
2000 || (defined(LINUX) && defined(SPARC))) && !defined(PCR)
2001 ptr_t GC_SysVGetDataStart(size_t max_page_size, ptr_t etext_addr)
2002 {
2003 word text_end = ((word)(etext_addr) + sizeof(word) - 1)
2004 & ~(word)(sizeof(word) - 1);
2005 /* etext rounded to word boundary */
2006 word next_page = ((text_end + (word)max_page_size - 1)
2007 & ~((word)max_page_size - 1));
2008 word page_offset = (text_end & ((word)max_page_size - 1));
2009 volatile ptr_t result = (char *)(next_page + page_offset);
2010 /* Note that this isn't equivalent to just adding */
2011 /* max_page_size to &etext if &etext is at a page boundary */
2012
2013 GC_setup_temporary_fault_handler();
2014 if (SETJMP(GC_jmp_buf) == 0) {
2015 /* Try writing to the address. */
2016 # ifdef AO_HAVE_fetch_and_add
2017 volatile AO_t zero = 0;
2018 (void)AO_fetch_and_add((volatile AO_t *)result, zero);
2019 # else
2020 /* Fallback to non-atomic fetch-and-store. */
2021 char v = *result;
2022 # if defined(CPPCHECK)
2023 GC_noop1((word)&v);
2024 # endif
2025 *result = v;
2026 # endif
2027 GC_reset_fault_handler();
2028 } else {
2029 GC_reset_fault_handler();
2030 /* We got here via a longjmp. The address is not readable. */
2031 /* This is known to happen under Solaris 2.4 + gcc, which */
2032 /* places string constants in the text segment, but after */
2033 /* etext. Use plan B. Note that we now know there is a gap */
2034 /* between text and data segments, so plan A brought us */
2035 /* something. */
2036 result = (char *)GC_find_limit(DATAEND, FALSE);
2037 }
2038 return (/* no volatile */ ptr_t)result;
2039 }
2040 # endif
2041
2042 #ifdef DATASTART_USES_BSDGETDATASTART
2043 /* It's unclear whether this should be identical to the above, or */
2044 /* whether it should apply to non-x86 architectures. */
2045 /* For now we don't assume that there is always an empty page after */
2046 /* etext. But in some cases there actually seems to be slightly more. */
2047 /* This also deals with holes between read-only data and writable data. */
2048 GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t max_page_size,
2049 ptr_t etext_addr)
2050 {
2051 word text_end = ((word)(etext_addr) + sizeof(word) - 1)
2052 & ~(word)(sizeof(word) - 1);
2053 /* etext rounded to word boundary */
2054 volatile word next_page = (text_end + (word)max_page_size - 1)
2055 & ~((word)max_page_size - 1);
2056 volatile ptr_t result = (ptr_t)text_end;
2057 GC_setup_temporary_fault_handler();
2058 if (SETJMP(GC_jmp_buf) == 0) {
2059 /* Try reading at the address. */
2060 /* This should happen before there is another thread. */
2061 for (; next_page < (word)DATAEND; next_page += (word)max_page_size)
2062 GC_noop1((word)(*(volatile unsigned char *)next_page));
2063 GC_reset_fault_handler();
2064 } else {
2065 GC_reset_fault_handler();
2066 /* As above, we go to plan B */
2067 result = (ptr_t)GC_find_limit(DATAEND, FALSE);
2068 }
2069 return(result);
2070 }
2071 #endif /* DATASTART_USES_BSDGETDATASTART */
2072
2073 #ifdef AMIGA
2074
2075 # define GC_AMIGA_DS
2076 # include "extra/AmigaOS.c"
2077 # undef GC_AMIGA_DS
2078
2079 #elif defined(OPENBSD)
2080
2081 /* Depending on arch alignment, there can be multiple holes */
2082 /* between DATASTART and DATAEND. Scan in DATASTART .. DATAEND */
2083 /* and register each region. */
2084 void GC_register_data_segments(void)
2085 {
2086 ptr_t region_start = DATASTART;
2087
2088 if ((word)region_start - 1U >= (word)DATAEND)
2089 ABORT_ARG2("Wrong DATASTART/END pair",
2090 ": %p .. %p", (void *)region_start, (void *)DATAEND);
2091 for (;;) {
2092 # ifdef GC_OPENBSD_UTHREADS
2093 ptr_t region_end = GC_find_limit_openbsd(region_start, DATAEND);
2094 # else
2095 ptr_t region_end = GC_find_limit_with_bound(region_start, TRUE, DATAEND);
2096 # endif
2097
2098 GC_add_roots_inner(region_start, region_end, FALSE);
2099 if ((word)region_end >= (word)DATAEND)
2100 break;
2101 region_start = GC_skip_hole_openbsd(region_end, DATAEND);
2102 }
2103 }
2104
2105 # else /* !OS2 && !Windows && !AMIGA && !OPENBSD */
2106
2107 # if !defined(PCR) && !defined(MACOS) && defined(REDIRECT_MALLOC) \
2108 && defined(GC_SOLARIS_THREADS)
2109 EXTERN_C_BEGIN
2110 extern caddr_t sbrk(int);
2111 EXTERN_C_END
2112 # endif
2113
2114 void GC_register_data_segments(void)
2115 {
2116 # if !defined(DYNAMIC_LOADING) && defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
2117 /* Avoid even referencing DATASTART and DATAEND as they are */
2118 /* unnecessary and cause linker errors when bitcode is enabled. */
2119 /* GC_register_data_segments() is not called anyway. */
2120 # elif !defined(PCR) && !defined(MACOS)
2121 # if defined(REDIRECT_MALLOC) && defined(GC_SOLARIS_THREADS)
2122 /* As of Solaris 2.3, the Solaris threads implementation */
2123 /* allocates the data structure for the initial thread with */
2124 /* sbrk at process startup. It needs to be scanned, so that */
2125 /* we don't lose some malloc allocated data structures */
2126 /* hanging from it. We're on thin ice here ... */
2127 GC_ASSERT(DATASTART);
2128 {
2129 ptr_t p = (ptr_t)sbrk(0);
2130 if ((word)DATASTART < (word)p)
2131 GC_add_roots_inner(DATASTART, p, FALSE);
2132 }
2133 # else
2134 if ((word)DATASTART - 1U >= (word)DATAEND) {
2135 /* Subtract one to check also for NULL */
2136 /* without a compiler warning. */
2137 ABORT_ARG2("Wrong DATASTART/END pair",
2138 ": %p .. %p", (void *)DATASTART, (void *)DATAEND);
2139 }
2140 GC_add_roots_inner(DATASTART, DATAEND, FALSE);
2141 # ifdef GC_HAVE_DATAREGION2
2142 if ((word)DATASTART2 - 1U >= (word)DATAEND2)
2143 ABORT_ARG2("Wrong DATASTART/END2 pair",
2144 ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2);
2145 GC_add_roots_inner(DATASTART2, DATAEND2, FALSE);
2146 # endif
2147 # endif
2148 # endif
2149 # if defined(MACOS)
2150 {
2151 # if defined(THINK_C)
2152 extern void* GC_MacGetDataStart(void);
2153 /* globals begin above stack and end at a5. */
2154 GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
2155 (ptr_t)LMGetCurrentA5(), FALSE);
2156 # else
2157 # if defined(__MWERKS__)
2158 # if !__POWERPC__
2159 extern void* GC_MacGetDataStart(void);
2160 /* MATTHEW: Function to handle Far Globals (CW Pro 3) */
2161 # if __option(far_data)
2162 extern void* GC_MacGetDataEnd(void);
2163 # endif
2164 /* globals begin above stack and end at a5. */
2165 GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
2166 (ptr_t)LMGetCurrentA5(), FALSE);
2167 /* MATTHEW: Handle Far Globals */
2168 # if __option(far_data)
2169 /* Far globals follow the QD globals: */
2170 GC_add_roots_inner((ptr_t)LMGetCurrentA5(),
2171 (ptr_t)GC_MacGetDataEnd(), FALSE);
2172 # endif
2173 # else
2174 extern char __data_start__[], __data_end__[];
2175 GC_add_roots_inner((ptr_t)&__data_start__,
2176 (ptr_t)&__data_end__, FALSE);
2177 # endif /* __POWERPC__ */
2178 # endif /* __MWERKS__ */
2179 # endif /* !THINK_C */
2180 }
2181 # endif /* MACOS */
2182
2183 /* Dynamic libraries are added at every collection, since they may */
2184 /* change. */
2185 }
2186
2187 # endif /* !AMIGA */
2188 # endif /* !MSWIN32 && !MSWINCE */
2189 # endif /* !OS2 */
2190
2191 /*
2192 * Auxiliary routines for obtaining memory from OS.
2193 */
2194
2195 # if !defined(OS2) && !defined(PCR) && !defined(AMIGA) \
2196 && !defined(USE_WINALLOC) && !defined(MACOS) && !defined(DOS4GW) \
2197 && !defined(NINTENDO_SWITCH) && !defined(NONSTOP) \
2198 && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PS3) \
2199 && !defined(SN_TARGET_PSP2) && !defined(RTEMS) && !defined(__CC_ARM)
2200
2201 # define SBRK_ARG_T ptrdiff_t
2202
2203 #if defined(MMAP_SUPPORTED)
2204
2205 #ifdef USE_MMAP_FIXED
2206 # define GC_MMAP_FLAGS MAP_FIXED | MAP_PRIVATE
2207 /* Seems to yield better performance on Solaris 2, but can */
2208 /* be unreliable if something is already mapped at the address. */
2209 #else
2210 # define GC_MMAP_FLAGS MAP_PRIVATE
2211 #endif
2212
2213 #ifdef USE_MMAP_ANON
2214 # define zero_fd -1
2215 # if defined(MAP_ANONYMOUS) && !defined(CPPCHECK)
2216 # define OPT_MAP_ANON MAP_ANONYMOUS
2217 # else
2218 # define OPT_MAP_ANON MAP_ANON
2219 # endif
2220 #else
2221 static int zero_fd = -1;
2222 # define OPT_MAP_ANON 0
2223 #endif
2224
2225 # ifndef MSWIN_XBOX1
2226 # if defined(SYMBIAN) && !defined(USE_MMAP_ANON)
2227 EXTERN_C_BEGIN
2228 extern char *GC_get_private_path_and_zero_file(void);
2229 EXTERN_C_END
2230 # endif
2231
2232 STATIC ptr_t GC_unix_mmap_get_mem(size_t bytes)
2233 {
2234 void *result;
2235 static ptr_t last_addr = HEAP_START;
2236
2237 # ifndef USE_MMAP_ANON
2238 static GC_bool initialized = FALSE;
2239
2240 if (!EXPECT(initialized, TRUE)) {
2241 # ifdef SYMBIAN
2242 char *path = GC_get_private_path_and_zero_file();
2243 if (path != NULL) {
2244 zero_fd = open(path, O_RDWR | O_CREAT, 0644);
2245 free(path);
2246 }
2247 # else
2248 zero_fd = open("/dev/zero", O_RDONLY);
2249 # endif
2250 if (zero_fd == -1)
2251 ABORT("Could not open /dev/zero");
2252 if (fcntl(zero_fd, F_SETFD, FD_CLOEXEC) == -1)
2253 WARN("Could not set FD_CLOEXEC for /dev/zero\n", 0);
2254
2255 initialized = TRUE;
2256 }
2257 # endif
2258
2259 GC_ASSERT(GC_page_size != 0);
2260 if (bytes & (GC_page_size - 1)) ABORT("Bad GET_MEM arg");
2261 result = mmap(last_addr, bytes, (PROT_READ | PROT_WRITE)
2262 | (GC_pages_executable ? PROT_EXEC : 0),
2263 GC_MMAP_FLAGS | OPT_MAP_ANON, zero_fd, 0/* offset */);
2264 # undef IGNORE_PAGES_EXECUTABLE
2265
2266 if (EXPECT(MAP_FAILED == result, FALSE)) {
2267 if (HEAP_START == last_addr && GC_pages_executable
2268 && (EACCES == errno || EPERM == errno))
2269 ABORT("Cannot allocate executable pages");
2270 return NULL;
2271 }
2272 last_addr = (ptr_t)(((word)result + bytes + GC_page_size - 1)
2273 & ~(word)(GC_page_size - 1));
2274 # if !defined(LINUX)
2275 if (last_addr == 0) {
2276 /* Oops. We got the end of the address space. This isn't */
2277 /* usable by arbitrary C code, since one-past-end pointers */
2278 /* don't work, so we discard it and try again. */
2279 munmap(result, ~GC_page_size - (size_t)result + 1);
2280 /* Leave last page mapped, so we can't repeat. */
2281 return GC_unix_mmap_get_mem(bytes);
2282 }
2283 # else
2284 GC_ASSERT(last_addr != 0);
2285 # endif
2286 if (((word)result % HBLKSIZE) != 0)
2287 ABORT(
2288 "GC_unix_get_mem: Memory returned by mmap is not aligned to HBLKSIZE.");
2289 return((ptr_t)result);
2290 }
2291 # endif /* !MSWIN_XBOX1 */
2292
2293 #endif /* MMAP_SUPPORTED */
2294
2295 #if defined(USE_MMAP)
2296 ptr_t GC_unix_get_mem(size_t bytes)
2297 {
2298 return GC_unix_mmap_get_mem(bytes);
2299 }
2300 #else /* !USE_MMAP */
2301
2302 STATIC ptr_t GC_unix_sbrk_get_mem(size_t bytes)
2303 {
2304 ptr_t result;
2305 # ifdef IRIX5
2306 /* Bare sbrk isn't thread safe. Play by malloc rules. */
2307 /* The equivalent may be needed on other systems as well. */
2308 __LOCK_MALLOC();
2309 # endif
2310 {
2311 ptr_t cur_brk = (ptr_t)sbrk(0);
2312 SBRK_ARG_T lsbs = (word)cur_brk & (GC_page_size-1);
2313
2314 GC_ASSERT(GC_page_size != 0);
2315 if ((SBRK_ARG_T)bytes < 0) {
2316 result = 0; /* too big */
2317 goto out;
2318 }
2319 if (lsbs != 0) {
2320 if ((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs) == (ptr_t)(-1)) {
2321 result = 0;
2322 goto out;
2323 }
2324 }
2325 # ifdef ADD_HEAP_GUARD_PAGES
2326 /* This is useful for catching severe memory overwrite problems that */
2327 /* span heap sections. It shouldn't otherwise be turned on. */
2328 {
2329 ptr_t guard = (ptr_t)sbrk((SBRK_ARG_T)GC_page_size);
2330 if (mprotect(guard, GC_page_size, PROT_NONE) != 0)
2331 ABORT("ADD_HEAP_GUARD_PAGES: mprotect failed");
2332 }
2333 # endif /* ADD_HEAP_GUARD_PAGES */
2334 result = (ptr_t)sbrk((SBRK_ARG_T)bytes);
2335 if (result == (ptr_t)(-1)) result = 0;
2336 }
2337 out:
2338 # ifdef IRIX5
2339 __UNLOCK_MALLOC();
2340 # endif
2341 return(result);
2342 }
2343
2344 ptr_t GC_unix_get_mem(size_t bytes)
2345 {
2346 # if defined(MMAP_SUPPORTED)
2347 /* By default, we try both sbrk and mmap, in that order. */
2348 static GC_bool sbrk_failed = FALSE;
2349 ptr_t result = 0;
2350
2351 if (GC_pages_executable) {
2352 /* If the allocated memory should have the execute permission */
2353 /* then sbrk() cannot be used. */
2354 return GC_unix_mmap_get_mem(bytes);
2355 }
2356 if (!sbrk_failed) result = GC_unix_sbrk_get_mem(bytes);
2357 if (0 == result) {
2358 sbrk_failed = TRUE;
2359 result = GC_unix_mmap_get_mem(bytes);
2360 }
2361 if (0 == result) {
2362 /* Try sbrk again, in case sbrk memory became available. */
2363 result = GC_unix_sbrk_get_mem(bytes);
2364 }
2365 return result;
2366 # else /* !MMAP_SUPPORTED */
2367 return GC_unix_sbrk_get_mem(bytes);
2368 # endif
2369 }
2370
2371 #endif /* !USE_MMAP */
2372
2373 # endif /* UN*X */
2374
2375 # ifdef OS2
2376
2377 void * os2_alloc(size_t bytes)
2378 {
2379 void * result;
2380
2381 if (DosAllocMem(&result, bytes, (PAG_READ | PAG_WRITE | PAG_COMMIT)
2382 | (GC_pages_executable ? PAG_EXECUTE : 0))
2383 != NO_ERROR) {
2384 return(0);
2385 }
2386 /* FIXME: What's the purpose of this recursion? (Probably, if */
2387 /* DosAllocMem returns memory at 0 address then just retry once.) */
2388 if (result == 0) return(os2_alloc(bytes));
2389 return(result);
2390 }
2391
2392 # endif /* OS2 */
2393
2394 #ifdef MSWIN_XBOX1
2395 ptr_t GC_durango_get_mem(size_t bytes)
2396 {
2397 if (0 == bytes) return NULL;
2398 return (ptr_t)VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_TOP_DOWN,
2399 PAGE_READWRITE);
2400 }
2401 #elif defined(MSWINCE)
2402 ptr_t GC_wince_get_mem(size_t bytes)
2403 {
2404 ptr_t result = 0; /* initialized to prevent warning. */
2405 word i;
2406
2407 GC_ASSERT(GC_page_size != 0);
2408 bytes = ROUNDUP_PAGESIZE(bytes);
2409
2410 /* Try to find reserved, uncommitted pages */
2411 for (i = 0; i < GC_n_heap_bases; i++) {
2412 if (((word)(-(signed_word)GC_heap_lengths[i])
2413 & (GC_sysinfo.dwAllocationGranularity-1))
2414 >= bytes) {
2415 result = GC_heap_bases[i] + GC_heap_lengths[i];
2416 break;
2417 }
2418 }
2419
2420 if (i == GC_n_heap_bases) {
2421 /* Reserve more pages */
2422 size_t res_bytes =
2423 SIZET_SAT_ADD(bytes, (size_t)GC_sysinfo.dwAllocationGranularity-1)
2424 & ~((size_t)GC_sysinfo.dwAllocationGranularity-1);
2425 /* If we ever support MPROTECT_VDB here, we will probably need to */
2426 /* ensure that res_bytes is strictly > bytes, so that VirtualProtect */
2427 /* never spans regions. It seems to be OK for a VirtualFree */
2428 /* argument to span regions, so we should be OK for now. */
2429 result = (ptr_t) VirtualAlloc(NULL, res_bytes,
2430 MEM_RESERVE | MEM_TOP_DOWN,
2431 GC_pages_executable ? PAGE_EXECUTE_READWRITE :
2432 PAGE_READWRITE);
2433 if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
2434 /* If I read the documentation correctly, this can */
2435 /* only happen if HBLKSIZE > 64 KB or not a power of 2. */
2436 if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections");
2437 if (result == NULL) return NULL;
2438 GC_heap_bases[GC_n_heap_bases] = result;
2439 GC_heap_lengths[GC_n_heap_bases] = 0;
2440 GC_n_heap_bases++;
2441 }
2442
2443 /* Commit pages */
2444 result = (ptr_t) VirtualAlloc(result, bytes, MEM_COMMIT,
2445 GC_pages_executable ? PAGE_EXECUTE_READWRITE :
2446 PAGE_READWRITE);
2447 # undef IGNORE_PAGES_EXECUTABLE
2448
2449 if (result != NULL) {
2450 if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
2451 GC_heap_lengths[i] += bytes;
2452 }
2453
2454 return(result);
2455 }
2456
2457 #elif defined(USE_WINALLOC) /* && !MSWIN_XBOX1 */ || defined(CYGWIN32)
2458
2459 # ifdef USE_GLOBAL_ALLOC
2460 # define GLOBAL_ALLOC_TEST 1
2461 # else
2462 # define GLOBAL_ALLOC_TEST GC_no_win32_dlls
2463 # endif
2464
2465 # if (defined(GC_USE_MEM_TOP_DOWN) && defined(USE_WINALLOC)) \
2466 || defined(CPPCHECK)
2467 DWORD GC_mem_top_down = MEM_TOP_DOWN;
2468 /* Use GC_USE_MEM_TOP_DOWN for better 64-bit */
2469 /* testing. Otherwise all addresses tend to */
2470 /* end up in first 4 GB, hiding bugs. */
2471 # else
2472 # define GC_mem_top_down 0
2473 # endif /* !GC_USE_MEM_TOP_DOWN */
2474
2475 ptr_t GC_win32_get_mem(size_t bytes)
2476 {
2477 ptr_t result;
2478
2479 # ifndef USE_WINALLOC
2480 result = GC_unix_get_mem(bytes);
2481 # else
2482 # if defined(MSWIN32) && !defined(MSWINRT_FLAVOR)
2483 if (GLOBAL_ALLOC_TEST) {
2484 /* VirtualAlloc doesn't like PAGE_EXECUTE_READWRITE. */
2485 /* There are also unconfirmed rumors of other */
2486 /* problems, so we dodge the issue. */
2487 result = (ptr_t)GlobalAlloc(0, SIZET_SAT_ADD(bytes, HBLKSIZE));
2488 /* Align it at HBLKSIZE boundary. */
2489 result = (ptr_t)(((word)result + HBLKSIZE - 1)
2490 & ~(word)(HBLKSIZE - 1));
2491 } else
2492 # endif
2493 /* else */ {
2494 /* VirtualProtect only works on regions returned by a */
2495 /* single VirtualAlloc call. Thus we allocate one */
2496 /* extra page, which will prevent merging of blocks */
2497 /* in separate regions, and eliminate any temptation */
2498 /* to call VirtualProtect on a range spanning regions. */
2499 /* This wastes a small amount of memory, and risks */
2500 /* increased fragmentation. But better alternatives */
2501 /* would require effort. */
2502 # ifdef MPROTECT_VDB
2503 /* We can't check for GC_incremental here (because */
2504 /* GC_enable_incremental() might be called some time */
2505 /* later after the GC initialization). */
2506 # ifdef GWW_VDB
2507 # define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE() ? 0 : 1)
2508 # else
2509 # define VIRTUAL_ALLOC_PAD 1
2510 # endif
2511 # else
2512 # define VIRTUAL_ALLOC_PAD 0
2513 # endif
2514 /* Pass the MEM_WRITE_WATCH only if GetWriteWatch-based */
2515 /* VDBs are enabled and the GetWriteWatch function is */
2516 /* available. Otherwise we waste resources or possibly */
2517 /* cause VirtualAlloc to fail (observed in Windows 2000 */
2518 /* SP2). */
2519 result = (ptr_t) VirtualAlloc(NULL,
2520 SIZET_SAT_ADD(bytes, VIRTUAL_ALLOC_PAD),
2521 GetWriteWatch_alloc_flag
2522 | (MEM_COMMIT | MEM_RESERVE)
2523 | GC_mem_top_down,
2524 GC_pages_executable ? PAGE_EXECUTE_READWRITE :
2525 PAGE_READWRITE);
2526 # undef IGNORE_PAGES_EXECUTABLE
2527 }
2528 # endif /* USE_WINALLOC */
2529 if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
2530 /* If I read the documentation correctly, this can */
2531 /* only happen if HBLKSIZE > 64 KB or not a power of 2. */
2532 if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections");
2533 if (0 != result) GC_heap_bases[GC_n_heap_bases++] = result;
2534 return(result);
2535 }
2536 #endif /* USE_WINALLOC || CYGWIN32 */
2537
2538 #if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) \
2539 || defined(MSWIN_XBOX1)
2540 GC_API void GC_CALL GC_win32_free_heap(void)
2541 {
2542 # if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC) \
2543 && !defined(MSWIN_XBOX1)
2544 GC_free_malloc_heap_list();
2545 # endif
2546 # if (defined(USE_WINALLOC) && !defined(MSWIN_XBOX1) \
2547 && !defined(MSWINCE)) || defined(CYGWIN32)
2548 # ifndef MSWINRT_FLAVOR
2549 # ifndef CYGWIN32
2550 if (GLOBAL_ALLOC_TEST)
2551 # endif
2552 {
2553 while (GC_n_heap_bases-- > 0) {
2554 # ifdef CYGWIN32
2555 /* FIXME: Is it OK to use non-GC free() here? */
2556 # else
2557 GlobalFree(GC_heap_bases[GC_n_heap_bases]);
2558 # endif
2559 GC_heap_bases[GC_n_heap_bases] = 0;
2560 }
2561 return;
2562 }
2563 # endif /* !MSWINRT_FLAVOR */
2564 # ifndef CYGWIN32
2565 /* Avoiding VirtualAlloc leak. */
2566 while (GC_n_heap_bases > 0) {
2567 VirtualFree(GC_heap_bases[--GC_n_heap_bases], 0, MEM_RELEASE);
2568 GC_heap_bases[GC_n_heap_bases] = 0;
2569 }
2570 # endif
2571 # endif /* USE_WINALLOC || CYGWIN32 */
2572 }
2573 #endif /* Windows */
2574
2575 #ifdef AMIGA
2576 # define GC_AMIGA_AM
2577 # include "extra/AmigaOS.c"
2578 # undef GC_AMIGA_AM
2579 #endif
2580
2581 #if defined(HAIKU)
2582 # include <stdlib.h>
2583 ptr_t GC_haiku_get_mem(size_t bytes)
2584 {
2585 void* mem;
2586
2587 GC_ASSERT(GC_page_size != 0);
2588 if (posix_memalign(&mem, GC_page_size, bytes) == 0)
2589 return mem;
2590 return NULL;
2591 }
2592 #endif /* HAIKU */
2593
2594 #if (defined(USE_MUNMAP) || defined(MPROTECT_VDB)) && !defined(USE_WINALLOC)
2595 # define ABORT_ON_REMAP_FAIL(C_msg_prefix, start_addr, len) \
2596 ABORT_ARG3(C_msg_prefix " failed", \
2597 " at %p (length %lu), errno= %d", \
2598 (void *)(start_addr), (unsigned long)(len), errno)
2599 #endif
2600
2601 #ifdef USE_MUNMAP
2602
2603 /* For now, this only works on Win32/WinCE and some Unix-like */
2604 /* systems. If you have something else, don't define */
2605 /* USE_MUNMAP. */
2606
2607 #if !defined(NN_PLATFORM_CTR) && !defined(MSWIN32) && !defined(MSWINCE) \
2608 && !defined(MSWIN_XBOX1)
2609 # include <unistd.h>
2610 # ifdef SN_TARGET_PS3
2611 # include <sys/memory.h>
2612 # else
2613 # include <sys/mman.h>
2614 # endif
2615 # include <sys/stat.h>
2616 # include <sys/types.h>
2617 #endif
2618
2619 /* Compute a page aligned starting address for the unmap */
2620 /* operation on a block of size bytes starting at start. */
2621 /* Return 0 if the block is too small to make this feasible. */
2622 STATIC ptr_t GC_unmap_start(ptr_t start, size_t bytes)
2623 {
2624 ptr_t result = (ptr_t)(((word)start + GC_page_size - 1)
2625 & ~(word)(GC_page_size - 1));
2626
2627 GC_ASSERT(GC_page_size != 0);
2628 if ((word)(result + GC_page_size) > (word)(start + bytes)) return 0;
2629 return result;
2630 }
2631
2632 /* Under Win32/WinCE we commit (map) and decommit (unmap) */
2633 /* memory using VirtualAlloc and VirtualFree. These functions */
2634 /* work on individual allocations of virtual memory, made */
2635 /* previously using VirtualAlloc with the MEM_RESERVE flag. */
2636 /* The ranges we need to (de)commit may span several of these */
2637 /* allocations; therefore we use VirtualQuery to check */
2638 /* allocation lengths, and split up the range as necessary. */
2639
2640 /* We assume that GC_remap is called on exactly the same range */
2641 /* as a previous call to GC_unmap. It is safe to consistently */
2642 /* round the endpoints in both places. */
2643
2644 static void block_unmap_inner(ptr_t start_addr, size_t len)
2645 {
2646 if (0 == start_addr) return;
2647
2648 # ifdef USE_WINALLOC
2649 while (len != 0) {
2650 MEMORY_BASIC_INFORMATION mem_info;
2651 word free_len;
2652
2653 if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
2654 != sizeof(mem_info))
2655 ABORT("Weird VirtualQuery result");
2656 free_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
2657 if (!VirtualFree(start_addr, free_len, MEM_DECOMMIT))
2658 ABORT("VirtualFree failed");
2659 GC_unmapped_bytes += free_len;
2660 start_addr += free_len;
2661 len -= free_len;
2662 }
2663 # else
2664 /* We immediately remap it to prevent an intervening mmap from */
2665 /* accidentally grabbing the same address space. */
2666 if (len != 0) {
2667 # ifdef SN_TARGET_PS3
2668 ps3_free_mem(start_addr, len);
2669 # elif defined(AIX) || defined(CYGWIN32) || defined(HAIKU) \
2670 || (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE)) \
2671 || defined(HPUX)
2672 /* On AIX, mmap(PROT_NONE) fails with ENOMEM unless the */
2673 /* environment variable XPG_SUS_ENV is set to ON. */
2674 /* On Cygwin, calling mmap() with the new protection flags on */
2675 /* an existing memory map with MAP_FIXED is broken. */
2676 /* However, calling mprotect() on the given address range */
2677 /* with PROT_NONE seems to work fine. */
2678 /* On Linux, low RLIMIT_AS value may lead to mmap failure. */
2679 # if defined(LINUX) && !defined(FORCE_MPROTECT_BEFORE_MADVISE)
2680 /* On Linux, at least, madvise() should be sufficient. */
2681 # else
2682 if (mprotect(start_addr, len, PROT_NONE))
2683 ABORT_ON_REMAP_FAIL("unmap: mprotect", start_addr, len);
2684 # endif
2685 # if !defined(CYGWIN32)
2686 /* On Linux (and some other platforms probably), */
2687 /* mprotect(PROT_NONE) is just disabling access to */
2688 /* the pages but not returning them to OS. */
2689 if (madvise(start_addr, len, MADV_DONTNEED) == -1)
2690 ABORT_ON_REMAP_FAIL("unmap: madvise", start_addr, len);
2691 # endif
2692 # elif defined(EMSCRIPTEN)
2693 /* Nothing to do, mmap(PROT_NONE) is not supported and */
2694 /* mprotect() is just a no-op. */
2695 # else
2696 void * result = mmap(start_addr, len, PROT_NONE,
2697 MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON,
2698 zero_fd, 0/* offset */);
2699
2700 if (EXPECT(MAP_FAILED == result, FALSE))
2701 ABORT_ON_REMAP_FAIL("unmap: mmap", start_addr, len);
2702 if (result != (void *)start_addr)
2703 ABORT("unmap: mmap() result differs from start_addr");
2704 # if defined(CPPCHECK) || defined(LINT2)
2705 /* Explicitly store the resource handle to a global variable. */
2706 GC_noop1((word)result);
2707 # endif
2708 # endif
2709 GC_unmapped_bytes += len;
2710 }
2711 # endif
2712 }
2713
2714 GC_INNER void GC_unmap(ptr_t start, size_t bytes)
2715 {
2716 ptr_t start_addr = GC_unmap_start(start, bytes);
2717 ptr_t end_addr = GC_unmap_end(start, bytes);
2718
2719 block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
2720 }
2721
2722 GC_INNER void GC_remap(ptr_t start, size_t bytes)
2723 {
2724 ptr_t start_addr = GC_unmap_start(start, bytes);
2725 ptr_t end_addr = GC_unmap_end(start, bytes);
2726 word len = end_addr - start_addr;
2727 if (0 == start_addr) return;
2728
2729 /* FIXME: Handle out-of-memory correctly (at least for Win32) */
2730 # ifdef USE_WINALLOC
2731 while (len != 0) {
2732 MEMORY_BASIC_INFORMATION mem_info;
2733 word alloc_len;
2734 ptr_t result;
2735
2736 if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
2737 != sizeof(mem_info))
2738 ABORT("Weird VirtualQuery result");
2739 alloc_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
2740 result = (ptr_t)VirtualAlloc(start_addr, alloc_len, MEM_COMMIT,
2741 GC_pages_executable
2742 ? PAGE_EXECUTE_READWRITE
2743 : PAGE_READWRITE);
2744 if (result != start_addr) {
2745 if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY ||
2746 GetLastError() == ERROR_OUTOFMEMORY) {
2747 ABORT("Not enough memory to process remapping");
2748 } else {
2749 ABORT("VirtualAlloc remapping failed");
2750 }
2751 }
2752 # ifdef LINT2
2753 GC_noop1((word)result);
2754 # endif
2755 GC_unmapped_bytes -= alloc_len;
2756 start_addr += alloc_len;
2757 len -= alloc_len;
2758 }
2759 # undef IGNORE_PAGES_EXECUTABLE
2760 # else
2761 /* It was already remapped with PROT_NONE. */
2762 {
2763 # if !defined(SN_TARGET_PS3) && !defined(FORCE_MPROTECT_BEFORE_MADVISE) \
2764 && defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE)
2765 /* Nothing to unprotect as madvise() is just a hint. */
2766 # elif defined(NACL) || defined(NETBSD)
2767 /* NaCl does not expose mprotect, but mmap should work fine. */
2768 /* In case of NetBSD, mprotect fails (unlike mmap) even */
2769 /* without PROT_EXEC if PaX MPROTECT feature is enabled. */
2770 void *result = mmap(start_addr, len, (PROT_READ | PROT_WRITE)
2771 | (GC_pages_executable ? PROT_EXEC : 0),
2772 MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON,
2773 zero_fd, 0 /* offset */);
2774 if (EXPECT(MAP_FAILED == result, FALSE))
2775 ABORT_ON_REMAP_FAIL("remap: mmap", start_addr, len);
2776 if (result != (void *)start_addr)
2777 ABORT("remap: mmap() result differs from start_addr");
2778 # if defined(CPPCHECK) || defined(LINT2)
2779 GC_noop1((word)result);
2780 # endif
2781 # undef IGNORE_PAGES_EXECUTABLE
2782 # else
2783 if (mprotect(start_addr, len, (PROT_READ | PROT_WRITE)
2784 | (GC_pages_executable ? PROT_EXEC : 0)))
2785 ABORT_ON_REMAP_FAIL("remap: mprotect", start_addr, len);
2786 # undef IGNORE_PAGES_EXECUTABLE
2787 # endif /* !NACL */
2788 }
2789 GC_unmapped_bytes -= len;
2790 # endif
2791 }
2792
2793 /* Two adjacent blocks have already been unmapped and are about to */
2794 /* be merged. Unmap the whole block. This typically requires */
2795 /* that we unmap a small section in the middle that was not previously */
2796 /* unmapped due to alignment constraints. */
2797 GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
2798 size_t bytes2)
2799 {
2800 ptr_t start1_addr = GC_unmap_start(start1, bytes1);
2801 ptr_t end1_addr = GC_unmap_end(start1, bytes1);
2802 ptr_t start2_addr = GC_unmap_start(start2, bytes2);
2803 ptr_t start_addr = end1_addr;
2804 ptr_t end_addr = start2_addr;
2805
2806 GC_ASSERT(start1 + bytes1 == start2);
2807 if (0 == start1_addr) start_addr = GC_unmap_start(start1, bytes1 + bytes2);
2808 if (0 == start2_addr) end_addr = GC_unmap_end(start1, bytes1 + bytes2);
2809 block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
2810 }
2811
2812 #endif /* USE_MUNMAP */
2813
2814 /* Routine for pushing any additional roots. In THREADS */
2815 /* environment, this is also responsible for marking from */
2816 /* thread stacks. */
2817 #ifndef THREADS
2818
2819 # if defined(EMSCRIPTEN) && defined(EMSCRIPTEN_ASYNCIFY)
2820 # include <emscripten.h>
2821
2822 static void scan_regs_cb(void *begin, void *end)
2823 {
2824 GC_push_all_stack((ptr_t)begin, (ptr_t)end);
2825 }
2826
2827 STATIC void GC_CALLBACK GC_default_push_other_roots(void)
2828 {
2829 /* Note: this needs -sASYNCIFY linker flag. */
2830 emscripten_scan_registers(scan_regs_cb);
2831 }
2832
2833 # else
2834 # define GC_default_push_other_roots 0
2835 # endif
2836
2837 #else /* THREADS */
2838
2839 # ifdef PCR
2840 PCR_ERes GC_push_thread_stack(PCR_Th_T *t, PCR_Any dummy)
2841 {
2842 struct PCR_ThCtl_TInfoRep info;
2843 PCR_ERes result;
2844
2845 info.ti_stkLow = info.ti_stkHi = 0;
2846 result = PCR_ThCtl_GetInfo(t, &info);
2847 GC_push_all_stack((ptr_t)(info.ti_stkLow), (ptr_t)(info.ti_stkHi));
2848 return(result);
2849 }
2850
2851 /* Push the contents of an old object. We treat this as stack */
2852 /* data only because that makes it robust against mark stack */
2853 /* overflow. */
2854 PCR_ERes GC_push_old_obj(void *p, size_t size, PCR_Any data)
2855 {
2856 GC_push_all_stack((ptr_t)p, (ptr_t)p + size);
2857 return(PCR_ERes_okay);
2858 }
2859
2860 extern struct PCR_MM_ProcsRep * GC_old_allocator;
2861 /* defined in pcr_interface.c. */
2862
2863 STATIC void GC_CALLBACK GC_default_push_other_roots(void)
2864 {
2865 /* Traverse data allocated by previous memory managers. */
2866 if ((*(GC_old_allocator->mmp_enumerate))(PCR_Bool_false,
2867 GC_push_old_obj, 0)
2868 != PCR_ERes_okay) {
2869 ABORT("Old object enumeration failed");
2870 }
2871 /* Traverse all thread stacks. */
2872 if (PCR_ERes_IsErr(
2873 PCR_ThCtl_ApplyToAllOtherThreads(GC_push_thread_stack,0))
2874 || PCR_ERes_IsErr(GC_push_thread_stack(PCR_Th_CurrThread(), 0))) {
2875 ABORT("Thread stack marking failed");
2876 }
2877 }
2878
2879 # elif defined(SN_TARGET_PS3)
2880 STATIC void GC_CALLBACK GC_default_push_other_roots(void)
2881 {
2882 ABORT("GC_default_push_other_roots is not implemented");
2883 }
2884
2885 void GC_push_thread_structures(void)
2886 {
2887 ABORT("GC_push_thread_structures is not implemented");
2888 }
2889
2890 # else /* GC_PTHREADS, or GC_WIN32_THREADS, etc. */
2891 STATIC void GC_CALLBACK GC_default_push_other_roots(void)
2892 {
2893 GC_push_all_stacks();
2894 }
2895 # endif
2896
2897 #endif /* THREADS */
2898
2899 GC_push_other_roots_proc GC_push_other_roots = GC_default_push_other_roots;
2900
2901 GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc fn)
2902 {
2903 GC_push_other_roots = fn;
2904 }
2905
2906 GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void)
2907 {
2908 return GC_push_other_roots;
2909 }
2910
2911 #if defined(SOFT_VDB) && !defined(NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK) \
2912 || (defined(GLIBC_2_19_TSX_BUG) && defined(PARALLEL_MARK))
2913 GC_INNER int GC_parse_version(int *pminor, const char *pverstr) {
2914 char *endp;
2915 unsigned long value = strtoul(pverstr, &endp, 10);
2916 int major = (int)value;
2917
2918 if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
2919 /* Parse error. */
2920 return -1;
2921 }
2922 if (*endp != '.') {
2923 /* No minor part. */
2924 *pminor = -1;
2925 } else {
2926 value = strtoul(endp + 1, &endp, 10);
2927 *pminor = (int)value;
2928 if (*pminor < 0 || (unsigned)(*pminor) != value) {
2929 return -1;
2930 }
2931 }
2932 return major;
2933 }
2934 #endif
2935
2936 /*
2937 * Routines for accessing dirty bits on virtual pages.
2938 * There are six ways to maintain this information:
2939 * DEFAULT_VDB: A simple dummy implementation that treats every page
2940 * as possibly dirty. This makes incremental collection
2941 * useless, but the implementation is still correct.
2942 * Manual VDB: Stacks and static data are always considered dirty.
2943 * Heap pages are considered dirty if GC_dirty(p) has been
2944 * called on some pointer p pointing to somewhere inside
2945 * an object on that page. A GC_dirty() call on a large
2946 * object directly dirties only a single page, but for the
2947 * manual VDB we are careful to treat an object with a dirty
2948 * page as completely dirty.
2949 * In order to avoid races, an object must be marked dirty
2950 * after it is written, and a reference to the object
2951 * must be kept on a stack or in a register in the interim.
2952 * With threads enabled, an object directly reachable from the
2953 * stack at the time of a collection is treated as dirty.
2954 * In single-threaded mode, it suffices to ensure that no
2955 * collection can take place between the pointer assignment
2956 * and the GC_dirty() call.
2957 * PCR_VDB: Use PPCRs virtual dirty bit facility.
2958 * PROC_VDB: Use the /proc facility for reading dirty bits. Only
2959 * works under some SVR4 variants. Even then, it may be
2960 * too slow to be entirely satisfactory. Requires reading
2961 * dirty bits for entire address space. Implementations tend
2962 * to assume that the client is a (slow) debugger.
2963 * SOFT_VDB: Use the /proc facility for reading soft-dirty PTEs.
2964 * Works on Linux 3.18+ if the kernel is properly configured.
2965 * The proposed implementation iterates over GC_heap_sects and
2966 * GC_static_roots examining the soft-dirty bit of the words
2967 * in /proc/self/pagemap corresponding to the pages of the
2968 * sections; finally all soft-dirty bits of the process are
2969 * cleared (by writing some special value to
2970 * /proc/self/clear_refs file). In case the soft-dirty bit is
2971 * not supported by the kernel, MPROTECT_VDB may be defined as
2972 * a fallback strategy.
2973 * MPROTECT_VDB:Protect pages and then catch the faults to keep track of
2974 * dirtied pages. The implementation (and implementability)
2975 * is highly system dependent. This usually fails when system
2976 * calls write to a protected page. We prevent the read system
2977 * call from doing so. It is the clients responsibility to
2978 * make sure that other system calls are similarly protected
2979 * or write only to the stack.
2980 * GWW_VDB: Use the Win32 GetWriteWatch functions, if available, to
2981 * read dirty bits. In case it is not available (because we
2982 * are running on Windows 95, Windows 2000 or earlier),
2983 * MPROTECT_VDB may be defined as a fallback strategy.
2984 */
2985
2986 #if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) \
2987 || defined(PROC_VDB)
2988 /* Add all pages in pht2 to pht1. */
2989 STATIC void GC_or_pages(page_hash_table pht1, page_hash_table pht2)
2990 {
2991 unsigned i;
2992 for (i = 0; i < PHT_SIZE; i++) pht1[i] |= pht2[i];
2993 }
2994 #endif /* CHECKSUMS && (GWW_VDB || SOFT_VDB) || PROC_VDB */
2995
2996 #ifdef GWW_VDB
2997
2998 # define GC_GWW_BUF_LEN (MAXHINCR * HBLKSIZE / 4096 /* x86 page size */)
2999 /* Still susceptible to overflow, if there are very large allocations, */
3000 /* and everything is dirty. */
3001 static PVOID gww_buf[GC_GWW_BUF_LEN];
3002
3003 # ifndef MPROTECT_VDB
3004 # define GC_gww_dirty_init GC_dirty_init
3005 # endif
3006
3007 GC_INNER GC_bool GC_gww_dirty_init(void)
3008 {
3009 detect_GetWriteWatch();
3010 return GC_GWW_AVAILABLE();
3011 }
3012
3013 GC_INLINE void GC_gww_read_dirty(GC_bool output_unneeded)
3014 {
3015 word i;
3016
3017 if (!output_unneeded)
3018 BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
3019
3020 for (i = 0; i != GC_n_heap_sects; ++i) {
3021 GC_ULONG_PTR count;
3022
3023 do {
3024 PVOID * pages = gww_buf;
3025 DWORD page_size;
3026
3027 count = GC_GWW_BUF_LEN;
3028 /* GetWriteWatch is documented as returning non-zero when it */
3029 /* fails, but the documentation doesn't explicitly say why it */
3030 /* would fail or what its behavior will be if it fails. It */
3031 /* does appear to fail, at least on recent Win2K instances, if */
3032 /* the underlying memory was not allocated with the appropriate */
3033 /* flag. This is common if GC_enable_incremental is called */
3034 /* shortly after GC initialization. To avoid modifying the */
3035 /* interface, we silently work around such a failure, it only */
3036 /* affects the initial (small) heap allocation. If there are */
3037 /* more dirty pages than will fit in the buffer, this is not */
3038 /* treated as a failure; we must check the page count in the */
3039 /* loop condition. Since each partial call will reset the */
3040 /* status of some pages, this should eventually terminate even */
3041 /* in the overflow case. */
3042 if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)(
3043 WRITE_WATCH_FLAG_RESET,
3044 GC_heap_sects[i].hs_start,
3045 GC_heap_sects[i].hs_bytes,
3046 pages, &count, &page_size) != 0) {
3047 static int warn_count = 0;
3048 struct hblk * start = (struct hblk *)GC_heap_sects[i].hs_start;
3049 static struct hblk *last_warned = 0;
3050 size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes);
3051
3052 if (i != 0 && last_warned != start && warn_count++ < 5) {
3053 last_warned = start;
3054 WARN("GC_gww_read_dirty unexpectedly failed at %p: "
3055 "Falling back to marking all pages dirty\n", start);
3056 }
3057 if (!output_unneeded) {
3058 unsigned j;
3059
3060 for (j = 0; j < nblocks; ++j) {
3061 word hash = PHT_HASH(start + j);
3062 set_pht_entry_from_index(GC_grungy_pages, hash);
3063 }
3064 }
3065 count = 1; /* Done with this section. */
3066 } else /* succeeded */ if (!output_unneeded) {
3067 PVOID * pages_end = pages + count;
3068
3069 while (pages != pages_end) {
3070 struct hblk * h = (struct hblk *) *pages++;
3071 struct hblk * h_end = (struct hblk *) ((char *) h + page_size);
3072 do {
3073 set_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
3074 } while ((word)(++h) < (word)h_end);
3075 }
3076 }
3077 } while (count == GC_GWW_BUF_LEN);
3078 /* FIXME: It's unclear from Microsoft's documentation if this loop */
3079 /* is useful. We suspect the call just fails if the buffer fills */
3080 /* up. But that should still be handled correctly. */
3081 }
3082
3083 # ifdef CHECKSUMS
3084 GC_ASSERT(!output_unneeded);
3085 GC_or_pages(GC_written_pages, GC_grungy_pages);
3086 # endif
3087 }
3088
3089 #elif defined(SOFT_VDB)
3090 static int clear_refs_fd = -1;
3091 # define GC_GWW_AVAILABLE() (clear_refs_fd != -1)
3092 #else
3093 # define GC_GWW_AVAILABLE() FALSE
3094 #endif /* !GWW_VDB && !SOFT_VDB */
3095
3096 #ifdef DEFAULT_VDB
3097 /* All of the following assume the allocation lock is held. */
3098
3099 /* The client asserts that unallocated pages in the heap are never */
3100 /* written. */
3101
3102 /* Initialize virtual dirty bit implementation. */
3103 GC_INNER GC_bool GC_dirty_init(void)
3104 {
3105 GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n");
3106 /* GC_dirty_pages and GC_grungy_pages are already cleared. */
3107 return TRUE;
3108 }
3109 #endif /* DEFAULT_VDB */
3110
3111 #ifndef GC_DISABLE_INCREMENTAL
3112 # if !defined(THREADS) || defined(HAVE_LOCKFREE_AO_OR)
3113 # define async_set_pht_entry_from_index(db, index) \
3114 set_pht_entry_from_index_concurrent(db, index)
3115 # elif defined(AO_HAVE_test_and_set_acquire)
3116 /* We need to lock around the bitmap update (in the write fault */
3117 /* handler or GC_dirty) in order to avoid the risk of losing a bit. */
3118 /* We do this with a test-and-set spin lock if possible. */
3119 GC_INNER volatile AO_TS_t GC_fault_handler_lock = AO_TS_INITIALIZER;
3120
3121 static void async_set_pht_entry_from_index(volatile page_hash_table db,
3122 size_t index)
3123 {
3124 GC_acquire_dirty_lock();
3125 set_pht_entry_from_index(db, index);
3126 GC_release_dirty_lock();
3127 }
3128 # else
3129 # error No test_and_set operation: Introduces a race.
3130 # endif /* THREADS && !AO_HAVE_test_and_set_acquire */
3131 #endif /* !GC_DISABLE_INCREMENTAL */
3132
3133 #ifdef MPROTECT_VDB
3134 /*
3135 * This implementation maintains dirty bits itself by catching write
3136 * faults and keeping track of them. We assume nobody else catches
3137 * SIGBUS or SIGSEGV. We assume no write faults occur in system calls.
3138 * This means that clients must ensure that system calls don't write
3139 * to the write-protected heap. Probably the best way to do this is to
3140 * ensure that system calls write at most to pointer-free objects in the
3141 * heap, and do even that only if we are on a platform on which those
3142 * are not protected.
3143 * We assume the page size is a multiple of HBLKSIZE.
3144 * We prefer them to be the same. We avoid protecting pointer-free
3145 * objects only if they are the same.
3146 */
3147 # ifdef DARWIN
3148 /* Using vm_protect (mach syscall) over mprotect (BSD syscall) seems to
3149 decrease the likelihood of some of the problems described below. */
3150 # include <mach/vm_map.h>
3151 STATIC mach_port_t GC_task_self = 0;
3152 # define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
3153 if (vm_protect(GC_task_self, (vm_address_t)(addr), (vm_size_t)(len), \
3154 FALSE, VM_PROT_READ \
3155 | ((allow_write) ? VM_PROT_WRITE : 0) \
3156 | (GC_pages_executable ? VM_PROT_EXECUTE : 0)) \
3157 == KERN_SUCCESS) {} else ABORT(C_msg_prefix \
3158 "vm_protect() failed")
3159
3160 # elif !defined(USE_WINALLOC)
3161 # include <sys/mman.h>
3162 # include <signal.h>
3163 # if !defined(CYGWIN32) && !defined(HAIKU)
3164 # include <sys/syscall.h>
3165 # endif
3166
3167 # define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
3168 if (mprotect((caddr_t)(addr), (size_t)(len), \
3169 PROT_READ | ((allow_write) ? PROT_WRITE : 0) \
3170 | (GC_pages_executable ? PROT_EXEC : 0)) >= 0) { \
3171 } else if (GC_pages_executable) { \
3172 ABORT_ON_REMAP_FAIL(C_msg_prefix \
3173 "mprotect vdb executable pages", \
3174 addr, len); \
3175 } else ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb", addr, len)
3176 # undef IGNORE_PAGES_EXECUTABLE
3177
3178 # else /* USE_WINALLOC */
3179 # ifndef MSWINCE
3180 # include <signal.h>
3181 # endif
3182
3183 static DWORD protect_junk;
3184 # define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
3185 if (VirtualProtect(addr, len, \
3186 GC_pages_executable ? \
3187 ((allow_write) ? PAGE_EXECUTE_READWRITE : \
3188 PAGE_EXECUTE_READ) : \
3189 (allow_write) ? PAGE_READWRITE : \
3190 PAGE_READONLY, \
3191 &protect_junk)) { \
3192 } else ABORT_ARG1(C_msg_prefix "VirtualProtect failed", \
3193 ": errcode= 0x%X", (unsigned)GetLastError())
3194 # endif /* USE_WINALLOC */
3195
3196 # define PROTECT(addr, len) PROTECT_INNER(addr, len, FALSE, "")
3197 # define UNPROTECT(addr, len) PROTECT_INNER(addr, len, TRUE, "un-")
3198
3199 # if defined(MSWIN32)
3200 typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR;
3201 # undef SIG_DFL
3202 # define SIG_DFL (LPTOP_LEVEL_EXCEPTION_FILTER)((signed_word)-1)
3203 # elif defined(MSWINCE)
3204 typedef LONG (WINAPI *SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS *);
3205 # undef SIG_DFL
3206 # define SIG_DFL (SIG_HNDLR_PTR) (-1)
3207 # elif defined(DARWIN)
3208 typedef void (* SIG_HNDLR_PTR)();
3209 # else
3210 typedef void (* SIG_HNDLR_PTR)(int, siginfo_t *, void *);
3211 typedef void (* PLAIN_HNDLR_PTR)(int);
3212 # endif
3213
3214 #ifndef DARWIN
3215 STATIC SIG_HNDLR_PTR GC_old_segv_handler = 0;
3216 /* Also old MSWIN32 ACCESS_VIOLATION filter */
3217 # if defined(FREEBSD) || defined(HPUX) || defined(HURD) || defined(LINUX)
3218 STATIC SIG_HNDLR_PTR GC_old_bus_handler = 0;
3219 # ifndef LINUX
3220 STATIC GC_bool GC_old_bus_handler_used_si = FALSE;
3221 # endif
3222 # endif
3223 # if !defined(MSWIN32) && !defined(MSWINCE)
3224 STATIC GC_bool GC_old_segv_handler_used_si = FALSE;
3225 # endif /* !MSWIN32 */
3226 #endif /* !DARWIN */
3227
3228 #ifdef THREADS
3229 /* This function is used only by the fault handler. Potential data */
3230 /* race between this function and GC_install_header, GC_remove_header */
3231 /* should not be harmful because the added or removed header should */
3232 /* be already unprotected. */
3233 GC_ATTR_NO_SANITIZE_THREAD
3234 static GC_bool is_header_found_async(void *addr)
3235 {
3236 # ifdef HASH_TL
3237 hdr *result;
3238 GET_HDR((ptr_t)addr, result);
3239 return result != NULL;
3240 # else
3241 return HDR_INNER(addr) != NULL;
3242 # endif
3243 }
3244 #else
3245 # define is_header_found_async(addr) (HDR(addr) != NULL)
3246 #endif /* !THREADS */
3247
3248 #ifndef DARWIN
3249
3250 # if !defined(MSWIN32) && !defined(MSWINCE)
3251 # include <errno.h>
3252 # if defined(FREEBSD) || defined(HURD) || defined(HPUX)
3253 # define SIG_OK (sig == SIGBUS || sig == SIGSEGV)
3254 # else
3255 # define SIG_OK (sig == SIGSEGV)
3256 /* Catch SIGSEGV but ignore SIGBUS. */
3257 # endif
3258 # if defined(FREEBSD)
3259 # ifndef SEGV_ACCERR
3260 # define SEGV_ACCERR 2
3261 # endif
3262 # if defined(AARCH64) || defined(ARM32) || defined(MIPS) \
3263 || __FreeBSD__ >= 7
3264 # define CODE_OK (si -> si_code == SEGV_ACCERR)
3265 # elif defined(POWERPC)
3266 # define AIM /* Pretend that we're AIM. */
3267 # include <machine/trap.h>
3268 # define CODE_OK (si -> si_code == EXC_DSI \
3269 || si -> si_code == SEGV_ACCERR)
3270 # else
3271 # define CODE_OK (si -> si_code == BUS_PAGE_FAULT \
3272 || si -> si_code == SEGV_ACCERR)
3273 # endif
3274 # elif defined(OSF1)
3275 # define CODE_OK (si -> si_code == 2 /* experimentally determined */)
3276 # elif defined(IRIX5)
3277 # define CODE_OK (si -> si_code == EACCES)
3278 # elif defined(CYGWIN32) || defined(HAIKU) || defined(HURD)
3279 # define CODE_OK TRUE
3280 # elif defined(LINUX)
3281 # define CODE_OK TRUE
3282 /* Empirically c.trapno == 14, on IA32, but is that useful? */
3283 /* Should probably consider alignment issues on other */
3284 /* architectures. */
3285 # elif defined(HPUX)
3286 # define CODE_OK (si -> si_code == SEGV_ACCERR \
3287 || si -> si_code == BUS_ADRERR \
3288 || si -> si_code == BUS_UNKNOWN \
3289 || si -> si_code == SEGV_UNKNOWN \
3290 || si -> si_code == BUS_OBJERR)
3291 # elif defined(SUNOS5SIGS)
3292 # define CODE_OK (si -> si_code == SEGV_ACCERR)
3293 # endif
3294 # ifndef NO_GETCONTEXT
3295 # include <ucontext.h>
3296 # endif
3297 STATIC void GC_write_fault_handler(int sig, siginfo_t *si, void *raw_sc)
3298 # else
3299 # define SIG_OK (exc_info -> ExceptionRecord -> ExceptionCode \
3300 == STATUS_ACCESS_VIOLATION)
3301 # define CODE_OK (exc_info -> ExceptionRecord -> ExceptionInformation[0] \
3302 == 1) /* Write fault */
3303 STATIC LONG WINAPI GC_write_fault_handler(
3304 struct _EXCEPTION_POINTERS *exc_info)
3305 # endif /* MSWIN32 || MSWINCE */
3306 {
3307 # if !defined(MSWIN32) && !defined(MSWINCE)
3308 char *addr = (char *)si->si_addr;
3309 # else
3310 char * addr = (char *) (exc_info -> ExceptionRecord
3311 -> ExceptionInformation[1]);
3312 # endif
3313
3314 if (SIG_OK && CODE_OK) {
3315 struct hblk * h = (struct hblk *)((word)addr
3316 & ~(word)(GC_page_size-1));
3317 GC_bool in_allocd_block;
3318 size_t i;
3319
3320 GC_ASSERT(GC_page_size != 0);
3321 # ifdef CHECKSUMS
3322 GC_record_fault(h);
3323 # endif
3324 # ifdef SUNOS5SIGS
3325 /* Address is only within the correct physical page. */
3326 in_allocd_block = FALSE;
3327 for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
3328 if (is_header_found_async(&h[i])) {
3329 in_allocd_block = TRUE;
3330 break;
3331 }
3332 }
3333 # else
3334 in_allocd_block = is_header_found_async(addr);
3335 # endif
3336 if (!in_allocd_block) {
3337 /* FIXME - We should make sure that we invoke the */
3338 /* old handler with the appropriate calling */
3339 /* sequence, which often depends on SA_SIGINFO. */
3340
3341 /* Heap blocks now begin and end on page boundaries */
3342 SIG_HNDLR_PTR old_handler;
3343
3344 # if defined(MSWIN32) || defined(MSWINCE)
3345 old_handler = GC_old_segv_handler;
3346 # else
3347 GC_bool used_si;
3348
3349 # if defined(FREEBSD) || defined(HURD) || defined(HPUX)
3350 if (sig == SIGBUS) {
3351 old_handler = GC_old_bus_handler;
3352 used_si = GC_old_bus_handler_used_si;
3353 } else
3354 # endif
3355 /* else */ {
3356 old_handler = GC_old_segv_handler;
3357 used_si = GC_old_segv_handler_used_si;
3358 }
3359 # endif
3360
3361 if (old_handler == (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
3362 # if !defined(MSWIN32) && !defined(MSWINCE)
3363 ABORT_ARG1("Unexpected bus error or segmentation fault",
3364 " at %p", (void *)addr);
3365 # else
3366 return(EXCEPTION_CONTINUE_SEARCH);
3367 # endif
3368 } else {
3369 /*
3370 * FIXME: This code should probably check if the
3371 * old signal handler used the traditional style and
3372 * if so call it using that style.
3373 */
3374 # if defined(MSWIN32) || defined(MSWINCE)
3375 return((*old_handler)(exc_info));
3376 # else
3377 if (used_si)
3378 ((SIG_HNDLR_PTR)old_handler) (sig, si, raw_sc);
3379 else
3380 /* FIXME: should pass nonstandard args as well. */
3381 ((PLAIN_HNDLR_PTR)(signed_word)old_handler)(sig);
3382 return;
3383 # endif
3384 }
3385 }
3386 UNPROTECT(h, GC_page_size);
3387 /* We need to make sure that no collection occurs between */
3388 /* the UNPROTECT and the setting of the dirty bit. Otherwise */
3389 /* a write by a third thread might go unnoticed. Reversing */
3390 /* the order is just as bad, since we would end up unprotecting */
3391 /* a page in a GC cycle during which it's not marked. */
3392 /* Currently we do this by disabling the thread stopping */
3393 /* signals while this handler is running. An alternative might */
3394 /* be to record the fact that we're about to unprotect, or */
3395 /* have just unprotected a page in the GC's thread structure, */
3396 /* and then to have the thread stopping code set the dirty */
3397 /* flag, if necessary. */
3398 for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
3399 word index = PHT_HASH(h+i);
3400
3401 async_set_pht_entry_from_index(GC_dirty_pages, index);
3402 }
3403 /* The write may not take place before dirty bits are read. */
3404 /* But then we'll fault again ... */
3405 # if defined(MSWIN32) || defined(MSWINCE)
3406 return(EXCEPTION_CONTINUE_EXECUTION);
3407 # else
3408 return;
3409 # endif
3410 }
3411 # if defined(MSWIN32) || defined(MSWINCE)
3412 return EXCEPTION_CONTINUE_SEARCH;
3413 # else
3414 ABORT_ARG1("Unexpected bus error or segmentation fault",
3415 " at %p", (void *)addr);
3416 # endif
3417 }
3418
3419 # if defined(GC_WIN32_THREADS) && !defined(CYGWIN32)
3420 GC_INNER void GC_set_write_fault_handler(void)
3421 {
3422 SetUnhandledExceptionFilter(GC_write_fault_handler);
3423 }
3424 # endif
3425
3426 # ifdef SOFT_VDB
3427 static GC_bool soft_dirty_init(void);
3428 # endif
3429
3430 GC_INNER GC_bool GC_dirty_init(void)
3431 {
3432 # if !defined(MSWIN32) && !defined(MSWINCE)
3433 struct sigaction act, oldact;
3434 # endif
3435
3436 # ifdef COUNT_PROTECTED_REGIONS
3437 GC_ASSERT(GC_page_size != 0);
3438 if ((signed_word)(GC_heapsize / (word)GC_page_size)
3439 >= ((signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
3440 - GC_num_unmapped_regions) * 2) {
3441 GC_COND_LOG_PRINTF("Cannot turn on GC incremental mode"
3442 " as heap contains too many pages\n");
3443 return FALSE;
3444 }
3445 # endif
3446 # if !defined(MSWIN32) && !defined(MSWINCE)
3447 act.sa_flags = SA_RESTART | SA_SIGINFO;
3448 act.sa_sigaction = GC_write_fault_handler;
3449 (void)sigemptyset(&act.sa_mask);
3450 # if defined(THREADS) && !defined(GC_OPENBSD_UTHREADS) \
3451 && !defined(GC_WIN32_THREADS) && !defined(NACL)
3452 /* Arrange to postpone the signal while we are in a write fault */
3453 /* handler. This effectively makes the handler atomic w.r.t. */
3454 /* stopping the world for GC. */
3455 (void)sigaddset(&act.sa_mask, GC_get_suspend_signal());
3456 # endif
3457 # endif /* !MSWIN32 */
3458 GC_VERBOSE_LOG_PRINTF(
3459 "Initializing mprotect virtual dirty bit implementation\n");
3460 if (GC_page_size % HBLKSIZE != 0) {
3461 ABORT("Page size not multiple of HBLKSIZE");
3462 }
3463 # ifdef GWW_VDB
3464 if (GC_gww_dirty_init()) {
3465 GC_COND_LOG_PRINTF("Using GetWriteWatch()\n");
3466 return TRUE;
3467 }
3468 # elif defined(SOFT_VDB)
3469 if (soft_dirty_init()) {
3470 GC_COND_LOG_PRINTF("Using soft-dirty bit feature\n");
3471 return TRUE;
3472 }
3473 # endif
3474 # ifdef MSWIN32
3475 GC_old_segv_handler = SetUnhandledExceptionFilter(
3476 GC_write_fault_handler);
3477 if (GC_old_segv_handler != NULL) {
3478 GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n");
3479 } else {
3480 GC_old_segv_handler = SIG_DFL;
3481 }
3482 # elif defined(MSWINCE)
3483 /* MPROTECT_VDB is unsupported for WinCE at present. */
3484 /* FIXME: implement it (if possible). */
3485 # else
3486 /* act.sa_restorer is deprecated and should not be initialized. */
3487 # if defined(GC_IRIX_THREADS)
3488 sigaction(SIGSEGV, 0, &oldact);
3489 sigaction(SIGSEGV, &act, 0);
3490 # else
3491 {
3492 int res = sigaction(SIGSEGV, &act, &oldact);
3493 if (res != 0) ABORT("Sigaction failed");
3494 }
3495 # endif
3496 if (oldact.sa_flags & SA_SIGINFO) {
3497 GC_old_segv_handler = oldact.sa_sigaction;
3498 GC_old_segv_handler_used_si = TRUE;
3499 } else {
3500 GC_old_segv_handler = (SIG_HNDLR_PTR)(signed_word)oldact.sa_handler;
3501 GC_old_segv_handler_used_si = FALSE;
3502 }
3503 if (GC_old_segv_handler == (SIG_HNDLR_PTR)(signed_word)SIG_IGN) {
3504 WARN("Previously ignored segmentation violation!?\n", 0);
3505 GC_old_segv_handler = (SIG_HNDLR_PTR)(signed_word)SIG_DFL;
3506 }
3507 if (GC_old_segv_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
3508 GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n");
3509 }
3510 # if defined(HPUX) || defined(LINUX) || defined(HURD) \
3511 || (defined(FREEBSD) && defined(SUNOS5SIGS))
3512 sigaction(SIGBUS, &act, &oldact);
3513 if ((oldact.sa_flags & SA_SIGINFO) != 0) {
3514 GC_old_bus_handler = oldact.sa_sigaction;
3515 # if !defined(LINUX)
3516 GC_old_bus_handler_used_si = TRUE;
3517 # endif
3518 } else {
3519 GC_old_bus_handler = (SIG_HNDLR_PTR)(signed_word)oldact.sa_handler;
3520 }
3521 if (GC_old_bus_handler == (SIG_HNDLR_PTR)(signed_word)SIG_IGN) {
3522 WARN("Previously ignored bus error!?\n", 0);
3523 # if !defined(LINUX)
3524 GC_old_bus_handler = (SIG_HNDLR_PTR)(signed_word)SIG_DFL;
3525 # else
3526 /* GC_old_bus_handler is not used by GC_write_fault_handler. */
3527 # endif
3528 } else if (GC_old_bus_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
3529 GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
3530 }
3531 # endif /* HPUX || LINUX || HURD || (FREEBSD && SUNOS5SIGS) */
3532 # endif /* !MSWIN32 && !MSWINCE */
3533 # if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
3534 GC_noop1((word)&__asan_default_options);
3535 # endif
3536 return TRUE;
3537 }
3538 #endif /* !DARWIN */
3539
3540 GC_API int GC_CALL GC_incremental_protection_needs(void)
3541 {
3542 GC_ASSERT(GC_is_initialized);
3543 # if defined(GWW_VDB) || defined(SOFT_VDB)
3544 /* Only if the incremental mode is already switched on. */
3545 if (GC_GWW_AVAILABLE())
3546 return GC_PROTECTS_NONE;
3547 # endif
3548 if (GC_page_size == HBLKSIZE) {
3549 return GC_PROTECTS_POINTER_HEAP;
3550 } else {
3551 return GC_PROTECTS_POINTER_HEAP | GC_PROTECTS_PTRFREE_HEAP;
3552 }
3553 }
3554 #define HAVE_INCREMENTAL_PROTECTION_NEEDS
3555
3556 #define IS_PTRFREE(hhdr) ((hhdr)->hb_descr == 0)
3557 #define PAGE_ALIGNED(x) !((word)(x) & (GC_page_size - 1))
3558
3559 STATIC void GC_protect_heap(void)
3560 {
3561 unsigned i;
3562 GC_bool protect_all =
3563 (0 != (GC_incremental_protection_needs() & GC_PROTECTS_PTRFREE_HEAP));
3564
3565 GC_ASSERT(GC_page_size != 0);
3566 for (i = 0; i < GC_n_heap_sects; i++) {
3567 ptr_t start = GC_heap_sects[i].hs_start;
3568 size_t len = GC_heap_sects[i].hs_bytes;
3569
3570 if (protect_all) {
3571 PROTECT(start, len);
3572 } else {
3573 struct hblk * current;
3574 struct hblk * current_start; /* Start of block to be protected. */
3575 struct hblk * limit;
3576
3577 GC_ASSERT(PAGE_ALIGNED(len));
3578 GC_ASSERT(PAGE_ALIGNED(start));
3579 current_start = current = (struct hblk *)start;
3580 limit = (struct hblk *)(start + len);
3581 while ((word)current < (word)limit) {
3582 hdr * hhdr;
3583 word nhblks;
3584 GC_bool is_ptrfree;
3585
3586 GC_ASSERT(PAGE_ALIGNED(current));
3587 GET_HDR(current, hhdr);
3588 if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
3589 /* This can happen only if we're at the beginning of a */
3590 /* heap segment, and a block spans heap segments. */
3591 /* We will handle that block as part of the preceding */
3592 /* segment. */
3593 GC_ASSERT(current_start == current);
3594 current_start = ++current;
3595 continue;
3596 }
3597 if (HBLK_IS_FREE(hhdr)) {
3598 GC_ASSERT(PAGE_ALIGNED(hhdr -> hb_sz));
3599 nhblks = divHBLKSZ(hhdr -> hb_sz);
3600 is_ptrfree = TRUE; /* dirty on alloc */
3601 } else {
3602 nhblks = OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
3603 is_ptrfree = IS_PTRFREE(hhdr);
3604 }
3605 if (is_ptrfree) {
3606 if ((word)current_start < (word)current) {
3607 PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
3608 }
3609 current_start = (current += nhblks);
3610 } else {
3611 current += nhblks;
3612 }
3613 }
3614 if ((word)current_start < (word)current) {
3615 PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
3616 }
3617 }
3618 }
3619 }
3620
3621 # if defined(CAN_HANDLE_FORK) && defined(DARWIN) && defined(THREADS) \
3622 || defined(COUNT_PROTECTED_REGIONS)
3623 /* Remove protection for the entire heap not updating GC_dirty_pages. */
3624 STATIC void GC_unprotect_all_heap(void)
3625 {
3626 unsigned i;
3627
3628 GC_ASSERT(I_HOLD_LOCK());
3629 GC_ASSERT(GC_auto_incremental);
3630 for (i = 0; i < GC_n_heap_sects; i++) {
3631 UNPROTECT(GC_heap_sects[i].hs_start, GC_heap_sects[i].hs_bytes);
3632 }
3633 }
3634 # endif /* CAN_HANDLE_FORK && DARWIN && THREADS || COUNT_PROTECTED_REGIONS */
3635
3636 # ifdef COUNT_PROTECTED_REGIONS
3637 GC_INNER void GC_handle_protected_regions_limit(void)
3638 {
3639 GC_ASSERT(GC_page_size != 0);
3640 /* To prevent exceeding the limit of vm.max_map_count, the most */
3641 /* trivial (though highly restrictive) way is to turn off the */
3642 /* incremental collection mode (based on mprotect) once the */
3643 /* number of pages in the heap reaches that limit. */
3644 if (GC_auto_incremental && !GC_GWW_AVAILABLE()
3645 && (signed_word)(GC_heapsize / (word)GC_page_size)
3646 >= ((signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
3647 - GC_num_unmapped_regions) * 2) {
3648 GC_unprotect_all_heap();
3649 # ifdef DARWIN
3650 GC_task_self = 0;
3651 # endif
3652 GC_incremental = FALSE;
3653 WARN("GC incremental mode is turned off"
3654 " to prevent hitting VM maps limit\n", 0);
3655 }
3656 }
3657 # endif /* COUNT_PROTECTED_REGIONS */
3658
3659 #endif /* MPROTECT_VDB */
3660
3661 #if !defined(THREADS) && (defined(PROC_VDB) || defined(SOFT_VDB))
3662 static pid_t saved_proc_pid; /* pid used to compose /proc file names */
3663 #endif
3664
3665 #ifdef PROC_VDB
3666 /* This implementation assumes a Solaris 2.X like /proc */
3667 /* pseudo-file-system from which we can read page modified bits. This */
3668 /* facility is far from optimal (e.g. we would like to get the info for */
3669 /* only some of the address space), but it avoids intercepting system */
3670 /* calls. */
3671
3672 # include <errno.h>
3673 # include <sys/types.h>
3674 # include <sys/signal.h>
3675 # include <sys/syscall.h>
3676 # include <sys/stat.h>
3677
3678 # ifdef GC_NO_SYS_FAULT_H
3679 /* This exists only to check PROC_VDB code compilation (on Linux). */
3680 # define PG_MODIFIED 1
3681 struct prpageheader {
3682 int dummy[2]; /* pr_tstamp */
3683 unsigned long pr_nmap;
3684 unsigned long pr_npage;
3685 };
3686 struct prasmap {
3687 char *pr_vaddr;
3688 size_t pr_npage;
3689 char dummy1[64+8]; /* pr_mapname, pr_offset */
3690 unsigned pr_mflags;
3691 unsigned pr_pagesize;
3692 int dummy2[2];
3693 };
3694 # else
3695 # include <sys/fault.h>
3696 # include <sys/procfs.h>
3697 # endif
3698
3699 # define INITIAL_BUF_SZ 16384
3700 STATIC size_t GC_proc_buf_size = INITIAL_BUF_SZ;
3701 STATIC char *GC_proc_buf = NULL;
3702 STATIC int GC_proc_fd = -1;
3703
3704 static GC_bool proc_dirty_open_files(void)
3705 {
3706 char buf[40];
3707 pid_t pid = getpid();
3708
3709 (void)snprintf(buf, sizeof(buf), "/proc/%ld/pagedata", (long)pid);
3710 buf[sizeof(buf) - 1] = '\0';
3711 GC_proc_fd = open(buf, O_RDONLY);
3712 if (-1 == GC_proc_fd) {
3713 WARN("/proc open failed; cannot enable GC incremental mode\n", 0);
3714 return FALSE;
3715 }
3716 if (syscall(SYS_fcntl, GC_proc_fd, F_SETFD, FD_CLOEXEC) == -1)
3717 WARN("Could not set FD_CLOEXEC for /proc\n", 0);
3718 # ifndef THREADS
3719 saved_proc_pid = pid; /* updated on success only */
3720 # endif
3721 return TRUE;
3722 }
3723
3724 # ifdef CAN_HANDLE_FORK
3725 GC_INNER void GC_dirty_update_child(void)
3726 {
3727 if (-1 == GC_proc_fd)
3728 return; /* GC incremental mode is off */
3729
3730 close(GC_proc_fd);
3731 if (!proc_dirty_open_files())
3732 GC_incremental = FALSE; /* should be safe to turn it off */
3733 }
3734 # endif /* CAN_HANDLE_FORK */
3735
3736 GC_INNER GC_bool GC_dirty_init(void)
3737 {
3738 if (GC_bytes_allocd != 0 || GC_bytes_allocd_before_gc != 0) {
3739 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
3740 GC_VERBOSE_LOG_PRINTF(
3741 "Allocated %lu bytes: all pages may have been written\n",
3742 (unsigned long)(GC_bytes_allocd + GC_bytes_allocd_before_gc));
3743 }
3744 if (!proc_dirty_open_files())
3745 return FALSE;
3746 GC_proc_buf = GC_scratch_alloc(GC_proc_buf_size);
3747 if (GC_proc_buf == NULL)
3748 ABORT("Insufficient space for /proc read");
3749 return TRUE;
3750 }
3751
3752 GC_INLINE void GC_proc_read_dirty(GC_bool output_unneeded)
3753 {
3754 int nmaps;
3755 char * bufp = GC_proc_buf;
3756 int i;
3757
3758 # ifndef THREADS
3759 /* If the current pid differs from the saved one, then we are in */
3760 /* the forked (child) process, the current /proc file should be */
3761 /* closed, the new one should be opened with the updated path. */
3762 /* Note, this is not needed for multi-threaded case because */
3763 /* fork_child_proc() reopens the file right after fork. */
3764 if (getpid() != saved_proc_pid
3765 && (-1 == GC_proc_fd /* no need to retry */
3766 || (close(GC_proc_fd), !proc_dirty_open_files()))) {
3767 /* Failed to reopen the file. Punt! */
3768 if (!output_unneeded)
3769 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
3770 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
3771 return;
3772 }
3773 # endif
3774
3775 BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
3776 if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) {
3777 /* Retry with larger buffer. */
3778 size_t new_size = 2 * GC_proc_buf_size;
3779 char *new_buf;
3780
3781 WARN("/proc read failed: GC_proc_buf_size= %" WARN_PRIuPTR "\n",
3782 GC_proc_buf_size);
3783 new_buf = GC_scratch_alloc(new_size);
3784 if (new_buf != 0) {
3785 GC_scratch_recycle_no_gww(bufp, GC_proc_buf_size);
3786 GC_proc_buf = bufp = new_buf;
3787 GC_proc_buf_size = new_size;
3788 }
3789 if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) {
3790 WARN("Insufficient space for /proc read\n", 0);
3791 /* Punt: */
3792 if (!output_unneeded)
3793 memset(GC_grungy_pages, 0xff, sizeof (page_hash_table));
3794 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
3795 return;
3796 }
3797 }
3798
3799 /* Copy dirty bits into GC_grungy_pages */
3800 nmaps = ((struct prpageheader *)bufp) -> pr_nmap;
3801 # ifdef DEBUG_DIRTY_BITS
3802 GC_log_printf("Proc VDB read: pr_nmap= %u, pr_npage= %lu\n",
3803 nmaps, ((struct prpageheader *)bufp)->pr_npage);
3804 # endif
3805 # if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
3806 GC_noop1(((struct prpageheader *)bufp)->dummy[0]);
3807 # endif
3808 bufp += sizeof(struct prpageheader);
3809 for (i = 0; i < nmaps; i++) {
3810 struct prasmap * map = (struct prasmap *)bufp;
3811 ptr_t vaddr = (ptr_t)(map -> pr_vaddr);
3812 unsigned long npages = map -> pr_npage;
3813 unsigned pagesize = map -> pr_pagesize;
3814 ptr_t limit;
3815
3816 # if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
3817 GC_noop1(map->dummy1[0] + map->dummy2[0]);
3818 # endif
3819 # ifdef DEBUG_DIRTY_BITS
3820 GC_log_printf(
3821 "pr_vaddr= %p, npage= %lu, mflags= 0x%x, pagesize= 0x%x\n",
3822 (void *)vaddr, npages, map->pr_mflags, pagesize);
3823 # endif
3824
3825 bufp += sizeof(struct prasmap);
3826 limit = vaddr + pagesize * npages;
3827 for (; (word)vaddr < (word)limit; vaddr += pagesize) {
3828 if ((*bufp++) & PG_MODIFIED) {
3829 struct hblk * h;
3830 ptr_t next_vaddr = vaddr + pagesize;
3831 # ifdef DEBUG_DIRTY_BITS
3832 GC_log_printf("dirty page at: %p\n", (void *)vaddr);
3833 # endif
3834 for (h = (struct hblk *)vaddr;
3835 (word)h < (word)next_vaddr; h++) {
3836 word index = PHT_HASH(h);
3837
3838 set_pht_entry_from_index(GC_grungy_pages, index);
3839 }
3840 }
3841 }
3842 bufp = (char *)(((word)bufp + (sizeof(long)-1))
3843 & ~(word)(sizeof(long)-1));
3844 }
3845 # ifdef DEBUG_DIRTY_BITS
3846 GC_log_printf("Proc VDB read done\n");
3847 # endif
3848
3849 /* Update GC_written_pages (even if output_unneeded). */
3850 GC_or_pages(GC_written_pages, GC_grungy_pages);
3851 }
3852
3853 #endif /* PROC_VDB */
3854
3855 #ifdef SOFT_VDB
3856 # ifndef VDB_BUF_SZ
3857 # define VDB_BUF_SZ 16384
3858 # endif
3859
3860 static int open_proc_fd(pid_t pid, const char *proc_filename, int mode)
3861 {
3862 int f;
3863 char buf[40];
3864
3865 (void)snprintf(buf, sizeof(buf), "/proc/%ld/%s", (long)pid,
3866 proc_filename);
3867 buf[sizeof(buf) - 1] = '\0';
3868 f = open(buf, mode);
3869 if (-1 == f) {
3870 WARN("/proc/self/%s open failed; cannot enable GC incremental mode\n",
3871 proc_filename);
3872 } else if (fcntl(f, F_SETFD, FD_CLOEXEC) == -1) {
3873 WARN("Could not set FD_CLOEXEC for /proc\n", 0);
3874 }
3875 return f;
3876 }
3877
3878 # include <stdint.h> /* for uint64_t */
3879
3880 typedef uint64_t pagemap_elem_t;
3881
3882 static pagemap_elem_t *soft_vdb_buf;
3883 static int pagemap_fd;
3884
3885 static GC_bool soft_dirty_open_files(void)
3886 {
3887 pid_t pid = getpid();
3888
3889 clear_refs_fd = open_proc_fd(pid, "clear_refs", O_WRONLY);
3890 if (-1 == clear_refs_fd)
3891 return FALSE;
3892 pagemap_fd = open_proc_fd(pid, "pagemap", O_RDONLY);
3893 if (-1 == pagemap_fd) {
3894 close(clear_refs_fd);
3895 clear_refs_fd = -1;
3896 return FALSE;
3897 }
3898 # ifndef THREADS
3899 saved_proc_pid = pid; /* updated on success only */
3900 # endif
3901 return TRUE;
3902 }
3903
3904 # ifdef CAN_HANDLE_FORK
3905 GC_INNER void GC_dirty_update_child(void)
3906 {
3907 if (-1 == clear_refs_fd)
3908 return; /* GC incremental mode is off */
3909
3910 close(clear_refs_fd);
3911 close(pagemap_fd);
3912 if (!soft_dirty_open_files())
3913 GC_incremental = FALSE;
3914 }
3915 # endif /* CAN_HANDLE_FORK */
3916
3917 /* Clear soft-dirty bits from the task's PTEs. */
3918 static void clear_soft_dirty_bits(void)
3919 {
3920 ssize_t res = write(clear_refs_fd, "4\n", 2);
3921
3922 if (res != 2)
3923 ABORT_ARG1("Failed to write to /proc/self/clear_refs",
3924 ": errno= %d", res < 0 ? errno : 0);
3925 }
3926
3927 /* The bit 55 of the 64-bit qword of pagemap file is the soft-dirty one. */
3928 # define PM_SOFTDIRTY_MASK ((pagemap_elem_t)1 << 55)
3929
3930 static GC_bool detect_soft_dirty_supported(ptr_t vaddr)
3931 {
3932 off_t fpos;
3933 pagemap_elem_t buf[1];
3934
3935 GC_ASSERT(GC_page_size != 0);
3936 *vaddr = 1; /* make it dirty */
3937 fpos = (off_t)((word)vaddr / GC_page_size * sizeof(pagemap_elem_t));
3938
3939 for (;;) {
3940 /* Read the relevant PTE from the pagemap file. */
3941 if (lseek(pagemap_fd, fpos, SEEK_SET) == (off_t)(-1))
3942 return FALSE;
3943 if (PROC_READ(pagemap_fd, buf, sizeof(buf)) != (int)sizeof(buf))
3944 return FALSE;
3945
3946 /* Is the soft-dirty bit unset? */
3947 if ((buf[0] & PM_SOFTDIRTY_MASK) == 0) return FALSE;
3948
3949 if (0 == *vaddr) break;
3950 /* Retry to check that writing to clear_refs works as expected. */
3951 /* This malfunction of the soft-dirty bits implementation is */
3952 /* observed on some Linux kernels on Power9 (e.g. in Fedora 36). */
3953 clear_soft_dirty_bits();
3954 *vaddr = 0;
3955 }
3956 return TRUE; /* success */
3957 }
3958
3959 # ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
3960 # include <sys/utsname.h>
3961 # include <string.h> /* for strcmp() */
3962
3963 /* Ensure the linux (kernel) major/minor version is as given or higher. */
3964 static GC_bool ensure_min_linux_ver(int major, int minor) {
3965 struct utsname info;
3966 int actual_major;
3967 int actual_minor = -1;
3968
3969 if (uname(&info) == -1) {
3970 return FALSE; /* uname() failed, should not happen actually. */
3971 }
3972 if (strcmp(info.sysname, "Linux")) {
3973 WARN("Cannot ensure Linux version as running on other OS: %s\n",
3974 info.sysname);
3975 return FALSE;
3976 }
3977 actual_major = GC_parse_version(&actual_minor, info.release);
3978 return actual_major > major
3979 || (actual_major == major && actual_minor >= minor);
3980 }
3981 # endif
3982
3983 # ifdef MPROTECT_VDB
3984 static GC_bool soft_dirty_init(void)
3985 # else
3986 GC_INNER GC_bool GC_dirty_init(void)
3987 # endif
3988 {
3989 # ifdef MPROTECT_VDB
3990 char * str = GETENV("GC_USE_GETWRITEWATCH");
3991
3992 # ifdef GC_PREFER_MPROTECT_VDB
3993 if (str == NULL || (*str == '0' && *(str + 1) == '\0'))
3994 return FALSE; /* the environment variable is unset or set to "0" */
3995 # else
3996 if (str != NULL && *str == '0' && *(str + 1) == '\0')
3997 return FALSE; /* the environment variable is set "0" */
3998 # endif
3999 # endif
4000 GC_ASSERT(NULL == soft_vdb_buf);
4001 # ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
4002 if (!ensure_min_linux_ver(3, 18)) {
4003 GC_COND_LOG_PRINTF(
4004 "Running on old kernel lacking correct soft-dirty bit support\n");
4005 return FALSE;
4006 }
4007 # endif
4008 if (!soft_dirty_open_files())
4009 return FALSE;
4010 soft_vdb_buf = (pagemap_elem_t *)GC_scratch_alloc(VDB_BUF_SZ);
4011 if (NULL == soft_vdb_buf)
4012 ABORT("Insufficient space for /proc pagemap buffer");
4013 if (!detect_soft_dirty_supported((ptr_t)soft_vdb_buf)) {
4014 GC_COND_LOG_PRINTF("Soft-dirty bit is not supported by kernel\n");
4015 /* Release the resources. */
4016 GC_scratch_recycle_no_gww(soft_vdb_buf, VDB_BUF_SZ);
4017 soft_vdb_buf = NULL;
4018 close(clear_refs_fd);
4019 clear_refs_fd = -1;
4020 close(pagemap_fd);
4021 return FALSE;
4022 }
4023 return TRUE;
4024 }
4025
4026 static off_t pagemap_buf_fpos; /* valid only if pagemap_buf_len > 0 */
4027 static size_t pagemap_buf_len;
4028
4029 /* Read bytes from /proc/self/pagemap at given file position. */
4030 /* len - the maximum number of bytes to read; (*pres) - amount of */
4031 /* bytes actually read, always bigger than 0 but never exceeds len; */
4032 /* next_fpos_hint - the file position of the next bytes block to read */
4033 /* ahead if possible (0 means no information provided). */
4034 static const pagemap_elem_t *pagemap_buffered_read(size_t *pres,
4035 off_t fpos, size_t len,
4036 off_t next_fpos_hint)
4037 {
4038 ssize_t res;
4039 size_t ofs;
4040
4041 GC_ASSERT(len > 0);
4042 if (pagemap_buf_fpos <= fpos
4043 && fpos < pagemap_buf_fpos + (off_t)pagemap_buf_len) {
4044 /* The requested data is already in the buffer. */
4045 ofs = (size_t)(fpos - pagemap_buf_fpos);
4046 res = (ssize_t)(pagemap_buf_fpos + pagemap_buf_len - fpos);
4047 } else {
4048 off_t aligned_pos = fpos & ~(off_t)(GC_page_size < VDB_BUF_SZ
4049 ? GC_page_size-1 : VDB_BUF_SZ-1);
4050
4051 for (;;) {
4052 size_t count;
4053
4054 if ((0 == pagemap_buf_len
4055 || pagemap_buf_fpos + (off_t)pagemap_buf_len != aligned_pos)
4056 && lseek(pagemap_fd, aligned_pos, SEEK_SET) == (off_t)(-1))
4057 ABORT_ARG2("Failed to lseek /proc/self/pagemap",
4058 ": offset= %lu, errno= %d", (unsigned long)fpos, errno);
4059
4060 /* How much to read at once? */
4061 ofs = (size_t)(fpos - aligned_pos);
4062 GC_ASSERT(ofs < VDB_BUF_SZ);
4063 if (next_fpos_hint > aligned_pos
4064 && next_fpos_hint - aligned_pos < VDB_BUF_SZ) {
4065 count = VDB_BUF_SZ;
4066 } else {
4067 count = len + ofs;
4068 if (count > VDB_BUF_SZ)
4069 count = VDB_BUF_SZ;
4070 }
4071
4072 GC_ASSERT(count % sizeof(pagemap_elem_t) == 0);
4073 res = PROC_READ(pagemap_fd, soft_vdb_buf, count);
4074 if (res > (ssize_t)ofs)
4075 break;
4076 if (res <= 0)
4077 ABORT_ARG1("Failed to read /proc/self/pagemap",
4078 ": errno= %d", res < 0 ? errno : 0);
4079 /* Retry (once) w/o page-alignment. */
4080 aligned_pos = fpos;
4081 }
4082
4083 /* Save the buffer (file window) position and size. */
4084 pagemap_buf_fpos = aligned_pos;
4085 pagemap_buf_len = (size_t)res;
4086 res -= (ssize_t)ofs;
4087 }
4088
4089 GC_ASSERT(ofs % sizeof(pagemap_elem_t) == 0);
4090 *pres = (size_t)res < len ? (size_t)res : len;
4091 return &soft_vdb_buf[ofs / sizeof(pagemap_elem_t)];
4092 }
4093
4094 static void soft_set_grungy_pages(ptr_t start, ptr_t limit,
4095 ptr_t next_start_hint)
4096 {
4097 word vaddr = (word)start & ~(word)(GC_page_size-1);
4098 off_t next_fpos_hint;
4099
4100 GC_ASSERT(modHBLKSZ((word)start) == 0);
4101 GC_ASSERT(GC_page_size != 0);
4102 next_fpos_hint = (off_t)((word)next_start_hint / GC_page_size
4103 * sizeof(pagemap_elem_t));
4104 while (vaddr < (word)limit) {
4105 size_t res;
4106 word limit_buf;
4107 const pagemap_elem_t *bufp = pagemap_buffered_read(&res,
4108 (off_t)(vaddr / GC_page_size * sizeof(pagemap_elem_t)),
4109 (size_t)(((word)limit - vaddr + GC_page_size-1) / GC_page_size
4110 * sizeof(pagemap_elem_t)),
4111 next_fpos_hint);
4112
4113 if (res % sizeof(pagemap_elem_t) != 0) {
4114 /* Punt: */
4115 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
4116 WARN("Incomplete read of pagemap, not multiple of entry size\n", 0);
4117 break;
4118 }
4119
4120 limit_buf = vaddr + (res / sizeof(pagemap_elem_t)) * GC_page_size;
4121 for (; vaddr < limit_buf; vaddr += GC_page_size, bufp++)
4122 if ((*bufp & PM_SOFTDIRTY_MASK) != 0) {
4123 struct hblk * h;
4124 word next_vaddr = vaddr + GC_page_size;
4125
4126 if (EXPECT(next_vaddr > (word)limit, FALSE))
4127 next_vaddr = (word)limit;
4128 /* If the bit is set, the respective PTE was written to */
4129 /* since clearing the soft-dirty bits. */
4130 # ifdef DEBUG_DIRTY_BITS
4131 GC_log_printf("dirty page at: %p\n", (void *)vaddr);
4132 # endif
4133 h = (struct hblk *)vaddr;
4134 if (EXPECT(vaddr < (word)start, FALSE))
4135 h = (struct hblk *)start;
4136 for (; (word)h < next_vaddr; h++) {
4137 word index = PHT_HASH(h);
4138 set_pht_entry_from_index(GC_grungy_pages, index);
4139 }
4140 }
4141 /* Read the next portion of pagemap file if incomplete. */
4142 }
4143 }
4144
4145 GC_INLINE void GC_soft_read_dirty(GC_bool output_unneeded)
4146 {
4147 # ifndef THREADS
4148 /* Similar as for GC_proc_read_dirty. */
4149 if (getpid() != saved_proc_pid
4150 && (-1 == clear_refs_fd /* no need to retry */
4151 || (close(clear_refs_fd), close(pagemap_fd),
4152 !soft_dirty_open_files()))) {
4153 /* Failed to reopen the files. */
4154 if (!output_unneeded) {
4155 /* Punt: */
4156 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
4157 # ifdef CHECKSUMS
4158 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
4159 # endif
4160 }
4161 return;
4162 }
4163 # endif
4164
4165 if (!output_unneeded) {
4166 word i;
4167
4168 BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
4169 pagemap_buf_len = 0; /* invalidate soft_vdb_buf */
4170
4171 for (i = 0; i != GC_n_heap_sects; ++i) {
4172 ptr_t start = GC_heap_sects[i].hs_start;
4173
4174 soft_set_grungy_pages(start, start + GC_heap_sects[i].hs_bytes,
4175 i < GC_n_heap_sects-1 ?
4176 GC_heap_sects[i+1].hs_start : NULL);
4177 }
4178 # ifdef CHECKSUMS
4179 GC_or_pages(GC_written_pages, GC_grungy_pages);
4180 # endif
4181
4182 # ifndef NO_VDB_FOR_STATIC_ROOTS
4183 for (i = 0; (int)i < n_root_sets; ++i) {
4184 soft_set_grungy_pages((ptr_t)HBLKPTR(GC_static_roots[i].r_start),
4185 GC_static_roots[i].r_end,
4186 (int)i < n_root_sets-1 ?
4187 GC_static_roots[i+1].r_start : NULL);
4188 }
4189 # endif
4190 }
4191
4192 clear_soft_dirty_bits();
4193 }
4194 #endif /* SOFT_VDB */
4195
4196 #ifdef PCR_VDB
4197
4198 # include "vd/PCR_VD.h"
4199
4200 # define NPAGES (32*1024) /* 128 MB */
4201
4202 PCR_VD_DB GC_grungy_bits[NPAGES];
4203
4204 STATIC ptr_t GC_vd_base = NULL;
4205 /* Address corresponding to GC_grungy_bits[0] */
4206 /* HBLKSIZE aligned. */
4207
4208 GC_INNER GC_bool GC_dirty_init(void)
4209 {
4210 /* For the time being, we assume the heap generally grows up */
4211 GC_vd_base = GC_heap_sects[0].hs_start;
4212 if (GC_vd_base == 0) {
4213 ABORT("Bad initial heap segment");
4214 }
4215 if (PCR_VD_Start(HBLKSIZE, GC_vd_base, NPAGES*HBLKSIZE)
4216 != PCR_ERes_okay) {
4217 ABORT("Dirty bit initialization failed");
4218 }
4219 return TRUE;
4220 }
4221 #endif /* PCR_VDB */
4222
4223 #ifndef GC_DISABLE_INCREMENTAL
4224 GC_INNER GC_bool GC_manual_vdb = FALSE;
4225
4226 /* Manually mark the page containing p as dirty. Logically, this */
4227 /* dirties the entire object. */
4228 GC_INNER void GC_dirty_inner(const void *p)
4229 {
4230 word index = PHT_HASH(p);
4231
4232 # if defined(MPROTECT_VDB)
4233 /* Do not update GC_dirty_pages if it should be followed by the */
4234 /* page unprotection. */
4235 GC_ASSERT(GC_manual_vdb);
4236 # endif
4237 async_set_pht_entry_from_index(GC_dirty_pages, index);
4238 }
4239
4240 /* Retrieve system dirty bits for the heap to a local buffer (unless */
4241 /* output_unneeded). Restore the systems notion of which pages are */
4242 /* dirty. We assume that either the world is stopped or it is OK to */
4243 /* lose dirty bits while it's happening (as in GC_enable_incremental).*/
4244 GC_INNER void GC_read_dirty(GC_bool output_unneeded)
4245 {
4246 if (GC_manual_vdb
4247 # if defined(MPROTECT_VDB)
4248 || !GC_GWW_AVAILABLE()
4249 # endif
4250 ) {
4251 if (!output_unneeded)
4252 BCOPY((/* no volatile */ void *)GC_dirty_pages, GC_grungy_pages,
4253 sizeof(GC_dirty_pages));
4254 BZERO((/* no volatile */ void *)GC_dirty_pages,
4255 sizeof(GC_dirty_pages));
4256 # ifdef MPROTECT_VDB
4257 if (!GC_manual_vdb)
4258 GC_protect_heap();
4259 # endif
4260 return;
4261 }
4262
4263 # ifdef GWW_VDB
4264 GC_gww_read_dirty(output_unneeded);
4265 # elif defined(PROC_VDB)
4266 GC_proc_read_dirty(output_unneeded);
4267 # elif defined(SOFT_VDB)
4268 GC_soft_read_dirty(output_unneeded);
4269 # elif defined(PCR_VDB)
4270 /* lazily enable dirty bits on newly added heap sects */
4271 {
4272 static int onhs = 0;
4273 int nhs = GC_n_heap_sects;
4274 for (; onhs < nhs; onhs++) {
4275 PCR_VD_WriteProtectEnable(
4276 GC_heap_sects[onhs].hs_start,
4277 GC_heap_sects[onhs].hs_bytes);
4278 }
4279 }
4280 if (PCR_VD_Clear(GC_vd_base, NPAGES*HBLKSIZE, GC_grungy_bits)
4281 != PCR_ERes_okay) {
4282 ABORT("Dirty bit read failed");
4283 }
4284 # endif
4285 }
4286
4287 # if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
4288 GC_INNER GC_bool GC_is_vdb_for_static_roots(void)
4289 {
4290 if (GC_manual_vdb) return FALSE;
4291 # if defined(MPROTECT_VDB)
4292 /* Currently used only in conjunction with SOFT_VDB. */
4293 return GC_GWW_AVAILABLE();
4294 # else
4295 GC_ASSERT(GC_incremental);
4296 return TRUE;
4297 # endif
4298 }
4299 # endif
4300
4301 /* Is the HBLKSIZE sized page at h marked dirty in the local buffer? */
4302 /* If the actual page size is different, this returns TRUE if any */
4303 /* of the pages overlapping h are dirty. This routine may err on the */
4304 /* side of labeling pages as dirty (and this implementation does). */
4305 GC_INNER GC_bool GC_page_was_dirty(struct hblk *h)
4306 {
4307 word index;
4308
4309 # ifdef PCR_VDB
4310 if (!GC_manual_vdb) {
4311 if ((word)h < (word)GC_vd_base
4312 || (word)h >= (word)(GC_vd_base + NPAGES * HBLKSIZE)) {
4313 return TRUE;
4314 }
4315 return GC_grungy_bits[h-(struct hblk*)GC_vd_base] & PCR_VD_DB_dirtyBit;
4316 }
4317 # elif defined(DEFAULT_VDB)
4318 if (!GC_manual_vdb)
4319 return TRUE;
4320 # elif defined(PROC_VDB)
4321 /* Unless manual VDB is on, the bitmap covers all process memory. */
4322 if (GC_manual_vdb)
4323 # endif
4324 {
4325 if (NULL == HDR(h))
4326 return TRUE;
4327 }
4328 index = PHT_HASH(h);
4329 return get_pht_entry_from_index(GC_grungy_pages, index);
4330 }
4331
4332 # if defined(CHECKSUMS) || defined(PROC_VDB)
4333 /* Could any valid GC heap pointer ever have been written to this page? */
4334 GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h)
4335 {
4336 # if defined(GWW_VDB) || defined(PROC_VDB) || defined(SOFT_VDB)
4337 word index;
4338
4339 # ifdef MPROTECT_VDB
4340 if (!GC_GWW_AVAILABLE())
4341 return TRUE;
4342 # endif
4343 # if defined(PROC_VDB)
4344 if (GC_manual_vdb)
4345 # endif
4346 {
4347 if (NULL == HDR(h))
4348 return TRUE;
4349 }
4350 index = PHT_HASH(h);
4351 return get_pht_entry_from_index(GC_written_pages, index);
4352 # else
4353 /* TODO: implement me for MANUAL_VDB. */
4354 (void)h;
4355 return TRUE;
4356 # endif
4357 }
4358 # endif /* CHECKSUMS || PROC_VDB */
4359
4360 /* We expect block h to be written shortly. Ensure that all pages */
4361 /* containing any part of the n hblks starting at h are no longer */
4362 /* protected. If is_ptrfree is false, also ensure that they will */
4363 /* subsequently appear to be dirty. Not allowed to call GC_printf */
4364 /* (and the friends) here, see Win32 GC_stop_world for the details. */
4365 GC_INNER void GC_remove_protection(struct hblk *h, word nblocks,
4366 GC_bool is_ptrfree)
4367 {
4368 # ifdef PCR_VDB
4369 (void)is_ptrfree;
4370 if (!GC_auto_incremental)
4371 return;
4372 PCR_VD_WriteProtectDisable(h, nblocks*HBLKSIZE);
4373 PCR_VD_WriteProtectEnable(h, nblocks*HBLKSIZE);
4374 # elif defined(MPROTECT_VDB)
4375 struct hblk * h_trunc; /* Truncated to page boundary */
4376 struct hblk * h_end; /* Page boundary following block end */
4377 struct hblk * current;
4378
4379 if (!GC_auto_incremental || GC_GWW_AVAILABLE())
4380 return;
4381 GC_ASSERT(GC_page_size != 0);
4382 h_trunc = (struct hblk *)((word)h & ~(word)(GC_page_size-1));
4383 h_end = (struct hblk *)(((word)(h + nblocks) + GC_page_size - 1)
4384 & ~(word)(GC_page_size - 1));
4385 /* Note that we cannot examine GC_dirty_pages to check */
4386 /* whether the page at h_trunc has already been marked */
4387 /* dirty as there could be a hash collision. */
4388 for (current = h_trunc; (word)current < (word)h_end; ++current) {
4389 word index = PHT_HASH(current);
4390
4391 if (!is_ptrfree || (word)current < (word)h
4392 || (word)current >= (word)(h + nblocks)) {
4393 async_set_pht_entry_from_index(GC_dirty_pages, index);
4394 }
4395 }
4396 UNPROTECT(h_trunc, (ptr_t)h_end - (ptr_t)h_trunc);
4397 # else
4398 /* Ignore write hints. They don't help us here. */
4399 (void)h; (void)nblocks; (void)is_ptrfree;
4400 # endif
4401 }
4402 #endif /* !GC_DISABLE_INCREMENTAL */
4403
4404 #if defined(MPROTECT_VDB) && defined(DARWIN)
4405 /* The following sources were used as a "reference" for this exception
4406 handling code:
4407 1. Apple's mach/xnu documentation
4408 2. Timothy J. Wood's "Mach Exception Handlers 101" post to the
4409 omnigroup's macosx-dev list.
4410 www.omnigroup.com/mailman/archive/macosx-dev/2000-June/014178.html
4411 3. macosx-nat.c from Apple's GDB source code.
4412 */
4413
4414 /* The bug that caused all this trouble should now be fixed. This should
4415 eventually be removed if all goes well. */
4416
4417 /* #define BROKEN_EXCEPTION_HANDLING */
4418
4419 #include <mach/mach.h>
4420 #include <mach/mach_error.h>
4421 #include <mach/exception.h>
4422 #include <mach/task.h>
4423
4424 EXTERN_C_BEGIN
4425
4426 /* Some of the following prototypes are missing in any header, although */
4427 /* they are documented. Some are in mach/exc.h file. */
4428 extern boolean_t
4429 exc_server(mach_msg_header_t *, mach_msg_header_t *);
4430
4431 extern kern_return_t
4432 exception_raise(mach_port_t, mach_port_t, mach_port_t, exception_type_t,
4433 exception_data_t, mach_msg_type_number_t);
4434
4435 extern kern_return_t
4436 exception_raise_state(mach_port_t, mach_port_t, mach_port_t, exception_type_t,
4437 exception_data_t, mach_msg_type_number_t,
4438 thread_state_flavor_t*, thread_state_t,
4439 mach_msg_type_number_t, thread_state_t,
4440 mach_msg_type_number_t*);
4441
4442 extern kern_return_t
4443 exception_raise_state_identity(mach_port_t, mach_port_t, mach_port_t,
4444 exception_type_t, exception_data_t,
4445 mach_msg_type_number_t, thread_state_flavor_t*,
4446 thread_state_t, mach_msg_type_number_t,
4447 thread_state_t, mach_msg_type_number_t*);
4448
4449 GC_API_OSCALL kern_return_t
4450 catch_exception_raise(mach_port_t exception_port, mach_port_t thread,
4451 mach_port_t task, exception_type_t exception,
4452 exception_data_t code,
4453 mach_msg_type_number_t code_count);
4454
4455 GC_API_OSCALL kern_return_t
4456 catch_exception_raise_state(mach_port_name_t exception_port,
4457 int exception, exception_data_t code,
4458 mach_msg_type_number_t codeCnt, int flavor,
4459 thread_state_t old_state, int old_stateCnt,
4460 thread_state_t new_state, int new_stateCnt);
4461
4462 GC_API_OSCALL kern_return_t
4463 catch_exception_raise_state_identity(mach_port_name_t exception_port,
4464 mach_port_t thread, mach_port_t task, int exception,
4465 exception_data_t code, mach_msg_type_number_t codeCnt,
4466 int flavor, thread_state_t old_state, int old_stateCnt,
4467 thread_state_t new_state, int new_stateCnt);
4468
4469 EXTERN_C_END
4470
4471 /* These should never be called, but just in case... */
4472 GC_API_OSCALL kern_return_t
4473 catch_exception_raise_state(mach_port_name_t exception_port GC_ATTR_UNUSED,
4474 int exception GC_ATTR_UNUSED, exception_data_t code GC_ATTR_UNUSED,
4475 mach_msg_type_number_t codeCnt GC_ATTR_UNUSED, int flavor GC_ATTR_UNUSED,
4476 thread_state_t old_state GC_ATTR_UNUSED, int old_stateCnt GC_ATTR_UNUSED,
4477 thread_state_t new_state GC_ATTR_UNUSED, int new_stateCnt GC_ATTR_UNUSED)
4478 {
4479 ABORT_RET("Unexpected catch_exception_raise_state invocation");
4480 return(KERN_INVALID_ARGUMENT);
4481 }
4482
4483 GC_API_OSCALL kern_return_t
4484 catch_exception_raise_state_identity(
4485 mach_port_name_t exception_port GC_ATTR_UNUSED,
4486 mach_port_t thread GC_ATTR_UNUSED, mach_port_t task GC_ATTR_UNUSED,
4487 int exception GC_ATTR_UNUSED, exception_data_t code GC_ATTR_UNUSED,
4488 mach_msg_type_number_t codeCnt GC_ATTR_UNUSED, int flavor GC_ATTR_UNUSED,
4489 thread_state_t old_state GC_ATTR_UNUSED, int old_stateCnt GC_ATTR_UNUSED,
4490 thread_state_t new_state GC_ATTR_UNUSED, int new_stateCnt GC_ATTR_UNUSED)
4491 {
4492 ABORT_RET("Unexpected catch_exception_raise_state_identity invocation");
4493 return(KERN_INVALID_ARGUMENT);
4494 }
4495
4496 #define MAX_EXCEPTION_PORTS 16
4497
4498 static struct {
4499 mach_msg_type_number_t count;
4500 exception_mask_t masks[MAX_EXCEPTION_PORTS];
4501 exception_handler_t ports[MAX_EXCEPTION_PORTS];
4502 exception_behavior_t behaviors[MAX_EXCEPTION_PORTS];
4503 thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS];
4504 } GC_old_exc_ports;
4505
4506 STATIC struct ports_s {
4507 void (*volatile os_callback[3])(void);
4508 mach_port_t exception;
4509 # if defined(THREADS)
4510 mach_port_t reply;
4511 # endif
4512 } GC_ports = {
4513 {
4514 /* This is to prevent stripping these routines as dead. */
4515 (void (*)(void))catch_exception_raise,
4516 (void (*)(void))catch_exception_raise_state,
4517 (void (*)(void))catch_exception_raise_state_identity
4518 },
4519 # ifdef THREADS
4520 0, /* for 'exception' */
4521 # endif
4522 0
4523 };
4524
4525 typedef struct {
4526 mach_msg_header_t head;
4527 } GC_msg_t;
4528
4529 typedef enum {
4530 GC_MP_NORMAL,
4531 GC_MP_DISCARDING,
4532 GC_MP_STOPPED
4533 } GC_mprotect_state_t;
4534
4535 #ifdef THREADS
4536 /* FIXME: 1 and 2 seem to be safe to use in the msgh_id field, but it */
4537 /* is not documented. Use the source and see if they should be OK. */
4538 # define ID_STOP 1
4539 # define ID_RESUME 2
4540
4541 /* This value is only used on the reply port. */
4542 # define ID_ACK 3
4543
4544 STATIC GC_mprotect_state_t GC_mprotect_state = GC_MP_NORMAL;
4545
4546 /* The following should ONLY be called when the world is stopped. */
4547 STATIC void GC_mprotect_thread_notify(mach_msg_id_t id)
4548 {
4549 struct buf_s {
4550 GC_msg_t msg;
4551 mach_msg_trailer_t trailer;
4552 } buf;
4553 mach_msg_return_t r;
4554
4555 /* remote, local */
4556 buf.msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
4557 buf.msg.head.msgh_size = sizeof(buf.msg);
4558 buf.msg.head.msgh_remote_port = GC_ports.exception;
4559 buf.msg.head.msgh_local_port = MACH_PORT_NULL;
4560 buf.msg.head.msgh_id = id;
4561
4562 r = mach_msg(&buf.msg.head, MACH_SEND_MSG | MACH_RCV_MSG | MACH_RCV_LARGE,
4563 sizeof(buf.msg), sizeof(buf), GC_ports.reply,
4564 MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
4565 if (r != MACH_MSG_SUCCESS)
4566 ABORT("mach_msg failed in GC_mprotect_thread_notify");
4567 if (buf.msg.head.msgh_id != ID_ACK)
4568 ABORT("Invalid ack in GC_mprotect_thread_notify");
4569 }
4570
4571 /* Should only be called by the mprotect thread */
4572 STATIC void GC_mprotect_thread_reply(void)
4573 {
4574 GC_msg_t msg;
4575 mach_msg_return_t r;
4576 /* remote, local */
4577
4578 msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
4579 msg.head.msgh_size = sizeof(msg);
4580 msg.head.msgh_remote_port = GC_ports.reply;
4581 msg.head.msgh_local_port = MACH_PORT_NULL;
4582 msg.head.msgh_id = ID_ACK;
4583
4584 r = mach_msg(&msg.head, MACH_SEND_MSG, sizeof(msg), 0, MACH_PORT_NULL,
4585 MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
4586 if (r != MACH_MSG_SUCCESS)
4587 ABORT("mach_msg failed in GC_mprotect_thread_reply");
4588 }
4589
4590 GC_INNER void GC_mprotect_stop(void)
4591 {
4592 GC_mprotect_thread_notify(ID_STOP);
4593 }
4594
4595 GC_INNER void GC_mprotect_resume(void)
4596 {
4597 GC_mprotect_thread_notify(ID_RESUME);
4598 }
4599
4600 # ifdef CAN_HANDLE_FORK
4601 GC_INNER void GC_dirty_update_child(void)
4602 {
4603 GC_ASSERT(I_HOLD_LOCK());
4604 if (0 == GC_task_self) return; /* GC incremental mode is off */
4605
4606 GC_ASSERT(GC_mprotect_state == GC_MP_NORMAL);
4607 GC_task_self = mach_task_self(); /* needed by UNPROTECT() */
4608 GC_unprotect_all_heap();
4609
4610 /* Restore the old task exception ports. */
4611 /* TODO: Should we do it in fork_prepare/parent_proc? */
4612 if (GC_old_exc_ports.count > 0) {
4613 /* TODO: Should we check GC_old_exc_ports.count<=1? */
4614 if (task_set_exception_ports(GC_task_self, GC_old_exc_ports.masks[0],
4615 GC_old_exc_ports.ports[0], GC_old_exc_ports.behaviors[0],
4616 GC_old_exc_ports.flavors[0]) != KERN_SUCCESS)
4617 ABORT("task_set_exception_ports failed (in child)");
4618 }
4619
4620 /* TODO: Re-enable incremental mode in child. */
4621 GC_task_self = 0;
4622 GC_incremental = FALSE;
4623 }
4624 # endif /* CAN_HANDLE_FORK */
4625
4626 #else
4627 /* The compiler should optimize away any GC_mprotect_state computations */
4628 # define GC_mprotect_state GC_MP_NORMAL
4629 #endif /* !THREADS */
4630
4631 struct mp_reply_s {
4632 mach_msg_header_t head;
4633 char data[256];
4634 };
4635
4636 struct mp_msg_s {
4637 mach_msg_header_t head;
4638 mach_msg_body_t msgh_body;
4639 char data[1024];
4640 };
4641
4642 STATIC void *GC_mprotect_thread(void *arg)
4643 {
4644 mach_msg_return_t r;
4645 /* These two structures contain some private kernel data. We don't */
4646 /* need to access any of it so we don't bother defining a proper */
4647 /* struct. The correct definitions are in the xnu source code. */
4648 struct mp_reply_s reply;
4649 struct mp_msg_s msg;
4650 mach_msg_id_t id;
4651
4652 if ((word)arg == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
4653 # if defined(CPPCHECK)
4654 reply.data[0] = 0; /* to prevent "field unused" warnings */
4655 msg.data[0] = 0;
4656 # endif
4657
4658 # if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
4659 (void)pthread_setname_np("GC-mprotect");
4660 # endif
4661 # if defined(THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
4662 GC_darwin_register_mach_handler_thread(mach_thread_self());
4663 # endif
4664
4665 for (;;) {
4666 r = mach_msg(&msg.head, MACH_RCV_MSG | MACH_RCV_LARGE |
4667 (GC_mprotect_state == GC_MP_DISCARDING ? MACH_RCV_TIMEOUT
4668 : 0), 0, sizeof(msg), GC_ports.exception,
4669 GC_mprotect_state == GC_MP_DISCARDING ? 0
4670 : MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
4671 id = r == MACH_MSG_SUCCESS ? msg.head.msgh_id : -1;
4672
4673 # if defined(THREADS)
4674 if (GC_mprotect_state == GC_MP_DISCARDING) {
4675 if (r == MACH_RCV_TIMED_OUT) {
4676 GC_mprotect_state = GC_MP_STOPPED;
4677 GC_mprotect_thread_reply();
4678 continue;
4679 }
4680 if (r == MACH_MSG_SUCCESS && (id == ID_STOP || id == ID_RESUME))
4681 ABORT("Out of order mprotect thread request");
4682 }
4683 # endif /* THREADS */
4684
4685 if (r != MACH_MSG_SUCCESS) {
4686 ABORT_ARG2("mach_msg failed",
4687 ": errcode= %d (%s)", (int)r, mach_error_string(r));
4688 }
4689
4690 switch (id) {
4691 # if defined(THREADS)
4692 case ID_STOP:
4693 if (GC_mprotect_state != GC_MP_NORMAL)
4694 ABORT("Called mprotect_stop when state wasn't normal");
4695 GC_mprotect_state = GC_MP_DISCARDING;
4696 break;
4697 case ID_RESUME:
4698 if (GC_mprotect_state != GC_MP_STOPPED)
4699 ABORT("Called mprotect_resume when state wasn't stopped");
4700 GC_mprotect_state = GC_MP_NORMAL;
4701 GC_mprotect_thread_reply();
4702 break;
4703 # endif /* THREADS */
4704 default:
4705 /* Handle the message (calls catch_exception_raise) */
4706 if (!exc_server(&msg.head, &reply.head))
4707 ABORT("exc_server failed");
4708 /* Send the reply */
4709 r = mach_msg(&reply.head, MACH_SEND_MSG, reply.head.msgh_size, 0,
4710 MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE,
4711 MACH_PORT_NULL);
4712 if (r != MACH_MSG_SUCCESS) {
4713 /* This will fail if the thread dies, but the thread */
4714 /* shouldn't die... */
4715 # ifdef BROKEN_EXCEPTION_HANDLING
4716 GC_err_printf("mach_msg failed with %d %s while sending "
4717 "exc reply\n", (int)r, mach_error_string(r));
4718 # else
4719 ABORT("mach_msg failed while sending exception reply");
4720 # endif
4721 }
4722 } /* switch */
4723 } /* for */
4724 }
4725
4726 /* All this SIGBUS code shouldn't be necessary. All protection faults should
4727 be going through the mach exception handler. However, it seems a SIGBUS is
4728 occasionally sent for some unknown reason. Even more odd, it seems to be
4729 meaningless and safe to ignore. */
4730 #ifdef BROKEN_EXCEPTION_HANDLING
4731
4732 /* Updates to this aren't atomic, but the SIGBUS'es seem pretty rare. */
4733 /* Even if this doesn't get updated property, it isn't really a problem. */
4734 STATIC int GC_sigbus_count = 0;
4735
4736 STATIC void GC_darwin_sigbus(int num, siginfo_t *sip, void *context)
4737 {
4738 if (num != SIGBUS)
4739 ABORT("Got a non-sigbus signal in the sigbus handler");
4740
4741 /* Ugh... some seem safe to ignore, but too many in a row probably means
4742 trouble. GC_sigbus_count is reset for each mach exception that is
4743 handled */
4744 if (GC_sigbus_count >= 8) {
4745 ABORT("Got more than 8 SIGBUSs in a row!");
4746 } else {
4747 GC_sigbus_count++;
4748 WARN("Ignoring SIGBUS\n", 0);
4749 }
4750 }
4751 #endif /* BROKEN_EXCEPTION_HANDLING */
4752
4753 GC_INNER GC_bool GC_dirty_init(void)
4754 {
4755 kern_return_t r;
4756 mach_port_t me;
4757 pthread_t thread;
4758 pthread_attr_t attr;
4759 exception_mask_t mask;
4760
4761 # if defined(CAN_HANDLE_FORK) && !defined(THREADS)
4762 if (GC_handle_fork) {
4763 /* To both support GC incremental mode and GC functions usage in */
4764 /* the forked child, pthread_atfork should be used to install */
4765 /* handlers that switch off GC_incremental in the child */
4766 /* gracefully (unprotecting all pages and clearing */
4767 /* GC_mach_handler_thread). For now, we just disable incremental */
4768 /* mode if fork() handling is requested by the client. */
4769 WARN("Can't turn on GC incremental mode as fork()"
4770 " handling requested\n", 0);
4771 return FALSE;
4772 }
4773 # endif
4774
4775 GC_VERBOSE_LOG_PRINTF("Initializing mach/darwin mprotect"
4776 " virtual dirty bit implementation\n");
4777 # ifdef BROKEN_EXCEPTION_HANDLING
4778 WARN("Enabling workarounds for various darwin "
4779 "exception handling bugs\n", 0);
4780 # endif
4781 if (GC_page_size % HBLKSIZE != 0) {
4782 ABORT("Page size not multiple of HBLKSIZE");
4783 }
4784
4785 GC_task_self = me = mach_task_self();
4786 GC_ASSERT(me != 0);
4787
4788 r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.exception);
4789 /* TODO: WARN and return FALSE in case of a failure. */
4790 if (r != KERN_SUCCESS)
4791 ABORT("mach_port_allocate failed (exception port)");
4792
4793 r = mach_port_insert_right(me, GC_ports.exception, GC_ports.exception,
4794 MACH_MSG_TYPE_MAKE_SEND);
4795 if (r != KERN_SUCCESS)
4796 ABORT("mach_port_insert_right failed (exception port)");
4797
4798 # if defined(THREADS)
4799 r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.reply);
4800 if(r != KERN_SUCCESS)
4801 ABORT("mach_port_allocate failed (reply port)");
4802 # endif
4803
4804 /* The exceptions we want to catch. */
4805 mask = EXC_MASK_BAD_ACCESS;
4806 r = task_get_exception_ports(me, mask, GC_old_exc_ports.masks,
4807 &GC_old_exc_ports.count, GC_old_exc_ports.ports,
4808 GC_old_exc_ports.behaviors,
4809 GC_old_exc_ports.flavors);
4810 if (r != KERN_SUCCESS)
4811 ABORT("task_get_exception_ports failed");
4812
4813 r = task_set_exception_ports(me, mask, GC_ports.exception, EXCEPTION_DEFAULT,
4814 GC_MACH_THREAD_STATE);
4815 if (r != KERN_SUCCESS)
4816 ABORT("task_set_exception_ports failed");
4817
4818 if (pthread_attr_init(&attr) != 0)
4819 ABORT("pthread_attr_init failed");
4820 if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
4821 ABORT("pthread_attr_setdetachedstate failed");
4822 /* This will call the real pthread function, not our wrapper. */
4823 if (GC_inner_pthread_create(&thread, &attr, GC_mprotect_thread, NULL) != 0)
4824 ABORT("pthread_create failed");
4825 (void)pthread_attr_destroy(&attr);
4826
4827 /* Setup the sigbus handler for ignoring the meaningless SIGBUSs */
4828 # ifdef BROKEN_EXCEPTION_HANDLING
4829 {
4830 struct sigaction sa, oldsa;
4831 sa.sa_handler = (SIG_HNDLR_PTR)GC_darwin_sigbus;
4832 sigemptyset(&sa.sa_mask);
4833 sa.sa_flags = SA_RESTART|SA_SIGINFO;
4834 /* sa.sa_restorer is deprecated and should not be initialized. */
4835 if (sigaction(SIGBUS, &sa, &oldsa) < 0)
4836 ABORT("sigaction failed");
4837 if (oldsa.sa_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) {
4838 GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
4839 }
4840 }
4841 # endif /* BROKEN_EXCEPTION_HANDLING */
4842 # if defined(CPPCHECK)
4843 GC_noop1((word)GC_ports.os_callback[0]);
4844 # endif
4845 return TRUE;
4846 }
4847
4848 /* The source code for Apple's GDB was used as a reference for the */
4849 /* exception forwarding code. This code is similar to be GDB code only */
4850 /* because there is only one way to do it. */
4851 STATIC kern_return_t GC_forward_exception(mach_port_t thread, mach_port_t task,
4852 exception_type_t exception,
4853 exception_data_t data,
4854 mach_msg_type_number_t data_count)
4855 {
4856 unsigned int i;
4857 kern_return_t r;
4858 mach_port_t port;
4859 exception_behavior_t behavior;
4860 thread_state_flavor_t flavor;
4861
4862 thread_state_data_t thread_state;
4863 mach_msg_type_number_t thread_state_count = THREAD_STATE_MAX;
4864
4865 for (i = 0; i < GC_old_exc_ports.count; i++) {
4866 if ((GC_old_exc_ports.masks[i] & ((exception_mask_t)1 << exception)) != 0)
4867 break;
4868 }
4869 if (i == GC_old_exc_ports.count)
4870 ABORT("No handler for exception!");
4871
4872 port = GC_old_exc_ports.ports[i];
4873 behavior = GC_old_exc_ports.behaviors[i];
4874 flavor = GC_old_exc_ports.flavors[i];
4875
4876 if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
4877 r = thread_get_state(thread, flavor, thread_state, &thread_state_count);
4878 if (r != KERN_SUCCESS)
4879 ABORT("thread_get_state failed in forward_exception");
4880 }
4881
4882 switch (behavior) {
4883 case EXCEPTION_STATE:
4884 r = exception_raise_state(port, thread, task, exception, data,
4885 data_count, &flavor, thread_state,
4886 thread_state_count, thread_state,
4887 &thread_state_count);
4888 break;
4889 case EXCEPTION_STATE_IDENTITY:
4890 r = exception_raise_state_identity(port, thread, task, exception, data,
4891 data_count, &flavor, thread_state,
4892 thread_state_count, thread_state,
4893 &thread_state_count);
4894 break;
4895 /* case EXCEPTION_DEFAULT: */ /* default signal handlers */
4896 default: /* user-supplied signal handlers */
4897 r = exception_raise(port, thread, task, exception, data, data_count);
4898 }
4899
4900 if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
4901 r = thread_set_state(thread, flavor, thread_state, thread_state_count);
4902 if (r != KERN_SUCCESS)
4903 ABORT("thread_set_state failed in forward_exception");
4904 }
4905 return r;
4906 }
4907
4908 #define FWD() GC_forward_exception(thread, task, exception, code, code_count)
4909
4910 #ifdef ARM32
4911 # define DARWIN_EXC_STATE ARM_EXCEPTION_STATE
4912 # define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE_COUNT
4913 # define DARWIN_EXC_STATE_T arm_exception_state_t
4914 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far)
4915 #elif defined(AARCH64)
4916 # define DARWIN_EXC_STATE ARM_EXCEPTION_STATE64
4917 # define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE64_COUNT
4918 # define DARWIN_EXC_STATE_T arm_exception_state64_t
4919 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far)
4920 #elif defined(POWERPC)
4921 # if CPP_WORDSZ == 32
4922 # define DARWIN_EXC_STATE PPC_EXCEPTION_STATE
4923 # define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE_COUNT
4924 # define DARWIN_EXC_STATE_T ppc_exception_state_t
4925 # else
4926 # define DARWIN_EXC_STATE PPC_EXCEPTION_STATE64
4927 # define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE64_COUNT
4928 # define DARWIN_EXC_STATE_T ppc_exception_state64_t
4929 # endif
4930 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(dar)
4931 #elif defined(I386) || defined(X86_64)
4932 # if CPP_WORDSZ == 32
4933 # if defined(i386_EXCEPTION_STATE_COUNT) \
4934 && !defined(x86_EXCEPTION_STATE32_COUNT)
4935 /* Use old naming convention for 32-bit x86. */
4936 # define DARWIN_EXC_STATE i386_EXCEPTION_STATE
4937 # define DARWIN_EXC_STATE_COUNT i386_EXCEPTION_STATE_COUNT
4938 # define DARWIN_EXC_STATE_T i386_exception_state_t
4939 # else
4940 # define DARWIN_EXC_STATE x86_EXCEPTION_STATE32
4941 # define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE32_COUNT
4942 # define DARWIN_EXC_STATE_T x86_exception_state32_t
4943 # endif
4944 # else
4945 # define DARWIN_EXC_STATE x86_EXCEPTION_STATE64
4946 # define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE64_COUNT
4947 # define DARWIN_EXC_STATE_T x86_exception_state64_t
4948 # endif
4949 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(faultvaddr)
4950 #elif !defined(CPPCHECK)
4951 # error FIXME for non-arm/ppc/x86 darwin
4952 #endif
4953
4954 /* This violates the namespace rules but there isn't anything that can */
4955 /* be done about it. The exception handling stuff is hard coded to */
4956 /* call this. catch_exception_raise, catch_exception_raise_state and */
4957 /* and catch_exception_raise_state_identity are called from OS. */
4958 GC_API_OSCALL kern_return_t
4959 catch_exception_raise(mach_port_t exception_port GC_ATTR_UNUSED,
4960 mach_port_t thread, mach_port_t task GC_ATTR_UNUSED,
4961 exception_type_t exception, exception_data_t code,
4962 mach_msg_type_number_t code_count GC_ATTR_UNUSED)
4963 {
4964 kern_return_t r;
4965 char *addr;
4966 thread_state_flavor_t flavor = DARWIN_EXC_STATE;
4967 mach_msg_type_number_t exc_state_count = DARWIN_EXC_STATE_COUNT;
4968 DARWIN_EXC_STATE_T exc_state;
4969
4970 if (exception != EXC_BAD_ACCESS || code[0] != KERN_PROTECTION_FAILURE) {
4971 # ifdef DEBUG_EXCEPTION_HANDLING
4972 /* We aren't interested, pass it on to the old handler */
4973 GC_log_printf("Exception: 0x%x Code: 0x%x 0x%x in catch...\n",
4974 exception, code_count > 0 ? code[0] : -1,
4975 code_count > 1 ? code[1] : -1);
4976 # endif
4977 return FWD();
4978 }
4979
4980 r = thread_get_state(thread, flavor, (natural_t*)&exc_state,
4981 &exc_state_count);
4982 if (r != KERN_SUCCESS) {
4983 /* The thread is supposed to be suspended while the exception */
4984 /* handler is called. This shouldn't fail. */
4985 # ifdef BROKEN_EXCEPTION_HANDLING
4986 GC_err_printf("thread_get_state failed in catch_exception_raise\n");
4987 return KERN_SUCCESS;
4988 # else
4989 ABORT("thread_get_state failed in catch_exception_raise");
4990 # endif
4991 }
4992
4993 /* This is the address that caused the fault */
4994 addr = (char*) exc_state.DARWIN_EXC_STATE_DAR;
4995 if (!is_header_found_async(addr)) {
4996 /* Ugh... just like the SIGBUS problem above, it seems we get */
4997 /* a bogus KERN_PROTECTION_FAILURE every once and a while. We wait */
4998 /* till we get a bunch in a row before doing anything about it. */
4999 /* If a "real" fault ever occurs it'll just keep faulting over and */
5000 /* over and we'll hit the limit pretty quickly. */
5001 # ifdef BROKEN_EXCEPTION_HANDLING
5002 static char *last_fault;
5003 static int last_fault_count;
5004
5005 if (addr != last_fault) {
5006 last_fault = addr;
5007 last_fault_count = 0;
5008 }
5009 if (++last_fault_count < 32) {
5010 if (last_fault_count == 1)
5011 WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n", addr);
5012 return KERN_SUCCESS;
5013 }
5014
5015 GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p; aborting...\n",
5016 (void *)addr);
5017 /* Can't pass it along to the signal handler because that is */
5018 /* ignoring SIGBUS signals. We also shouldn't call ABORT here as */
5019 /* signals don't always work too well from the exception handler. */
5020 EXIT();
5021 # else /* BROKEN_EXCEPTION_HANDLING */
5022 /* Pass it along to the next exception handler
5023 (which should call SIGBUS/SIGSEGV) */
5024 return FWD();
5025 # endif /* !BROKEN_EXCEPTION_HANDLING */
5026 }
5027
5028 # ifdef BROKEN_EXCEPTION_HANDLING
5029 /* Reset the number of consecutive SIGBUSs */
5030 GC_sigbus_count = 0;
5031 # endif
5032
5033 GC_ASSERT(GC_page_size != 0);
5034 if (GC_mprotect_state == GC_MP_NORMAL) { /* common case */
5035 struct hblk * h = (struct hblk *)((word)addr & ~(word)(GC_page_size-1));
5036 size_t i;
5037
5038 # ifdef CHECKSUMS
5039 GC_record_fault(h);
5040 # endif
5041 UNPROTECT(h, GC_page_size);
5042 for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
5043 word index = PHT_HASH(h+i);
5044 async_set_pht_entry_from_index(GC_dirty_pages, index);
5045 }
5046 } else if (GC_mprotect_state == GC_MP_DISCARDING) {
5047 /* Lie to the thread for now. No sense UNPROTECT()ing the memory
5048 when we're just going to PROTECT() it again later. The thread
5049 will just fault again once it resumes */
5050 } else {
5051 /* Shouldn't happen, i don't think */
5052 GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n");
5053 return FWD();
5054 }
5055 return KERN_SUCCESS;
5056 }
5057 #undef FWD
5058
5059 #ifndef NO_DESC_CATCH_EXCEPTION_RAISE
5060 /* These symbols should have REFERENCED_DYNAMICALLY (0x10) bit set to */
5061 /* let strip know they are not to be stripped. */
5062 __asm__(".desc _catch_exception_raise, 0x10");
5063 __asm__(".desc _catch_exception_raise_state, 0x10");
5064 __asm__(".desc _catch_exception_raise_state_identity, 0x10");
5065 #endif
5066
5067 #endif /* DARWIN && MPROTECT_VDB */
5068
5069 #ifndef HAVE_INCREMENTAL_PROTECTION_NEEDS
5070 GC_API int GC_CALL GC_incremental_protection_needs(void)
5071 {
5072 GC_ASSERT(GC_is_initialized);
5073 return GC_PROTECTS_NONE;
5074 }
5075 #endif /* !HAVE_INCREMENTAL_PROTECTION_NEEDS */
5076
5077 #ifdef ECOS
5078 /* Undo sbrk() redirection. */
5079 # undef sbrk
5080 #endif
5081
5082 /* If value is non-zero then allocate executable memory. */
5083 GC_API void GC_CALL GC_set_pages_executable(int value)
5084 {
5085 GC_ASSERT(!GC_is_initialized);
5086 /* Even if IGNORE_PAGES_EXECUTABLE is defined, GC_pages_executable is */
5087 /* touched here to prevent a compiler warning. */
5088 GC_pages_executable = (GC_bool)(value != 0);
5089 }
5090
5091 /* Returns non-zero if the GC-allocated memory is executable. */
5092 /* GC_get_pages_executable is defined after all the places */
5093 /* where GC_get_pages_executable is undefined. */
5094 GC_API int GC_CALL GC_get_pages_executable(void)
5095 {
5096 # ifdef IGNORE_PAGES_EXECUTABLE
5097 return 1; /* Always allocate executable memory. */
5098 # else
5099 return (int)GC_pages_executable;
5100 # endif
5101 }
5102
5103 /* Call stack save code for debugging. Should probably be in */
5104 /* mach_dep.c, but that requires reorganization. */
5105
5106 /* I suspect the following works for most *nix x86 variants, so */
5107 /* long as the frame pointer is explicitly stored. In the case of gcc, */
5108 /* compiler flags (e.g. -fomit-frame-pointer) determine whether it is. */
5109 #if defined(I386) && defined(LINUX) && defined(SAVE_CALL_CHAIN)
5110 # include <features.h>
5111
5112 struct frame {
5113 struct frame *fr_savfp;
5114 long fr_savpc;
5115 # if NARGS > 0
5116 long fr_arg[NARGS]; /* All the arguments go here. */
5117 # endif
5118 };
5119 #endif
5120
5121 #if defined(SPARC)
5122 # if defined(LINUX)
5123 # include <features.h>
5124
5125 # if defined(SAVE_CALL_CHAIN)
5126 struct frame {
5127 long fr_local[8];
5128 long fr_arg[6];
5129 struct frame *fr_savfp;
5130 long fr_savpc;
5131 # ifndef __arch64__
5132 char *fr_stret;
5133 # endif
5134 long fr_argd[6];
5135 long fr_argx[0];
5136 };
5137 # endif
5138 # elif defined (DRSNX)
5139 # include <sys/sparc/frame.h>
5140 # elif defined(OPENBSD)
5141 # include <frame.h>
5142 # elif defined(FREEBSD) || defined(NETBSD)
5143 # include <machine/frame.h>
5144 # else
5145 # include <sys/frame.h>
5146 # endif
5147 # if NARGS > 6
5148 # error We only know how to get the first 6 arguments
5149 # endif
5150 #endif /* SPARC */
5151
5152 #ifdef NEED_CALLINFO
5153 /* Fill in the pc and argument information for up to NFRAMES of my */
5154 /* callers. Ignore my frame and my callers frame. */
5155
5156 #ifdef LINUX
5157 # include <unistd.h>
5158 #endif
5159
5160 #endif /* NEED_CALLINFO */
5161
5162 #if defined(GC_HAVE_BUILTIN_BACKTRACE)
5163 # ifdef _MSC_VER
5164 # include "private/msvc_dbg.h"
5165 # else
5166 # include <execinfo.h>
5167 # endif
5168 #endif
5169
5170 #ifdef SAVE_CALL_CHAIN
5171
5172 #if NARGS == 0 && NFRAMES % 2 == 0 /* No padding */ \
5173 && defined(GC_HAVE_BUILTIN_BACKTRACE)
5174
5175 #ifdef REDIRECT_MALLOC
5176 /* Deal with possible malloc calls in backtrace by omitting */
5177 /* the infinitely recursing backtrace. */
5178 # ifdef THREADS
5179 __thread /* If your compiler doesn't understand this */
5180 /* you could use something like pthread_getspecific. */
5181 # endif
5182 GC_bool GC_in_save_callers = FALSE;
5183
5184 # if defined(THREADS) && defined(DBG_HDRS_ALL)
5185 # include "private/dbg_mlc.h"
5186
5187 /* A dummy version of GC_save_callers() which does not call */
5188 /* backtrace(). */
5189 GC_INNER void GC_save_callers_no_unlock(struct callinfo info[NFRAMES])
5190 {
5191 GC_ASSERT(I_HOLD_LOCK());
5192 info[0].ci_pc = (word)(&GC_save_callers_no_unlock);
5193 BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
5194 }
5195 # endif
5196 #endif /* REDIRECT_MALLOC */
5197
5198 GC_INNER void GC_save_callers(struct callinfo info[NFRAMES])
5199 {
5200 void * tmp_info[NFRAMES + 1];
5201 int npcs, i;
5202
5203 GC_ASSERT(I_HOLD_LOCK());
5204 /* backtrace may call dl_iterate_phdr which is also */
5205 /* used by GC_register_dynamic_libraries, and */
5206 /* dl_iterate_phdr is not guaranteed to be reentrant. */
5207
5208 GC_STATIC_ASSERT(sizeof(struct callinfo) == sizeof(void *));
5209 # ifdef REDIRECT_MALLOC
5210 if (GC_in_save_callers) {
5211 info[0].ci_pc = (word)(&GC_save_callers);
5212 for (i = 1; i < NFRAMES; ++i) info[i].ci_pc = 0;
5213 return;
5214 }
5215 GC_in_save_callers = TRUE;
5216 /* backtrace() might call a redirected malloc. */
5217 UNLOCK();
5218 npcs = backtrace((void **)tmp_info, NFRAMES + 1);
5219 LOCK();
5220 # else
5221 npcs = backtrace((void **)tmp_info, NFRAMES + 1);
5222 # endif
5223 /* We retrieve NFRAMES+1 pc values, but discard the first one, since */
5224 /* it points to our own frame. */
5225 i = 0;
5226 if (npcs > 1) {
5227 i = npcs - 1;
5228 BCOPY(&tmp_info[1], info, (unsigned)i * sizeof(void *));
5229 }
5230 for (; i < NFRAMES; ++i) info[i].ci_pc = 0;
5231 # ifdef REDIRECT_MALLOC
5232 GC_in_save_callers = FALSE;
5233 # endif
5234 }
5235
5236 #else /* No builtin backtrace; do it ourselves */
5237
5238 #if (defined(OPENBSD) || defined(NETBSD) || defined(FREEBSD)) && defined(SPARC)
5239 # define FR_SAVFP fr_fp
5240 # define FR_SAVPC fr_pc
5241 #else
5242 # define FR_SAVFP fr_savfp
5243 # define FR_SAVPC fr_savpc
5244 #endif
5245
5246 #if defined(SPARC) && (defined(__arch64__) || defined(__sparcv9))
5247 # define BIAS 2047
5248 #else
5249 # define BIAS 0
5250 #endif
5251
5252 GC_INNER void GC_save_callers(struct callinfo info[NFRAMES])
5253 {
5254 struct frame *frame;
5255 struct frame *fp;
5256 int nframes = 0;
5257 # ifdef I386
5258 /* We assume this is turned on only with gcc as the compiler. */
5259 asm("movl %%ebp,%0" : "=r"(frame));
5260 fp = frame;
5261 # else
5262 frame = (struct frame *)GC_save_regs_in_stack();
5263 fp = (struct frame *)((long) frame -> FR_SAVFP + BIAS);
5264 #endif
5265
5266 for (; !((word)fp HOTTER_THAN (word)frame)
5267 # ifndef THREADS
5268 && !((word)GC_stackbottom HOTTER_THAN (word)fp)
5269 # elif defined(STACK_GROWS_UP)
5270 && fp != NULL
5271 # endif
5272 && nframes < NFRAMES;
5273 fp = (struct frame *)((long) fp -> FR_SAVFP + BIAS), nframes++) {
5274 # if NARGS > 0
5275 int i;
5276 # endif
5277
5278 info[nframes].ci_pc = fp->FR_SAVPC;
5279 # if NARGS > 0
5280 for (i = 0; i < NARGS; i++) {
5281 info[nframes].ci_arg[i] = ~(fp->fr_arg[i]);
5282 }
5283 # endif /* NARGS > 0 */
5284 }
5285 if (nframes < NFRAMES) info[nframes].ci_pc = 0;
5286 }
5287
5288 #endif /* No builtin backtrace */
5289
5290 #endif /* SAVE_CALL_CHAIN */
5291
5292 #ifdef NEED_CALLINFO
5293
5294 /* Print info to stderr. We do NOT hold the allocation lock */
5295 GC_INNER void GC_print_callers(struct callinfo info[NFRAMES])
5296 {
5297 int i;
5298 static int reentry_count = 0;
5299 DCL_LOCK_STATE;
5300
5301 /* FIXME: This should probably use a different lock, so that we */
5302 /* become callable with or without the allocation lock. */
5303 LOCK();
5304 ++reentry_count;
5305 UNLOCK();
5306
5307 # if NFRAMES == 1
5308 GC_err_printf("\tCaller at allocation:\n");
5309 # else
5310 GC_err_printf("\tCall chain at allocation:\n");
5311 # endif
5312 for (i = 0; i < NFRAMES; i++) {
5313 # if defined(LINUX) && !defined(SMALL_CONFIG)
5314 GC_bool stop = FALSE;
5315 # endif
5316
5317 if (0 == info[i].ci_pc)
5318 break;
5319 # if NARGS > 0
5320 {
5321 int j;
5322
5323 GC_err_printf("\t\targs: ");
5324 for (j = 0; j < NARGS; j++) {
5325 if (j != 0) GC_err_printf(", ");
5326 GC_err_printf("%d (0x%X)", ~(info[i].ci_arg[j]),
5327 ~(info[i].ci_arg[j]));
5328 }
5329 GC_err_printf("\n");
5330 }
5331 # endif
5332 if (reentry_count > 1) {
5333 /* We were called during an allocation during */
5334 /* a previous GC_print_callers call; punt. */
5335 GC_err_printf("\t\t##PC##= 0x%lx\n",
5336 (unsigned long)info[i].ci_pc);
5337 continue;
5338 }
5339 {
5340 char buf[40];
5341 char *name;
5342 # if defined(GC_HAVE_BUILTIN_BACKTRACE) \
5343 && !defined(GC_BACKTRACE_SYMBOLS_BROKEN)
5344 char **sym_name =
5345 backtrace_symbols((void **)(&(info[i].ci_pc)), 1);
5346 if (sym_name != NULL) {
5347 name = sym_name[0];
5348 } else
5349 # endif
5350 /* else */ {
5351 (void)snprintf(buf, sizeof(buf), "##PC##= 0x%lx",
5352 (unsigned long)info[i].ci_pc);
5353 buf[sizeof(buf) - 1] = '\0';
5354 name = buf;
5355 }
5356 # if defined(LINUX) && !defined(SMALL_CONFIG)
5357 /* Try for a line number. */
5358 do {
5359 FILE *pipe;
5360 # define EXE_SZ 100
5361 static char exe_name[EXE_SZ];
5362 # define CMD_SZ 200
5363 char cmd_buf[CMD_SZ];
5364 # define RESULT_SZ 200
5365 static char result_buf[RESULT_SZ];
5366 size_t result_len;
5367 char *old_preload;
5368 # define PRELOAD_SZ 200
5369 char preload_buf[PRELOAD_SZ];
5370 static GC_bool found_exe_name = FALSE;
5371 static GC_bool will_fail = FALSE;
5372
5373 /* Try to get it via a hairy and expensive scheme. */
5374 /* First we get the name of the executable: */
5375 if (will_fail)
5376 break;
5377 if (!found_exe_name) {
5378 int ret_code = readlink("/proc/self/exe", exe_name, EXE_SZ);
5379
5380 if (ret_code < 0 || ret_code >= EXE_SZ
5381 || exe_name[0] != '/') {
5382 will_fail = TRUE; /* Don't try again. */
5383 break;
5384 }
5385 exe_name[ret_code] = '\0';
5386 found_exe_name = TRUE;
5387 }
5388 /* Then we use popen to start addr2line -e <exe> <addr> */
5389 /* There are faster ways to do this, but hopefully this */
5390 /* isn't time critical. */
5391 (void)snprintf(cmd_buf, sizeof(cmd_buf),
5392 "/usr/bin/addr2line -f -e %s 0x%lx",
5393 exe_name, (unsigned long)info[i].ci_pc);
5394 cmd_buf[sizeof(cmd_buf) - 1] = '\0';
5395 old_preload = GETENV("LD_PRELOAD");
5396 if (0 != old_preload) {
5397 size_t old_len = strlen(old_preload);
5398 if (old_len >= PRELOAD_SZ) {
5399 will_fail = TRUE;
5400 break;
5401 }
5402 BCOPY(old_preload, preload_buf, old_len + 1);
5403 unsetenv ("LD_PRELOAD");
5404 }
5405 pipe = popen(cmd_buf, "r");
5406 if (0 != old_preload
5407 && 0 != setenv ("LD_PRELOAD", preload_buf, 0)) {
5408 WARN("Failed to reset LD_PRELOAD\n", 0);
5409 }
5410 if (NULL == pipe) {
5411 will_fail = TRUE;
5412 break;
5413 }
5414 result_len = fread(result_buf, 1, RESULT_SZ - 1, pipe);
5415 (void)pclose(pipe);
5416 if (0 == result_len) {
5417 will_fail = TRUE;
5418 break;
5419 }
5420 if (result_buf[result_len - 1] == '\n') --result_len;
5421 result_buf[result_len] = 0;
5422 if (result_buf[0] == '?'
5423 || (result_buf[result_len-2] == ':'
5424 && result_buf[result_len-1] == '0'))
5425 break;
5426 /* Get rid of embedded newline, if any. Test for "main" */
5427 {
5428 char * nl = strchr(result_buf, '\n');
5429 if (nl != NULL
5430 && (word)nl < (word)(result_buf + result_len)) {
5431 *nl = ':';
5432 }
5433 if (strncmp(result_buf, "main",
5434 nl != NULL
5435 ? (size_t)((word)nl /* a cppcheck workaround */
5436 - COVERT_DATAFLOW(result_buf))
5437 : result_len) == 0) {
5438 stop = TRUE;
5439 }
5440 }
5441 if (result_len < RESULT_SZ - 25) {
5442 /* Add in hex address */
5443 (void)snprintf(&result_buf[result_len],
5444 sizeof(result_buf) - result_len,
5445 " [0x%lx]", (unsigned long)info[i].ci_pc);
5446 result_buf[sizeof(result_buf) - 1] = '\0';
5447 }
5448 # if defined(CPPCHECK)
5449 GC_noop1((unsigned char)name[0]);
5450 /* name computed previously is discarded */
5451 # endif
5452 name = result_buf;
5453 } while (0);
5454 # endif /* LINUX */
5455 GC_err_printf("\t\t%s\n", name);
5456 # if defined(GC_HAVE_BUILTIN_BACKTRACE) \
5457 && !defined(GC_BACKTRACE_SYMBOLS_BROKEN)
5458 if (sym_name != NULL)
5459 free(sym_name); /* May call GC_[debug_]free; that's OK */
5460 # endif
5461 }
5462 # if defined(LINUX) && !defined(SMALL_CONFIG)
5463 if (stop)
5464 break;
5465 # endif
5466 }
5467 LOCK();
5468 --reentry_count;
5469 UNLOCK();
5470 }
5471
5472 #endif /* NEED_CALLINFO */
5473
5474 #if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
5475 /* Dump /proc/self/maps to GC_stderr, to enable looking up names for */
5476 /* addresses in FIND_LEAK output. */
5477 void GC_print_address_map(void)
5478 {
5479 const char *maps = GC_get_maps();
5480
5481 GC_err_printf("---------- Begin address map ----------\n");
5482 GC_err_puts(maps);
5483 GC_err_printf("---------- End address map ----------\n");
5484 }
5485 #endif /* LINUX && ELF */