Debian Codegraph @

gc/pthread_support.c

raw
1 /*
2 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
5 * Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
6 * Copyright (c) 2008-2021 Ivan Maidanski
7 *
8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 *
11 * Permission is hereby granted to use or copy this program
12 * for any purpose, provided the above notices are retained on all copies.
13 * Permission to modify the code and to distribute modified code is granted,
14 * provided the above notices are retained, and a notice that the code was
15 * modified is included with the above copyright notice.
16 */
17
18 #include "private/pthread_support.h"
19
20 /*
21 * Support code originally for LinuxThreads, the clone()-based kernel
22 * thread package for Linux which is included in libc6.
23 *
24 * This code no doubt makes some assumptions beyond what is
25 * guaranteed by the pthread standard, though it now does
26 * very little of that. It now also supports NPTL, and many
27 * other Posix thread implementations. We are trying to merge
28 * all flavors of pthread support code into this file.
29 */
30
31 #if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
32
33 # include <stdlib.h>
34 # include <pthread.h>
35 # include <sched.h>
36 # include <time.h>
37 # include <errno.h>
38 # include <unistd.h>
39 # if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
40 # if !defined(GC_RTEMS_PTHREADS)
41 # include <sys/mman.h>
42 # endif
43 # include <sys/time.h>
44 # include <sys/types.h>
45 # include <sys/stat.h>
46 # include <fcntl.h>
47 # endif
48 # include <signal.h>
49
50 # include "gc_inline.h"
51
52 #if defined(GC_DARWIN_THREADS)
53 # include "private/darwin_semaphore.h"
54 #else
55 # include <semaphore.h>
56 #endif /* !GC_DARWIN_THREADS */
57
58 #if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
59 # include <sys/sysctl.h>
60 #endif /* GC_DARWIN_THREADS */
61
62 #if defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
63 # include <sys/param.h>
64 # include <sys/sysctl.h>
65 #endif /* GC_NETBSD_THREADS */
66
67 /* Allocator lock definitions. */
68 #if !defined(USE_SPIN_LOCK)
69 GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
70 #endif
71
72 #ifdef GC_ASSERTIONS
73 GC_INNER unsigned long GC_lock_holder = NO_THREAD;
74 /* Used only for assertions. */
75 #endif
76
77 #if defined(GC_DGUX386_THREADS)
78 # include <sys/dg_sys_info.h>
79 # include <sys/_int_psem.h>
80 /* sem_t is an uint in DG/UX */
81 typedef unsigned int sem_t;
82 #endif /* GC_DGUX386_THREADS */
83
84 /* Undefine macros used to redirect pthread primitives. */
85 # undef pthread_create
86 # ifndef GC_NO_PTHREAD_SIGMASK
87 # undef pthread_sigmask
88 # endif
89 # ifndef GC_NO_PTHREAD_CANCEL
90 # undef pthread_cancel
91 # endif
92 # ifdef GC_HAVE_PTHREAD_EXIT
93 # undef pthread_exit
94 # endif
95 # undef pthread_join
96 # undef pthread_detach
97 # if defined(GC_OSF1_THREADS) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
98 && !defined(_PTHREAD_USE_PTDNAM_)
99 /* Restore the original mangled names on Tru64 UNIX. */
100 # define pthread_create __pthread_create
101 # define pthread_join __pthread_join
102 # define pthread_detach __pthread_detach
103 # ifndef GC_NO_PTHREAD_CANCEL
104 # define pthread_cancel __pthread_cancel
105 # endif
106 # ifdef GC_HAVE_PTHREAD_EXIT
107 # define pthread_exit __pthread_exit
108 # endif
109 # endif
110
111 #ifdef GC_USE_LD_WRAP
112 # define WRAP_FUNC(f) __wrap_##f
113 # define REAL_FUNC(f) __real_##f
114 int REAL_FUNC(pthread_create)(pthread_t *,
115 GC_PTHREAD_CREATE_CONST pthread_attr_t *,
116 void *(*start_routine)(void *), void *);
117 int REAL_FUNC(pthread_join)(pthread_t, void **);
118 int REAL_FUNC(pthread_detach)(pthread_t);
119 # ifndef GC_NO_PTHREAD_SIGMASK
120 int REAL_FUNC(pthread_sigmask)(int, const sigset_t *, sigset_t *);
121 # endif
122 # ifndef GC_NO_PTHREAD_CANCEL
123 int REAL_FUNC(pthread_cancel)(pthread_t);
124 # endif
125 # ifdef GC_HAVE_PTHREAD_EXIT
126 void REAL_FUNC(pthread_exit)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
127 # endif
128 #else
129 # ifdef GC_USE_DLOPEN_WRAP
130 # include <dlfcn.h>
131 # define WRAP_FUNC(f) f
132 # define REAL_FUNC(f) GC_real_##f
133 /* We define both GC_f and plain f to be the wrapped function. */
134 /* In that way plain calls work, as do calls from files that */
135 /* included gc.h, which redefined f to GC_f. */
136 /* FIXME: Needs work for DARWIN and True64 (OSF1) */
137 typedef int (* GC_pthread_create_t)(pthread_t *,
138 GC_PTHREAD_CREATE_CONST pthread_attr_t *,
139 void * (*)(void *), void *);
140 static GC_pthread_create_t REAL_FUNC(pthread_create);
141 # ifndef GC_NO_PTHREAD_SIGMASK
142 typedef int (* GC_pthread_sigmask_t)(int, const sigset_t *,
143 sigset_t *);
144 static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask);
145 # endif
146 typedef int (* GC_pthread_join_t)(pthread_t, void **);
147 static GC_pthread_join_t REAL_FUNC(pthread_join);
148 typedef int (* GC_pthread_detach_t)(pthread_t);
149 static GC_pthread_detach_t REAL_FUNC(pthread_detach);
150 # ifndef GC_NO_PTHREAD_CANCEL
151 typedef int (* GC_pthread_cancel_t)(pthread_t);
152 static GC_pthread_cancel_t REAL_FUNC(pthread_cancel);
153 # endif
154 # ifdef GC_HAVE_PTHREAD_EXIT
155 typedef void (* GC_pthread_exit_t)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
156 static GC_pthread_exit_t REAL_FUNC(pthread_exit);
157 # endif
158 # else
159 # define WRAP_FUNC(f) GC_##f
160 # if !defined(GC_DGUX386_THREADS)
161 # define REAL_FUNC(f) f
162 # else /* GC_DGUX386_THREADS */
163 # define REAL_FUNC(f) __d10_##f
164 # endif /* GC_DGUX386_THREADS */
165 # endif
166 #endif
167
168 #if defined(GC_USE_LD_WRAP) || defined(GC_USE_DLOPEN_WRAP)
169 /* Define GC_ functions as aliases for the plain ones, which will */
170 /* be intercepted. This allows files which include gc.h, and hence */
171 /* generate references to the GC_ symbols, to see the right symbols. */
172 GC_API int GC_pthread_create(pthread_t * t,
173 GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
174 void * (* fn)(void *), void * arg)
175 {
176 return pthread_create(t, a, fn, arg);
177 }
178
179 # ifndef GC_NO_PTHREAD_SIGMASK
180 GC_API int GC_pthread_sigmask(int how, const sigset_t *mask,
181 sigset_t *old)
182 {
183 return pthread_sigmask(how, mask, old);
184 }
185 # endif /* !GC_NO_PTHREAD_SIGMASK */
186
187 GC_API int GC_pthread_join(pthread_t t, void **res)
188 {
189 return pthread_join(t, res);
190 }
191
192 GC_API int GC_pthread_detach(pthread_t t)
193 {
194 return pthread_detach(t);
195 }
196
197 # ifndef GC_NO_PTHREAD_CANCEL
198 GC_API int GC_pthread_cancel(pthread_t t)
199 {
200 return pthread_cancel(t);
201 }
202 # endif /* !GC_NO_PTHREAD_CANCEL */
203
204 # ifdef GC_HAVE_PTHREAD_EXIT
205 GC_API GC_PTHREAD_EXIT_ATTRIBUTE void GC_pthread_exit(void *retval)
206 {
207 pthread_exit(retval);
208 }
209 # endif
210 #endif /* Linker-based interception. */
211
212 #ifdef GC_USE_DLOPEN_WRAP
213 STATIC GC_bool GC_syms_initialized = FALSE;
214
215 STATIC void GC_init_real_syms(void)
216 {
217 void *dl_handle;
218
219 if (GC_syms_initialized) return;
220 # ifdef RTLD_NEXT
221 dl_handle = RTLD_NEXT;
222 # else
223 dl_handle = dlopen("libpthread.so.0", RTLD_LAZY);
224 if (NULL == dl_handle) {
225 dl_handle = dlopen("libpthread.so", RTLD_LAZY); /* without ".0" */
226 }
227 if (NULL == dl_handle) ABORT("Couldn't open libpthread");
228 # endif
229 REAL_FUNC(pthread_create) = (GC_pthread_create_t)(word)
230 dlsym(dl_handle, "pthread_create");
231 # ifdef RTLD_NEXT
232 if (REAL_FUNC(pthread_create) == 0)
233 ABORT("pthread_create not found"
234 " (probably -lgc is specified after -lpthread)");
235 # endif
236 # ifndef GC_NO_PTHREAD_SIGMASK
237 REAL_FUNC(pthread_sigmask) = (GC_pthread_sigmask_t)(word)
238 dlsym(dl_handle, "pthread_sigmask");
239 # endif
240 REAL_FUNC(pthread_join) = (GC_pthread_join_t)(word)
241 dlsym(dl_handle, "pthread_join");
242 REAL_FUNC(pthread_detach) = (GC_pthread_detach_t)(word)
243 dlsym(dl_handle, "pthread_detach");
244 # ifndef GC_NO_PTHREAD_CANCEL
245 REAL_FUNC(pthread_cancel) = (GC_pthread_cancel_t)(word)
246 dlsym(dl_handle, "pthread_cancel");
247 # endif
248 # ifdef GC_HAVE_PTHREAD_EXIT
249 REAL_FUNC(pthread_exit) = (GC_pthread_exit_t)(word)
250 dlsym(dl_handle, "pthread_exit");
251 # endif
252 GC_syms_initialized = TRUE;
253 }
254
255 # define INIT_REAL_SYMS() if (EXPECT(GC_syms_initialized, TRUE)) {} \
256 else GC_init_real_syms()
257 #else
258 # define INIT_REAL_SYMS() (void)0
259 #endif
260
261 static GC_bool parallel_initialized = FALSE;
262
263 #if defined(MPROTECT_VDB) && defined(DARWIN)
264 GC_INNER int GC_inner_pthread_create(pthread_t *t,
265 GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
266 void *(*fn)(void *), void *arg)
267 {
268 INIT_REAL_SYMS();
269 return REAL_FUNC(pthread_create)(t, a, fn, arg);
270 }
271 #endif
272
273 #ifndef GC_ALWAYS_MULTITHREADED
274 GC_INNER GC_bool GC_need_to_lock = FALSE;
275 #endif
276
277 STATIC int GC_nprocs = 1;
278 /* Number of processors. We may not have */
279 /* access to all of them, but this is as good */
280 /* a guess as any ... */
281
282 #ifdef THREAD_LOCAL_ALLOC
283 /* We must explicitly mark ptrfree and gcj free lists, since the free */
284 /* list links wouldn't otherwise be found. We also set them in the */
285 /* normal free lists, since that involves touching less memory than */
286 /* if we scanned them normally. */
287 GC_INNER void GC_mark_thread_local_free_lists(void)
288 {
289 int i;
290 GC_thread p;
291
292 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
293 for (p = GC_threads[i]; 0 != p; p = p -> next) {
294 if (!(p -> flags & FINISHED))
295 GC_mark_thread_local_fls_for(&(p->tlfs));
296 }
297 }
298 }
299
300 # if defined(GC_ASSERTIONS)
301 /* Check that all thread-local free-lists are completely marked. */
302 /* Also check that thread-specific-data structures are marked. */
303 void GC_check_tls(void)
304 {
305 int i;
306 GC_thread p;
307
308 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
309 for (p = GC_threads[i]; 0 != p; p = p -> next) {
310 if (!(p -> flags & FINISHED))
311 GC_check_tls_for(&(p->tlfs));
312 }
313 }
314 # if defined(USE_CUSTOM_SPECIFIC)
315 if (GC_thread_key != 0)
316 GC_check_tsd_marks(GC_thread_key);
317 # endif
318 }
319 # endif /* GC_ASSERTIONS */
320
321 #endif /* THREAD_LOCAL_ALLOC */
322
323 # ifndef MAX_MARKERS
324 # define MAX_MARKERS 16
325 # endif
326
327 #ifdef PARALLEL_MARK
328
329 static ptr_t marker_sp[MAX_MARKERS - 1] = {0};
330 #ifdef IA64
331 static ptr_t marker_bsp[MAX_MARKERS - 1] = {0};
332 #endif
333
334 #if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
335 static mach_port_t marker_mach_threads[MAX_MARKERS - 1] = {0};
336
337 /* Used only by GC_suspend_thread_list(). */
338 GC_INNER GC_bool GC_is_mach_marker(thread_act_t thread)
339 {
340 int i;
341 for (i = 0; i < GC_markers_m1; i++) {
342 if (marker_mach_threads[i] == thread)
343 return TRUE;
344 }
345 return FALSE;
346 }
347 #endif /* GC_DARWIN_THREADS */
348
349 #ifdef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG /* NetBSD */
350 static void set_marker_thread_name(unsigned id)
351 {
352 int err = pthread_setname_np(pthread_self(), "GC-marker-%zu",
353 (void*)(size_t)id);
354 if (err != 0)
355 WARN("pthread_setname_np failed, errno= %" WARN_PRIdPTR "\n",
356 (signed_word)err);
357 }
358 #elif defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) \
359 || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
360 static void set_marker_thread_name(unsigned id)
361 {
362 char name_buf[16]; /* pthread_setname_np may fail for longer names */
363 int len = sizeof("GC-marker-") - 1;
364
365 /* Compose the name manually as snprintf may be unavailable or */
366 /* "%u directive output may be truncated" warning may occur. */
367 BCOPY("GC-marker-", name_buf, len);
368 if (id >= 10)
369 name_buf[len++] = (char)('0' + (id / 10) % 10);
370 name_buf[len] = (char)('0' + id % 10);
371 name_buf[len + 1] = '\0';
372
373 # ifdef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID /* iOS, OS X */
374 (void)pthread_setname_np(name_buf);
375 # else /* Linux, Solaris, etc. */
376 if (pthread_setname_np(pthread_self(), name_buf) != 0)
377 WARN("pthread_setname_np failed\n", 0);
378 # endif
379 }
380 #else
381 # define set_marker_thread_name(id) (void)(id)
382 #endif
383
384 STATIC void * GC_mark_thread(void * id)
385 {
386 word my_mark_no = 0;
387 IF_CANCEL(int cancel_state;)
388
389 if ((word)id == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
390 DISABLE_CANCEL(cancel_state);
391 /* Mark threads are not cancellable; they */
392 /* should be invisible to client. */
393 set_marker_thread_name((unsigned)(word)id);
394 marker_sp[(word)id] = GC_approx_sp();
395 # ifdef IA64
396 marker_bsp[(word)id] = GC_save_regs_in_stack();
397 # endif
398 # if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
399 marker_mach_threads[(word)id] = mach_thread_self();
400 # endif
401
402 /* Inform GC_start_mark_threads about completion of marker data init. */
403 GC_acquire_mark_lock();
404 if (0 == --GC_fl_builder_count) /* count may have a negative value */
405 GC_notify_all_builder();
406
407 for (;; ++my_mark_no) {
408 /* GC_mark_no is passed only to allow GC_help_marker to terminate */
409 /* promptly. This is important if it were called from the signal */
410 /* handler or from the GC lock acquisition code. Under Linux, it's */
411 /* not safe to call it from a signal handler, since it uses mutexes */
412 /* and condition variables. Since it is called only here, the */
413 /* argument is unnecessary. */
414 if (my_mark_no < GC_mark_no || my_mark_no > GC_mark_no + 2) {
415 /* resynchronize if we get far off, e.g. because GC_mark_no */
416 /* wrapped. */
417 my_mark_no = GC_mark_no;
418 }
419 # ifdef DEBUG_THREADS
420 GC_log_printf("Starting mark helper for mark number %lu\n",
421 (unsigned long)my_mark_no);
422 # endif
423 GC_help_marker(my_mark_no);
424 }
425 }
426
427 STATIC pthread_t GC_mark_threads[MAX_MARKERS];
428
429 #ifdef GLIBC_2_1_MUTEX_HACK
430 /* Ugly workaround for a linux threads bug in the final versions */
431 /* of glibc2.1. Pthread_mutex_trylock sets the mutex owner */
432 /* field even when it fails to acquire the mutex. This causes */
433 /* pthread_cond_wait to die. Remove for glibc2.2. */
434 /* According to the man page, we should use */
435 /* PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, but that isn't actually */
436 /* defined. */
437 static pthread_mutex_t mark_mutex =
438 {0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, {0, 0}};
439 #else
440 static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
441 #endif
442
443 static int available_markers_m1 = 0;
444
445 #ifdef CAN_HANDLE_FORK
446 static pthread_cond_t mark_cv;
447 /* initialized by GC_start_mark_threads_inner */
448 #else
449 static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
450 #endif
451
452 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all);
453
454 GC_INNER void GC_start_mark_threads_inner(void)
455 {
456 int i;
457 pthread_attr_t attr;
458 # ifndef NO_MARKER_SPECIAL_SIGMASK
459 sigset_t set, oldset;
460 # endif
461
462 GC_ASSERT(I_HOLD_LOCK());
463 ASSERT_CANCEL_DISABLED();
464 if (available_markers_m1 <= 0 || GC_parallel) return;
465 /* Skip if parallel markers disabled or already started. */
466 GC_wait_for_gc_completion(TRUE);
467
468 # ifdef CAN_HANDLE_FORK
469 /* Initialize mark_cv (for the first time), or cleanup its value */
470 /* after forking in the child process. All the marker threads in */
471 /* the parent process were blocked on this variable at fork, so */
472 /* pthread_cond_wait() malfunction (hang) is possible in the */
473 /* child process without such a cleanup. */
474 /* TODO: This is not portable, it is better to shortly unblock */
475 /* all marker threads in the parent process at fork. */
476 {
477 pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
478 BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
479 }
480 # endif
481
482 GC_ASSERT(GC_fl_builder_count == 0);
483 INIT_REAL_SYMS(); /* for pthread_create */
484 if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
485 if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
486 ABORT("pthread_attr_setdetachstate failed");
487
488 # ifdef DEFAULT_STACK_MAYBE_SMALL
489 /* Default stack size is usually too small: increase it. */
490 /* Otherwise marker threads or GC may run out of space. */
491 {
492 size_t old_size;
493
494 if (pthread_attr_getstacksize(&attr, &old_size) != 0)
495 ABORT("pthread_attr_getstacksize failed");
496 if (old_size < MIN_STACK_SIZE
497 && old_size != 0 /* stack size is known */) {
498 if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
499 ABORT("pthread_attr_setstacksize failed");
500 }
501 }
502 # endif /* DEFAULT_STACK_MAYBE_SMALL */
503
504 # ifndef NO_MARKER_SPECIAL_SIGMASK
505 /* Apply special signal mask to GC marker threads, and don't drop */
506 /* user defined signals by GC marker threads. */
507 if (sigfillset(&set) != 0)
508 ABORT("sigfillset failed");
509
510 # if !defined(GC_DARWIN_THREADS) && !defined(GC_OPENBSD_UTHREADS) \
511 && !defined(NACL)
512 /* These are used by GC to stop and restart the world. */
513 if (sigdelset(&set, GC_get_suspend_signal()) != 0
514 || sigdelset(&set, GC_get_thr_restart_signal()) != 0)
515 ABORT("sigdelset failed");
516 # endif
517
518 if (REAL_FUNC(pthread_sigmask)(SIG_BLOCK, &set, &oldset) < 0) {
519 WARN("pthread_sigmask set failed, no markers started\n", 0);
520 GC_markers_m1 = 0;
521 (void)pthread_attr_destroy(&attr);
522 return;
523 }
524 # endif /* !NO_MARKER_SPECIAL_SIGMASK */
525
526 /* To have proper GC_parallel value in GC_help_marker. */
527 GC_markers_m1 = available_markers_m1;
528
529 for (i = 0; i < available_markers_m1; ++i) {
530 if (0 != REAL_FUNC(pthread_create)(GC_mark_threads + i, &attr,
531 GC_mark_thread, (void *)(word)i)) {
532 WARN("Marker thread creation failed\n", 0);
533 /* Don't try to create other marker threads. */
534 GC_markers_m1 = i;
535 break;
536 }
537 }
538
539 # ifndef NO_MARKER_SPECIAL_SIGMASK
540 /* Restore previous signal mask. */
541 if (REAL_FUNC(pthread_sigmask)(SIG_SETMASK, &oldset, NULL) < 0) {
542 WARN("pthread_sigmask restore failed\n", 0);
543 }
544 # endif
545
546 (void)pthread_attr_destroy(&attr);
547 GC_wait_for_markers_init();
548 GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
549 }
550
551 #endif /* PARALLEL_MARK */
552
553 GC_INNER GC_bool GC_thr_initialized = FALSE;
554
555 GC_INNER volatile GC_thread GC_threads[THREAD_TABLE_SZ] = {0};
556
557 /* It may not be safe to allocate when we register the first thread. */
558 /* As "next" and "status" fields are unused, no need to push this */
559 /* (but "backing_store_end" field should be pushed on E2K). */
560 static struct GC_Thread_Rep first_thread;
561
562 void GC_push_thread_structures(void)
563 {
564 GC_ASSERT(I_HOLD_LOCK());
565 GC_push_all((/* no volatile */ void *)&GC_threads,
566 (ptr_t)(&GC_threads) + sizeof(GC_threads));
567 # ifdef E2K
568 GC_PUSH_ALL_SYM(first_thread.backing_store_end);
569 # endif
570 # if defined(THREAD_LOCAL_ALLOC) && defined(USE_CUSTOM_SPECIFIC)
571 GC_PUSH_ALL_SYM(GC_thread_key);
572 # endif
573 }
574
575 #ifdef DEBUG_THREADS
576 STATIC int GC_count_threads(void)
577 {
578 int i;
579 int count = 0;
580 GC_ASSERT(I_HOLD_LOCK());
581 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
582 GC_thread th = GC_threads[i];
583 while (th) {
584 if (!(th->flags & FINISHED))
585 ++count;
586 th = th->next;
587 }
588 }
589 return count;
590 }
591 #endif /* DEBUG_THREADS */
592
593 /* Add a thread to GC_threads. We assume it wasn't already there. */
594 /* Caller holds allocation lock. */
595 STATIC GC_thread GC_new_thread(pthread_t id)
596 {
597 int hv = THREAD_TABLE_INDEX(id);
598 GC_thread result;
599 static GC_bool first_thread_used = FALSE;
600
601 # ifdef DEBUG_THREADS
602 GC_log_printf("Creating thread %p\n", (void *)id);
603 for (result = GC_threads[hv]; result != NULL; result = result->next)
604 if (!THREAD_EQUAL(result->id, id)) {
605 GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
606 break;
607 }
608 # endif
609 GC_ASSERT(I_HOLD_LOCK());
610 if (!EXPECT(first_thread_used, TRUE)) {
611 result = &first_thread;
612 first_thread_used = TRUE;
613 GC_ASSERT(NULL == GC_threads[hv]);
614 # if defined(THREAD_SANITIZER) && defined(CPPCHECK)
615 GC_noop1((unsigned char)result->dummy[0]);
616 # endif
617 } else {
618 result = (struct GC_Thread_Rep *)
619 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
620 if (result == 0) return(0);
621 }
622 result -> id = id;
623 # ifdef USE_TKILL_ON_ANDROID
624 result -> kernel_id = gettid();
625 # endif
626 result -> next = GC_threads[hv];
627 GC_threads[hv] = result;
628 # ifdef NACL
629 GC_nacl_gc_thread_self = result;
630 GC_nacl_initialize_gc_thread();
631 # endif
632 GC_ASSERT(result -> flags == 0 && result -> thread_blocked == 0);
633 if (EXPECT(result != &first_thread, TRUE))
634 GC_dirty(result);
635 return(result);
636 }
637
638 /* Delete a thread from GC_threads. We assume it is there. */
639 /* (The code intentionally traps if it wasn't.) */
640 /* It is safe to delete the main thread. */
641 STATIC void GC_delete_thread(pthread_t id)
642 {
643 int hv = THREAD_TABLE_INDEX(id);
644 GC_thread p = GC_threads[hv];
645 GC_thread prev = NULL;
646
647 # ifdef DEBUG_THREADS
648 GC_log_printf("Deleting thread %p, n_threads= %d\n",
649 (void *)id, GC_count_threads());
650 # endif
651 GC_ASSERT(I_HOLD_LOCK());
652 while (!THREAD_EQUAL(p -> id, id)) {
653 prev = p;
654 p = p -> next;
655 }
656 if (prev == 0) {
657 GC_threads[hv] = p -> next;
658 } else {
659 GC_ASSERT(prev != &first_thread);
660 prev -> next = p -> next;
661 GC_dirty(prev);
662 }
663 if (p != &first_thread) {
664 # ifdef GC_DARWIN_THREADS
665 mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
666 # endif
667 GC_INTERNAL_FREE(p);
668 }
669 }
670
671 /* If a thread has been joined, but we have not yet */
672 /* been notified, then there may be more than one thread */
673 /* in the table with the same pthread id. */
674 /* This is OK, but we need a way to delete a specific one. */
675 STATIC void GC_delete_gc_thread(GC_thread t)
676 {
677 pthread_t id = t -> id;
678 int hv = THREAD_TABLE_INDEX(id);
679 GC_thread p = GC_threads[hv];
680 GC_thread prev = NULL;
681
682 GC_ASSERT(I_HOLD_LOCK());
683 while (p != t) {
684 prev = p;
685 p = p -> next;
686 }
687 if (prev == 0) {
688 GC_threads[hv] = p -> next;
689 } else {
690 GC_ASSERT(prev != &first_thread);
691 prev -> next = p -> next;
692 GC_dirty(prev);
693 }
694 # ifdef GC_DARWIN_THREADS
695 mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
696 # endif
697 GC_INTERNAL_FREE(p);
698
699 # ifdef DEBUG_THREADS
700 GC_log_printf("Deleted thread %p, n_threads= %d\n",
701 (void *)id, GC_count_threads());
702 # endif
703 }
704
705 /* Return a GC_thread corresponding to a given pthread_t. */
706 /* Returns 0 if it's not there. */
707 /* Caller holds allocation lock or otherwise inhibits */
708 /* updates. */
709 /* If there is more than one thread with the given id we */
710 /* return the most recent one. */
711 GC_INNER GC_thread GC_lookup_thread(pthread_t id)
712 {
713 GC_thread p = GC_threads[THREAD_TABLE_INDEX(id)];
714
715 while (p != 0 && !THREAD_EQUAL(p -> id, id)) p = p -> next;
716 return(p);
717 }
718
719 #ifndef GC_NO_FINALIZATION
720 /* Called by GC_finalize() (in case of an allocation failure observed). */
721 GC_INNER void GC_reset_finalizer_nested(void)
722 {
723 GC_thread me = GC_lookup_thread(pthread_self());
724
725 me->finalizer_nested = 0;
726 }
727
728 /* Checks and updates the thread-local level of finalizers recursion. */
729 /* Returns NULL if GC_invoke_finalizers() should not be called by the */
730 /* collector (to minimize the risk of a deep finalizers recursion), */
731 /* otherwise returns a pointer to the thread-local finalizer_nested. */
732 /* Called by GC_notify_or_invoke_finalizers() only (the GC lock is */
733 /* held). */
734 GC_INNER unsigned char *GC_check_finalizer_nested(void)
735 {
736 GC_thread me = GC_lookup_thread(pthread_self());
737 unsigned nesting_level;
738
739 # if defined(INCLUDE_LINUX_THREAD_DESCR) && defined(REDIRECT_MALLOC)
740 /* As noted in GC_start_routine, an allocation may happen in */
741 /* GC_get_stack_base, causing GC_notify_or_invoke_finalizers */
742 /* to be called before the thread gets registered. */
743 if (EXPECT(NULL == me, FALSE)) return NULL;
744 # endif
745 nesting_level = me->finalizer_nested;
746 if (nesting_level) {
747 /* We are inside another GC_invoke_finalizers(). */
748 /* Skip some implicitly-called GC_invoke_finalizers() */
749 /* depending on the nesting (recursion) level. */
750 if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
751 me->finalizer_skipped = 0;
752 }
753 me->finalizer_nested = (unsigned char)(nesting_level + 1);
754 return &me->finalizer_nested;
755 }
756 #endif /* !GC_NO_FINALIZATION */
757
758 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
759 /* This is called from thread-local GC_malloc(). */
760 GC_bool GC_is_thread_tsd_valid(void *tsd)
761 {
762 GC_thread me;
763 DCL_LOCK_STATE;
764
765 LOCK();
766 me = GC_lookup_thread(pthread_self());
767 UNLOCK();
768 return (word)tsd >= (word)(&me->tlfs)
769 && (word)tsd < (word)(&me->tlfs) + sizeof(me->tlfs);
770 }
771 #endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
772
773 GC_API int GC_CALL GC_thread_is_registered(void)
774 {
775 pthread_t self = pthread_self();
776 GC_thread me;
777 DCL_LOCK_STATE;
778
779 LOCK();
780 me = GC_lookup_thread(self);
781 UNLOCK();
782 return me != NULL && !(me -> flags & FINISHED);
783 }
784
785 static pthread_t main_pthread_id;
786 static void *main_stack, *main_altstack;
787 static word main_stack_size, main_altstack_size;
788
789 GC_API void GC_CALL GC_register_altstack(void *stack, GC_word stack_size,
790 void *altstack,
791 GC_word altstack_size)
792 {
793 GC_thread me;
794 pthread_t self = pthread_self();
795 DCL_LOCK_STATE;
796
797 LOCK();
798 me = GC_lookup_thread(self);
799 if (me != NULL) {
800 me->stack = (ptr_t)stack;
801 me->stack_size = stack_size;
802 me->altstack = (ptr_t)altstack;
803 me->altstack_size = altstack_size;
804 } else {
805 /* This happens if we are called before GC_thr_init. */
806 main_pthread_id = self;
807 main_stack = stack;
808 main_stack_size = stack_size;
809 main_altstack = altstack;
810 main_altstack_size = altstack_size;
811 }
812 UNLOCK();
813 }
814
815 #ifdef CAN_HANDLE_FORK
816
817 /* Prevent TSan false positive about the race during items removal */
818 /* from GC_threads. (The race cannot happen since only one thread */
819 /* survives in the child.) */
820 # ifdef CAN_CALL_ATFORK
821 GC_ATTR_NO_SANITIZE_THREAD
822 # endif
823 static void store_to_threads_table(int hv, GC_thread me)
824 {
825 GC_threads[hv] = me;
826 }
827
828 /* Remove all entries from the GC_threads table, except the */
829 /* one for the current thread. We need to do this in the child */
830 /* process after a fork(), since only the current thread */
831 /* survives in the child. */
832 STATIC void GC_remove_all_threads_but_me(void)
833 {
834 pthread_t self = pthread_self();
835 int hv;
836
837 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
838 GC_thread p, next;
839 GC_thread me = NULL;
840
841 for (p = GC_threads[hv]; 0 != p; p = next) {
842 next = p -> next;
843 if (THREAD_EQUAL(p -> id, self)
844 && me == NULL) { /* ignore dead threads with the same id */
845 me = p;
846 p -> next = 0;
847 # ifdef GC_DARWIN_THREADS
848 /* Update thread Id after fork (it is OK to call */
849 /* GC_destroy_thread_local and GC_free_inner */
850 /* before update). */
851 me -> stop_info.mach_thread = mach_thread_self();
852 # endif
853 # ifdef USE_TKILL_ON_ANDROID
854 me -> kernel_id = gettid();
855 # endif
856 # if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
857 {
858 int res;
859
860 /* Some TLS implementations might be not fork-friendly, so */
861 /* we re-assign thread-local pointer to 'tlfs' for safety */
862 /* instead of the assertion check (again, it is OK to call */
863 /* GC_destroy_thread_local and GC_free_inner before). */
864 res = GC_setspecific(GC_thread_key, &me->tlfs);
865 if (COVERT_DATAFLOW(res) != 0)
866 ABORT("GC_setspecific failed (in child)");
867 }
868 # endif
869 } else {
870 # ifdef THREAD_LOCAL_ALLOC
871 if (!(p -> flags & FINISHED)) {
872 /* Cannot call GC_destroy_thread_local here. The free */
873 /* lists may be in an inconsistent state (as thread p may */
874 /* be updating one of the lists by GC_generic_malloc_many */
875 /* or GC_FAST_MALLOC_GRANS when fork is invoked). */
876 /* This should not be a problem because the lost elements */
877 /* of the free lists will be collected during GC. */
878 GC_remove_specific_after_fork(GC_thread_key, p -> id);
879 }
880 # endif
881 /* TODO: To avoid TSan hang (when updating GC_bytes_freed), */
882 /* we just skip explicit freeing of GC_threads entries. */
883 # if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
884 if (p != &first_thread) GC_INTERNAL_FREE(p);
885 # endif
886 }
887 }
888 store_to_threads_table(hv, me);
889 }
890 }
891 #endif /* CAN_HANDLE_FORK */
892
893 #ifdef USE_PROC_FOR_LIBRARIES
894 GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
895 {
896 int i;
897 GC_thread p;
898
899 GC_ASSERT(I_HOLD_LOCK());
900 # ifdef PARALLEL_MARK
901 for (i = 0; i < GC_markers_m1; ++i) {
902 if ((word)marker_sp[i] > (word)lo && (word)marker_sp[i] < (word)hi)
903 return TRUE;
904 # ifdef IA64
905 if ((word)marker_bsp[i] > (word)lo
906 && (word)marker_bsp[i] < (word)hi)
907 return TRUE;
908 # endif
909 }
910 # endif
911 for (i = 0; i < THREAD_TABLE_SZ; i++) {
912 for (p = GC_threads[i]; p != 0; p = p -> next) {
913 if (0 != p -> stack_end) {
914 # ifdef STACK_GROWS_UP
915 if ((word)p->stack_end >= (word)lo
916 && (word)p->stack_end < (word)hi)
917 return TRUE;
918 # else /* STACK_GROWS_DOWN */
919 if ((word)p->stack_end > (word)lo
920 && (word)p->stack_end <= (word)hi)
921 return TRUE;
922 # endif
923 }
924 }
925 }
926 return FALSE;
927 }
928 #endif /* USE_PROC_FOR_LIBRARIES */
929
930 #if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
931 && defined(IA64)
932 /* Find the largest stack_base smaller than bound. May be used */
933 /* to find the boundary between a register stack and adjacent */
934 /* immediately preceding memory stack. */
935 GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound)
936 {
937 int i;
938 GC_thread p;
939 ptr_t result = 0;
940
941 GC_ASSERT(I_HOLD_LOCK());
942 # ifdef PARALLEL_MARK
943 for (i = 0; i < GC_markers_m1; ++i) {
944 if ((word)marker_sp[i] > (word)result
945 && (word)marker_sp[i] < (word)bound)
946 result = marker_sp[i];
947 }
948 # endif
949 for (i = 0; i < THREAD_TABLE_SZ; i++) {
950 for (p = GC_threads[i]; p != 0; p = p -> next) {
951 if ((word)p->stack_end > (word)result
952 && (word)p->stack_end < (word)bound) {
953 result = p -> stack_end;
954 }
955 }
956 }
957 return result;
958 }
959 #endif /* IA64 */
960
961 #ifndef STAT_READ
962 # define STAT_READ read
963 /* If read is wrapped, this may need to be redefined to call */
964 /* the real one. */
965 #endif
966
967 #ifdef GC_HPUX_THREADS
968 # define GC_get_nprocs() pthread_num_processors_np()
969
970 #elif defined(GC_OSF1_THREADS) || defined(GC_AIX_THREADS) \
971 || defined(GC_HAIKU_THREADS) || defined(GC_SOLARIS_THREADS) \
972 || defined(HURD) || defined(HOST_ANDROID) || defined(NACL)
973 GC_INLINE int GC_get_nprocs(void)
974 {
975 int nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
976 return nprocs > 0 ? nprocs : 1; /* ignore error silently */
977 }
978
979 #elif defined(GC_IRIX_THREADS)
980 GC_INLINE int GC_get_nprocs(void)
981 {
982 int nprocs = (int)sysconf(_SC_NPROC_ONLN);
983 return nprocs > 0 ? nprocs : 1; /* ignore error silently */
984 }
985
986 #elif defined(GC_LINUX_THREADS) /* && !HOST_ANDROID && !NACL */
987 /* Return the number of processors. */
988 STATIC int GC_get_nprocs(void)
989 {
990 /* Should be "return sysconf(_SC_NPROCESSORS_ONLN);" but that */
991 /* appears to be buggy in many cases. */
992 /* We look for lines "cpu<n>" in /proc/stat. */
993 # define PROC_STAT_BUF_SZ ((1 + MAX_MARKERS) * 100) /* should be enough */
994 /* No need to read the entire /proc/stat to get maximum cpu<N> as */
995 /* - the requested lines are located at the beginning of the file; */
996 /* - the lines with cpu<N> where N > MAX_MARKERS are not needed. */
997 char stat_buf[PROC_STAT_BUF_SZ+1];
998 int f;
999 int result, i, len;
1000
1001 f = open("/proc/stat", O_RDONLY);
1002 if (f < 0) {
1003 WARN("Could not open /proc/stat\n", 0);
1004 return 1; /* assume an uniprocessor */
1005 }
1006 len = STAT_READ(f, stat_buf, sizeof(stat_buf)-1);
1007 /* Unlikely that we need to retry because of an incomplete read here. */
1008 if (len < 0) {
1009 WARN("Failed to read /proc/stat, errno= %" WARN_PRIdPTR "\n",
1010 (signed_word)errno);
1011 close(f);
1012 return 1;
1013 }
1014 stat_buf[len] = '\0'; /* to avoid potential buffer overrun by atoi() */
1015 close(f);
1016
1017 result = 1;
1018 /* Some old kernels only have a single "cpu nnnn ..." */
1019 /* entry in /proc/stat. We identify those as */
1020 /* uniprocessors. */
1021
1022 for (i = 0; i < len - 4; ++i) {
1023 if (stat_buf[i] == '\n' && stat_buf[i+1] == 'c'
1024 && stat_buf[i+2] == 'p' && stat_buf[i+3] == 'u') {
1025 int cpu_no = atoi(&stat_buf[i + 4]);
1026 if (cpu_no >= result)
1027 result = cpu_no + 1;
1028 }
1029 }
1030 return result;
1031 }
1032
1033 #elif defined(GC_DGUX386_THREADS)
1034 /* Return the number of processors, or i <= 0 if it can't be determined. */
1035 STATIC int GC_get_nprocs(void)
1036 {
1037 int numCpus;
1038 struct dg_sys_info_pm_info pm_sysinfo;
1039 int status = 0;
1040
1041 status = dg_sys_info((long int *) &pm_sysinfo,
1042 DG_SYS_INFO_PM_INFO_TYPE, DG_SYS_INFO_PM_CURRENT_VERSION);
1043 if (status < 0)
1044 /* set -1 for error */
1045 numCpus = -1;
1046 else
1047 /* Active CPUs */
1048 numCpus = pm_sysinfo.idle_vp_count;
1049 return(numCpus);
1050 }
1051
1052 #elif defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS) \
1053 || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
1054 STATIC int GC_get_nprocs(void)
1055 {
1056 int mib[] = {CTL_HW,HW_NCPU};
1057 int res;
1058 size_t len = sizeof(res);
1059
1060 sysctl(mib, sizeof(mib)/sizeof(int), &res, &len, NULL, 0);
1061 return res;
1062 }
1063
1064 #else
1065 /* E.g., GC_RTEMS_PTHREADS */
1066 # define GC_get_nprocs() 1 /* not implemented */
1067 #endif /* !GC_LINUX_THREADS && !GC_DARWIN_THREADS && ... */
1068
1069 #if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
1070 /* Some buggy Linux/arm kernels show only non-sleeping CPUs in */
1071 /* /proc/stat (and /proc/cpuinfo), so another data system source is */
1072 /* tried first. Result <= 0 on error. */
1073 STATIC int GC_get_nprocs_present(void)
1074 {
1075 char stat_buf[16];
1076 int f;
1077 int len;
1078
1079 f = open("/sys/devices/system/cpu/present", O_RDONLY);
1080 if (f < 0)
1081 return -1; /* cannot open the file */
1082
1083 len = STAT_READ(f, stat_buf, sizeof(stat_buf));
1084 close(f);
1085
1086 /* Recognized file format: "0\n" or "0-<max_cpu_id>\n" */
1087 /* The file might probably contain a comma-separated list */
1088 /* but we do not need to handle it (just silently ignore). */
1089 if (len < 2 || stat_buf[0] != '0' || stat_buf[len - 1] != '\n') {
1090 return 0; /* read error or unrecognized content */
1091 } else if (len == 2) {
1092 return 1; /* an uniprocessor */
1093 } else if (stat_buf[1] != '-') {
1094 return 0; /* unrecognized content */
1095 }
1096
1097 stat_buf[len - 1] = '\0'; /* terminate the string */
1098 return atoi(&stat_buf[2]) + 1; /* skip "0-" and parse max_cpu_num */
1099 }
1100 #endif /* ARM32 && GC_LINUX_THREADS && !NACL */
1101
1102 #if defined(CAN_HANDLE_FORK) && defined(THREAD_SANITIZER)
1103 # include "private/gc_pmark.h" /* for MS_NONE */
1104
1105 /* Workaround for TSan which does not notice that the GC lock */
1106 /* is acquired in fork_prepare_proc(). */
1107 GC_ATTR_NO_SANITIZE_THREAD
1108 static GC_bool collection_in_progress(void)
1109 {
1110 return GC_mark_state != MS_NONE;
1111 }
1112 #else
1113 # define collection_in_progress() GC_collection_in_progress()
1114 #endif
1115
1116 /* We hold the GC lock. Wait until an in-progress GC has finished. */
1117 /* Repeatedly RELEASES GC LOCK in order to wait. */
1118 /* If wait_for_all is true, then we exit with the GC lock held and no */
1119 /* collection in progress; otherwise we just wait for the current GC */
1120 /* to finish. */
1121 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
1122 {
1123 DCL_LOCK_STATE;
1124 # if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
1125 /* GC_lock_holder is accessed with the lock held, so there is no */
1126 /* data race actually (unlike what is reported by TSan). */
1127 GC_ASSERT(I_HOLD_LOCK());
1128 # endif
1129 ASSERT_CANCEL_DISABLED();
1130 if (GC_incremental && collection_in_progress()) {
1131 word old_gc_no = GC_gc_no;
1132
1133 /* Make sure that no part of our stack is still on the mark stack, */
1134 /* since it's about to be unmapped. */
1135 while (GC_incremental && collection_in_progress()
1136 && (wait_for_all || old_gc_no == GC_gc_no)) {
1137 ENTER_GC();
1138 GC_in_thread_creation = TRUE;
1139 GC_collect_a_little_inner(1);
1140 GC_in_thread_creation = FALSE;
1141 EXIT_GC();
1142 UNLOCK();
1143 sched_yield();
1144 LOCK();
1145 }
1146 }
1147 }
1148
1149 #ifdef CAN_HANDLE_FORK
1150 /* Procedures called before and after a fork. The goal here is to make */
1151 /* it safe to call GC_malloc() in a forked child. It's unclear that is */
1152 /* attainable, since the single UNIX spec seems to imply that one */
1153 /* should only call async-signal-safe functions, and we probably can't */
1154 /* quite guarantee that. But we give it our best shot. (That same */
1155 /* spec also implies that it's not safe to call the system malloc */
1156 /* between fork() and exec(). Thus we're doing no worse than it.) */
1157
1158 IF_CANCEL(static int fork_cancel_state;)
1159 /* protected by allocation lock. */
1160
1161 # ifdef PARALLEL_MARK
1162 # ifdef THREAD_SANITIZER
1163 # if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1164 STATIC void GC_generic_lock(pthread_mutex_t *);
1165 # endif
1166 GC_ATTR_NO_SANITIZE_THREAD
1167 static void wait_for_reclaim_atfork(void);
1168 # else
1169 # define wait_for_reclaim_atfork() GC_wait_for_reclaim()
1170 # endif
1171 # endif /* PARALLEL_MARK */
1172
1173 /* Called before a fork() */
1174 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1175 /* GC_lock_holder is updated safely (no data race actually). */
1176 GC_ATTR_NO_SANITIZE_THREAD
1177 #endif
1178 static void fork_prepare_proc(void)
1179 {
1180 /* Acquire all relevant locks, so that after releasing the locks */
1181 /* the child will see a consistent state in which monitor */
1182 /* invariants hold. Unfortunately, we can't acquire libc locks */
1183 /* we might need, and there seems to be no guarantee that libc */
1184 /* must install a suitable fork handler. */
1185 /* Wait for an ongoing GC to finish, since we can't finish it in */
1186 /* the (one remaining thread in) the child. */
1187 LOCK();
1188 DISABLE_CANCEL(fork_cancel_state);
1189 /* Following waits may include cancellation points. */
1190 # if defined(PARALLEL_MARK)
1191 if (GC_parallel)
1192 wait_for_reclaim_atfork();
1193 # endif
1194 GC_wait_for_gc_completion(TRUE);
1195 # if defined(PARALLEL_MARK)
1196 if (GC_parallel) {
1197 # if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
1198 && defined(CAN_CALL_ATFORK)
1199 /* Prevent TSan false positive about the data race */
1200 /* when updating GC_mark_lock_holder. */
1201 GC_generic_lock(&mark_mutex);
1202 # else
1203 GC_acquire_mark_lock();
1204 # endif
1205 }
1206 # endif
1207 GC_acquire_dirty_lock();
1208 }
1209
1210 /* Called in parent after a fork() (even if the latter failed). */
1211 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1212 GC_ATTR_NO_SANITIZE_THREAD
1213 #endif
1214 static void fork_parent_proc(void)
1215 {
1216 GC_release_dirty_lock();
1217 # if defined(PARALLEL_MARK)
1218 if (GC_parallel) {
1219 # if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
1220 && defined(CAN_CALL_ATFORK)
1221 /* To match that in fork_prepare_proc. */
1222 (void)pthread_mutex_unlock(&mark_mutex);
1223 # else
1224 GC_release_mark_lock();
1225 # endif
1226 }
1227 # endif
1228 RESTORE_CANCEL(fork_cancel_state);
1229 UNLOCK();
1230 }
1231
1232 /* Called in child after a fork() */
1233 #if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
1234 GC_ATTR_NO_SANITIZE_THREAD
1235 #endif
1236 static void fork_child_proc(void)
1237 {
1238 GC_release_dirty_lock();
1239 # ifndef GC_DISABLE_INCREMENTAL
1240 GC_dirty_update_child();
1241 # endif
1242 # ifdef PARALLEL_MARK
1243 if (GC_parallel) {
1244 # if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
1245 && defined(CAN_CALL_ATFORK)
1246 (void)pthread_mutex_unlock(&mark_mutex);
1247 # else
1248 GC_release_mark_lock();
1249 # endif
1250 /* Turn off parallel marking in the child, since we are probably */
1251 /* just going to exec, and we would have to restart mark threads. */
1252 GC_parallel = FALSE;
1253 }
1254 # ifdef THREAD_SANITIZER
1255 /* TSan does not support threads creation in the child process. */
1256 available_markers_m1 = 0;
1257 # endif
1258 # endif
1259 /* Clean up the thread table, so that just our thread is left. */
1260 GC_remove_all_threads_but_me();
1261 RESTORE_CANCEL(fork_cancel_state);
1262 UNLOCK();
1263 /* Even though after a fork the child only inherits the single */
1264 /* thread that called the fork(), if another thread in the parent */
1265 /* was attempting to lock the mutex while being held in */
1266 /* fork_child_prepare(), the mutex will be left in an inconsistent */
1267 /* state in the child after the UNLOCK. This is the case, at */
1268 /* least, in Mac OS X and leads to an unusable GC in the child */
1269 /* which will block when attempting to perform any GC operation */
1270 /* that acquires the allocation mutex. */
1271 # ifdef USE_PTHREAD_LOCKS
1272 GC_ASSERT(I_DONT_HOLD_LOCK());
1273 /* Reinitialize the mutex. It should be safe since we are */
1274 /* running this in the child which only inherits a single thread. */
1275 /* mutex_destroy() may return EBUSY, which makes no sense, but */
1276 /* that is the reason for the need of the reinitialization. */
1277 (void)pthread_mutex_destroy(&GC_allocate_ml);
1278 /* TODO: Probably some targets might need the default mutex */
1279 /* attribute to be passed instead of NULL. */
1280 if (0 != pthread_mutex_init(&GC_allocate_ml, NULL))
1281 ABORT("pthread_mutex_init failed (in child)");
1282 # endif
1283 }
1284
1285 /* Routines for fork handling by client (no-op if pthread_atfork works). */
1286 GC_API void GC_CALL GC_atfork_prepare(void)
1287 {
1288 if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
1289 if (GC_handle_fork <= 0)
1290 fork_prepare_proc();
1291 }
1292
1293 GC_API void GC_CALL GC_atfork_parent(void)
1294 {
1295 if (GC_handle_fork <= 0)
1296 fork_parent_proc();
1297 }
1298
1299 GC_API void GC_CALL GC_atfork_child(void)
1300 {
1301 if (GC_handle_fork <= 0)
1302 fork_child_proc();
1303 }
1304 #endif /* CAN_HANDLE_FORK */
1305
1306 #ifdef INCLUDE_LINUX_THREAD_DESCR
1307 __thread int GC_dummy_thread_local;
1308 #endif
1309
1310 #ifdef PARALLEL_MARK
1311 static void setup_mark_lock(void);
1312
1313 static unsigned required_markers_cnt = 0;
1314 /* The default value (0) means the number of */
1315 /* markers should be selected automatically. */
1316 #endif /* PARALLEL_MARK */
1317
1318 GC_API void GC_CALL GC_set_markers_count(unsigned markers GC_ATTR_UNUSED)
1319 {
1320 # ifdef PARALLEL_MARK
1321 required_markers_cnt = markers < MAX_MARKERS ? markers : MAX_MARKERS;
1322 # endif
1323 }
1324
1325 #ifndef DONT_USE_ATEXIT
1326 STATIC pthread_t GC_main_thread_id;
1327
1328 GC_INNER GC_bool GC_is_main_thread(void)
1329 {
1330 GC_ASSERT(GC_thr_initialized);
1331 return THREAD_EQUAL(GC_main_thread_id, pthread_self());
1332 }
1333 #endif /* !DONT_USE_ATEXIT */
1334
1335 GC_INNER void GC_thr_init(void)
1336 {
1337 GC_ASSERT(I_HOLD_LOCK());
1338 if (GC_thr_initialized) return;
1339 GC_thr_initialized = TRUE;
1340
1341 GC_ASSERT((word)&GC_threads % sizeof(word) == 0);
1342 # ifdef CAN_HANDLE_FORK
1343 /* Prepare for forks if requested. */
1344 if (GC_handle_fork) {
1345 # ifdef CAN_CALL_ATFORK
1346 if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
1347 fork_child_proc) == 0) {
1348 /* Handlers successfully registered. */
1349 GC_handle_fork = 1;
1350 } else
1351 # endif
1352 /* else */ if (GC_handle_fork != -1)
1353 ABORT("pthread_atfork failed");
1354 }
1355 # endif
1356 # ifdef INCLUDE_LINUX_THREAD_DESCR
1357 /* Explicitly register the region including the address */
1358 /* of a thread local variable. This should include thread */
1359 /* locals for the main thread, except for those allocated */
1360 /* in response to dlopen calls. */
1361 {
1362 ptr_t thread_local_addr = (ptr_t)(&GC_dummy_thread_local);
1363 ptr_t main_thread_start, main_thread_end;
1364 if (!GC_enclosing_mapping(thread_local_addr, &main_thread_start,
1365 &main_thread_end)) {
1366 ABORT("Failed to find mapping for main thread thread locals");
1367 } else {
1368 /* main_thread_start and main_thread_end are initialized. */
1369 GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
1370 }
1371 }
1372 # endif
1373 /* Add the initial thread, so we can stop it. */
1374 {
1375 pthread_t self = pthread_self();
1376 GC_thread t = GC_new_thread(self);
1377
1378 if (t == NULL)
1379 ABORT("Failed to allocate memory for the initial thread");
1380 # ifdef GC_DARWIN_THREADS
1381 t -> stop_info.mach_thread = mach_thread_self();
1382 # else
1383 t -> stop_info.stack_ptr = GC_approx_sp();
1384 # endif
1385 # ifndef DONT_USE_ATEXIT
1386 GC_main_thread_id = self;
1387 # endif
1388 t -> flags = DETACHED | MAIN_THREAD;
1389 if (THREAD_EQUAL(self, main_pthread_id)) {
1390 t -> stack = (ptr_t)main_stack;
1391 t -> stack_size = main_stack_size;
1392 t -> altstack = (ptr_t)main_altstack;
1393 t -> altstack_size = main_altstack_size;
1394 }
1395 }
1396
1397 /* Set GC_nprocs and available_markers_m1. */
1398 {
1399 char * nprocs_string = GETENV("GC_NPROCS");
1400 GC_nprocs = -1;
1401 if (nprocs_string != NULL) GC_nprocs = atoi(nprocs_string);
1402 }
1403 if (GC_nprocs <= 0
1404 # if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
1405 && (GC_nprocs = GC_get_nprocs_present()) <= 1
1406 /* Workaround for some Linux/arm kernels */
1407 # endif
1408 )
1409 {
1410 GC_nprocs = GC_get_nprocs();
1411 }
1412 if (GC_nprocs <= 0) {
1413 WARN("GC_get_nprocs() returned %" WARN_PRIdPTR "\n",
1414 (signed_word)GC_nprocs);
1415 GC_nprocs = 2; /* assume dual-core */
1416 # ifdef PARALLEL_MARK
1417 available_markers_m1 = 0; /* but use only one marker */
1418 # endif
1419 } else {
1420 # ifdef PARALLEL_MARK
1421 {
1422 char * markers_string = GETENV("GC_MARKERS");
1423 int markers = required_markers_cnt;
1424
1425 if (markers_string != NULL) {
1426 markers = atoi(markers_string);
1427 if (markers <= 0 || markers > MAX_MARKERS) {
1428 WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
1429 "; using maximum threads\n", (signed_word)markers);
1430 markers = MAX_MARKERS;
1431 }
1432 } else if (0 == markers) {
1433 /* Unless the client sets the desired number of */
1434 /* parallel markers, it is determined based on the */
1435 /* number of CPU cores. */
1436 markers = GC_nprocs;
1437 # if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
1438 /* This is primarily for targets without getenv(). */
1439 if (markers < GC_MIN_MARKERS)
1440 markers = GC_MIN_MARKERS;
1441 # endif
1442 if (markers > MAX_MARKERS)
1443 markers = MAX_MARKERS; /* silently limit the value */
1444 }
1445 available_markers_m1 = markers - 1;
1446 }
1447 # endif
1448 }
1449 GC_COND_LOG_PRINTF("Number of processors: %d\n", GC_nprocs);
1450
1451 # if defined(BASE_ATOMIC_OPS_EMULATED) && !defined(GC_DARWIN_THREADS) \
1452 && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
1453 && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
1454 /* Ensure the process is running on just one CPU core. */
1455 /* This is needed because the AO primitives emulated with */
1456 /* locks cannot be used inside signal handlers. */
1457 {
1458 cpu_set_t mask;
1459 int cpu_set_cnt = 0;
1460 int cpu_lowest_set = 0;
1461 int i = GC_nprocs > 1 ? GC_nprocs : 2; /* check at least 2 cores */
1462
1463 if (sched_getaffinity(0 /* current process */,
1464 sizeof(mask), &mask) == -1)
1465 ABORT_ARG1("sched_getaffinity failed", ": errno= %d", errno);
1466 while (i-- > 0)
1467 if (CPU_ISSET(i, &mask)) {
1468 cpu_lowest_set = i;
1469 cpu_set_cnt++;
1470 }
1471 if (0 == cpu_set_cnt)
1472 ABORT("sched_getaffinity returned empty mask");
1473 if (cpu_set_cnt > 1) {
1474 CPU_ZERO(&mask);
1475 CPU_SET(cpu_lowest_set, &mask); /* select just one CPU */
1476 if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
1477 ABORT_ARG1("sched_setaffinity failed", ": errno= %d", errno);
1478 WARN("CPU affinity mask is set to %p\n", (word)1 << cpu_lowest_set);
1479 }
1480 }
1481 # endif /* BASE_ATOMIC_OPS_EMULATED */
1482
1483 # ifndef GC_DARWIN_THREADS
1484 GC_stop_init();
1485 # endif
1486
1487 # ifdef PARALLEL_MARK
1488 if (available_markers_m1 <= 0) {
1489 /* Disable parallel marking. */
1490 GC_parallel = FALSE;
1491 GC_COND_LOG_PRINTF(
1492 "Single marker thread, turning off parallel marking\n");
1493 } else {
1494 setup_mark_lock();
1495 }
1496 # endif
1497 }
1498
1499 /* Perform all initializations, including those that */
1500 /* may require allocation. */
1501 /* Called without allocation lock. */
1502 /* Must be called before a second thread is created. */
1503 /* Did we say it's called without the allocation lock? */
1504 GC_INNER void GC_init_parallel(void)
1505 {
1506 # if defined(THREAD_LOCAL_ALLOC)
1507 DCL_LOCK_STATE;
1508 # endif
1509 if (parallel_initialized) return;
1510 parallel_initialized = TRUE;
1511
1512 /* GC_init() calls us back, so set flag first. */
1513 if (!GC_is_initialized) GC_init();
1514 /* Initialize thread local free lists if used. */
1515 # if defined(THREAD_LOCAL_ALLOC)
1516 LOCK();
1517 GC_init_thread_local(&(GC_lookup_thread(pthread_self())->tlfs));
1518 UNLOCK();
1519 # endif
1520 }
1521
1522 #ifndef GC_NO_PTHREAD_SIGMASK
1523 GC_API int WRAP_FUNC(pthread_sigmask)(int how, const sigset_t *set,
1524 sigset_t *oset)
1525 {
1526 sigset_t fudged_set;
1527
1528 INIT_REAL_SYMS();
1529 if (set != NULL && (how == SIG_BLOCK || how == SIG_SETMASK)) {
1530 int sig_suspend = GC_get_suspend_signal();
1531
1532 fudged_set = *set;
1533 GC_ASSERT(sig_suspend >= 0);
1534 if (sigdelset(&fudged_set, sig_suspend) != 0)
1535 ABORT("sigdelset failed");
1536 set = &fudged_set;
1537 }
1538 return(REAL_FUNC(pthread_sigmask)(how, set, oset));
1539 }
1540 #endif /* !GC_NO_PTHREAD_SIGMASK */
1541
1542 static GC_bool do_blocking_enter(GC_thread me)
1543 {
1544 # if defined(SPARC) || defined(IA64)
1545 ptr_t stack_ptr = GC_save_regs_in_stack();
1546 /* TODO: regs saving already done by GC_with_callee_saves_pushed */
1547 # elif defined(E2K)
1548 size_t stack_size;
1549 # endif
1550 GC_bool topOfStackUnset = FALSE;
1551
1552 GC_ASSERT(I_HOLD_LOCK());
1553 GC_ASSERT(!(me -> thread_blocked));
1554 # ifdef SPARC
1555 me -> stop_info.stack_ptr = stack_ptr;
1556 # else
1557 me -> stop_info.stack_ptr = GC_approx_sp();
1558 # endif
1559 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1560 if (me -> topOfStack == NULL) {
1561 /* GC_do_blocking_inner is not called recursively, */
1562 /* so topOfStack should be computed now. */
1563 topOfStackUnset = TRUE;
1564 me -> topOfStack = GC_FindTopOfStack(0);
1565 }
1566 # endif
1567 # ifdef IA64
1568 me -> backing_store_ptr = stack_ptr;
1569 # elif defined(E2K)
1570 GC_ASSERT(NULL == me -> backing_store_end);
1571 stack_size = GC_alloc_and_get_procedure_stack(&me->backing_store_end);
1572 me->backing_store_ptr = me->backing_store_end + stack_size;
1573 # endif
1574 me -> thread_blocked = (unsigned char)TRUE;
1575 /* Save context here if we want to support precise stack marking */
1576 return topOfStackUnset;
1577 }
1578
1579 static void do_blocking_leave(GC_thread me, GC_bool topOfStackUnset)
1580 {
1581 GC_ASSERT(I_HOLD_LOCK());
1582 # if defined(CPPCHECK)
1583 GC_noop1((word)&me->thread_blocked);
1584 # endif
1585 me -> thread_blocked = FALSE;
1586 # ifdef E2K
1587 GC_ASSERT(me -> backing_store_end != NULL);
1588 /* Note that me->backing_store_end value here may differ from */
1589 /* the one stored in this function previously. */
1590 GC_INTERNAL_FREE(me -> backing_store_end);
1591 me -> backing_store_ptr = NULL;
1592 me -> backing_store_end = NULL;
1593 # endif
1594 # if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
1595 if (topOfStackUnset)
1596 me -> topOfStack = NULL; /* make topOfStack unset again */
1597 # else
1598 (void)topOfStackUnset;
1599 # endif
1600 }
1601
1602 /* Wrapper for functions that are likely to block for an appreciable */
1603 /* length of time. */
1604 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context GC_ATTR_UNUSED)
1605 {
1606 struct blocking_data *d = (struct blocking_data *)data;
1607 GC_thread me;
1608 GC_bool topOfStackUnset;
1609 DCL_LOCK_STATE;
1610
1611 LOCK();
1612 me = GC_lookup_thread(pthread_self());
1613 topOfStackUnset = do_blocking_enter(me);
1614 UNLOCK();
1615
1616 d -> client_data = (d -> fn)(d -> client_data);
1617
1618 LOCK(); /* This will block if the world is stopped. */
1619 # ifdef LINT2
1620 {
1621 # ifdef GC_ASSERTIONS
1622 GC_thread saved_me = me;
1623 # endif
1624
1625 /* The pointer to the GC thread descriptor should not be */
1626 /* changed while the thread is registered but a static */
1627 /* analysis tool might complain that this pointer value */
1628 /* (obtained in the first locked section) is unreliable in */
1629 /* the second locked section. */
1630 me = GC_lookup_thread(pthread_self());
1631 GC_ASSERT(me == saved_me);
1632 }
1633 # endif
1634 # if defined(GC_ENABLE_SUSPEND_THREAD) && !defined(GC_DARWIN_THREADS) \
1635 && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
1636 && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
1637 /* Note: this code cannot be moved into do_blocking_leave() */
1638 /* otherwise there could be a static analysis tool warning */
1639 /* (false positive) about unlock without a matching lock. */
1640 while (EXPECT((me -> stop_info.ext_suspend_cnt & 1) != 0, FALSE)) {
1641 word suspend_cnt = (word)(me -> stop_info.ext_suspend_cnt);
1642 /* read suspend counter (number) before unlocking */
1643
1644 UNLOCK();
1645 GC_suspend_self_inner(me, suspend_cnt);
1646 LOCK();
1647 }
1648 # endif
1649 do_blocking_leave(me, topOfStackUnset);
1650 UNLOCK();
1651 }
1652
1653 #if defined(GC_ENABLE_SUSPEND_THREAD) && !defined(GC_DARWIN_THREADS) \
1654 && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
1655 && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
1656 /* Similar to GC_do_blocking_inner() but assuming the GC lock is held */
1657 /* and fn is GC_suspend_self_inner. */
1658 GC_INNER void GC_suspend_self_blocked(ptr_t thread_me,
1659 void * context GC_ATTR_UNUSED)
1660 {
1661 GC_thread me = (GC_thread)thread_me;
1662 GC_bool topOfStackUnset;
1663 DCL_LOCK_STATE;
1664
1665 GC_ASSERT(I_HOLD_LOCK());
1666 topOfStackUnset = do_blocking_enter(me);
1667 while ((me -> stop_info.ext_suspend_cnt & 1) != 0) {
1668 word suspend_cnt = (word)(me -> stop_info.ext_suspend_cnt);
1669
1670 UNLOCK();
1671 GC_suspend_self_inner(me, suspend_cnt);
1672 LOCK();
1673 }
1674 do_blocking_leave(me, topOfStackUnset);
1675 }
1676 #endif
1677
1678 GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
1679 const struct GC_stack_base *sb)
1680 {
1681 GC_thread t = (GC_thread)gc_thread_handle;
1682
1683 GC_ASSERT(sb -> mem_base != NULL);
1684 if (!EXPECT(GC_is_initialized, TRUE)) {
1685 GC_ASSERT(NULL == t);
1686 } else {
1687 GC_ASSERT(I_HOLD_LOCK());
1688 if (NULL == t) /* current thread? */
1689 t = GC_lookup_thread(pthread_self());
1690 GC_ASSERT((t -> flags & FINISHED) == 0);
1691 GC_ASSERT(!(t -> thread_blocked)
1692 && NULL == t -> traced_stack_sect); /* for now */
1693
1694 if ((t -> flags & MAIN_THREAD) == 0) {
1695 t -> stack_end = (ptr_t)sb->mem_base;
1696 # ifdef IA64
1697 t -> backing_store_end = (ptr_t)sb->reg_base;
1698 # endif
1699 return;
1700 }
1701 /* Otherwise alter the stack bottom of the primordial thread. */
1702 }
1703
1704 GC_stackbottom = (char*)sb->mem_base;
1705 # ifdef IA64
1706 GC_register_stackbottom = (ptr_t)sb->reg_base;
1707 # endif
1708 }
1709
1710 GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
1711 {
1712 pthread_t self = pthread_self();
1713 GC_thread me;
1714 DCL_LOCK_STATE;
1715
1716 LOCK();
1717 me = GC_lookup_thread(self);
1718 /* The thread is assumed to be registered. */
1719 if ((me -> flags & MAIN_THREAD) == 0) {
1720 sb -> mem_base = me -> stack_end;
1721 # ifdef IA64
1722 sb -> reg_base = me -> backing_store_end;
1723 # elif defined(E2K)
1724 sb -> reg_base = NULL;
1725 # endif
1726 } else {
1727 sb -> mem_base = GC_stackbottom;
1728 # ifdef IA64
1729 sb -> reg_base = GC_register_stackbottom;
1730 # elif defined(E2K)
1731 sb -> reg_base = NULL;
1732 # endif
1733 }
1734 UNLOCK();
1735 return (void *)me; /* gc_thread_handle */
1736 }
1737
1738 /* GC_call_with_gc_active() has the opposite to GC_do_blocking() */
1739 /* functionality. It might be called from a user function invoked by */
1740 /* GC_do_blocking() to temporarily back allow calling any GC function */
1741 /* and/or manipulating pointers to the garbage collected heap. */
1742 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
1743 void * client_data)
1744 {
1745 struct GC_traced_stack_sect_s stacksect;
1746 pthread_t self = pthread_self();
1747 GC_thread me;
1748 # ifdef E2K
1749 size_t stack_size;
1750 # endif
1751 DCL_LOCK_STATE;
1752
1753 LOCK(); /* This will block if the world is stopped. */
1754 me = GC_lookup_thread(self);
1755
1756 /* Adjust our stack bottom value (this could happen unless */
1757 /* GC_get_stack_base() was used which returned GC_SUCCESS). */
1758 if ((me -> flags & MAIN_THREAD) == 0) {
1759 GC_ASSERT(me -> stack_end != NULL);
1760 if ((word)me->stack_end HOTTER_THAN (word)(&stacksect))
1761 me -> stack_end = (ptr_t)(&stacksect);
1762 } else {
1763 /* The original stack. */
1764 if ((word)GC_stackbottom HOTTER_THAN (word)(&stacksect))
1765 GC_stackbottom = (ptr_t)COVERT_DATAFLOW(&stacksect);
1766 }
1767
1768 if (!me->thread_blocked) {
1769 /* We are not inside GC_do_blocking() - do nothing more. */
1770 UNLOCK();
1771 client_data = fn(client_data);
1772 /* Prevent treating the above as a tail call. */
1773 GC_noop1(COVERT_DATAFLOW(&stacksect));
1774 return client_data; /* result */
1775 }
1776
1777 # if defined(GC_ENABLE_SUSPEND_THREAD) && !defined(GC_DARWIN_THREADS) \
1778 && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
1779 && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
1780 while (EXPECT((me -> stop_info.ext_suspend_cnt & 1) != 0, FALSE)) {
1781 word suspend_cnt = (word)(me -> stop_info.ext_suspend_cnt);
1782 UNLOCK();
1783 GC_suspend_self_inner(me, suspend_cnt);
1784 LOCK();
1785 }
1786 # endif
1787
1788 /* Setup new "stack section". */
1789 stacksect.saved_stack_ptr = me -> stop_info.stack_ptr;
1790 # ifdef IA64
1791 /* This is the same as in GC_call_with_stack_base(). */
1792 stacksect.backing_store_end = GC_save_regs_in_stack();
1793 /* Unnecessarily flushes register stack, */
1794 /* but that probably doesn't hurt. */
1795 stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
1796 # elif defined(E2K)
1797 GC_ASSERT(me -> backing_store_end != NULL);
1798 GC_INTERNAL_FREE(me -> backing_store_end);
1799 me -> backing_store_ptr = NULL;
1800 me -> backing_store_end = NULL;
1801 # endif
1802 stacksect.prev = me -> traced_stack_sect;
1803 me -> thread_blocked = FALSE;
1804 me -> traced_stack_sect = &stacksect;
1805
1806 UNLOCK();
1807 client_data = fn(client_data);
1808 GC_ASSERT(me -> thread_blocked == FALSE);
1809 GC_ASSERT(me -> traced_stack_sect == &stacksect);
1810
1811 /* Restore original "stack section". */
1812 # if defined(CPPCHECK)
1813 GC_noop1((word)me->traced_stack_sect);
1814 # endif
1815 # ifdef E2K
1816 (void)GC_save_regs_in_stack();
1817 # endif
1818 LOCK();
1819 me -> traced_stack_sect = stacksect.prev;
1820 # ifdef IA64
1821 me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
1822 # elif defined(E2K)
1823 GC_ASSERT(NULL == me -> backing_store_end);
1824 stack_size = GC_alloc_and_get_procedure_stack(&me->backing_store_end);
1825 me->backing_store_ptr = me->backing_store_end + stack_size;
1826 # endif
1827 me -> thread_blocked = (unsigned char)TRUE;
1828 me -> stop_info.stack_ptr = stacksect.saved_stack_ptr;
1829 UNLOCK();
1830
1831 return client_data; /* result */
1832 }
1833
1834 STATIC void GC_unregister_my_thread_inner(GC_thread me)
1835 {
1836 GC_ASSERT(I_HOLD_LOCK());
1837 # ifdef DEBUG_THREADS
1838 GC_log_printf(
1839 "Unregistering thread %p, gc_thread= %p, n_threads= %d\n",
1840 (void *)me->id, (void *)me, GC_count_threads());
1841 # endif
1842 GC_ASSERT(!(me -> flags & FINISHED));
1843 # if defined(THREAD_LOCAL_ALLOC)
1844 GC_ASSERT(GC_getspecific(GC_thread_key) == &me->tlfs);
1845 GC_destroy_thread_local(&(me->tlfs));
1846 # endif
1847 # ifdef NACL
1848 GC_nacl_shutdown_gc_thread();
1849 GC_nacl_gc_thread_self = NULL;
1850 # endif
1851 # if defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)
1852 /* Handle DISABLED_GC flag which is set by the */
1853 /* intercepted pthread_cancel or pthread_exit. */
1854 if ((me -> flags & DISABLED_GC) != 0) {
1855 GC_dont_gc--;
1856 }
1857 # endif
1858 if (me -> flags & DETACHED) {
1859 GC_delete_thread(pthread_self());
1860 } else {
1861 me -> flags |= FINISHED;
1862 }
1863 # if defined(THREAD_LOCAL_ALLOC)
1864 /* It is required to call remove_specific defined in specific.c. */
1865 GC_remove_specific(GC_thread_key);
1866 # endif
1867 }
1868
1869 GC_API int GC_CALL GC_unregister_my_thread(void)
1870 {
1871 pthread_t self = pthread_self();
1872 GC_thread me;
1873 IF_CANCEL(int cancel_state;)
1874 DCL_LOCK_STATE;
1875
1876 LOCK();
1877 DISABLE_CANCEL(cancel_state);
1878 /* Wait for any GC that may be marking from our stack to */
1879 /* complete before we remove this thread. */
1880 GC_wait_for_gc_completion(FALSE);
1881 me = GC_lookup_thread(self);
1882 # ifdef DEBUG_THREADS
1883 GC_log_printf(
1884 "Called GC_unregister_my_thread on %p, gc_thread= %p\n",
1885 (void *)self, (void *)me);
1886 # endif
1887 GC_ASSERT(THREAD_EQUAL(me->id, self));
1888 GC_unregister_my_thread_inner(me);
1889 RESTORE_CANCEL(cancel_state);
1890 UNLOCK();
1891 return GC_SUCCESS;
1892 }
1893
1894 /* Called at thread exit. */
1895 /* Never called for main thread. That's OK, since it */
1896 /* results in at most a tiny one-time leak. And */
1897 /* linuxthreads doesn't reclaim the main threads */
1898 /* resources or id anyway. */
1899 GC_INNER_PTHRSTART void GC_thread_exit_proc(void *arg)
1900 {
1901 IF_CANCEL(int cancel_state;)
1902 DCL_LOCK_STATE;
1903
1904 # ifdef DEBUG_THREADS
1905 GC_log_printf("Called GC_thread_exit_proc on %p, gc_thread= %p\n",
1906 (void *)((GC_thread)arg)->id, arg);
1907 # endif
1908 LOCK();
1909 DISABLE_CANCEL(cancel_state);
1910 GC_wait_for_gc_completion(FALSE);
1911 GC_unregister_my_thread_inner((GC_thread)arg);
1912 RESTORE_CANCEL(cancel_state);
1913 UNLOCK();
1914 }
1915
1916 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
1917 GC_API int WRAP_FUNC(pthread_join)(pthread_t thread, void **retval)
1918 {
1919 int result;
1920 GC_thread t;
1921 DCL_LOCK_STATE;
1922
1923 INIT_REAL_SYMS();
1924 LOCK();
1925 t = (GC_thread)COVERT_DATAFLOW(GC_lookup_thread(thread));
1926 /* This is guaranteed to be the intended one, since the thread id */
1927 /* can't have been recycled by pthreads. */
1928 UNLOCK();
1929 result = REAL_FUNC(pthread_join)(thread, retval);
1930 # if defined(GC_FREEBSD_THREADS)
1931 /* On FreeBSD, the wrapped pthread_join() sometimes returns (what
1932 appears to be) a spurious EINTR which caused the test and real code
1933 to gratuitously fail. Having looked at system pthread library source
1934 code, I see how this return code may be generated. In one path of
1935 code, pthread_join() just returns the errno setting of the thread
1936 being joined. This does not match the POSIX specification or the
1937 local man pages thus I have taken the liberty to catch this one
1938 spurious return value properly conditionalized on GC_FREEBSD_THREADS. */
1939 if (result == EINTR) result = 0;
1940 # endif
1941 if (result == 0) {
1942 LOCK();
1943 /* Here the pthread thread id may have been recycled. */
1944 /* Delete the thread from GC_threads (unless it has been */
1945 /* registered again from the client thread key destructor). */
1946 if ((t -> flags & FINISHED) != 0)
1947 GC_delete_gc_thread(t);
1948 UNLOCK();
1949 }
1950 return result;
1951 }
1952
1953 GC_API int WRAP_FUNC(pthread_detach)(pthread_t thread)
1954 {
1955 int result;
1956 GC_thread t;
1957 DCL_LOCK_STATE;
1958
1959 INIT_REAL_SYMS();
1960 LOCK();
1961 t = (GC_thread)COVERT_DATAFLOW(GC_lookup_thread(thread));
1962 UNLOCK();
1963 result = REAL_FUNC(pthread_detach)(thread);
1964 if (result == 0) {
1965 LOCK();
1966 t -> flags |= DETACHED;
1967 /* Here the pthread thread id may have been recycled. */
1968 if ((t -> flags & FINISHED) != 0) {
1969 GC_delete_gc_thread(t);
1970 }
1971 UNLOCK();
1972 }
1973 return result;
1974 }
1975 #endif /* !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */
1976
1977 #ifndef GC_NO_PTHREAD_CANCEL
1978 /* We should deal with the fact that apparently on Solaris and, */
1979 /* probably, on some Linux we can't collect while a thread is */
1980 /* exiting, since signals aren't handled properly. This currently */
1981 /* gives rise to deadlocks. The only workaround seen is to intercept */
1982 /* pthread_cancel() and pthread_exit(), and disable the collections */
1983 /* until the thread exit handler is called. That's ugly, because we */
1984 /* risk growing the heap unnecessarily. But it seems that we don't */
1985 /* really have an option in that the process is not in a fully */
1986 /* functional state while a thread is exiting. */
1987 GC_API int WRAP_FUNC(pthread_cancel)(pthread_t thread)
1988 {
1989 # ifdef CANCEL_SAFE
1990 GC_thread t;
1991 DCL_LOCK_STATE;
1992 # endif
1993
1994 INIT_REAL_SYMS();
1995 # ifdef CANCEL_SAFE
1996 LOCK();
1997 t = GC_lookup_thread(thread);
1998 /* We test DISABLED_GC because pthread_exit could be called at */
1999 /* the same time. (If t is NULL then pthread_cancel should */
2000 /* return ESRCH.) */
2001 if (t != NULL && (t -> flags & DISABLED_GC) == 0) {
2002 t -> flags |= DISABLED_GC;
2003 GC_dont_gc++;
2004 }
2005 UNLOCK();
2006 # endif
2007 return REAL_FUNC(pthread_cancel)(thread);
2008 }
2009 #endif /* !GC_NO_PTHREAD_CANCEL */
2010
2011 #ifdef GC_HAVE_PTHREAD_EXIT
2012 GC_API GC_PTHREAD_EXIT_ATTRIBUTE void WRAP_FUNC(pthread_exit)(void *retval)
2013 {
2014 pthread_t self = pthread_self();
2015 GC_thread me;
2016 DCL_LOCK_STATE;
2017
2018 INIT_REAL_SYMS();
2019 LOCK();
2020 me = GC_lookup_thread(self);
2021 /* We test DISABLED_GC because someone else could call */
2022 /* pthread_cancel at the same time. */
2023 if (me != 0 && (me -> flags & DISABLED_GC) == 0) {
2024 me -> flags |= DISABLED_GC;
2025 GC_dont_gc++;
2026 }
2027 UNLOCK();
2028
2029 REAL_FUNC(pthread_exit)(retval);
2030 }
2031 #endif /* GC_HAVE_PTHREAD_EXIT */
2032
2033 GC_INNER GC_bool GC_in_thread_creation = FALSE;
2034 /* Protected by allocation lock. */
2035
2036 GC_INLINE void GC_record_stack_base(GC_thread me,
2037 const struct GC_stack_base *sb)
2038 {
2039 # ifndef GC_DARWIN_THREADS
2040 me -> stop_info.stack_ptr = (ptr_t)sb->mem_base;
2041 # endif
2042 me -> stack_end = (ptr_t)sb->mem_base;
2043 if (me -> stack_end == NULL)
2044 ABORT("Bad stack base in GC_register_my_thread");
2045 # ifdef IA64
2046 me -> backing_store_end = (ptr_t)sb->reg_base;
2047 # endif
2048 }
2049
2050 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
2051 pthread_t my_pthread)
2052 {
2053 GC_thread me;
2054
2055 GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */
2056 me = GC_new_thread(my_pthread);
2057 GC_in_thread_creation = FALSE;
2058 if (me == 0)
2059 ABORT("Failed to allocate memory for thread registering");
2060 # ifdef GC_DARWIN_THREADS
2061 me -> stop_info.mach_thread = mach_thread_self();
2062 # endif
2063 GC_record_stack_base(me, sb);
2064 # ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
2065 /* Since this could be executed from a detached thread */
2066 /* destructor, our signals might already be blocked. */
2067 GC_unblock_gc_signals();
2068 # endif
2069 return me;
2070 }
2071
2072 GC_API void GC_CALL GC_allow_register_threads(void)
2073 {
2074 # ifdef GC_ASSERTIONS
2075 DCL_LOCK_STATE;
2076
2077 /* Check GC is initialized and the current thread is registered. */
2078 LOCK(); /* just to match that in win32_threads.c */
2079 GC_ASSERT(GC_lookup_thread(pthread_self()) != 0);
2080 UNLOCK();
2081 # endif
2082 INIT_REAL_SYMS(); /* to initialize symbols while single-threaded */
2083 GC_init_lib_bounds();
2084 GC_start_mark_threads();
2085 set_need_to_lock();
2086 }
2087
2088 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
2089 {
2090 pthread_t self = pthread_self();
2091 GC_thread me;
2092 DCL_LOCK_STATE;
2093
2094 if (GC_need_to_lock == FALSE)
2095 ABORT("Threads explicit registering is not previously enabled");
2096
2097 LOCK();
2098 me = GC_lookup_thread(self);
2099 if (0 == me) {
2100 me = GC_register_my_thread_inner(sb, self);
2101 # if defined(CPPCHECK)
2102 GC_noop1(me->flags);
2103 # endif
2104 me -> flags |= DETACHED;
2105 /* Treat as detached, since we do not need to worry about */
2106 /* pointer results. */
2107 # if defined(THREAD_LOCAL_ALLOC)
2108 GC_init_thread_local(&(me->tlfs));
2109 # endif
2110 UNLOCK();
2111 return GC_SUCCESS;
2112 } else if ((me -> flags & FINISHED) != 0) {
2113 /* This code is executed when a thread is registered from the */
2114 /* client thread key destructor. */
2115 # ifdef NACL
2116 GC_nacl_gc_thread_self = me;
2117 GC_nacl_initialize_gc_thread();
2118 # endif
2119 # ifdef GC_DARWIN_THREADS
2120 /* Reinitialize mach_thread to avoid thread_suspend fail */
2121 /* with MACH_SEND_INVALID_DEST error. */
2122 me -> stop_info.mach_thread = mach_thread_self();
2123 # endif
2124 GC_record_stack_base(me, sb);
2125 me -> flags &= ~FINISHED; /* but not DETACHED */
2126 # ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
2127 /* Since this could be executed from a thread destructor, */
2128 /* our signals might be blocked. */
2129 GC_unblock_gc_signals();
2130 # endif
2131 # if defined(THREAD_LOCAL_ALLOC)
2132 GC_init_thread_local(&(me->tlfs));
2133 # endif
2134 # if defined(GC_ENABLE_SUSPEND_THREAD) && !defined(GC_DARWIN_THREADS) \
2135 && !defined(GC_OPENBSD_UTHREADS) && !defined(NACL) \
2136 && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
2137 if ((me -> stop_info.ext_suspend_cnt & 1) != 0) {
2138 GC_with_callee_saves_pushed(GC_suspend_self_blocked, (ptr_t)me);
2139 }
2140 # endif
2141 UNLOCK();
2142 return GC_SUCCESS;
2143 } else {
2144 UNLOCK();
2145 return GC_DUPLICATE;
2146 }
2147 }
2148
2149 struct start_info {
2150 void *(*start_routine)(void *);
2151 void *arg;
2152 word flags;
2153 sem_t registered; /* 1 ==> in our thread table, but */
2154 /* parent hasn't yet noticed. */
2155 };
2156
2157 /* Called from GC_inner_start_routine(). Defined in this file to */
2158 /* minimize the number of include files in pthread_start.c (because */
2159 /* sem_t and sem_post() are not used in that file directly). */
2160 GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
2161 void *(**pstart)(void *),
2162 void **pstart_arg,
2163 struct GC_stack_base *sb, void *arg)
2164 {
2165 struct start_info * si = (struct start_info *)arg;
2166 pthread_t self = pthread_self();
2167 GC_thread me;
2168 DCL_LOCK_STATE;
2169
2170 # ifdef DEBUG_THREADS
2171 GC_log_printf("Starting thread %p, pid= %ld, sp= %p\n",
2172 (void *)self, (long)getpid(), (void *)&arg);
2173 # endif
2174 LOCK();
2175 me = GC_register_my_thread_inner(sb, self);
2176 me -> flags = si -> flags;
2177 # if defined(THREAD_LOCAL_ALLOC)
2178 GC_init_thread_local(&(me->tlfs));
2179 # endif
2180 UNLOCK();
2181 *pstart = si -> start_routine;
2182 # ifdef DEBUG_THREADS
2183 GC_log_printf("start_routine= %p\n", (void *)(signed_word)(*pstart));
2184 # endif
2185 *pstart_arg = si -> arg;
2186 sem_post(&(si -> registered)); /* Last action on si. */
2187 /* OK to deallocate. */
2188 return me;
2189 }
2190
2191 #if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
2192 STATIC void * GC_start_routine(void * arg)
2193 {
2194 # ifdef INCLUDE_LINUX_THREAD_DESCR
2195 struct GC_stack_base sb;
2196
2197 # ifdef REDIRECT_MALLOC
2198 /* GC_get_stack_base may call pthread_getattr_np, which can */
2199 /* unfortunately call realloc, which may allocate from an */
2200 /* unregistered thread. This is unpleasant, since it might */
2201 /* force heap growth (or, even, heap overflow). */
2202 GC_disable();
2203 # endif
2204 if (GC_get_stack_base(&sb) != GC_SUCCESS)
2205 ABORT("Failed to get thread stack base");
2206 # ifdef REDIRECT_MALLOC
2207 GC_enable();
2208 # endif
2209 return GC_inner_start_routine(&sb, arg);
2210 # else
2211 return GC_call_with_stack_base(GC_inner_start_routine, arg);
2212 # endif
2213 }
2214
2215 GC_API int WRAP_FUNC(pthread_create)(pthread_t *new_thread,
2216 GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
2217 void *(*start_routine)(void *), void *arg)
2218 {
2219 int result;
2220 int detachstate;
2221 word my_flags = 0;
2222 struct start_info si;
2223 DCL_LOCK_STATE;
2224 /* This is otherwise saved only in an area mmapped by the thread */
2225 /* library, which isn't visible to the collector. */
2226
2227 /* We resist the temptation to muck with the stack size here, */
2228 /* even if the default is unreasonably small. That's the client's */
2229 /* responsibility. */
2230
2231 INIT_REAL_SYMS();
2232 if (!EXPECT(parallel_initialized, TRUE))
2233 GC_init_parallel();
2234 GC_init_lib_bounds();
2235 if (sem_init(&si.registered, GC_SEM_INIT_PSHARED, 0) != 0)
2236 ABORT("sem_init failed");
2237
2238 si.start_routine = start_routine;
2239 si.arg = arg;
2240 LOCK();
2241 if (!EXPECT(GC_thr_initialized, TRUE))
2242 GC_thr_init();
2243 # ifdef GC_ASSERTIONS
2244 {
2245 size_t stack_size = 0;
2246 if (NULL != attr) {
2247 if (pthread_attr_getstacksize(attr, &stack_size) != 0)
2248 ABORT("pthread_attr_getstacksize failed");
2249 }
2250 if (0 == stack_size) {
2251 pthread_attr_t my_attr;
2252
2253 if (pthread_attr_init(&my_attr) != 0)
2254 ABORT("pthread_attr_init failed");
2255 if (pthread_attr_getstacksize(&my_attr, &stack_size) != 0)
2256 ABORT("pthread_attr_getstacksize failed");
2257 (void)pthread_attr_destroy(&my_attr);
2258 }
2259 /* On Solaris 10, with default attr initialization, */
2260 /* stack_size remains 0. Fudge it. */
2261 if (0 == stack_size) {
2262 # ifndef SOLARIS
2263 WARN("Failed to get stack size for assertion checking\n", 0);
2264 # endif
2265 stack_size = 1000000;
2266 }
2267 GC_ASSERT(stack_size >= 65536);
2268 /* Our threads may need to do some work for the GC. */
2269 /* Ridiculously small threads won't work, and they */
2270 /* probably wouldn't work anyway. */
2271 }
2272 # endif
2273 if (NULL == attr) {
2274 detachstate = PTHREAD_CREATE_JOINABLE;
2275 } else {
2276 if (pthread_attr_getdetachstate(attr, &detachstate) != 0)
2277 ABORT("pthread_attr_getdetachstate failed");
2278 }
2279 if (PTHREAD_CREATE_DETACHED == detachstate) my_flags |= DETACHED;
2280 si.flags = my_flags;
2281 UNLOCK();
2282 # ifdef DEBUG_THREADS
2283 GC_log_printf("About to start new thread from thread %p\n",
2284 (void *)pthread_self());
2285 # endif
2286 # ifdef PARALLEL_MARK
2287 if (EXPECT(!GC_parallel && available_markers_m1 > 0, FALSE))
2288 GC_start_mark_threads();
2289 # endif
2290 set_need_to_lock();
2291 result = REAL_FUNC(pthread_create)(new_thread, attr, GC_start_routine,
2292 &si);
2293
2294 /* Wait until child has been added to the thread table. */
2295 /* This also ensures that we hold onto the stack-allocated si until */
2296 /* the child is done with it. */
2297 if (0 == result) {
2298 IF_CANCEL(int cancel_state;)
2299
2300 # ifdef DEBUG_THREADS
2301 /* new_thread is non-NULL because pthread_create requires it. */
2302 GC_log_printf("Started thread %p\n", (void *)(*new_thread));
2303 # endif
2304 DISABLE_CANCEL(cancel_state);
2305 /* pthread_create is not a cancellation point. */
2306 while (0 != sem_wait(&si.registered)) {
2307 # if defined(GC_HAIKU_THREADS)
2308 /* To workaround some bug in Haiku semaphores. */
2309 if (EACCES == errno) continue;
2310 # endif
2311 if (EINTR != errno) ABORT("sem_wait failed");
2312 }
2313 RESTORE_CANCEL(cancel_state);
2314 }
2315 sem_destroy(&si.registered);
2316 return(result);
2317 }
2318 #endif /* !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */
2319
2320 #if defined(USE_SPIN_LOCK) || !defined(NO_PTHREAD_TRYLOCK)
2321 /* Spend a few cycles in a way that can't introduce contention with */
2322 /* other threads. */
2323 #define GC_PAUSE_SPIN_CYCLES 10
2324 STATIC void GC_pause(void)
2325 {
2326 int i;
2327
2328 for (i = 0; i < GC_PAUSE_SPIN_CYCLES; ++i) {
2329 /* Something that's unlikely to be optimized away. */
2330 # if defined(AO_HAVE_compiler_barrier) \
2331 && !defined(BASE_ATOMIC_OPS_EMULATED)
2332 AO_compiler_barrier();
2333 # else
2334 GC_noop1(i);
2335 # endif
2336 }
2337 }
2338 #endif
2339
2340 #ifndef SPIN_MAX
2341 # define SPIN_MAX 128 /* Maximum number of calls to GC_pause before */
2342 /* give up. */
2343 #endif
2344
2345 GC_INNER volatile GC_bool GC_collecting = FALSE;
2346 /* A hint that we're in the collector and */
2347 /* holding the allocation lock for an */
2348 /* extended period. */
2349
2350 #if (!defined(USE_SPIN_LOCK) && !defined(NO_PTHREAD_TRYLOCK)) \
2351 || defined(PARALLEL_MARK)
2352 /* If we don't want to use the below spinlock implementation, either */
2353 /* because we don't have a GC_test_and_set implementation, or because */
2354 /* we don't want to risk sleeping, we can still try spinning on */
2355 /* pthread_mutex_trylock for a while. This appears to be very */
2356 /* beneficial in many cases. */
2357 /* I suspect that under high contention this is nearly always better */
2358 /* than the spin lock. But it's a bit slower on a uniprocessor. */
2359 /* Hence we still default to the spin lock. */
2360 /* This is also used to acquire the mark lock for the parallel */
2361 /* marker. */
2362
2363 /* Here we use a strict exponential backoff scheme. I don't know */
2364 /* whether that's better or worse than the above. We eventually */
2365 /* yield by calling pthread_mutex_lock(); it never makes sense to */
2366 /* explicitly sleep. */
2367
2368 /* #define LOCK_STATS */
2369 /* Note that LOCK_STATS requires AO_HAVE_test_and_set. */
2370 #ifdef LOCK_STATS
2371 volatile AO_t GC_spin_count = 0;
2372 volatile AO_t GC_block_count = 0;
2373 volatile AO_t GC_unlocked_count = 0;
2374 #endif
2375
2376 STATIC void GC_generic_lock(pthread_mutex_t * lock)
2377 {
2378 #ifndef NO_PTHREAD_TRYLOCK
2379 unsigned pause_length = 1;
2380 unsigned i;
2381
2382 if (0 == pthread_mutex_trylock(lock)) {
2383 # ifdef LOCK_STATS
2384 (void)AO_fetch_and_add1(&GC_unlocked_count);
2385 # endif
2386 return;
2387 }
2388 for (; pause_length <= SPIN_MAX; pause_length <<= 1) {
2389 for (i = 0; i < pause_length; ++i) {
2390 GC_pause();
2391 }
2392 switch(pthread_mutex_trylock(lock)) {
2393 case 0:
2394 # ifdef LOCK_STATS
2395 (void)AO_fetch_and_add1(&GC_spin_count);
2396 # endif
2397 return;
2398 case EBUSY:
2399 break;
2400 default:
2401 ABORT("Unexpected error from pthread_mutex_trylock");
2402 }
2403 }
2404 #endif /* !NO_PTHREAD_TRYLOCK */
2405 # ifdef LOCK_STATS
2406 (void)AO_fetch_and_add1(&GC_block_count);
2407 # endif
2408 pthread_mutex_lock(lock);
2409 }
2410
2411 #endif /* !USE_SPIN_LOCK || ... */
2412
2413 #if defined(AO_HAVE_char_load) && !defined(BASE_ATOMIC_OPS_EMULATED)
2414 # define is_collecting() \
2415 ((GC_bool)AO_char_load((unsigned char *)&GC_collecting))
2416 #else
2417 /* GC_collecting is a hint, a potential data race between */
2418 /* GC_lock() and ENTER/EXIT_GC() is OK to ignore. */
2419 # define is_collecting() GC_collecting
2420 #endif
2421
2422 #if defined(USE_SPIN_LOCK)
2423
2424 /* Reasonably fast spin locks. Basically the same implementation */
2425 /* as STL alloc.h. This isn't really the right way to do this. */
2426 /* but until the POSIX scheduling mess gets straightened out ... */
2427
2428 GC_INNER volatile AO_TS_t GC_allocate_lock = AO_TS_INITIALIZER;
2429
2430 # define low_spin_max 30 /* spin cycles if we suspect uniprocessor */
2431 # define high_spin_max SPIN_MAX /* spin cycles for multiprocessor */
2432
2433 static volatile AO_t spin_max = low_spin_max;
2434 static volatile AO_t last_spins = 0;
2435 /* A potential data race between */
2436 /* threads invoking GC_lock which reads */
2437 /* and updates spin_max and last_spins */
2438 /* could be ignored because these */
2439 /* variables are hints only. */
2440
2441 GC_INNER void GC_lock(void)
2442 {
2443 unsigned my_spin_max;
2444 unsigned my_last_spins;
2445 unsigned i;
2446
2447 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2448 return;
2449 }
2450 my_spin_max = (unsigned)AO_load(&spin_max);
2451 my_last_spins = (unsigned)AO_load(&last_spins);
2452 for (i = 0; i < my_spin_max; i++) {
2453 if (is_collecting() || GC_nprocs == 1)
2454 goto yield;
2455 if (i < my_last_spins/2) {
2456 GC_pause();
2457 continue;
2458 }
2459 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2460 /*
2461 * got it!
2462 * Spinning worked. Thus we're probably not being scheduled
2463 * against the other process with which we were contending.
2464 * Thus it makes sense to spin longer the next time.
2465 */
2466 AO_store(&last_spins, (AO_t)i);
2467 AO_store(&spin_max, (AO_t)high_spin_max);
2468 return;
2469 }
2470 }
2471 /* We are probably being scheduled against the other process. Sleep. */
2472 AO_store(&spin_max, (AO_t)low_spin_max);
2473 yield:
2474 for (i = 0;; ++i) {
2475 if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
2476 return;
2477 }
2478 # define SLEEP_THRESHOLD 12
2479 /* Under Linux very short sleeps tend to wait until */
2480 /* the current time quantum expires. On old Linux */
2481 /* kernels nanosleep (<= 2 ms) just spins. */
2482 /* (Under 2.4, this happens only for real-time */
2483 /* processes.) We want to minimize both behaviors */
2484 /* here. */
2485 if (i < SLEEP_THRESHOLD) {
2486 sched_yield();
2487 } else {
2488 struct timespec ts;
2489
2490 if (i > 24) i = 24;
2491 /* Don't wait for more than about 15 ms, */
2492 /* even under extreme contention. */
2493 ts.tv_sec = 0;
2494 ts.tv_nsec = 1 << i;
2495 nanosleep(&ts, 0);
2496 }
2497 }
2498 }
2499
2500 #elif defined(USE_PTHREAD_LOCKS)
2501
2502 # ifndef NO_PTHREAD_TRYLOCK
2503 GC_INNER void GC_lock(void)
2504 {
2505 if (1 == GC_nprocs || is_collecting()) {
2506 pthread_mutex_lock(&GC_allocate_ml);
2507 } else {
2508 GC_generic_lock(&GC_allocate_ml);
2509 }
2510 }
2511 # elif defined(GC_ASSERTIONS)
2512 GC_INNER void GC_lock(void)
2513 {
2514 pthread_mutex_lock(&GC_allocate_ml);
2515 }
2516 # endif
2517
2518 #endif /* !USE_SPIN_LOCK && USE_PTHREAD_LOCKS */
2519
2520 #ifdef PARALLEL_MARK
2521
2522 # ifdef GC_ASSERTIONS
2523 STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
2524 # define SET_MARK_LOCK_HOLDER \
2525 (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
2526 # define UNSET_MARK_LOCK_HOLDER \
2527 do { \
2528 GC_ASSERT(GC_mark_lock_holder \
2529 == NUMERIC_THREAD_ID(pthread_self())); \
2530 GC_mark_lock_holder = NO_THREAD; \
2531 } while (0)
2532 # else
2533 # define SET_MARK_LOCK_HOLDER (void)0
2534 # define UNSET_MARK_LOCK_HOLDER (void)0
2535 # endif /* !GC_ASSERTIONS */
2536
2537 static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
2538
2539 static void setup_mark_lock(void)
2540 {
2541 # ifdef GLIBC_2_19_TSX_BUG
2542 pthread_mutexattr_t mattr;
2543 int glibc_minor = -1;
2544 int glibc_major = GC_parse_version(&glibc_minor, gnu_get_libc_version());
2545
2546 if (glibc_major > 2 || (glibc_major == 2 && glibc_minor >= 19)) {
2547 /* TODO: disable this workaround for glibc with fixed TSX */
2548 /* This disables lock elision to workaround a bug in glibc 2.19+ */
2549 if (0 != pthread_mutexattr_init(&mattr)) {
2550 ABORT("pthread_mutexattr_init failed");
2551 }
2552 if (0 != pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL)) {
2553 ABORT("pthread_mutexattr_settype failed");
2554 }
2555 if (0 != pthread_mutex_init(&mark_mutex, &mattr)) {
2556 ABORT("pthread_mutex_init failed");
2557 }
2558 (void)pthread_mutexattr_destroy(&mattr);
2559 }
2560 # endif
2561 }
2562
2563 GC_INNER void GC_acquire_mark_lock(void)
2564 {
2565 # if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
2566 GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
2567 # endif
2568 GC_generic_lock(&mark_mutex);
2569 SET_MARK_LOCK_HOLDER;
2570 }
2571
2572 GC_INNER void GC_release_mark_lock(void)
2573 {
2574 UNSET_MARK_LOCK_HOLDER;
2575 if (pthread_mutex_unlock(&mark_mutex) != 0) {
2576 ABORT("pthread_mutex_unlock failed");
2577 }
2578 }
2579
2580 /* Collector must wait for a freelist builders for 2 reasons: */
2581 /* 1) Mark bits may still be getting examined without lock. */
2582 /* 2) Partial free lists referenced only by locals may not be scanned */
2583 /* correctly, e.g. if they contain "pointer-free" objects, since the */
2584 /* free-list link may be ignored. */
2585 STATIC void GC_wait_builder(void)
2586 {
2587 ASSERT_CANCEL_DISABLED();
2588 UNSET_MARK_LOCK_HOLDER;
2589 if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
2590 ABORT("pthread_cond_wait failed");
2591 }
2592 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2593 SET_MARK_LOCK_HOLDER;
2594 }
2595
2596 GC_INNER void GC_wait_for_reclaim(void)
2597 {
2598 GC_acquire_mark_lock();
2599 while (GC_fl_builder_count > 0) {
2600 GC_wait_builder();
2601 }
2602 GC_release_mark_lock();
2603 }
2604
2605 # if defined(CAN_HANDLE_FORK) && defined(THREAD_SANITIZER)
2606 /* Identical to GC_wait_for_reclaim() but with the no_sanitize */
2607 /* attribute as a workaround for TSan which does not notice that */
2608 /* the GC lock is acquired in fork_prepare_proc(). */
2609 GC_ATTR_NO_SANITIZE_THREAD
2610 static void wait_for_reclaim_atfork(void)
2611 {
2612 GC_acquire_mark_lock();
2613 while (GC_fl_builder_count > 0)
2614 GC_wait_builder();
2615 GC_release_mark_lock();
2616 }
2617 # endif /* CAN_HANDLE_FORK && THREAD_SANITIZER */
2618
2619 GC_INNER void GC_notify_all_builder(void)
2620 {
2621 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
2622 if (pthread_cond_broadcast(&builder_cv) != 0) {
2623 ABORT("pthread_cond_broadcast failed");
2624 }
2625 }
2626
2627 GC_INNER void GC_wait_marker(void)
2628 {
2629 ASSERT_CANCEL_DISABLED();
2630 GC_ASSERT(GC_parallel);
2631 UNSET_MARK_LOCK_HOLDER;
2632 if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
2633 ABORT("pthread_cond_wait failed");
2634 }
2635 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2636 SET_MARK_LOCK_HOLDER;
2637 }
2638
2639 GC_INNER void GC_notify_all_marker(void)
2640 {
2641 GC_ASSERT(GC_parallel);
2642 if (pthread_cond_broadcast(&mark_cv) != 0) {
2643 ABORT("pthread_cond_broadcast failed");
2644 }
2645 }
2646
2647 #endif /* PARALLEL_MARK */
2648
2649 #ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS
2650 /* Workaround "undefined reference" linkage errors on some targets. */
2651 EXTERN_C_BEGIN
2652 extern void __pthread_register_cancel(void) __attribute__((__weak__));
2653 extern void __pthread_unregister_cancel(void) __attribute__((__weak__));
2654 EXTERN_C_END
2655
2656 void __pthread_register_cancel(void) {}
2657 void __pthread_unregister_cancel(void) {}
2658 #endif
2659
2660 #endif /* GC_PTHREADS */