Debian Codegraph @

libc-top-half/musl/src/thread/pthread_create.c

raw
1 #define _GNU_SOURCE
2 #include "pthread_impl.h"
3 #include "stdio_impl.h"
4 #include "libc.h"
5 #include "lock.h"
6 #ifdef __wasilibc_unmodified_upstream
7 #include <sys/mman.h>
8 #endif
9 #include <string.h>
10 #include <stddef.h>
11 #ifndef __wasilibc_unmodified_upstream
12 #include <stdatomic.h>
13 #endif
14
15 #include <stdalign.h>
16 #include <assert.h>
17 #include <wasi/api.h>
18
19 static void dummy_0()
20 {
21 }
22 weak_alias(dummy_0, __acquire_ptc);
23 weak_alias(dummy_0, __release_ptc);
24 weak_alias(dummy_0, __pthread_tsd_run_dtors);
25 weak_alias(dummy_0, __do_orphaned_stdio_locks);
26 #ifdef __wasilibc_unmodified_upstream
27 weak_alias(dummy_0, __dl_thread_cleanup);
28 weak_alias(dummy_0, __membarrier_init);
29 #endif
30
31 static int tl_lock_count;
32 static int tl_lock_waiters;
33
34 void __tl_lock(void)
35 {
36 int tid = __pthread_self()->tid;
37 int val = __thread_list_lock;
38 if (val == tid) {
39 tl_lock_count++;
40 return;
41 }
42 while ((val = a_cas(&__thread_list_lock, 0, tid)))
43 __wait(&__thread_list_lock, &tl_lock_waiters, val, 0);
44 }
45
46 void __tl_unlock(void)
47 {
48 if (tl_lock_count) {
49 tl_lock_count--;
50 return;
51 }
52 a_store(&__thread_list_lock, 0);
53 if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0);
54 }
55
56 void __tl_sync(pthread_t td)
57 {
58 a_barrier();
59 int val = __thread_list_lock;
60 if (!val) return;
61 __wait(&__thread_list_lock, &tl_lock_waiters, val, 0);
62 if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0);
63 }
64
65 #ifndef __wasilibc_unmodified_upstream
66 static void *map_base_deferred_free;
67
68 static void process_map_base_deferred_free()
69 {
70 /* called with __tl_lock held */
71 free(map_base_deferred_free);
72 map_base_deferred_free = NULL;
73 }
74 #endif
75
76 #ifdef __wasilibc_unmodified_upstream
77 _Noreturn void __pthread_exit(void *result)
78 #else
79 static void __pthread_exit(void *result)
80 #endif
81 {
82 pthread_t self = __pthread_self();
83 sigset_t set;
84
85 self->canceldisable = 1;
86 self->cancelasync = 0;
87 self->result = result;
88
89 while (self->cancelbuf) {
90 void (*f)(void *) = self->cancelbuf->__f;
91 void *x = self->cancelbuf->__x;
92 self->cancelbuf = self->cancelbuf->__next;
93 f(x);
94 }
95
96 __pthread_tsd_run_dtors();
97
98 #ifdef __wasilibc_unmodified_upstream
99 __block_app_sigs(&set);
100 #endif
101
102 /* This atomic potentially competes with a concurrent pthread_detach
103 * call; the loser is responsible for freeing thread resources. */
104 int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING);
105
106 if (state==DT_DETACHED && self->map_base) {
107 /* Since __unmapself bypasses the normal munmap code path,
108 * explicitly wait for vmlock holders first. This must be
109 * done before any locks are taken, to avoid lock ordering
110 * issues that could lead to deadlock. */
111 #ifdef __wasilibc_unmodified_upstream
112 __vm_wait();
113 #endif
114 }
115
116 /* Access to target the exiting thread with syscalls that use
117 * its kernel tid is controlled by killlock. For detached threads,
118 * any use past this point would have undefined behavior, but for
119 * joinable threads it's a valid usage that must be handled.
120 * Signals must be blocked since pthread_kill must be AS-safe. */
121 LOCK(self->killlock);
122
123 /* The thread list lock must be AS-safe, and thus depends on
124 * application signals being blocked above. */
125 __tl_lock();
126
127 /* If this is the only thread in the list, don't proceed with
128 * termination of the thread, but restore the previous lock and
129 * signal state to prepare for exit to call atexit handlers. */
130 if (self->next == self) {
131 __tl_unlock();
132 UNLOCK(self->killlock);
133 self->detach_state = state;
134 #ifdef __wasilibc_unmodified_upstream
135 __restore_sigs(&set);
136 #endif
137 exit(0);
138 }
139
140 /* At this point we are committed to thread termination. */
141
142 /* After the kernel thread exits, its tid may be reused. Clear it
143 * to prevent inadvertent use and inform functions that would use
144 * it that it's no longer available. At this point the killlock
145 * may be released, since functions that use it will consistently
146 * see the thread as having exited. Release it now so that no
147 * remaining locks (except thread list) are held if we end up
148 * resetting need_locks below. */
149 self->tid = 0;
150 UNLOCK(self->killlock);
151
152 #ifdef __wasilibc_unmodified_upstream
153 /* Process robust list in userspace to handle non-pshared mutexes
154 * and the detached thread case where the robust list head will
155 * be invalid when the kernel would process it. */
156 __vm_lock();
157 #endif
158 volatile void *volatile *rp;
159 while ((rp=self->robust_list.head) && rp != &self->robust_list.head) {
160 pthread_mutex_t *m = (void *)((char *)rp
161 - offsetof(pthread_mutex_t, _m_next));
162 int waiters = m->_m_waiters;
163 int priv = (m->_m_type & 128) ^ 128;
164 self->robust_list.pending = rp;
165 self->robust_list.head = *rp;
166 int cont = a_swap(&m->_m_lock, 0x40000000);
167 self->robust_list.pending = 0;
168 if (cont < 0 || waiters)
169 __wake(&m->_m_lock, 1, priv);
170 }
171 #ifdef __wasilibc_unmodified_upstream
172 __vm_unlock();
173 #endif
174
175 __do_orphaned_stdio_locks();
176 #ifdef __wasilibc_unmodified_upstream
177 __dl_thread_cleanup();
178 #endif
179
180 /* Last, unlink thread from the list. This change will not be visible
181 * until the lock is released, which only happens after SYS_exit
182 * has been called, via the exit futex address pointing at the lock.
183 * This needs to happen after any possible calls to LOCK() that might
184 * skip locking if process appears single-threaded. */
185 if (!--libc.threads_minus_1) libc.need_locks = -1;
186 self->next->prev = self->prev;
187 self->prev->next = self->next;
188 self->prev = self->next = self;
189
190 #ifdef __wasilibc_unmodified_upstream
191 if (state==DT_DETACHED && self->map_base) {
192 /* Detached threads must block even implementation-internal
193 * signals, since they will not have a stack in their last
194 * moments of existence. */
195 __block_all_sigs(&set);
196
197 /* Robust list will no longer be valid, and was already
198 * processed above, so unregister it with the kernel. */
199 if (self->robust_list.off)
200 __syscall(SYS_set_robust_list, 0, 3*sizeof(long));
201
202 /* The following call unmaps the thread's stack mapping
203 * and then exits without touching the stack. */
204 __unmapself(self->map_base, self->map_size);
205 }
206 #else
207 if (state==DT_DETACHED && self->map_base) {
208 /* As we use malloc/free which is considerably more complex
209 * than mmap/munmap to call and can even require a valid
210 * thread context, it's difficult to implement __unmapself.
211 *
212 * Here we take an alternative approach which simply defers
213 * the deallocation. An obvious downside of this approach is
214 * that it keeps the stack longer. (possibly forever.)
215 * To avoid wasting too much memory, we only defer a single
216 * item at most. */
217 process_map_base_deferred_free();
218 map_base_deferred_free = self->map_base;
219 // Can't use `exit()` here, because it is too high level
220 return;
221 }
222 #endif
223
224 /* Wake any joiner. */
225 a_store(&self->detach_state, DT_EXITED);
226 __wake(&self->detach_state, 1, 1);
227
228 #ifdef __wasilibc_unmodified_upstream
229 for (;;) __syscall(SYS_exit, 0);
230 #else
231 // Can't use `exit()` here, because it is too high level
232
233 /* On Linux, the thread is created with CLONE_CHILD_CLEARTID,
234 * and the lock (__thread_list_lock) will be unlocked by kernel when
235 * this thread terminates.
236 * See also set_tid_address(2)
237 *
238 * In WebAssembly, we leave it to wasi_thread_start instead.
239 */
240 #endif
241 }
242
243 void __do_cleanup_push(struct __ptcb *cb)
244 {
245 struct pthread *self = __pthread_self();
246 cb->__next = self->cancelbuf;
247 self->cancelbuf = cb;
248 }
249
250 void __do_cleanup_pop(struct __ptcb *cb)
251 {
252 __pthread_self()->cancelbuf = cb->__next;
253 }
254
255 struct start_args {
256 #ifdef __wasilibc_unmodified_upstream
257 void *(*start_func)(void *);
258 void *start_arg;
259 volatile int control;
260 unsigned long sig_mask[_NSIG/8/sizeof(long)];
261 #else
262 /*
263 * Note: the offset of the "stack" and "tls_base" members
264 * in this structure is hardcoded in wasi_thread_start.
265 */
266 void *stack;
267 void *tls_base;
268 void *(*start_func)(void *);
269 void *start_arg;
270 #endif
271 };
272
273 #ifdef __wasilibc_unmodified_upstream
274 static int start(void *p)
275 {
276 struct start_args *args = p;
277 int state = args->control;
278 if (state) {
279 if (a_cas(&args->control, 1, 2)==1)
280 __wait(&args->control, 0, 2, 1);
281 if (args->control) {
282 #ifdef __wasilibc_unmodified_upstream
283 __syscall(SYS_set_tid_address, &args->control);
284 for (;;) __syscall(SYS_exit, 0);
285 #endif
286 }
287 }
288 #ifdef __wasilibc_unmodified_upstream
289 __syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->sig_mask, 0, _NSIG/8);
290 #endif
291 __pthread_exit(args->start_func(args->start_arg));
292 return 0;
293 }
294
295 static int start_c11(void *p)
296 {
297 struct start_args *args = p;
298 int (*start)(void*) = (int(*)(void*)) args->start_func;
299 __pthread_exit((void *)(uintptr_t)start(args->start_arg));
300 return 0;
301 }
302 #else
303
304 /*
305 * We want to ensure wasi_thread_start is linked whenever
306 * pthread_create is used. The following reference is to ensure that.
307 * Otherwise, the linker doesn't notice the dependency because
308 * wasi_thread_start is used indirectly via a wasm export.
309 */
310 void wasi_thread_start(int tid, void *p);
311 hidden void *__dummy_reference = wasi_thread_start;
312
313 hidden void __wasi_thread_start_C(int tid, void *p)
314 {
315 struct start_args *args = p;
316 pthread_t self = __pthread_self();
317 // Set the thread ID (TID) on the pthread structure. The TID is stored
318 // atomically since it is also stored by the parent thread; this way,
319 // whichever thread (parent or child) reaches this point first can proceed
320 // without waiting.
321 atomic_store((atomic_int *) &(self->tid), tid);
322 // Execute the user's start function.
323 __pthread_exit(args->start_func(args->start_arg));
324 }
325 #endif
326
327 #ifdef __wasilibc_unmodified_upstream
328 #define ROUND(x) (((x)+PAGE_SIZE-1)&-PAGE_SIZE)
329 #else
330 /*
331 * As we allocate stack with malloc() instead of mmap/mprotect,
332 * there is no point to round it up to PAGE_SIZE.
333 * Instead, round up to a sane alignment.
334 * Note: PAGE_SIZE is rather big on WASM. (65536)
335 */
336 #define ROUND(x) (((x)+16-1)&-16)
337 #endif
338
339 /* pthread_key_create.c overrides this */
340 static volatile size_t dummy = 0;
341 weak_alias(dummy, __pthread_tsd_size);
342 static void *dummy_tsd[1] = { 0 };
343 weak_alias(dummy_tsd, __pthread_tsd_main);
344
345 static FILE *volatile dummy_file = 0;
346 weak_alias(dummy_file, __stdin_used);
347 weak_alias(dummy_file, __stdout_used);
348 weak_alias(dummy_file, __stderr_used);
349
350 static void init_file_lock(FILE *f)
351 {
352 if (f && f->lock<0) f->lock = 0;
353 }
354
355 int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp, void *(*entry)(void *), void *restrict arg)
356 {
357 int ret, c11 = (attrp == __ATTRP_C11_THREAD);
358 size_t size, guard;
359 struct pthread *self, *new;
360 unsigned char *map = 0, *stack = 0, *tsd = 0, *stack_limit;
361 #ifdef __wasilibc_unmodified_upstream
362 unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND
363 | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS
364 | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED;
365 #endif
366 pthread_attr_t attr = { 0 };
367 sigset_t set;
368 #ifndef __wasilibc_unmodified_upstream
369 size_t tls_size = __builtin_wasm_tls_size();
370 size_t tls_align = __builtin_wasm_tls_align();
371 void* tls_base = __builtin_wasm_tls_base();
372 void* new_tls_base;
373 size_t tls_offset;
374 tls_size += tls_align;
375 #endif
376
377 #ifdef __wasilibc_unmodified_upstream
378 if (!libc.can_do_threads) return ENOSYS;
379 #endif
380 self = __pthread_self();
381 if (!libc.threaded) {
382 for (FILE *f=*__ofl_lock(); f; f=f->next)
383 init_file_lock(f);
384 __ofl_unlock();
385 init_file_lock(__stdin_used);
386 init_file_lock(__stdout_used);
387 init_file_lock(__stderr_used);
388 #ifdef __wasilibc_unmodified_upstream
389 __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8);
390 #endif
391 self->tsd = (void **)__pthread_tsd_main;
392 #ifdef __wasilibc_unmodified_upstream
393 __membarrier_init();
394 #endif
395 libc.threaded = 1;
396 }
397 if (attrp && !c11) attr = *attrp;
398
399 __acquire_ptc();
400 if (!attrp || c11) {
401 attr._a_stacksize = __default_stacksize;
402 attr._a_guardsize = __default_guardsize;
403 }
404
405 if (attr._a_stackaddr) {
406 #ifdef __wasilibc_unmodified_upstream
407 size_t need = libc.tls_size + __pthread_tsd_size;
408 #else
409 size_t need = tls_size + __pthread_tsd_size;
410 #endif
411 size = attr._a_stacksize;
412 stack = (void *)(attr._a_stackaddr & -16);
413 stack_limit = (void *)(attr._a_stackaddr - size);
414 /* Use application-provided stack for TLS only when
415 * it does not take more than ~12% or 2k of the
416 * application's stack space. */
417 if (need < size/8 && need < 2048) {
418 tsd = stack - __pthread_tsd_size;
419 #ifdef __wasilibc_unmodified_upstream
420 stack = tsd - libc.tls_size;
421 #else
422 stack = tsd - tls_size;
423 #endif
424 memset(stack, 0, need);
425 } else {
426 size = ROUND(need);
427 }
428 guard = 0;
429 } else {
430 guard = ROUND(attr._a_guardsize);
431 size = guard + ROUND(attr._a_stacksize
432 #ifdef __wasilibc_unmodified_upstream
433 + libc.tls_size + __pthread_tsd_size);
434 #else
435 + tls_size + __pthread_tsd_size);
436 #endif
437 }
438
439 if (!tsd) {
440 #ifdef __wasilibc_unmodified_upstream
441 if (guard) {
442 map = __mmap(0, size, PROT_NONE, MAP_PRIVATE|MAP_ANON, -1, 0);
443 if (map == MAP_FAILED) goto fail;
444 if (__mprotect(map+guard, size-guard, PROT_READ|PROT_WRITE)
445 && errno != ENOSYS) {
446 __munmap(map, size);
447 goto fail;
448 }
449 } else {
450 map = __mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
451 if (map == MAP_FAILED) goto fail;
452 }
453 #else
454 /* Process the deferred free request if any before
455 * allocationg a new one. Hopefully it enables a reuse of the memory.
456 *
457 * Note: We can't perform a simple "handoff" becasue allocation
458 * sizes might be different. (eg. the stack size might differ) */
459 __tl_lock();
460 process_map_base_deferred_free();
461 __tl_unlock();
462 map = malloc(size);
463 if (!map) goto fail;
464 #endif
465 tsd = map + size - __pthread_tsd_size;
466 #ifndef __wasilibc_unmodified_upstream
467 memset(tsd, 0, __pthread_tsd_size);
468 #endif
469 if (!stack) {
470 #ifdef __wasilibc_unmodified_upstream
471 stack = tsd - libc.tls_size;
472 #else
473 stack = tsd - tls_size;
474 #endif
475 stack_limit = map + guard;
476 }
477 }
478
479 #ifdef __wasilibc_unmodified_upstream
480 new = __copy_tls(tsd - libc.tls_size);
481 #else
482 new_tls_base = __copy_tls(tsd - tls_size);
483 tls_offset = new_tls_base - tls_base;
484 new = (void*)((uintptr_t)self + tls_offset);
485 #endif
486 new->map_base = map;
487 new->map_size = size;
488 new->stack = stack;
489 new->stack_size = stack - stack_limit;
490 new->guard_size = guard;
491 new->self = new;
492 new->tsd = (void *)tsd;
493 new->locale = &libc.global_locale;
494 if (attr._a_detach) {
495 new->detach_state = DT_DETACHED;
496 } else {
497 new->detach_state = DT_JOINABLE;
498 }
499 new->robust_list.head = &new->robust_list.head;
500 new->canary = self->canary;
501 new->sysinfo = self->sysinfo;
502
503 /* Setup argument structure for the new thread on its stack.
504 * It's safe to access from the caller only until the thread
505 * list is unlocked. */
506 #ifdef __wasilibc_unmodified_upstream
507 stack -= (uintptr_t)stack % sizeof(uintptr_t);
508 stack -= sizeof(struct start_args);
509 struct start_args *args = (void *)stack;
510 args->start_func = entry;
511 args->start_arg = arg;
512 args->control = attr._a_sched ? 1 : 0;
513
514 /* Application signals (but not the synccall signal) must be
515 * blocked before the thread list lock can be taken, to ensure
516 * that the lock is AS-safe. */
517 __block_app_sigs(&set);
518
519 /* Ensure SIGCANCEL is unblocked in new thread. This requires
520 * working with a copy of the set so we can restore the
521 * original mask in the calling thread. */
522 memcpy(&args->sig_mask, &set, sizeof args->sig_mask);
523 args->sig_mask[(SIGCANCEL-1)/8/sizeof(long)] &=
524 ~(1UL<<((SIGCANCEL-1)%(8*sizeof(long))));
525 #else
526 /* Align the stack to struct start_args */
527 stack -= sizeof(struct start_args);
528 stack -= (uintptr_t)stack % alignof(struct start_args);
529 struct start_args *args = (void *)stack;
530
531 /* Align the stack to 16 and store it */
532 new->stack = (void *)((uintptr_t) stack & -16);
533 /* Correct the stack size */
534 new->stack_size = stack - stack_limit;
535
536 args->stack = new->stack; /* just for convenience of asm trampoline */
537 args->start_func = entry;
538 args->start_arg = arg;
539 args->tls_base = (void*)new_tls_base;
540 #endif
541
542 __tl_lock();
543 if (!libc.threads_minus_1++) libc.need_locks = 1;
544 #ifdef __wasilibc_unmodified_upstream
545 ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock);
546 #else
547 /* Instead of `__clone`, WASI uses a host API to instantiate a new version
548 * of the current module and start executing the entry function. The
549 * wasi-threads specification requires the module to export a
550 * `wasi_thread_start` function, which is invoked with `args`. */
551 ret = __wasi_thread_spawn((void *) args);
552 #endif
553
554 #ifdef __wasilibc_unmodified_upstream
555 /* All clone failures translate to EAGAIN. If explicit scheduling
556 * was requested, attempt it before unlocking the thread list so
557 * that the failed thread is never exposed and so that we can
558 * clean up all transient resource usage before returning. */
559 if (ret < 0) {
560 ret = -EAGAIN;
561 } else if (attr._a_sched) {
562 ret = __syscall(SYS_sched_setscheduler,
563 new->tid, attr._a_policy, &attr._a_prio);
564 if (a_swap(&args->control, ret ? 3 : 0)==2)
565 __wake(&args->control, 1, 1);
566 if (ret)
567 __wait(&args->control, 0, 3, 0);
568 }
569 #else
570 #define WASI_THREADS_MAX_TID 0x1FFFFFFF
571 /* `wasi_thread_spawn` will either return a host-provided thread ID (TID)
572 * (`<1, 0x1FFFFFFF>`) or an error code (`< 0`). Please note that `0` is
573 * reserved for compatibility reasons and must not be returned by the runtime.
574 * As in the unmodified version, all spawn failures translate to EAGAIN;
575 * unlike the modified version, there is no need to "start up" the child
576 * thread--the host does this. If the spawn did succeed, then we store the
577 * TID atomically, since this parent thread is racing with the child thread
578 * to set this field; this way, whichever thread reaches this point first
579 * can continue without waiting. */
580 assert(ret != 0 && ret <= WASI_THREADS_MAX_TID);
581 if (ret < 0) {
582 ret = -EAGAIN;
583 } else {
584 atomic_store((atomic_int *) &(new->tid), ret);
585 }
586 #endif
587
588 if (ret >= 0) {
589 new->next = self->next;
590 new->prev = self;
591 new->next->prev = new;
592 new->prev->next = new;
593 } else {
594 if (!--libc.threads_minus_1) libc.need_locks = 0;
595 }
596 __tl_unlock();
597 #ifdef __wasilibc_unmodified_upstream
598 __restore_sigs(&set);
599 #endif
600 __release_ptc();
601
602 if (ret < 0) {
603 #ifdef __wasilibc_unmodified_upstream
604 if (map) __munmap(map, size);
605 #else
606 free(map);
607 #endif
608 return -ret;
609 }
610
611 *res = new;
612 return 0;
613 fail:
614 __release_ptc();
615 return EAGAIN;
616 }
617
618 #ifdef __wasilibc_unmodified_upstream
619 weak_alias(__pthread_exit, pthread_exit);
620 #endif
621 weak_alias(__pthread_create, pthread_create);