| 1 |
#define _GNU_SOURCE |
| 2 |
#include "pthread_impl.h" |
| 3 |
#include "stdio_impl.h" |
| 4 |
#include "libc.h" |
| 5 |
#include "lock.h" |
| 6 |
#ifdef __wasilibc_unmodified_upstream |
| 7 |
#include <sys/mman.h> |
| 8 |
#endif |
| 9 |
#include <string.h> |
| 10 |
#include <stddef.h> |
| 11 |
#ifndef __wasilibc_unmodified_upstream |
| 12 |
#include <stdatomic.h> |
| 13 |
#endif |
| 14 |
|
| 15 |
#include <stdalign.h> |
| 16 |
#include <assert.h> |
| 17 |
#include <wasi/api.h> |
| 18 |
|
| 19 |
static void dummy_0() |
| 20 |
{ |
| 21 |
} |
| 22 |
weak_alias(dummy_0, __acquire_ptc); |
| 23 |
weak_alias(dummy_0, __release_ptc); |
| 24 |
weak_alias(dummy_0, __pthread_tsd_run_dtors); |
| 25 |
weak_alias(dummy_0, __do_orphaned_stdio_locks); |
| 26 |
#ifdef __wasilibc_unmodified_upstream |
| 27 |
weak_alias(dummy_0, __dl_thread_cleanup); |
| 28 |
weak_alias(dummy_0, __membarrier_init); |
| 29 |
#endif |
| 30 |
|
| 31 |
static int tl_lock_count; |
| 32 |
static int tl_lock_waiters; |
| 33 |
|
| 34 |
void __tl_lock(void) |
| 35 |
{ |
| 36 |
int tid = __pthread_self()->tid; |
| 37 |
int val = __thread_list_lock; |
| 38 |
if (val == tid) { |
| 39 |
tl_lock_count++; |
| 40 |
return; |
| 41 |
} |
| 42 |
while ((val = a_cas(&__thread_list_lock, 0, tid))) |
| 43 |
__wait(&__thread_list_lock, &tl_lock_waiters, val, 0); |
| 44 |
} |
| 45 |
|
| 46 |
void __tl_unlock(void) |
| 47 |
{ |
| 48 |
if (tl_lock_count) { |
| 49 |
tl_lock_count--; |
| 50 |
return; |
| 51 |
} |
| 52 |
a_store(&__thread_list_lock, 0); |
| 53 |
if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0); |
| 54 |
} |
| 55 |
|
| 56 |
void __tl_sync(pthread_t td) |
| 57 |
{ |
| 58 |
a_barrier(); |
| 59 |
int val = __thread_list_lock; |
| 60 |
if (!val) return; |
| 61 |
__wait(&__thread_list_lock, &tl_lock_waiters, val, 0); |
| 62 |
if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0); |
| 63 |
} |
| 64 |
|
| 65 |
#ifndef __wasilibc_unmodified_upstream |
| 66 |
static void *map_base_deferred_free; |
| 67 |
|
| 68 |
static void process_map_base_deferred_free() |
| 69 |
{ |
| 70 |
/* called with __tl_lock held */ |
| 71 |
free(map_base_deferred_free); |
| 72 |
map_base_deferred_free = NULL; |
| 73 |
} |
| 74 |
#endif |
| 75 |
|
| 76 |
#ifdef __wasilibc_unmodified_upstream |
| 77 |
_Noreturn void __pthread_exit(void *result) |
| 78 |
#else |
| 79 |
static void __pthread_exit(void *result) |
| 80 |
#endif |
| 81 |
{ |
| 82 |
pthread_t self = __pthread_self(); |
| 83 |
sigset_t set; |
| 84 |
|
| 85 |
self->canceldisable = 1; |
| 86 |
self->cancelasync = 0; |
| 87 |
self->result = result; |
| 88 |
|
| 89 |
while (self->cancelbuf) { |
| 90 |
void (*f)(void *) = self->cancelbuf->__f; |
| 91 |
void *x = self->cancelbuf->__x; |
| 92 |
self->cancelbuf = self->cancelbuf->__next; |
| 93 |
f(x); |
| 94 |
} |
| 95 |
|
| 96 |
__pthread_tsd_run_dtors(); |
| 97 |
|
| 98 |
#ifdef __wasilibc_unmodified_upstream |
| 99 |
__block_app_sigs(&set); |
| 100 |
#endif |
| 101 |
|
| 102 |
/* This atomic potentially competes with a concurrent pthread_detach |
| 103 |
* call; the loser is responsible for freeing thread resources. */ |
| 104 |
int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); |
| 105 |
|
| 106 |
if (state==DT_DETACHED && self->map_base) { |
| 107 |
/* Since __unmapself bypasses the normal munmap code path, |
| 108 |
* explicitly wait for vmlock holders first. This must be |
| 109 |
* done before any locks are taken, to avoid lock ordering |
| 110 |
* issues that could lead to deadlock. */ |
| 111 |
#ifdef __wasilibc_unmodified_upstream |
| 112 |
__vm_wait(); |
| 113 |
#endif |
| 114 |
} |
| 115 |
|
| 116 |
/* Access to target the exiting thread with syscalls that use |
| 117 |
* its kernel tid is controlled by killlock. For detached threads, |
| 118 |
* any use past this point would have undefined behavior, but for |
| 119 |
* joinable threads it's a valid usage that must be handled. |
| 120 |
* Signals must be blocked since pthread_kill must be AS-safe. */ |
| 121 |
LOCK(self->killlock); |
| 122 |
|
| 123 |
/* The thread list lock must be AS-safe, and thus depends on |
| 124 |
* application signals being blocked above. */ |
| 125 |
__tl_lock(); |
| 126 |
|
| 127 |
/* If this is the only thread in the list, don't proceed with |
| 128 |
* termination of the thread, but restore the previous lock and |
| 129 |
* signal state to prepare for exit to call atexit handlers. */ |
| 130 |
if (self->next == self) { |
| 131 |
__tl_unlock(); |
| 132 |
UNLOCK(self->killlock); |
| 133 |
self->detach_state = state; |
| 134 |
#ifdef __wasilibc_unmodified_upstream |
| 135 |
__restore_sigs(&set); |
| 136 |
#endif |
| 137 |
exit(0); |
| 138 |
} |
| 139 |
|
| 140 |
/* At this point we are committed to thread termination. */ |
| 141 |
|
| 142 |
/* After the kernel thread exits, its tid may be reused. Clear it |
| 143 |
* to prevent inadvertent use and inform functions that would use |
| 144 |
* it that it's no longer available. At this point the killlock |
| 145 |
* may be released, since functions that use it will consistently |
| 146 |
* see the thread as having exited. Release it now so that no |
| 147 |
* remaining locks (except thread list) are held if we end up |
| 148 |
* resetting need_locks below. */ |
| 149 |
self->tid = 0; |
| 150 |
UNLOCK(self->killlock); |
| 151 |
|
| 152 |
#ifdef __wasilibc_unmodified_upstream |
| 153 |
/* Process robust list in userspace to handle non-pshared mutexes |
| 154 |
* and the detached thread case where the robust list head will |
| 155 |
* be invalid when the kernel would process it. */ |
| 156 |
__vm_lock(); |
| 157 |
#endif |
| 158 |
volatile void *volatile *rp; |
| 159 |
while ((rp=self->robust_list.head) && rp != &self->robust_list.head) { |
| 160 |
pthread_mutex_t *m = (void *)((char *)rp |
| 161 |
- offsetof(pthread_mutex_t, _m_next)); |
| 162 |
int waiters = m->_m_waiters; |
| 163 |
int priv = (m->_m_type & 128) ^ 128; |
| 164 |
self->robust_list.pending = rp; |
| 165 |
self->robust_list.head = *rp; |
| 166 |
int cont = a_swap(&m->_m_lock, 0x40000000); |
| 167 |
self->robust_list.pending = 0; |
| 168 |
if (cont < 0 || waiters) |
| 169 |
__wake(&m->_m_lock, 1, priv); |
| 170 |
} |
| 171 |
#ifdef __wasilibc_unmodified_upstream |
| 172 |
__vm_unlock(); |
| 173 |
#endif |
| 174 |
|
| 175 |
__do_orphaned_stdio_locks(); |
| 176 |
#ifdef __wasilibc_unmodified_upstream |
| 177 |
__dl_thread_cleanup(); |
| 178 |
#endif |
| 179 |
|
| 180 |
/* Last, unlink thread from the list. This change will not be visible |
| 181 |
* until the lock is released, which only happens after SYS_exit |
| 182 |
* has been called, via the exit futex address pointing at the lock. |
| 183 |
* This needs to happen after any possible calls to LOCK() that might |
| 184 |
* skip locking if process appears single-threaded. */ |
| 185 |
if (!--libc.threads_minus_1) libc.need_locks = -1; |
| 186 |
self->next->prev = self->prev; |
| 187 |
self->prev->next = self->next; |
| 188 |
self->prev = self->next = self; |
| 189 |
|
| 190 |
#ifdef __wasilibc_unmodified_upstream |
| 191 |
if (state==DT_DETACHED && self->map_base) { |
| 192 |
/* Detached threads must block even implementation-internal |
| 193 |
* signals, since they will not have a stack in their last |
| 194 |
* moments of existence. */ |
| 195 |
__block_all_sigs(&set); |
| 196 |
|
| 197 |
/* Robust list will no longer be valid, and was already |
| 198 |
* processed above, so unregister it with the kernel. */ |
| 199 |
if (self->robust_list.off) |
| 200 |
__syscall(SYS_set_robust_list, 0, 3*sizeof(long)); |
| 201 |
|
| 202 |
/* The following call unmaps the thread's stack mapping |
| 203 |
* and then exits without touching the stack. */ |
| 204 |
__unmapself(self->map_base, self->map_size); |
| 205 |
} |
| 206 |
#else |
| 207 |
if (state==DT_DETACHED && self->map_base) { |
| 208 |
/* As we use malloc/free which is considerably more complex |
| 209 |
* than mmap/munmap to call and can even require a valid |
| 210 |
* thread context, it's difficult to implement __unmapself. |
| 211 |
* |
| 212 |
* Here we take an alternative approach which simply defers |
| 213 |
* the deallocation. An obvious downside of this approach is |
| 214 |
* that it keeps the stack longer. (possibly forever.) |
| 215 |
* To avoid wasting too much memory, we only defer a single |
| 216 |
* item at most. */ |
| 217 |
process_map_base_deferred_free(); |
| 218 |
map_base_deferred_free = self->map_base; |
| 219 |
// Can't use `exit()` here, because it is too high level |
| 220 |
return; |
| 221 |
} |
| 222 |
#endif |
| 223 |
|
| 224 |
/* Wake any joiner. */ |
| 225 |
a_store(&self->detach_state, DT_EXITED); |
| 226 |
__wake(&self->detach_state, 1, 1); |
| 227 |
|
| 228 |
#ifdef __wasilibc_unmodified_upstream |
| 229 |
for (;;) __syscall(SYS_exit, 0); |
| 230 |
#else |
| 231 |
// Can't use `exit()` here, because it is too high level |
| 232 |
|
| 233 |
/* On Linux, the thread is created with CLONE_CHILD_CLEARTID, |
| 234 |
* and the lock (__thread_list_lock) will be unlocked by kernel when |
| 235 |
* this thread terminates. |
| 236 |
* See also set_tid_address(2) |
| 237 |
* |
| 238 |
* In WebAssembly, we leave it to wasi_thread_start instead. |
| 239 |
*/ |
| 240 |
#endif |
| 241 |
} |
| 242 |
|
| 243 |
void __do_cleanup_push(struct __ptcb *cb) |
| 244 |
{ |
| 245 |
struct pthread *self = __pthread_self(); |
| 246 |
cb->__next = self->cancelbuf; |
| 247 |
self->cancelbuf = cb; |
| 248 |
} |
| 249 |
|
| 250 |
void __do_cleanup_pop(struct __ptcb *cb) |
| 251 |
{ |
| 252 |
__pthread_self()->cancelbuf = cb->__next; |
| 253 |
} |
| 254 |
|
| 255 |
struct start_args { |
| 256 |
#ifdef __wasilibc_unmodified_upstream |
| 257 |
void *(*start_func)(void *); |
| 258 |
void *start_arg; |
| 259 |
volatile int control; |
| 260 |
unsigned long sig_mask[_NSIG/8/sizeof(long)]; |
| 261 |
#else |
| 262 |
/* |
| 263 |
* Note: the offset of the "stack" and "tls_base" members |
| 264 |
* in this structure is hardcoded in wasi_thread_start. |
| 265 |
*/ |
| 266 |
void *stack; |
| 267 |
void *tls_base; |
| 268 |
void *(*start_func)(void *); |
| 269 |
void *start_arg; |
| 270 |
#endif |
| 271 |
}; |
| 272 |
|
| 273 |
#ifdef __wasilibc_unmodified_upstream |
| 274 |
static int start(void *p) |
| 275 |
{ |
| 276 |
struct start_args *args = p; |
| 277 |
int state = args->control; |
| 278 |
if (state) { |
| 279 |
if (a_cas(&args->control, 1, 2)==1) |
| 280 |
__wait(&args->control, 0, 2, 1); |
| 281 |
if (args->control) { |
| 282 |
#ifdef __wasilibc_unmodified_upstream |
| 283 |
__syscall(SYS_set_tid_address, &args->control); |
| 284 |
for (;;) __syscall(SYS_exit, 0); |
| 285 |
#endif |
| 286 |
} |
| 287 |
} |
| 288 |
#ifdef __wasilibc_unmodified_upstream |
| 289 |
__syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->sig_mask, 0, _NSIG/8); |
| 290 |
#endif |
| 291 |
__pthread_exit(args->start_func(args->start_arg)); |
| 292 |
return 0; |
| 293 |
} |
| 294 |
|
| 295 |
static int start_c11(void *p) |
| 296 |
{ |
| 297 |
struct start_args *args = p; |
| 298 |
int (*start)(void*) = (int(*)(void*)) args->start_func; |
| 299 |
__pthread_exit((void *)(uintptr_t)start(args->start_arg)); |
| 300 |
return 0; |
| 301 |
} |
| 302 |
#else |
| 303 |
|
| 304 |
/* |
| 305 |
* We want to ensure wasi_thread_start is linked whenever |
| 306 |
* pthread_create is used. The following reference is to ensure that. |
| 307 |
* Otherwise, the linker doesn't notice the dependency because |
| 308 |
* wasi_thread_start is used indirectly via a wasm export. |
| 309 |
*/ |
| 310 |
void wasi_thread_start(int tid, void *p); |
| 311 |
hidden void *__dummy_reference = wasi_thread_start; |
| 312 |
|
| 313 |
hidden void __wasi_thread_start_C(int tid, void *p) |
| 314 |
{ |
| 315 |
struct start_args *args = p; |
| 316 |
pthread_t self = __pthread_self(); |
| 317 |
// Set the thread ID (TID) on the pthread structure. The TID is stored |
| 318 |
// atomically since it is also stored by the parent thread; this way, |
| 319 |
// whichever thread (parent or child) reaches this point first can proceed |
| 320 |
// without waiting. |
| 321 |
atomic_store((atomic_int *) &(self->tid), tid); |
| 322 |
// Execute the user's start function. |
| 323 |
__pthread_exit(args->start_func(args->start_arg)); |
| 324 |
} |
| 325 |
#endif |
| 326 |
|
| 327 |
#ifdef __wasilibc_unmodified_upstream |
| 328 |
#define ROUND(x) (((x)+PAGE_SIZE-1)&-PAGE_SIZE) |
| 329 |
#else |
| 330 |
/* |
| 331 |
* As we allocate stack with malloc() instead of mmap/mprotect, |
| 332 |
* there is no point to round it up to PAGE_SIZE. |
| 333 |
* Instead, round up to a sane alignment. |
| 334 |
* Note: PAGE_SIZE is rather big on WASM. (65536) |
| 335 |
*/ |
| 336 |
#define ROUND(x) (((x)+16-1)&-16) |
| 337 |
#endif |
| 338 |
|
| 339 |
/* pthread_key_create.c overrides this */ |
| 340 |
static volatile size_t dummy = 0; |
| 341 |
weak_alias(dummy, __pthread_tsd_size); |
| 342 |
static void *dummy_tsd[1] = { 0 }; |
| 343 |
weak_alias(dummy_tsd, __pthread_tsd_main); |
| 344 |
|
| 345 |
static FILE *volatile dummy_file = 0; |
| 346 |
weak_alias(dummy_file, __stdin_used); |
| 347 |
weak_alias(dummy_file, __stdout_used); |
| 348 |
weak_alias(dummy_file, __stderr_used); |
| 349 |
|
| 350 |
static void init_file_lock(FILE *f) |
| 351 |
{ |
| 352 |
if (f && f->lock<0) f->lock = 0; |
| 353 |
} |
| 354 |
|
| 355 |
int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp, void *(*entry)(void *), void *restrict arg) |
| 356 |
{ |
| 357 |
int ret, c11 = (attrp == __ATTRP_C11_THREAD); |
| 358 |
size_t size, guard; |
| 359 |
struct pthread *self, *new; |
| 360 |
unsigned char *map = 0, *stack = 0, *tsd = 0, *stack_limit; |
| 361 |
#ifdef __wasilibc_unmodified_upstream |
| 362 |
unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND |
| 363 |
| CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS |
| 364 |
| CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED; |
| 365 |
#endif |
| 366 |
pthread_attr_t attr = { 0 }; |
| 367 |
sigset_t set; |
| 368 |
#ifndef __wasilibc_unmodified_upstream |
| 369 |
size_t tls_size = __builtin_wasm_tls_size(); |
| 370 |
size_t tls_align = __builtin_wasm_tls_align(); |
| 371 |
void* tls_base = __builtin_wasm_tls_base(); |
| 372 |
void* new_tls_base; |
| 373 |
size_t tls_offset; |
| 374 |
tls_size += tls_align; |
| 375 |
#endif |
| 376 |
|
| 377 |
#ifdef __wasilibc_unmodified_upstream |
| 378 |
if (!libc.can_do_threads) return ENOSYS; |
| 379 |
#endif |
| 380 |
self = __pthread_self(); |
| 381 |
if (!libc.threaded) { |
| 382 |
for (FILE *f=*__ofl_lock(); f; f=f->next) |
| 383 |
init_file_lock(f); |
| 384 |
__ofl_unlock(); |
| 385 |
init_file_lock(__stdin_used); |
| 386 |
init_file_lock(__stdout_used); |
| 387 |
init_file_lock(__stderr_used); |
| 388 |
#ifdef __wasilibc_unmodified_upstream |
| 389 |
__syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8); |
| 390 |
#endif |
| 391 |
self->tsd = (void **)__pthread_tsd_main; |
| 392 |
#ifdef __wasilibc_unmodified_upstream |
| 393 |
__membarrier_init(); |
| 394 |
#endif |
| 395 |
libc.threaded = 1; |
| 396 |
} |
| 397 |
if (attrp && !c11) attr = *attrp; |
| 398 |
|
| 399 |
__acquire_ptc(); |
| 400 |
if (!attrp || c11) { |
| 401 |
attr._a_stacksize = __default_stacksize; |
| 402 |
attr._a_guardsize = __default_guardsize; |
| 403 |
} |
| 404 |
|
| 405 |
if (attr._a_stackaddr) { |
| 406 |
#ifdef __wasilibc_unmodified_upstream |
| 407 |
size_t need = libc.tls_size + __pthread_tsd_size; |
| 408 |
#else |
| 409 |
size_t need = tls_size + __pthread_tsd_size; |
| 410 |
#endif |
| 411 |
size = attr._a_stacksize; |
| 412 |
stack = (void *)(attr._a_stackaddr & -16); |
| 413 |
stack_limit = (void *)(attr._a_stackaddr - size); |
| 414 |
/* Use application-provided stack for TLS only when |
| 415 |
* it does not take more than ~12% or 2k of the |
| 416 |
* application's stack space. */ |
| 417 |
if (need < size/8 && need < 2048) { |
| 418 |
tsd = stack - __pthread_tsd_size; |
| 419 |
#ifdef __wasilibc_unmodified_upstream |
| 420 |
stack = tsd - libc.tls_size; |
| 421 |
#else |
| 422 |
stack = tsd - tls_size; |
| 423 |
#endif |
| 424 |
memset(stack, 0, need); |
| 425 |
} else { |
| 426 |
size = ROUND(need); |
| 427 |
} |
| 428 |
guard = 0; |
| 429 |
} else { |
| 430 |
guard = ROUND(attr._a_guardsize); |
| 431 |
size = guard + ROUND(attr._a_stacksize |
| 432 |
#ifdef __wasilibc_unmodified_upstream |
| 433 |
+ libc.tls_size + __pthread_tsd_size); |
| 434 |
#else |
| 435 |
+ tls_size + __pthread_tsd_size); |
| 436 |
#endif |
| 437 |
} |
| 438 |
|
| 439 |
if (!tsd) { |
| 440 |
#ifdef __wasilibc_unmodified_upstream |
| 441 |
if (guard) { |
| 442 |
map = __mmap(0, size, PROT_NONE, MAP_PRIVATE|MAP_ANON, -1, 0); |
| 443 |
if (map == MAP_FAILED) goto fail; |
| 444 |
if (__mprotect(map+guard, size-guard, PROT_READ|PROT_WRITE) |
| 445 |
&& errno != ENOSYS) { |
| 446 |
__munmap(map, size); |
| 447 |
goto fail; |
| 448 |
} |
| 449 |
} else { |
| 450 |
map = __mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); |
| 451 |
if (map == MAP_FAILED) goto fail; |
| 452 |
} |
| 453 |
#else |
| 454 |
/* Process the deferred free request if any before |
| 455 |
* allocationg a new one. Hopefully it enables a reuse of the memory. |
| 456 |
* |
| 457 |
* Note: We can't perform a simple "handoff" becasue allocation |
| 458 |
* sizes might be different. (eg. the stack size might differ) */ |
| 459 |
__tl_lock(); |
| 460 |
process_map_base_deferred_free(); |
| 461 |
__tl_unlock(); |
| 462 |
map = malloc(size); |
| 463 |
if (!map) goto fail; |
| 464 |
#endif |
| 465 |
tsd = map + size - __pthread_tsd_size; |
| 466 |
#ifndef __wasilibc_unmodified_upstream |
| 467 |
memset(tsd, 0, __pthread_tsd_size); |
| 468 |
#endif |
| 469 |
if (!stack) { |
| 470 |
#ifdef __wasilibc_unmodified_upstream |
| 471 |
stack = tsd - libc.tls_size; |
| 472 |
#else |
| 473 |
stack = tsd - tls_size; |
| 474 |
#endif |
| 475 |
stack_limit = map + guard; |
| 476 |
} |
| 477 |
} |
| 478 |
|
| 479 |
#ifdef __wasilibc_unmodified_upstream |
| 480 |
new = __copy_tls(tsd - libc.tls_size); |
| 481 |
#else |
| 482 |
new_tls_base = __copy_tls(tsd - tls_size); |
| 483 |
tls_offset = new_tls_base - tls_base; |
| 484 |
new = (void*)((uintptr_t)self + tls_offset); |
| 485 |
#endif |
| 486 |
new->map_base = map; |
| 487 |
new->map_size = size; |
| 488 |
new->stack = stack; |
| 489 |
new->stack_size = stack - stack_limit; |
| 490 |
new->guard_size = guard; |
| 491 |
new->self = new; |
| 492 |
new->tsd = (void *)tsd; |
| 493 |
new->locale = &libc.global_locale; |
| 494 |
if (attr._a_detach) { |
| 495 |
new->detach_state = DT_DETACHED; |
| 496 |
} else { |
| 497 |
new->detach_state = DT_JOINABLE; |
| 498 |
} |
| 499 |
new->robust_list.head = &new->robust_list.head; |
| 500 |
new->canary = self->canary; |
| 501 |
new->sysinfo = self->sysinfo; |
| 502 |
|
| 503 |
/* Setup argument structure for the new thread on its stack. |
| 504 |
* It's safe to access from the caller only until the thread |
| 505 |
* list is unlocked. */ |
| 506 |
#ifdef __wasilibc_unmodified_upstream |
| 507 |
stack -= (uintptr_t)stack % sizeof(uintptr_t); |
| 508 |
stack -= sizeof(struct start_args); |
| 509 |
struct start_args *args = (void *)stack; |
| 510 |
args->start_func = entry; |
| 511 |
args->start_arg = arg; |
| 512 |
args->control = attr._a_sched ? 1 : 0; |
| 513 |
|
| 514 |
/* Application signals (but not the synccall signal) must be |
| 515 |
* blocked before the thread list lock can be taken, to ensure |
| 516 |
* that the lock is AS-safe. */ |
| 517 |
__block_app_sigs(&set); |
| 518 |
|
| 519 |
/* Ensure SIGCANCEL is unblocked in new thread. This requires |
| 520 |
* working with a copy of the set so we can restore the |
| 521 |
* original mask in the calling thread. */ |
| 522 |
memcpy(&args->sig_mask, &set, sizeof args->sig_mask); |
| 523 |
args->sig_mask[(SIGCANCEL-1)/8/sizeof(long)] &= |
| 524 |
~(1UL<<((SIGCANCEL-1)%(8*sizeof(long)))); |
| 525 |
#else |
| 526 |
/* Align the stack to struct start_args */ |
| 527 |
stack -= sizeof(struct start_args); |
| 528 |
stack -= (uintptr_t)stack % alignof(struct start_args); |
| 529 |
struct start_args *args = (void *)stack; |
| 530 |
|
| 531 |
/* Align the stack to 16 and store it */ |
| 532 |
new->stack = (void *)((uintptr_t) stack & -16); |
| 533 |
/* Correct the stack size */ |
| 534 |
new->stack_size = stack - stack_limit; |
| 535 |
|
| 536 |
args->stack = new->stack; /* just for convenience of asm trampoline */ |
| 537 |
args->start_func = entry; |
| 538 |
args->start_arg = arg; |
| 539 |
args->tls_base = (void*)new_tls_base; |
| 540 |
#endif |
| 541 |
|
| 542 |
__tl_lock(); |
| 543 |
if (!libc.threads_minus_1++) libc.need_locks = 1; |
| 544 |
#ifdef __wasilibc_unmodified_upstream |
| 545 |
ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock); |
| 546 |
#else |
| 547 |
/* Instead of `__clone`, WASI uses a host API to instantiate a new version |
| 548 |
* of the current module and start executing the entry function. The |
| 549 |
* wasi-threads specification requires the module to export a |
| 550 |
* `wasi_thread_start` function, which is invoked with `args`. */ |
| 551 |
ret = __wasi_thread_spawn((void *) args); |
| 552 |
#endif |
| 553 |
|
| 554 |
#ifdef __wasilibc_unmodified_upstream |
| 555 |
/* All clone failures translate to EAGAIN. If explicit scheduling |
| 556 |
* was requested, attempt it before unlocking the thread list so |
| 557 |
* that the failed thread is never exposed and so that we can |
| 558 |
* clean up all transient resource usage before returning. */ |
| 559 |
if (ret < 0) { |
| 560 |
ret = -EAGAIN; |
| 561 |
} else if (attr._a_sched) { |
| 562 |
ret = __syscall(SYS_sched_setscheduler, |
| 563 |
new->tid, attr._a_policy, &attr._a_prio); |
| 564 |
if (a_swap(&args->control, ret ? 3 : 0)==2) |
| 565 |
__wake(&args->control, 1, 1); |
| 566 |
if (ret) |
| 567 |
__wait(&args->control, 0, 3, 0); |
| 568 |
} |
| 569 |
#else |
| 570 |
#define WASI_THREADS_MAX_TID 0x1FFFFFFF |
| 571 |
/* `wasi_thread_spawn` will either return a host-provided thread ID (TID) |
| 572 |
* (`<1, 0x1FFFFFFF>`) or an error code (`< 0`). Please note that `0` is |
| 573 |
* reserved for compatibility reasons and must not be returned by the runtime. |
| 574 |
* As in the unmodified version, all spawn failures translate to EAGAIN; |
| 575 |
* unlike the modified version, there is no need to "start up" the child |
| 576 |
* thread--the host does this. If the spawn did succeed, then we store the |
| 577 |
* TID atomically, since this parent thread is racing with the child thread |
| 578 |
* to set this field; this way, whichever thread reaches this point first |
| 579 |
* can continue without waiting. */ |
| 580 |
assert(ret != 0 && ret <= WASI_THREADS_MAX_TID); |
| 581 |
if (ret < 0) { |
| 582 |
ret = -EAGAIN; |
| 583 |
} else { |
| 584 |
atomic_store((atomic_int *) &(new->tid), ret); |
| 585 |
} |
| 586 |
#endif |
| 587 |
|
| 588 |
if (ret >= 0) { |
| 589 |
new->next = self->next; |
| 590 |
new->prev = self; |
| 591 |
new->next->prev = new; |
| 592 |
new->prev->next = new; |
| 593 |
} else { |
| 594 |
if (!--libc.threads_minus_1) libc.need_locks = 0; |
| 595 |
} |
| 596 |
__tl_unlock(); |
| 597 |
#ifdef __wasilibc_unmodified_upstream |
| 598 |
__restore_sigs(&set); |
| 599 |
#endif |
| 600 |
__release_ptc(); |
| 601 |
|
| 602 |
if (ret < 0) { |
| 603 |
#ifdef __wasilibc_unmodified_upstream |
| 604 |
if (map) __munmap(map, size); |
| 605 |
#else |
| 606 |
free(map); |
| 607 |
#endif |
| 608 |
return -ret; |
| 609 |
} |
| 610 |
|
| 611 |
*res = new; |
| 612 |
return 0; |
| 613 |
fail: |
| 614 |
__release_ptc(); |
| 615 |
return EAGAIN; |
| 616 |
} |
| 617 |
|
| 618 |
#ifdef __wasilibc_unmodified_upstream |
| 619 |
weak_alias(__pthread_exit, pthread_exit); |
| 620 |
#endif |
| 621 |
weak_alias(__pthread_create, pthread_create); |