Debian Codegraph @

libc-top-half/musl/src/env/__init_tls.c

raw
1 #ifdef __wasilibc_unmodified_upstream
2 #define SYSCALL_NO_TLS 1
3 #include <elf.h>
4 #endif
5 #include <limits.h>
6 #ifdef __wasilibc_unmodified_upstream
7 #include <sys/mman.h>
8 #endif
9 #include <string.h>
10 #include <stddef.h>
11 #include "pthread_impl.h"
12 #include "libc.h"
13 #include "atomic.h"
14 #include "syscall.h"
15
16 #if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
17 volatile int __thread_list_lock;
18 #endif
19
20 #ifndef __wasilibc_unmodified_upstream
21
22 /* These symbols are generated by wasm-ld. __stack_high/__stack_low
23 * symbols are only available in LLVM v16 and higher, therefore they're
24 * defined as weak symbols and if not available, __heap_base/__data_end
25 * is used instead.
26 *
27 * TODO: remove usage of __heap_base/__data_end for stack size calculation
28 * once we drop support for LLVM v15 and older.
29 */
30 #if !defined(__pic__)
31 extern unsigned char __heap_base;
32 extern unsigned char __data_end;
33 extern unsigned char __global_base;
34 #endif
35 extern weak unsigned char __stack_high;
36 extern weak unsigned char __stack_low;
37
38 struct stack_bounds {
39 void *base;
40 size_t size;
41 };
42
43 static inline struct stack_bounds get_stack_bounds()
44 {
45 struct stack_bounds bounds;
46
47 if (&__stack_high) {
48 bounds.base = &__stack_high;
49 bounds.size = &__stack_high - &__stack_low;
50 } else {
51 /* For non-pic, make a guess using the knowledge about
52 * how wasm-ld lays out things. For pic, just give up.
53 */
54 #if !defined(__pic__)
55 unsigned char *sp;
56 __asm__(
57 ".globaltype __stack_pointer, i32\n"
58 "global.get __stack_pointer\n"
59 "local.set %0\n"
60 : "=r"(sp));
61 if (sp > &__global_base) {
62 bounds.base = &__heap_base;
63 bounds.size = &__heap_base - &__data_end;
64 } else {
65 bounds.base = &__global_base;
66 bounds.size = (size_t)&__global_base;
67 }
68 #else
69 bounds.base = 0;
70 bounds.size = 0;
71 #endif
72 }
73
74 return bounds;
75 }
76
77 void __wasi_init_tp() {
78 __init_tp((void *)__get_tp());
79 }
80 #endif
81
82 int __init_tp(void *p)
83 {
84 pthread_t td = p;
85 td->self = td;
86 #ifdef __wasilibc_unmodified_upstream
87 int r = __set_thread_area(TP_ADJ(p));
88 if (r < 0) return -1;
89 if (!r) libc.can_do_threads = 1;
90 td->detach_state = DT_JOINABLE;
91 td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock);
92 #else
93 struct stack_bounds bounds = get_stack_bounds();
94 __default_stacksize =
95 bounds.size < DEFAULT_STACK_MAX ?
96 bounds.size : DEFAULT_STACK_MAX;
97 td->stack = bounds.base;
98 td->stack_size = bounds.size;
99 td->guard_size = 0;
100 #ifdef _REENTRANT
101 td->detach_state = DT_JOINABLE;
102 /*
103 * Initialize the TID to a value which doesn't conflict with
104 * host-allocated TIDs, so that TID-based locks can work.
105 *
106 * Note:
107 * - Host-allocated TIDs range from 1 to 0x1fffffff. (inclusive)
108 * - __tl_lock and __lockfile uses TID 0 as "unlocked".
109 * - __lockfile relies on the fact the most significant two bits
110 * of TIDs are 0.
111 */
112 td->tid = 0x3fffffff;
113 #endif
114 #endif
115 #if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
116 td->locale = &libc.global_locale;
117 td->robust_list.head = &td->robust_list.head;
118 #endif
119 td->sysinfo = __sysinfo;
120 td->next = td->prev = td;
121 return 0;
122 }
123
124 #ifdef __wasilibc_unmodified_upstream
125
126 static struct builtin_tls {
127 char c;
128 struct pthread pt;
129 void *space[16];
130 } builtin_tls[1];
131 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
132
133 static struct tls_module main_tls;
134 #endif
135
136 #ifndef __wasilibc_unmodified_upstream
137 extern void __wasm_init_tls(void*);
138 #endif
139
140 #if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
141 void *__copy_tls(unsigned char *mem)
142 {
143 #ifdef __wasilibc_unmodified_upstream
144 pthread_t td;
145 struct tls_module *p;
146 size_t i;
147 uintptr_t *dtv;
148
149 #ifdef TLS_ABOVE_TP
150 dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1);
151
152 mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1);
153 td = (pthread_t)mem;
154 mem += sizeof(struct pthread);
155
156 for (i=1, p=libc.tls_head; p; i++, p=p->next) {
157 dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET;
158 memcpy(mem + p->offset, p->image, p->len);
159 }
160 #else
161 dtv = (uintptr_t *)mem;
162
163 mem += libc.tls_size - sizeof(struct pthread);
164 mem -= (uintptr_t)mem & (libc.tls_align-1);
165 td = (pthread_t)mem;
166
167 for (i=1, p=libc.tls_head; p; i++, p=p->next) {
168 dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET;
169 memcpy(mem - p->offset, p->image, p->len);
170 }
171 #endif
172 dtv[0] = libc.tls_cnt;
173 td->dtv = dtv;
174 return td;
175 #else
176 size_t tls_align = __builtin_wasm_tls_align();
177 volatile void* tls_base = __builtin_wasm_tls_base();
178 mem += tls_align;
179 mem -= (uintptr_t)mem & (tls_align-1);
180 __wasm_init_tls(mem);
181 __asm__("local.get %0\n"
182 "global.set __tls_base\n"
183 :: "r"(tls_base));
184 return mem;
185 #endif
186 }
187 #endif /* defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT) */
188
189 #ifdef __wasilibc_unmodified_upstream
190 #if ULONG_MAX == 0xffffffff
191 typedef Elf32_Phdr Phdr;
192 #else
193 typedef Elf64_Phdr Phdr;
194 #endif
195
196 extern weak hidden const size_t _DYNAMIC[];
197
198 static void static_init_tls(size_t *aux)
199 {
200 unsigned char *p;
201 size_t n;
202 Phdr *phdr, *tls_phdr=0;
203 size_t base = 0;
204 void *mem;
205
206 for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) {
207 phdr = (void *)p;
208 if (phdr->p_type == PT_PHDR)
209 base = aux[AT_PHDR] - phdr->p_vaddr;
210 if (phdr->p_type == PT_DYNAMIC && _DYNAMIC)
211 base = (size_t)_DYNAMIC - phdr->p_vaddr;
212 if (phdr->p_type == PT_TLS)
213 tls_phdr = phdr;
214 if (phdr->p_type == PT_GNU_STACK &&
215 phdr->p_memsz > __default_stacksize)
216 __default_stacksize =
217 phdr->p_memsz < DEFAULT_STACK_MAX ?
218 phdr->p_memsz : DEFAULT_STACK_MAX;
219 }
220
221 if (tls_phdr) {
222 main_tls.image = (void *)(base + tls_phdr->p_vaddr);
223 main_tls.len = tls_phdr->p_filesz;
224 main_tls.size = tls_phdr->p_memsz;
225 main_tls.align = tls_phdr->p_align;
226 libc.tls_cnt = 1;
227 libc.tls_head = &main_tls;
228 }
229
230 main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image)
231 & (main_tls.align-1);
232 #ifdef TLS_ABOVE_TP
233 main_tls.offset = GAP_ABOVE_TP;
234 main_tls.offset += (-GAP_ABOVE_TP + (uintptr_t)main_tls.image)
235 & (main_tls.align-1);
236 #else
237 main_tls.offset = main_tls.size;
238 #endif
239 if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN;
240
241 libc.tls_align = main_tls.align;
242 libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread)
243 #ifdef TLS_ABOVE_TP
244 + main_tls.offset
245 #endif
246 + main_tls.size + main_tls.align
247 + MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN;
248
249 if (libc.tls_size > sizeof builtin_tls) {
250 #ifndef SYS_mmap2
251 #define SYS_mmap2 SYS_mmap
252 #endif
253 mem = (void *)__syscall(
254 SYS_mmap2,
255 0, libc.tls_size, PROT_READ|PROT_WRITE,
256 MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
257 /* -4095...-1 cast to void * will crash on dereference anyway,
258 * so don't bloat the init code checking for error codes and
259 * explicitly calling a_crash(). */
260 } else {
261 mem = builtin_tls;
262 }
263
264 /* Failure to initialize thread pointer is always fatal. */
265 if (__init_tp(__copy_tls(mem)) < 0)
266 a_crash();
267 }
268
269 weak_alias(static_init_tls, __init_tls);
270 #endif