Debian Codegraph @

libgcc/config/aarch64/heap-trampoline.c

raw
1 /* Copyright The GNU Toolchain Authors. */
2
3 /* libc is required to allocate trampolines. */
4 #ifndef inhibit_libc
5
6 #include <unistd.h>
7 #include <sys/mman.h>
8 #include <stdint.h>
9 #include <stdlib.h>
10 #include <stdio.h>
11 #include <string.h>
12
13 #if __APPLE__
14 /* For pthread_jit_write_protect_np */
15 #include <pthread.h>
16 #endif
17
18 /* HEAP_T_ATTR is provided to allow targets to build the exported functions
19 as weak definitions. */
20 #ifndef HEAP_T_ATTR
21 # define HEAP_T_ATTR
22 #endif
23
24 void *allocate_trampoline_page (void);
25 int get_trampolines_per_page (void);
26 struct tramp_ctrl_data *allocate_tramp_ctrl (struct tramp_ctrl_data *parent);
27 void *allocate_trampoline_page (void);
28
29 void __gcc_nested_func_ptr_created (void *chain, void *func, void *dst);
30 void __gcc_nested_func_ptr_deleted (void);
31
32 #if defined(__linux__) || defined(__gnu_hurd__)
33 static const unsigned char aarch64_trampoline_insns[6][4] = {
34 {0x5f, 0x24, 0x03, 0xd5}, /* hint 34 */
35 {0xb1, 0x00, 0x00, 0x58}, /* ldr x17, .+20 */
36 {0xd2, 0x00, 0x00, 0x58}, /* ldr x18, .+24 */
37 {0x20, 0x02, 0x1f, 0xd6}, /* br x17 */
38 {0x9f, 0x3f, 0x03, 0xd5}, /* dsb sy */
39 {0xdf, 0x3f, 0x03, 0xd5} /* isb */
40 };
41
42 #elif __APPLE__
43 static const unsigned char aarch64_trampoline_insns[6][4] = {
44 {0x5f, 0x24, 0x03, 0xd5}, /* hint 34 */
45 {0xb1, 0x00, 0x00, 0x58}, /* ldr x17, .+20 */
46 {0xd0, 0x00, 0x00, 0x58}, /* ldr x16, .+24 */
47 {0x20, 0x02, 0x1f, 0xd6}, /* br x17 */
48 {0x9f, 0x3f, 0x03, 0xd5}, /* dsb sy */
49 {0xdf, 0x3f, 0x03, 0xd5} /* isb */
50 };
51
52 #else
53 #error "Unsupported AArch64 platform for heap trampolines"
54 #endif
55
56 struct aarch64_trampoline {
57 unsigned char insns[6][4];
58 void *func_ptr;
59 void *chain_ptr;
60 };
61
62 struct tramp_ctrl_data
63 {
64 struct tramp_ctrl_data *prev;
65
66 int free_trampolines;
67
68 /* This will be pointing to an executable mmap'ed page. */
69 struct aarch64_trampoline *trampolines;
70 };
71
72 int
73 get_trampolines_per_page (void)
74 {
75 return getpagesize() / sizeof(struct aarch64_trampoline);
76 }
77
78 static _Thread_local struct tramp_ctrl_data *tramp_ctrl_curr = NULL;
79
80 void *
81 allocate_trampoline_page (void)
82 {
83 void *page;
84
85 #if defined(__linux__) || defined(__gnu_hurd__)
86 page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
87 MAP_ANON | MAP_PRIVATE, 0, 0);
88 #elif __APPLE__
89 page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
90 MAP_ANON | MAP_PRIVATE | MAP_JIT, 0, 0);
91 #else
92 page = MAP_FAILED;
93 #endif
94
95 return page;
96 }
97
98 struct tramp_ctrl_data *
99 allocate_tramp_ctrl (struct tramp_ctrl_data *parent)
100 {
101 struct tramp_ctrl_data *p = malloc (sizeof (struct tramp_ctrl_data));
102 if (p == NULL)
103 return NULL;
104
105 p->trampolines = allocate_trampoline_page ();
106
107 if (p->trampolines == MAP_FAILED)
108 return NULL;
109
110 p->prev = parent;
111 p->free_trampolines = get_trampolines_per_page();
112
113 return p;
114 }
115
116 HEAP_T_ATTR
117 void
118 __gcc_nested_func_ptr_created (void *chain, void *func, void *dst)
119 {
120 if (tramp_ctrl_curr == NULL)
121 {
122 tramp_ctrl_curr = allocate_tramp_ctrl (NULL);
123 if (tramp_ctrl_curr == NULL)
124 abort ();
125 }
126
127 if (tramp_ctrl_curr->free_trampolines == 0)
128 {
129 void *tramp_ctrl = allocate_tramp_ctrl (tramp_ctrl_curr);
130 if (!tramp_ctrl)
131 abort ();
132
133 tramp_ctrl_curr = tramp_ctrl;
134 }
135
136 struct aarch64_trampoline *trampoline
137 = &tramp_ctrl_curr->trampolines[get_trampolines_per_page ()
138 - tramp_ctrl_curr->free_trampolines];
139
140 #if __APPLE__
141 /* Disable write protection for the MAP_JIT regions in this thread (see
142 https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon) */
143 pthread_jit_write_protect_np (0);
144 #endif
145
146 memcpy (trampoline->insns, aarch64_trampoline_insns,
147 sizeof(aarch64_trampoline_insns));
148 trampoline->func_ptr = func;
149 trampoline->chain_ptr = chain;
150
151 #if __APPLE__
152 /* Re-enable write protection. */
153 pthread_jit_write_protect_np (1);
154 #endif
155
156 tramp_ctrl_curr->free_trampolines -= 1;
157
158 __builtin___clear_cache ((void *)trampoline->insns,
159 ((void *)trampoline->insns + sizeof(trampoline->insns)));
160
161 *(void **) dst = &trampoline->insns;
162 }
163
164 HEAP_T_ATTR
165 void
166 __gcc_nested_func_ptr_deleted (void)
167 {
168 if (tramp_ctrl_curr == NULL)
169 abort ();
170
171 tramp_ctrl_curr->free_trampolines += 1;
172
173 if (tramp_ctrl_curr->free_trampolines == get_trampolines_per_page ())
174 {
175 if (tramp_ctrl_curr->prev == NULL)
176 return;
177
178 munmap (tramp_ctrl_curr->trampolines, getpagesize());
179 struct tramp_ctrl_data *prev = tramp_ctrl_curr->prev;
180 free (tramp_ctrl_curr);
181 tramp_ctrl_curr = prev;
182 }
183 }
184
185 #endif /* !inhibit_libc */