Debian Codegraph @

src/aarch64/ffi.c

raw
1 /* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
2
3 Permission is hereby granted, free of charge, to any person obtaining
4 a copy of this software and associated documentation files (the
5 ``Software''), to deal in the Software without restriction, including
6 without limitation the rights to use, copy, modify, merge, publish,
7 distribute, sublicense, and/or sell copies of the Software, and to
8 permit persons to whom the Software is furnished to do so, subject to
9 the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
15 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
17 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
18 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
19 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
21
22 #if defined(__aarch64__) || defined(__arm64__)|| defined (_M_ARM64)
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <stdint.h>
26 #include <fficonfig.h>
27 #include <ffi.h>
28 #include <ffi_common.h>
29 #include "internal.h"
30 #ifdef _WIN32
31 #include <windows.h> /* FlushInstructionCache */
32 #endif
33 #include <tramp.h>
34
35 /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE;
36 all further uses in this file will refer to the 128-bit type. */
37 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
38 # if FFI_TYPE_LONGDOUBLE != 4
39 # error FFI_TYPE_LONGDOUBLE out of date
40 # endif
41 #else
42 # undef FFI_TYPE_LONGDOUBLE
43 # define FFI_TYPE_LONGDOUBLE 4
44 #endif
45
46 union _d
47 {
48 UINT64 d;
49 UINT32 s[2];
50 };
51
52 struct _v
53 {
54 union _d d[2] __attribute__((aligned(16)));
55 };
56
57 struct call_context
58 {
59 struct _v v[N_V_ARG_REG];
60 UINT64 x[N_X_ARG_REG];
61 };
62
63 #if FFI_EXEC_TRAMPOLINE_TABLE
64
65 #ifdef __MACH__
66 #ifdef HAVE_ARM64E_PTRAUTH
67 #include <ptrauth.h>
68 #endif
69 #include <mach/vm_param.h>
70 #endif
71
72 #else
73
74 #if defined (__clang__) && defined (__APPLE__)
75 extern void sys_icache_invalidate (void *start, size_t len);
76 #endif
77
78 static inline void
79 ffi_clear_cache (void *start, void *end)
80 {
81 #if defined (__clang__) && defined (__APPLE__)
82 sys_icache_invalidate (start, (char *)end - (char *)start);
83 #elif defined (__GNUC__)
84 __builtin___clear_cache (start, end);
85 #elif defined (_WIN32)
86 FlushInstructionCache(GetCurrentProcess(), start, (char*)end - (char*)start);
87 #else
88 #error "Missing builtin to flush instruction cache"
89 #endif
90 }
91
92 #endif
93
94 /* A subroutine of is_vfp_type. Given a structure type, return the type code
95 of the first non-structure element. Recurse for structure elements.
96 Return -1 if the structure is in fact empty, i.e. no nested elements. */
97
98 static int
99 is_hfa0 (const ffi_type *ty)
100 {
101 ffi_type **elements = ty->elements;
102 int i, ret = -1;
103
104 if (elements != NULL)
105 for (i = 0; elements[i]; ++i)
106 {
107 ret = elements[i]->type;
108 if (ret == FFI_TYPE_STRUCT || ret == FFI_TYPE_COMPLEX)
109 {
110 ret = is_hfa0 (elements[i]);
111 if (ret < 0)
112 continue;
113 }
114 break;
115 }
116
117 return ret;
118 }
119
120 /* A subroutine of is_vfp_type. Given a structure type, return true if all
121 of the non-structure elements are the same as CANDIDATE. */
122
123 static int
124 is_hfa1 (const ffi_type *ty, int candidate)
125 {
126 ffi_type **elements = ty->elements;
127 int i;
128
129 if (elements != NULL)
130 for (i = 0; elements[i]; ++i)
131 {
132 int t = elements[i]->type;
133 if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX)
134 {
135 if (!is_hfa1 (elements[i], candidate))
136 return 0;
137 }
138 else if (t != candidate)
139 return 0;
140 }
141
142 return 1;
143 }
144
145 /* Determine if TY may be allocated to the FP registers. This is both an
146 fp scalar type as well as an homogenous floating point aggregate (HFA).
147 That is, a structure consisting of 1 to 4 members of all the same type,
148 where that type is an fp scalar.
149
150 Returns non-zero iff TY is an HFA. The result is the AARCH64_RET_*
151 constant for the type. */
152
153 static int
154 is_vfp_type (const ffi_type *ty)
155 {
156 ffi_type **elements;
157 int candidate, i;
158 size_t size, ele_count;
159
160 /* Quickest tests first. */
161 candidate = ty->type;
162 switch (candidate)
163 {
164 default:
165 return 0;
166 case FFI_TYPE_FLOAT:
167 case FFI_TYPE_DOUBLE:
168 case FFI_TYPE_LONGDOUBLE:
169 ele_count = 1;
170 goto done;
171 case FFI_TYPE_COMPLEX:
172 candidate = ty->elements[0]->type;
173 switch (candidate)
174 {
175 case FFI_TYPE_FLOAT:
176 case FFI_TYPE_DOUBLE:
177 case FFI_TYPE_LONGDOUBLE:
178 ele_count = 2;
179 goto done;
180 }
181 return 0;
182 case FFI_TYPE_STRUCT:
183 break;
184 }
185
186 /* No HFA types are smaller than 4 bytes, or larger than 64 bytes. */
187 size = ty->size;
188 if (size < 4 || size > 64)
189 return 0;
190
191 /* Find the type of the first non-structure member. */
192 elements = ty->elements;
193 candidate = elements[0]->type;
194 if (candidate == FFI_TYPE_STRUCT || candidate == FFI_TYPE_COMPLEX)
195 {
196 for (i = 0; ; ++i)
197 {
198 candidate = is_hfa0 (elements[i]);
199 if (candidate >= 0)
200 break;
201 }
202 }
203
204 /* If the first member is not a floating point type, it's not an HFA.
205 Also quickly re-check the size of the structure. */
206 switch (candidate)
207 {
208 case FFI_TYPE_FLOAT:
209 ele_count = size / sizeof(float);
210 if (size != ele_count * sizeof(float))
211 return 0;
212 break;
213 case FFI_TYPE_DOUBLE:
214 ele_count = size / sizeof(double);
215 if (size != ele_count * sizeof(double))
216 return 0;
217 break;
218 case FFI_TYPE_LONGDOUBLE:
219 ele_count = size / sizeof(long double);
220 if (size != ele_count * sizeof(long double))
221 return 0;
222 break;
223 default:
224 return 0;
225 }
226 if (ele_count > 4)
227 return 0;
228
229 /* Finally, make sure that all scalar elements are the same type. */
230 for (i = 0; elements[i]; ++i)
231 {
232 int t = elements[i]->type;
233 if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX)
234 {
235 if (!is_hfa1 (elements[i], candidate))
236 return 0;
237 }
238 else if (t != candidate)
239 return 0;
240 }
241
242 /* All tests succeeded. Encode the result. */
243 done:
244 return candidate * 4 + (4 - (int)ele_count);
245 }
246
247 /* Representation of the procedure call argument marshalling
248 state.
249
250 The terse state variable names match the names used in the AARCH64
251 PCS.
252
253 The struct area is allocated downwards from the top of the argument
254 area. It is used to hold copies of structures passed by value that are
255 bigger than 16 bytes. */
256
257 struct arg_state
258 {
259 unsigned ngrn; /* Next general-purpose register number. */
260 unsigned nsrn; /* Next vector register number. */
261 size_t nsaa; /* Next stack offset. */
262 size_t next_struct_area; /* Place to allocate big structs. */
263
264 #if defined (__APPLE__)
265 unsigned allocating_variadic;
266 #endif
267 };
268
269 /* Initialize a procedure call argument marshalling state. */
270 static void
271 arg_init (struct arg_state *state, size_t size)
272 {
273 state->ngrn = 0;
274 state->nsrn = 0;
275 state->nsaa = 0;
276 state->next_struct_area = size;
277 #if defined (__APPLE__)
278 state->allocating_variadic = 0;
279 #endif
280 }
281
282 /* Allocate an aligned slot on the stack and return a pointer to it. */
283 static void *
284 allocate_to_stack (struct arg_state *state, void *stack,
285 size_t alignment, size_t size)
286 {
287 size_t nsaa = state->nsaa;
288
289 /* Round up the NSAA to the larger of 8 or the natural
290 alignment of the argument's type. */
291 #if defined (__APPLE__)
292 if (state->allocating_variadic && alignment < 8)
293 alignment = 8;
294 #else
295 if (alignment < 8)
296 alignment = 8;
297 #endif
298
299 nsaa = FFI_ALIGN (nsaa, alignment);
300 state->nsaa = nsaa + size;
301
302 return (char *)stack + nsaa;
303 }
304
305 /* Allocate and copy a structure that is passed by value on the stack and
306 return a pointer to it. */
307 static void *
308 allocate_and_copy_struct_to_stack (struct arg_state *state, void *stack,
309 size_t alignment, size_t size, void *value)
310 {
311 size_t dest = state->next_struct_area - size;
312
313 /* Round down to the natural alignment of the value. */
314 dest = FFI_ALIGN_DOWN (dest, alignment);
315 state->next_struct_area = dest;
316
317 return memcpy ((char *) stack + dest, value, size);
318 }
319
320 static ffi_arg
321 extend_integer_type (void *source, int type)
322 {
323 switch (type)
324 {
325 case FFI_TYPE_UINT8:
326 {
327 UINT8 u8;
328 memcpy (&u8, source, sizeof (u8));
329 return u8;
330 }
331 case FFI_TYPE_SINT8:
332 {
333 SINT8 s8;
334 memcpy (&s8, source, sizeof (s8));
335 return s8;
336 }
337 case FFI_TYPE_UINT16:
338 {
339 UINT16 u16;
340 memcpy (&u16, source, sizeof (u16));
341 return u16;
342 }
343 case FFI_TYPE_SINT16:
344 {
345 SINT16 s16;
346 memcpy (&s16, source, sizeof (s16));
347 return s16;
348 }
349 case FFI_TYPE_UINT32:
350 {
351 UINT32 u32;
352 memcpy (&u32, source, sizeof (u32));
353 return u32;
354 }
355 case FFI_TYPE_INT:
356 case FFI_TYPE_SINT32:
357 {
358 SINT32 s32;
359 memcpy (&s32, source, sizeof (s32));
360 return s32;
361 }
362 case FFI_TYPE_UINT64:
363 case FFI_TYPE_SINT64:
364 {
365 UINT64 u64;
366 memcpy (&u64, source, sizeof (u64));
367 return u64;
368 }
369 case FFI_TYPE_POINTER:
370 {
371 uintptr_t uptr;
372 memcpy (&uptr, source, sizeof (uptr));
373 return uptr;
374 }
375 default:
376 abort();
377 }
378 }
379
380 #if defined(_MSC_VER)
381 void extend_hfa_type (void *dest, void *src, int h);
382 #else
383 static void
384 extend_hfa_type (void *dest, void *src, int h)
385 {
386 ssize_t f = h - AARCH64_RET_S4;
387 void *x0;
388
389 #define BTI_J "hint #36"
390 asm volatile (
391 "adr %0, 0f\n"
392 " add %0, %0, %1\n"
393 " br %0\n"
394 "0: "BTI_J"\n" /* S4 */
395 " ldp s16, s17, [%3]\n"
396 " ldp s18, s19, [%3, #8]\n"
397 " b 4f\n"
398 " "BTI_J"\n" /* S3 */
399 " ldp s16, s17, [%3]\n"
400 " ldr s18, [%3, #8]\n"
401 " b 3f\n"
402 " "BTI_J"\n" /* S2 */
403 " ldp s16, s17, [%3]\n"
404 " b 2f\n"
405 " nop\n"
406 " "BTI_J"\n" /* S1 */
407 " ldr s16, [%3]\n"
408 " b 1f\n"
409 " nop\n"
410 " "BTI_J"\n" /* D4 */
411 " ldp d16, d17, [%3]\n"
412 " ldp d18, d19, [%3, #16]\n"
413 " b 4f\n"
414 " "BTI_J"\n" /* D3 */
415 " ldp d16, d17, [%3]\n"
416 " ldr d18, [%3, #16]\n"
417 " b 3f\n"
418 " "BTI_J"\n" /* D2 */
419 " ldp d16, d17, [%3]\n"
420 " b 2f\n"
421 " nop\n"
422 " "BTI_J"\n" /* D1 */
423 " ldr d16, [%3]\n"
424 " b 1f\n"
425 " nop\n"
426 " "BTI_J"\n" /* Q4 */
427 " ldp q16, q17, [%3]\n"
428 " ldp q18, q19, [%3, #32]\n"
429 " b 4f\n"
430 " "BTI_J"\n" /* Q3 */
431 " ldp q16, q17, [%3]\n"
432 " ldr q18, [%3, #32]\n"
433 " b 3f\n"
434 " "BTI_J"\n" /* Q2 */
435 " ldp q16, q17, [%3]\n"
436 " b 2f\n"
437 " nop\n"
438 " "BTI_J"\n" /* Q1 */
439 " ldr q16, [%3]\n"
440 " b 1f\n"
441 "4: str q19, [%2, #48]\n"
442 "3: str q18, [%2, #32]\n"
443 "2: str q17, [%2, #16]\n"
444 "1: str q16, [%2]"
445 : "=&r"(x0)
446 : "r"(f * 16), "r"(dest), "r"(src)
447 : "memory", "v16", "v17", "v18", "v19");
448 }
449 #endif
450
451 #if defined(_MSC_VER)
452 void* compress_hfa_type (void *dest, void *src, int h);
453 #else
454 static void *
455 compress_hfa_type (void *dest, void *reg, int h)
456 {
457 switch (h)
458 {
459 case AARCH64_RET_S1:
460 if (dest == reg)
461 {
462 #ifdef __AARCH64EB__
463 dest += 12;
464 #endif
465 }
466 else
467 *(float *)dest = *(float *)reg;
468 break;
469 case AARCH64_RET_S2:
470 asm ("ldp q16, q17, [%1]\n\t"
471 "st2 { v16.s, v17.s }[0], [%0]"
472 : : "r"(dest), "r"(reg) : "memory", "v16", "v17");
473 break;
474 case AARCH64_RET_S3:
475 asm ("ldp q16, q17, [%1]\n\t"
476 "ldr q18, [%1, #32]\n\t"
477 "st3 { v16.s, v17.s, v18.s }[0], [%0]"
478 : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18");
479 break;
480 case AARCH64_RET_S4:
481 asm ("ldp q16, q17, [%1]\n\t"
482 "ldp q18, q19, [%1, #32]\n\t"
483 "st4 { v16.s, v17.s, v18.s, v19.s }[0], [%0]"
484 : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19");
485 break;
486
487 case AARCH64_RET_D1:
488 if (dest == reg)
489 {
490 #ifdef __AARCH64EB__
491 dest += 8;
492 #endif
493 }
494 else
495 *(double *)dest = *(double *)reg;
496 break;
497 case AARCH64_RET_D2:
498 asm ("ldp q16, q17, [%1]\n\t"
499 "st2 { v16.d, v17.d }[0], [%0]"
500 : : "r"(dest), "r"(reg) : "memory", "v16", "v17");
501 break;
502 case AARCH64_RET_D3:
503 asm ("ldp q16, q17, [%1]\n\t"
504 "ldr q18, [%1, #32]\n\t"
505 "st3 { v16.d, v17.d, v18.d }[0], [%0]"
506 : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18");
507 break;
508 case AARCH64_RET_D4:
509 asm ("ldp q16, q17, [%1]\n\t"
510 "ldp q18, q19, [%1, #32]\n\t"
511 "st4 { v16.d, v17.d, v18.d, v19.d }[0], [%0]"
512 : : "r"(dest), "r"(reg) : "memory", "v16", "v17", "v18", "v19");
513 break;
514
515 default:
516 if (dest != reg)
517 return memcpy (dest, reg, 16 * (4 - (h & 3)));
518 break;
519 }
520 return dest;
521 }
522 #endif
523
524 /* Either allocate an appropriate register for the argument type, or if
525 none are available, allocate a stack slot and return a pointer
526 to the allocated space. */
527
528 static void *
529 allocate_int_to_reg_or_stack (struct call_context *context,
530 struct arg_state *state,
531 void *stack, size_t size)
532 {
533 if (state->ngrn < N_X_ARG_REG)
534 return &context->x[state->ngrn++];
535
536 state->ngrn = N_X_ARG_REG;
537 return allocate_to_stack (state, stack, size, size);
538 }
539
540 ffi_status FFI_HIDDEN
541 ffi_prep_cif_machdep (ffi_cif *cif)
542 {
543 ffi_type *rtype = cif->rtype;
544 size_t bytes = cif->bytes;
545 int flags, i, n;
546
547 switch (rtype->type)
548 {
549 case FFI_TYPE_VOID:
550 flags = AARCH64_RET_VOID;
551 break;
552 case FFI_TYPE_UINT8:
553 flags = AARCH64_RET_UINT8;
554 break;
555 case FFI_TYPE_UINT16:
556 flags = AARCH64_RET_UINT16;
557 break;
558 case FFI_TYPE_UINT32:
559 flags = AARCH64_RET_UINT32;
560 break;
561 case FFI_TYPE_SINT8:
562 flags = AARCH64_RET_SINT8;
563 break;
564 case FFI_TYPE_SINT16:
565 flags = AARCH64_RET_SINT16;
566 break;
567 case FFI_TYPE_INT:
568 case FFI_TYPE_SINT32:
569 flags = AARCH64_RET_SINT32;
570 break;
571 case FFI_TYPE_SINT64:
572 case FFI_TYPE_UINT64:
573 flags = AARCH64_RET_INT64;
574 break;
575 case FFI_TYPE_POINTER:
576 flags = (sizeof(void *) == 4 ? AARCH64_RET_UINT32 : AARCH64_RET_INT64);
577 break;
578
579 case FFI_TYPE_FLOAT:
580 case FFI_TYPE_DOUBLE:
581 case FFI_TYPE_LONGDOUBLE:
582 case FFI_TYPE_STRUCT:
583 case FFI_TYPE_COMPLEX:
584 flags = is_vfp_type (rtype);
585 if (flags == 0)
586 {
587 size_t s = rtype->size;
588 if (s > 16)
589 {
590 flags = AARCH64_RET_VOID | AARCH64_RET_IN_MEM;
591 bytes += 8;
592 }
593 else if (s == 16)
594 flags = AARCH64_RET_INT128;
595 else if (s == 8)
596 flags = AARCH64_RET_INT64;
597 else
598 flags = AARCH64_RET_INT128 | AARCH64_RET_NEED_COPY;
599 }
600 break;
601
602 default:
603 abort();
604 }
605
606 for (i = 0, n = cif->nargs; i < n; i++)
607 if (is_vfp_type (cif->arg_types[i]))
608 {
609 flags |= AARCH64_FLAG_ARG_V;
610 break;
611 }
612
613 /* Round the stack up to a multiple of the stack alignment requirement. */
614 cif->bytes = (unsigned) FFI_ALIGN(bytes, 16);
615 cif->flags = flags;
616 #if defined (__APPLE__)
617 cif->aarch64_nfixedargs = 0;
618 #endif
619
620 return FFI_OK;
621 }
622
623 #if defined (__APPLE__)
624 /* Perform Apple-specific cif processing for variadic calls */
625 ffi_status FFI_HIDDEN
626 ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs,
627 unsigned int ntotalargs)
628 {
629 ffi_status status = ffi_prep_cif_machdep (cif);
630 cif->aarch64_nfixedargs = nfixedargs;
631 return status;
632 }
633 #else
634 ffi_status FFI_HIDDEN
635 ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs)
636 {
637 ffi_status status = ffi_prep_cif_machdep (cif);
638 cif->flags |= AARCH64_FLAG_VARARG;
639 return status;
640 }
641 #endif /* __APPLE__ */
642
643 extern void ffi_call_SYSV (struct call_context *context, void *frame,
644 void (*fn)(void), void *rvalue, int flags,
645 void *closure) FFI_HIDDEN;
646
647 /* Call a function with the provided arguments and capture the return
648 value.
649 n.b. ffi_call_SYSV will steal the alloca'd `stack` variable here for use
650 _as its own stack_ - so we need to compile this function without ASAN */
651 FFI_ASAN_NO_SANITIZE
652 static void
653 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *orig_rvalue,
654 void **avalue, void *closure)
655 {
656 struct call_context *context;
657 void *stack, *frame, *rvalue;
658 struct arg_state state;
659 size_t stack_bytes, rtype_size, rsize;
660 int i, nargs, flags, isvariadic = 0;
661 ffi_type *rtype;
662
663 flags = cif->flags;
664 rtype = cif->rtype;
665 rtype_size = rtype->size;
666 stack_bytes = cif->bytes;
667
668 if (flags & AARCH64_FLAG_VARARG)
669 {
670 isvariadic = 1;
671 flags &= ~AARCH64_FLAG_VARARG;
672 }
673
674 /* If the target function returns a structure via hidden pointer,
675 then we cannot allow a null rvalue. Otherwise, mash a null
676 rvalue to void return type. */
677 rsize = 0;
678 if (flags & AARCH64_RET_IN_MEM)
679 {
680 if (orig_rvalue == NULL)
681 rsize = rtype_size;
682 }
683 else if (orig_rvalue == NULL)
684 flags &= AARCH64_FLAG_ARG_V;
685 else if (flags & AARCH64_RET_NEED_COPY)
686 rsize = 16;
687
688 /* Allocate consecutive stack for everything we'll need.
689 The frame uses 40 bytes for: lr, fp, rvalue, flags, sp */
690 context = alloca (sizeof(struct call_context) + stack_bytes + 40 + rsize);
691 stack = context + 1;
692 frame = (void*)((uintptr_t)stack + (uintptr_t)stack_bytes);
693 rvalue = (rsize ? (void*)((uintptr_t)frame + 40) : orig_rvalue);
694
695 arg_init (&state, stack_bytes);
696 for (i = 0, nargs = cif->nargs; i < nargs; i++)
697 {
698 ffi_type *ty = cif->arg_types[i];
699 size_t s = ty->size;
700 void *a = avalue[i];
701 int h, t;
702 void *dest;
703
704 t = ty->type;
705 switch (t)
706 {
707 case FFI_TYPE_VOID:
708 FFI_ASSERT (0);
709 break;
710
711 /* If the argument is a basic type the argument is allocated to an
712 appropriate register, or if none are available, to the stack. */
713 case FFI_TYPE_INT:
714 case FFI_TYPE_UINT8:
715 case FFI_TYPE_SINT8:
716 case FFI_TYPE_UINT16:
717 case FFI_TYPE_SINT16:
718 case FFI_TYPE_UINT32:
719 case FFI_TYPE_SINT32:
720 case FFI_TYPE_UINT64:
721 case FFI_TYPE_SINT64:
722 case FFI_TYPE_POINTER:
723 do_pointer:
724 {
725 ffi_arg ext = extend_integer_type (a, t);
726 if (state.ngrn < N_X_ARG_REG)
727 context->x[state.ngrn++] = ext;
728 else
729 {
730 void *d = allocate_to_stack (&state, stack, ty->alignment, s);
731 state.ngrn = N_X_ARG_REG;
732 /* Note that the default abi extends each argument
733 to a full 64-bit slot, while the iOS abi allocates
734 only enough space. */
735 #ifdef __APPLE__
736 memcpy(d, a, s);
737 #else
738 *(ffi_arg *)d = ext;
739 #endif
740 }
741 }
742 break;
743
744 case FFI_TYPE_FLOAT:
745 case FFI_TYPE_DOUBLE:
746 case FFI_TYPE_LONGDOUBLE:
747 case FFI_TYPE_STRUCT:
748 case FFI_TYPE_COMPLEX:
749 {
750 h = is_vfp_type (ty);
751 if (h)
752 {
753 int elems = 4 - (h & 3);
754 if (cif->abi == FFI_WIN64 && isvariadic)
755 {
756 if (state.ngrn + elems <= N_X_ARG_REG)
757 {
758 dest = &context->x[state.ngrn];
759 state.ngrn += elems;
760 extend_hfa_type(dest, a, h);
761 break;
762 }
763 state.nsrn = N_X_ARG_REG;
764 dest = allocate_to_stack(&state, stack, ty->alignment, s);
765 }
766 else
767 {
768 if (state.nsrn + elems <= N_V_ARG_REG)
769 {
770 dest = &context->v[state.nsrn];
771 state.nsrn += elems;
772 extend_hfa_type (dest, a, h);
773 break;
774 }
775 state.nsrn = N_V_ARG_REG;
776 dest = allocate_to_stack (&state, stack, ty->alignment, s);
777 }
778 }
779 else if (s > 16)
780 {
781 /* If the argument is a composite type that is larger than 16
782 bytes, then the argument is copied to memory, and
783 the argument is replaced by a pointer to the copy. */
784 dest = allocate_and_copy_struct_to_stack (&state, stack,
785 ty->alignment, s,
786 avalue[i]);
787 a = &dest;
788 t = FFI_TYPE_POINTER;
789 s = sizeof (void *);
790 goto do_pointer;
791 }
792 else
793 {
794 size_t n = (s + 7) / 8;
795 if (state.ngrn + n <= N_X_ARG_REG)
796 {
797 /* If the argument is a composite type and the size in
798 double-words is not more than the number of available
799 X registers, then the argument is copied into
800 consecutive X registers. */
801 dest = &context->x[state.ngrn];
802 state.ngrn += (unsigned int)n;
803 }
804 else
805 {
806 /* Otherwise, there are insufficient X registers. Further
807 X register allocations are prevented, the NSAA is
808 adjusted and the argument is copied to memory at the
809 adjusted NSAA. */
810 state.ngrn = N_X_ARG_REG;
811 dest = allocate_to_stack (&state, stack, ty->alignment, s);
812 }
813 }
814 memcpy (dest, a, s);
815 }
816 break;
817
818 default:
819 abort();
820 }
821
822 #if defined (__APPLE__)
823 if (i + 1 == cif->aarch64_nfixedargs)
824 {
825 state.ngrn = N_X_ARG_REG;
826 state.nsrn = N_V_ARG_REG;
827 state.allocating_variadic = 1;
828 }
829 #endif
830 }
831
832 ffi_call_SYSV (context, frame, fn, rvalue, flags, closure);
833
834 if (flags & AARCH64_RET_NEED_COPY)
835 memcpy (orig_rvalue, rvalue, rtype_size);
836 }
837
838 void
839 ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
840 {
841 ffi_call_int (cif, fn, rvalue, avalue, NULL);
842 }
843
844 #if FFI_CLOSURES
845
846 #ifdef FFI_GO_CLOSURES
847 void
848 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
849 void **avalue, void *closure)
850 {
851 ffi_call_int (cif, fn, rvalue, avalue, closure);
852 }
853 #endif /* FFI_GO_CLOSURES */
854
855 /* Build a trampoline. */
856
857 extern void ffi_closure_SYSV (void) FFI_HIDDEN;
858 extern void ffi_closure_SYSV_V (void) FFI_HIDDEN;
859 #if defined(FFI_EXEC_STATIC_TRAMP)
860 extern void ffi_closure_SYSV_alt (void) FFI_HIDDEN;
861 extern void ffi_closure_SYSV_V_alt (void) FFI_HIDDEN;
862 #endif
863
864 ffi_status
865 ffi_prep_closure_loc (ffi_closure *closure,
866 ffi_cif* cif,
867 void (*fun)(ffi_cif*,void*,void**,void*),
868 void *user_data,
869 void *codeloc)
870 {
871 if (cif->abi != FFI_SYSV && cif->abi != FFI_WIN64)
872 return FFI_BAD_ABI;
873
874 void (*start)(void);
875
876 if (cif->flags & AARCH64_FLAG_ARG_V)
877 start = ffi_closure_SYSV_V;
878 else
879 start = ffi_closure_SYSV;
880
881 #if FFI_EXEC_TRAMPOLINE_TABLE
882 # ifdef __MACH__
883 # ifdef HAVE_ARM64E_PTRAUTH
884 codeloc = ptrauth_auth_data(codeloc, ptrauth_key_function_pointer, 0);
885 # endif
886 void **config = (void **)((uint8_t *)codeloc - PAGE_MAX_SIZE);
887 config[0] = closure;
888 config[1] = start;
889 # endif
890 #else
891 static const unsigned char trampoline[16] = {
892 0x90, 0x00, 0x00, 0x58, /* ldr x16, tramp+16 */
893 0xf1, 0xff, 0xff, 0x10, /* adr x17, tramp+0 */
894 0x00, 0x02, 0x1f, 0xd6 /* br x16 */
895 };
896 char *tramp = closure->tramp;
897
898 # if defined(FFI_EXEC_STATIC_TRAMP)
899 if (ffi_tramp_is_present(closure))
900 {
901 /* Initialize the static trampoline's parameters. */
902 if (start == ffi_closure_SYSV_V)
903 start = ffi_closure_SYSV_V_alt;
904 else
905 start = ffi_closure_SYSV_alt;
906 ffi_tramp_set_parms (closure->ftramp, start, closure);
907 goto out;
908 }
909 # endif
910
911 /* Initialize the dynamic trampoline. */
912 memcpy (tramp, trampoline, sizeof(trampoline));
913
914 *(UINT64 *)(tramp + 16) = (uintptr_t)start;
915
916 ffi_clear_cache(tramp, tramp + FFI_TRAMPOLINE_SIZE);
917
918 /* Also flush the cache for code mapping. */
919 # ifdef _WIN32
920 // Not using dlmalloc.c for Windows ARM64 builds
921 // so calling ffi_data_to_code_pointer() isn't necessary
922 unsigned char *tramp_code = tramp;
923 # else
924 unsigned char *tramp_code = ffi_data_to_code_pointer (tramp);
925 # endif
926 ffi_clear_cache (tramp_code, tramp_code + FFI_TRAMPOLINE_SIZE);
927 # if defined(FFI_EXEC_STATIC_TRAMP)
928 out:
929 # endif
930 #endif
931
932 closure->cif = cif;
933 closure->fun = fun;
934 closure->user_data = user_data;
935
936 return FFI_OK;
937 }
938
939 #ifdef FFI_GO_CLOSURES
940 extern void ffi_go_closure_SYSV (void) FFI_HIDDEN;
941 extern void ffi_go_closure_SYSV_V (void) FFI_HIDDEN;
942
943 ffi_status
944 ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif* cif,
945 void (*fun)(ffi_cif*,void*,void**,void*))
946 {
947 void (*start)(void);
948
949 if (cif->abi != FFI_SYSV && cif->abi != FFI_WIN64)
950 return FFI_BAD_ABI;
951
952 if (cif->flags & AARCH64_FLAG_ARG_V)
953 start = ffi_go_closure_SYSV_V;
954 else
955 start = ffi_go_closure_SYSV;
956
957 closure->tramp = start;
958 closure->cif = cif;
959 closure->fun = fun;
960
961 return FFI_OK;
962 }
963 #endif /* FFI_GO_CLOSURES */
964
965 /* Primary handler to setup and invoke a function within a closure.
966
967 A closure when invoked enters via the assembler wrapper
968 ffi_closure_SYSV(). The wrapper allocates a call context on the
969 stack, saves the interesting registers (from the perspective of
970 the calling convention) into the context then passes control to
971 ffi_closure_SYSV_inner() passing the saved context and a pointer to
972 the stack at the point ffi_closure_SYSV() was invoked.
973
974 On the return path the assembler wrapper will reload call context
975 registers.
976
977 ffi_closure_SYSV_inner() marshalls the call context into ffi value
978 descriptors, invokes the wrapped function, then marshalls the return
979 value back into the call context. */
980
981 int FFI_HIDDEN
982 ffi_closure_SYSV_inner (ffi_cif *cif,
983 void (*fun)(ffi_cif*,void*,void**,void*),
984 void *user_data,
985 struct call_context *context,
986 void *stack, void *rvalue, void *struct_rvalue)
987 {
988 void **avalue = (void**) alloca (cif->nargs * sizeof (void*));
989 int i, h, nargs, flags, isvariadic = 0;
990 struct arg_state state;
991
992 arg_init (&state, cif->bytes);
993
994 flags = cif->flags;
995 if (flags & AARCH64_FLAG_VARARG)
996 {
997 isvariadic = 1;
998 flags &= ~AARCH64_FLAG_VARARG;
999 }
1000
1001 for (i = 0, nargs = cif->nargs; i < nargs; i++)
1002 {
1003 ffi_type *ty = cif->arg_types[i];
1004 int t = ty->type;
1005 size_t n, s = ty->size;
1006
1007 switch (t)
1008 {
1009 case FFI_TYPE_VOID:
1010 FFI_ASSERT (0);
1011 break;
1012
1013 case FFI_TYPE_INT:
1014 case FFI_TYPE_UINT8:
1015 case FFI_TYPE_SINT8:
1016 case FFI_TYPE_UINT16:
1017 case FFI_TYPE_SINT16:
1018 case FFI_TYPE_UINT32:
1019 case FFI_TYPE_SINT32:
1020 case FFI_TYPE_UINT64:
1021 case FFI_TYPE_SINT64:
1022 case FFI_TYPE_POINTER:
1023 avalue[i] = allocate_int_to_reg_or_stack (context, &state, stack, s);
1024 break;
1025
1026 case FFI_TYPE_FLOAT:
1027 case FFI_TYPE_DOUBLE:
1028 case FFI_TYPE_LONGDOUBLE:
1029 case FFI_TYPE_STRUCT:
1030 case FFI_TYPE_COMPLEX:
1031 h = is_vfp_type (ty);
1032 if (h)
1033 {
1034 n = 4 - (h & 3);
1035 if (cif->abi == FFI_WIN64 && isvariadic)
1036 {
1037 if (state.ngrn + n <= N_X_ARG_REG)
1038 {
1039 void *reg = &context->x[state.ngrn];
1040 state.ngrn += (unsigned int)n;
1041
1042 /* Eeek! We need a pointer to the structure, however the
1043 homogeneous float elements are being passed in individual
1044 registers, therefore for float and double the structure
1045 is not represented as a contiguous sequence of bytes in
1046 our saved register context. We don't need the original
1047 contents of the register storage, so we reformat the
1048 structure into the same memory. */
1049 avalue[i] = compress_hfa_type(reg, reg, h);
1050 }
1051 else
1052 {
1053 state.ngrn = N_X_ARG_REG;
1054 state.nsrn = N_V_ARG_REG;
1055 avalue[i] = allocate_to_stack(&state, stack,
1056 ty->alignment, s);
1057 }
1058 }
1059 else
1060 {
1061 if (state.nsrn + n <= N_V_ARG_REG)
1062 {
1063 void *reg = &context->v[state.nsrn];
1064 state.nsrn += (unsigned int)n;
1065 avalue[i] = compress_hfa_type(reg, reg, h);
1066 }
1067 else
1068 {
1069 state.nsrn = N_V_ARG_REG;
1070 avalue[i] = allocate_to_stack(&state, stack,
1071 ty->alignment, s);
1072 }
1073 }
1074 }
1075 else if (s > 16)
1076 {
1077 /* Replace Composite type of size greater than 16 with a
1078 pointer. */
1079 #ifdef __ILP32__
1080 UINT64 avalue_tmp;
1081 memcpy (&avalue_tmp,
1082 allocate_int_to_reg_or_stack (context, &state,
1083 stack, sizeof (void *)),
1084 sizeof (UINT64));
1085 avalue[i] = (void *)(UINT32)avalue_tmp;
1086 #else
1087 avalue[i] = *(void **)
1088 allocate_int_to_reg_or_stack (context, &state, stack,
1089 sizeof (void *));
1090 #endif
1091 }
1092 else
1093 {
1094 n = (s + 7) / 8;
1095 if (state.ngrn + n <= N_X_ARG_REG)
1096 {
1097 avalue[i] = &context->x[state.ngrn];
1098 state.ngrn += (unsigned int)n;
1099 }
1100 else
1101 {
1102 state.ngrn = N_X_ARG_REG;
1103 avalue[i] = allocate_to_stack(&state, stack,
1104 ty->alignment, s);
1105 }
1106 }
1107 break;
1108
1109 default:
1110 abort();
1111 }
1112
1113 #if defined (__APPLE__)
1114 if (i + 1 == cif->aarch64_nfixedargs)
1115 {
1116 state.ngrn = N_X_ARG_REG;
1117 state.nsrn = N_V_ARG_REG;
1118 state.allocating_variadic = 1;
1119 }
1120 #endif
1121 }
1122
1123 if (flags & AARCH64_RET_IN_MEM)
1124 rvalue = struct_rvalue;
1125
1126 fun (cif, rvalue, avalue, user_data);
1127
1128 return flags;
1129 }
1130
1131 #if defined(FFI_EXEC_STATIC_TRAMP)
1132 void *
1133 ffi_tramp_arch (size_t *tramp_size, size_t *map_size)
1134 {
1135 extern void *trampoline_code_table;
1136
1137 *tramp_size = AARCH64_TRAMP_SIZE;
1138 *map_size = AARCH64_TRAMP_MAP_SIZE;
1139 return &trampoline_code_table;
1140 }
1141 #endif
1142
1143 #endif /* FFI_CLOSURES */
1144
1145 #endif /* (__aarch64__) || defined(__arm64__)|| defined (_M_ARM64)*/