Debian Codegraph @

src/x86/ffi64.c

raw
1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2011, 2018, 2022 Anthony Green
3 Copyright (c) 2013 The Written Word, Inc.
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <[email protected]>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include <tramp.h>
37 #include "internal64.h"
38
39 #ifdef __x86_64__
40
41 #define MAX_GPR_REGS 6
42 #define MAX_SSE_REGS 8
43
44 #if defined(__INTEL_COMPILER)
45 #include "xmmintrin.h"
46 #define UINT128 __m128
47 #else
48 #if defined(__SUNPRO_C)
49 #include <sunmedia_types.h>
50 #define UINT128 __m128i
51 #else
52 #define UINT128 __int128_t
53 #endif
54 #endif
55
56 union big_int_union
57 {
58 UINT32 i32;
59 UINT64 i64;
60 UINT128 i128;
61 };
62
63 struct register_args
64 {
65 /* Registers for argument passing. */
66 UINT64 gpr[MAX_GPR_REGS];
67 union big_int_union sse[MAX_SSE_REGS];
68 UINT64 rax; /* ssecount */
69 UINT64 r10; /* static chain */
70 };
71
72 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
73 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
74
75 /* All reference to register classes here is identical to the code in
76 gcc/config/i386/i386.c. Do *not* change one without the other. */
77
78 /* Register class used for passing given 64bit part of the argument.
79 These represent classes as documented by the PS ABI, with the
80 exception of SSESF, SSEDF classes, that are basically SSE class,
81 just gcc will use SF or DFmode move instead of DImode to avoid
82 reformatting penalties.
83
84 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
85 whenever possible (upper half does contain padding). */
86 enum x86_64_reg_class
87 {
88 X86_64_NO_CLASS,
89 X86_64_INTEGER_CLASS,
90 X86_64_INTEGERSI_CLASS,
91 X86_64_SSE_CLASS,
92 X86_64_SSESF_CLASS,
93 X86_64_SSEDF_CLASS,
94 X86_64_SSEUP_CLASS,
95 X86_64_X87_CLASS,
96 X86_64_X87UP_CLASS,
97 X86_64_COMPLEX_X87_CLASS,
98 X86_64_MEMORY_CLASS
99 };
100
101 #define MAX_CLASSES 4
102
103 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
104
105 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
106 of this code is to classify each 8bytes of incoming argument by the register
107 class and assign registers accordingly. */
108
109 /* Return the union class of CLASS1 and CLASS2.
110 See the x86-64 PS ABI for details. */
111
112 static enum x86_64_reg_class
113 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
114 {
115 /* Rule #1: If both classes are equal, this is the resulting class. */
116 if (class1 == class2)
117 return class1;
118
119 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
120 the other class. */
121 if (class1 == X86_64_NO_CLASS)
122 return class2;
123 if (class2 == X86_64_NO_CLASS)
124 return class1;
125
126 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
127 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
128 return X86_64_MEMORY_CLASS;
129
130 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
131 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
132 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
133 return X86_64_INTEGERSI_CLASS;
134 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
135 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
136 return X86_64_INTEGER_CLASS;
137
138 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
139 MEMORY is used. */
140 if (class1 == X86_64_X87_CLASS
141 || class1 == X86_64_X87UP_CLASS
142 || class1 == X86_64_COMPLEX_X87_CLASS
143 || class2 == X86_64_X87_CLASS
144 || class2 == X86_64_X87UP_CLASS
145 || class2 == X86_64_COMPLEX_X87_CLASS)
146 return X86_64_MEMORY_CLASS;
147
148 /* Rule #6: Otherwise class SSE is used. */
149 return X86_64_SSE_CLASS;
150 }
151
152 /* Classify the argument of type TYPE and mode MODE.
153 CLASSES will be filled by the register class used to pass each word
154 of the operand. The number of words is returned. In case the parameter
155 should be passed in memory, 0 is returned. As a special case for zero
156 sized containers, classes[0] will be NO_CLASS and 1 is returned.
157
158 See the x86-64 PS ABI for details.
159 */
160 static size_t
161 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
162 size_t byte_offset)
163 {
164 switch (type->type)
165 {
166 case FFI_TYPE_UINT8:
167 case FFI_TYPE_SINT8:
168 case FFI_TYPE_UINT16:
169 case FFI_TYPE_SINT16:
170 case FFI_TYPE_UINT32:
171 case FFI_TYPE_SINT32:
172 case FFI_TYPE_UINT64:
173 case FFI_TYPE_SINT64:
174 case FFI_TYPE_POINTER:
175 do_integer:
176 {
177 size_t size = byte_offset + type->size;
178
179 if (size <= 4)
180 {
181 classes[0] = X86_64_INTEGERSI_CLASS;
182 return 1;
183 }
184 else if (size <= 8)
185 {
186 classes[0] = X86_64_INTEGER_CLASS;
187 return 1;
188 }
189 else if (size <= 12)
190 {
191 classes[0] = X86_64_INTEGER_CLASS;
192 classes[1] = X86_64_INTEGERSI_CLASS;
193 return 2;
194 }
195 else if (size <= 16)
196 {
197 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
198 return 2;
199 }
200 else
201 FFI_ASSERT (0);
202 }
203 case FFI_TYPE_FLOAT:
204 if (!(byte_offset % 8))
205 classes[0] = X86_64_SSESF_CLASS;
206 else
207 classes[0] = X86_64_SSE_CLASS;
208 return 1;
209 case FFI_TYPE_DOUBLE:
210 classes[0] = X86_64_SSEDF_CLASS;
211 return 1;
212 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
213 case FFI_TYPE_LONGDOUBLE:
214 classes[0] = X86_64_X87_CLASS;
215 classes[1] = X86_64_X87UP_CLASS;
216 return 2;
217 #endif
218 case FFI_TYPE_STRUCT:
219 {
220 const size_t UNITS_PER_WORD = 8;
221 size_t words = (type->size + byte_offset + UNITS_PER_WORD - 1)
222 / UNITS_PER_WORD;
223 ffi_type **ptr;
224 unsigned int i;
225 enum x86_64_reg_class subclasses[MAX_CLASSES];
226
227 /* If the struct is larger than 32 bytes, pass it on the stack. */
228 if (type->size > 32)
229 return 0;
230
231 for (i = 0; i < words; i++)
232 classes[i] = X86_64_NO_CLASS;
233
234 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
235 signalize memory class, so handle it as special case. */
236 if (!words)
237 {
238 case FFI_TYPE_VOID:
239 classes[0] = X86_64_NO_CLASS;
240 return 1;
241 }
242
243 /* Merge the fields of structure. */
244 for (ptr = type->elements; *ptr != NULL; ptr++)
245 {
246 size_t num, pos;
247
248 byte_offset = FFI_ALIGN (byte_offset, (*ptr)->alignment);
249
250 num = classify_argument (*ptr, subclasses, byte_offset % 8);
251 if (num == 0)
252 return 0;
253 pos = byte_offset / 8;
254 for (i = 0; i < num && (i + pos) < words; i++)
255 {
256 size_t pos = byte_offset / 8;
257 classes[i + pos] =
258 merge_classes (subclasses[i], classes[i + pos]);
259 }
260
261 byte_offset += (*ptr)->size;
262 }
263
264 if (words > 2)
265 {
266 /* When size > 16 bytes, if the first one isn't
267 X86_64_SSE_CLASS or any other ones aren't
268 X86_64_SSEUP_CLASS, everything should be passed in
269 memory. */
270 if (classes[0] != X86_64_SSE_CLASS)
271 return 0;
272
273 for (i = 1; i < words; i++)
274 if (classes[i] != X86_64_SSEUP_CLASS)
275 return 0;
276 }
277
278 /* Final merger cleanup. */
279 for (i = 0; i < words; i++)
280 {
281 /* If one class is MEMORY, everything should be passed in
282 memory. */
283 if (classes[i] == X86_64_MEMORY_CLASS)
284 return 0;
285
286 /* The X86_64_SSEUP_CLASS should be always preceded by
287 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
288 if (i > 1 && classes[i] == X86_64_SSEUP_CLASS
289 && classes[i - 1] != X86_64_SSE_CLASS
290 && classes[i - 1] != X86_64_SSEUP_CLASS)
291 {
292 /* The first one should never be X86_64_SSEUP_CLASS. */
293 FFI_ASSERT (i != 0);
294 classes[i] = X86_64_SSE_CLASS;
295 }
296
297 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
298 everything should be passed in memory. */
299 if (i > 1 && classes[i] == X86_64_X87UP_CLASS
300 && (classes[i - 1] != X86_64_X87_CLASS))
301 {
302 /* The first one should never be X86_64_X87UP_CLASS. */
303 FFI_ASSERT (i != 0);
304 return 0;
305 }
306 }
307 return words;
308 }
309 case FFI_TYPE_COMPLEX:
310 {
311 ffi_type *inner = type->elements[0];
312 switch (inner->type)
313 {
314 case FFI_TYPE_INT:
315 case FFI_TYPE_UINT8:
316 case FFI_TYPE_SINT8:
317 case FFI_TYPE_UINT16:
318 case FFI_TYPE_SINT16:
319 case FFI_TYPE_UINT32:
320 case FFI_TYPE_SINT32:
321 case FFI_TYPE_UINT64:
322 case FFI_TYPE_SINT64:
323 goto do_integer;
324
325 case FFI_TYPE_FLOAT:
326 classes[0] = X86_64_SSE_CLASS;
327 if (byte_offset % 8)
328 {
329 classes[1] = X86_64_SSESF_CLASS;
330 return 2;
331 }
332 return 1;
333 case FFI_TYPE_DOUBLE:
334 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
335 return 2;
336 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
337 case FFI_TYPE_LONGDOUBLE:
338 classes[0] = X86_64_COMPLEX_X87_CLASS;
339 return 1;
340 #endif
341 }
342 }
343 }
344 abort();
345 }
346
347 /* Examine the argument and return set number of register required in each
348 class. Return zero iff parameter should be passed in memory, otherwise
349 the number of registers. */
350
351 static size_t
352 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
353 _Bool in_return, int *pngpr, int *pnsse)
354 {
355 size_t n;
356 unsigned int i;
357 int ngpr, nsse;
358
359 n = classify_argument (type, classes, 0);
360 if (n == 0)
361 return 0;
362
363 ngpr = nsse = 0;
364 for (i = 0; i < n; ++i)
365 switch (classes[i])
366 {
367 case X86_64_INTEGER_CLASS:
368 case X86_64_INTEGERSI_CLASS:
369 ngpr++;
370 break;
371 case X86_64_SSE_CLASS:
372 case X86_64_SSESF_CLASS:
373 case X86_64_SSEDF_CLASS:
374 nsse++;
375 break;
376 case X86_64_NO_CLASS:
377 case X86_64_SSEUP_CLASS:
378 break;
379 case X86_64_X87_CLASS:
380 case X86_64_X87UP_CLASS:
381 case X86_64_COMPLEX_X87_CLASS:
382 return in_return != 0;
383 default:
384 abort ();
385 }
386
387 *pngpr = ngpr;
388 *pnsse = nsse;
389
390 return n;
391 }
392
393 /* Perform machine dependent cif processing. */
394
395 #ifndef __ILP32__
396 extern ffi_status
397 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
398 #endif
399
400 ffi_status FFI_HIDDEN
401 ffi_prep_cif_machdep (ffi_cif *cif)
402 {
403 int gprcount, ssecount, i, avn, ngpr, nsse;
404 unsigned flags;
405 enum x86_64_reg_class classes[MAX_CLASSES];
406 size_t bytes, n, rtype_size;
407 ffi_type *rtype;
408
409 #ifndef __ILP32__
410 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
411 return ffi_prep_cif_machdep_efi64(cif);
412 #endif
413 if (cif->abi != FFI_UNIX64)
414 return FFI_BAD_ABI;
415
416 gprcount = ssecount = 0;
417
418 rtype = cif->rtype;
419 rtype_size = rtype->size;
420 switch (rtype->type)
421 {
422 case FFI_TYPE_VOID:
423 flags = UNIX64_RET_VOID;
424 break;
425 case FFI_TYPE_UINT8:
426 flags = UNIX64_RET_UINT8;
427 break;
428 case FFI_TYPE_SINT8:
429 flags = UNIX64_RET_SINT8;
430 break;
431 case FFI_TYPE_UINT16:
432 flags = UNIX64_RET_UINT16;
433 break;
434 case FFI_TYPE_SINT16:
435 flags = UNIX64_RET_SINT16;
436 break;
437 case FFI_TYPE_UINT32:
438 flags = UNIX64_RET_UINT32;
439 break;
440 case FFI_TYPE_INT:
441 case FFI_TYPE_SINT32:
442 flags = UNIX64_RET_SINT32;
443 break;
444 case FFI_TYPE_UINT64:
445 case FFI_TYPE_SINT64:
446 flags = UNIX64_RET_INT64;
447 break;
448 case FFI_TYPE_POINTER:
449 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
450 break;
451 case FFI_TYPE_FLOAT:
452 flags = UNIX64_RET_XMM32;
453 break;
454 case FFI_TYPE_DOUBLE:
455 flags = UNIX64_RET_XMM64;
456 break;
457 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
458 case FFI_TYPE_LONGDOUBLE:
459 flags = UNIX64_RET_X87;
460 break;
461 #endif
462 case FFI_TYPE_STRUCT:
463 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
464 if (n == 0)
465 {
466 /* The return value is passed in memory. A pointer to that
467 memory is the first argument. Allocate a register for it. */
468 gprcount++;
469 /* We don't have to do anything in asm for the return. */
470 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
471 }
472 else
473 {
474 _Bool sse0 = SSE_CLASS_P (classes[0]);
475
476 if (rtype_size == 4 && sse0)
477 flags = UNIX64_RET_XMM32;
478 else if (rtype_size == 8)
479 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
480 else
481 {
482 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
483 if (sse0 && sse1)
484 flags = UNIX64_RET_ST_XMM0_XMM1;
485 else if (sse0)
486 flags = UNIX64_RET_ST_XMM0_RAX;
487 else if (sse1)
488 flags = UNIX64_RET_ST_RAX_XMM0;
489 else
490 flags = UNIX64_RET_ST_RAX_RDX;
491 flags |= rtype_size << UNIX64_SIZE_SHIFT;
492 }
493 }
494 break;
495 case FFI_TYPE_COMPLEX:
496 switch (rtype->elements[0]->type)
497 {
498 case FFI_TYPE_UINT8:
499 case FFI_TYPE_SINT8:
500 case FFI_TYPE_UINT16:
501 case FFI_TYPE_SINT16:
502 case FFI_TYPE_INT:
503 case FFI_TYPE_UINT32:
504 case FFI_TYPE_SINT32:
505 case FFI_TYPE_UINT64:
506 case FFI_TYPE_SINT64:
507 flags = UNIX64_RET_ST_RAX_RDX | ((unsigned) rtype_size << UNIX64_SIZE_SHIFT);
508 break;
509 case FFI_TYPE_FLOAT:
510 flags = UNIX64_RET_XMM64;
511 break;
512 case FFI_TYPE_DOUBLE:
513 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
514 break;
515 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
516 case FFI_TYPE_LONGDOUBLE:
517 flags = UNIX64_RET_X87_2;
518 break;
519 #endif
520 default:
521 return FFI_BAD_TYPEDEF;
522 }
523 break;
524 default:
525 return FFI_BAD_TYPEDEF;
526 }
527
528 /* Go over all arguments and determine the way they should be passed.
529 If it's in a register and there is space for it, let that be so. If
530 not, add it's size to the stack byte count. */
531 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
532 {
533 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
534 || gprcount + ngpr > MAX_GPR_REGS
535 || ssecount + nsse > MAX_SSE_REGS)
536 {
537 long align = cif->arg_types[i]->alignment;
538
539 if (align < 8)
540 align = 8;
541
542 bytes = FFI_ALIGN (bytes, align);
543 bytes += cif->arg_types[i]->size;
544 }
545 else
546 {
547 gprcount += ngpr;
548 ssecount += nsse;
549 }
550 }
551 if (ssecount)
552 flags |= UNIX64_FLAG_XMM_ARGS;
553
554 cif->flags = flags;
555 cif->bytes = (unsigned) FFI_ALIGN (bytes, 8);
556
557 return FFI_OK;
558 }
559
560 /* n.b. ffi_call_unix64 will steal the alloca'd `stack` variable here for use
561 _as its own stack_ - so we need to compile this function without ASAN */
562 FFI_ASAN_NO_SANITIZE
563 static void
564 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
565 void **avalue, void *closure)
566 {
567 enum x86_64_reg_class classes[MAX_CLASSES];
568 char *stack, *argp;
569 ffi_type **arg_types;
570 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
571 struct register_args *reg_args;
572
573 /* Can't call 32-bit mode from 64-bit mode. */
574 FFI_ASSERT (cif->abi == FFI_UNIX64);
575
576 /* If the return value is a struct and we don't have a return value
577 address then we need to make one. Otherwise we can ignore it. */
578 flags = cif->flags;
579 if (rvalue == NULL)
580 {
581 if (flags & UNIX64_FLAG_RET_IN_MEM)
582 rvalue = alloca (cif->rtype->size);
583 else
584 flags = UNIX64_RET_VOID;
585 }
586
587 arg_types = cif->arg_types;
588 avn = cif->nargs;
589
590 /* Allocate the space for the arguments, plus 4 words of temp space. */
591 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
592 reg_args = (struct register_args *) stack;
593 argp = stack + sizeof (struct register_args);
594
595 reg_args->r10 = (uintptr_t) closure;
596
597 gprcount = ssecount = 0;
598
599 /* If the return value is passed in memory, add the pointer as the
600 first integer argument. */
601 if (flags & UNIX64_FLAG_RET_IN_MEM)
602 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
603
604 for (i = 0; i < avn; ++i)
605 {
606 size_t n, size = arg_types[i]->size;
607
608 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
609 if (n == 0
610 || gprcount + ngpr > MAX_GPR_REGS
611 || ssecount + nsse > MAX_SSE_REGS)
612 {
613 long align = arg_types[i]->alignment;
614
615 /* Stack arguments are *always* at least 8 byte aligned. */
616 if (align < 8)
617 align = 8;
618
619 /* Pass this argument in memory. */
620 argp = (void *) FFI_ALIGN (argp, align);
621 memcpy (argp, avalue[i], size);
622
623 argp += size;
624 }
625 else
626 {
627 /* The argument is passed entirely in registers. */
628 char *a = (char *) avalue[i];
629 unsigned int j;
630
631 for (j = 0; j < n; j++, a += 8, size -= 8)
632 {
633 switch (classes[j])
634 {
635 case X86_64_NO_CLASS:
636 case X86_64_SSEUP_CLASS:
637 break;
638 case X86_64_INTEGER_CLASS:
639 case X86_64_INTEGERSI_CLASS:
640 /* Sign-extend integer arguments passed in general
641 purpose registers, to cope with the fact that
642 LLVM incorrectly assumes that this will be done
643 (the x86-64 PS ABI does not specify this). */
644 switch (arg_types[i]->type)
645 {
646 case FFI_TYPE_SINT8:
647 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
648 break;
649 case FFI_TYPE_SINT16:
650 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
651 break;
652 case FFI_TYPE_SINT32:
653 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
654 break;
655 default:
656 reg_args->gpr[gprcount] = 0;
657 memcpy (&reg_args->gpr[gprcount], a, size <= 8 ? size : 8);
658 }
659 gprcount++;
660 break;
661 case X86_64_SSE_CLASS:
662 case X86_64_SSEDF_CLASS:
663 memcpy (&reg_args->sse[ssecount++].i64, a, sizeof(UINT64));
664 break;
665 case X86_64_SSESF_CLASS:
666 memcpy (&reg_args->sse[ssecount++].i32, a, sizeof(UINT32));
667 break;
668 default:
669 abort();
670 }
671 }
672 }
673 }
674 reg_args->rax = ssecount;
675
676 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
677 flags, rvalue, fn);
678 }
679
680 #ifndef __ILP32__
681 extern void
682 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
683 #endif
684
685 void
686 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
687 {
688 ffi_type **arg_types = cif->arg_types;
689 int i, nargs = cif->nargs;
690 const int max_reg_struct_size = cif->abi == FFI_GNUW64 ? 8 : 16;
691
692 /* If we have any large structure arguments, make a copy so we are passing
693 by value. */
694 for (i = 0; i < nargs; i++)
695 {
696 ffi_type *at = arg_types[i];
697 int size = at->size;
698 if (at->type == FFI_TYPE_STRUCT && size > max_reg_struct_size)
699 {
700 char *argcopy = alloca (size);
701 memcpy (argcopy, avalue[i], size);
702 avalue[i] = argcopy;
703 }
704 }
705
706 #ifndef __ILP32__
707 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
708 {
709 ffi_call_efi64(cif, fn, rvalue, avalue);
710 return;
711 }
712 #endif
713 ffi_call_int (cif, fn, rvalue, avalue, NULL);
714 }
715
716 #ifdef FFI_GO_CLOSURES
717
718 #ifndef __ILP32__
719 extern void
720 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
721 void **avalue, void *closure);
722 #endif
723
724 void
725 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
726 void **avalue, void *closure)
727 {
728 #ifndef __ILP32__
729 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
730 {
731 ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
732 return;
733 }
734 #endif
735 ffi_call_int (cif, fn, rvalue, avalue, closure);
736 }
737
738 #endif /* FFI_GO_CLOSURES */
739
740 extern void ffi_closure_unix64(void) FFI_HIDDEN;
741 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
742 #if defined(FFI_EXEC_STATIC_TRAMP)
743 extern void ffi_closure_unix64_alt(void) FFI_HIDDEN;
744 extern void ffi_closure_unix64_sse_alt(void) FFI_HIDDEN;
745 #endif
746
747 #ifndef __ILP32__
748 extern ffi_status
749 ffi_prep_closure_loc_efi64(ffi_closure* closure,
750 ffi_cif* cif,
751 void (*fun)(ffi_cif*, void*, void**, void*),
752 void *user_data,
753 void *codeloc);
754 #endif
755
756 ffi_status
757 ffi_prep_closure_loc (ffi_closure* closure,
758 ffi_cif* cif,
759 void (*fun)(ffi_cif*, void*, void**, void*),
760 void *user_data,
761 void *codeloc)
762 {
763 static const unsigned char trampoline[24] = {
764 /* endbr64 */
765 0xf3, 0x0f, 0x1e, 0xfa,
766 /* leaq -0xb(%rip),%r10 # 0x0 */
767 0x4c, 0x8d, 0x15, 0xf5, 0xff, 0xff, 0xff,
768 /* jmpq *0x7(%rip) # 0x18 */
769 0xff, 0x25, 0x07, 0x00, 0x00, 0x00,
770 /* nopl 0(%rax) */
771 0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00
772 };
773 void (*dest)(void);
774 char *tramp = closure->tramp;
775
776 #ifndef __ILP32__
777 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
778 return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
779 #endif
780 if (cif->abi != FFI_UNIX64)
781 return FFI_BAD_ABI;
782
783 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
784 dest = ffi_closure_unix64_sse;
785 else
786 dest = ffi_closure_unix64;
787
788 #if defined(FFI_EXEC_STATIC_TRAMP)
789 if (ffi_tramp_is_present(closure))
790 {
791 /* Initialize the static trampoline's parameters. */
792 if (dest == ffi_closure_unix64_sse)
793 dest = ffi_closure_unix64_sse_alt;
794 else
795 dest = ffi_closure_unix64_alt;
796 ffi_tramp_set_parms (closure->ftramp, dest, closure);
797 goto out;
798 }
799 #endif
800
801 /* Initialize the dynamic trampoline. */
802 memcpy (tramp, trampoline, sizeof(trampoline));
803 *(UINT64 *)(tramp + sizeof (trampoline)) = (uintptr_t)dest;
804
805 #if defined(FFI_EXEC_STATIC_TRAMP)
806 out:
807 #endif
808 closure->cif = cif;
809 closure->fun = fun;
810 closure->user_data = user_data;
811
812 return FFI_OK;
813 }
814
815 int FFI_HIDDEN
816 ffi_closure_unix64_inner(ffi_cif *cif,
817 void (*fun)(ffi_cif*, void*, void**, void*),
818 void *user_data,
819 void *rvalue,
820 struct register_args *reg_args,
821 char *argp)
822 {
823 void **avalue;
824 ffi_type **arg_types;
825 long i, avn;
826 int gprcount, ssecount, ngpr, nsse;
827 int flags;
828
829 avn = cif->nargs;
830 flags = cif->flags;
831 avalue = alloca(avn * sizeof(void *));
832 gprcount = ssecount = 0;
833
834 if (flags & UNIX64_FLAG_RET_IN_MEM)
835 {
836 /* On return, %rax will contain the address that was passed
837 by the caller in %rdi. */
838 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
839 *(void **)rvalue = r;
840 rvalue = r;
841 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
842 }
843
844 arg_types = cif->arg_types;
845 for (i = 0; i < avn; ++i)
846 {
847 enum x86_64_reg_class classes[MAX_CLASSES];
848 size_t n;
849
850 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
851 if (n == 0
852 || gprcount + ngpr > MAX_GPR_REGS
853 || ssecount + nsse > MAX_SSE_REGS)
854 {
855 long align = arg_types[i]->alignment;
856
857 /* Stack arguments are *always* at least 8 byte aligned. */
858 if (align < 8)
859 align = 8;
860
861 /* Pass this argument in memory. */
862 argp = (void *) FFI_ALIGN (argp, align);
863 avalue[i] = argp;
864 argp += arg_types[i]->size;
865 }
866 /* If the argument is in a single register, or two consecutive
867 integer registers, then we can use that address directly. */
868 else if (n == 1
869 || (n == 2 && !(SSE_CLASS_P (classes[0])
870 || SSE_CLASS_P (classes[1]))))
871 {
872 /* The argument is in a single register. */
873 if (SSE_CLASS_P (classes[0]))
874 {
875 avalue[i] = &reg_args->sse[ssecount];
876 ssecount += n;
877 }
878 else
879 {
880 avalue[i] = &reg_args->gpr[gprcount];
881 gprcount += n;
882 }
883 }
884 /* Otherwise, allocate space to make them consecutive. */
885 else
886 {
887 char *a = alloca (16);
888 unsigned int j;
889
890 avalue[i] = a;
891 for (j = 0; j < n; j++, a += 8)
892 {
893 if (SSE_CLASS_P (classes[j]))
894 memcpy (a, &reg_args->sse[ssecount++], 8);
895 else
896 memcpy (a, &reg_args->gpr[gprcount++], 8);
897 }
898 }
899 }
900
901 /* Invoke the closure. */
902 fun (cif, rvalue, avalue, user_data);
903
904 /* Tell assembly how to perform return type promotions. */
905 return flags;
906 }
907
908 #ifdef FFI_GO_CLOSURES
909
910 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
911 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
912
913 #ifndef __ILP32__
914 extern ffi_status
915 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
916 void (*fun)(ffi_cif*, void*, void**, void*));
917 #endif
918
919 ffi_status
920 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
921 void (*fun)(ffi_cif*, void*, void**, void*))
922 {
923 #ifndef __ILP32__
924 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
925 return ffi_prep_go_closure_efi64(closure, cif, fun);
926 #endif
927 if (cif->abi != FFI_UNIX64)
928 return FFI_BAD_ABI;
929
930 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
931 ? ffi_go_closure_unix64_sse
932 : ffi_go_closure_unix64);
933 closure->cif = cif;
934 closure->fun = fun;
935
936 return FFI_OK;
937 }
938
939 #endif /* FFI_GO_CLOSURES */
940
941 #if defined(FFI_EXEC_STATIC_TRAMP)
942 void *
943 ffi_tramp_arch (size_t *tramp_size, size_t *map_size)
944 {
945 extern void *trampoline_code_table;
946
947 *map_size = UNIX64_TRAMP_MAP_SIZE;
948 *tramp_size = UNIX64_TRAMP_SIZE;
949 return &trampoline_code_table;
950 }
951 #endif
952
953 #endif /* __x86_64__ */