Debian Codegraph @

src/powerpc/ffi_linux64.c

raw
1 /* -----------------------------------------------------------------------
2 ffi_linux64.c - Copyright (C) 2013 IBM
3 Copyright (C) 2011 Anthony Green
4 Copyright (C) 2011 Kyle Moffett
5 Copyright (C) 2008 Red Hat, Inc
6 Copyright (C) 2007, 2008 Free Software Foundation, Inc
7 Copyright (c) 1998 Geoffrey Keating
8
9 PowerPC Foreign Function Interface
10
11 Permission is hereby granted, free of charge, to any person obtaining
12 a copy of this software and associated documentation files (the
13 ``Software''), to deal in the Software without restriction, including
14 without limitation the rights to use, copy, modify, merge, publish,
15 distribute, sublicense, and/or sell copies of the Software, and to
16 permit persons to whom the Software is furnished to do so, subject to
17 the following conditions:
18
19 The above copyright notice and this permission notice shall be included
20 in all copies or substantial portions of the Software.
21
22 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
25 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
26 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
27 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 OTHER DEALINGS IN THE SOFTWARE.
29 ----------------------------------------------------------------------- */
30
31 #include "ffi.h"
32 #include <tramp.h>
33
34 #ifdef POWERPC64
35 #include "ffi_common.h"
36 #include "ffi_powerpc.h"
37
38
39 /* About the LINUX64 ABI. */
40 enum {
41 NUM_GPR_ARG_REGISTERS64 = 8,
42 NUM_FPR_ARG_REGISTERS64 = 13,
43 NUM_VEC_ARG_REGISTERS64 = 12,
44 };
45 enum { ASM_NEEDS_REGISTERS64 = 4 };
46
47
48 #if HAVE_LONG_DOUBLE_VARIANT && FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
49 /* Adjust size of ffi_type_longdouble. */
50 void FFI_HIDDEN
51 ffi_prep_types_linux64 (ffi_abi abi)
52 {
53 if ((abi & (FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128)) == FFI_LINUX)
54 {
55 ffi_type_longdouble.size = 8;
56 ffi_type_longdouble.alignment = 8;
57 }
58 else
59 {
60 ffi_type_longdouble.size = 16;
61 ffi_type_longdouble.alignment = 16;
62 }
63 }
64 #endif
65
66
67 static unsigned int
68 discover_homogeneous_aggregate (ffi_abi abi,
69 const ffi_type *t,
70 unsigned int *elnum)
71 {
72 switch (t->type)
73 {
74 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
75 case FFI_TYPE_LONGDOUBLE:
76 /* 64-bit long doubles are equivalent to doubles. */
77 if ((abi & FFI_LINUX_LONG_DOUBLE_128) == 0)
78 {
79 *elnum = 1;
80 return FFI_TYPE_DOUBLE;
81 }
82 /* IBM extended precision values use unaligned pairs
83 of FPRs, but according to the ABI must be considered
84 distinct from doubles. They are also limited to a
85 maximum of four members in a homogeneous aggregate. */
86 else if ((abi & FFI_LINUX_LONG_DOUBLE_IEEE128) == 0)
87 {
88 *elnum = 2;
89 return FFI_TYPE_LONGDOUBLE;
90 }
91 /* Fall through. */
92 #endif
93 case FFI_TYPE_FLOAT:
94 case FFI_TYPE_DOUBLE:
95 *elnum = 1;
96 return (int) t->type;
97
98 case FFI_TYPE_STRUCT:;
99 {
100 unsigned int base_elt = 0, total_elnum = 0;
101 ffi_type **el = t->elements;
102 while (*el)
103 {
104 unsigned int el_elt, el_elnum = 0;
105 el_elt = discover_homogeneous_aggregate (abi, *el, &el_elnum);
106 if (el_elt == 0
107 || (base_elt && base_elt != el_elt))
108 return 0;
109 base_elt = el_elt;
110 total_elnum += el_elnum;
111 #if _CALL_ELF == 2
112 if (total_elnum > 8)
113 return 0;
114 #else
115 if (total_elnum > 1)
116 return 0;
117 #endif
118 el++;
119 }
120 *elnum = total_elnum;
121 return base_elt;
122 }
123
124 default:
125 return 0;
126 }
127 }
128
129
130 /* Perform machine dependent cif processing */
131 static ffi_status
132 ffi_prep_cif_linux64_core (ffi_cif *cif)
133 {
134 ffi_type **ptr;
135 unsigned bytes;
136 unsigned i, fparg_count = 0, intarg_count = 0, vecarg_count = 0;
137 unsigned flags = cif->flags;
138 unsigned elt, elnum, rtype;
139
140 #if FFI_TYPE_LONGDOUBLE == FFI_TYPE_DOUBLE
141 /* If compiled without long double support... */
142 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0 ||
143 (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
144 return FFI_BAD_ABI;
145 #elif !defined(__VEC__)
146 /* If compiled without vector register support (used by assembly)... */
147 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
148 return FFI_BAD_ABI;
149 #else
150 /* If the IEEE128 flag is set, but long double is only 64 bits wide... */
151 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) == 0 &&
152 (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
153 return FFI_BAD_ABI;
154 #endif
155
156 /* The machine-independent calculation of cif->bytes doesn't work
157 for us. Redo the calculation. */
158 #if _CALL_ELF == 2
159 /* Space for backchain, CR, LR, TOC and the asm's temp regs. */
160 bytes = (4 + ASM_NEEDS_REGISTERS64) * sizeof (long);
161
162 /* Space for the general registers. */
163 bytes += NUM_GPR_ARG_REGISTERS64 * sizeof (long);
164 #else
165 /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
166 regs. */
167 bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof (long);
168
169 /* Space for the mandatory parm save area and general registers. */
170 bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof (long);
171 #endif
172
173 /* Return value handling. */
174 rtype = cif->rtype->type;
175 #if _CALL_ELF == 2
176 homogeneous:
177 #endif
178 switch (rtype)
179 {
180 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
181 case FFI_TYPE_LONGDOUBLE:
182 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
183 {
184 flags |= FLAG_RETURNS_VEC;
185 break;
186 }
187 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
188 flags |= FLAG_RETURNS_128BITS;
189 /* Fall through. */
190 #endif
191 case FFI_TYPE_DOUBLE:
192 flags |= FLAG_RETURNS_64BITS;
193 /* Fall through. */
194 case FFI_TYPE_FLOAT:
195 flags |= FLAG_RETURNS_FP;
196 break;
197
198 case FFI_TYPE_UINT128:
199 flags |= FLAG_RETURNS_128BITS;
200 /* Fall through. */
201 case FFI_TYPE_UINT64:
202 case FFI_TYPE_SINT64:
203 case FFI_TYPE_POINTER:
204 flags |= FLAG_RETURNS_64BITS;
205 break;
206
207 case FFI_TYPE_STRUCT:
208 #if _CALL_ELF == 2
209 elt = discover_homogeneous_aggregate (cif->abi, cif->rtype, &elnum);
210 if (elt)
211 {
212 flags |= FLAG_RETURNS_SMST;
213 rtype = elt;
214 goto homogeneous;
215 }
216 if (cif->rtype->size <= 16)
217 {
218 flags |= FLAG_RETURNS_SMST;
219 break;
220 }
221 #endif
222 intarg_count++;
223 flags |= FLAG_RETVAL_REFERENCE;
224 /* Fall through. */
225 case FFI_TYPE_VOID:
226 flags |= FLAG_RETURNS_NOTHING;
227 break;
228
229 default:
230 /* Returns 32-bit integer, or similar. Nothing to do here. */
231 break;
232 }
233
234 for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
235 {
236 unsigned int align;
237
238 switch ((*ptr)->type)
239 {
240 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
241 case FFI_TYPE_LONGDOUBLE:
242 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
243 {
244 vecarg_count++;
245 /* Align to 16 bytes, plus the 16-byte argument. */
246 intarg_count = (intarg_count + 3) & ~0x1;
247 if (vecarg_count > NUM_VEC_ARG_REGISTERS64)
248 flags |= FLAG_ARG_NEEDS_PSAVE;
249 break;
250 }
251 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
252 {
253 fparg_count++;
254 intarg_count++;
255 }
256 /* Fall through. */
257 #endif
258 case FFI_TYPE_DOUBLE:
259 case FFI_TYPE_FLOAT:
260 fparg_count++;
261 intarg_count++;
262 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
263 flags |= FLAG_ARG_NEEDS_PSAVE;
264 break;
265
266 case FFI_TYPE_STRUCT:
267 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
268 {
269 align = (*ptr)->alignment;
270 if (align > 16)
271 align = 16;
272 align = align / 8;
273 if (align > 1)
274 intarg_count = FFI_ALIGN (intarg_count, align);
275 }
276 intarg_count += ((*ptr)->size + 7) / 8;
277 elt = discover_homogeneous_aggregate (cif->abi, *ptr, &elnum);
278 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
279 if (elt == FFI_TYPE_LONGDOUBLE &&
280 (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
281 {
282 vecarg_count += elnum;
283 if (vecarg_count > NUM_VEC_ARG_REGISTERS64)
284 flags |= FLAG_ARG_NEEDS_PSAVE;
285 break;
286 }
287 else
288 #endif
289 if (elt)
290 {
291 fparg_count += elnum;
292 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
293 flags |= FLAG_ARG_NEEDS_PSAVE;
294 }
295 else
296 {
297 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
298 flags |= FLAG_ARG_NEEDS_PSAVE;
299 }
300 break;
301
302 case FFI_TYPE_POINTER:
303 case FFI_TYPE_UINT64:
304 case FFI_TYPE_SINT64:
305 case FFI_TYPE_INT:
306 case FFI_TYPE_UINT32:
307 case FFI_TYPE_SINT32:
308 case FFI_TYPE_UINT16:
309 case FFI_TYPE_SINT16:
310 case FFI_TYPE_UINT8:
311 case FFI_TYPE_SINT8:
312 /* Everything else is passed as a 8-byte word in a GPR, either
313 the object itself or a pointer to it. */
314 intarg_count++;
315 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
316 flags |= FLAG_ARG_NEEDS_PSAVE;
317 break;
318 default:
319 FFI_ASSERT (0);
320 }
321 }
322
323 if (fparg_count != 0)
324 flags |= FLAG_FP_ARGUMENTS;
325 if (intarg_count > 4)
326 flags |= FLAG_4_GPR_ARGUMENTS;
327 if (vecarg_count != 0)
328 flags |= FLAG_VEC_ARGUMENTS;
329
330 /* Space for the FPR registers, if needed. */
331 if (fparg_count != 0)
332 bytes += NUM_FPR_ARG_REGISTERS64 * sizeof (double);
333 /* Space for the vector registers, if needed, aligned to 16 bytes. */
334 if (vecarg_count != 0) {
335 bytes = (bytes + 15) & ~0xF;
336 bytes += NUM_VEC_ARG_REGISTERS64 * sizeof (float128);
337 }
338
339 /* Stack space. */
340 #if _CALL_ELF == 2
341 if ((flags & FLAG_ARG_NEEDS_PSAVE) != 0)
342 bytes += intarg_count * sizeof (long);
343 #else
344 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
345 bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof (long);
346 #endif
347
348 /* The stack space allocated needs to be a multiple of 16 bytes. */
349 bytes = (bytes + 15) & ~0xF;
350
351 cif->flags = flags;
352 cif->bytes = bytes;
353
354 return FFI_OK;
355 }
356
357 ffi_status FFI_HIDDEN
358 ffi_prep_cif_linux64 (ffi_cif *cif)
359 {
360 if ((cif->abi & FFI_LINUX) != 0)
361 cif->nfixedargs = cif->nargs;
362 #if _CALL_ELF != 2
363 else if (cif->abi == FFI_COMPAT_LINUX64)
364 {
365 /* This call is from old code. Don't touch cif->nfixedargs
366 since old code will be using a smaller cif. */
367 cif->flags |= FLAG_COMPAT;
368 /* Translate to new abi value. */
369 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
370 }
371 #endif
372 else
373 return FFI_BAD_ABI;
374 return ffi_prep_cif_linux64_core (cif);
375 }
376
377 ffi_status FFI_HIDDEN
378 ffi_prep_cif_linux64_var (ffi_cif *cif,
379 unsigned int nfixedargs,
380 unsigned int ntotalargs MAYBE_UNUSED)
381 {
382 if ((cif->abi & FFI_LINUX) != 0)
383 cif->nfixedargs = nfixedargs;
384 #if _CALL_ELF != 2
385 else if (cif->abi == FFI_COMPAT_LINUX64)
386 {
387 /* This call is from old code. Don't touch cif->nfixedargs
388 since old code will be using a smaller cif. */
389 cif->flags |= FLAG_COMPAT;
390 /* Translate to new abi value. */
391 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
392 }
393 #endif
394 else
395 return FFI_BAD_ABI;
396 #if _CALL_ELF == 2
397 cif->flags |= FLAG_ARG_NEEDS_PSAVE;
398 #endif
399 return ffi_prep_cif_linux64_core (cif);
400 }
401
402
403 /* ffi_prep_args64 is called by the assembly routine once stack space
404 has been allocated for the function's arguments.
405
406 The stack layout we want looks like this:
407
408 | Ret addr from ffi_call_LINUX64 8bytes | higher addresses
409 |--------------------------------------------|
410 | CR save area 8bytes |
411 |--------------------------------------------|
412 | Previous backchain pointer 8 | stack pointer here
413 |--------------------------------------------|<+ <<< on entry to
414 | Saved r28-r31 4*8 | | ffi_call_LINUX64
415 |--------------------------------------------| |
416 | GPR registers r3-r10 8*8 | |
417 |--------------------------------------------| |
418 | FPR registers f1-f13 (optional) 13*8 | |
419 |--------------------------------------------| |
420 | VEC registers v2-v13 (optional) 12*16 | |
421 |--------------------------------------------| |
422 | Parameter save area | |
423 |--------------------------------------------| |
424 | TOC save area 8 | |
425 |--------------------------------------------| | stack |
426 | Linker doubleword 8 | | grows |
427 |--------------------------------------------| | down V
428 | Compiler doubleword 8 | |
429 |--------------------------------------------| | lower addresses
430 | Space for callee's LR 8 | |
431 |--------------------------------------------| |
432 | CR save area 8 | |
433 |--------------------------------------------| | stack pointer here
434 | Current backchain pointer 8 |-/ during
435 |--------------------------------------------| <<< ffi_call_LINUX64
436
437 */
438
439 void FFI_HIDDEN
440 ffi_prep_args64 (extended_cif *ecif, unsigned long *const stack)
441 {
442 const unsigned long bytes = ecif->cif->bytes;
443 const unsigned long flags = ecif->cif->flags;
444
445 typedef union
446 {
447 char *c;
448 unsigned long *ul;
449 float *f;
450 double *d;
451 float128 *f128;
452 size_t p;
453 } valp;
454
455 /* 'stacktop' points at the previous backchain pointer. */
456 valp stacktop;
457
458 /* 'next_arg' points at the space for gpr3, and grows upwards as
459 we use GPR registers, then continues at rest. */
460 valp gpr_base;
461 valp gpr_end;
462 valp rest;
463 valp next_arg;
464
465 /* 'fpr_base' points at the space for f1, and grows upwards as
466 we use FPR registers. */
467 valp fpr_base;
468 unsigned int fparg_count;
469
470 /* 'vec_base' points at the space for v2, and grows upwards as
471 we use vector registers. */
472 valp vec_base;
473 unsigned int vecarg_count;
474
475 unsigned int i, words, nargs, nfixedargs;
476 ffi_type **ptr;
477 double double_tmp;
478 union
479 {
480 void **v;
481 char **c;
482 signed char **sc;
483 unsigned char **uc;
484 signed short **ss;
485 unsigned short **us;
486 signed int **si;
487 unsigned int **ui;
488 unsigned long **ul;
489 float **f;
490 double **d;
491 float128 **f128;
492 } p_argv;
493 unsigned long gprvalue;
494 unsigned long align;
495
496 stacktop.c = (char *) stack + bytes;
497 gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
498 gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
499 #if _CALL_ELF == 2
500 rest.ul = stack + 4 + NUM_GPR_ARG_REGISTERS64;
501 #else
502 rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
503 #endif
504 fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
505 fparg_count = 0;
506 /* Place the vector args below the FPRs, if used, else the GPRs. */
507 if (ecif->cif->flags & FLAG_FP_ARGUMENTS)
508 vec_base.p = fpr_base.p & ~0xF;
509 else
510 vec_base.p = gpr_base.p;
511 vec_base.f128 -= NUM_VEC_ARG_REGISTERS64;
512 vecarg_count = 0;
513 next_arg.ul = gpr_base.ul;
514
515 /* Check that everything starts aligned properly. */
516 FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
517 FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
518 FFI_ASSERT (((unsigned long) gpr_base.c & 0xF) == 0);
519 FFI_ASSERT (((unsigned long) gpr_end.c & 0xF) == 0);
520 FFI_ASSERT (((unsigned long) vec_base.c & 0xF) == 0);
521 FFI_ASSERT ((bytes & 0xF) == 0);
522
523 /* Deal with return values that are actually pass-by-reference. */
524 if (flags & FLAG_RETVAL_REFERENCE)
525 *next_arg.ul++ = (unsigned long) (char *) ecif->rvalue;
526
527 /* Now for the arguments. */
528 p_argv.v = ecif->avalue;
529 nargs = ecif->cif->nargs;
530 #if _CALL_ELF != 2
531 nfixedargs = (unsigned) -1;
532 if ((flags & FLAG_COMPAT) == 0)
533 #endif
534 nfixedargs = ecif->cif->nfixedargs;
535 for (ptr = ecif->cif->arg_types, i = 0;
536 i < nargs;
537 i++, ptr++, p_argv.v++)
538 {
539 unsigned int elt, elnum;
540
541 switch ((*ptr)->type)
542 {
543 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
544 case FFI_TYPE_LONGDOUBLE:
545 if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
546 {
547 next_arg.p = FFI_ALIGN (next_arg.p, 16);
548 if (next_arg.ul == gpr_end.ul)
549 next_arg.ul = rest.ul;
550 if (vecarg_count < NUM_VEC_ARG_REGISTERS64 && i < nfixedargs)
551 memcpy (vec_base.f128++, *p_argv.f128, sizeof (float128));
552 else
553 memcpy (next_arg.f128, *p_argv.f128, sizeof (float128));
554 if (++next_arg.f128 == gpr_end.f128)
555 next_arg.f128 = rest.f128;
556 vecarg_count++;
557 FFI_ASSERT (__LDBL_MANT_DIG__ == 113);
558 FFI_ASSERT (flags & FLAG_VEC_ARGUMENTS);
559 break;
560 }
561 if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
562 {
563 double_tmp = (*p_argv.d)[0];
564 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
565 {
566 *fpr_base.d++ = double_tmp;
567 # if _CALL_ELF != 2
568 if ((flags & FLAG_COMPAT) != 0)
569 *next_arg.d = double_tmp;
570 # endif
571 }
572 else
573 *next_arg.d = double_tmp;
574 if (++next_arg.ul == gpr_end.ul)
575 next_arg.ul = rest.ul;
576 fparg_count++;
577 double_tmp = (*p_argv.d)[1];
578 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
579 {
580 *fpr_base.d++ = double_tmp;
581 # if _CALL_ELF != 2
582 if ((flags & FLAG_COMPAT) != 0)
583 *next_arg.d = double_tmp;
584 # endif
585 }
586 else
587 *next_arg.d = double_tmp;
588 if (++next_arg.ul == gpr_end.ul)
589 next_arg.ul = rest.ul;
590 fparg_count++;
591 FFI_ASSERT (__LDBL_MANT_DIG__ == 106);
592 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
593 break;
594 }
595 /* Fall through. */
596 #endif
597 case FFI_TYPE_DOUBLE:
598 #if _CALL_ELF != 2
599 do_double:
600 #endif
601 double_tmp = **p_argv.d;
602 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
603 {
604 *fpr_base.d++ = double_tmp;
605 #if _CALL_ELF != 2
606 if ((flags & FLAG_COMPAT) != 0)
607 *next_arg.d = double_tmp;
608 #endif
609 }
610 else
611 *next_arg.d = double_tmp;
612 if (++next_arg.ul == gpr_end.ul)
613 next_arg.ul = rest.ul;
614 fparg_count++;
615 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
616 break;
617
618 case FFI_TYPE_FLOAT:
619 #if _CALL_ELF != 2
620 do_float:
621 #endif
622 double_tmp = **p_argv.f;
623 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
624 {
625 *fpr_base.d++ = double_tmp;
626 #if _CALL_ELF != 2
627 if ((flags & FLAG_COMPAT) != 0)
628 {
629 # ifndef __LITTLE_ENDIAN__
630 next_arg.f[1] = (float) double_tmp;
631 # else
632 next_arg.f[0] = (float) double_tmp;
633 # endif
634 }
635 #endif
636 }
637 else
638 {
639 # ifndef __LITTLE_ENDIAN__
640 next_arg.f[1] = (float) double_tmp;
641 # else
642 next_arg.f[0] = (float) double_tmp;
643 # endif
644 }
645 if (++next_arg.ul == gpr_end.ul)
646 next_arg.ul = rest.ul;
647 fparg_count++;
648 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
649 break;
650
651 case FFI_TYPE_STRUCT:
652 if ((ecif->cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
653 {
654 align = (*ptr)->alignment;
655 if (align > 16)
656 align = 16;
657 if (align > 1)
658 {
659 next_arg.p = FFI_ALIGN (next_arg.p, align);
660 if (next_arg.ul == gpr_end.ul)
661 next_arg.ul = rest.ul;
662 }
663 }
664 elt = discover_homogeneous_aggregate (ecif->cif->abi, *ptr, &elnum);
665 if (elt)
666 {
667 #if _CALL_ELF == 2
668 union {
669 void *v;
670 float *f;
671 double *d;
672 float128 *f128;
673 } arg;
674
675 arg.v = *p_argv.v;
676 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
677 if (elt == FFI_TYPE_LONGDOUBLE &&
678 (ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
679 {
680 do
681 {
682 if (vecarg_count < NUM_VEC_ARG_REGISTERS64
683 && i < nfixedargs)
684 memcpy (vec_base.f128++, arg.f128++, sizeof (float128));
685 else
686 memcpy (next_arg.f128, arg.f128++, sizeof (float128));
687 if (++next_arg.f128 == gpr_end.f128)
688 next_arg.f128 = rest.f128;
689 vecarg_count++;
690 }
691 while (--elnum != 0);
692 }
693 else
694 #endif
695 if (elt == FFI_TYPE_FLOAT)
696 {
697 do
698 {
699 double_tmp = *arg.f++;
700 if (fparg_count < NUM_FPR_ARG_REGISTERS64
701 && i < nfixedargs)
702 *fpr_base.d++ = double_tmp;
703 else
704 *next_arg.f = (float) double_tmp;
705 if (++next_arg.f == gpr_end.f)
706 next_arg.f = rest.f;
707 fparg_count++;
708 }
709 while (--elnum != 0);
710 if ((next_arg.p & 7) != 0)
711 if (++next_arg.f == gpr_end.f)
712 next_arg.f = rest.f;
713 }
714 else
715 do
716 {
717 double_tmp = *arg.d++;
718 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
719 *fpr_base.d++ = double_tmp;
720 else
721 *next_arg.d = double_tmp;
722 if (++next_arg.d == gpr_end.d)
723 next_arg.d = rest.d;
724 fparg_count++;
725 }
726 while (--elnum != 0);
727 #else
728 if (elt == FFI_TYPE_FLOAT)
729 goto do_float;
730 else
731 goto do_double;
732 #endif
733 }
734 else
735 {
736 words = ((*ptr)->size + 7) / 8;
737 if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
738 {
739 size_t first = gpr_end.c - next_arg.c;
740 memcpy (next_arg.c, *p_argv.c, first);
741 memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
742 next_arg.c = rest.c + words * 8 - first;
743 }
744 else
745 {
746 char *where = next_arg.c;
747
748 #ifndef __LITTLE_ENDIAN__
749 /* Structures with size less than eight bytes are passed
750 left-padded. */
751 if ((*ptr)->size < 8)
752 where += 8 - (*ptr)->size;
753 #endif
754 memcpy (where, *p_argv.c, (*ptr)->size);
755 next_arg.ul += words;
756 if (next_arg.ul == gpr_end.ul)
757 next_arg.ul = rest.ul;
758 }
759 }
760 break;
761
762 case FFI_TYPE_UINT8:
763 gprvalue = **p_argv.uc;
764 goto putgpr;
765 case FFI_TYPE_SINT8:
766 gprvalue = **p_argv.sc;
767 goto putgpr;
768 case FFI_TYPE_UINT16:
769 gprvalue = **p_argv.us;
770 goto putgpr;
771 case FFI_TYPE_SINT16:
772 gprvalue = **p_argv.ss;
773 goto putgpr;
774 case FFI_TYPE_UINT32:
775 gprvalue = **p_argv.ui;
776 goto putgpr;
777 case FFI_TYPE_INT:
778 case FFI_TYPE_SINT32:
779 gprvalue = **p_argv.si;
780 goto putgpr;
781
782 case FFI_TYPE_UINT64:
783 case FFI_TYPE_SINT64:
784 case FFI_TYPE_POINTER:
785 gprvalue = **p_argv.ul;
786 putgpr:
787 *next_arg.ul++ = gprvalue;
788 if (next_arg.ul == gpr_end.ul)
789 next_arg.ul = rest.ul;
790 break;
791 }
792 }
793
794 FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS
795 || (next_arg.ul >= gpr_base.ul
796 && next_arg.ul <= gpr_base.ul + 4));
797 }
798
799
800 #if _CALL_ELF == 2
801 #define MIN_CACHE_LINE_SIZE 8
802
803 static void
804 flush_icache (char *wraddr, char *xaddr, int size)
805 {
806 int i;
807 for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE)
808 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;"
809 : : "r" (xaddr + i), "r" (wraddr + i) : "memory");
810 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
811 : : "r"(xaddr + size - 1), "r"(wraddr + size - 1)
812 : "memory");
813 }
814 #endif
815
816
817 ffi_status FFI_HIDDEN
818 ffi_prep_closure_loc_linux64 (ffi_closure *closure,
819 ffi_cif *cif,
820 void (*fun) (ffi_cif *, void *, void **, void *),
821 void *user_data,
822 void *codeloc)
823 {
824 if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
825 return FFI_BAD_ABI;
826
827 #ifdef FFI_EXEC_STATIC_TRAMP
828 if (ffi_tramp_is_present(closure))
829 {
830 /* Initialize the static trampoline's parameters. */
831 void (*dest)(void) = ffi_closure_LINUX64;
832 ffi_tramp_set_parms (closure->ftramp, dest, closure);
833 }
834 else
835 #endif
836 {
837 #if _CALL_ELF == 2
838 unsigned int *tramp = (unsigned int *) &closure->tramp[0];
839 tramp[0] = 0xe96c0018; /* 0: ld 11,2f-0b(12) */
840 tramp[1] = 0xe98c0010; /* ld 12,1f-0b(12) */
841 tramp[2] = 0x7d8903a6; /* mtctr 12 */
842 tramp[3] = 0x4e800420; /* bctr */
843 /* 1: .quad function_addr */
844 /* 2: .quad context */
845 *(void **) &tramp[4] = (void *) ffi_closure_LINUX64;
846 *(void **) &tramp[6] = codeloc;
847 flush_icache ((char *) tramp, (char *) codeloc, 4 * 4);
848 #else
849 /* Copy function address and TOC from ffi_closure_LINUX64 OPD. */
850 void **tramp = (void **) &closure->tramp[0];
851 memcpy (&tramp[0], (void **) ffi_closure_LINUX64, sizeof (void *));
852 tramp[1] = codeloc;
853 memcpy (&tramp[2], (void **) ffi_closure_LINUX64 + 1, sizeof (void *));
854 #endif
855 }
856
857 closure->cif = cif;
858 closure->fun = fun;
859 closure->user_data = user_data;
860
861 return FFI_OK;
862 }
863
864
865 int FFI_HIDDEN
866 ffi_closure_helper_LINUX64 (ffi_cif *cif,
867 void (*fun) (ffi_cif *, void *, void **, void *),
868 void *user_data,
869 void *rvalue,
870 unsigned long *pst,
871 ffi_dblfl *pfr,
872 float128 *pvec)
873 {
874 /* rvalue is the pointer to space for return value in closure assembly */
875 /* pst is the pointer to parameter save area
876 (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
877 /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
878 /* pvec is the pointer to where v2-v13 are stored in ffi_closure_LINUX64 */
879
880 void **avalue;
881 ffi_type **arg_types;
882 unsigned long i, avn, nfixedargs;
883 ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
884 float128 *end_pvec = pvec + NUM_VEC_ARG_REGISTERS64;
885 unsigned long align;
886
887 avalue = alloca (cif->nargs * sizeof (void *));
888
889 /* Copy the caller's structure return value address so that the
890 closure returns the data directly to the caller. */
891 if (cif->rtype->type == FFI_TYPE_STRUCT
892 && (cif->flags & FLAG_RETURNS_SMST) == 0)
893 {
894 rvalue = (void *) *pst;
895 pst++;
896 }
897
898 i = 0;
899 avn = cif->nargs;
900 #if _CALL_ELF != 2
901 nfixedargs = (unsigned) -1;
902 if ((cif->flags & FLAG_COMPAT) == 0)
903 #endif
904 nfixedargs = cif->nfixedargs;
905 arg_types = cif->arg_types;
906
907 /* Grab the addresses of the arguments from the stack frame. */
908 while (i < avn)
909 {
910 unsigned int elt, elnum;
911
912 switch (arg_types[i]->type)
913 {
914 case FFI_TYPE_SINT8:
915 case FFI_TYPE_UINT8:
916 #ifndef __LITTLE_ENDIAN__
917 avalue[i] = (char *) pst + 7;
918 pst++;
919 break;
920 #endif
921
922 case FFI_TYPE_SINT16:
923 case FFI_TYPE_UINT16:
924 #ifndef __LITTLE_ENDIAN__
925 avalue[i] = (char *) pst + 6;
926 pst++;
927 break;
928 #endif
929
930 case FFI_TYPE_SINT32:
931 case FFI_TYPE_UINT32:
932 #ifndef __LITTLE_ENDIAN__
933 avalue[i] = (char *) pst + 4;
934 pst++;
935 break;
936 #endif
937
938 case FFI_TYPE_SINT64:
939 case FFI_TYPE_UINT64:
940 case FFI_TYPE_POINTER:
941 avalue[i] = pst;
942 pst++;
943 break;
944
945 case FFI_TYPE_STRUCT:
946 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
947 {
948 align = arg_types[i]->alignment;
949 if (align > 16)
950 align = 16;
951 if (align > 1)
952 pst = (unsigned long *) FFI_ALIGN ((size_t) pst, align);
953 }
954 elt = discover_homogeneous_aggregate (cif->abi, arg_types[i], &elnum);
955 if (elt)
956 {
957 #if _CALL_ELF == 2
958 union {
959 void *v;
960 unsigned long *ul;
961 float *f;
962 double *d;
963 float128 *f128;
964 size_t p;
965 } to, from;
966
967 /* Repackage the aggregate from its parts. The
968 aggregate size is not greater than the space taken by
969 the registers so store back to the register/parameter
970 save arrays. */
971 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
972 if (elt == FFI_TYPE_LONGDOUBLE &&
973 (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
974 {
975 if (pvec + elnum <= end_pvec)
976 to.v = pvec;
977 else
978 to.v = pst;
979 }
980 else
981 #endif
982 if (pfr + elnum <= end_pfr)
983 to.v = pfr;
984 else
985 to.v = pst;
986
987 avalue[i] = to.v;
988 from.ul = pst;
989 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
990 if (elt == FFI_TYPE_LONGDOUBLE &&
991 (cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
992 {
993 do
994 {
995 if (pvec < end_pvec && i < nfixedargs)
996 memcpy (to.f128, pvec++, sizeof (float128));
997 else
998 memcpy (to.f128, from.f128, sizeof (float128));
999 to.f128++;
1000 from.f128++;
1001 }
1002 while (--elnum != 0);
1003 }
1004 else
1005 #endif
1006 if (elt == FFI_TYPE_FLOAT)
1007 {
1008 do
1009 {
1010 if (pfr < end_pfr && i < nfixedargs)
1011 {
1012 *to.f = (float) pfr->d;
1013 pfr++;
1014 }
1015 else
1016 *to.f = *from.f;
1017 to.f++;
1018 from.f++;
1019 }
1020 while (--elnum != 0);
1021 }
1022 else
1023 {
1024 do
1025 {
1026 if (pfr < end_pfr && i < nfixedargs)
1027 {
1028 *to.d = pfr->d;
1029 pfr++;
1030 }
1031 else
1032 *to.d = *from.d;
1033 to.d++;
1034 from.d++;
1035 }
1036 while (--elnum != 0);
1037 }
1038 #else
1039 if (elt == FFI_TYPE_FLOAT)
1040 goto do_float;
1041 else
1042 goto do_double;
1043 #endif
1044 }
1045 else
1046 {
1047 #ifndef __LITTLE_ENDIAN__
1048 /* Structures with size less than eight bytes are passed
1049 left-padded. */
1050 if (arg_types[i]->size < 8)
1051 avalue[i] = (char *) pst + 8 - arg_types[i]->size;
1052 else
1053 #endif
1054 avalue[i] = pst;
1055 }
1056 pst += (arg_types[i]->size + 7) / 8;
1057 break;
1058
1059 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1060 case FFI_TYPE_LONGDOUBLE:
1061 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_IEEE128) != 0)
1062 {
1063 if (((unsigned long) pst & 0xF) != 0)
1064 ++pst;
1065 if (pvec < end_pvec && i < nfixedargs)
1066 avalue[i] = pvec++;
1067 else
1068 avalue[i] = pst;
1069 pst += 2;
1070 break;
1071 }
1072 else if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
1073 {
1074 if (pfr + 1 < end_pfr && i + 1 < nfixedargs)
1075 {
1076 avalue[i] = pfr;
1077 pfr += 2;
1078 }
1079 else
1080 {
1081 if (pfr < end_pfr && i < nfixedargs)
1082 {
1083 /* Passed partly in f13 and partly on the stack.
1084 Move it all to the stack. */
1085 *pst = *(unsigned long *) pfr;
1086 pfr++;
1087 }
1088 avalue[i] = pst;
1089 }
1090 pst += 2;
1091 break;
1092 }
1093 /* Fall through. */
1094 #endif
1095 case FFI_TYPE_DOUBLE:
1096 #if _CALL_ELF != 2
1097 do_double:
1098 #endif
1099 /* On the outgoing stack all values are aligned to 8 */
1100 /* there are 13 64bit floating point registers */
1101
1102 if (pfr < end_pfr && i < nfixedargs)
1103 {
1104 avalue[i] = pfr;
1105 pfr++;
1106 }
1107 else
1108 avalue[i] = pst;
1109 pst++;
1110 break;
1111
1112 case FFI_TYPE_FLOAT:
1113 #if _CALL_ELF != 2
1114 do_float:
1115 #endif
1116 if (pfr < end_pfr && i < nfixedargs)
1117 {
1118 /* Float values are stored as doubles in the
1119 ffi_closure_LINUX64 code. Fix them here. */
1120 pfr->f = (float) pfr->d;
1121 avalue[i] = pfr;
1122 pfr++;
1123 }
1124 else
1125 {
1126 #ifndef __LITTLE_ENDIAN__
1127 avalue[i] = (char *) pst + 4;
1128 #else
1129 avalue[i] = pst;
1130 #endif
1131 }
1132 pst++;
1133 break;
1134
1135 default:
1136 FFI_ASSERT (0);
1137 }
1138
1139 i++;
1140 }
1141
1142 (*fun) (cif, rvalue, avalue, user_data);
1143
1144 /* Tell ffi_closure_LINUX64 how to perform return type promotions. */
1145 if ((cif->flags & FLAG_RETURNS_SMST) != 0)
1146 {
1147 if ((cif->flags & (FLAG_RETURNS_FP | FLAG_RETURNS_VEC)) == 0)
1148 return FFI_V2_TYPE_SMALL_STRUCT + cif->rtype->size - 1;
1149 else if ((cif->flags & FLAG_RETURNS_VEC) != 0)
1150 return FFI_V2_TYPE_VECTOR_HOMOG;
1151 else if ((cif->flags & FLAG_RETURNS_64BITS) != 0)
1152 return FFI_V2_TYPE_DOUBLE_HOMOG;
1153 else
1154 return FFI_V2_TYPE_FLOAT_HOMOG;
1155 }
1156 if ((cif->flags & FLAG_RETURNS_VEC) != 0)
1157 return FFI_V2_TYPE_VECTOR;
1158 return cif->rtype->type;
1159 }
1160 #endif