Debian Codegraph @

src/loongarch64/ffi.c

raw
1 /* -----------------------------------------------------------------------
2 ffi.c - Copyright (c) 2022 Xu Chenghua <[email protected]>
3 2022 Cheng Lulu <[email protected]>
4 Based on RISC-V port
5
6 LoongArch Foreign Function Interface
7
8 Permission is hereby granted, free of charge, to any person obtaining
9 a copy of this software and associated documentation files (the
10 ``Software''), to deal in the Software without restriction, including
11 without limitation the rights to use, copy, modify, merge, publish,
12 distribute, sublicense, and/or sell copies of the Software, and to
13 permit persons to whom the Software is furnished to do so, subject to
14 the following conditions:
15
16 The above copyright notice and this permission notice shall be included
17 in all copies or substantial portions of the Software.
18
19 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
20 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
23 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
24 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 DEALINGS IN THE SOFTWARE.
27 ----------------------------------------------------------------------- */
28
29 #include <ffi.h>
30 #include <ffi_common.h>
31 #include <tramp.h>
32
33 #include <stdlib.h>
34 #include <stdint.h>
35
36 #if defined(__loongarch_soft_float)
37 # define ABI_FRLEN 0
38 #elif defined(__loongarch_single_float)
39 # define ABI_FRLEN 32
40 # define ABI_FLOAT float
41 #elif defined(__loongarch_double_float)
42 # define ABI_FRLEN 64
43 # define ABI_FLOAT double
44 #else
45 #error unsupported LoongArch floating-point ABI
46 #endif
47
48 #define NARGREG 8
49 #define STKALIGN 16
50 #define MAXCOPYARG (2 * sizeof (double))
51
52 /* call_context registers
53 - 8 floating point parameter/result registers.
54 - 8 integer parameter/result registers.
55 - 2 registers used by the assembly code to in-place construct its own
56 stack frame
57 - frame register
58 - return register
59 */
60 typedef struct call_context
61 {
62 #if !defined(__loongarch_soft_float)
63 ABI_FLOAT fa[8];
64 #endif
65 size_t a[10];
66 } call_context;
67
68 typedef struct call_builder
69 {
70 call_context *aregs;
71 int used_integer;
72 int used_float;
73 size_t *used_stack;
74 size_t *stack;
75 size_t next_struct_area;
76 } call_builder;
77
78 /* Integer (not pointer) less than ABI GRLEN. */
79 /* FFI_TYPE_INT does not appear to be used. */
80 #if __SIZEOF_POINTER__ == 8
81 # define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT64)
82 #else
83 # define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT32)
84 #endif
85
86 #if ABI_FRLEN
87 typedef struct float_struct_info
88 {
89 char as_elements;
90 char type1;
91 char offset2;
92 char type2;
93 } float_struct_info;
94
95 #if ABI_FRLEN >= 64
96 # define IS_FLOAT(type) ((type) >= FFI_TYPE_FLOAT && (type) <= FFI_TYPE_DOUBLE)
97 #else
98 # define IS_FLOAT(type) ((type) == FFI_TYPE_FLOAT)
99 #endif
100
101 static ffi_type **
102 flatten_struct (ffi_type *in, ffi_type **out, ffi_type **out_end)
103 {
104 int i;
105
106 if (out == out_end)
107 return out;
108 if (in->type != FFI_TYPE_STRUCT)
109 *(out++) = in;
110 else
111 for (i = 0; in->elements[i]; i++)
112 out = flatten_struct (in->elements[i], out, out_end);
113 return out;
114 }
115
116 /* Structs with at most two fields after flattening, one of which is of
117 floating point type, are passed in multiple registers if sufficient
118 registers are available. */
119 static float_struct_info
120 struct_passed_as_elements (call_builder *cb, ffi_type *top)
121 {
122 float_struct_info ret = {0, 0, 0, 0};
123 ffi_type *fields[3];
124 int num_floats, num_ints;
125 int num_fields = flatten_struct (top, fields, fields + 3) - fields;
126
127 if (num_fields == 1)
128 {
129 if (IS_FLOAT (fields[0]->type))
130 {
131 ret.as_elements = 1;
132 ret.type1 = fields[0]->type;
133 }
134 }
135 else if (num_fields == 2)
136 {
137 num_floats = IS_FLOAT (fields[0]->type) + IS_FLOAT (fields[1]->type);
138 num_ints = IS_INT (fields[0]->type) + IS_INT (fields[1]->type);
139 if (num_floats == 0 || num_floats + num_ints != 2)
140 return ret;
141 if (cb->used_float + num_floats > NARGREG
142 || cb->used_integer + (2 - num_floats) > NARGREG)
143 return ret;
144 if (!IS_FLOAT (fields[0]->type) && !IS_FLOAT (fields[1]->type))
145 return ret;
146
147 ret.type1 = fields[0]->type;
148 ret.type2 = fields[1]->type;
149 ret.offset2 = FFI_ALIGN (fields[0]->size, fields[1]->alignment);
150 ret.as_elements = 1;
151 }
152 return ret;
153 }
154 #endif
155
156 /* Allocates a single register, float register, or GRLEN-sized stack slot to a
157 datum. */
158 static void
159 marshal_atom (call_builder *cb, int type, void *data)
160 {
161 size_t value = 0;
162 switch (type)
163 {
164 case FFI_TYPE_UINT8:
165 value = *(uint8_t *) data;
166 break;
167 case FFI_TYPE_SINT8:
168 value = *(int8_t *) data;
169 break;
170 case FFI_TYPE_UINT16:
171 value = *(uint16_t *) data;
172 break;
173 case FFI_TYPE_SINT16:
174 value = *(int16_t *) data;
175 break;
176 /* 32-bit quantities are always sign-extended in the ABI. */
177 case FFI_TYPE_UINT32:
178 value = *(int32_t *) data;
179 break;
180 case FFI_TYPE_SINT32:
181 value = *(int32_t *) data;
182 break;
183 #if __SIZEOF_POINTER__ == 8
184 case FFI_TYPE_UINT64:
185 value = *(uint64_t *) data;
186 break;
187 case FFI_TYPE_SINT64:
188 value = *(int64_t *) data;
189 break;
190 #endif
191 case FFI_TYPE_POINTER:
192 value = *(size_t *) data;
193 break;
194
195 #if ABI_FRLEN >= 32
196 case FFI_TYPE_FLOAT:
197 *(float *)(cb->aregs->fa + cb->used_float++) = *(float *) data;
198 return;
199 #endif
200 #if ABI_FRLEN >= 64
201 case FFI_TYPE_DOUBLE:
202 (cb->aregs->fa[cb->used_float++]) = *(double *) data;
203 return;
204 #endif
205 default:
206 FFI_ASSERT (0);
207 break;
208 }
209
210 if (cb->used_integer == NARGREG)
211 *cb->used_stack++ = value;
212 else
213 cb->aregs->a[cb->used_integer++] = value;
214 }
215
216 static void
217 unmarshal_atom (call_builder *cb, int type, void *data)
218 {
219 size_t value;
220 switch (type)
221 {
222 #if ABI_FRLEN >= 32
223 case FFI_TYPE_FLOAT:
224 *(float *) data = *(float *)(cb->aregs->fa + cb->used_float++);
225 return;
226 #endif
227 #if ABI_FRLEN >= 64
228 case FFI_TYPE_DOUBLE:
229 *(double *) data = cb->aregs->fa[cb->used_float++];
230 return;
231 #endif
232 }
233
234 if (cb->used_integer == NARGREG)
235 value = *cb->used_stack++;
236 else
237 value = cb->aregs->a[cb->used_integer++];
238
239 switch (type)
240 {
241 case FFI_TYPE_UINT8:
242 case FFI_TYPE_SINT8:
243 case FFI_TYPE_UINT16:
244 case FFI_TYPE_SINT16:
245 case FFI_TYPE_UINT32:
246 case FFI_TYPE_SINT32:
247 #if __SIZEOF_POINTER__ == 8
248 case FFI_TYPE_UINT64:
249 case FFI_TYPE_SINT64:
250 #endif
251 case FFI_TYPE_POINTER:
252 *(ffi_arg *)data = value;
253 break;
254 default:
255 FFI_ASSERT (0);
256 break;
257 }
258 }
259
260 /* Allocate and copy a structure that is passed by value on the stack and
261 return a pointer to it. */
262 static void *
263 allocate_and_copy_struct_to_stack (call_builder *cb, void *data,
264 ffi_type *type)
265 {
266 size_t dest = cb->next_struct_area - type->size;
267
268 dest = FFI_ALIGN_DOWN (dest, type->alignment);
269 cb->next_struct_area = dest;
270
271 return memcpy ((char *)cb->stack + dest, data, type->size);
272 }
273
274 /* Adds an argument to a call, or a not by reference return value. */
275 static void
276 marshal (call_builder *cb, ffi_type *type, int var, void *data)
277 {
278 size_t realign[2];
279
280 #if ABI_FRLEN
281 if (!var && type->type == FFI_TYPE_STRUCT)
282 {
283 float_struct_info fsi = struct_passed_as_elements (cb, type);
284 if (fsi.as_elements)
285 {
286 marshal_atom (cb, fsi.type1, data);
287 if (fsi.offset2)
288 marshal_atom (cb, fsi.type2, ((char *) data) + fsi.offset2);
289 return;
290 }
291 }
292
293 if (!var && cb->used_float < NARGREG
294 && IS_FLOAT (type->type))
295 {
296 marshal_atom (cb, type->type, data);
297 return;
298 }
299
300 double promoted;
301 if (var && type->type == FFI_TYPE_FLOAT)
302 {
303 /* C standard requires promoting float -> double for variable arg. */
304 promoted = *(float *) data;
305 type = &ffi_type_double;
306 data = &promoted;
307 }
308 #endif
309
310 if (type->size > 2 * __SIZEOF_POINTER__)
311 /* Pass by reference. */
312 {
313 allocate_and_copy_struct_to_stack (cb, data, type);
314 data = (char *)cb->stack + cb->next_struct_area;
315 marshal_atom (cb, FFI_TYPE_POINTER, &data);
316 }
317 else if (IS_INT (type->type) || type->type == FFI_TYPE_POINTER)
318 marshal_atom (cb, type->type, data);
319 else
320 {
321 /* Overlong integers, soft-float floats, and structs without special
322 float handling are treated identically from this point on. */
323
324 /* Variadics are aligned even in registers. */
325 if (type->alignment > __SIZEOF_POINTER__)
326 {
327 if (var)
328 cb->used_integer = FFI_ALIGN (cb->used_integer, 2);
329 cb->used_stack
330 = (size_t *) FFI_ALIGN (cb->used_stack, 2 * __SIZEOF_POINTER__);
331 }
332
333 memcpy (realign, data, type->size);
334 if (type->size > 0)
335 marshal_atom (cb, FFI_TYPE_POINTER, realign);
336 if (type->size > __SIZEOF_POINTER__)
337 marshal_atom (cb, FFI_TYPE_POINTER, realign + 1);
338 }
339 }
340
341 /* For arguments passed by reference returns the pointer, otherwise the arg
342 is copied (up to MAXCOPYARG bytes). */
343 static void *
344 unmarshal (call_builder *cb, ffi_type *type, int var, void *data)
345 {
346 size_t realign[2];
347 void *pointer;
348
349 #if ABI_FRLEN
350 if (!var && type->type == FFI_TYPE_STRUCT)
351 {
352 float_struct_info fsi = struct_passed_as_elements (cb, type);
353 if (fsi.as_elements)
354 {
355 unmarshal_atom (cb, fsi.type1, data);
356 if (fsi.offset2)
357 unmarshal_atom (cb, fsi.type2, ((char *) data) + fsi.offset2);
358 return data;
359 }
360 }
361
362 if (!var && cb->used_float < NARGREG
363 && IS_FLOAT (type->type))
364 {
365 unmarshal_atom (cb, type->type, data);
366 return data;
367 }
368
369 if (var && type->type == FFI_TYPE_FLOAT)
370 {
371 int m = cb->used_integer;
372 void *promoted
373 = m < NARGREG ? cb->aregs->a + m : cb->used_stack + m - NARGREG + 1;
374 *(float *) promoted = *(double *) promoted;
375 }
376 #endif
377
378 if (type->size > 2 * __SIZEOF_POINTER__)
379 {
380 /* Pass by reference. */
381 unmarshal_atom (cb, FFI_TYPE_POINTER, (char *) &pointer);
382 return pointer;
383 }
384 else if (IS_INT (type->type) || type->type == FFI_TYPE_POINTER)
385 {
386 unmarshal_atom (cb, type->type, data);
387 return data;
388 }
389 else
390 {
391 /* Overlong integers, soft-float floats, and structs without special
392 float handling are treated identically from this point on. */
393
394 /* Variadics are aligned even in registers. */
395 if (type->alignment > __SIZEOF_POINTER__)
396 {
397 if (var)
398 cb->used_integer = FFI_ALIGN (cb->used_integer, 2);
399 cb->used_stack
400 = (size_t *) FFI_ALIGN (cb->used_stack, 2 * __SIZEOF_POINTER__);
401 }
402
403 if (type->size > 0)
404 unmarshal_atom (cb, FFI_TYPE_POINTER, realign);
405 if (type->size > __SIZEOF_POINTER__)
406 unmarshal_atom (cb, FFI_TYPE_POINTER, realign + 1);
407 memcpy (data, realign, type->size);
408 return data;
409 }
410 }
411
412 static int
413 passed_by_ref (call_builder *cb, ffi_type *type, int var)
414 {
415 #if ABI_FRLEN
416 if (!var && type->type == FFI_TYPE_STRUCT)
417 {
418 float_struct_info fsi = struct_passed_as_elements (cb, type);
419 if (fsi.as_elements)
420 return 0;
421 }
422 #endif
423
424 return type->size > 2 * __SIZEOF_POINTER__;
425 }
426
427 /* Perform machine dependent cif processing. */
428 ffi_status
429 ffi_prep_cif_machdep (ffi_cif *cif)
430 {
431 cif->loongarch_nfixedargs = cif->nargs;
432 return FFI_OK;
433 }
434
435 /* Perform machine dependent cif processing when we have a variadic
436 function. */
437 ffi_status
438 ffi_prep_cif_machdep_var (ffi_cif *cif, unsigned int nfixedargs,
439 unsigned int ntotalargs)
440 {
441 cif->loongarch_nfixedargs = nfixedargs;
442 return FFI_OK;
443 }
444
445 /* Low level routine for calling functions. */
446 extern void ffi_call_asm (void *stack, struct call_context *regs,
447 void (*fn) (void), void *closure) FFI_HIDDEN;
448
449 static void
450 ffi_call_int (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue,
451 void *closure)
452 {
453 /* This is a conservative estimate, assuming a complex return value and
454 that all remaining arguments are long long / __int128 */
455 size_t arg_bytes = cif->bytes;
456 size_t rval_bytes = 0;
457 if (rvalue == NULL && cif->rtype->size > 2 * __SIZEOF_POINTER__)
458 rval_bytes = FFI_ALIGN (cif->rtype->size, STKALIGN);
459 size_t alloc_size = arg_bytes + rval_bytes + sizeof (call_context);
460
461 /* The assembly code will deallocate all stack data at lower addresses
462 than the argument region, so we need to allocate the frame and the
463 return value after the arguments in a single allocation. */
464 size_t alloc_base;
465 /* Argument region must be 16-byte aligned in LP64 ABIs. */
466 if (_Alignof(max_align_t) >= STKALIGN)
467 /* Since sizeof long double is normally 16, the compiler will
468 guarantee alloca alignment to at least that much. */
469 alloc_base = (size_t) alloca (alloc_size);
470 else
471 alloc_base = FFI_ALIGN (alloca (alloc_size + STKALIGN - 1), STKALIGN);
472
473 if (rval_bytes)
474 rvalue = (void *) (alloc_base + arg_bytes);
475
476 call_builder cb;
477 cb.used_float = cb.used_integer = 0;
478 cb.aregs = (call_context *) (alloc_base + arg_bytes + rval_bytes);
479 cb.used_stack = (void *) alloc_base;
480 cb.stack = (void *) alloc_base;
481 cb.next_struct_area = arg_bytes;
482
483 int return_by_ref = passed_by_ref (&cb, cif->rtype, 0);
484 if (return_by_ref)
485 cb.aregs->a[cb.used_integer++] = (size_t)rvalue;
486
487 int i;
488 for (i = 0; i < cif->nargs; i++)
489 marshal (&cb, cif->arg_types[i], i >= cif->loongarch_nfixedargs,
490 avalue[i]);
491
492 ffi_call_asm ((void *) alloc_base, cb.aregs, fn, closure);
493
494 cb.used_float = cb.used_integer = 0;
495 if (!return_by_ref && rvalue)
496 unmarshal (&cb, cif->rtype, 0, rvalue);
497 }
498
499 void
500 ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
501 {
502 ffi_call_int (cif, fn, rvalue, avalue, NULL);
503 }
504
505 void
506 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue,
507 void *closure)
508 {
509 ffi_call_int (cif, fn, rvalue, avalue, closure);
510 }
511
512 extern void ffi_closure_asm (void) FFI_HIDDEN;
513
514 ffi_status
515 ffi_prep_closure_loc (ffi_closure *closure, ffi_cif *cif,
516 void (*fun) (ffi_cif *, void *, void **, void *),
517 void *user_data, void *codeloc)
518 {
519 uint32_t *tramp = (uint32_t *) &closure->tramp[0];
520 uint64_t fn = (uint64_t) (uintptr_t) ffi_closure_asm;
521
522 if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
523 return FFI_BAD_ABI;
524
525 #if defined(FFI_EXEC_STATIC_TRAMP)
526 if (ffi_tramp_is_present(closure))
527 {
528 ffi_tramp_set_parms (closure->ftramp, ffi_closure_asm, closure);
529 goto out;
530 }
531 #endif
532
533 /* Fill the dynamic trampoline. We will call ffi_closure_inner with codeloc,
534 not closure, but as long as the memory is readable it should work. */
535 tramp[0] = 0x1800000c; /* pcaddi $t0, 0 (i.e. $t0 <- tramp) */
536 tramp[1] = 0x28c0418d; /* ld.d $t1, $t0, 16 */
537 tramp[2] = 0x4c0001a0; /* jirl $zero, $t1, 0 */
538 tramp[3] = 0x03400000; /* nop */
539 tramp[4] = fn;
540 tramp[5] = fn >> 32;
541
542 __builtin___clear_cache (codeloc, codeloc + FFI_TRAMPOLINE_SIZE);
543
544 #if defined(FFI_EXEC_STATIC_TRAMP)
545 out:
546 #endif
547 closure->cif = cif;
548 closure->fun = fun;
549 closure->user_data = user_data;
550
551 return FFI_OK;
552 }
553
554 extern void ffi_go_closure_asm (void) FFI_HIDDEN;
555
556 ffi_status
557 ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
558 void (*fun) (ffi_cif *, void *, void **, void *))
559 {
560 if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
561 return FFI_BAD_ABI;
562
563 closure->tramp = (void *) ffi_go_closure_asm;
564 closure->cif = cif;
565 closure->fun = fun;
566 return FFI_OK;
567 }
568
569 /* Called by the assembly code with aregs pointing to saved argument registers
570 and stack pointing to the stacked arguments. Return values passed in
571 registers will be reloaded from aregs. */
572 void FFI_HIDDEN
573 ffi_closure_inner (ffi_cif *cif,
574 void (*fun) (ffi_cif *, void *, void **, void *),
575 void *user_data, size_t *stack, call_context *aregs)
576 {
577 void **avalue = alloca (cif->nargs * sizeof (void *));
578 /* Storage for arguments which will be copied by unmarshal(). We could
579 theoretically avoid the copies in many cases and use at most 128 bytes
580 of memory, but allocating disjoint storage for each argument is
581 simpler. */
582 char *astorage = alloca (cif->nargs * MAXCOPYARG);
583 void *rvalue;
584 call_builder cb;
585 int return_by_ref;
586 int i;
587
588 cb.aregs = aregs;
589 cb.used_integer = cb.used_float = 0;
590 cb.used_stack = stack;
591
592 return_by_ref = passed_by_ref (&cb, cif->rtype, 0);
593 if (return_by_ref)
594 unmarshal (&cb, &ffi_type_pointer, 0, &rvalue);
595 else
596 rvalue = alloca (cif->rtype->size);
597
598 for (i = 0; i < cif->nargs; i++)
599 avalue[i]
600 = unmarshal (&cb, cif->arg_types[i], i >= cif->loongarch_nfixedargs,
601 astorage + i * MAXCOPYARG);
602
603 fun (cif, rvalue, avalue, user_data);
604
605 if (!return_by_ref && cif->rtype->type != FFI_TYPE_VOID)
606 {
607 cb.used_integer = cb.used_float = 0;
608 marshal (&cb, cif->rtype, 0, rvalue);
609 }
610 }
611
612 #if defined(FFI_EXEC_STATIC_TRAMP)
613 void *
614 ffi_tramp_arch (size_t *tramp_size, size_t *map_size)
615 {
616 extern void *trampoline_code_table;
617
618 *tramp_size = 16;
619 /* A mapping size of 64K is chosen to cover the page sizes of 4K, 16K, and
620 64K. */
621 *map_size = 1 << 16;
622 return &trampoline_code_table;
623 }
624 #endif