Debian Codegraph @

src/riscv/ffi.c

raw
1 /* -----------------------------------------------------------------------
2 ffi.c - Copyright (c) 2015 Michael Knyszek <[email protected]>
3 2015 Andrew Waterman <[email protected]>
4 2018 Stef O'Rear <[email protected]>
5 Based on MIPS N32/64 port
6
7 RISC-V Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdint.h>
35
36 #if __riscv_float_abi_double
37 #define ABI_FLEN 64
38 #define ABI_FLOAT double
39 #elif __riscv_float_abi_single
40 #define ABI_FLEN 32
41 #define ABI_FLOAT float
42 #endif
43
44 #define NARGREG 8
45 #define STKALIGN 16
46 #define MAXCOPYARG (2 * sizeof(double))
47
48 typedef struct call_context
49 {
50 #if ABI_FLEN
51 ABI_FLOAT fa[8];
52 #endif
53 size_t a[8];
54 /* used by the assembly code to in-place construct its own stack frame */
55 char frame[16];
56 } call_context;
57
58 typedef struct call_builder
59 {
60 call_context *aregs;
61 int used_integer;
62 int used_float;
63 size_t *used_stack;
64 void *struct_stack;
65 } call_builder;
66
67 /* integer (not pointer) less than ABI XLEN */
68 /* FFI_TYPE_INT does not appear to be used */
69 #if __SIZEOF_POINTER__ == 8
70 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT64)
71 #else
72 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT32)
73 #endif
74
75 #if ABI_FLEN
76 typedef struct {
77 char as_elements, type1, offset2, type2;
78 } float_struct_info;
79
80 #if ABI_FLEN >= 64
81 #define IS_FLOAT(type) ((type) >= FFI_TYPE_FLOAT && (type) <= FFI_TYPE_DOUBLE)
82 #else
83 #define IS_FLOAT(type) ((type) == FFI_TYPE_FLOAT)
84 #endif
85
86 static ffi_type **flatten_struct(ffi_type *in, ffi_type **out, ffi_type **out_end) {
87 int i;
88 if (out == out_end) return out;
89 if (in->type != FFI_TYPE_STRUCT) {
90 *(out++) = in;
91 } else {
92 for (i = 0; in->elements[i]; i++)
93 out = flatten_struct(in->elements[i], out, out_end);
94 }
95 return out;
96 }
97
98 /* Structs with at most two fields after flattening, one of which is of
99 floating point type, are passed in multiple registers if sufficient
100 registers are available. */
101 static float_struct_info struct_passed_as_elements(call_builder *cb, ffi_type *top) {
102 float_struct_info ret = {0, 0, 0, 0};
103 ffi_type *fields[3];
104 int num_floats, num_ints;
105 int num_fields = flatten_struct(top, fields, fields + 3) - fields;
106
107 if (num_fields == 1) {
108 if (IS_FLOAT(fields[0]->type)) {
109 ret.as_elements = 1;
110 ret.type1 = fields[0]->type;
111 }
112 } else if (num_fields == 2) {
113 num_floats = IS_FLOAT(fields[0]->type) + IS_FLOAT(fields[1]->type);
114 num_ints = IS_INT(fields[0]->type) + IS_INT(fields[1]->type);
115 if (num_floats == 0 || num_floats + num_ints != 2)
116 return ret;
117 if (cb->used_float + num_floats > NARGREG || cb->used_integer + (2 - num_floats) > NARGREG)
118 return ret;
119 if (!IS_FLOAT(fields[0]->type) && !IS_FLOAT(fields[1]->type))
120 return ret;
121
122 ret.type1 = fields[0]->type;
123 ret.type2 = fields[1]->type;
124 ret.offset2 = FFI_ALIGN(fields[0]->size, fields[1]->alignment);
125 ret.as_elements = 1;
126 }
127
128 return ret;
129 }
130 #endif
131
132 /* allocates a single register, float register, or XLEN-sized stack slot to a datum */
133 static void marshal_atom(call_builder *cb, int type, void *data) {
134 size_t value = 0;
135 switch (type) {
136 case FFI_TYPE_UINT8: value = *(uint8_t *)data; break;
137 case FFI_TYPE_SINT8: value = *(int8_t *)data; break;
138 case FFI_TYPE_UINT16: value = *(uint16_t *)data; break;
139 case FFI_TYPE_SINT16: value = *(int16_t *)data; break;
140 /* 32-bit quantities are always sign-extended in the ABI */
141 case FFI_TYPE_UINT32: value = *(int32_t *)data; break;
142 case FFI_TYPE_SINT32: value = *(int32_t *)data; break;
143 #if __SIZEOF_POINTER__ == 8
144 case FFI_TYPE_UINT64: value = *(uint64_t *)data; break;
145 case FFI_TYPE_SINT64: value = *(int64_t *)data; break;
146 #endif
147 case FFI_TYPE_POINTER: value = *(size_t *)data; break;
148
149 /* float values may be recoded in an implementation-defined way
150 by hardware conforming to 2.1 or earlier, so use asm to
151 reinterpret floats as doubles */
152 #if ABI_FLEN >= 32
153 case FFI_TYPE_FLOAT:
154 asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(float *)data));
155 return;
156 #endif
157 #if ABI_FLEN >= 64
158 case FFI_TYPE_DOUBLE:
159 asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(double *)data));
160 return;
161 #endif
162 default: FFI_ASSERT(0); break;
163 }
164
165 if (cb->used_integer == NARGREG) {
166 *cb->used_stack++ = value;
167 } else {
168 cb->aregs->a[cb->used_integer++] = value;
169 }
170 }
171
172 static void unmarshal_atom(call_builder *cb, int type, void *data) {
173 size_t value;
174 switch (type) {
175 #if ABI_FLEN >= 32
176 case FFI_TYPE_FLOAT:
177 asm("" : "=f"(*(float *)data) : "0"(cb->aregs->fa[cb->used_float++]));
178 return;
179 #endif
180 #if ABI_FLEN >= 64
181 case FFI_TYPE_DOUBLE:
182 asm("" : "=f"(*(double *)data) : "0"(cb->aregs->fa[cb->used_float++]));
183 return;
184 #endif
185 }
186
187 if (cb->used_integer == NARGREG) {
188 value = *cb->used_stack++;
189 } else {
190 value = cb->aregs->a[cb->used_integer++];
191 }
192
193 switch (type) {
194 case FFI_TYPE_UINT8: *(uint8_t *)data = value; break;
195 case FFI_TYPE_SINT8: *(uint8_t *)data = value; break;
196 case FFI_TYPE_UINT16: *(uint16_t *)data = value; break;
197 case FFI_TYPE_SINT16: *(uint16_t *)data = value; break;
198 case FFI_TYPE_UINT32: *(uint32_t *)data = value; break;
199 case FFI_TYPE_SINT32: *(uint32_t *)data = value; break;
200 #if __SIZEOF_POINTER__ == 8
201 case FFI_TYPE_UINT64: *(uint64_t *)data = value; break;
202 case FFI_TYPE_SINT64: *(uint64_t *)data = value; break;
203 #endif
204 case FFI_TYPE_POINTER: *(size_t *)data = value; break;
205 default: FFI_ASSERT(0); break;
206 }
207 }
208
209 /* adds an argument to a call, or a not by reference return value */
210 static void marshal(call_builder *cb, ffi_type *type, int var, void *data) {
211 size_t realign[2];
212
213 #if ABI_FLEN
214 if (!var && type->type == FFI_TYPE_STRUCT) {
215 float_struct_info fsi = struct_passed_as_elements(cb, type);
216 if (fsi.as_elements) {
217 marshal_atom(cb, fsi.type1, data);
218 if (fsi.offset2)
219 marshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
220 return;
221 }
222 }
223
224 if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
225 marshal_atom(cb, type->type, data);
226 return;
227 }
228 #endif
229
230 if (type->size > 2 * __SIZEOF_POINTER__) {
231 /* copy to stack and pass by reference */
232 data = memcpy (cb->struct_stack, data, type->size);
233 cb->struct_stack = (size_t *) FFI_ALIGN ((char *) cb->struct_stack + type->size, __SIZEOF_POINTER__);
234 marshal_atom(cb, FFI_TYPE_POINTER, &data);
235 } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
236 marshal_atom(cb, type->type, data);
237 } else {
238 /* overlong integers, soft-float floats, and structs without special
239 float handling are treated identically from this point on */
240
241 /* variadics are aligned even in registers */
242 if (type->alignment > __SIZEOF_POINTER__) {
243 if (var)
244 cb->used_integer = FFI_ALIGN(cb->used_integer, 2);
245 cb->used_stack = (size_t *)FFI_ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
246 }
247
248 memcpy(realign, data, type->size);
249 if (type->size > 0)
250 marshal_atom(cb, FFI_TYPE_POINTER, realign);
251 if (type->size > __SIZEOF_POINTER__)
252 marshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
253 }
254 }
255
256 /* for arguments passed by reference returns the pointer, otherwise the arg is copied (up to MAXCOPYARG bytes) */
257 static void *unmarshal(call_builder *cb, ffi_type *type, int var, void *data) {
258 size_t realign[2];
259 void *pointer;
260
261 #if ABI_FLEN
262 if (!var && type->type == FFI_TYPE_STRUCT) {
263 float_struct_info fsi = struct_passed_as_elements(cb, type);
264 if (fsi.as_elements) {
265 unmarshal_atom(cb, fsi.type1, data);
266 if (fsi.offset2)
267 unmarshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
268 return data;
269 }
270 }
271
272 if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
273 unmarshal_atom(cb, type->type, data);
274 return data;
275 }
276 #endif
277
278 if (type->size > 2 * __SIZEOF_POINTER__) {
279 /* pass by reference */
280 unmarshal_atom(cb, FFI_TYPE_POINTER, (char*)&pointer);
281 return pointer;
282 } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
283 unmarshal_atom(cb, type->type, data);
284 return data;
285 } else {
286 /* overlong integers, soft-float floats, and structs without special
287 float handling are treated identically from this point on */
288
289 /* variadics are aligned even in registers */
290 if (type->alignment > __SIZEOF_POINTER__) {
291 if (var)
292 cb->used_integer = FFI_ALIGN(cb->used_integer, 2);
293 cb->used_stack = (size_t *)FFI_ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
294 }
295
296 if (type->size > 0)
297 unmarshal_atom(cb, FFI_TYPE_POINTER, realign);
298 if (type->size > __SIZEOF_POINTER__)
299 unmarshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
300 memcpy(data, realign, type->size);
301 return data;
302 }
303 }
304
305 static int passed_by_ref(call_builder *cb, ffi_type *type, int var) {
306 #if ABI_FLEN
307 if (!var && type->type == FFI_TYPE_STRUCT) {
308 float_struct_info fsi = struct_passed_as_elements(cb, type);
309 if (fsi.as_elements) return 0;
310 }
311 #endif
312
313 return type->size > 2 * __SIZEOF_POINTER__;
314 }
315
316 /* Perform machine dependent cif processing */
317 ffi_status ffi_prep_cif_machdep(ffi_cif *cif) {
318 cif->riscv_nfixedargs = cif->nargs;
319 return FFI_OK;
320 }
321
322 /* Perform machine dependent cif processing when we have a variadic function */
323
324 ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs) {
325 cif->riscv_nfixedargs = nfixedargs;
326 return FFI_OK;
327 }
328
329 /* Low level routine for calling functions */
330 extern void ffi_call_asm (void *stack, struct call_context *regs,
331 void (*fn) (void), void *closure) FFI_HIDDEN;
332
333 static void
334 ffi_call_int (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue,
335 void *closure)
336 {
337 /* this is a conservative estimate, assuming a complex return value and
338 that all remaining arguments are long long / __int128 */
339 size_t arg_bytes = cif->nargs <= 3 ? 0 :
340 FFI_ALIGN(2 * sizeof(size_t) * (cif->nargs - 3), STKALIGN);
341 /* Allocate space for copies of big structures. */
342 size_t struct_bytes = FFI_ALIGN (cif->bytes, STKALIGN);
343 size_t rval_bytes = 0;
344 if (rvalue == NULL && cif->rtype->size > 2*__SIZEOF_POINTER__)
345 rval_bytes = FFI_ALIGN(cif->rtype->size, STKALIGN);
346 size_t alloc_size = arg_bytes + rval_bytes + struct_bytes + sizeof(call_context);
347
348 /* the assembly code will deallocate all stack data at lower addresses
349 than the argument region, so we need to allocate the frame and the
350 return value after the arguments in a single allocation */
351 size_t alloc_base;
352 /* Argument region must be 16-byte aligned */
353 if (_Alignof(max_align_t) >= STKALIGN) {
354 /* since sizeof long double is normally 16, the compiler will
355 guarantee alloca alignment to at least that much */
356 alloc_base = (size_t)alloca(alloc_size);
357 } else {
358 alloc_base = FFI_ALIGN(alloca(alloc_size + STKALIGN - 1), STKALIGN);
359 }
360
361 if (rval_bytes)
362 rvalue = (void*)(alloc_base + arg_bytes);
363
364 call_builder cb;
365 cb.used_float = cb.used_integer = 0;
366 cb.aregs = (call_context*)(alloc_base + arg_bytes + rval_bytes + struct_bytes);
367 cb.used_stack = (void*)alloc_base;
368 cb.struct_stack = (void *) (alloc_base + arg_bytes + rval_bytes);
369
370 int return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
371 if (return_by_ref)
372 marshal(&cb, &ffi_type_pointer, 0, &rvalue);
373
374 int i;
375 for (i = 0; i < cif->nargs; i++)
376 marshal(&cb, cif->arg_types[i], i >= cif->riscv_nfixedargs, avalue[i]);
377
378 ffi_call_asm ((void *) alloc_base, cb.aregs, fn, closure);
379
380 cb.used_float = cb.used_integer = 0;
381 if (!return_by_ref && rvalue)
382 {
383 if (IS_INT(cif->rtype->type)
384 && cif->rtype->size < sizeof (ffi_arg))
385 {
386 /* Integer types smaller than ffi_arg need to be extended. */
387 switch (cif->rtype->type)
388 {
389 case FFI_TYPE_SINT8:
390 case FFI_TYPE_SINT16:
391 case FFI_TYPE_SINT32:
392 unmarshal_atom (&cb, (sizeof (ffi_arg) > 4
393 ? FFI_TYPE_SINT64 : FFI_TYPE_SINT32),
394 rvalue);
395 break;
396 case FFI_TYPE_UINT8:
397 case FFI_TYPE_UINT16:
398 case FFI_TYPE_UINT32:
399 unmarshal_atom (&cb, (sizeof (ffi_arg) > 4
400 ? FFI_TYPE_UINT64 : FFI_TYPE_UINT32),
401 rvalue);
402 break;
403 }
404 }
405 else
406 unmarshal(&cb, cif->rtype, 0, rvalue);
407 }
408 }
409
410 void
411 ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
412 {
413 ffi_call_int(cif, fn, rvalue, avalue, NULL);
414 }
415
416 void
417 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
418 void **avalue, void *closure)
419 {
420 ffi_call_int(cif, fn, rvalue, avalue, closure);
421 }
422
423 extern void ffi_closure_asm(void) FFI_HIDDEN;
424
425 ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, void *codeloc)
426 {
427 uint32_t *tramp = (uint32_t *) &closure->tramp[0];
428 uint64_t fn = (uint64_t) (uintptr_t) ffi_closure_asm;
429
430 if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
431 return FFI_BAD_ABI;
432
433 /* we will call ffi_closure_inner with codeloc, not closure, but as long
434 as the memory is readable it should work */
435
436 tramp[0] = 0x00000317; /* auipc t1, 0 (i.e. t0 <- codeloc) */
437 #if __SIZEOF_POINTER__ == 8
438 tramp[1] = 0x01033383; /* ld t2, 16(t1) */
439 #else
440 tramp[1] = 0x01032383; /* lw t2, 16(t1) */
441 #endif
442 tramp[2] = 0x00038067; /* jr t2 */
443 tramp[3] = 0x00000013; /* nop */
444 tramp[4] = fn;
445 tramp[5] = fn >> 32;
446
447 closure->cif = cif;
448 closure->fun = fun;
449 closure->user_data = user_data;
450
451 #if !defined(__FreeBSD__)
452 __builtin___clear_cache(codeloc, codeloc + FFI_TRAMPOLINE_SIZE);
453 #endif
454
455 return FFI_OK;
456 }
457
458 extern void ffi_go_closure_asm (void) FFI_HIDDEN;
459
460 ffi_status
461 ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
462 void (*fun) (ffi_cif *, void *, void **, void *))
463 {
464 if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
465 return FFI_BAD_ABI;
466
467 closure->tramp = (void *) ffi_go_closure_asm;
468 closure->cif = cif;
469 closure->fun = fun;
470
471 return FFI_OK;
472 }
473
474 /* Called by the assembly code with aregs pointing to saved argument registers
475 and stack pointing to the stacked arguments. Return values passed in
476 registers will be reloaded from aregs. */
477 void FFI_HIDDEN
478 ffi_closure_inner (ffi_cif *cif,
479 void (*fun) (ffi_cif *, void *, void **, void *),
480 void *user_data,
481 size_t *stack, call_context *aregs)
482 {
483 void **avalue = alloca(cif->nargs * sizeof(void*));
484 /* storage for arguments which will be copied by unmarshal(). We could
485 theoretically avoid the copies in many cases and use at most 128 bytes
486 of memory, but allocating disjoint storage for each argument is
487 simpler. */
488 char *astorage = alloca(cif->nargs * MAXCOPYARG);
489 void *rvalue;
490 call_builder cb;
491 int return_by_ref;
492 int i;
493
494 cb.aregs = aregs;
495 cb.used_integer = cb.used_float = 0;
496 cb.used_stack = stack;
497
498 return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
499 if (return_by_ref)
500 unmarshal(&cb, &ffi_type_pointer, 0, &rvalue);
501 else
502 rvalue = alloca(cif->rtype->size);
503
504 for (i = 0; i < cif->nargs; i++)
505 avalue[i] = unmarshal(&cb, cif->arg_types[i],
506 i >= cif->riscv_nfixedargs, astorage + i*MAXCOPYARG);
507
508 fun (cif, rvalue, avalue, user_data);
509
510 if (!return_by_ref && cif->rtype->type != FFI_TYPE_VOID) {
511 cb.used_integer = cb.used_float = 0;
512 marshal(&cb, cif->rtype, 0, rvalue);
513 }
514 }