Debian Codegraph @

src/x86/unix64.S

raw
1 /* -----------------------------------------------------------------------
2 unix64.S - Copyright (c) 2013 The Written Word, Inc.
3 - Copyright (c) 2008 Red Hat, Inc
4 - Copyright (c) 2002 Bo Thorsen <[email protected]>
5
6 x86-64 Foreign Function Interface
7
8 Permission is hereby granted, free of charge, to any person obtaining
9 a copy of this software and associated documentation files (the
10 ``Software''), to deal in the Software without restriction, including
11 without limitation the rights to use, copy, modify, merge, publish,
12 distribute, sublicense, and/or sell copies of the Software, and to
13 permit persons to whom the Software is furnished to do so, subject to
14 the following conditions:
15
16 The above copyright notice and this permission notice shall be included
17 in all copies or substantial portions of the Software.
18
19 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
20 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
23 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
24 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 DEALINGS IN THE SOFTWARE.
27 ----------------------------------------------------------------------- */
28
29 #ifdef __x86_64__
30 #define LIBFFI_ASM
31 #include <fficonfig.h>
32 #include <ffi.h>
33 #include "internal64.h"
34 #include "asmnames.h"
35
36 .text
37
38 /* This macro allows the safe creation of jump tables without an
39 actual table. The entry points into the table are all 8 bytes.
40 The use of ORG asserts that we're at the correct location. */
41 /* ??? The clang assembler doesn't handle .org with symbolic expressions. */
42 #ifdef __CET__
43 /* Double slot size to 16 byte to add 4 bytes of ENDBR64. */
44 # define E(BASE, X) .balign 8; .org BASE + X * 16
45 #elif defined(__clang__) || defined(__APPLE__) || (defined (__sun__) && defined(__svr4__))
46 # define E(BASE, X) .balign 8
47 #else
48 # define E(BASE, X) .balign 8; .org BASE + X * 8
49 #endif
50
51 /* ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
52 void *raddr, void (*fnaddr)(void));
53
54 Bit o trickiness here -- ARGS+BYTES is the base of the stack frame
55 for this function. This has been allocated by ffi_call. We also
56 deallocate some of the stack that has been alloca'd. */
57
58 .balign 8
59 .globl C(ffi_call_unix64)
60 FFI_HIDDEN(C(ffi_call_unix64))
61
62 C(ffi_call_unix64):
63 L(UW0):
64 _CET_ENDBR
65 movq (%rsp), %r10 /* Load return address. */
66 leaq (%rdi, %rsi), %rax /* Find local stack base. */
67 movq %rdx, (%rax) /* Save flags. */
68 movq %rcx, 8(%rax) /* Save raddr. */
69 movq %rbp, 16(%rax) /* Save old frame pointer. */
70 movq %r10, 24(%rax) /* Relocate return address. */
71 movq %rax, %rbp /* Finalize local stack frame. */
72
73 /* New stack frame based off rbp. This is a itty bit of unwind
74 trickery in that the CFA *has* changed. There is no easy way
75 to describe it correctly on entry to the function. Fortunately,
76 it doesn't matter too much since at all points we can correctly
77 unwind back to ffi_call. Note that the location to which we
78 moved the return address is (the new) CFA-8, so from the
79 perspective of the unwind info, it hasn't moved. */
80 L(UW1):
81 /* cfi_def_cfa(%rbp, 32) */
82 /* cfi_rel_offset(%rbp, 16) */
83
84 movq %rdi, %r10 /* Save a copy of the register area. */
85 movq %r8, %r11 /* Save a copy of the target fn. */
86
87 /* Load up all argument registers. */
88 movq (%r10), %rdi
89 movq 0x08(%r10), %rsi
90 movq 0x10(%r10), %rdx
91 movq 0x18(%r10), %rcx
92 movq 0x20(%r10), %r8
93 movq 0x28(%r10), %r9
94 movl 0xb0(%r10), %eax /* Set number of SSE registers. */
95 testl %eax, %eax
96 jnz L(load_sse)
97 L(ret_from_load_sse):
98
99 /* Deallocate the reg arg area, except for r10, then load via pop. */
100 leaq 0xb8(%r10), %rsp
101 popq %r10
102
103 /* Call the user function. */
104 call *%r11
105
106 /* Deallocate stack arg area; local stack frame in redzone. */
107 leaq 24(%rbp), %rsp
108
109 movq 0(%rbp), %rcx /* Reload flags. */
110 movq 8(%rbp), %rdi /* Reload raddr. */
111 movq 16(%rbp), %rbp /* Reload old frame pointer. */
112 L(UW2):
113 /* cfi_remember_state */
114 /* cfi_def_cfa(%rsp, 8) */
115 /* cfi_restore(%rbp) */
116
117 /* The first byte of the flags contains the FFI_TYPE. */
118 cmpb $UNIX64_RET_LAST, %cl
119 movzbl %cl, %r10d
120 leaq L(store_table)(%rip), %r11
121 ja L(sa)
122 #ifdef __CET__
123 /* NB: Originally, each slot is 8 byte. 4 bytes of ENDBR64 +
124 4 bytes NOP padding double slot size to 16 bytes. */
125 addl %r10d, %r10d
126 #endif
127 leaq (%r11, %r10, 8), %r10
128
129 /* Prep for the structure cases: scratch area in redzone. */
130 leaq -20(%rsp), %rsi
131 jmp *%r10
132
133 .balign 8
134 L(store_table):
135 E(L(store_table), UNIX64_RET_VOID)
136 _CET_ENDBR
137 ret
138 E(L(store_table), UNIX64_RET_UINT8)
139 _CET_ENDBR
140 movzbl %al, %eax
141 movq %rax, (%rdi)
142 ret
143 E(L(store_table), UNIX64_RET_UINT16)
144 _CET_ENDBR
145 movzwl %ax, %eax
146 movq %rax, (%rdi)
147 ret
148 E(L(store_table), UNIX64_RET_UINT32)
149 _CET_ENDBR
150 movl %eax, %eax
151 movq %rax, (%rdi)
152 ret
153 E(L(store_table), UNIX64_RET_SINT8)
154 _CET_ENDBR
155 movsbq %al, %rax
156 movq %rax, (%rdi)
157 ret
158 E(L(store_table), UNIX64_RET_SINT16)
159 _CET_ENDBR
160 movswq %ax, %rax
161 movq %rax, (%rdi)
162 ret
163 E(L(store_table), UNIX64_RET_SINT32)
164 _CET_ENDBR
165 cltq
166 movq %rax, (%rdi)
167 ret
168 E(L(store_table), UNIX64_RET_INT64)
169 _CET_ENDBR
170 movq %rax, (%rdi)
171 ret
172 E(L(store_table), UNIX64_RET_XMM32)
173 _CET_ENDBR
174 movd %xmm0, (%rdi)
175 ret
176 E(L(store_table), UNIX64_RET_XMM64)
177 _CET_ENDBR
178 movq %xmm0, (%rdi)
179 ret
180 E(L(store_table), UNIX64_RET_X87)
181 _CET_ENDBR
182 fstpt (%rdi)
183 ret
184 E(L(store_table), UNIX64_RET_X87_2)
185 _CET_ENDBR
186 fstpt (%rdi)
187 fstpt 16(%rdi)
188 ret
189 E(L(store_table), UNIX64_RET_ST_XMM0_RAX)
190 _CET_ENDBR
191 movq %rax, 8(%rsi)
192 jmp L(s3)
193 E(L(store_table), UNIX64_RET_ST_RAX_XMM0)
194 _CET_ENDBR
195 movq %xmm0, 8(%rsi)
196 jmp L(s2)
197 E(L(store_table), UNIX64_RET_ST_XMM0_XMM1)
198 _CET_ENDBR
199 movq %xmm1, 8(%rsi)
200 jmp L(s3)
201 E(L(store_table), UNIX64_RET_ST_RAX_RDX)
202 _CET_ENDBR
203 movq %rdx, 8(%rsi)
204 L(s2):
205 movq %rax, (%rsi)
206 shrl $UNIX64_SIZE_SHIFT, %ecx
207 rep movsb
208 ret
209 .balign 8
210 L(s3):
211 movq %xmm0, (%rsi)
212 shrl $UNIX64_SIZE_SHIFT, %ecx
213 rep movsb
214 ret
215
216 L(sa): call PLT(C(abort))
217
218 /* Many times we can avoid loading any SSE registers at all.
219 It's not worth an indirect jump to load the exact set of
220 SSE registers needed; zero or all is a good compromise. */
221 .balign 2
222 L(UW3):
223 /* cfi_restore_state */
224 L(load_sse):
225 movdqa 0x30(%r10), %xmm0
226 movdqa 0x40(%r10), %xmm1
227 movdqa 0x50(%r10), %xmm2
228 movdqa 0x60(%r10), %xmm3
229 movdqa 0x70(%r10), %xmm4
230 movdqa 0x80(%r10), %xmm5
231 movdqa 0x90(%r10), %xmm6
232 movdqa 0xa0(%r10), %xmm7
233 jmp L(ret_from_load_sse)
234
235 L(UW4):
236 ENDF(C(ffi_call_unix64))
237
238 /* 6 general registers, 8 vector registers,
239 32 bytes of rvalue, 8 bytes of alignment. */
240 #define ffi_closure_OFS_G 0
241 #define ffi_closure_OFS_V (6*8)
242 #define ffi_closure_OFS_RVALUE (ffi_closure_OFS_V + 8*16)
243 #define ffi_closure_FS (ffi_closure_OFS_RVALUE + 32 + 8)
244
245 /* The location of rvalue within the red zone after deallocating the frame. */
246 #define ffi_closure_RED_RVALUE (ffi_closure_OFS_RVALUE - ffi_closure_FS)
247
248 .balign 2
249 .globl C(ffi_closure_unix64_sse)
250 FFI_HIDDEN(C(ffi_closure_unix64_sse))
251
252 C(ffi_closure_unix64_sse):
253 L(UW5):
254 _CET_ENDBR
255 subq $ffi_closure_FS, %rsp
256 L(UW6):
257 /* cfi_adjust_cfa_offset(ffi_closure_FS) */
258
259 movdqa %xmm0, ffi_closure_OFS_V+0x00(%rsp)
260 movdqa %xmm1, ffi_closure_OFS_V+0x10(%rsp)
261 movdqa %xmm2, ffi_closure_OFS_V+0x20(%rsp)
262 movdqa %xmm3, ffi_closure_OFS_V+0x30(%rsp)
263 movdqa %xmm4, ffi_closure_OFS_V+0x40(%rsp)
264 movdqa %xmm5, ffi_closure_OFS_V+0x50(%rsp)
265 movdqa %xmm6, ffi_closure_OFS_V+0x60(%rsp)
266 movdqa %xmm7, ffi_closure_OFS_V+0x70(%rsp)
267 jmp L(sse_entry1)
268
269 L(UW7):
270 ENDF(C(ffi_closure_unix64_sse))
271
272 .balign 2
273 .globl C(ffi_closure_unix64)
274 FFI_HIDDEN(C(ffi_closure_unix64))
275
276 C(ffi_closure_unix64):
277 L(UW8):
278 _CET_ENDBR
279 subq $ffi_closure_FS, %rsp
280 L(UW9):
281 /* cfi_adjust_cfa_offset(ffi_closure_FS) */
282 L(sse_entry1):
283 movq %rdi, ffi_closure_OFS_G+0x00(%rsp)
284 movq %rsi, ffi_closure_OFS_G+0x08(%rsp)
285 movq %rdx, ffi_closure_OFS_G+0x10(%rsp)
286 movq %rcx, ffi_closure_OFS_G+0x18(%rsp)
287 movq %r8, ffi_closure_OFS_G+0x20(%rsp)
288 movq %r9, ffi_closure_OFS_G+0x28(%rsp)
289
290 #ifdef __ILP32__
291 movl FFI_TRAMPOLINE_SIZE(%r10), %edi /* Load cif */
292 movl FFI_TRAMPOLINE_SIZE+4(%r10), %esi /* Load fun */
293 movl FFI_TRAMPOLINE_SIZE+8(%r10), %edx /* Load user_data */
294 #else
295 movq FFI_TRAMPOLINE_SIZE(%r10), %rdi /* Load cif */
296 movq FFI_TRAMPOLINE_SIZE+8(%r10), %rsi /* Load fun */
297 movq FFI_TRAMPOLINE_SIZE+16(%r10), %rdx /* Load user_data */
298 #endif
299 L(do_closure):
300 leaq ffi_closure_OFS_RVALUE(%rsp), %rcx /* Load rvalue */
301 movq %rsp, %r8 /* Load reg_args */
302 leaq ffi_closure_FS+8(%rsp), %r9 /* Load argp */
303 call PLT(C(ffi_closure_unix64_inner))
304
305 /* Deallocate stack frame early; return value is now in redzone. */
306 addq $ffi_closure_FS, %rsp
307 L(UW10):
308 /* cfi_adjust_cfa_offset(-ffi_closure_FS) */
309
310 /* The first byte of the return value contains the FFI_TYPE. */
311 cmpb $UNIX64_RET_LAST, %al
312 movzbl %al, %r10d
313 leaq L(load_table)(%rip), %r11
314 ja L(la)
315 #ifdef __CET__
316 /* NB: Originally, each slot is 8 byte. 4 bytes of ENDBR64 +
317 4 bytes NOP padding double slot size to 16 bytes. */
318 addl %r10d, %r10d
319 #endif
320 leaq (%r11, %r10, 8), %r10
321 leaq ffi_closure_RED_RVALUE(%rsp), %rsi
322 jmp *%r10
323
324 .balign 8
325 L(load_table):
326 E(L(load_table), UNIX64_RET_VOID)
327 _CET_ENDBR
328 ret
329 E(L(load_table), UNIX64_RET_UINT8)
330 _CET_ENDBR
331 movzbl (%rsi), %eax
332 ret
333 E(L(load_table), UNIX64_RET_UINT16)
334 _CET_ENDBR
335 movzwl (%rsi), %eax
336 ret
337 E(L(load_table), UNIX64_RET_UINT32)
338 _CET_ENDBR
339 movl (%rsi), %eax
340 ret
341 E(L(load_table), UNIX64_RET_SINT8)
342 _CET_ENDBR
343 movsbl (%rsi), %eax
344 ret
345 E(L(load_table), UNIX64_RET_SINT16)
346 _CET_ENDBR
347 movswl (%rsi), %eax
348 ret
349 E(L(load_table), UNIX64_RET_SINT32)
350 _CET_ENDBR
351 movl (%rsi), %eax
352 ret
353 E(L(load_table), UNIX64_RET_INT64)
354 _CET_ENDBR
355 movq (%rsi), %rax
356 ret
357 E(L(load_table), UNIX64_RET_XMM32)
358 _CET_ENDBR
359 movd (%rsi), %xmm0
360 ret
361 E(L(load_table), UNIX64_RET_XMM64)
362 _CET_ENDBR
363 movq (%rsi), %xmm0
364 ret
365 E(L(load_table), UNIX64_RET_X87)
366 _CET_ENDBR
367 fldt (%rsi)
368 ret
369 E(L(load_table), UNIX64_RET_X87_2)
370 _CET_ENDBR
371 fldt 16(%rsi)
372 fldt (%rsi)
373 ret
374 E(L(load_table), UNIX64_RET_ST_XMM0_RAX)
375 _CET_ENDBR
376 movq 8(%rsi), %rax
377 jmp L(l3)
378 E(L(load_table), UNIX64_RET_ST_RAX_XMM0)
379 _CET_ENDBR
380 movq 8(%rsi), %xmm0
381 jmp L(l2)
382 E(L(load_table), UNIX64_RET_ST_XMM0_XMM1)
383 _CET_ENDBR
384 movq 8(%rsi), %xmm1
385 jmp L(l3)
386 E(L(load_table), UNIX64_RET_ST_RAX_RDX)
387 _CET_ENDBR
388 movq 8(%rsi), %rdx
389 L(l2):
390 movq (%rsi), %rax
391 ret
392 .balign 8
393 L(l3):
394 movq (%rsi), %xmm0
395 ret
396
397 L(la): call PLT(C(abort))
398
399 L(UW11):
400 ENDF(C(ffi_closure_unix64))
401
402 .balign 2
403 .globl C(ffi_go_closure_unix64_sse)
404 FFI_HIDDEN(C(ffi_go_closure_unix64_sse))
405
406 C(ffi_go_closure_unix64_sse):
407 L(UW12):
408 _CET_ENDBR
409 subq $ffi_closure_FS, %rsp
410 L(UW13):
411 /* cfi_adjust_cfa_offset(ffi_closure_FS) */
412
413 movdqa %xmm0, ffi_closure_OFS_V+0x00(%rsp)
414 movdqa %xmm1, ffi_closure_OFS_V+0x10(%rsp)
415 movdqa %xmm2, ffi_closure_OFS_V+0x20(%rsp)
416 movdqa %xmm3, ffi_closure_OFS_V+0x30(%rsp)
417 movdqa %xmm4, ffi_closure_OFS_V+0x40(%rsp)
418 movdqa %xmm5, ffi_closure_OFS_V+0x50(%rsp)
419 movdqa %xmm6, ffi_closure_OFS_V+0x60(%rsp)
420 movdqa %xmm7, ffi_closure_OFS_V+0x70(%rsp)
421 jmp L(sse_entry2)
422
423 L(UW14):
424 ENDF(C(ffi_go_closure_unix64_sse))
425
426 .balign 2
427 .globl C(ffi_go_closure_unix64)
428 FFI_HIDDEN(C(ffi_go_closure_unix64))
429
430 C(ffi_go_closure_unix64):
431 L(UW15):
432 _CET_ENDBR
433 subq $ffi_closure_FS, %rsp
434 L(UW16):
435 /* cfi_adjust_cfa_offset(ffi_closure_FS) */
436 L(sse_entry2):
437 movq %rdi, ffi_closure_OFS_G+0x00(%rsp)
438 movq %rsi, ffi_closure_OFS_G+0x08(%rsp)
439 movq %rdx, ffi_closure_OFS_G+0x10(%rsp)
440 movq %rcx, ffi_closure_OFS_G+0x18(%rsp)
441 movq %r8, ffi_closure_OFS_G+0x20(%rsp)
442 movq %r9, ffi_closure_OFS_G+0x28(%rsp)
443
444 #ifdef __ILP32__
445 movl 4(%r10), %edi /* Load cif */
446 movl 8(%r10), %esi /* Load fun */
447 movl %r10d, %edx /* Load closure (user_data) */
448 #else
449 movq 8(%r10), %rdi /* Load cif */
450 movq 16(%r10), %rsi /* Load fun */
451 movq %r10, %rdx /* Load closure (user_data) */
452 #endif
453 jmp L(do_closure)
454
455 L(UW17):
456 ENDF(C(ffi_go_closure_unix64))
457
458 #if defined(FFI_EXEC_STATIC_TRAMP)
459 .balign 8
460 .globl C(ffi_closure_unix64_sse_alt)
461 FFI_HIDDEN(C(ffi_closure_unix64_sse_alt))
462
463 C(ffi_closure_unix64_sse_alt):
464 /* See the comments above trampoline_code_table. */
465 _CET_ENDBR
466 movq 8(%rsp), %r10 /* Load closure in r10 */
467 addq $16, %rsp /* Restore the stack */
468 jmp C(ffi_closure_unix64_sse)
469 ENDF(C(ffi_closure_unix64_sse_alt))
470
471 .balign 8
472 .globl C(ffi_closure_unix64_alt)
473 FFI_HIDDEN(C(ffi_closure_unix64_alt))
474
475 C(ffi_closure_unix64_alt):
476 /* See the comments above trampoline_code_table. */
477 _CET_ENDBR
478 movq 8(%rsp), %r10 /* Load closure in r10 */
479 addq $16, %rsp /* Restore the stack */
480 jmp C(ffi_closure_unix64)
481 ENDF(C(ffi_closure_unix64_alt))
482
483 /*
484 * Below is the definition of the trampoline code table. Each element in
485 * the code table is a trampoline.
486 *
487 * Because we jump to the trampoline, we place a _CET_ENDBR at the
488 * beginning of the trampoline to mark it as a valid branch target. This is
489 * part of the the Intel CET (Control Flow Enforcement Technology).
490 */
491 /*
492 * The trampoline uses register r10. It saves the original value of r10 on
493 * the stack.
494 *
495 * The trampoline has two parameters - target code to jump to and data for
496 * the target code. The trampoline extracts the parameters from its parameter
497 * block (see tramp_table_map()). The trampoline saves the data address on
498 * the stack. Finally, it jumps to the target code.
499 *
500 * The target code can choose to:
501 *
502 * - restore the value of r10
503 * - load the data address in a register
504 * - restore the stack pointer to what it was when the trampoline was invoked.
505 */
506 #ifdef ENDBR_PRESENT
507 # define X86_DATA_OFFSET 4077
508 # ifdef __ILP32__
509 # define X86_CODE_OFFSET 4069
510 # else
511 # define X86_CODE_OFFSET 4073
512 # endif
513 #else
514 # define X86_DATA_OFFSET 4081
515 # ifdef __ILP32__
516 # define X86_CODE_OFFSET 4073
517 # else
518 # define X86_CODE_OFFSET 4077
519 # endif
520 #endif
521
522 .align UNIX64_TRAMP_MAP_SIZE
523 .globl trampoline_code_table
524 FFI_HIDDEN(C(trampoline_code_table))
525
526 C(trampoline_code_table):
527 .rept UNIX64_TRAMP_MAP_SIZE / UNIX64_TRAMP_SIZE
528 _CET_ENDBR
529 subq $16, %rsp /* Make space on the stack */
530 movq %r10, (%rsp) /* Save %r10 on stack */
531 #ifdef __ILP32__
532 movl X86_DATA_OFFSET(%rip), %r10d /* Copy data into %r10 */
533 #else
534 movq X86_DATA_OFFSET(%rip), %r10 /* Copy data into %r10 */
535 #endif
536 movq %r10, 8(%rsp) /* Save data on stack */
537 #ifdef __ILP32__
538 movl X86_CODE_OFFSET(%rip), %r10d /* Copy code into %r10 */
539 #else
540 movq X86_CODE_OFFSET(%rip), %r10 /* Copy code into %r10 */
541 #endif
542 jmp *%r10 /* Jump to code */
543 .align 8
544 .endr
545 ENDF(C(trampoline_code_table))
546 .align UNIX64_TRAMP_MAP_SIZE
547 #endif /* FFI_EXEC_STATIC_TRAMP */
548
549 /* Sadly, OSX cctools-as doesn't understand .cfi directives at all. */
550
551 #ifdef __APPLE__
552 .section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support
553 EHFrame0:
554 #elif defined(HAVE_AS_X86_64_UNWIND_SECTION_TYPE)
555 .section .eh_frame,"a",@unwind
556 #else
557 .section .eh_frame,"a",@progbits
558 #endif
559
560 #ifdef HAVE_AS_X86_PCREL
561 # define PCREL(X) X - .
562 #else
563 # define PCREL(X) X@rel
564 #endif
565
566 /* Simplify advancing between labels. Assume DW_CFA_advance_loc1 fits. */
567 #ifdef __CET__
568 /* Use DW_CFA_advance_loc2 when IBT is enabled. */
569 # define ADV(N, P) .byte 3; .2byte L(N)-L(P)
570 #else
571 # define ADV(N, P) .byte 2, L(N)-L(P)
572 #endif
573
574 .balign 8
575 L(CIE):
576 .set L(set0),L(ECIE)-L(SCIE)
577 .long L(set0) /* CIE Length */
578 L(SCIE):
579 .long 0 /* CIE Identifier Tag */
580 .byte 1 /* CIE Version */
581 .ascii "zR\0" /* CIE Augmentation */
582 .byte 1 /* CIE Code Alignment Factor */
583 .byte 0x78 /* CIE Data Alignment Factor */
584 .byte 0x10 /* CIE RA Column */
585 .byte 1 /* Augmentation size */
586 .byte 0x1b /* FDE Encoding (pcrel sdata4) */
587 .byte 0xc, 7, 8 /* DW_CFA_def_cfa, %rsp offset 8 */
588 .byte 0x80+16, 1 /* DW_CFA_offset, %rip offset 1*-8 */
589 .balign 8
590 L(ECIE):
591
592 .set L(set1),L(EFDE1)-L(SFDE1)
593 .long L(set1) /* FDE Length */
594 L(SFDE1):
595 .long L(SFDE1)-L(CIE) /* FDE CIE offset */
596 .long PCREL(L(UW0)) /* Initial location */
597 .long L(UW4)-L(UW0) /* Address range */
598 .byte 0 /* Augmentation size */
599 ADV(UW1, UW0)
600 .byte 0xc, 6, 32 /* DW_CFA_def_cfa, %rbp 32 */
601 .byte 0x80+6, 2 /* DW_CFA_offset, %rbp 2*-8 */
602 ADV(UW2, UW1)
603 .byte 0xa /* DW_CFA_remember_state */
604 .byte 0xc, 7, 8 /* DW_CFA_def_cfa, %rsp 8 */
605 .byte 0xc0+6 /* DW_CFA_restore, %rbp */
606 ADV(UW3, UW2)
607 .byte 0xb /* DW_CFA_restore_state */
608 .balign 8
609 L(EFDE1):
610
611 .set L(set2),L(EFDE2)-L(SFDE2)
612 .long L(set2) /* FDE Length */
613 L(SFDE2):
614 .long L(SFDE2)-L(CIE) /* FDE CIE offset */
615 .long PCREL(L(UW5)) /* Initial location */
616 .long L(UW7)-L(UW5) /* Address range */
617 .byte 0 /* Augmentation size */
618 ADV(UW6, UW5)
619 .byte 0xe /* DW_CFA_def_cfa_offset */
620 .byte ffi_closure_FS + 8, 1 /* uleb128, assuming 128 <= FS < 255 */
621 .balign 8
622 L(EFDE2):
623
624 .set L(set3),L(EFDE3)-L(SFDE3)
625 .long L(set3) /* FDE Length */
626 L(SFDE3):
627 .long L(SFDE3)-L(CIE) /* FDE CIE offset */
628 .long PCREL(L(UW8)) /* Initial location */
629 .long L(UW11)-L(UW8) /* Address range */
630 .byte 0 /* Augmentation size */
631 ADV(UW9, UW8)
632 .byte 0xe /* DW_CFA_def_cfa_offset */
633 .byte ffi_closure_FS + 8, 1 /* uleb128, assuming 128 <= FS < 255 */
634 ADV(UW10, UW9)
635 .byte 0xe, 8 /* DW_CFA_def_cfa_offset 8 */
636 L(EFDE3):
637
638 .set L(set4),L(EFDE4)-L(SFDE4)
639 .long L(set4) /* FDE Length */
640 L(SFDE4):
641 .long L(SFDE4)-L(CIE) /* FDE CIE offset */
642 .long PCREL(L(UW12)) /* Initial location */
643 .long L(UW14)-L(UW12) /* Address range */
644 .byte 0 /* Augmentation size */
645 ADV(UW13, UW12)
646 .byte 0xe /* DW_CFA_def_cfa_offset */
647 .byte ffi_closure_FS + 8, 1 /* uleb128, assuming 128 <= FS < 255 */
648 .balign 8
649 L(EFDE4):
650
651 .set L(set5),L(EFDE5)-L(SFDE5)
652 .long L(set5) /* FDE Length */
653 L(SFDE5):
654 .long L(SFDE5)-L(CIE) /* FDE CIE offset */
655 .long PCREL(L(UW15)) /* Initial location */
656 .long L(UW17)-L(UW15) /* Address range */
657 .byte 0 /* Augmentation size */
658 ADV(UW16, UW15)
659 .byte 0xe /* DW_CFA_def_cfa_offset */
660 .byte ffi_closure_FS + 8, 1 /* uleb128, assuming 128 <= FS < 255 */
661 .balign 8
662 L(EFDE5):
663 #ifdef __APPLE__
664 .subsections_via_symbols
665 .section __LD,__compact_unwind,regular,debug
666
667 /* compact unwind for ffi_call_unix64 */
668 .quad C(ffi_call_unix64)
669 .set L1,L(UW4)-L(UW0)
670 .long L1
671 .long 0x04000000 /* use dwarf unwind info */
672 .quad 0
673 .quad 0
674
675 /* compact unwind for ffi_closure_unix64_sse */
676 .quad C(ffi_closure_unix64_sse)
677 .set L2,L(UW7)-L(UW5)
678 .long L2
679 .long 0x04000000 /* use dwarf unwind info */
680 .quad 0
681 .quad 0
682
683 /* compact unwind for ffi_closure_unix64 */
684 .quad C(ffi_closure_unix64)
685 .set L3,L(UW11)-L(UW8)
686 .long L3
687 .long 0x04000000 /* use dwarf unwind info */
688 .quad 0
689 .quad 0
690
691 /* compact unwind for ffi_go_closure_unix64_sse */
692 .quad C(ffi_go_closure_unix64_sse)
693 .set L4,L(UW14)-L(UW12)
694 .long L4
695 .long 0x04000000 /* use dwarf unwind info */
696 .quad 0
697 .quad 0
698
699 /* compact unwind for ffi_go_closure_unix64 */
700 .quad C(ffi_go_closure_unix64)
701 .set L5,L(UW17)-L(UW15)
702 .long L5
703 .long 0x04000000 /* use dwarf unwind info */
704 .quad 0
705 .quad 0
706 #endif
707
708 #endif /* __x86_64__ */
709 #if defined __ELF__ && defined __linux__
710 .section .note.GNU-stack,"",@progbits
711 #endif