Debian Codegraph @

internal/engine/wazevo/backend/isa/amd64/abi_go_call.go

raw
1 package amd64
2
3 import (
4 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
5 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
8 )
9
10 var calleeSavedVRegs = []regalloc.VReg{
11 rdxVReg, r12VReg, r13VReg, r14VReg, r15VReg,
12 xmm8VReg, xmm9VReg, xmm10VReg, xmm11VReg, xmm12VReg, xmm13VReg, xmm14VReg, xmm15VReg,
13 }
14
15 // CompileGoFunctionTrampoline implements backend.Machine.
16 func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte {
17 argBegin := 1 // Skips exec context by default.
18 if needModuleContextPtr {
19 argBegin++
20 }
21
22 abi := &backend.FunctionABI{}
23 abi.Init(sig, intArgResultRegs, floatArgResultRegs)
24 m.currentABI = abi
25
26 cur := m.allocateNop()
27 m.rootInstr = cur
28
29 // Execution context is always the first argument.
30 execCtrPtr := raxVReg
31
32 // First we update RBP and RSP just like the normal prologue.
33 //
34 // (high address) (high address)
35 // RBP ----> +-----------------+ +-----------------+
36 // | ....... | | ....... |
37 // | ret Y | | ret Y |
38 // | ....... | | ....... |
39 // | ret 0 | | ret 0 |
40 // | arg X | | arg X |
41 // | ....... | ====> | ....... |
42 // | arg 1 | | arg 1 |
43 // | arg 0 | | arg 0 |
44 // | Return Addr | | Return Addr |
45 // RSP ----> +-----------------+ | Caller_RBP |
46 // (low address) +-----------------+ <----- RSP, RBP
47 //
48 cur = m.setupRBPRSP(cur)
49
50 goSliceSizeAligned, goSliceSizeAlignedUnaligned := backend.GoFunctionCallRequiredStackSize(sig, argBegin)
51 cur = m.insertStackBoundsCheck(goSliceSizeAligned+8 /* size of the Go slice */, cur)
52
53 // Save the callee saved registers.
54 cur = m.saveRegistersInExecutionContext(cur, execCtrPtr, calleeSavedVRegs)
55
56 if needModuleContextPtr {
57 moduleCtrPtr := rbxVReg // Module context is always the second argument.
58 mem := m.newAmodeImmReg(
59 wazevoapi.ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque.U32(),
60 execCtrPtr)
61 store := m.allocateInstr().asMovRM(moduleCtrPtr, newOperandMem(mem), 8)
62 cur = linkInstr(cur, store)
63 }
64
65 // Now let's advance the RSP to the stack slot for the arguments.
66 //
67 // (high address) (high address)
68 // +-----------------+ +-----------------+
69 // | ....... | | ....... |
70 // | ret Y | | ret Y |
71 // | ....... | | ....... |
72 // | ret 0 | | ret 0 |
73 // | arg X | | arg X |
74 // | ....... | =======> | ....... |
75 // | arg 1 | | arg 1 |
76 // | arg 0 | | arg 0 |
77 // | Return Addr | | Return Addr |
78 // | Caller_RBP | | Caller_RBP |
79 // RBP,RSP --> +-----------------+ +-----------------+ <----- RBP
80 // (low address) | arg[N]/ret[M] |
81 // | .......... |
82 // | arg[1]/ret[1] |
83 // | arg[0]/ret[0] |
84 // +-----------------+ <----- RSP
85 // (low address)
86 //
87 // where the region of "arg[0]/ret[0] ... arg[N]/ret[M]" is the stack used by the Go functions,
88 // therefore will be accessed as the usual []uint64. So that's where we need to pass/receive
89 // the arguments/return values to/from Go function.
90 cur = m.addRSP(-int32(goSliceSizeAligned), cur)
91
92 // Next, we need to store all the arguments to the stack in the typical Wasm stack style.
93 var offsetInGoSlice int32
94 for i := range abi.Args[argBegin:] {
95 arg := &abi.Args[argBegin+i]
96 var v regalloc.VReg
97 if arg.Kind == backend.ABIArgKindReg {
98 v = arg.Reg
99 } else {
100 // We have saved callee saved registers, so we can use them.
101 if arg.Type.IsInt() {
102 v = r15VReg
103 } else {
104 v = xmm15VReg
105 }
106 mem := newOperandMem(m.newAmodeImmReg(uint32(arg.Offset+16 /* to skip caller_rbp and ret_addr */), rbpVReg))
107 load := m.allocateInstr()
108 switch arg.Type {
109 case ssa.TypeI32:
110 load.asMovzxRmR(extModeLQ, mem, v)
111 case ssa.TypeI64:
112 load.asMov64MR(mem, v)
113 case ssa.TypeF32:
114 load.asXmmUnaryRmR(sseOpcodeMovss, mem, v)
115 case ssa.TypeF64:
116 load.asXmmUnaryRmR(sseOpcodeMovsd, mem, v)
117 case ssa.TypeV128:
118 load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, v)
119 default:
120 panic("BUG")
121 }
122 cur = linkInstr(cur, load)
123 }
124
125 store := m.allocateInstr()
126 mem := newOperandMem(m.newAmodeImmReg(uint32(offsetInGoSlice), rspVReg))
127 switch arg.Type {
128 case ssa.TypeI32:
129 store.asMovRM(v, mem, 4)
130 offsetInGoSlice += 8 // always uint64 rep.
131 case ssa.TypeI64:
132 store.asMovRM(v, mem, 8)
133 offsetInGoSlice += 8
134 case ssa.TypeF32:
135 store.asXmmMovRM(sseOpcodeMovss, v, mem)
136 offsetInGoSlice += 8 // always uint64 rep.
137 case ssa.TypeF64:
138 store.asXmmMovRM(sseOpcodeMovsd, v, mem)
139 offsetInGoSlice += 8
140 case ssa.TypeV128:
141 store.asXmmMovRM(sseOpcodeMovdqu, v, mem)
142 offsetInGoSlice += 16
143 default:
144 panic("BUG")
145 }
146 cur = linkInstr(cur, store)
147 }
148
149 // Finally we push the size of the slice to the stack so the stack looks like:
150 //
151 // (high address)
152 // +-----------------+
153 // | ....... |
154 // | ret Y |
155 // | ....... |
156 // | ret 0 |
157 // | arg X |
158 // | ....... |
159 // | arg 1 |
160 // | arg 0 |
161 // | Return Addr |
162 // | Caller_RBP |
163 // +-----------------+ <----- RBP
164 // | arg[N]/ret[M] |
165 // | .......... |
166 // | arg[1]/ret[1] |
167 // | arg[0]/ret[0] |
168 // | slice size |
169 // +-----------------+ <----- RSP
170 // (low address)
171 //
172 // push $sliceSize
173 cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandImm32(uint32(goSliceSizeAlignedUnaligned))))
174
175 // Load the exitCode to the register.
176 exitCodeReg := r12VReg // Callee saved which is already saved.
177 cur = linkInstr(cur, m.allocateInstr().asImm(exitCodeReg, uint64(exitCode), false))
178
179 saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtrPtr, exitCodeReg)
180 cur = linkInstr(cur, setExitCode)
181 cur = linkInstr(cur, saveRsp)
182 cur = linkInstr(cur, saveRbp)
183
184 // Ready to exit the execution.
185 cur = m.storeReturnAddressAndExit(cur, execCtrPtr)
186
187 // We don't need the slice size anymore, so pop it.
188 cur = m.addRSP(8, cur)
189
190 // Ready to set up the results.
191 offsetInGoSlice = 0
192 // To avoid overwriting with the execution context pointer by the result, we need to track the offset,
193 // and defer the restoration of the result to the end of this function.
194 var argOverlapWithExecCtxOffset int32 = -1
195 for i := range abi.Rets {
196 r := &abi.Rets[i]
197 var v regalloc.VReg
198 isRegResult := r.Kind == backend.ABIArgKindReg
199 if isRegResult {
200 v = r.Reg
201 if v.RealReg() == execCtrPtr.RealReg() {
202 argOverlapWithExecCtxOffset = offsetInGoSlice
203 offsetInGoSlice += 8 // always uint64 rep.
204 continue
205 }
206 } else {
207 if r.Type.IsInt() {
208 v = r15VReg
209 } else {
210 v = xmm15VReg
211 }
212 }
213
214 load := m.allocateInstr()
215 mem := newOperandMem(m.newAmodeImmReg(uint32(offsetInGoSlice), rspVReg))
216 switch r.Type {
217 case ssa.TypeI32:
218 load.asMovzxRmR(extModeLQ, mem, v)
219 offsetInGoSlice += 8 // always uint64 rep.
220 case ssa.TypeI64:
221 load.asMov64MR(mem, v)
222 offsetInGoSlice += 8
223 case ssa.TypeF32:
224 load.asXmmUnaryRmR(sseOpcodeMovss, mem, v)
225 offsetInGoSlice += 8 // always uint64 rep.
226 case ssa.TypeF64:
227 load.asXmmUnaryRmR(sseOpcodeMovsd, mem, v)
228 offsetInGoSlice += 8
229 case ssa.TypeV128:
230 load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, v)
231 offsetInGoSlice += 16
232 default:
233 panic("BUG")
234 }
235 cur = linkInstr(cur, load)
236
237 if !isRegResult {
238 // We need to store it back to the result slot above rbp.
239 store := m.allocateInstr()
240 mem := newOperandMem(m.newAmodeImmReg(uint32(abi.ArgStackSize+r.Offset+16 /* to skip caller_rbp and ret_addr */), rbpVReg))
241 switch r.Type {
242 case ssa.TypeI32:
243 store.asMovRM(v, mem, 4)
244 case ssa.TypeI64:
245 store.asMovRM(v, mem, 8)
246 case ssa.TypeF32:
247 store.asXmmMovRM(sseOpcodeMovss, v, mem)
248 case ssa.TypeF64:
249 store.asXmmMovRM(sseOpcodeMovsd, v, mem)
250 case ssa.TypeV128:
251 store.asXmmMovRM(sseOpcodeMovdqu, v, mem)
252 default:
253 panic("BUG")
254 }
255 cur = linkInstr(cur, store)
256 }
257 }
258
259 // Before return, we need to restore the callee saved registers.
260 cur = m.restoreRegistersInExecutionContext(cur, execCtrPtr, calleeSavedVRegs)
261
262 if argOverlapWithExecCtxOffset >= 0 {
263 // At this point execCtt is not used anymore, so we can finally store the
264 // result to the register which overlaps with the execution context pointer.
265 mem := newOperandMem(m.newAmodeImmReg(uint32(argOverlapWithExecCtxOffset), rspVReg))
266 load := m.allocateInstr().asMov64MR(mem, execCtrPtr)
267 cur = linkInstr(cur, load)
268 }
269
270 // Finally ready to return.
271 cur = m.revertRBPRSP(cur)
272 linkInstr(cur, m.allocateInstr().asRet())
273
274 m.encodeWithoutSSA(m.rootInstr)
275 return m.c.Buf()
276 }
277
278 func (m *machine) saveRegistersInExecutionContext(cur *instruction, execCtx regalloc.VReg, regs []regalloc.VReg) *instruction {
279 offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64()
280 for _, v := range regs {
281 store := m.allocateInstr()
282 mem := newOperandMem(m.newAmodeImmReg(uint32(offset), execCtx))
283 switch v.RegType() {
284 case regalloc.RegTypeInt:
285 store.asMovRM(v, mem, 8)
286 case regalloc.RegTypeFloat:
287 store.asXmmMovRM(sseOpcodeMovdqu, v, mem)
288 default:
289 panic("BUG")
290 }
291 cur = linkInstr(cur, store)
292 offset += 16 // See execution context struct. Each register is 16 bytes-aligned unconditionally.
293 }
294 return cur
295 }
296
297 func (m *machine) restoreRegistersInExecutionContext(cur *instruction, execCtx regalloc.VReg, regs []regalloc.VReg) *instruction {
298 offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64()
299 for _, v := range regs {
300 load := m.allocateInstr()
301 mem := newOperandMem(m.newAmodeImmReg(uint32(offset), execCtx))
302 switch v.RegType() {
303 case regalloc.RegTypeInt:
304 load.asMov64MR(mem, v)
305 case regalloc.RegTypeFloat:
306 load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, v)
307 default:
308 panic("BUG")
309 }
310 cur = linkInstr(cur, load)
311 offset += 16 // See execution context struct. Each register is 16 bytes-aligned unconditionally.
312 }
313 return cur
314 }
315
316 func (m *machine) storeReturnAddressAndExit(cur *instruction, execCtx regalloc.VReg) *instruction {
317 readRip := m.allocateInstr()
318 cur = linkInstr(cur, readRip)
319
320 ripReg := r12VReg // Callee saved which is already saved.
321 saveRip := m.allocateInstr().asMovRM(
322 ripReg,
323 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)),
324 8,
325 )
326 cur = linkInstr(cur, saveRip)
327
328 exit := m.allocateExitSeq(execCtx)
329 cur = linkInstr(cur, exit)
330
331 nop, l := m.allocateBrTarget()
332 cur = linkInstr(cur, nop)
333 readRip.asLEA(newOperandLabel(l), ripReg)
334 return cur
335 }
336
337 // saveRequiredRegs is the set of registers that must be saved/restored during growing stack when there's insufficient
338 // stack space left. Basically this is the all allocatable registers except for RSP and RBP, and RAX which contains the
339 // execution context pointer. ExecCtx pointer is always the first argument so we don't need to save it.
340 var stackGrowSaveVRegs = []regalloc.VReg{
341 rdxVReg, r12VReg, r13VReg, r14VReg, r15VReg,
342 rcxVReg, rbxVReg, rsiVReg, rdiVReg, r8VReg, r9VReg, r10VReg, r11VReg,
343 xmm8VReg, xmm9VReg, xmm10VReg, xmm11VReg, xmm12VReg, xmm13VReg, xmm14VReg, xmm15VReg,
344 xmm0VReg, xmm1VReg, xmm2VReg, xmm3VReg, xmm4VReg, xmm5VReg, xmm6VReg, xmm7VReg,
345 }
346
347 // CompileStackGrowCallSequence implements backend.Machine.
348 func (m *machine) CompileStackGrowCallSequence() []byte {
349 cur := m.allocateNop()
350 m.rootInstr = cur
351
352 cur = m.setupRBPRSP(cur)
353
354 // Execution context is always the first argument.
355 execCtrPtr := raxVReg
356
357 // Save the callee saved and argument registers.
358 cur = m.saveRegistersInExecutionContext(cur, execCtrPtr, stackGrowSaveVRegs)
359
360 // Load the exitCode to the register.
361 exitCodeReg := r12VReg // Already saved.
362 cur = linkInstr(cur, m.allocateInstr().asImm(exitCodeReg, uint64(wazevoapi.ExitCodeGrowStack), false))
363
364 saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtrPtr, exitCodeReg)
365 cur = linkInstr(cur, setExitCode)
366 cur = linkInstr(cur, saveRsp)
367 cur = linkInstr(cur, saveRbp)
368
369 // Ready to exit the execution.
370 cur = m.storeReturnAddressAndExit(cur, execCtrPtr)
371
372 // After the exit, restore the saved registers.
373 cur = m.restoreRegistersInExecutionContext(cur, execCtrPtr, stackGrowSaveVRegs)
374
375 // Finally ready to return.
376 cur = m.revertRBPRSP(cur)
377 linkInstr(cur, m.allocateInstr().asRet())
378
379 m.encodeWithoutSSA(m.rootInstr)
380 return m.c.Buf()
381 }
382
383 // insertStackBoundsCheck will insert the instructions after `cur` to check the
384 // stack bounds, and if there's no sufficient spaces required for the function,
385 // exit the execution and try growing it in Go world.
386 func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instruction) *instruction {
387 // add $requiredStackSize, %rsp ;; Temporarily update the sp.
388 // cmp ExecutionContextOffsetStackBottomPtr(%rax), %rsp ;; Compare the stack bottom and the sp.
389 // ja .ok
390 // sub $requiredStackSize, %rsp ;; Reverse the temporary update.
391 // pushq r15 ;; save the temporary.
392 // mov $requiredStackSize, %r15
393 // mov %15, ExecutionContextOffsetStackGrowRequiredSize(%rax) ;; Set the required size in the execution context.
394 // popq r15 ;; restore the temporary.
395 // callq *ExecutionContextOffsetStackGrowCallTrampolineAddress(%rax) ;; Call the Go function to grow the stack.
396 // jmp .cont
397 // .ok:
398 // sub $requiredStackSize, %rsp ;; Reverse the temporary update.
399 // .cont:
400 cur = m.addRSP(-int32(requiredStackSize), cur)
401 cur = linkInstr(cur, m.allocateInstr().asCmpRmiR(true,
402 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackBottomPtr.U32(), raxVReg)),
403 rspVReg, true))
404
405 ja := m.allocateInstr()
406 cur = linkInstr(cur, ja)
407
408 cur = m.addRSP(int32(requiredStackSize), cur)
409
410 // Save the temporary.
411
412 cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandReg(r15VReg)))
413 // Load the required size to the temporary.
414 cur = linkInstr(cur, m.allocateInstr().asImm(r15VReg, uint64(requiredStackSize), true))
415 // Set the required size in the execution context.
416 cur = linkInstr(cur, m.allocateInstr().asMovRM(r15VReg,
417 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackGrowRequiredSize.U32(), raxVReg)), 8))
418 // Restore the temporary.
419 cur = linkInstr(cur, m.allocateInstr().asPop64(r15VReg))
420 // Call the Go function to grow the stack.
421 cur = linkInstr(cur, m.allocateInstr().asCallIndirect(newOperandMem(m.newAmodeImmReg(
422 wazevoapi.ExecutionContextOffsetStackGrowCallTrampolineAddress.U32(), raxVReg)), nil))
423 // Jump to the continuation.
424 jmpToCont := m.allocateInstr()
425 cur = linkInstr(cur, jmpToCont)
426
427 // .ok:
428 okInstr, ok := m.allocateBrTarget()
429 cur = linkInstr(cur, okInstr)
430 ja.asJmpIf(condNBE, newOperandLabel(ok))
431 // On the ok path, we only need to reverse the temporary update.
432 cur = m.addRSP(int32(requiredStackSize), cur)
433
434 // .cont:
435 contInstr, cont := m.allocateBrTarget()
436 cur = linkInstr(cur, contInstr)
437 jmpToCont.asJmp(newOperandLabel(cont))
438
439 return cur
440 }