Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/abi_go_call.go

raw
1 package arm64
2
3 import (
4 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
5 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
8 )
9
10 var calleeSavedRegistersSorted = []regalloc.VReg{
11 x19VReg, x20VReg, x21VReg, x22VReg, x23VReg, x24VReg, x25VReg, x26VReg, x28VReg,
12 v18VReg, v19VReg, v20VReg, v21VReg, v22VReg, v23VReg, v24VReg, v25VReg, v26VReg, v27VReg, v28VReg, v29VReg, v30VReg, v31VReg,
13 }
14
15 // CompileGoFunctionTrampoline implements backend.Machine.
16 func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte {
17 argBegin := 1 // Skips exec context by default.
18 if needModuleContextPtr {
19 argBegin++
20 }
21
22 abi := &backend.FunctionABI{}
23 abi.Init(sig, intParamResultRegs, floatParamResultRegs)
24 m.currentABI = abi
25
26 cur := m.allocateInstr()
27 cur.asNop0()
28 m.rootInstr = cur
29
30 // Execution context is always the first argument.
31 execCtrPtr := x0VReg
32
33 // In the following, we create the following stack layout:
34 //
35 // (high address)
36 // SP ------> +-----------------+ <----+
37 // | ....... | |
38 // | ret Y | |
39 // | ....... | |
40 // | ret 0 | |
41 // | arg X | | size_of_arg_ret
42 // | ....... | |
43 // | arg 1 | |
44 // | arg 0 | <----+ <-------- originalArg0Reg
45 // | size_of_arg_ret |
46 // | ReturnAddress |
47 // +-----------------+ <----+
48 // | xxxx | | ;; might be padded to make it 16-byte aligned.
49 // +--->| arg[N]/ret[M] | |
50 // sliceSize| | ............ | | goCallStackSize
51 // | | arg[1]/ret[1] | |
52 // +--->| arg[0]/ret[0] | <----+ <-------- arg0ret0AddrReg
53 // | sliceSize |
54 // | frame_size |
55 // +-----------------+
56 // (low address)
57 //
58 // where the region of "arg[0]/ret[0] ... arg[N]/ret[M]" is the stack used by the Go functions,
59 // therefore will be accessed as the usual []uint64. So that's where we need to pass/receive
60 // the arguments/return values.
61
62 // First of all, to update the SP, and create "ReturnAddress + size_of_arg_ret".
63 cur = m.createReturnAddrAndSizeOfArgRetSlot(cur)
64
65 const frameInfoSize = 16 // == frame_size + sliceSize.
66
67 // Next, we should allocate the stack for the Go function call if necessary.
68 goCallStackSize, sliceSizeInBytes := backend.GoFunctionCallRequiredStackSize(sig, argBegin)
69 cur = m.insertStackBoundsCheck(goCallStackSize+frameInfoSize, cur)
70
71 originalArg0Reg := x17VReg // Caller save, so we can use it for whatever we want.
72 if m.currentABI.AlignedArgResultStackSlotSize() > 0 {
73 // At this point, SP points to `ReturnAddress`, so add 16 to get the original arg 0 slot.
74 cur = m.addsAddOrSubStackPointer(cur, originalArg0Reg, frameInfoSize, true)
75 }
76
77 // Save the callee saved registers.
78 cur = m.saveRegistersInExecutionContext(cur, calleeSavedRegistersSorted)
79
80 if needModuleContextPtr {
81 offset := wazevoapi.ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque.I64()
82 if !offsetFitsInAddressModeKindRegUnsignedImm12(64, offset) {
83 panic("BUG: too large or un-aligned offset for goFunctionCallCalleeModuleContextOpaque in execution context")
84 }
85
86 // Module context is always the second argument.
87 moduleCtrPtr := x1VReg
88 store := m.allocateInstr()
89 amode := m.amodePool.Allocate()
90 *amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: execCtrPtr, imm: offset}
91 store.asStore(operandNR(moduleCtrPtr), amode, 64)
92 cur = linkInstr(cur, store)
93 }
94
95 // Advances the stack pointer.
96 cur = m.addsAddOrSubStackPointer(cur, spVReg, goCallStackSize, false)
97
98 // Copy the pointer to x15VReg.
99 arg0ret0AddrReg := x15VReg // Caller save, so we can use it for whatever we want.
100 copySp := m.allocateInstr()
101 copySp.asMove64(arg0ret0AddrReg, spVReg)
102 cur = linkInstr(cur, copySp)
103
104 // Next, we need to store all the arguments to the stack in the typical Wasm stack style.
105 for i := range abi.Args[argBegin:] {
106 arg := &abi.Args[argBegin+i]
107 store := m.allocateInstr()
108 var v regalloc.VReg
109 if arg.Kind == backend.ABIArgKindReg {
110 v = arg.Reg
111 } else {
112 cur, v = m.goFunctionCallLoadStackArg(cur, originalArg0Reg, arg,
113 // Caller save, so we can use it for whatever we want.
114 x11VReg, v11VReg)
115 }
116
117 var sizeInBits byte
118 if arg.Type == ssa.TypeV128 {
119 sizeInBits = 128
120 } else {
121 sizeInBits = 64
122 }
123 amode := m.amodePool.Allocate()
124 *amode = addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg, imm: int64(sizeInBits / 8)}
125 store.asStore(operandNR(v), amode, sizeInBits)
126 cur = linkInstr(cur, store)
127 }
128
129 // Finally, now that we've advanced SP to arg[0]/ret[0], we allocate `frame_size + sliceSize`.
130 var frameSizeReg, sliceSizeReg regalloc.VReg
131 if goCallStackSize > 0 {
132 cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, goCallStackSize)
133 frameSizeReg = tmpRegVReg
134 cur = m.lowerConstantI64AndInsert(cur, x16VReg, sliceSizeInBytes/8)
135 sliceSizeReg = x16VReg
136 } else {
137 frameSizeReg = xzrVReg
138 sliceSizeReg = xzrVReg
139 }
140 _amode := addressModePreOrPostIndex(m, spVReg, -16, true)
141 storeP := m.allocateInstr()
142 storeP.asStorePair64(frameSizeReg, sliceSizeReg, _amode)
143 cur = linkInstr(cur, storeP)
144
145 // Set the exit status on the execution context.
146 cur = m.setExitCode(cur, x0VReg, exitCode)
147
148 // Save the current stack pointer.
149 cur = m.saveCurrentStackPointer(cur, x0VReg)
150
151 // Exit the execution.
152 cur = m.storeReturnAddressAndExit(cur)
153
154 // After the call, we need to restore the callee saved registers.
155 cur = m.restoreRegistersInExecutionContext(cur, calleeSavedRegistersSorted)
156
157 // Get the pointer to the arg[0]/ret[0]: We need to skip `frame_size + sliceSize`.
158 if len(abi.Rets) > 0 {
159 cur = m.addsAddOrSubStackPointer(cur, arg0ret0AddrReg, frameInfoSize, true)
160 }
161
162 // Advances the SP so that it points to `ReturnAddress`.
163 cur = m.addsAddOrSubStackPointer(cur, spVReg, frameInfoSize+goCallStackSize, true)
164 ldr := m.allocateInstr()
165 // And load the return address.
166 amode := addressModePreOrPostIndex(m, spVReg, 16 /* stack pointer must be 16-byte aligned. */, false /* increment after loads */)
167 ldr.asULoad(lrVReg, amode, 64)
168 cur = linkInstr(cur, ldr)
169
170 originalRet0Reg := x17VReg // Caller save, so we can use it for whatever we want.
171 if m.currentABI.RetStackSize > 0 {
172 cur = m.addsAddOrSubStackPointer(cur, originalRet0Reg, m.currentABI.ArgStackSize, true)
173 }
174
175 // Make the SP point to the original address (above the result slot).
176 if s := int64(m.currentABI.AlignedArgResultStackSlotSize()); s > 0 {
177 cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true)
178 }
179
180 for i := range abi.Rets {
181 r := &abi.Rets[i]
182 if r.Kind == backend.ABIArgKindReg {
183 loadIntoReg := m.allocateInstr()
184 mode := m.amodePool.Allocate()
185 *mode = addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg}
186 switch r.Type {
187 case ssa.TypeI32:
188 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
189 loadIntoReg.asULoad(r.Reg, mode, 32)
190 case ssa.TypeI64:
191 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
192 loadIntoReg.asULoad(r.Reg, mode, 64)
193 case ssa.TypeF32:
194 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
195 loadIntoReg.asFpuLoad(r.Reg, mode, 32)
196 case ssa.TypeF64:
197 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
198 loadIntoReg.asFpuLoad(r.Reg, mode, 64)
199 case ssa.TypeV128:
200 mode.imm = 16
201 loadIntoReg.asFpuLoad(r.Reg, mode, 128)
202 default:
203 panic("TODO")
204 }
205 cur = linkInstr(cur, loadIntoReg)
206 } else {
207 // First we need to load the value to a temporary just like ^^.
208 intTmp, floatTmp := x11VReg, v11VReg
209 loadIntoTmpReg := m.allocateInstr()
210 mode := m.amodePool.Allocate()
211 *mode = addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg}
212 var resultReg regalloc.VReg
213 switch r.Type {
214 case ssa.TypeI32:
215 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
216 loadIntoTmpReg.asULoad(intTmp, mode, 32)
217 resultReg = intTmp
218 case ssa.TypeI64:
219 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
220 loadIntoTmpReg.asULoad(intTmp, mode, 64)
221 resultReg = intTmp
222 case ssa.TypeF32:
223 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
224 loadIntoTmpReg.asFpuLoad(floatTmp, mode, 32)
225 resultReg = floatTmp
226 case ssa.TypeF64:
227 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
228 loadIntoTmpReg.asFpuLoad(floatTmp, mode, 64)
229 resultReg = floatTmp
230 case ssa.TypeV128:
231 mode.imm = 16
232 loadIntoTmpReg.asFpuLoad(floatTmp, mode, 128)
233 resultReg = floatTmp
234 default:
235 panic("TODO")
236 }
237 cur = linkInstr(cur, loadIntoTmpReg)
238 cur = m.goFunctionCallStoreStackResult(cur, originalRet0Reg, r, resultReg)
239 }
240 }
241
242 ret := m.allocateInstr()
243 ret.asRet()
244 linkInstr(cur, ret)
245
246 m.encode(m.rootInstr)
247 return m.compiler.Buf()
248 }
249
250 func (m *machine) saveRegistersInExecutionContext(cur *instruction, regs []regalloc.VReg) *instruction {
251 offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64()
252 for _, v := range regs {
253 store := m.allocateInstr()
254 var sizeInBits byte
255 switch v.RegType() {
256 case regalloc.RegTypeInt:
257 sizeInBits = 64
258 case regalloc.RegTypeFloat:
259 sizeInBits = 128
260 }
261 mode := m.amodePool.Allocate()
262 *mode = addressMode{
263 kind: addressModeKindRegUnsignedImm12,
264 // Execution context is always the first argument.
265 rn: x0VReg, imm: offset,
266 }
267 store.asStore(operandNR(v), mode, sizeInBits)
268 store.prev = cur
269 cur.next = store
270 cur = store
271 offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally store regs at the offset of multiple of 16.
272 }
273 return cur
274 }
275
276 func (m *machine) restoreRegistersInExecutionContext(cur *instruction, regs []regalloc.VReg) *instruction {
277 offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64()
278 for _, v := range regs {
279 load := m.allocateInstr()
280 var as func(dst regalloc.VReg, amode *addressMode, sizeInBits byte)
281 var sizeInBits byte
282 switch v.RegType() {
283 case regalloc.RegTypeInt:
284 as = load.asULoad
285 sizeInBits = 64
286 case regalloc.RegTypeFloat:
287 as = load.asFpuLoad
288 sizeInBits = 128
289 }
290 mode := m.amodePool.Allocate()
291 *mode = addressMode{
292 kind: addressModeKindRegUnsignedImm12,
293 // Execution context is always the first argument.
294 rn: x0VReg, imm: offset,
295 }
296 as(v, mode, sizeInBits)
297 cur = linkInstr(cur, load)
298 offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally load regs at the offset of multiple of 16.
299 }
300 return cur
301 }
302
303 func (m *machine) lowerConstantI64AndInsert(cur *instruction, dst regalloc.VReg, v int64) *instruction {
304 m.pendingInstructions = m.pendingInstructions[:0]
305 m.lowerConstantI64(dst, v)
306 for _, instr := range m.pendingInstructions {
307 cur = linkInstr(cur, instr)
308 }
309 return cur
310 }
311
312 func (m *machine) lowerConstantI32AndInsert(cur *instruction, dst regalloc.VReg, v int32) *instruction {
313 m.pendingInstructions = m.pendingInstructions[:0]
314 m.lowerConstantI32(dst, v)
315 for _, instr := range m.pendingInstructions {
316 cur = linkInstr(cur, instr)
317 }
318 return cur
319 }
320
321 func (m *machine) setExitCode(cur *instruction, execCtr regalloc.VReg, exitCode wazevoapi.ExitCode) *instruction {
322 constReg := x17VReg // caller-saved, so we can use it.
323 cur = m.lowerConstantI32AndInsert(cur, constReg, int32(exitCode))
324
325 // Set the exit status on the execution context.
326 setExistStatus := m.allocateInstr()
327 mode := m.amodePool.Allocate()
328 *mode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: execCtr, imm: wazevoapi.ExecutionContextOffsetExitCodeOffset.I64()}
329 setExistStatus.asStore(operandNR(constReg), mode, 32)
330 cur = linkInstr(cur, setExistStatus)
331 return cur
332 }
333
334 func (m *machine) storeReturnAddressAndExit(cur *instruction) *instruction {
335 // Read the return address into tmp, and store it in the execution context.
336 adr := m.allocateInstr()
337 adr.asAdr(tmpRegVReg, exitSequenceSize+8)
338 cur = linkInstr(cur, adr)
339
340 storeReturnAddr := m.allocateInstr()
341 mode := m.amodePool.Allocate()
342 *mode = addressMode{
343 kind: addressModeKindRegUnsignedImm12,
344 // Execution context is always the first argument.
345 rn: x0VReg, imm: wazevoapi.ExecutionContextOffsetGoCallReturnAddress.I64(),
346 }
347 storeReturnAddr.asStore(operandNR(tmpRegVReg), mode, 64)
348 cur = linkInstr(cur, storeReturnAddr)
349
350 // Exit the execution.
351 trapSeq := m.allocateInstr()
352 trapSeq.asExitSequence(x0VReg)
353 cur = linkInstr(cur, trapSeq)
354 return cur
355 }
356
357 func (m *machine) saveCurrentStackPointer(cur *instruction, execCtr regalloc.VReg) *instruction {
358 // Save the current stack pointer:
359 // mov tmp, sp,
360 // str tmp, [exec_ctx, #stackPointerBeforeGoCall]
361 movSp := m.allocateInstr()
362 movSp.asMove64(tmpRegVReg, spVReg)
363 cur = linkInstr(cur, movSp)
364
365 strSp := m.allocateInstr()
366 mode := m.amodePool.Allocate()
367 *mode = addressMode{
368 kind: addressModeKindRegUnsignedImm12,
369 rn: execCtr, imm: wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.I64(),
370 }
371 strSp.asStore(operandNR(tmpRegVReg), mode, 64)
372 cur = linkInstr(cur, strSp)
373 return cur
374 }
375
376 func (m *machine) goFunctionCallLoadStackArg(cur *instruction, originalArg0Reg regalloc.VReg, arg *backend.ABIArg, intVReg, floatVReg regalloc.VReg) (*instruction, regalloc.VReg) {
377 load := m.allocateInstr()
378 var result regalloc.VReg
379 mode := m.amodePool.Allocate()
380 *mode = addressMode{kind: addressModeKindPostIndex, rn: originalArg0Reg}
381 switch arg.Type {
382 case ssa.TypeI32:
383 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
384 load.asULoad(intVReg, mode, 32)
385 result = intVReg
386 case ssa.TypeI64:
387 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
388 load.asULoad(intVReg, mode, 64)
389 result = intVReg
390 case ssa.TypeF32:
391 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
392 load.asFpuLoad(floatVReg, mode, 32)
393 result = floatVReg
394 case ssa.TypeF64:
395 mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
396 load.asFpuLoad(floatVReg, mode, 64)
397 result = floatVReg
398 case ssa.TypeV128:
399 mode.imm = 16
400 load.asFpuLoad(floatVReg, mode, 128)
401 result = floatVReg
402 default:
403 panic("TODO")
404 }
405
406 cur = linkInstr(cur, load)
407 return cur, result
408 }
409
410 func (m *machine) goFunctionCallStoreStackResult(cur *instruction, originalRet0Reg regalloc.VReg, result *backend.ABIArg, resultVReg regalloc.VReg) *instruction {
411 store := m.allocateInstr()
412 mode := m.amodePool.Allocate()
413 *mode = addressMode{kind: addressModeKindPostIndex, rn: originalRet0Reg}
414 var sizeInBits byte
415 switch result.Type {
416 case ssa.TypeI32, ssa.TypeF32:
417 mode.imm = 8
418 sizeInBits = 32
419 case ssa.TypeI64, ssa.TypeF64:
420 mode.imm = 8
421 sizeInBits = 64
422 case ssa.TypeV128:
423 mode.imm = 16
424 sizeInBits = 128
425 default:
426 panic("TODO")
427 }
428 store.asStore(operandNR(resultVReg), mode, sizeInBits)
429 return linkInstr(cur, store)
430 }