Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go

raw
1 package arm64
2
3 import (
4 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
5 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
8 )
9
10 // CompileEntryPreamble implements backend.Machine. This assumes `entrypoint` function (in abi_go_entry_arm64.s) passes:
11 //
12 // 1. First (execution context ptr) and Second arguments are already passed in x0, and x1.
13 // 2. param/result slice ptr in x19; the pointer to []uint64{} which is used to pass arguments and accept return values.
14 // 3. Go-allocated stack slice ptr in x26.
15 // 4. Function executable in x24.
16 //
17 // also SP and FP are correct Go-runtime-based values, and LR is the return address to the Go-side caller.
18 func (m *machine) CompileEntryPreamble(signature *ssa.Signature) []byte {
19 root := m.constructEntryPreamble(signature)
20 m.encode(root)
21 return m.compiler.Buf()
22 }
23
24 var (
25 executionContextPtrReg = x0VReg
26 // callee-saved regs so that they can be used in the prologue and epilogue.
27 paramResultSlicePtr = x19VReg
28 savedExecutionContextPtr = x20VReg
29 // goAllocatedStackPtr is not used in the epilogue.
30 goAllocatedStackPtr = x26VReg
31 // paramResultSliceCopied is not used in the epilogue.
32 paramResultSliceCopied = x25VReg
33 // tmpRegVReg is not used in the epilogue.
34 functionExecutable = x24VReg
35 )
36
37 func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, arg *backend.ABIArg, argStartOffsetFromSP int64) *instruction {
38 typ := arg.Type
39 bits := typ.Bits()
40 isStackArg := arg.Kind == backend.ABIArgKindStack
41
42 var loadTargetReg operand
43 if !isStackArg {
44 loadTargetReg = operandNR(arg.Reg)
45 } else {
46 switch typ {
47 case ssa.TypeI32, ssa.TypeI64:
48 loadTargetReg = operandNR(x15VReg)
49 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
50 loadTargetReg = operandNR(v15VReg)
51 default:
52 panic("TODO?")
53 }
54 }
55
56 var postIndexImm int64
57 if typ == ssa.TypeV128 {
58 postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice.
59 } else {
60 postIndexImm = 8
61 }
62 loadMode := m.amodePool.Allocate()
63 *loadMode = addressMode{kind: addressModeKindPostIndex, rn: paramSlicePtr, imm: postIndexImm}
64
65 instr := m.allocateInstr()
66 switch typ {
67 case ssa.TypeI32:
68 instr.asULoad(loadTargetReg.reg(), loadMode, 32)
69 case ssa.TypeI64:
70 instr.asULoad(loadTargetReg.reg(), loadMode, 64)
71 case ssa.TypeF32:
72 instr.asFpuLoad(loadTargetReg.reg(), loadMode, 32)
73 case ssa.TypeF64:
74 instr.asFpuLoad(loadTargetReg.reg(), loadMode, 64)
75 case ssa.TypeV128:
76 instr.asFpuLoad(loadTargetReg.reg(), loadMode, 128)
77 }
78 cur = linkInstr(cur, instr)
79
80 if isStackArg {
81 var storeMode *addressMode
82 cur, storeMode = m.resolveAddressModeForOffsetAndInsert(cur, argStartOffsetFromSP+arg.Offset, bits, spVReg, true)
83 toStack := m.allocateInstr()
84 toStack.asStore(loadTargetReg, storeMode, bits)
85 cur = linkInstr(cur, toStack)
86 }
87 return cur
88 }
89
90 func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, result *backend.ABIArg, resultStartOffsetFromSP int64) *instruction {
91 isStackArg := result.Kind == backend.ABIArgKindStack
92 typ := result.Type
93 bits := typ.Bits()
94
95 var storeTargetReg operand
96 if !isStackArg {
97 storeTargetReg = operandNR(result.Reg)
98 } else {
99 switch typ {
100 case ssa.TypeI32, ssa.TypeI64:
101 storeTargetReg = operandNR(x15VReg)
102 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
103 storeTargetReg = operandNR(v15VReg)
104 default:
105 panic("TODO?")
106 }
107 }
108
109 var postIndexImm int64
110 if typ == ssa.TypeV128 {
111 postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice.
112 } else {
113 postIndexImm = 8
114 }
115
116 if isStackArg {
117 var loadMode *addressMode
118 cur, loadMode = m.resolveAddressModeForOffsetAndInsert(cur, resultStartOffsetFromSP+result.Offset, bits, spVReg, true)
119 toReg := m.allocateInstr()
120 switch typ {
121 case ssa.TypeI32, ssa.TypeI64:
122 toReg.asULoad(storeTargetReg.reg(), loadMode, bits)
123 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
124 toReg.asFpuLoad(storeTargetReg.reg(), loadMode, bits)
125 default:
126 panic("TODO?")
127 }
128 cur = linkInstr(cur, toReg)
129 }
130
131 mode := m.amodePool.Allocate()
132 *mode = addressMode{kind: addressModeKindPostIndex, rn: resultSlicePtr, imm: postIndexImm}
133 instr := m.allocateInstr()
134 instr.asStore(storeTargetReg, mode, bits)
135 cur = linkInstr(cur, instr)
136 return cur
137 }
138
139 func (m *machine) constructEntryPreamble(sig *ssa.Signature) (root *instruction) {
140 abi := backend.FunctionABI{}
141 abi.Init(sig, intParamResultRegs, floatParamResultRegs)
142
143 root = m.allocateNop()
144
145 //// ----------------------------------- prologue ----------------------------------- ////
146
147 // First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well.
148 // mov savedExecutionContextPtr, x0
149 cur := m.move64(savedExecutionContextPtr, executionContextPtrReg, root)
150
151 // Next, save the current FP, SP and LR into the wazevo.executionContext:
152 // str fp, [savedExecutionContextPtr, #OriginalFramePointer]
153 // mov tmp, sp ;; sp cannot be str'ed directly.
154 // str sp, [savedExecutionContextPtr, #OriginalStackPointer]
155 // str lr, [savedExecutionContextPtr, #GoReturnAddress]
156 cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, true, cur)
157 cur = m.move64(tmpRegVReg, spVReg, cur)
158 cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, true, cur)
159 cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, true, cur)
160
161 // Then, move the Go-allocated stack pointer to SP:
162 // mov sp, goAllocatedStackPtr
163 cur = m.move64(spVReg, goAllocatedStackPtr, cur)
164
165 prReg := paramResultSlicePtr
166 if len(abi.Args) > 2 && len(abi.Rets) > 0 {
167 // paramResultSlicePtr is modified during the execution of goEntryPreamblePassArg,
168 // so copy it to another reg.
169 cur = m.move64(paramResultSliceCopied, paramResultSlicePtr, cur)
170 prReg = paramResultSliceCopied
171 }
172
173 stackSlotSize := int64(abi.AlignedArgResultStackSlotSize())
174 for i := range abi.Args {
175 if i < 2 {
176 // module context ptr and execution context ptr are passed in x0 and x1 by the Go assembly function.
177 continue
178 }
179 arg := &abi.Args[i]
180 cur = m.goEntryPreamblePassArg(cur, prReg, arg, -stackSlotSize)
181 }
182
183 // Call the real function.
184 bl := m.allocateInstr()
185 bl.asCallIndirect(functionExecutable, &abi)
186 cur = linkInstr(cur, bl)
187
188 ///// ----------------------------------- epilogue ----------------------------------- /////
189
190 // Store the register results into paramResultSlicePtr.
191 for i := range abi.Rets {
192 cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, &abi.Rets[i], abi.ArgStackSize-stackSlotSize)
193 }
194
195 // Finally, restore the FP, SP and LR, and return to the Go code.
196 // ldr fp, [savedExecutionContextPtr, #OriginalFramePointer]
197 // ldr tmp, [savedExecutionContextPtr, #OriginalStackPointer]
198 // mov sp, tmp ;; sp cannot be str'ed directly.
199 // ldr lr, [savedExecutionContextPtr, #GoReturnAddress]
200 // ret ;; --> return to the Go code
201 cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, false, cur)
202 cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, false, cur)
203 cur = m.move64(spVReg, tmpRegVReg, cur)
204 cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, false, cur)
205 retInst := m.allocateInstr()
206 retInst.asRet()
207 linkInstr(cur, retInst)
208 return
209 }
210
211 func (m *machine) move64(dst, src regalloc.VReg, prev *instruction) *instruction {
212 instr := m.allocateInstr()
213 instr.asMove64(dst, src)
214 return linkInstr(prev, instr)
215 }
216
217 func (m *machine) loadOrStoreAtExecutionContext(d regalloc.VReg, offset wazevoapi.Offset, store bool, prev *instruction) *instruction {
218 instr := m.allocateInstr()
219 mode := m.amodePool.Allocate()
220 *mode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: savedExecutionContextPtr, imm: offset.I64()}
221 if store {
222 instr.asStore(operandNR(d), mode, 64)
223 } else {
224 instr.asULoad(d, mode, 64)
225 }
226 return linkInstr(prev, instr)
227 }
228
229 func linkInstr(prev, next *instruction) *instruction {
230 prev.next = next
231 next.prev = prev
232 return next
233 }