Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/abi.go

raw
1 package arm64
2
3 import (
4 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
5 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
7 )
8
9 // References:
10 // * https://github.com/golang/go/blob/go1.24.0/src/cmd/compile/abi-internal.md#arm64-architecture
11 // * https://developer.arm.com/documentation/102374/0101/Procedure-Call-Standard
12
13 var (
14 intParamResultRegs = []regalloc.RealReg{x0, x1, x2, x3, x4, x5, x6, x7}
15 floatParamResultRegs = []regalloc.RealReg{v0, v1, v2, v3, v4, v5, v6, v7}
16 )
17
18 var regInfo = &regalloc.RegisterInfo{
19 AllocatableRegisters: [regalloc.NumRegType][]regalloc.RealReg{
20 // We don't allocate:
21 // - x18: Reserved by the macOS: https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Respect-the-purpose-of-specific-CPU-registers
22 // - x28: Reserved by Go runtime.
23 // - x27(=tmpReg): because of the reason described on tmpReg.
24 regalloc.RegTypeInt: {
25 x8, x9, x10, x11, x12, x13, x14, x15,
26 x16, x17, x19, x20, x21, x22, x23, x24, x25,
27 x26, x29, x30,
28 // These are the argument/return registers. Less preferred in the allocation.
29 x7, x6, x5, x4, x3, x2, x1, x0,
30 },
31 regalloc.RegTypeFloat: {
32 v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19,
33 v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30,
34 // These are the argument/return registers. Less preferred in the allocation.
35 v7, v6, v5, v4, v3, v2, v1, v0,
36 },
37 },
38 CalleeSavedRegisters: regalloc.NewRegSet(
39 x19, x20, x21, x22, x23, x24, x25, x26, x28,
40 v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31,
41 ),
42 CallerSavedRegisters: regalloc.NewRegSet(
43 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x29, x30,
44 v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17,
45 ),
46 RealRegToVReg: []regalloc.VReg{
47 x0: x0VReg, x1: x1VReg, x2: x2VReg, x3: x3VReg, x4: x4VReg, x5: x5VReg, x6: x6VReg, x7: x7VReg, x8: x8VReg, x9: x9VReg, x10: x10VReg, x11: x11VReg, x12: x12VReg, x13: x13VReg, x14: x14VReg, x15: x15VReg, x16: x16VReg, x17: x17VReg, x18: x18VReg, x19: x19VReg, x20: x20VReg, x21: x21VReg, x22: x22VReg, x23: x23VReg, x24: x24VReg, x25: x25VReg, x26: x26VReg, x27: x27VReg, x28: x28VReg, x29: x29VReg, x30: x30VReg,
48 v0: v0VReg, v1: v1VReg, v2: v2VReg, v3: v3VReg, v4: v4VReg, v5: v5VReg, v6: v6VReg, v7: v7VReg, v8: v8VReg, v9: v9VReg, v10: v10VReg, v11: v11VReg, v12: v12VReg, v13: v13VReg, v14: v14VReg, v15: v15VReg, v16: v16VReg, v17: v17VReg, v18: v18VReg, v19: v19VReg, v20: v20VReg, v21: v21VReg, v22: v22VReg, v23: v23VReg, v24: v24VReg, v25: v25VReg, v26: v26VReg, v27: v27VReg, v28: v28VReg, v29: v29VReg, v30: v30VReg, v31: v31VReg,
49 },
50 RealRegName: func(r regalloc.RealReg) string { return regNames[r] },
51 RealRegType: func(r regalloc.RealReg) regalloc.RegType {
52 if r < v0 {
53 return regalloc.RegTypeInt
54 }
55 return regalloc.RegTypeFloat
56 },
57 }
58
59 // ArgsResultsRegs implements backend.Machine.
60 func (m *machine) ArgsResultsRegs() (argResultInts, argResultFloats []regalloc.RealReg) {
61 return intParamResultRegs, floatParamResultRegs
62 }
63
64 // LowerParams implements backend.FunctionABI.
65 func (m *machine) LowerParams(args []ssa.Value) {
66 a := m.currentABI
67
68 for i, ssaArg := range args {
69 if !ssaArg.Valid() {
70 continue
71 }
72 reg := m.compiler.VRegOf(ssaArg)
73 arg := &a.Args[i]
74 if arg.Kind == backend.ABIArgKindReg {
75 m.InsertMove(reg, arg.Reg, arg.Type)
76 } else {
77 // TODO: we could use pair load if there's consecutive loads for the same type.
78 //
79 // (high address)
80 // +-----------------+
81 // | ....... |
82 // | ret Y |
83 // | ....... |
84 // | ret 0 |
85 // | arg X |
86 // | ....... |
87 // | arg 1 |
88 // | arg 0 | <-|
89 // | ReturnAddress | |
90 // +-----------------+ |
91 // | ........... | |
92 // | clobbered M | | argStackOffset: is unknown at this point of compilation.
93 // | ............ | |
94 // | clobbered 0 | |
95 // | spill slot N | |
96 // | ........... | |
97 // | spill slot 0 | |
98 // SP---> +-----------------+ <-+
99 // (low address)
100
101 bits := arg.Type.Bits()
102 // At this point of compilation, we don't yet know how much space exist below the return address.
103 // So we instruct the address mode to add the `argStackOffset` to the offset at the later phase of compilation.
104 amode := m.amodePool.Allocate()
105 *amode = addressMode{imm: arg.Offset, rn: spVReg, kind: addressModeKindArgStackSpace}
106 load := m.allocateInstr()
107 switch arg.Type {
108 case ssa.TypeI32, ssa.TypeI64:
109 load.asULoad(reg, amode, bits)
110 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
111 load.asFpuLoad(reg, amode, bits)
112 default:
113 panic("BUG")
114 }
115 m.insert(load)
116 m.unresolvedAddressModes = append(m.unresolvedAddressModes, load)
117 }
118 }
119 }
120
121 // LowerReturns lowers the given returns.
122 func (m *machine) LowerReturns(rets []ssa.Value) {
123 a := m.currentABI
124
125 l := len(rets) - 1
126 for i := range rets {
127 // Reverse order in order to avoid overwriting the stack returns existing in the return registers.
128 ret := rets[l-i]
129 r := &a.Rets[l-i]
130 reg := m.compiler.VRegOf(ret)
131 if def := m.compiler.ValueDefinition(ret); def.IsFromInstr() {
132 // Constant instructions are inlined.
133 if inst := def.Instr; inst.Constant() {
134 val := inst.Return()
135 valType := val.Type()
136 v := inst.ConstantVal()
137 m.insertLoadConstant(v, valType, reg)
138 }
139 }
140 if r.Kind == backend.ABIArgKindReg {
141 m.InsertMove(r.Reg, reg, ret.Type())
142 } else {
143 // TODO: we could use pair store if there's consecutive stores for the same type.
144 //
145 // (high address)
146 // +-----------------+
147 // | ....... |
148 // | ret Y |
149 // | ....... |
150 // | ret 0 | <-+
151 // | arg X | |
152 // | ....... | |
153 // | arg 1 | |
154 // | arg 0 | |
155 // | ReturnAddress | |
156 // +-----------------+ |
157 // | ........... | |
158 // | spill slot M | | retStackOffset: is unknown at this point of compilation.
159 // | ............ | |
160 // | spill slot 2 | |
161 // | spill slot 1 | |
162 // | clobbered 0 | |
163 // | clobbered 1 | |
164 // | ........... | |
165 // | clobbered N | |
166 // SP---> +-----------------+ <-+
167 // (low address)
168
169 bits := r.Type.Bits()
170
171 // At this point of compilation, we don't yet know how much space exist below the return address.
172 // So we instruct the address mode to add the `retStackOffset` to the offset at the later phase of compilation.
173 amode := m.amodePool.Allocate()
174 *amode = addressMode{imm: r.Offset, rn: spVReg, kind: addressModeKindResultStackSpace}
175 store := m.allocateInstr()
176 store.asStore(operandNR(reg), amode, bits)
177 m.insert(store)
178 m.unresolvedAddressModes = append(m.unresolvedAddressModes, store)
179 }
180 }
181 }
182
183 // callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the
184 // caller side of the function call.
185 func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, slotBegin int64) {
186 arg := &a.Args[argIndex]
187 if def.IsFromInstr() {
188 // Constant instructions are inlined.
189 if inst := def.Instr; inst.Constant() {
190 val := inst.Return()
191 valType := val.Type()
192 v := inst.ConstantVal()
193 m.insertLoadConstant(v, valType, reg)
194 }
195 }
196 if arg.Kind == backend.ABIArgKindReg {
197 m.InsertMove(arg.Reg, reg, arg.Type)
198 } else {
199 // TODO: we could use pair store if there's consecutive stores for the same type.
200 //
201 // Note that at this point, stack pointer is already adjusted.
202 bits := arg.Type.Bits()
203 amode := m.resolveAddressModeForOffset(arg.Offset-slotBegin, bits, spVReg, false)
204 store := m.allocateInstr()
205 store.asStore(operandNR(reg), amode, bits)
206 m.insert(store)
207 }
208 }
209
210 func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, slotBegin int64) {
211 r := &a.Rets[retIndex]
212 if r.Kind == backend.ABIArgKindReg {
213 m.InsertMove(reg, r.Reg, r.Type)
214 } else {
215 // TODO: we could use pair load if there's consecutive loads for the same type.
216 amode := m.resolveAddressModeForOffset(a.ArgStackSize+r.Offset-slotBegin, r.Type.Bits(), spVReg, false)
217 ldr := m.allocateInstr()
218 switch r.Type {
219 case ssa.TypeI32, ssa.TypeI64:
220 ldr.asULoad(reg, amode, r.Type.Bits())
221 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
222 ldr.asFpuLoad(reg, amode, r.Type.Bits())
223 default:
224 panic("BUG")
225 }
226 m.insert(ldr)
227 }
228 }
229
230 func (m *machine) resolveAddressModeForOffsetAndInsert(cur *instruction, offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) (*instruction, *addressMode) {
231 m.pendingInstructions = m.pendingInstructions[:0]
232 mode := m.resolveAddressModeForOffset(offset, dstBits, rn, allowTmpRegUse)
233 for _, instr := range m.pendingInstructions {
234 cur = linkInstr(cur, instr)
235 }
236 return cur, mode
237 }
238
239 func (m *machine) resolveAddressModeForOffset(offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) *addressMode {
240 if rn.RegType() != regalloc.RegTypeInt {
241 panic("BUG: rn should be a pointer: " + formatVRegSized(rn, 64))
242 }
243 amode := m.amodePool.Allocate()
244 if offsetFitsInAddressModeKindRegUnsignedImm12(dstBits, offset) {
245 *amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: rn, imm: offset}
246 } else if offsetFitsInAddressModeKindRegSignedImm9(offset) {
247 *amode = addressMode{kind: addressModeKindRegSignedImm9, rn: rn, imm: offset}
248 } else {
249 var indexReg regalloc.VReg
250 if allowTmpRegUse {
251 m.lowerConstantI64(tmpRegVReg, offset)
252 indexReg = tmpRegVReg
253 } else {
254 indexReg = m.compiler.AllocateVReg(ssa.TypeI64)
255 m.lowerConstantI64(indexReg, offset)
256 }
257 *amode = addressMode{kind: addressModeKindRegReg, rn: rn, rm: indexReg, extOp: extendOpUXTX /* indicates index rm is 64-bit */}
258 }
259 return amode
260 }
261
262 func (m *machine) lowerCall(si *ssa.Instruction) {
263 isDirectCall := si.Opcode() == ssa.OpcodeCall
264 indirectCalleePtr, directCallee, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
265
266 if isDirectCall {
267 call := m.allocateInstr()
268 call.asCall(directCallee, calleeABI)
269 m.insert(call)
270 } else {
271 ptr := m.compiler.VRegOf(indirectCalleePtr)
272 callInd := m.allocateInstr()
273 callInd.asCallIndirect(ptr, calleeABI)
274 m.insert(callInd)
275 }
276
277 m.insertReturns(si, calleeABI, stackSlotSize)
278 }
279
280 func (m *machine) prepareCall(si *ssa.Instruction, isDirectCall bool) (ssa.Value, ssa.FuncRef, *backend.FunctionABI, int64) {
281 var indirectCalleePtr ssa.Value
282 var directCallee ssa.FuncRef
283 var sigID ssa.SignatureID
284 var args []ssa.Value
285 if isDirectCall {
286 directCallee, sigID, args = si.CallData()
287 } else {
288 indirectCalleePtr, sigID, args, _ /* on arm64, the calling convention is compatible with the Go runtime */ = si.CallIndirectData()
289 }
290 calleeABI := m.compiler.GetFunctionABI(m.compiler.SSABuilder().ResolveSignature(sigID))
291
292 stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize())
293 if m.maxRequiredStackSizeForCalls < stackSlotSize+16 {
294 m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // return address frame.
295 }
296
297 for i, arg := range args {
298 reg := m.compiler.VRegOf(arg)
299 def := m.compiler.ValueDefinition(arg)
300 m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize)
301 }
302 return indirectCalleePtr, directCallee, calleeABI, stackSlotSize
303 }
304
305 func (m *machine) insertReturns(si *ssa.Instruction, calleeABI *backend.FunctionABI, stackSlotSize int64) {
306 var index int
307 r1, rs := si.Returns()
308 if r1.Valid() {
309 m.callerGenFunctionReturnVReg(calleeABI, 0, m.compiler.VRegOf(r1), stackSlotSize)
310 index++
311 }
312
313 for _, r := range rs {
314 m.callerGenFunctionReturnVReg(calleeABI, index, m.compiler.VRegOf(r), stackSlotSize)
315 index++
316 }
317 }
318
319 func (m *machine) lowerTailCall(si *ssa.Instruction) {
320 isDirectCall := si.Opcode() == ssa.OpcodeTailCallReturnCall
321 indirectCalleePtr, directCallee, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
322
323 // We currently support tail calls only when the args are passed via registers
324 // otherwise we fall back to a plain call.
325 // For details, see internal/engine/RATIONALE.md
326 isAllRegs := stackSlotSize == 0
327
328 switch {
329 case isDirectCall && isAllRegs:
330 tailJump := m.allocateInstr()
331 tailJump.asTailCall(directCallee, calleeABI)
332 m.insert(tailJump)
333 case !isDirectCall && isAllRegs:
334 ptr := m.compiler.VRegOf(indirectCalleePtr)
335 callInd := m.allocateInstr()
336 callInd.asTailCallIndirect(ptr, calleeABI)
337 m.insert(callInd)
338 case isDirectCall && !isAllRegs:
339 tailJump := m.allocateInstr()
340 tailJump.asCall(directCallee, calleeABI)
341 m.insert(tailJump)
342 case !isDirectCall && !isAllRegs:
343 ptr := m.compiler.VRegOf(indirectCalleePtr)
344 callInd := m.allocateInstr()
345 callInd.asCallIndirect(ptr, calleeABI)
346 m.insert(callInd)
347 }
348
349 // If this is a proper tail call, returns will be cleared in the postRegAlloc phase.
350 m.insertReturns(si, calleeABI, stackSlotSize)
351 }
352
353 func (m *machine) insertAddOrSubStackPointer(rd regalloc.VReg, diff int64, add bool) {
354 if imm12Operand, ok := asImm12Operand(uint64(diff)); ok {
355 alu := m.allocateInstr()
356 var ao aluOp
357 if add {
358 ao = aluOpAdd
359 } else {
360 ao = aluOpSub
361 }
362 alu.asALU(ao, rd, operandNR(spVReg), imm12Operand, true)
363 m.insert(alu)
364 } else {
365 m.lowerConstantI64(tmpRegVReg, diff)
366 alu := m.allocateInstr()
367 var ao aluOp
368 if add {
369 ao = aluOpAdd
370 } else {
371 ao = aluOpSub
372 }
373 alu.asALU(ao, rd, operandNR(spVReg), operandNR(tmpRegVReg), true)
374 m.insert(alu)
375 }
376 }