| 1 |
package arm64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 5 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 6 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 7 |
) |
| 8 |
|
| 9 |
// References: |
| 10 |
// * https://github.com/golang/go/blob/go1.24.0/src/cmd/compile/abi-internal.md#arm64-architecture |
| 11 |
// * https://developer.arm.com/documentation/102374/0101/Procedure-Call-Standard |
| 12 |
|
| 13 |
var ( |
| 14 |
intParamResultRegs = []regalloc.RealReg{x0, x1, x2, x3, x4, x5, x6, x7} |
| 15 |
floatParamResultRegs = []regalloc.RealReg{v0, v1, v2, v3, v4, v5, v6, v7} |
| 16 |
) |
| 17 |
|
| 18 |
var regInfo = ®alloc.RegisterInfo{ |
| 19 |
AllocatableRegisters: [regalloc.NumRegType][]regalloc.RealReg{ |
| 20 |
// We don't allocate: |
| 21 |
// - x18: Reserved by the macOS: https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Respect-the-purpose-of-specific-CPU-registers |
| 22 |
// - x28: Reserved by Go runtime. |
| 23 |
// - x27(=tmpReg): because of the reason described on tmpReg. |
| 24 |
regalloc.RegTypeInt: { |
| 25 |
x8, x9, x10, x11, x12, x13, x14, x15, |
| 26 |
x16, x17, x19, x20, x21, x22, x23, x24, x25, |
| 27 |
x26, x29, x30, |
| 28 |
// These are the argument/return registers. Less preferred in the allocation. |
| 29 |
x7, x6, x5, x4, x3, x2, x1, x0, |
| 30 |
}, |
| 31 |
regalloc.RegTypeFloat: { |
| 32 |
v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, |
| 33 |
v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, |
| 34 |
// These are the argument/return registers. Less preferred in the allocation. |
| 35 |
v7, v6, v5, v4, v3, v2, v1, v0, |
| 36 |
}, |
| 37 |
}, |
| 38 |
CalleeSavedRegisters: regalloc.NewRegSet( |
| 39 |
x19, x20, x21, x22, x23, x24, x25, x26, x28, |
| 40 |
v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, |
| 41 |
), |
| 42 |
CallerSavedRegisters: regalloc.NewRegSet( |
| 43 |
x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x29, x30, |
| 44 |
v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, |
| 45 |
), |
| 46 |
RealRegToVReg: []regalloc.VReg{ |
| 47 |
x0: x0VReg, x1: x1VReg, x2: x2VReg, x3: x3VReg, x4: x4VReg, x5: x5VReg, x6: x6VReg, x7: x7VReg, x8: x8VReg, x9: x9VReg, x10: x10VReg, x11: x11VReg, x12: x12VReg, x13: x13VReg, x14: x14VReg, x15: x15VReg, x16: x16VReg, x17: x17VReg, x18: x18VReg, x19: x19VReg, x20: x20VReg, x21: x21VReg, x22: x22VReg, x23: x23VReg, x24: x24VReg, x25: x25VReg, x26: x26VReg, x27: x27VReg, x28: x28VReg, x29: x29VReg, x30: x30VReg, |
| 48 |
v0: v0VReg, v1: v1VReg, v2: v2VReg, v3: v3VReg, v4: v4VReg, v5: v5VReg, v6: v6VReg, v7: v7VReg, v8: v8VReg, v9: v9VReg, v10: v10VReg, v11: v11VReg, v12: v12VReg, v13: v13VReg, v14: v14VReg, v15: v15VReg, v16: v16VReg, v17: v17VReg, v18: v18VReg, v19: v19VReg, v20: v20VReg, v21: v21VReg, v22: v22VReg, v23: v23VReg, v24: v24VReg, v25: v25VReg, v26: v26VReg, v27: v27VReg, v28: v28VReg, v29: v29VReg, v30: v30VReg, v31: v31VReg, |
| 49 |
}, |
| 50 |
RealRegName: func(r regalloc.RealReg) string { return regNames[r] }, |
| 51 |
RealRegType: func(r regalloc.RealReg) regalloc.RegType { |
| 52 |
if r < v0 { |
| 53 |
return regalloc.RegTypeInt |
| 54 |
} |
| 55 |
return regalloc.RegTypeFloat |
| 56 |
}, |
| 57 |
} |
| 58 |
|
| 59 |
// ArgsResultsRegs implements backend.Machine. |
| 60 |
func (m *machine) ArgsResultsRegs() (argResultInts, argResultFloats []regalloc.RealReg) { |
| 61 |
return intParamResultRegs, floatParamResultRegs |
| 62 |
} |
| 63 |
|
| 64 |
// LowerParams implements backend.FunctionABI. |
| 65 |
func (m *machine) LowerParams(args []ssa.Value) { |
| 66 |
a := m.currentABI |
| 67 |
|
| 68 |
for i, ssaArg := range args { |
| 69 |
if !ssaArg.Valid() { |
| 70 |
continue |
| 71 |
} |
| 72 |
reg := m.compiler.VRegOf(ssaArg) |
| 73 |
arg := &a.Args[i] |
| 74 |
if arg.Kind == backend.ABIArgKindReg { |
| 75 |
m.InsertMove(reg, arg.Reg, arg.Type) |
| 76 |
} else { |
| 77 |
// TODO: we could use pair load if there's consecutive loads for the same type. |
| 78 |
// |
| 79 |
// (high address) |
| 80 |
// +-----------------+ |
| 81 |
// | ....... | |
| 82 |
// | ret Y | |
| 83 |
// | ....... | |
| 84 |
// | ret 0 | |
| 85 |
// | arg X | |
| 86 |
// | ....... | |
| 87 |
// | arg 1 | |
| 88 |
// | arg 0 | <-| |
| 89 |
// | ReturnAddress | | |
| 90 |
// +-----------------+ | |
| 91 |
// | ........... | | |
| 92 |
// | clobbered M | | argStackOffset: is unknown at this point of compilation. |
| 93 |
// | ............ | | |
| 94 |
// | clobbered 0 | | |
| 95 |
// | spill slot N | | |
| 96 |
// | ........... | | |
| 97 |
// | spill slot 0 | | |
| 98 |
// SP---> +-----------------+ <-+ |
| 99 |
// (low address) |
| 100 |
|
| 101 |
bits := arg.Type.Bits() |
| 102 |
// At this point of compilation, we don't yet know how much space exist below the return address. |
| 103 |
// So we instruct the address mode to add the `argStackOffset` to the offset at the later phase of compilation. |
| 104 |
amode := m.amodePool.Allocate() |
| 105 |
*amode = addressMode{imm: arg.Offset, rn: spVReg, kind: addressModeKindArgStackSpace} |
| 106 |
load := m.allocateInstr() |
| 107 |
switch arg.Type { |
| 108 |
case ssa.TypeI32, ssa.TypeI64: |
| 109 |
load.asULoad(reg, amode, bits) |
| 110 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 111 |
load.asFpuLoad(reg, amode, bits) |
| 112 |
default: |
| 113 |
panic("BUG") |
| 114 |
} |
| 115 |
m.insert(load) |
| 116 |
m.unresolvedAddressModes = append(m.unresolvedAddressModes, load) |
| 117 |
} |
| 118 |
} |
| 119 |
} |
| 120 |
|
| 121 |
// LowerReturns lowers the given returns. |
| 122 |
func (m *machine) LowerReturns(rets []ssa.Value) { |
| 123 |
a := m.currentABI |
| 124 |
|
| 125 |
l := len(rets) - 1 |
| 126 |
for i := range rets { |
| 127 |
// Reverse order in order to avoid overwriting the stack returns existing in the return registers. |
| 128 |
ret := rets[l-i] |
| 129 |
r := &a.Rets[l-i] |
| 130 |
reg := m.compiler.VRegOf(ret) |
| 131 |
if def := m.compiler.ValueDefinition(ret); def.IsFromInstr() { |
| 132 |
// Constant instructions are inlined. |
| 133 |
if inst := def.Instr; inst.Constant() { |
| 134 |
val := inst.Return() |
| 135 |
valType := val.Type() |
| 136 |
v := inst.ConstantVal() |
| 137 |
m.insertLoadConstant(v, valType, reg) |
| 138 |
} |
| 139 |
} |
| 140 |
if r.Kind == backend.ABIArgKindReg { |
| 141 |
m.InsertMove(r.Reg, reg, ret.Type()) |
| 142 |
} else { |
| 143 |
// TODO: we could use pair store if there's consecutive stores for the same type. |
| 144 |
// |
| 145 |
// (high address) |
| 146 |
// +-----------------+ |
| 147 |
// | ....... | |
| 148 |
// | ret Y | |
| 149 |
// | ....... | |
| 150 |
// | ret 0 | <-+ |
| 151 |
// | arg X | | |
| 152 |
// | ....... | | |
| 153 |
// | arg 1 | | |
| 154 |
// | arg 0 | | |
| 155 |
// | ReturnAddress | | |
| 156 |
// +-----------------+ | |
| 157 |
// | ........... | | |
| 158 |
// | spill slot M | | retStackOffset: is unknown at this point of compilation. |
| 159 |
// | ............ | | |
| 160 |
// | spill slot 2 | | |
| 161 |
// | spill slot 1 | | |
| 162 |
// | clobbered 0 | | |
| 163 |
// | clobbered 1 | | |
| 164 |
// | ........... | | |
| 165 |
// | clobbered N | | |
| 166 |
// SP---> +-----------------+ <-+ |
| 167 |
// (low address) |
| 168 |
|
| 169 |
bits := r.Type.Bits() |
| 170 |
|
| 171 |
// At this point of compilation, we don't yet know how much space exist below the return address. |
| 172 |
// So we instruct the address mode to add the `retStackOffset` to the offset at the later phase of compilation. |
| 173 |
amode := m.amodePool.Allocate() |
| 174 |
*amode = addressMode{imm: r.Offset, rn: spVReg, kind: addressModeKindResultStackSpace} |
| 175 |
store := m.allocateInstr() |
| 176 |
store.asStore(operandNR(reg), amode, bits) |
| 177 |
m.insert(store) |
| 178 |
m.unresolvedAddressModes = append(m.unresolvedAddressModes, store) |
| 179 |
} |
| 180 |
} |
| 181 |
} |
| 182 |
|
| 183 |
// callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the |
| 184 |
// caller side of the function call. |
| 185 |
func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, slotBegin int64) { |
| 186 |
arg := &a.Args[argIndex] |
| 187 |
if def.IsFromInstr() { |
| 188 |
// Constant instructions are inlined. |
| 189 |
if inst := def.Instr; inst.Constant() { |
| 190 |
val := inst.Return() |
| 191 |
valType := val.Type() |
| 192 |
v := inst.ConstantVal() |
| 193 |
m.insertLoadConstant(v, valType, reg) |
| 194 |
} |
| 195 |
} |
| 196 |
if arg.Kind == backend.ABIArgKindReg { |
| 197 |
m.InsertMove(arg.Reg, reg, arg.Type) |
| 198 |
} else { |
| 199 |
// TODO: we could use pair store if there's consecutive stores for the same type. |
| 200 |
// |
| 201 |
// Note that at this point, stack pointer is already adjusted. |
| 202 |
bits := arg.Type.Bits() |
| 203 |
amode := m.resolveAddressModeForOffset(arg.Offset-slotBegin, bits, spVReg, false) |
| 204 |
store := m.allocateInstr() |
| 205 |
store.asStore(operandNR(reg), amode, bits) |
| 206 |
m.insert(store) |
| 207 |
} |
| 208 |
} |
| 209 |
|
| 210 |
func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, slotBegin int64) { |
| 211 |
r := &a.Rets[retIndex] |
| 212 |
if r.Kind == backend.ABIArgKindReg { |
| 213 |
m.InsertMove(reg, r.Reg, r.Type) |
| 214 |
} else { |
| 215 |
// TODO: we could use pair load if there's consecutive loads for the same type. |
| 216 |
amode := m.resolveAddressModeForOffset(a.ArgStackSize+r.Offset-slotBegin, r.Type.Bits(), spVReg, false) |
| 217 |
ldr := m.allocateInstr() |
| 218 |
switch r.Type { |
| 219 |
case ssa.TypeI32, ssa.TypeI64: |
| 220 |
ldr.asULoad(reg, amode, r.Type.Bits()) |
| 221 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 222 |
ldr.asFpuLoad(reg, amode, r.Type.Bits()) |
| 223 |
default: |
| 224 |
panic("BUG") |
| 225 |
} |
| 226 |
m.insert(ldr) |
| 227 |
} |
| 228 |
} |
| 229 |
|
| 230 |
func (m *machine) resolveAddressModeForOffsetAndInsert(cur *instruction, offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) (*instruction, *addressMode) { |
| 231 |
m.pendingInstructions = m.pendingInstructions[:0] |
| 232 |
mode := m.resolveAddressModeForOffset(offset, dstBits, rn, allowTmpRegUse) |
| 233 |
for _, instr := range m.pendingInstructions { |
| 234 |
cur = linkInstr(cur, instr) |
| 235 |
} |
| 236 |
return cur, mode |
| 237 |
} |
| 238 |
|
| 239 |
func (m *machine) resolveAddressModeForOffset(offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) *addressMode { |
| 240 |
if rn.RegType() != regalloc.RegTypeInt { |
| 241 |
panic("BUG: rn should be a pointer: " + formatVRegSized(rn, 64)) |
| 242 |
} |
| 243 |
amode := m.amodePool.Allocate() |
| 244 |
if offsetFitsInAddressModeKindRegUnsignedImm12(dstBits, offset) { |
| 245 |
*amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: rn, imm: offset} |
| 246 |
} else if offsetFitsInAddressModeKindRegSignedImm9(offset) { |
| 247 |
*amode = addressMode{kind: addressModeKindRegSignedImm9, rn: rn, imm: offset} |
| 248 |
} else { |
| 249 |
var indexReg regalloc.VReg |
| 250 |
if allowTmpRegUse { |
| 251 |
m.lowerConstantI64(tmpRegVReg, offset) |
| 252 |
indexReg = tmpRegVReg |
| 253 |
} else { |
| 254 |
indexReg = m.compiler.AllocateVReg(ssa.TypeI64) |
| 255 |
m.lowerConstantI64(indexReg, offset) |
| 256 |
} |
| 257 |
*amode = addressMode{kind: addressModeKindRegReg, rn: rn, rm: indexReg, extOp: extendOpUXTX /* indicates index rm is 64-bit */} |
| 258 |
} |
| 259 |
return amode |
| 260 |
} |
| 261 |
|
| 262 |
func (m *machine) lowerCall(si *ssa.Instruction) { |
| 263 |
isDirectCall := si.Opcode() == ssa.OpcodeCall |
| 264 |
indirectCalleePtr, directCallee, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall) |
| 265 |
|
| 266 |
if isDirectCall { |
| 267 |
call := m.allocateInstr() |
| 268 |
call.asCall(directCallee, calleeABI) |
| 269 |
m.insert(call) |
| 270 |
} else { |
| 271 |
ptr := m.compiler.VRegOf(indirectCalleePtr) |
| 272 |
callInd := m.allocateInstr() |
| 273 |
callInd.asCallIndirect(ptr, calleeABI) |
| 274 |
m.insert(callInd) |
| 275 |
} |
| 276 |
|
| 277 |
m.insertReturns(si, calleeABI, stackSlotSize) |
| 278 |
} |
| 279 |
|
| 280 |
func (m *machine) prepareCall(si *ssa.Instruction, isDirectCall bool) (ssa.Value, ssa.FuncRef, *backend.FunctionABI, int64) { |
| 281 |
var indirectCalleePtr ssa.Value |
| 282 |
var directCallee ssa.FuncRef |
| 283 |
var sigID ssa.SignatureID |
| 284 |
var args []ssa.Value |
| 285 |
if isDirectCall { |
| 286 |
directCallee, sigID, args = si.CallData() |
| 287 |
} else { |
| 288 |
indirectCalleePtr, sigID, args, _ /* on arm64, the calling convention is compatible with the Go runtime */ = si.CallIndirectData() |
| 289 |
} |
| 290 |
calleeABI := m.compiler.GetFunctionABI(m.compiler.SSABuilder().ResolveSignature(sigID)) |
| 291 |
|
| 292 |
stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize()) |
| 293 |
if m.maxRequiredStackSizeForCalls < stackSlotSize+16 { |
| 294 |
m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // return address frame. |
| 295 |
} |
| 296 |
|
| 297 |
for i, arg := range args { |
| 298 |
reg := m.compiler.VRegOf(arg) |
| 299 |
def := m.compiler.ValueDefinition(arg) |
| 300 |
m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize) |
| 301 |
} |
| 302 |
return indirectCalleePtr, directCallee, calleeABI, stackSlotSize |
| 303 |
} |
| 304 |
|
| 305 |
func (m *machine) insertReturns(si *ssa.Instruction, calleeABI *backend.FunctionABI, stackSlotSize int64) { |
| 306 |
var index int |
| 307 |
r1, rs := si.Returns() |
| 308 |
if r1.Valid() { |
| 309 |
m.callerGenFunctionReturnVReg(calleeABI, 0, m.compiler.VRegOf(r1), stackSlotSize) |
| 310 |
index++ |
| 311 |
} |
| 312 |
|
| 313 |
for _, r := range rs { |
| 314 |
m.callerGenFunctionReturnVReg(calleeABI, index, m.compiler.VRegOf(r), stackSlotSize) |
| 315 |
index++ |
| 316 |
} |
| 317 |
} |
| 318 |
|
| 319 |
func (m *machine) lowerTailCall(si *ssa.Instruction) { |
| 320 |
isDirectCall := si.Opcode() == ssa.OpcodeTailCallReturnCall |
| 321 |
indirectCalleePtr, directCallee, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall) |
| 322 |
|
| 323 |
// We currently support tail calls only when the args are passed via registers |
| 324 |
// otherwise we fall back to a plain call. |
| 325 |
// For details, see internal/engine/RATIONALE.md |
| 326 |
isAllRegs := stackSlotSize == 0 |
| 327 |
|
| 328 |
switch { |
| 329 |
case isDirectCall && isAllRegs: |
| 330 |
tailJump := m.allocateInstr() |
| 331 |
tailJump.asTailCall(directCallee, calleeABI) |
| 332 |
m.insert(tailJump) |
| 333 |
case !isDirectCall && isAllRegs: |
| 334 |
ptr := m.compiler.VRegOf(indirectCalleePtr) |
| 335 |
callInd := m.allocateInstr() |
| 336 |
callInd.asTailCallIndirect(ptr, calleeABI) |
| 337 |
m.insert(callInd) |
| 338 |
case isDirectCall && !isAllRegs: |
| 339 |
tailJump := m.allocateInstr() |
| 340 |
tailJump.asCall(directCallee, calleeABI) |
| 341 |
m.insert(tailJump) |
| 342 |
case !isDirectCall && !isAllRegs: |
| 343 |
ptr := m.compiler.VRegOf(indirectCalleePtr) |
| 344 |
callInd := m.allocateInstr() |
| 345 |
callInd.asCallIndirect(ptr, calleeABI) |
| 346 |
m.insert(callInd) |
| 347 |
} |
| 348 |
|
| 349 |
// If this is a proper tail call, returns will be cleared in the postRegAlloc phase. |
| 350 |
m.insertReturns(si, calleeABI, stackSlotSize) |
| 351 |
} |
| 352 |
|
| 353 |
func (m *machine) insertAddOrSubStackPointer(rd regalloc.VReg, diff int64, add bool) { |
| 354 |
if imm12Operand, ok := asImm12Operand(uint64(diff)); ok { |
| 355 |
alu := m.allocateInstr() |
| 356 |
var ao aluOp |
| 357 |
if add { |
| 358 |
ao = aluOpAdd |
| 359 |
} else { |
| 360 |
ao = aluOpSub |
| 361 |
} |
| 362 |
alu.asALU(ao, rd, operandNR(spVReg), imm12Operand, true) |
| 363 |
m.insert(alu) |
| 364 |
} else { |
| 365 |
m.lowerConstantI64(tmpRegVReg, diff) |
| 366 |
alu := m.allocateInstr() |
| 367 |
var ao aluOp |
| 368 |
if add { |
| 369 |
ao = aluOpAdd |
| 370 |
} else { |
| 371 |
ao = aluOpSub |
| 372 |
} |
| 373 |
alu.asALU(ao, rd, operandNR(spVReg), operandNR(tmpRegVReg), true) |
| 374 |
m.insert(alu) |
| 375 |
} |
| 376 |
} |