| 1 |
package arm64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 5 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 6 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 7 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" |
| 8 |
) |
| 9 |
|
| 10 |
// CompileEntryPreamble implements backend.Machine. This assumes `entrypoint` function (in abi_go_entry_arm64.s) passes: |
| 11 |
// |
| 12 |
// 1. First (execution context ptr) and Second arguments are already passed in x0, and x1. |
| 13 |
// 2. param/result slice ptr in x19; the pointer to []uint64{} which is used to pass arguments and accept return values. |
| 14 |
// 3. Go-allocated stack slice ptr in x26. |
| 15 |
// 4. Function executable in x24. |
| 16 |
// |
| 17 |
// also SP and FP are correct Go-runtime-based values, and LR is the return address to the Go-side caller. |
| 18 |
func (m *machine) CompileEntryPreamble(signature *ssa.Signature) []byte { |
| 19 |
root := m.constructEntryPreamble(signature) |
| 20 |
m.encode(root) |
| 21 |
return m.compiler.Buf() |
| 22 |
} |
| 23 |
|
| 24 |
var ( |
| 25 |
executionContextPtrReg = x0VReg |
| 26 |
// callee-saved regs so that they can be used in the prologue and epilogue. |
| 27 |
paramResultSlicePtr = x19VReg |
| 28 |
savedExecutionContextPtr = x20VReg |
| 29 |
// goAllocatedStackPtr is not used in the epilogue. |
| 30 |
goAllocatedStackPtr = x26VReg |
| 31 |
// paramResultSliceCopied is not used in the epilogue. |
| 32 |
paramResultSliceCopied = x25VReg |
| 33 |
// tmpRegVReg is not used in the epilogue. |
| 34 |
functionExecutable = x24VReg |
| 35 |
) |
| 36 |
|
| 37 |
func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, arg *backend.ABIArg, argStartOffsetFromSP int64) *instruction { |
| 38 |
typ := arg.Type |
| 39 |
bits := typ.Bits() |
| 40 |
isStackArg := arg.Kind == backend.ABIArgKindStack |
| 41 |
|
| 42 |
var loadTargetReg operand |
| 43 |
if !isStackArg { |
| 44 |
loadTargetReg = operandNR(arg.Reg) |
| 45 |
} else { |
| 46 |
switch typ { |
| 47 |
case ssa.TypeI32, ssa.TypeI64: |
| 48 |
loadTargetReg = operandNR(x15VReg) |
| 49 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 50 |
loadTargetReg = operandNR(v15VReg) |
| 51 |
default: |
| 52 |
panic("TODO?") |
| 53 |
} |
| 54 |
} |
| 55 |
|
| 56 |
var postIndexImm int64 |
| 57 |
if typ == ssa.TypeV128 { |
| 58 |
postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice. |
| 59 |
} else { |
| 60 |
postIndexImm = 8 |
| 61 |
} |
| 62 |
loadMode := m.amodePool.Allocate() |
| 63 |
*loadMode = addressMode{kind: addressModeKindPostIndex, rn: paramSlicePtr, imm: postIndexImm} |
| 64 |
|
| 65 |
instr := m.allocateInstr() |
| 66 |
switch typ { |
| 67 |
case ssa.TypeI32: |
| 68 |
instr.asULoad(loadTargetReg.reg(), loadMode, 32) |
| 69 |
case ssa.TypeI64: |
| 70 |
instr.asULoad(loadTargetReg.reg(), loadMode, 64) |
| 71 |
case ssa.TypeF32: |
| 72 |
instr.asFpuLoad(loadTargetReg.reg(), loadMode, 32) |
| 73 |
case ssa.TypeF64: |
| 74 |
instr.asFpuLoad(loadTargetReg.reg(), loadMode, 64) |
| 75 |
case ssa.TypeV128: |
| 76 |
instr.asFpuLoad(loadTargetReg.reg(), loadMode, 128) |
| 77 |
} |
| 78 |
cur = linkInstr(cur, instr) |
| 79 |
|
| 80 |
if isStackArg { |
| 81 |
var storeMode *addressMode |
| 82 |
cur, storeMode = m.resolveAddressModeForOffsetAndInsert(cur, argStartOffsetFromSP+arg.Offset, bits, spVReg, true) |
| 83 |
toStack := m.allocateInstr() |
| 84 |
toStack.asStore(loadTargetReg, storeMode, bits) |
| 85 |
cur = linkInstr(cur, toStack) |
| 86 |
} |
| 87 |
return cur |
| 88 |
} |
| 89 |
|
| 90 |
func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, result *backend.ABIArg, resultStartOffsetFromSP int64) *instruction { |
| 91 |
isStackArg := result.Kind == backend.ABIArgKindStack |
| 92 |
typ := result.Type |
| 93 |
bits := typ.Bits() |
| 94 |
|
| 95 |
var storeTargetReg operand |
| 96 |
if !isStackArg { |
| 97 |
storeTargetReg = operandNR(result.Reg) |
| 98 |
} else { |
| 99 |
switch typ { |
| 100 |
case ssa.TypeI32, ssa.TypeI64: |
| 101 |
storeTargetReg = operandNR(x15VReg) |
| 102 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 103 |
storeTargetReg = operandNR(v15VReg) |
| 104 |
default: |
| 105 |
panic("TODO?") |
| 106 |
} |
| 107 |
} |
| 108 |
|
| 109 |
var postIndexImm int64 |
| 110 |
if typ == ssa.TypeV128 { |
| 111 |
postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice. |
| 112 |
} else { |
| 113 |
postIndexImm = 8 |
| 114 |
} |
| 115 |
|
| 116 |
if isStackArg { |
| 117 |
var loadMode *addressMode |
| 118 |
cur, loadMode = m.resolveAddressModeForOffsetAndInsert(cur, resultStartOffsetFromSP+result.Offset, bits, spVReg, true) |
| 119 |
toReg := m.allocateInstr() |
| 120 |
switch typ { |
| 121 |
case ssa.TypeI32, ssa.TypeI64: |
| 122 |
toReg.asULoad(storeTargetReg.reg(), loadMode, bits) |
| 123 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 124 |
toReg.asFpuLoad(storeTargetReg.reg(), loadMode, bits) |
| 125 |
default: |
| 126 |
panic("TODO?") |
| 127 |
} |
| 128 |
cur = linkInstr(cur, toReg) |
| 129 |
} |
| 130 |
|
| 131 |
mode := m.amodePool.Allocate() |
| 132 |
*mode = addressMode{kind: addressModeKindPostIndex, rn: resultSlicePtr, imm: postIndexImm} |
| 133 |
instr := m.allocateInstr() |
| 134 |
instr.asStore(storeTargetReg, mode, bits) |
| 135 |
cur = linkInstr(cur, instr) |
| 136 |
return cur |
| 137 |
} |
| 138 |
|
| 139 |
func (m *machine) constructEntryPreamble(sig *ssa.Signature) (root *instruction) { |
| 140 |
abi := backend.FunctionABI{} |
| 141 |
abi.Init(sig, intParamResultRegs, floatParamResultRegs) |
| 142 |
|
| 143 |
root = m.allocateNop() |
| 144 |
|
| 145 |
//// ----------------------------------- prologue ----------------------------------- //// |
| 146 |
|
| 147 |
// First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well. |
| 148 |
// mov savedExecutionContextPtr, x0 |
| 149 |
cur := m.move64(savedExecutionContextPtr, executionContextPtrReg, root) |
| 150 |
|
| 151 |
// Next, save the current FP, SP and LR into the wazevo.executionContext: |
| 152 |
// str fp, [savedExecutionContextPtr, #OriginalFramePointer] |
| 153 |
// mov tmp, sp ;; sp cannot be str'ed directly. |
| 154 |
// str sp, [savedExecutionContextPtr, #OriginalStackPointer] |
| 155 |
// str lr, [savedExecutionContextPtr, #GoReturnAddress] |
| 156 |
cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, true, cur) |
| 157 |
cur = m.move64(tmpRegVReg, spVReg, cur) |
| 158 |
cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, true, cur) |
| 159 |
cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, true, cur) |
| 160 |
|
| 161 |
// Then, move the Go-allocated stack pointer to SP: |
| 162 |
// mov sp, goAllocatedStackPtr |
| 163 |
cur = m.move64(spVReg, goAllocatedStackPtr, cur) |
| 164 |
|
| 165 |
prReg := paramResultSlicePtr |
| 166 |
if len(abi.Args) > 2 && len(abi.Rets) > 0 { |
| 167 |
// paramResultSlicePtr is modified during the execution of goEntryPreamblePassArg, |
| 168 |
// so copy it to another reg. |
| 169 |
cur = m.move64(paramResultSliceCopied, paramResultSlicePtr, cur) |
| 170 |
prReg = paramResultSliceCopied |
| 171 |
} |
| 172 |
|
| 173 |
stackSlotSize := int64(abi.AlignedArgResultStackSlotSize()) |
| 174 |
for i := range abi.Args { |
| 175 |
if i < 2 { |
| 176 |
// module context ptr and execution context ptr are passed in x0 and x1 by the Go assembly function. |
| 177 |
continue |
| 178 |
} |
| 179 |
arg := &abi.Args[i] |
| 180 |
cur = m.goEntryPreamblePassArg(cur, prReg, arg, -stackSlotSize) |
| 181 |
} |
| 182 |
|
| 183 |
// Call the real function. |
| 184 |
bl := m.allocateInstr() |
| 185 |
bl.asCallIndirect(functionExecutable, &abi) |
| 186 |
cur = linkInstr(cur, bl) |
| 187 |
|
| 188 |
///// ----------------------------------- epilogue ----------------------------------- ///// |
| 189 |
|
| 190 |
// Store the register results into paramResultSlicePtr. |
| 191 |
for i := range abi.Rets { |
| 192 |
cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, &abi.Rets[i], abi.ArgStackSize-stackSlotSize) |
| 193 |
} |
| 194 |
|
| 195 |
// Finally, restore the FP, SP and LR, and return to the Go code. |
| 196 |
// ldr fp, [savedExecutionContextPtr, #OriginalFramePointer] |
| 197 |
// ldr tmp, [savedExecutionContextPtr, #OriginalStackPointer] |
| 198 |
// mov sp, tmp ;; sp cannot be str'ed directly. |
| 199 |
// ldr lr, [savedExecutionContextPtr, #GoReturnAddress] |
| 200 |
// ret ;; --> return to the Go code |
| 201 |
cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, false, cur) |
| 202 |
cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, false, cur) |
| 203 |
cur = m.move64(spVReg, tmpRegVReg, cur) |
| 204 |
cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, false, cur) |
| 205 |
retInst := m.allocateInstr() |
| 206 |
retInst.asRet() |
| 207 |
linkInstr(cur, retInst) |
| 208 |
return |
| 209 |
} |
| 210 |
|
| 211 |
func (m *machine) move64(dst, src regalloc.VReg, prev *instruction) *instruction { |
| 212 |
instr := m.allocateInstr() |
| 213 |
instr.asMove64(dst, src) |
| 214 |
return linkInstr(prev, instr) |
| 215 |
} |
| 216 |
|
| 217 |
func (m *machine) loadOrStoreAtExecutionContext(d regalloc.VReg, offset wazevoapi.Offset, store bool, prev *instruction) *instruction { |
| 218 |
instr := m.allocateInstr() |
| 219 |
mode := m.amodePool.Allocate() |
| 220 |
*mode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: savedExecutionContextPtr, imm: offset.I64()} |
| 221 |
if store { |
| 222 |
instr.asStore(operandNR(d), mode, 64) |
| 223 |
} else { |
| 224 |
instr.asULoad(d, mode, 64) |
| 225 |
} |
| 226 |
return linkInstr(prev, instr) |
| 227 |
} |
| 228 |
|
| 229 |
func linkInstr(prev, next *instruction) *instruction { |
| 230 |
prev.next = next |
| 231 |
next.prev = prev |
| 232 |
return next |
| 233 |
} |