| 1 |
package amd64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 5 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 6 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 7 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" |
| 8 |
) |
| 9 |
|
| 10 |
var ( |
| 11 |
executionContextPtrReg = raxVReg |
| 12 |
|
| 13 |
// Followings are callee saved registers. They can be used freely in the entry preamble |
| 14 |
// since the preamble is called via Go assembly function which has stack-based ABI. |
| 15 |
|
| 16 |
// savedExecutionContextPtr also must be a callee-saved reg so that they can be used in the prologue and epilogue. |
| 17 |
savedExecutionContextPtr = rdxVReg |
| 18 |
// paramResultSlicePtr must match with entrypoint function in abi_entry_amd64.s. |
| 19 |
paramResultSlicePtr = r12VReg |
| 20 |
// goAllocatedStackPtr must match with entrypoint function in abi_entry_amd64.s. |
| 21 |
goAllocatedStackPtr = r13VReg |
| 22 |
// functionExecutable must match with entrypoint function in abi_entry_amd64.s. |
| 23 |
functionExecutable = r14VReg |
| 24 |
tmpIntReg = r15VReg |
| 25 |
tmpXmmReg = xmm15VReg |
| 26 |
) |
| 27 |
|
| 28 |
// CompileEntryPreamble implements backend.Machine. |
| 29 |
func (m *machine) CompileEntryPreamble(sig *ssa.Signature) []byte { |
| 30 |
root := m.compileEntryPreamble(sig) |
| 31 |
m.encodeWithoutSSA(root) |
| 32 |
buf := m.c.Buf() |
| 33 |
return buf |
| 34 |
} |
| 35 |
|
| 36 |
func (m *machine) compileEntryPreamble(sig *ssa.Signature) *instruction { |
| 37 |
abi := backend.FunctionABI{} |
| 38 |
abi.Init(sig, intArgResultRegs, floatArgResultRegs) |
| 39 |
|
| 40 |
root := m.allocateNop() |
| 41 |
|
| 42 |
//// ----------------------------------- prologue ----------------------------------- //// |
| 43 |
|
| 44 |
// First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well. |
| 45 |
// mov %executionContextPtrReg, %savedExecutionContextPtr |
| 46 |
cur := m.move64(executionContextPtrReg, savedExecutionContextPtr, root) |
| 47 |
|
| 48 |
// Next is to save the original RBP and RSP into the execution context. |
| 49 |
cur = m.saveOriginalRSPRBP(cur) |
| 50 |
|
| 51 |
// Now set the RSP to the Go-allocated stack pointer. |
| 52 |
// mov %goAllocatedStackPtr, %rsp |
| 53 |
cur = m.move64(goAllocatedStackPtr, rspVReg, cur) |
| 54 |
|
| 55 |
if stackSlotSize := abi.AlignedArgResultStackSlotSize(); stackSlotSize > 0 { |
| 56 |
// Allocate stack slots for the arguments and return values. |
| 57 |
// sub $stackSlotSize, %rsp |
| 58 |
spDec := m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(uint32(stackSlotSize)), rspVReg, true) |
| 59 |
cur = linkInstr(cur, spDec) |
| 60 |
} |
| 61 |
|
| 62 |
var offset uint32 |
| 63 |
for i := range abi.Args { |
| 64 |
if i < 2 { |
| 65 |
// module context ptr and execution context ptr are passed in rax and rbx by the Go assembly function. |
| 66 |
continue |
| 67 |
} |
| 68 |
arg := &abi.Args[i] |
| 69 |
cur = m.goEntryPreamblePassArg(cur, paramResultSlicePtr, offset, arg) |
| 70 |
if arg.Type == ssa.TypeV128 { |
| 71 |
offset += 16 |
| 72 |
} else { |
| 73 |
offset += 8 |
| 74 |
} |
| 75 |
} |
| 76 |
|
| 77 |
// Zero out RBP so that the unwind/stack growth code can correctly detect the end of the stack. |
| 78 |
zerosRbp := m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(rbpVReg), rbpVReg, true) |
| 79 |
cur = linkInstr(cur, zerosRbp) |
| 80 |
|
| 81 |
// Now ready to call the real function. Note that at this point stack pointer is already set to the Go-allocated, |
| 82 |
// which is aligned to 16 bytes. |
| 83 |
call := m.allocateInstr().asCallIndirect(newOperandReg(functionExecutable), &abi) |
| 84 |
cur = linkInstr(cur, call) |
| 85 |
|
| 86 |
//// ----------------------------------- epilogue ----------------------------------- //// |
| 87 |
|
| 88 |
// Read the results from regs and the stack, and set them correctly into the paramResultSlicePtr. |
| 89 |
offset = 0 |
| 90 |
for i := range abi.Rets { |
| 91 |
r := &abi.Rets[i] |
| 92 |
cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, offset, r, uint32(abi.ArgStackSize)) |
| 93 |
if r.Type == ssa.TypeV128 { |
| 94 |
offset += 16 |
| 95 |
} else { |
| 96 |
offset += 8 |
| 97 |
} |
| 98 |
} |
| 99 |
|
| 100 |
// Finally, restore the original RBP and RSP. |
| 101 |
cur = m.restoreOriginalRSPRBP(cur) |
| 102 |
|
| 103 |
ret := m.allocateInstr().asRet() |
| 104 |
linkInstr(cur, ret) |
| 105 |
return root |
| 106 |
} |
| 107 |
|
| 108 |
// saveOriginalRSPRBP saves the original RSP and RBP into the execution context. |
| 109 |
func (m *machine) saveOriginalRSPRBP(cur *instruction) *instruction { |
| 110 |
// mov %rbp, wazevoapi.ExecutionContextOffsetOriginalFramePointer(%executionContextPtrReg) |
| 111 |
// mov %rsp, wazevoapi.ExecutionContextOffsetOriginalStackPointer(%executionContextPtrReg) |
| 112 |
cur = m.loadOrStore64AtExecutionCtx(executionContextPtrReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, rbpVReg, true, cur) |
| 113 |
cur = m.loadOrStore64AtExecutionCtx(executionContextPtrReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, rspVReg, true, cur) |
| 114 |
return cur |
| 115 |
} |
| 116 |
|
| 117 |
// restoreOriginalRSPRBP restores the original RSP and RBP from the execution context. |
| 118 |
func (m *machine) restoreOriginalRSPRBP(cur *instruction) *instruction { |
| 119 |
// mov wazevoapi.ExecutionContextOffsetOriginalFramePointer(%executionContextPtrReg), %rbp |
| 120 |
// mov wazevoapi.ExecutionContextOffsetOriginalStackPointer(%executionContextPtrReg), %rsp |
| 121 |
cur = m.loadOrStore64AtExecutionCtx(savedExecutionContextPtr, wazevoapi.ExecutionContextOffsetOriginalFramePointer, rbpVReg, false, cur) |
| 122 |
cur = m.loadOrStore64AtExecutionCtx(savedExecutionContextPtr, wazevoapi.ExecutionContextOffsetOriginalStackPointer, rspVReg, false, cur) |
| 123 |
return cur |
| 124 |
} |
| 125 |
|
| 126 |
func (m *machine) move64(src, dst regalloc.VReg, prev *instruction) *instruction { |
| 127 |
mov := m.allocateInstr().asMovRR(src, dst, true) |
| 128 |
return linkInstr(prev, mov) |
| 129 |
} |
| 130 |
|
| 131 |
func (m *machine) loadOrStore64AtExecutionCtx(execCtx regalloc.VReg, offset wazevoapi.Offset, r regalloc.VReg, store bool, prev *instruction) *instruction { |
| 132 |
mem := newOperandMem(m.newAmodeImmReg(offset.U32(), execCtx)) |
| 133 |
instr := m.allocateInstr() |
| 134 |
if store { |
| 135 |
instr.asMovRM(r, mem, 8) |
| 136 |
} else { |
| 137 |
instr.asMov64MR(mem, r) |
| 138 |
} |
| 139 |
return linkInstr(prev, instr) |
| 140 |
} |
| 141 |
|
| 142 |
// This is for debugging. |
| 143 |
func (m *machine) linkUD2(cur *instruction) *instruction { //nolint |
| 144 |
return linkInstr(cur, m.allocateInstr().asUD2()) |
| 145 |
} |
| 146 |
|
| 147 |
func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, offsetInParamSlice uint32, arg *backend.ABIArg) *instruction { |
| 148 |
var dst regalloc.VReg |
| 149 |
argTyp := arg.Type |
| 150 |
if arg.Kind == backend.ABIArgKindStack { |
| 151 |
// Caller saved registers ca |
| 152 |
switch argTyp { |
| 153 |
case ssa.TypeI32, ssa.TypeI64: |
| 154 |
dst = tmpIntReg |
| 155 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 156 |
dst = tmpXmmReg |
| 157 |
default: |
| 158 |
panic("BUG") |
| 159 |
} |
| 160 |
} else { |
| 161 |
dst = arg.Reg |
| 162 |
} |
| 163 |
|
| 164 |
load := m.allocateInstr() |
| 165 |
a := newOperandMem(m.newAmodeImmReg(offsetInParamSlice, paramSlicePtr)) |
| 166 |
switch arg.Type { |
| 167 |
case ssa.TypeI32: |
| 168 |
load.asMovzxRmR(extModeLQ, a, dst) |
| 169 |
case ssa.TypeI64: |
| 170 |
load.asMov64MR(a, dst) |
| 171 |
case ssa.TypeF32: |
| 172 |
load.asXmmUnaryRmR(sseOpcodeMovss, a, dst) |
| 173 |
case ssa.TypeF64: |
| 174 |
load.asXmmUnaryRmR(sseOpcodeMovsd, a, dst) |
| 175 |
case ssa.TypeV128: |
| 176 |
load.asXmmUnaryRmR(sseOpcodeMovdqu, a, dst) |
| 177 |
} |
| 178 |
|
| 179 |
cur = linkInstr(cur, load) |
| 180 |
if arg.Kind == backend.ABIArgKindStack { |
| 181 |
// Store back to the stack. |
| 182 |
store := m.allocateInstr() |
| 183 |
a := newOperandMem(m.newAmodeImmReg(uint32(arg.Offset), rspVReg)) |
| 184 |
switch arg.Type { |
| 185 |
case ssa.TypeI32: |
| 186 |
store.asMovRM(dst, a, 4) |
| 187 |
case ssa.TypeI64: |
| 188 |
store.asMovRM(dst, a, 8) |
| 189 |
case ssa.TypeF32: |
| 190 |
store.asXmmMovRM(sseOpcodeMovss, dst, a) |
| 191 |
case ssa.TypeF64: |
| 192 |
store.asXmmMovRM(sseOpcodeMovsd, dst, a) |
| 193 |
case ssa.TypeV128: |
| 194 |
store.asXmmMovRM(sseOpcodeMovdqu, dst, a) |
| 195 |
} |
| 196 |
cur = linkInstr(cur, store) |
| 197 |
} |
| 198 |
return cur |
| 199 |
} |
| 200 |
|
| 201 |
func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, offsetInResultSlice uint32, result *backend.ABIArg, resultStackSlotBeginOffset uint32) *instruction { |
| 202 |
var r regalloc.VReg |
| 203 |
if result.Kind == backend.ABIArgKindStack { |
| 204 |
// Load the value to the temporary. |
| 205 |
load := m.allocateInstr() |
| 206 |
offset := resultStackSlotBeginOffset + uint32(result.Offset) |
| 207 |
a := newOperandMem(m.newAmodeImmReg(offset, rspVReg)) |
| 208 |
switch result.Type { |
| 209 |
case ssa.TypeI32: |
| 210 |
r = tmpIntReg |
| 211 |
load.asMovzxRmR(extModeLQ, a, r) |
| 212 |
case ssa.TypeI64: |
| 213 |
r = tmpIntReg |
| 214 |
load.asMov64MR(a, r) |
| 215 |
case ssa.TypeF32: |
| 216 |
r = tmpXmmReg |
| 217 |
load.asXmmUnaryRmR(sseOpcodeMovss, a, r) |
| 218 |
case ssa.TypeF64: |
| 219 |
r = tmpXmmReg |
| 220 |
load.asXmmUnaryRmR(sseOpcodeMovsd, a, r) |
| 221 |
case ssa.TypeV128: |
| 222 |
r = tmpXmmReg |
| 223 |
load.asXmmUnaryRmR(sseOpcodeMovdqu, a, r) |
| 224 |
default: |
| 225 |
panic("BUG") |
| 226 |
} |
| 227 |
cur = linkInstr(cur, load) |
| 228 |
} else { |
| 229 |
r = result.Reg |
| 230 |
} |
| 231 |
|
| 232 |
store := m.allocateInstr() |
| 233 |
a := newOperandMem(m.newAmodeImmReg(offsetInResultSlice, resultSlicePtr)) |
| 234 |
switch result.Type { |
| 235 |
case ssa.TypeI32: |
| 236 |
store.asMovRM(r, a, 4) |
| 237 |
case ssa.TypeI64: |
| 238 |
store.asMovRM(r, a, 8) |
| 239 |
case ssa.TypeF32: |
| 240 |
store.asXmmMovRM(sseOpcodeMovss, r, a) |
| 241 |
case ssa.TypeF64: |
| 242 |
store.asXmmMovRM(sseOpcodeMovsd, r, a) |
| 243 |
case ssa.TypeV128: |
| 244 |
store.asXmmMovRM(sseOpcodeMovdqu, r, a) |
| 245 |
} |
| 246 |
|
| 247 |
return linkInstr(cur, store) |
| 248 |
} |