Debian Codegraph @

internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go

raw
1 package amd64
2
3 import (
4 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
5 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
8 )
9
10 var (
11 executionContextPtrReg = raxVReg
12
13 // Followings are callee saved registers. They can be used freely in the entry preamble
14 // since the preamble is called via Go assembly function which has stack-based ABI.
15
16 // savedExecutionContextPtr also must be a callee-saved reg so that they can be used in the prologue and epilogue.
17 savedExecutionContextPtr = rdxVReg
18 // paramResultSlicePtr must match with entrypoint function in abi_entry_amd64.s.
19 paramResultSlicePtr = r12VReg
20 // goAllocatedStackPtr must match with entrypoint function in abi_entry_amd64.s.
21 goAllocatedStackPtr = r13VReg
22 // functionExecutable must match with entrypoint function in abi_entry_amd64.s.
23 functionExecutable = r14VReg
24 tmpIntReg = r15VReg
25 tmpXmmReg = xmm15VReg
26 )
27
28 // CompileEntryPreamble implements backend.Machine.
29 func (m *machine) CompileEntryPreamble(sig *ssa.Signature) []byte {
30 root := m.compileEntryPreamble(sig)
31 m.encodeWithoutSSA(root)
32 buf := m.c.Buf()
33 return buf
34 }
35
36 func (m *machine) compileEntryPreamble(sig *ssa.Signature) *instruction {
37 abi := backend.FunctionABI{}
38 abi.Init(sig, intArgResultRegs, floatArgResultRegs)
39
40 root := m.allocateNop()
41
42 //// ----------------------------------- prologue ----------------------------------- ////
43
44 // First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well.
45 // mov %executionContextPtrReg, %savedExecutionContextPtr
46 cur := m.move64(executionContextPtrReg, savedExecutionContextPtr, root)
47
48 // Next is to save the original RBP and RSP into the execution context.
49 cur = m.saveOriginalRSPRBP(cur)
50
51 // Now set the RSP to the Go-allocated stack pointer.
52 // mov %goAllocatedStackPtr, %rsp
53 cur = m.move64(goAllocatedStackPtr, rspVReg, cur)
54
55 if stackSlotSize := abi.AlignedArgResultStackSlotSize(); stackSlotSize > 0 {
56 // Allocate stack slots for the arguments and return values.
57 // sub $stackSlotSize, %rsp
58 spDec := m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(uint32(stackSlotSize)), rspVReg, true)
59 cur = linkInstr(cur, spDec)
60 }
61
62 var offset uint32
63 for i := range abi.Args {
64 if i < 2 {
65 // module context ptr and execution context ptr are passed in rax and rbx by the Go assembly function.
66 continue
67 }
68 arg := &abi.Args[i]
69 cur = m.goEntryPreamblePassArg(cur, paramResultSlicePtr, offset, arg)
70 if arg.Type == ssa.TypeV128 {
71 offset += 16
72 } else {
73 offset += 8
74 }
75 }
76
77 // Zero out RBP so that the unwind/stack growth code can correctly detect the end of the stack.
78 zerosRbp := m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(rbpVReg), rbpVReg, true)
79 cur = linkInstr(cur, zerosRbp)
80
81 // Now ready to call the real function. Note that at this point stack pointer is already set to the Go-allocated,
82 // which is aligned to 16 bytes.
83 call := m.allocateInstr().asCallIndirect(newOperandReg(functionExecutable), &abi)
84 cur = linkInstr(cur, call)
85
86 //// ----------------------------------- epilogue ----------------------------------- ////
87
88 // Read the results from regs and the stack, and set them correctly into the paramResultSlicePtr.
89 offset = 0
90 for i := range abi.Rets {
91 r := &abi.Rets[i]
92 cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, offset, r, uint32(abi.ArgStackSize))
93 if r.Type == ssa.TypeV128 {
94 offset += 16
95 } else {
96 offset += 8
97 }
98 }
99
100 // Finally, restore the original RBP and RSP.
101 cur = m.restoreOriginalRSPRBP(cur)
102
103 ret := m.allocateInstr().asRet()
104 linkInstr(cur, ret)
105 return root
106 }
107
108 // saveOriginalRSPRBP saves the original RSP and RBP into the execution context.
109 func (m *machine) saveOriginalRSPRBP(cur *instruction) *instruction {
110 // mov %rbp, wazevoapi.ExecutionContextOffsetOriginalFramePointer(%executionContextPtrReg)
111 // mov %rsp, wazevoapi.ExecutionContextOffsetOriginalStackPointer(%executionContextPtrReg)
112 cur = m.loadOrStore64AtExecutionCtx(executionContextPtrReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, rbpVReg, true, cur)
113 cur = m.loadOrStore64AtExecutionCtx(executionContextPtrReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, rspVReg, true, cur)
114 return cur
115 }
116
117 // restoreOriginalRSPRBP restores the original RSP and RBP from the execution context.
118 func (m *machine) restoreOriginalRSPRBP(cur *instruction) *instruction {
119 // mov wazevoapi.ExecutionContextOffsetOriginalFramePointer(%executionContextPtrReg), %rbp
120 // mov wazevoapi.ExecutionContextOffsetOriginalStackPointer(%executionContextPtrReg), %rsp
121 cur = m.loadOrStore64AtExecutionCtx(savedExecutionContextPtr, wazevoapi.ExecutionContextOffsetOriginalFramePointer, rbpVReg, false, cur)
122 cur = m.loadOrStore64AtExecutionCtx(savedExecutionContextPtr, wazevoapi.ExecutionContextOffsetOriginalStackPointer, rspVReg, false, cur)
123 return cur
124 }
125
126 func (m *machine) move64(src, dst regalloc.VReg, prev *instruction) *instruction {
127 mov := m.allocateInstr().asMovRR(src, dst, true)
128 return linkInstr(prev, mov)
129 }
130
131 func (m *machine) loadOrStore64AtExecutionCtx(execCtx regalloc.VReg, offset wazevoapi.Offset, r regalloc.VReg, store bool, prev *instruction) *instruction {
132 mem := newOperandMem(m.newAmodeImmReg(offset.U32(), execCtx))
133 instr := m.allocateInstr()
134 if store {
135 instr.asMovRM(r, mem, 8)
136 } else {
137 instr.asMov64MR(mem, r)
138 }
139 return linkInstr(prev, instr)
140 }
141
142 // This is for debugging.
143 func (m *machine) linkUD2(cur *instruction) *instruction { //nolint
144 return linkInstr(cur, m.allocateInstr().asUD2())
145 }
146
147 func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, offsetInParamSlice uint32, arg *backend.ABIArg) *instruction {
148 var dst regalloc.VReg
149 argTyp := arg.Type
150 if arg.Kind == backend.ABIArgKindStack {
151 // Caller saved registers ca
152 switch argTyp {
153 case ssa.TypeI32, ssa.TypeI64:
154 dst = tmpIntReg
155 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
156 dst = tmpXmmReg
157 default:
158 panic("BUG")
159 }
160 } else {
161 dst = arg.Reg
162 }
163
164 load := m.allocateInstr()
165 a := newOperandMem(m.newAmodeImmReg(offsetInParamSlice, paramSlicePtr))
166 switch arg.Type {
167 case ssa.TypeI32:
168 load.asMovzxRmR(extModeLQ, a, dst)
169 case ssa.TypeI64:
170 load.asMov64MR(a, dst)
171 case ssa.TypeF32:
172 load.asXmmUnaryRmR(sseOpcodeMovss, a, dst)
173 case ssa.TypeF64:
174 load.asXmmUnaryRmR(sseOpcodeMovsd, a, dst)
175 case ssa.TypeV128:
176 load.asXmmUnaryRmR(sseOpcodeMovdqu, a, dst)
177 }
178
179 cur = linkInstr(cur, load)
180 if arg.Kind == backend.ABIArgKindStack {
181 // Store back to the stack.
182 store := m.allocateInstr()
183 a := newOperandMem(m.newAmodeImmReg(uint32(arg.Offset), rspVReg))
184 switch arg.Type {
185 case ssa.TypeI32:
186 store.asMovRM(dst, a, 4)
187 case ssa.TypeI64:
188 store.asMovRM(dst, a, 8)
189 case ssa.TypeF32:
190 store.asXmmMovRM(sseOpcodeMovss, dst, a)
191 case ssa.TypeF64:
192 store.asXmmMovRM(sseOpcodeMovsd, dst, a)
193 case ssa.TypeV128:
194 store.asXmmMovRM(sseOpcodeMovdqu, dst, a)
195 }
196 cur = linkInstr(cur, store)
197 }
198 return cur
199 }
200
201 func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, offsetInResultSlice uint32, result *backend.ABIArg, resultStackSlotBeginOffset uint32) *instruction {
202 var r regalloc.VReg
203 if result.Kind == backend.ABIArgKindStack {
204 // Load the value to the temporary.
205 load := m.allocateInstr()
206 offset := resultStackSlotBeginOffset + uint32(result.Offset)
207 a := newOperandMem(m.newAmodeImmReg(offset, rspVReg))
208 switch result.Type {
209 case ssa.TypeI32:
210 r = tmpIntReg
211 load.asMovzxRmR(extModeLQ, a, r)
212 case ssa.TypeI64:
213 r = tmpIntReg
214 load.asMov64MR(a, r)
215 case ssa.TypeF32:
216 r = tmpXmmReg
217 load.asXmmUnaryRmR(sseOpcodeMovss, a, r)
218 case ssa.TypeF64:
219 r = tmpXmmReg
220 load.asXmmUnaryRmR(sseOpcodeMovsd, a, r)
221 case ssa.TypeV128:
222 r = tmpXmmReg
223 load.asXmmUnaryRmR(sseOpcodeMovdqu, a, r)
224 default:
225 panic("BUG")
226 }
227 cur = linkInstr(cur, load)
228 } else {
229 r = result.Reg
230 }
231
232 store := m.allocateInstr()
233 a := newOperandMem(m.newAmodeImmReg(offsetInResultSlice, resultSlicePtr))
234 switch result.Type {
235 case ssa.TypeI32:
236 store.asMovRM(r, a, 4)
237 case ssa.TypeI64:
238 store.asMovRM(r, a, 8)
239 case ssa.TypeF32:
240 store.asXmmMovRM(sseOpcodeMovss, r, a)
241 case ssa.TypeF64:
242 store.asXmmMovRM(sseOpcodeMovsd, r, a)
243 case ssa.TypeV128:
244 store.asXmmMovRM(sseOpcodeMovdqu, r, a)
245 }
246
247 return linkInstr(cur, store)
248 }