Debian Codegraph @

internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go

raw
1 package amd64
2
3 import (
4 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
5 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
6 )
7
8 // PostRegAlloc implements backend.Machine.
9 func (m *machine) PostRegAlloc() {
10 m.setupPrologue()
11 m.postRegAlloc()
12 }
13
14 func (m *machine) setupPrologue() {
15 cur := m.rootInstr
16 prevInitInst := cur.next
17
18 // At this point, we have the stack layout as follows:
19 //
20 // (high address)
21 // +-----------------+ <----- RBP (somewhere in the middle of the stack)
22 // | ....... |
23 // | ret Y |
24 // | ....... |
25 // | ret 0 |
26 // | arg X |
27 // | ....... |
28 // | arg 1 |
29 // | arg 0 |
30 // | Return Addr |
31 // RSP ----> +-----------------+
32 // (low address)
33
34 // First, we push the RBP, and update the RBP to the current RSP.
35 //
36 // (high address) (high address)
37 // RBP ----> +-----------------+ +-----------------+
38 // | ....... | | ....... |
39 // | ret Y | | ret Y |
40 // | ....... | | ....... |
41 // | ret 0 | | ret 0 |
42 // | arg X | | arg X |
43 // | ....... | ====> | ....... |
44 // | arg 1 | | arg 1 |
45 // | arg 0 | | arg 0 |
46 // | Return Addr | | Return Addr |
47 // RSP ----> +-----------------+ | Caller_RBP |
48 // (low address) +-----------------+ <----- RSP, RBP
49 //
50 cur = m.setupRBPRSP(cur)
51
52 if !m.stackBoundsCheckDisabled {
53 cur = m.insertStackBoundsCheck(m.requiredStackSize(), cur)
54 }
55
56 //
57 // (high address)
58 // +-----------------+ +-----------------+
59 // | ....... | | ....... |
60 // | ret Y | | ret Y |
61 // | ....... | | ....... |
62 // | ret 0 | | ret 0 |
63 // | arg X | | arg X |
64 // | ....... | | ....... |
65 // | arg 1 | | arg 1 |
66 // | arg 0 | | arg 0 |
67 // | xxxxx | | xxxxx |
68 // | Return Addr | | Return Addr |
69 // | Caller_RBP | ====> | Caller_RBP |
70 // RBP,RSP->+-----------------+ +-----------------+ <----- RBP
71 // (low address) | clobbered M |
72 // | clobbered 1 |
73 // | ........... |
74 // | clobbered 0 |
75 // +-----------------+ <----- RSP
76 //
77 if regs := m.clobberedRegs; len(regs) > 0 {
78 for i := range regs {
79 r := regs[len(regs)-1-i] // Reverse order.
80 if r.RegType() == regalloc.RegTypeInt {
81 cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandReg(r)))
82 } else {
83 // Push the XMM register is not supported by the PUSH instruction.
84 cur = m.addRSP(-16, cur)
85 push := m.allocateInstr().asXmmMovRM(
86 sseOpcodeMovdqu, r, newOperandMem(m.newAmodeImmReg(0, rspVReg)),
87 )
88 cur = linkInstr(cur, push)
89 }
90 }
91 }
92
93 if size := m.spillSlotSize; size > 0 {
94 // Simply decrease the RSP to allocate the spill slots.
95 // sub $size, %rsp
96 cur = linkInstr(cur, m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(uint32(size)), rspVReg, true))
97
98 // At this point, we have the stack layout as follows:
99 //
100 // (high address)
101 // +-----------------+
102 // | ....... |
103 // | ret Y |
104 // | ....... |
105 // | ret 0 |
106 // | arg X |
107 // | ....... |
108 // | arg 1 |
109 // | arg 0 |
110 // | ReturnAddress |
111 // | Caller_RBP |
112 // +-----------------+ <--- RBP
113 // | clobbered M |
114 // | ............ |
115 // | clobbered 1 |
116 // | clobbered 0 |
117 // | spill slot N |
118 // | ............ |
119 // | spill slot 0 |
120 // +-----------------+ <--- RSP
121 // (low address)
122 }
123
124 linkInstr(cur, prevInitInst)
125 }
126
127 // postRegAlloc does multiple things while walking through the instructions:
128 // 1. Inserts the epilogue code.
129 // 2. Removes the redundant copy instruction.
130 // 3. Inserts the dec/inc RSP instruction right before/after the call instruction.
131 // 4. Lowering that is supposed to be done after regalloc.
132 func (m *machine) postRegAlloc() {
133 for cur := m.rootInstr; cur != nil; cur = cur.next {
134 switch k := cur.kind; k {
135 case ret:
136 m.setupEpilogueAfter(cur.prev)
137 continue
138 case fcvtToSintSequence, fcvtToUintSequence:
139 m.pendingInstructions = m.pendingInstructions[:0]
140 if k == fcvtToSintSequence {
141 m.lowerFcvtToSintSequenceAfterRegalloc(cur)
142 } else {
143 m.lowerFcvtToUintSequenceAfterRegalloc(cur)
144 }
145 prev := cur.prev
146 next := cur.next
147 cur := prev
148 for _, instr := range m.pendingInstructions {
149 cur = linkInstr(cur, instr)
150 }
151 linkInstr(cur, next)
152 continue
153 case xmmCMov:
154 m.pendingInstructions = m.pendingInstructions[:0]
155 m.lowerXmmCmovAfterRegAlloc(cur)
156 prev := cur.prev
157 next := cur.next
158 cur := prev
159 for _, instr := range m.pendingInstructions {
160 cur = linkInstr(cur, instr)
161 }
162 linkInstr(cur, next)
163 continue
164 case idivRemSequence:
165 m.pendingInstructions = m.pendingInstructions[:0]
166 m.lowerIDivRemSequenceAfterRegAlloc(cur)
167 prev := cur.prev
168 next := cur.next
169 cur := prev
170 for _, instr := range m.pendingInstructions {
171 cur = linkInstr(cur, instr)
172 }
173 linkInstr(cur, next)
174 continue
175 case call, callIndirect:
176 // At this point, reg alloc is done, therefore we can safely insert dec/inc RPS instruction
177 // right before/after the call instruction. If this is done before reg alloc, the stack slot
178 // can point to the wrong location and therefore results in a wrong value.
179 call := cur
180 next := call.next
181 _, _, _, _, size := backend.ABIInfoFromUint64(call.u2)
182 if size > 0 {
183 dec := m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(size), rspVReg, true)
184 linkInstr(call.prev, dec)
185 linkInstr(dec, call)
186 inc := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, newOperandImm32(size), rspVReg, true)
187 linkInstr(call, inc)
188 linkInstr(inc, next)
189 }
190 continue
191 case tailCall, tailCallIndirect:
192 // At this point, reg alloc is done, therefore we can safely insert dec RPS instruction
193 // right before the tail call (jump) instruction. If this is done before reg alloc, the stack slot
194 // can point to the wrong location and therefore results in a wrong value.
195 tailCall := cur
196 _, _, _, _, size := backend.ABIInfoFromUint64(tailCall.u2)
197 if size > 0 {
198 dec := m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(size), rspVReg, true)
199 linkInstr(tailCall.prev, dec)
200 linkInstr(dec, tailCall)
201 }
202 // In a tail call, we insert the epilogue before the jump instruction.
203 m.setupEpilogueAfter(tailCall.prev)
204 // If this has been encoded as a proper tail call, we can remove the trailing instructions
205 // For details, see internal/engine/RATIONALE.md
206 m.removeUntilRet(cur.next)
207 continue
208 }
209
210 // Removes the redundant copy instruction.
211 if cur.IsCopy() && cur.op1.reg().RealReg() == cur.op2.reg().RealReg() {
212 prev, next := cur.prev, cur.next
213 // Remove the copy instruction.
214 prev.next = next
215 if next != nil {
216 next.prev = prev
217 }
218 }
219 }
220 }
221
222 func (m *machine) setupEpilogueAfter(cur *instruction) {
223 prevNext := cur.next
224
225 // At this point, we have the stack layout as follows:
226 //
227 // (high address)
228 // +-----------------+
229 // | ....... |
230 // | ret Y |
231 // | ....... |
232 // | ret 0 |
233 // | arg X |
234 // | ....... |
235 // | arg 1 |
236 // | arg 0 |
237 // | ReturnAddress |
238 // | Caller_RBP |
239 // +-----------------+ <--- RBP
240 // | clobbered M |
241 // | ............ |
242 // | clobbered 1 |
243 // | clobbered 0 |
244 // | spill slot N |
245 // | ............ |
246 // | spill slot 0 |
247 // +-----------------+ <--- RSP
248 // (low address)
249
250 if size := m.spillSlotSize; size > 0 {
251 // Simply increase the RSP to free the spill slots.
252 // add $size, %rsp
253 cur = linkInstr(cur, m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, newOperandImm32(uint32(size)), rspVReg, true))
254 }
255
256 //
257 // (high address)
258 // +-----------------+ +-----------------+
259 // | ....... | | ....... |
260 // | ret Y | | ret Y |
261 // | ....... | | ....... |
262 // | ret 0 | | ret 0 |
263 // | arg X | | arg X |
264 // | ....... | | ....... |
265 // | arg 1 | | arg 1 |
266 // | arg 0 | | arg 0 |
267 // | ReturnAddress | | ReturnAddress |
268 // | Caller_RBP | | Caller_RBP |
269 // RBP ---> +-----------------+ ========> +-----------------+ <---- RSP, RBP
270 // | clobbered M |
271 // | ............ |
272 // | clobbered 1 |
273 // | clobbered 0 |
274 // RSP ---> +-----------------+
275 // (low address)
276 //
277 if regs := m.clobberedRegs; len(regs) > 0 {
278 for _, r := range regs {
279 if r.RegType() == regalloc.RegTypeInt {
280 cur = linkInstr(cur, m.allocateInstr().asPop64(r))
281 } else {
282 // Pop the XMM register is not supported by the POP instruction.
283 pop := m.allocateInstr().asXmmUnaryRmR(
284 sseOpcodeMovdqu, newOperandMem(m.newAmodeImmReg(0, rspVReg)), r,
285 )
286 cur = linkInstr(cur, pop)
287 cur = m.addRSP(16, cur)
288 }
289 }
290 }
291
292 // Now roll back the RSP to RBP, and pop the caller's RBP.
293 cur = m.revertRBPRSP(cur)
294
295 linkInstr(cur, prevNext)
296 }
297
298 // removeUntilRet removes the instructions starting from `cur` until the first `ret` instruction.
299 func (m *machine) removeUntilRet(cur *instruction) {
300 for ; cur != nil; cur = cur.next {
301 prev, next := cur.prev, cur.next
302 prev.next = next
303 if next != nil {
304 next.prev = prev
305 }
306 if cur.kind == ret {
307 return
308 }
309 }
310 }
311
312 func (m *machine) addRSP(offset int32, cur *instruction) *instruction {
313 if offset == 0 {
314 return cur
315 }
316 opcode := aluRmiROpcodeAdd
317 if offset < 0 {
318 opcode = aluRmiROpcodeSub
319 offset = -offset
320 }
321 return linkInstr(cur, m.allocateInstr().asAluRmiR(opcode, newOperandImm32(uint32(offset)), rspVReg, true))
322 }
323
324 func (m *machine) setupRBPRSP(cur *instruction) *instruction {
325 cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandReg(rbpVReg)))
326 cur = linkInstr(cur, m.allocateInstr().asMovRR(rspVReg, rbpVReg, true))
327 return cur
328 }
329
330 func (m *machine) revertRBPRSP(cur *instruction) *instruction {
331 cur = linkInstr(cur, m.allocateInstr().asMovRR(rbpVReg, rspVReg, true))
332 cur = linkInstr(cur, m.allocateInstr().asPop64(rbpVReg))
333 return cur
334 }