Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go

raw
1 package arm64
2
3 import (
4 "fmt"
5
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
8 )
9
10 // PostRegAlloc implements backend.Machine.
11 func (m *machine) PostRegAlloc() {
12 m.setupPrologue()
13 m.postRegAlloc()
14 }
15
16 // setupPrologue initializes the prologue of the function.
17 func (m *machine) setupPrologue() {
18 cur := m.rootInstr
19 prevInitInst := cur.next
20
21 //
22 // (high address) (high address)
23 // SP----> +-----------------+ +------------------+ <----+
24 // | ....... | | ....... | |
25 // | ret Y | | ret Y | |
26 // | ....... | | ....... | |
27 // | ret 0 | | ret 0 | |
28 // | arg X | | arg X | | size_of_arg_ret.
29 // | ....... | ====> | ....... | |
30 // | arg 1 | | arg 1 | |
31 // | arg 0 | | arg 0 | <----+
32 // |-----------------| | size_of_arg_ret |
33 // | return address |
34 // +------------------+ <---- SP
35 // (low address) (low address)
36
37 // Saves the return address (lr) and the size_of_arg_ret below the SP.
38 // size_of_arg_ret is used for stack unwinding.
39 cur = m.createReturnAddrAndSizeOfArgRetSlot(cur)
40
41 if !m.stackBoundsCheckDisabled {
42 cur = m.insertStackBoundsCheck(m.requiredStackSize(), cur)
43 }
44
45 // Decrement SP if spillSlotSize > 0.
46 if m.spillSlotSize == 0 && len(m.spillSlots) != 0 {
47 panic(fmt.Sprintf("BUG: spillSlotSize=%d, spillSlots=%v\n", m.spillSlotSize, m.spillSlots))
48 }
49
50 if regs := m.clobberedRegs; len(regs) > 0 {
51 //
52 // (high address) (high address)
53 // +-----------------+ +-----------------+
54 // | ....... | | ....... |
55 // | ret Y | | ret Y |
56 // | ....... | | ....... |
57 // | ret 0 | | ret 0 |
58 // | arg X | | arg X |
59 // | ....... | | ....... |
60 // | arg 1 | | arg 1 |
61 // | arg 0 | | arg 0 |
62 // | size_of_arg_ret | | size_of_arg_ret |
63 // | ReturnAddress | | ReturnAddress |
64 // SP----> +-----------------+ ====> +-----------------+
65 // (low address) | clobbered M |
66 // | ............ |
67 // | clobbered 0 |
68 // +-----------------+ <----- SP
69 // (low address)
70 //
71 _amode := addressModePreOrPostIndex(m, spVReg,
72 -16, // stack pointer must be 16-byte aligned.
73 true, // Decrement before store.
74 )
75 for _, vr := range regs {
76 // TODO: pair stores to reduce the number of instructions.
77 store := m.allocateInstr()
78 store.asStore(operandNR(vr), _amode, regTypeToRegisterSizeInBits(vr.RegType()))
79 cur = linkInstr(cur, store)
80 }
81 }
82
83 if size := m.spillSlotSize; size > 0 {
84 // Check if size is 16-byte aligned.
85 if size&0xf != 0 {
86 panic(fmt.Errorf("BUG: spill slot size %d is not 16-byte aligned", size))
87 }
88
89 cur = m.addsAddOrSubStackPointer(cur, spVReg, size, false)
90
91 // At this point, the stack looks like:
92 //
93 // (high address)
94 // +------------------+
95 // | ....... |
96 // | ret Y |
97 // | ....... |
98 // | ret 0 |
99 // | arg X |
100 // | ....... |
101 // | arg 1 |
102 // | arg 0 |
103 // | size_of_arg_ret |
104 // | ReturnAddress |
105 // +------------------+
106 // | clobbered M |
107 // | ............ |
108 // | clobbered 0 |
109 // | spill slot N |
110 // | ............ |
111 // | spill slot 2 |
112 // | spill slot 0 |
113 // SP----> +------------------+
114 // (low address)
115 }
116
117 // We push the frame size into the stack to make it possible to unwind stack:
118 //
119 //
120 // (high address) (high address)
121 // +-----------------+ +-----------------+
122 // | ....... | | ....... |
123 // | ret Y | | ret Y |
124 // | ....... | | ....... |
125 // | ret 0 | | ret 0 |
126 // | arg X | | arg X |
127 // | ....... | | ....... |
128 // | arg 1 | | arg 1 |
129 // | arg 0 | | arg 0 |
130 // | size_of_arg_ret | | size_of_arg_ret |
131 // | ReturnAddress | | ReturnAddress |
132 // +-----------------+ ==> +-----------------+ <----+
133 // | clobbered M | | clobbered M | |
134 // | ............ | | ............ | |
135 // | clobbered 2 | | clobbered 2 | |
136 // | clobbered 1 | | clobbered 1 | | frame size
137 // | clobbered 0 | | clobbered 0 | |
138 // | spill slot N | | spill slot N | |
139 // | ............ | | ............ | |
140 // | spill slot 0 | | spill slot 0 | <----+
141 // SP---> +-----------------+ | xxxxxx | ;; unused space to make it 16-byte aligned.
142 // | frame_size |
143 // +-----------------+ <---- SP
144 // (low address)
145 //
146 cur = m.createFrameSizeSlot(cur, m.frameSize())
147
148 linkInstr(cur, prevInitInst)
149 }
150
151 func (m *machine) createReturnAddrAndSizeOfArgRetSlot(cur *instruction) *instruction {
152 // First we decrement the stack pointer to point the arg0 slot.
153 var sizeOfArgRetReg regalloc.VReg
154 s := int64(m.currentABI.AlignedArgResultStackSlotSize())
155 if s > 0 {
156 cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, s)
157 sizeOfArgRetReg = tmpRegVReg
158
159 subSp := m.allocateInstr()
160 subSp.asALU(aluOpSub, spVReg, operandNR(spVReg), operandNR(sizeOfArgRetReg), true)
161 cur = linkInstr(cur, subSp)
162 } else {
163 sizeOfArgRetReg = xzrVReg
164 }
165
166 // Saves the return address (lr) and the size_of_arg_ret below the SP.
167 // size_of_arg_ret is used for stack unwinding.
168 pstr := m.allocateInstr()
169 amode := addressModePreOrPostIndex(m, spVReg, -16, true /* decrement before store */)
170 pstr.asStorePair64(lrVReg, sizeOfArgRetReg, amode)
171 cur = linkInstr(cur, pstr)
172 return cur
173 }
174
175 func (m *machine) createFrameSizeSlot(cur *instruction, s int64) *instruction {
176 var frameSizeReg regalloc.VReg
177 if s > 0 {
178 cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, s)
179 frameSizeReg = tmpRegVReg
180 } else {
181 frameSizeReg = xzrVReg
182 }
183 _amode := addressModePreOrPostIndex(m, spVReg,
184 -16, // stack pointer must be 16-byte aligned.
185 true, // Decrement before store.
186 )
187 store := m.allocateInstr()
188 store.asStore(operandNR(frameSizeReg), _amode, 64)
189 cur = linkInstr(cur, store)
190 return cur
191 }
192
193 // postRegAlloc does multiple things while walking through the instructions:
194 // 1. Removes the redundant copy instruction.
195 // 2. Inserts the epilogue.
196 func (m *machine) postRegAlloc() {
197 for cur := m.rootInstr; cur != nil; cur = cur.next {
198 switch cur.kind {
199 case ret:
200 m.setupEpilogueAfter(cur.prev)
201 case tailCall, tailCallInd:
202 m.setupEpilogueAfter(cur.prev)
203 // If this has been encoded as a proper tail call, we can remove the trailing instructions.
204 // For details, see internal/engine/RATIONALE.md
205 m.removeUntilRet(cur.next)
206 case loadConstBlockArg:
207 lc := cur
208 next := lc.next
209 m.pendingInstructions = m.pendingInstructions[:0]
210 m.lowerLoadConstantBlockArgAfterRegAlloc(lc)
211 for _, instr := range m.pendingInstructions {
212 cur = linkInstr(cur, instr)
213 }
214 linkInstr(cur, next)
215 m.pendingInstructions = m.pendingInstructions[:0]
216 default:
217 // Removes the redundant copy instruction.
218 if cur.IsCopy() && cur.rn.realReg() == cur.rd.RealReg() {
219 prev, next := cur.prev, cur.next
220 // Remove the copy instruction.
221 prev.next = next
222 if next != nil {
223 next.prev = prev
224 }
225 }
226 }
227 }
228 }
229
230 func (m *machine) setupEpilogueAfter(cur *instruction) {
231 prevNext := cur.next
232
233 // We've stored the frame size in the prologue, and now that we are about to return from this function, we won't need it anymore.
234 cur = m.addsAddOrSubStackPointer(cur, spVReg, 16, true)
235
236 if s := m.spillSlotSize; s > 0 {
237 // Adjust SP to the original value:
238 //
239 // (high address) (high address)
240 // +-----------------+ +-----------------+
241 // | ....... | | ....... |
242 // | ret Y | | ret Y |
243 // | ....... | | ....... |
244 // | ret 0 | | ret 0 |
245 // | arg X | | arg X |
246 // | ....... | | ....... |
247 // | arg 1 | | arg 1 |
248 // | arg 0 | | arg 0 |
249 // | xxxxx | | xxxxx |
250 // | ReturnAddress | | ReturnAddress |
251 // +-----------------+ ====> +-----------------+
252 // | clobbered M | | clobbered M |
253 // | ............ | | ............ |
254 // | clobbered 1 | | clobbered 1 |
255 // | clobbered 0 | | clobbered 0 |
256 // | spill slot N | +-----------------+ <---- SP
257 // | ............ |
258 // | spill slot 0 |
259 // SP---> +-----------------+
260 // (low address)
261 //
262 cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true)
263 }
264
265 // First we need to restore the clobbered registers.
266 if len(m.clobberedRegs) > 0 {
267 // (high address)
268 // +-----------------+ +-----------------+
269 // | ....... | | ....... |
270 // | ret Y | | ret Y |
271 // | ....... | | ....... |
272 // | ret 0 | | ret 0 |
273 // | arg X | | arg X |
274 // | ....... | | ....... |
275 // | arg 1 | | arg 1 |
276 // | arg 0 | | arg 0 |
277 // | xxxxx | | xxxxx |
278 // | ReturnAddress | | ReturnAddress |
279 // +-----------------+ ========> +-----------------+ <---- SP
280 // | clobbered M |
281 // | ........... |
282 // | clobbered 1 |
283 // | clobbered 0 |
284 // SP---> +-----------------+
285 // (low address)
286
287 l := len(m.clobberedRegs) - 1
288 for i := range m.clobberedRegs {
289 vr := m.clobberedRegs[l-i] // reverse order to restore.
290 load := m.allocateInstr()
291 amode := addressModePreOrPostIndex(m, spVReg,
292 16, // stack pointer must be 16-byte aligned.
293 false, // Increment after store.
294 )
295 // TODO: pair loads to reduce the number of instructions.
296 switch regTypeToRegisterSizeInBits(vr.RegType()) {
297 case 64: // save int reg.
298 load.asULoad(vr, amode, 64)
299 case 128: // save vector reg.
300 load.asFpuLoad(vr, amode, 128)
301 }
302 cur = linkInstr(cur, load)
303 }
304 }
305
306 // Reload the return address (lr).
307 //
308 // +-----------------+ +-----------------+
309 // | ....... | | ....... |
310 // | ret Y | | ret Y |
311 // | ....... | | ....... |
312 // | ret 0 | | ret 0 |
313 // | arg X | | arg X |
314 // | ....... | ===> | ....... |
315 // | arg 1 | | arg 1 |
316 // | arg 0 | | arg 0 |
317 // | xxxxx | +-----------------+ <---- SP
318 // | ReturnAddress |
319 // SP----> +-----------------+
320
321 ldr := m.allocateInstr()
322 ldr.asULoad(lrVReg,
323 addressModePreOrPostIndex(m, spVReg, 16 /* stack pointer must be 16-byte aligned. */, false /* increment after loads */), 64)
324 cur = linkInstr(cur, ldr)
325
326 if s := int64(m.currentABI.AlignedArgResultStackSlotSize()); s > 0 {
327 cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true)
328 }
329
330 linkInstr(cur, prevNext)
331 }
332
333 // removeUntilRet removes the instructions starting from `cur` until the first `ret` instruction.
334 func (m *machine) removeUntilRet(cur *instruction) {
335 for ; cur != nil; cur = cur.next {
336 prev, next := cur.prev, cur.next
337 prev.next = next
338 if next != nil {
339 next.prev = prev
340 }
341 if cur.kind == ret {
342 return
343 }
344 }
345 }
346
347 // saveRequiredRegs is the set of registers that must be saved/restored during growing stack when there's insufficient
348 // stack space left. Basically this is the combination of CalleeSavedRegisters plus argument registers execpt for x0,
349 // which always points to the execution context whenever the native code is entered from Go.
350 var saveRequiredRegs = []regalloc.VReg{
351 x1VReg, x2VReg, x3VReg, x4VReg, x5VReg, x6VReg, x7VReg,
352 x19VReg, x20VReg, x21VReg, x22VReg, x23VReg, x24VReg, x25VReg, x26VReg, x28VReg, lrVReg,
353 v0VReg, v1VReg, v2VReg, v3VReg, v4VReg, v5VReg, v6VReg, v7VReg,
354 v18VReg, v19VReg, v20VReg, v21VReg, v22VReg, v23VReg, v24VReg, v25VReg, v26VReg, v27VReg, v28VReg, v29VReg, v30VReg, v31VReg,
355 }
356
357 // insertStackBoundsCheck will insert the instructions after `cur` to check the
358 // stack bounds, and if there's no sufficient spaces required for the function,
359 // exit the execution and try growing it in Go world.
360 //
361 // TODO: we should be able to share the instructions across all the functions to reduce the size of compiled executable.
362 func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instruction) *instruction {
363 if requiredStackSize%16 != 0 {
364 panic("BUG")
365 }
366
367 if immm12op, ok := asImm12Operand(uint64(requiredStackSize)); ok {
368 // sub tmp, sp, #requiredStackSize
369 sub := m.allocateInstr()
370 sub.asALU(aluOpSub, tmpRegVReg, operandNR(spVReg), immm12op, true)
371 cur = linkInstr(cur, sub)
372 } else {
373 // This case, we first load the requiredStackSize into the temporary register,
374 cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, requiredStackSize)
375 // Then subtract it.
376 sub := m.allocateInstr()
377 sub.asALU(aluOpSub, tmpRegVReg, operandNR(spVReg), operandNR(tmpRegVReg), true)
378 cur = linkInstr(cur, sub)
379 }
380
381 tmp2 := x11VReg // Caller save, so it is safe to use it here in the prologue.
382
383 // ldr tmp2, [executionContext #StackBottomPtr]
384 ldr := m.allocateInstr()
385 amode := m.amodePool.Allocate()
386 *amode = addressMode{
387 kind: addressModeKindRegUnsignedImm12,
388 rn: x0VReg, // execution context is always the first argument.
389 imm: wazevoapi.ExecutionContextOffsetStackBottomPtr.I64(),
390 }
391 ldr.asULoad(tmp2, amode, 64)
392 cur = linkInstr(cur, ldr)
393
394 // subs xzr, tmp, tmp2
395 subs := m.allocateInstr()
396 subs.asALU(aluOpSubS, xzrVReg, operandNR(tmpRegVReg), operandNR(tmp2), true)
397 cur = linkInstr(cur, subs)
398
399 // b.ge #imm
400 cbr := m.allocateInstr()
401 cbr.asCondBr(ge.asCond(), labelInvalid, false /* ignored */)
402 cur = linkInstr(cur, cbr)
403
404 // Set the required stack size and set it to the exec context.
405 {
406 // First load the requiredStackSize into the temporary register,
407 cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, requiredStackSize)
408 setRequiredStackSize := m.allocateInstr()
409 amode := m.amodePool.Allocate()
410 *amode = addressMode{
411 kind: addressModeKindRegUnsignedImm12,
412 // Execution context is always the first argument.
413 rn: x0VReg, imm: wazevoapi.ExecutionContextOffsetStackGrowRequiredSize.I64(),
414 }
415 setRequiredStackSize.asStore(operandNR(tmpRegVReg), amode, 64)
416
417 cur = linkInstr(cur, setRequiredStackSize)
418 }
419
420 ldrAddress := m.allocateInstr()
421 amode2 := m.amodePool.Allocate()
422 *amode2 = addressMode{
423 kind: addressModeKindRegUnsignedImm12,
424 rn: x0VReg, // execution context is always the first argument
425 imm: wazevoapi.ExecutionContextOffsetStackGrowCallTrampolineAddress.I64(),
426 }
427 ldrAddress.asULoad(tmpRegVReg, amode2, 64)
428 cur = linkInstr(cur, ldrAddress)
429
430 // Then jumps to the stack grow call sequence's address, meaning
431 // transferring the control to the code compiled by CompileStackGrowCallSequence.
432 bl := m.allocateInstr()
433 bl.asCallIndirect(tmpRegVReg, nil)
434 cur = linkInstr(cur, bl)
435
436 // Now that we know the entire code, we can finalize how many bytes
437 // we have to skip when the stack size is sufficient.
438 var cbrOffset int64
439 for _cur := cbr; ; _cur = _cur.next {
440 cbrOffset += _cur.size()
441 if _cur == cur {
442 break
443 }
444 }
445 cbr.condBrOffsetResolve(cbrOffset)
446 return cur
447 }
448
449 // CompileStackGrowCallSequence implements backend.Machine.
450 func (m *machine) CompileStackGrowCallSequence() []byte {
451 cur := m.allocateInstr()
452 cur.asNop0()
453 m.rootInstr = cur
454
455 // Save the callee saved and argument registers.
456 cur = m.saveRegistersInExecutionContext(cur, saveRequiredRegs)
457
458 // Save the current stack pointer.
459 cur = m.saveCurrentStackPointer(cur, x0VReg)
460
461 // Set the exit status on the execution context.
462 cur = m.setExitCode(cur, x0VReg, wazevoapi.ExitCodeGrowStack)
463
464 // Exit the execution.
465 cur = m.storeReturnAddressAndExit(cur)
466
467 // After the exit, restore the saved registers.
468 cur = m.restoreRegistersInExecutionContext(cur, saveRequiredRegs)
469
470 // Then goes back the original address of this stack grow call.
471 ret := m.allocateInstr()
472 ret.asRet()
473 linkInstr(cur, ret)
474
475 m.encode(m.rootInstr)
476 return m.compiler.Buf()
477 }
478
479 func (m *machine) addsAddOrSubStackPointer(cur *instruction, rd regalloc.VReg, diff int64, add bool) *instruction {
480 m.pendingInstructions = m.pendingInstructions[:0]
481 m.insertAddOrSubStackPointer(rd, diff, add)
482 for _, inserted := range m.pendingInstructions {
483 cur = linkInstr(cur, inserted)
484 }
485 return cur
486 }