Debian Codegraph @

internal/engine/wazevo/backend/isa/amd64/lower_mem.go

raw
1 package amd64
2
3 import (
4 "fmt"
5
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
8 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
9 )
10
11 var addendsMatchOpcodes = [...]ssa.Opcode{ssa.OpcodeUExtend, ssa.OpcodeSExtend, ssa.OpcodeIadd, ssa.OpcodeIconst, ssa.OpcodeIshl}
12
13 type addend struct {
14 r regalloc.VReg
15 off int64
16 shift byte
17 }
18
19 func (a addend) String() string {
20 return fmt.Sprintf("addend{r=%s, off=%d, shift=%d}", a.r, a.off, a.shift)
21 }
22
23 // lowerToAddressMode converts a pointer to an addressMode that can be used as an operand for load/store instructions.
24 func (m *machine) lowerToAddressMode(ptr ssa.Value, offsetBase uint32) (am *amode) {
25 def := m.c.ValueDefinition(ptr)
26
27 if offsetBase&0x80000000 != 0 {
28 // Special casing the huge base offset whose MSB is set. In x64, the immediate is always
29 // sign-extended, but our IR semantics requires the offset base is always unsigned.
30 // Note that this should be extremely rare or even this shouldn't hit in the real application,
31 // therefore we don't need to optimize this case in my opinion.
32
33 a := m.lowerAddend(def)
34 off64 := a.off + int64(offsetBase)
35 offsetBaseReg := m.c.AllocateVReg(ssa.TypeI64)
36 m.lowerIconst(offsetBaseReg, uint64(off64), true)
37 if a.r != regalloc.VRegInvalid {
38 return m.newAmodeRegRegShift(0, offsetBaseReg, a.r, a.shift)
39 } else {
40 return m.newAmodeImmReg(0, offsetBaseReg)
41 }
42 }
43
44 if op := m.c.MatchInstrOneOf(def, addendsMatchOpcodes[:]); op == ssa.OpcodeIadd {
45 add := def.Instr
46 x, y := add.Arg2()
47 xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
48 ax := m.lowerAddend(xDef)
49 ay := m.lowerAddend(yDef)
50 add.MarkLowered()
51 return m.lowerAddendsToAmode(ax, ay, offsetBase)
52 } else {
53 // If it is not an Iadd, then we lower the one addend.
54 a := m.lowerAddend(def)
55 // off is always 0 if r is valid.
56 if a.r != regalloc.VRegInvalid {
57 if a.shift != 0 {
58 tmpReg := m.c.AllocateVReg(ssa.TypeI64)
59 m.lowerIconst(tmpReg, 0, true)
60 return m.newAmodeRegRegShift(offsetBase, tmpReg, a.r, a.shift)
61 }
62 return m.newAmodeImmReg(offsetBase, a.r)
63 } else {
64 off64 := a.off + int64(offsetBase)
65 tmpReg := m.c.AllocateVReg(ssa.TypeI64)
66 m.lowerIconst(tmpReg, uint64(off64), true)
67 return m.newAmodeImmReg(0, tmpReg)
68 }
69 }
70 }
71
72 func (m *machine) lowerAddendsToAmode(x, y addend, offBase uint32) *amode {
73 if x.r != regalloc.VRegInvalid && x.off != 0 || y.r != regalloc.VRegInvalid && y.off != 0 {
74 panic("invalid input")
75 }
76
77 u64 := uint64(x.off+y.off) + uint64(offBase)
78 if u64 != 0 {
79 if _, ok := asImm32(u64, false); !ok {
80 tmpReg := m.c.AllocateVReg(ssa.TypeI64)
81 m.lowerIconst(tmpReg, u64, true)
82 // Blank u64 as it has been already lowered.
83 u64 = 0
84
85 if x.r == regalloc.VRegInvalid {
86 x.r = tmpReg
87 } else if y.r == regalloc.VRegInvalid {
88 y.r = tmpReg
89 } else {
90 // We already know that either rx or ry is invalid,
91 // so we overwrite it with the temporary register.
92 panic("BUG")
93 }
94 }
95 }
96
97 u32 := uint32(u64)
98 switch {
99 // We assume rx, ry are valid iff offx, offy are 0.
100 case x.r != regalloc.VRegInvalid && y.r != regalloc.VRegInvalid:
101 switch {
102 case x.shift != 0 && y.shift != 0:
103 // Cannot absorb two shifted registers, must lower one to a shift instruction.
104 shifted := m.allocateInstr()
105 shifted.asShiftR(shiftROpShiftLeft, newOperandImm32(uint32(x.shift)), x.r, true)
106 m.insert(shifted)
107
108 return m.newAmodeRegRegShift(u32, x.r, y.r, y.shift)
109 case x.shift != 0 && y.shift == 0:
110 // Swap base and index.
111 x, y = y, x
112 fallthrough
113 default:
114 return m.newAmodeRegRegShift(u32, x.r, y.r, y.shift)
115 }
116 case x.r == regalloc.VRegInvalid && y.r != regalloc.VRegInvalid:
117 x, y = y, x
118 fallthrough
119 case x.r != regalloc.VRegInvalid && y.r == regalloc.VRegInvalid:
120 if x.shift != 0 {
121 zero := m.c.AllocateVReg(ssa.TypeI64)
122 m.lowerIconst(zero, 0, true)
123 return m.newAmodeRegRegShift(u32, zero, x.r, x.shift)
124 }
125 return m.newAmodeImmReg(u32, x.r)
126 default: // Both are invalid: use the offset.
127 tmpReg := m.c.AllocateVReg(ssa.TypeI64)
128 m.lowerIconst(tmpReg, u64, true)
129 return m.newAmodeImmReg(0, tmpReg)
130 }
131 }
132
133 func (m *machine) lowerAddend(x backend.SSAValueDefinition) addend {
134 if !x.IsFromInstr() {
135 return addend{m.c.VRegOf(x.V), 0, 0}
136 }
137 // Ensure the addend is not referenced in multiple places; we will discard nested Iadds.
138 op := m.c.MatchInstrOneOf(x, addendsMatchOpcodes[:])
139 if op != ssa.OpcodeInvalid && op != ssa.OpcodeIadd {
140 return m.lowerAddendFromInstr(x.Instr)
141 }
142 p := m.getOperand_Reg(x)
143 return addend{p.reg(), 0, 0}
144 }
145
146 // lowerAddendFromInstr takes an instruction returns a Vreg and an offset that can be used in an address mode.
147 // The Vreg is regalloc.VRegInvalid if the addend cannot be lowered to a register.
148 // The offset is 0 if the addend can be lowered to a register.
149 func (m *machine) lowerAddendFromInstr(instr *ssa.Instruction) addend {
150 instr.MarkLowered()
151 switch op := instr.Opcode(); op {
152 case ssa.OpcodeIconst:
153 u64 := instr.ConstantVal()
154 if instr.Return().Type().Bits() == 32 {
155 return addend{regalloc.VRegInvalid, int64(int32(u64)), 0} // sign-extend.
156 } else {
157 return addend{regalloc.VRegInvalid, int64(u64), 0}
158 }
159 case ssa.OpcodeUExtend, ssa.OpcodeSExtend:
160 input := instr.Arg()
161 inputDef := m.c.ValueDefinition(input)
162 if input.Type().Bits() != 32 {
163 panic("BUG: invalid input type " + input.Type().String())
164 }
165 constInst := inputDef.IsFromInstr() && inputDef.Instr.Constant()
166 switch {
167 case constInst && op == ssa.OpcodeSExtend:
168 return addend{regalloc.VRegInvalid, int64(uint32(inputDef.Instr.ConstantVal())), 0}
169 case constInst && op == ssa.OpcodeUExtend:
170 return addend{regalloc.VRegInvalid, int64(int32(inputDef.Instr.ConstantVal())), 0} // sign-extend!
171 default:
172 r := m.getOperand_Reg(inputDef)
173 return addend{r.reg(), 0, 0}
174 }
175 case ssa.OpcodeIshl:
176 // If the addend is a shift, we can only handle it if the shift amount is a constant.
177 x, amount := instr.Arg2()
178 amountDef := m.c.ValueDefinition(amount)
179 if amountDef.IsFromInstr() && amountDef.Instr.Constant() && amountDef.Instr.ConstantVal() <= 3 {
180 r := m.getOperand_Reg(m.c.ValueDefinition(x))
181 return addend{r.reg(), 0, uint8(amountDef.Instr.ConstantVal())}
182 }
183 r := m.getOperand_Reg(m.c.ValueDefinition(x))
184 return addend{r.reg(), 0, 0}
185 }
186 panic("BUG: invalid opcode")
187 }