| 1 |
package amd64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"fmt" |
| 5 |
|
| 6 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 7 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 8 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 9 |
) |
| 10 |
|
| 11 |
var addendsMatchOpcodes = [...]ssa.Opcode{ssa.OpcodeUExtend, ssa.OpcodeSExtend, ssa.OpcodeIadd, ssa.OpcodeIconst, ssa.OpcodeIshl} |
| 12 |
|
| 13 |
type addend struct { |
| 14 |
r regalloc.VReg |
| 15 |
off int64 |
| 16 |
shift byte |
| 17 |
} |
| 18 |
|
| 19 |
func (a addend) String() string { |
| 20 |
return fmt.Sprintf("addend{r=%s, off=%d, shift=%d}", a.r, a.off, a.shift) |
| 21 |
} |
| 22 |
|
| 23 |
// lowerToAddressMode converts a pointer to an addressMode that can be used as an operand for load/store instructions. |
| 24 |
func (m *machine) lowerToAddressMode(ptr ssa.Value, offsetBase uint32) (am *amode) { |
| 25 |
def := m.c.ValueDefinition(ptr) |
| 26 |
|
| 27 |
if offsetBase&0x80000000 != 0 { |
| 28 |
// Special casing the huge base offset whose MSB is set. In x64, the immediate is always |
| 29 |
// sign-extended, but our IR semantics requires the offset base is always unsigned. |
| 30 |
// Note that this should be extremely rare or even this shouldn't hit in the real application, |
| 31 |
// therefore we don't need to optimize this case in my opinion. |
| 32 |
|
| 33 |
a := m.lowerAddend(def) |
| 34 |
off64 := a.off + int64(offsetBase) |
| 35 |
offsetBaseReg := m.c.AllocateVReg(ssa.TypeI64) |
| 36 |
m.lowerIconst(offsetBaseReg, uint64(off64), true) |
| 37 |
if a.r != regalloc.VRegInvalid { |
| 38 |
return m.newAmodeRegRegShift(0, offsetBaseReg, a.r, a.shift) |
| 39 |
} else { |
| 40 |
return m.newAmodeImmReg(0, offsetBaseReg) |
| 41 |
} |
| 42 |
} |
| 43 |
|
| 44 |
if op := m.c.MatchInstrOneOf(def, addendsMatchOpcodes[:]); op == ssa.OpcodeIadd { |
| 45 |
add := def.Instr |
| 46 |
x, y := add.Arg2() |
| 47 |
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 48 |
ax := m.lowerAddend(xDef) |
| 49 |
ay := m.lowerAddend(yDef) |
| 50 |
add.MarkLowered() |
| 51 |
return m.lowerAddendsToAmode(ax, ay, offsetBase) |
| 52 |
} else { |
| 53 |
// If it is not an Iadd, then we lower the one addend. |
| 54 |
a := m.lowerAddend(def) |
| 55 |
// off is always 0 if r is valid. |
| 56 |
if a.r != regalloc.VRegInvalid { |
| 57 |
if a.shift != 0 { |
| 58 |
tmpReg := m.c.AllocateVReg(ssa.TypeI64) |
| 59 |
m.lowerIconst(tmpReg, 0, true) |
| 60 |
return m.newAmodeRegRegShift(offsetBase, tmpReg, a.r, a.shift) |
| 61 |
} |
| 62 |
return m.newAmodeImmReg(offsetBase, a.r) |
| 63 |
} else { |
| 64 |
off64 := a.off + int64(offsetBase) |
| 65 |
tmpReg := m.c.AllocateVReg(ssa.TypeI64) |
| 66 |
m.lowerIconst(tmpReg, uint64(off64), true) |
| 67 |
return m.newAmodeImmReg(0, tmpReg) |
| 68 |
} |
| 69 |
} |
| 70 |
} |
| 71 |
|
| 72 |
func (m *machine) lowerAddendsToAmode(x, y addend, offBase uint32) *amode { |
| 73 |
if x.r != regalloc.VRegInvalid && x.off != 0 || y.r != regalloc.VRegInvalid && y.off != 0 { |
| 74 |
panic("invalid input") |
| 75 |
} |
| 76 |
|
| 77 |
u64 := uint64(x.off+y.off) + uint64(offBase) |
| 78 |
if u64 != 0 { |
| 79 |
if _, ok := asImm32(u64, false); !ok { |
| 80 |
tmpReg := m.c.AllocateVReg(ssa.TypeI64) |
| 81 |
m.lowerIconst(tmpReg, u64, true) |
| 82 |
// Blank u64 as it has been already lowered. |
| 83 |
u64 = 0 |
| 84 |
|
| 85 |
if x.r == regalloc.VRegInvalid { |
| 86 |
x.r = tmpReg |
| 87 |
} else if y.r == regalloc.VRegInvalid { |
| 88 |
y.r = tmpReg |
| 89 |
} else { |
| 90 |
// We already know that either rx or ry is invalid, |
| 91 |
// so we overwrite it with the temporary register. |
| 92 |
panic("BUG") |
| 93 |
} |
| 94 |
} |
| 95 |
} |
| 96 |
|
| 97 |
u32 := uint32(u64) |
| 98 |
switch { |
| 99 |
// We assume rx, ry are valid iff offx, offy are 0. |
| 100 |
case x.r != regalloc.VRegInvalid && y.r != regalloc.VRegInvalid: |
| 101 |
switch { |
| 102 |
case x.shift != 0 && y.shift != 0: |
| 103 |
// Cannot absorb two shifted registers, must lower one to a shift instruction. |
| 104 |
shifted := m.allocateInstr() |
| 105 |
shifted.asShiftR(shiftROpShiftLeft, newOperandImm32(uint32(x.shift)), x.r, true) |
| 106 |
m.insert(shifted) |
| 107 |
|
| 108 |
return m.newAmodeRegRegShift(u32, x.r, y.r, y.shift) |
| 109 |
case x.shift != 0 && y.shift == 0: |
| 110 |
// Swap base and index. |
| 111 |
x, y = y, x |
| 112 |
fallthrough |
| 113 |
default: |
| 114 |
return m.newAmodeRegRegShift(u32, x.r, y.r, y.shift) |
| 115 |
} |
| 116 |
case x.r == regalloc.VRegInvalid && y.r != regalloc.VRegInvalid: |
| 117 |
x, y = y, x |
| 118 |
fallthrough |
| 119 |
case x.r != regalloc.VRegInvalid && y.r == regalloc.VRegInvalid: |
| 120 |
if x.shift != 0 { |
| 121 |
zero := m.c.AllocateVReg(ssa.TypeI64) |
| 122 |
m.lowerIconst(zero, 0, true) |
| 123 |
return m.newAmodeRegRegShift(u32, zero, x.r, x.shift) |
| 124 |
} |
| 125 |
return m.newAmodeImmReg(u32, x.r) |
| 126 |
default: // Both are invalid: use the offset. |
| 127 |
tmpReg := m.c.AllocateVReg(ssa.TypeI64) |
| 128 |
m.lowerIconst(tmpReg, u64, true) |
| 129 |
return m.newAmodeImmReg(0, tmpReg) |
| 130 |
} |
| 131 |
} |
| 132 |
|
| 133 |
func (m *machine) lowerAddend(x backend.SSAValueDefinition) addend { |
| 134 |
if !x.IsFromInstr() { |
| 135 |
return addend{m.c.VRegOf(x.V), 0, 0} |
| 136 |
} |
| 137 |
// Ensure the addend is not referenced in multiple places; we will discard nested Iadds. |
| 138 |
op := m.c.MatchInstrOneOf(x, addendsMatchOpcodes[:]) |
| 139 |
if op != ssa.OpcodeInvalid && op != ssa.OpcodeIadd { |
| 140 |
return m.lowerAddendFromInstr(x.Instr) |
| 141 |
} |
| 142 |
p := m.getOperand_Reg(x) |
| 143 |
return addend{p.reg(), 0, 0} |
| 144 |
} |
| 145 |
|
| 146 |
// lowerAddendFromInstr takes an instruction returns a Vreg and an offset that can be used in an address mode. |
| 147 |
// The Vreg is regalloc.VRegInvalid if the addend cannot be lowered to a register. |
| 148 |
// The offset is 0 if the addend can be lowered to a register. |
| 149 |
func (m *machine) lowerAddendFromInstr(instr *ssa.Instruction) addend { |
| 150 |
instr.MarkLowered() |
| 151 |
switch op := instr.Opcode(); op { |
| 152 |
case ssa.OpcodeIconst: |
| 153 |
u64 := instr.ConstantVal() |
| 154 |
if instr.Return().Type().Bits() == 32 { |
| 155 |
return addend{regalloc.VRegInvalid, int64(int32(u64)), 0} // sign-extend. |
| 156 |
} else { |
| 157 |
return addend{regalloc.VRegInvalid, int64(u64), 0} |
| 158 |
} |
| 159 |
case ssa.OpcodeUExtend, ssa.OpcodeSExtend: |
| 160 |
input := instr.Arg() |
| 161 |
inputDef := m.c.ValueDefinition(input) |
| 162 |
if input.Type().Bits() != 32 { |
| 163 |
panic("BUG: invalid input type " + input.Type().String()) |
| 164 |
} |
| 165 |
constInst := inputDef.IsFromInstr() && inputDef.Instr.Constant() |
| 166 |
switch { |
| 167 |
case constInst && op == ssa.OpcodeSExtend: |
| 168 |
return addend{regalloc.VRegInvalid, int64(uint32(inputDef.Instr.ConstantVal())), 0} |
| 169 |
case constInst && op == ssa.OpcodeUExtend: |
| 170 |
return addend{regalloc.VRegInvalid, int64(int32(inputDef.Instr.ConstantVal())), 0} // sign-extend! |
| 171 |
default: |
| 172 |
r := m.getOperand_Reg(inputDef) |
| 173 |
return addend{r.reg(), 0, 0} |
| 174 |
} |
| 175 |
case ssa.OpcodeIshl: |
| 176 |
// If the addend is a shift, we can only handle it if the shift amount is a constant. |
| 177 |
x, amount := instr.Arg2() |
| 178 |
amountDef := m.c.ValueDefinition(amount) |
| 179 |
if amountDef.IsFromInstr() && amountDef.Instr.Constant() && amountDef.Instr.ConstantVal() <= 3 { |
| 180 |
r := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 181 |
return addend{r.reg(), 0, uint8(amountDef.Instr.ConstantVal())} |
| 182 |
} |
| 183 |
r := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 184 |
return addend{r.reg(), 0, 0} |
| 185 |
} |
| 186 |
panic("BUG: invalid opcode") |
| 187 |
} |