| 1 |
package arm64 |
| 2 |
|
| 3 |
// This file contains the logic to "find and determine operands" for instructions. |
| 4 |
// In order to finalize the form of an operand, we might end up merging/eliminating |
| 5 |
// the source instructions into an operand whenever possible. |
| 6 |
|
| 7 |
import ( |
| 8 |
"fmt" |
| 9 |
|
| 10 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 11 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 12 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 13 |
) |
| 14 |
|
| 15 |
type ( |
| 16 |
// operand represents an operand of an instruction whose type is determined by the kind. |
| 17 |
operand struct { |
| 18 |
kind operandKind |
| 19 |
data, data2 uint64 |
| 20 |
} |
| 21 |
operandKind byte |
| 22 |
) |
| 23 |
|
| 24 |
// Here's the list of operand kinds. We use the abbreviation of the kind name not only for these consts, |
| 25 |
// but also names of functions which return the operand of the kind. |
| 26 |
const ( |
| 27 |
// operandKindNR represents "NormalRegister" (NR). This is literally the register without any special operation unlike others. |
| 28 |
operandKindNR operandKind = iota |
| 29 |
// operandKindSR represents "Shifted Register" (SR). This is a register which is shifted by a constant. |
| 30 |
// Some of the arm64 instructions can take this kind of operand. |
| 31 |
operandKindSR |
| 32 |
// operandKindER represents "Extended Register (ER). This is a register which is sign/zero-extended to a larger size. |
| 33 |
// Some of the arm64 instructions can take this kind of operand. |
| 34 |
operandKindER |
| 35 |
// operandKindImm12 represents "Immediate 12" (Imm12). This is a 12-bit immediate value which can be either shifted by 12 or not. |
| 36 |
// See asImm12 function for detail. |
| 37 |
operandKindImm12 |
| 38 |
// operandKindShiftImm represents "Shifted Immediate" (ShiftImm) used by shift operations. |
| 39 |
operandKindShiftImm |
| 40 |
) |
| 41 |
|
| 42 |
// String implements fmt.Stringer for debugging. |
| 43 |
func (o operand) format(size byte) string { |
| 44 |
switch o.kind { |
| 45 |
case operandKindNR: |
| 46 |
return formatVRegSized(o.nr(), size) |
| 47 |
case operandKindSR: |
| 48 |
r, amt, sop := o.sr() |
| 49 |
return fmt.Sprintf("%s, %s #%d", formatVRegSized(r, size), sop, amt) |
| 50 |
case operandKindER: |
| 51 |
r, eop, _ := o.er() |
| 52 |
return fmt.Sprintf("%s %s", formatVRegSized(r, size), eop) |
| 53 |
case operandKindImm12: |
| 54 |
imm12, shiftBit := o.imm12() |
| 55 |
if shiftBit == 1 { |
| 56 |
return fmt.Sprintf("#%#x", uint64(imm12)<<12) |
| 57 |
} else { |
| 58 |
return fmt.Sprintf("#%#x", imm12) |
| 59 |
} |
| 60 |
default: |
| 61 |
panic(fmt.Sprintf("unknown operand kind: %d", o.kind)) |
| 62 |
} |
| 63 |
} |
| 64 |
|
| 65 |
// operandNR encodes the given VReg as an operand of operandKindNR. |
| 66 |
func operandNR(r regalloc.VReg) operand { |
| 67 |
return operand{kind: operandKindNR, data: uint64(r)} |
| 68 |
} |
| 69 |
|
| 70 |
// nr decodes the underlying VReg assuming the operand is of operandKindNR. |
| 71 |
func (o operand) nr() regalloc.VReg { |
| 72 |
return regalloc.VReg(o.data) |
| 73 |
} |
| 74 |
|
| 75 |
// operandER encodes the given VReg as an operand of operandKindER. |
| 76 |
func operandER(r regalloc.VReg, eop extendOp, to byte) operand { |
| 77 |
if to < 32 { |
| 78 |
panic("TODO?BUG?: when we need to extend to less than 32 bits?") |
| 79 |
} |
| 80 |
return operand{kind: operandKindER, data: uint64(r), data2: uint64(eop)<<32 | uint64(to)} |
| 81 |
} |
| 82 |
|
| 83 |
// er decodes the underlying VReg, extend operation, and the target size assuming the operand is of operandKindER. |
| 84 |
func (o operand) er() (r regalloc.VReg, eop extendOp, to byte) { |
| 85 |
return regalloc.VReg(o.data), extendOp(o.data2>>32) & 0xff, byte(o.data2 & 0xff) |
| 86 |
} |
| 87 |
|
| 88 |
// operandSR encodes the given VReg as an operand of operandKindSR. |
| 89 |
func operandSR(r regalloc.VReg, amt byte, sop shiftOp) operand { |
| 90 |
return operand{kind: operandKindSR, data: uint64(r), data2: uint64(amt)<<32 | uint64(sop)} |
| 91 |
} |
| 92 |
|
| 93 |
// sr decodes the underlying VReg, shift amount, and shift operation assuming the operand is of operandKindSR. |
| 94 |
func (o operand) sr() (r regalloc.VReg, amt byte, sop shiftOp) { |
| 95 |
return regalloc.VReg(o.data), byte(o.data2>>32) & 0xff, shiftOp(o.data2) & 0xff |
| 96 |
} |
| 97 |
|
| 98 |
// operandImm12 encodes the given imm12 as an operand of operandKindImm12. |
| 99 |
func operandImm12(imm12 uint16, shiftBit byte) operand { |
| 100 |
return operand{kind: operandKindImm12, data: uint64(imm12) | uint64(shiftBit)<<32} |
| 101 |
} |
| 102 |
|
| 103 |
// imm12 decodes the underlying imm12 data assuming the operand is of operandKindImm12. |
| 104 |
func (o operand) imm12() (v uint16, shiftBit byte) { |
| 105 |
return uint16(o.data), byte(o.data >> 32) |
| 106 |
} |
| 107 |
|
| 108 |
// operandShiftImm encodes the given amount as an operand of operandKindShiftImm. |
| 109 |
func operandShiftImm(amount byte) operand { |
| 110 |
return operand{kind: operandKindShiftImm, data: uint64(amount)} |
| 111 |
} |
| 112 |
|
| 113 |
// shiftImm decodes the underlying shift amount data assuming the operand is of operandKindShiftImm. |
| 114 |
func (o operand) shiftImm() byte { |
| 115 |
return byte(o.data) |
| 116 |
} |
| 117 |
|
| 118 |
// reg returns the register of the operand if applicable. |
| 119 |
func (o operand) reg() regalloc.VReg { |
| 120 |
switch o.kind { |
| 121 |
case operandKindNR: |
| 122 |
return o.nr() |
| 123 |
case operandKindSR: |
| 124 |
r, _, _ := o.sr() |
| 125 |
return r |
| 126 |
case operandKindER: |
| 127 |
r, _, _ := o.er() |
| 128 |
return r |
| 129 |
case operandKindImm12: |
| 130 |
// Does not have a register. |
| 131 |
case operandKindShiftImm: |
| 132 |
// Does not have a register. |
| 133 |
default: |
| 134 |
panic(o.kind) |
| 135 |
} |
| 136 |
return regalloc.VRegInvalid |
| 137 |
} |
| 138 |
|
| 139 |
func (o operand) realReg() regalloc.RealReg { |
| 140 |
return o.nr().RealReg() |
| 141 |
} |
| 142 |
|
| 143 |
func (o operand) assignReg(v regalloc.VReg) operand { |
| 144 |
switch o.kind { |
| 145 |
case operandKindNR: |
| 146 |
return operandNR(v) |
| 147 |
case operandKindSR: |
| 148 |
_, amt, sop := o.sr() |
| 149 |
return operandSR(v, amt, sop) |
| 150 |
case operandKindER: |
| 151 |
_, eop, to := o.er() |
| 152 |
return operandER(v, eop, to) |
| 153 |
case operandKindImm12: |
| 154 |
// Does not have a register. |
| 155 |
case operandKindShiftImm: |
| 156 |
// Does not have a register. |
| 157 |
} |
| 158 |
panic(o.kind) |
| 159 |
} |
| 160 |
|
| 161 |
// ensureValueNR returns an operand of either operandKindER, operandKindSR, or operandKindNR from the given value (defined by `def). |
| 162 |
// |
| 163 |
// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). |
| 164 |
// If the operand can be expressed as operandKindImm12, `mode` is ignored. |
| 165 |
func (m *machine) getOperand_Imm12_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { |
| 166 |
if !def.IsFromInstr() { |
| 167 |
return operandNR(m.compiler.VRegOf(def.V)) |
| 168 |
} |
| 169 |
|
| 170 |
instr := def.Instr |
| 171 |
if instr.Opcode() == ssa.OpcodeIconst { |
| 172 |
if imm12Op, ok := asImm12Operand(instr.ConstantVal()); ok { |
| 173 |
instr.MarkLowered() |
| 174 |
return imm12Op |
| 175 |
} |
| 176 |
} |
| 177 |
return m.getOperand_ER_SR_NR(def, mode) |
| 178 |
} |
| 179 |
|
| 180 |
// getOperand_MaybeNegatedImm12_ER_SR_NR is almost the same as getOperand_Imm12_ER_SR_NR, but this might negate the immediate value. |
| 181 |
// If the immediate value is negated, the second return value is true, otherwise always false. |
| 182 |
func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand, negatedImm12 bool) { |
| 183 |
if !def.IsFromInstr() { |
| 184 |
return operandNR(m.compiler.VRegOf(def.V)), false |
| 185 |
} |
| 186 |
|
| 187 |
instr := def.Instr |
| 188 |
if instr.Opcode() == ssa.OpcodeIconst { |
| 189 |
c := instr.ConstantVal() |
| 190 |
if imm12Op, ok := asImm12Operand(c); ok { |
| 191 |
instr.MarkLowered() |
| 192 |
return imm12Op, false |
| 193 |
} |
| 194 |
|
| 195 |
signExtended := int64(c) |
| 196 |
if def.V.Type().Bits() == 32 { |
| 197 |
signExtended = (signExtended << 32) >> 32 |
| 198 |
} |
| 199 |
negatedWithoutSign := -signExtended |
| 200 |
if imm12Op, ok := asImm12Operand(uint64(negatedWithoutSign)); ok { |
| 201 |
instr.MarkLowered() |
| 202 |
return imm12Op, true |
| 203 |
} |
| 204 |
} |
| 205 |
return m.getOperand_ER_SR_NR(def, mode), false |
| 206 |
} |
| 207 |
|
| 208 |
// ensureValueNR returns an operand of either operandKindER, operandKindSR, or operandKindNR from the given value (defined by `def). |
| 209 |
// |
| 210 |
// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). |
| 211 |
func (m *machine) getOperand_ER_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { |
| 212 |
if !def.IsFromInstr() { |
| 213 |
return operandNR(m.compiler.VRegOf(def.V)) |
| 214 |
} |
| 215 |
|
| 216 |
if m.compiler.MatchInstr(def, ssa.OpcodeSExtend) || m.compiler.MatchInstr(def, ssa.OpcodeUExtend) { |
| 217 |
extInstr := def.Instr |
| 218 |
|
| 219 |
signed := extInstr.Opcode() == ssa.OpcodeSExtend |
| 220 |
innerExtFromBits, innerExtToBits := extInstr.ExtendFromToBits() |
| 221 |
modeBits, modeSigned := mode.bits(), mode.signed() |
| 222 |
if mode == extModeNone || innerExtToBits == modeBits { |
| 223 |
eop := extendOpFrom(signed, innerExtFromBits) |
| 224 |
extArg := m.getOperand_NR(m.compiler.ValueDefinition(extInstr.Arg()), extModeNone) |
| 225 |
op = operandER(extArg.nr(), eop, innerExtToBits) |
| 226 |
extInstr.MarkLowered() |
| 227 |
return |
| 228 |
} |
| 229 |
|
| 230 |
if innerExtToBits > modeBits { |
| 231 |
panic("BUG?TODO?: need the results of inner extension to be larger than the mode") |
| 232 |
} |
| 233 |
|
| 234 |
switch { |
| 235 |
case (!signed && !modeSigned) || (signed && modeSigned): |
| 236 |
// Two sign/zero extensions are equivalent to one sign/zero extension for the larger size. |
| 237 |
eop := extendOpFrom(modeSigned, innerExtFromBits) |
| 238 |
op = operandER(m.compiler.VRegOf(extInstr.Arg()), eop, modeBits) |
| 239 |
extInstr.MarkLowered() |
| 240 |
case (signed && !modeSigned) || (!signed && modeSigned): |
| 241 |
// We need to {sign, zero}-extend the result of the {zero,sign} extension. |
| 242 |
eop := extendOpFrom(modeSigned, innerExtToBits) |
| 243 |
op = operandER(m.compiler.VRegOf(extInstr.Return()), eop, modeBits) |
| 244 |
// Note that we failed to merge the inner extension instruction this case. |
| 245 |
} |
| 246 |
return |
| 247 |
} |
| 248 |
return m.getOperand_SR_NR(def, mode) |
| 249 |
} |
| 250 |
|
| 251 |
// ensureValueNR returns an operand of either operandKindSR or operandKindNR from the given value (defined by `def). |
| 252 |
// |
| 253 |
// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). |
| 254 |
func (m *machine) getOperand_SR_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { |
| 255 |
if !def.IsFromInstr() { |
| 256 |
return operandNR(m.compiler.VRegOf(def.V)) |
| 257 |
} |
| 258 |
|
| 259 |
if m.compiler.MatchInstr(def, ssa.OpcodeIshl) { |
| 260 |
// Check if the shift amount is constant instruction. |
| 261 |
targetVal, amountVal := def.Instr.Arg2() |
| 262 |
targetVReg := m.getOperand_NR(m.compiler.ValueDefinition(targetVal), extModeNone).nr() |
| 263 |
amountDef := m.compiler.ValueDefinition(amountVal) |
| 264 |
if amountDef.IsFromInstr() && amountDef.Instr.Constant() { |
| 265 |
// If that is the case, we can use the shifted register operand (SR). |
| 266 |
c := byte(amountDef.Instr.ConstantVal()) & (targetVal.Type().Bits() - 1) // Clears the unnecessary bits. |
| 267 |
def.Instr.MarkLowered() |
| 268 |
amountDef.Instr.MarkLowered() |
| 269 |
return operandSR(targetVReg, c, shiftOpLSL) |
| 270 |
} |
| 271 |
} |
| 272 |
return m.getOperand_NR(def, mode) |
| 273 |
} |
| 274 |
|
| 275 |
// getOperand_ShiftImm_NR returns an operand of either operandKindShiftImm or operandKindNR from the given value (defined by `def). |
| 276 |
func (m *machine) getOperand_ShiftImm_NR(def backend.SSAValueDefinition, mode extMode, shiftBitWidth byte) (op operand) { |
| 277 |
if !def.IsFromInstr() { |
| 278 |
return operandNR(m.compiler.VRegOf(def.V)) |
| 279 |
} |
| 280 |
|
| 281 |
instr := def.Instr |
| 282 |
if instr.Constant() { |
| 283 |
amount := byte(instr.ConstantVal()) & (shiftBitWidth - 1) // Clears the unnecessary bits. |
| 284 |
return operandShiftImm(amount) |
| 285 |
} |
| 286 |
return m.getOperand_NR(def, mode) |
| 287 |
} |
| 288 |
|
| 289 |
// ensureValueNR returns an operand of operandKindNR from the given value (defined by `def). |
| 290 |
// |
| 291 |
// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). |
| 292 |
func (m *machine) getOperand_NR(def backend.SSAValueDefinition, mode extMode) (op operand) { |
| 293 |
var v regalloc.VReg |
| 294 |
if def.IsFromInstr() && def.Instr.Constant() { |
| 295 |
// We inline all the constant instructions so that we could reduce the register usage. |
| 296 |
v = m.lowerConstant(def.Instr) |
| 297 |
def.Instr.MarkLowered() |
| 298 |
} else { |
| 299 |
v = m.compiler.VRegOf(def.V) |
| 300 |
} |
| 301 |
|
| 302 |
r := v |
| 303 |
switch inBits := def.V.Type().Bits(); { |
| 304 |
case mode == extModeNone: |
| 305 |
case inBits == 32 && (mode == extModeZeroExtend32 || mode == extModeSignExtend32): |
| 306 |
case inBits == 32 && mode == extModeZeroExtend64: |
| 307 |
extended := m.compiler.AllocateVReg(ssa.TypeI64) |
| 308 |
ext := m.allocateInstr() |
| 309 |
ext.asExtend(extended, v, 32, 64, false) |
| 310 |
m.insert(ext) |
| 311 |
r = extended |
| 312 |
case inBits == 32 && mode == extModeSignExtend64: |
| 313 |
extended := m.compiler.AllocateVReg(ssa.TypeI64) |
| 314 |
ext := m.allocateInstr() |
| 315 |
ext.asExtend(extended, v, 32, 64, true) |
| 316 |
m.insert(ext) |
| 317 |
r = extended |
| 318 |
case inBits == 64 && (mode == extModeZeroExtend64 || mode == extModeSignExtend64): |
| 319 |
} |
| 320 |
return operandNR(r) |
| 321 |
} |
| 322 |
|
| 323 |
func asImm12Operand(val uint64) (op operand, ok bool) { |
| 324 |
v, shiftBit, ok := asImm12(val) |
| 325 |
if !ok { |
| 326 |
return operand{}, false |
| 327 |
} |
| 328 |
return operandImm12(v, shiftBit), true |
| 329 |
} |
| 330 |
|
| 331 |
func asImm12(val uint64) (v uint16, shiftBit byte, ok bool) { |
| 332 |
const mask1, mask2 uint64 = 0xfff, 0xfff_000 |
| 333 |
if val&^mask1 == 0 { |
| 334 |
return uint16(val), 0, true |
| 335 |
} else if val&^mask2 == 0 { |
| 336 |
return uint16(val >> 12), 1, true |
| 337 |
} else { |
| 338 |
return 0, 0, false |
| 339 |
} |
| 340 |
} |