| 1 |
package arm64 |
| 2 |
|
| 3 |
// Files prefixed as lower_instr** do the instruction selection, meaning that lowering SSA level instructions |
| 4 |
// into machine specific instructions. |
| 5 |
// |
| 6 |
// Importantly, what the lower** functions does includes tree-matching; find the pattern from the given instruction tree, |
| 7 |
// and merge the multiple instructions if possible. It can be considered as "N:1" instruction selection. |
| 8 |
|
| 9 |
import ( |
| 10 |
"fmt" |
| 11 |
"math" |
| 12 |
|
| 13 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 14 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 15 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" |
| 16 |
) |
| 17 |
|
| 18 |
// LowerSingleBranch implements backend.Machine. |
| 19 |
func (m *machine) LowerSingleBranch(br *ssa.Instruction) { |
| 20 |
switch br.Opcode() { |
| 21 |
case ssa.OpcodeJump: |
| 22 |
_, _, targetBlkID := br.BranchData() |
| 23 |
if br.IsFallthroughJump() { |
| 24 |
return |
| 25 |
} |
| 26 |
b := m.allocateInstr() |
| 27 |
targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID) |
| 28 |
if targetBlk.ReturnBlock() { |
| 29 |
b.asRet() |
| 30 |
} else { |
| 31 |
b.asBr(ssaBlockLabel(targetBlk)) |
| 32 |
} |
| 33 |
m.insert(b) |
| 34 |
case ssa.OpcodeBrTable: |
| 35 |
m.lowerBrTable(br) |
| 36 |
default: |
| 37 |
panic("BUG: unexpected branch opcode" + br.Opcode().String()) |
| 38 |
} |
| 39 |
} |
| 40 |
|
| 41 |
func (m *machine) lowerBrTable(i *ssa.Instruction) { |
| 42 |
index, targetBlockIDs := i.BrTableData() |
| 43 |
targetBlockCount := len(targetBlockIDs.View()) |
| 44 |
indexOperand := m.getOperand_NR(m.compiler.ValueDefinition(index), extModeNone) |
| 45 |
|
| 46 |
// Firstly, we have to do the bounds check of the index, and |
| 47 |
// set it to the default target (sitting at the end of the list) if it's out of bounds. |
| 48 |
|
| 49 |
// mov maxIndexReg #maximum_index |
| 50 |
// subs wzr, index, maxIndexReg |
| 51 |
// csel adjustedIndex, maxIndexReg, index, hs ;; if index is higher or equal than maxIndexReg. |
| 52 |
maxIndexReg := m.compiler.AllocateVReg(ssa.TypeI32) |
| 53 |
m.lowerConstantI32(maxIndexReg, int32(targetBlockCount-1)) |
| 54 |
subs := m.allocateInstr() |
| 55 |
subs.asALU(aluOpSubS, xzrVReg, indexOperand, operandNR(maxIndexReg), false) |
| 56 |
m.insert(subs) |
| 57 |
csel := m.allocateInstr() |
| 58 |
adjustedIndex := m.compiler.AllocateVReg(ssa.TypeI32) |
| 59 |
csel.asCSel(adjustedIndex, operandNR(maxIndexReg), indexOperand, hs, false) |
| 60 |
m.insert(csel) |
| 61 |
|
| 62 |
brSequence := m.allocateInstr() |
| 63 |
|
| 64 |
tableIndex := m.addJmpTableTarget(targetBlockIDs) |
| 65 |
brSequence.asBrTableSequence(adjustedIndex, tableIndex, targetBlockCount) |
| 66 |
m.insert(brSequence) |
| 67 |
} |
| 68 |
|
| 69 |
// LowerConditionalBranch implements backend.Machine. |
| 70 |
func (m *machine) LowerConditionalBranch(b *ssa.Instruction) { |
| 71 |
cval, args, targetBlkID := b.BranchData() |
| 72 |
if len(args) > 0 { |
| 73 |
panic(fmt.Sprintf( |
| 74 |
"conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s", |
| 75 |
m.currentLabelPos.sb, |
| 76 |
targetBlkID, |
| 77 |
)) |
| 78 |
} |
| 79 |
|
| 80 |
targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID) |
| 81 |
target := ssaBlockLabel(targetBlk) |
| 82 |
cvalDef := m.compiler.ValueDefinition(cval) |
| 83 |
|
| 84 |
switch { |
| 85 |
case m.compiler.MatchInstr(cvalDef, ssa.OpcodeIcmp): // This case, we can use the ALU flag set by SUBS instruction. |
| 86 |
cvalInstr := cvalDef.Instr |
| 87 |
x, y, c := cvalInstr.IcmpData() |
| 88 |
cc, signed := condFlagFromSSAIntegerCmpCond(c), c.Signed() |
| 89 |
if b.Opcode() == ssa.OpcodeBrz { |
| 90 |
cc = cc.invert() |
| 91 |
} |
| 92 |
|
| 93 |
if !m.tryLowerBandToFlag(x, y) { |
| 94 |
m.lowerIcmpToFlag(x, y, signed) |
| 95 |
} |
| 96 |
cbr := m.allocateInstr() |
| 97 |
cbr.asCondBr(cc.asCond(), target, false /* ignored */) |
| 98 |
m.insert(cbr) |
| 99 |
cvalDef.Instr.MarkLowered() |
| 100 |
case m.compiler.MatchInstr(cvalDef, ssa.OpcodeFcmp): // This case we can use the Fpu flag directly. |
| 101 |
cvalInstr := cvalDef.Instr |
| 102 |
x, y, c := cvalInstr.FcmpData() |
| 103 |
cc := condFlagFromSSAFloatCmpCond(c) |
| 104 |
if b.Opcode() == ssa.OpcodeBrz { |
| 105 |
cc = cc.invert() |
| 106 |
} |
| 107 |
m.lowerFcmpToFlag(x, y) |
| 108 |
cbr := m.allocateInstr() |
| 109 |
cbr.asCondBr(cc.asCond(), target, false /* ignored */) |
| 110 |
m.insert(cbr) |
| 111 |
cvalDef.Instr.MarkLowered() |
| 112 |
default: |
| 113 |
rn := m.getOperand_NR(cvalDef, extModeNone) |
| 114 |
var c cond |
| 115 |
if b.Opcode() == ssa.OpcodeBrz { |
| 116 |
c = registerAsRegZeroCond(rn.nr()) |
| 117 |
} else { |
| 118 |
c = registerAsRegNotZeroCond(rn.nr()) |
| 119 |
} |
| 120 |
cbr := m.allocateInstr() |
| 121 |
cbr.asCondBr(c, target, false) |
| 122 |
m.insert(cbr) |
| 123 |
} |
| 124 |
} |
| 125 |
|
| 126 |
func (m *machine) tryLowerBandToFlag(x, y ssa.Value) (ok bool) { |
| 127 |
xx := m.compiler.ValueDefinition(x) |
| 128 |
yy := m.compiler.ValueDefinition(y) |
| 129 |
if xx.IsFromInstr() && xx.Instr.Constant() && xx.Instr.ConstantVal() == 0 { |
| 130 |
if m.compiler.MatchInstr(yy, ssa.OpcodeBand) { |
| 131 |
bandInstr := yy.Instr |
| 132 |
m.lowerBitwiseAluOp(bandInstr, aluOpAnds, true) |
| 133 |
ok = true |
| 134 |
bandInstr.MarkLowered() |
| 135 |
return |
| 136 |
} |
| 137 |
} |
| 138 |
|
| 139 |
if yy.IsFromInstr() && yy.Instr.Constant() && yy.Instr.ConstantVal() == 0 { |
| 140 |
if m.compiler.MatchInstr(xx, ssa.OpcodeBand) { |
| 141 |
bandInstr := xx.Instr |
| 142 |
m.lowerBitwiseAluOp(bandInstr, aluOpAnds, true) |
| 143 |
ok = true |
| 144 |
bandInstr.MarkLowered() |
| 145 |
return |
| 146 |
} |
| 147 |
} |
| 148 |
return |
| 149 |
} |
| 150 |
|
| 151 |
// LowerInstr implements backend.Machine. |
| 152 |
func (m *machine) LowerInstr(instr *ssa.Instruction) { |
| 153 |
if l := instr.SourceOffset(); l.Valid() { |
| 154 |
info := m.allocateInstr().asEmitSourceOffsetInfo(l) |
| 155 |
m.insert(info) |
| 156 |
} |
| 157 |
|
| 158 |
switch op := instr.Opcode(); op { |
| 159 |
case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable: |
| 160 |
panic("BUG: branching instructions are handled by LowerBranches") |
| 161 |
case ssa.OpcodeReturn: |
| 162 |
panic("BUG: return must be handled by backend.Compiler") |
| 163 |
case ssa.OpcodeIadd, ssa.OpcodeIsub: |
| 164 |
m.lowerSubOrAdd(instr, op == ssa.OpcodeIadd) |
| 165 |
case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv, ssa.OpcodeFmax, ssa.OpcodeFmin: |
| 166 |
m.lowerFpuBinOp(instr) |
| 167 |
case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined. |
| 168 |
case ssa.OpcodeExitWithCode: |
| 169 |
execCtx, code := instr.ExitWithCodeData() |
| 170 |
m.lowerExitWithCode(m.compiler.VRegOf(execCtx), code) |
| 171 |
case ssa.OpcodeExitIfTrueWithCode: |
| 172 |
execCtx, c, code := instr.ExitIfTrueWithCodeData() |
| 173 |
m.lowerExitIfTrueWithCode(m.compiler.VRegOf(execCtx), c, code) |
| 174 |
case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32: |
| 175 |
m.lowerStore(instr) |
| 176 |
case ssa.OpcodeLoad: |
| 177 |
dst := instr.Return() |
| 178 |
ptr, offset, typ := instr.LoadData() |
| 179 |
m.lowerLoad(ptr, offset, typ, dst) |
| 180 |
case ssa.OpcodeVZeroExtLoad: |
| 181 |
dst := instr.Return() |
| 182 |
ptr, offset, typ := instr.VZeroExtLoadData() |
| 183 |
m.lowerLoad(ptr, offset, typ, dst) |
| 184 |
case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32: |
| 185 |
ptr, offset, _ := instr.LoadData() |
| 186 |
ret := m.compiler.VRegOf(instr.Return()) |
| 187 |
m.lowerExtLoad(op, ptr, offset, ret) |
| 188 |
case ssa.OpcodeCall, ssa.OpcodeCallIndirect: |
| 189 |
m.lowerCall(instr) |
| 190 |
case ssa.OpcodeIcmp: |
| 191 |
m.lowerIcmp(instr) |
| 192 |
case ssa.OpcodeVIcmp: |
| 193 |
m.lowerVIcmp(instr) |
| 194 |
case ssa.OpcodeVFcmp: |
| 195 |
m.lowerVFcmp(instr) |
| 196 |
case ssa.OpcodeVCeil: |
| 197 |
m.lowerVecMisc(vecOpFrintp, instr) |
| 198 |
case ssa.OpcodeVFloor: |
| 199 |
m.lowerVecMisc(vecOpFrintm, instr) |
| 200 |
case ssa.OpcodeVTrunc: |
| 201 |
m.lowerVecMisc(vecOpFrintz, instr) |
| 202 |
case ssa.OpcodeVNearest: |
| 203 |
m.lowerVecMisc(vecOpFrintn, instr) |
| 204 |
case ssa.OpcodeVMaxPseudo: |
| 205 |
m.lowerVMinMaxPseudo(instr, true) |
| 206 |
case ssa.OpcodeVMinPseudo: |
| 207 |
m.lowerVMinMaxPseudo(instr, false) |
| 208 |
case ssa.OpcodeBand: |
| 209 |
m.lowerBitwiseAluOp(instr, aluOpAnd, false) |
| 210 |
case ssa.OpcodeBor: |
| 211 |
m.lowerBitwiseAluOp(instr, aluOpOrr, false) |
| 212 |
case ssa.OpcodeBxor: |
| 213 |
m.lowerBitwiseAluOp(instr, aluOpEor, false) |
| 214 |
case ssa.OpcodeIshl: |
| 215 |
m.lowerShifts(instr, extModeNone, aluOpLsl) |
| 216 |
case ssa.OpcodeSshr: |
| 217 |
if instr.Return().Type().Bits() == 64 { |
| 218 |
m.lowerShifts(instr, extModeSignExtend64, aluOpAsr) |
| 219 |
} else { |
| 220 |
m.lowerShifts(instr, extModeSignExtend32, aluOpAsr) |
| 221 |
} |
| 222 |
case ssa.OpcodeUshr: |
| 223 |
if instr.Return().Type().Bits() == 64 { |
| 224 |
m.lowerShifts(instr, extModeZeroExtend64, aluOpLsr) |
| 225 |
} else { |
| 226 |
m.lowerShifts(instr, extModeZeroExtend32, aluOpLsr) |
| 227 |
} |
| 228 |
case ssa.OpcodeRotl: |
| 229 |
m.lowerRotl(instr) |
| 230 |
case ssa.OpcodeRotr: |
| 231 |
m.lowerRotr(instr) |
| 232 |
case ssa.OpcodeSExtend, ssa.OpcodeUExtend: |
| 233 |
from, to, signed := instr.ExtendData() |
| 234 |
m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed) |
| 235 |
case ssa.OpcodeFcmp: |
| 236 |
x, y, c := instr.FcmpData() |
| 237 |
m.lowerFcmp(x, y, instr.Return(), c) |
| 238 |
case ssa.OpcodeImul: |
| 239 |
x, y := instr.Arg2() |
| 240 |
result := instr.Return() |
| 241 |
m.lowerImul(x, y, result) |
| 242 |
case ssa.OpcodeUndefined: |
| 243 |
undef := m.allocateInstr() |
| 244 |
undef.asUDF() |
| 245 |
m.insert(undef) |
| 246 |
case ssa.OpcodeSelect: |
| 247 |
c, x, y := instr.SelectData() |
| 248 |
if x.Type() == ssa.TypeV128 { |
| 249 |
rc := m.getOperand_NR(m.compiler.ValueDefinition(c), extModeNone) |
| 250 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 251 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 252 |
rd := m.compiler.VRegOf(instr.Return()) |
| 253 |
m.lowerSelectVec(rc, rn, rm, rd) |
| 254 |
} else { |
| 255 |
m.lowerSelect(c, x, y, instr.Return()) |
| 256 |
} |
| 257 |
case ssa.OpcodeClz: |
| 258 |
x := instr.Arg() |
| 259 |
result := instr.Return() |
| 260 |
m.lowerClz(x, result) |
| 261 |
case ssa.OpcodeCtz: |
| 262 |
x := instr.Arg() |
| 263 |
result := instr.Return() |
| 264 |
m.lowerCtz(x, result) |
| 265 |
case ssa.OpcodePopcnt: |
| 266 |
x := instr.Arg() |
| 267 |
result := instr.Return() |
| 268 |
m.lowerPopcnt(x, result) |
| 269 |
case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat: |
| 270 |
x, ctx := instr.Arg2() |
| 271 |
result := instr.Return() |
| 272 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 273 |
rd := m.compiler.VRegOf(result) |
| 274 |
ctxVReg := m.compiler.VRegOf(ctx) |
| 275 |
m.lowerFpuToInt(rd, rn, ctxVReg, true, x.Type() == ssa.TypeF64, |
| 276 |
result.Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat) |
| 277 |
case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat: |
| 278 |
x, ctx := instr.Arg2() |
| 279 |
result := instr.Return() |
| 280 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 281 |
rd := m.compiler.VRegOf(result) |
| 282 |
ctxVReg := m.compiler.VRegOf(ctx) |
| 283 |
m.lowerFpuToInt(rd, rn, ctxVReg, false, x.Type() == ssa.TypeF64, |
| 284 |
result.Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat) |
| 285 |
case ssa.OpcodeFcvtFromSint: |
| 286 |
x := instr.Arg() |
| 287 |
result := instr.Return() |
| 288 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 289 |
rd := m.compiler.VRegOf(result) |
| 290 |
m.lowerIntToFpu(rd, rn, true, x.Type() == ssa.TypeI64, result.Type().Bits() == 64) |
| 291 |
case ssa.OpcodeFcvtFromUint: |
| 292 |
x := instr.Arg() |
| 293 |
result := instr.Return() |
| 294 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 295 |
rd := m.compiler.VRegOf(result) |
| 296 |
m.lowerIntToFpu(rd, rn, false, x.Type() == ssa.TypeI64, result.Type().Bits() == 64) |
| 297 |
case ssa.OpcodeFdemote: |
| 298 |
v := instr.Arg() |
| 299 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) |
| 300 |
rd := m.compiler.VRegOf(instr.Return()) |
| 301 |
cnt := m.allocateInstr() |
| 302 |
cnt.asFpuRR(fpuUniOpCvt64To32, rd, rn, false) |
| 303 |
m.insert(cnt) |
| 304 |
case ssa.OpcodeFpromote: |
| 305 |
v := instr.Arg() |
| 306 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) |
| 307 |
rd := m.compiler.VRegOf(instr.Return()) |
| 308 |
cnt := m.allocateInstr() |
| 309 |
cnt.asFpuRR(fpuUniOpCvt32To64, rd, rn, true) |
| 310 |
m.insert(cnt) |
| 311 |
case ssa.OpcodeIreduce: |
| 312 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(instr.Arg()), extModeNone) |
| 313 |
retVal := instr.Return() |
| 314 |
rd := m.compiler.VRegOf(retVal) |
| 315 |
|
| 316 |
if retVal.Type() != ssa.TypeI32 { |
| 317 |
panic("TODO?: Ireduce to non-i32") |
| 318 |
} |
| 319 |
mov := m.allocateInstr() |
| 320 |
mov.asMove32(rd, rn.reg()) |
| 321 |
m.insert(mov) |
| 322 |
case ssa.OpcodeFneg: |
| 323 |
m.lowerFpuUniOp(fpuUniOpNeg, instr.Arg(), instr.Return()) |
| 324 |
case ssa.OpcodeSqrt: |
| 325 |
m.lowerFpuUniOp(fpuUniOpSqrt, instr.Arg(), instr.Return()) |
| 326 |
case ssa.OpcodeCeil: |
| 327 |
m.lowerFpuUniOp(fpuUniOpRoundPlus, instr.Arg(), instr.Return()) |
| 328 |
case ssa.OpcodeFloor: |
| 329 |
m.lowerFpuUniOp(fpuUniOpRoundMinus, instr.Arg(), instr.Return()) |
| 330 |
case ssa.OpcodeTrunc: |
| 331 |
m.lowerFpuUniOp(fpuUniOpRoundZero, instr.Arg(), instr.Return()) |
| 332 |
case ssa.OpcodeNearest: |
| 333 |
m.lowerFpuUniOp(fpuUniOpRoundNearest, instr.Arg(), instr.Return()) |
| 334 |
case ssa.OpcodeFabs: |
| 335 |
m.lowerFpuUniOp(fpuUniOpAbs, instr.Arg(), instr.Return()) |
| 336 |
case ssa.OpcodeBitcast: |
| 337 |
m.lowerBitcast(instr) |
| 338 |
case ssa.OpcodeFcopysign: |
| 339 |
x, y := instr.Arg2() |
| 340 |
m.lowerFcopysign(x, y, instr.Return()) |
| 341 |
case ssa.OpcodeSdiv, ssa.OpcodeUdiv: |
| 342 |
x, y, ctx := instr.Arg3() |
| 343 |
ctxVReg := m.compiler.VRegOf(ctx) |
| 344 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 345 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 346 |
rd := m.compiler.VRegOf(instr.Return()) |
| 347 |
m.lowerIDiv(ctxVReg, rd, rn, rm, x.Type() == ssa.TypeI64, op == ssa.OpcodeSdiv) |
| 348 |
case ssa.OpcodeSrem, ssa.OpcodeUrem: |
| 349 |
x, y, ctx := instr.Arg3() |
| 350 |
ctxVReg := m.compiler.VRegOf(ctx) |
| 351 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 352 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 353 |
rd := m.compiler.VRegOf(instr.Return()) |
| 354 |
m.lowerIRem(ctxVReg, rd, rn.nr(), rm, x.Type() == ssa.TypeI64, op == ssa.OpcodeSrem) |
| 355 |
case ssa.OpcodeVconst: |
| 356 |
result := m.compiler.VRegOf(instr.Return()) |
| 357 |
lo, hi := instr.VconstData() |
| 358 |
v := m.allocateInstr() |
| 359 |
v.asLoadFpuConst128(result, lo, hi) |
| 360 |
m.insert(v) |
| 361 |
case ssa.OpcodeVbnot: |
| 362 |
x := instr.Arg() |
| 363 |
ins := m.allocateInstr() |
| 364 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 365 |
rd := m.compiler.VRegOf(instr.Return()) |
| 366 |
ins.asVecMisc(vecOpNot, rd, rn, vecArrangement16B) |
| 367 |
m.insert(ins) |
| 368 |
case ssa.OpcodeVbxor: |
| 369 |
x, y := instr.Arg2() |
| 370 |
m.lowerVecRRR(vecOpEOR, x, y, instr.Return(), vecArrangement16B) |
| 371 |
case ssa.OpcodeVbor: |
| 372 |
x, y := instr.Arg2() |
| 373 |
m.lowerVecRRR(vecOpOrr, x, y, instr.Return(), vecArrangement16B) |
| 374 |
case ssa.OpcodeVband: |
| 375 |
x, y := instr.Arg2() |
| 376 |
m.lowerVecRRR(vecOpAnd, x, y, instr.Return(), vecArrangement16B) |
| 377 |
case ssa.OpcodeVbandnot: |
| 378 |
x, y := instr.Arg2() |
| 379 |
m.lowerVecRRR(vecOpBic, x, y, instr.Return(), vecArrangement16B) |
| 380 |
case ssa.OpcodeVbitselect: |
| 381 |
c, x, y := instr.SelectData() |
| 382 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 383 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 384 |
creg := m.getOperand_NR(m.compiler.ValueDefinition(c), extModeNone) |
| 385 |
tmp := m.compiler.AllocateVReg(ssa.TypeV128) |
| 386 |
|
| 387 |
// creg is overwritten by BSL, so we need to move it to the result register before the instruction |
| 388 |
// in case when it is used somewhere else. |
| 389 |
mov := m.allocateInstr() |
| 390 |
mov.asFpuMov128(tmp, creg.nr()) |
| 391 |
m.insert(mov) |
| 392 |
|
| 393 |
ins := m.allocateInstr() |
| 394 |
ins.asVecRRRRewrite(vecOpBsl, tmp, rn, rm, vecArrangement16B) |
| 395 |
m.insert(ins) |
| 396 |
|
| 397 |
mov2 := m.allocateInstr() |
| 398 |
rd := m.compiler.VRegOf(instr.Return()) |
| 399 |
mov2.asFpuMov128(rd, tmp) |
| 400 |
m.insert(mov2) |
| 401 |
case ssa.OpcodeVanyTrue, ssa.OpcodeVallTrue: |
| 402 |
x, lane := instr.ArgWithLane() |
| 403 |
var arr vecArrangement |
| 404 |
if op == ssa.OpcodeVallTrue { |
| 405 |
arr = ssaLaneToArrangement(lane) |
| 406 |
} |
| 407 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 408 |
rd := m.compiler.VRegOf(instr.Return()) |
| 409 |
m.lowerVcheckTrue(op, rm, rd, arr) |
| 410 |
case ssa.OpcodeVhighBits: |
| 411 |
x, lane := instr.ArgWithLane() |
| 412 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 413 |
rd := m.compiler.VRegOf(instr.Return()) |
| 414 |
arr := ssaLaneToArrangement(lane) |
| 415 |
m.lowerVhighBits(rm, rd, arr) |
| 416 |
case ssa.OpcodeVIadd: |
| 417 |
x, y, lane := instr.Arg2WithLane() |
| 418 |
arr := ssaLaneToArrangement(lane) |
| 419 |
m.lowerVecRRR(vecOpAdd, x, y, instr.Return(), arr) |
| 420 |
case ssa.OpcodeExtIaddPairwise: |
| 421 |
v, lane, signed := instr.ExtIaddPairwiseData() |
| 422 |
vv := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) |
| 423 |
|
| 424 |
tmpLo, tmpHi := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 425 |
var widen vecOp |
| 426 |
if signed { |
| 427 |
widen = vecOpSshll |
| 428 |
} else { |
| 429 |
widen = vecOpUshll |
| 430 |
} |
| 431 |
|
| 432 |
var loArr, hiArr, dstArr vecArrangement |
| 433 |
switch lane { |
| 434 |
case ssa.VecLaneI8x16: |
| 435 |
loArr, hiArr, dstArr = vecArrangement8B, vecArrangement16B, vecArrangement8H |
| 436 |
case ssa.VecLaneI16x8: |
| 437 |
loArr, hiArr, dstArr = vecArrangement4H, vecArrangement8H, vecArrangement4S |
| 438 |
case ssa.VecLaneI32x4: |
| 439 |
loArr, hiArr, dstArr = vecArrangement2S, vecArrangement4S, vecArrangement2D |
| 440 |
default: |
| 441 |
panic("unsupported lane " + lane.String()) |
| 442 |
} |
| 443 |
|
| 444 |
widenLo := m.allocateInstr().asVecShiftImm(widen, tmpLo.nr(), vv, operandShiftImm(0), loArr) |
| 445 |
widenHi := m.allocateInstr().asVecShiftImm(widen, tmpHi.nr(), vv, operandShiftImm(0), hiArr) |
| 446 |
addp := m.allocateInstr().asVecRRR(vecOpAddp, m.compiler.VRegOf(instr.Return()), tmpLo, tmpHi, dstArr) |
| 447 |
m.insert(widenLo) |
| 448 |
m.insert(widenHi) |
| 449 |
m.insert(addp) |
| 450 |
|
| 451 |
case ssa.OpcodeVSaddSat: |
| 452 |
x, y, lane := instr.Arg2WithLane() |
| 453 |
arr := ssaLaneToArrangement(lane) |
| 454 |
m.lowerVecRRR(vecOpSqadd, x, y, instr.Return(), arr) |
| 455 |
case ssa.OpcodeVUaddSat: |
| 456 |
x, y, lane := instr.Arg2WithLane() |
| 457 |
arr := ssaLaneToArrangement(lane) |
| 458 |
m.lowerVecRRR(vecOpUqadd, x, y, instr.Return(), arr) |
| 459 |
case ssa.OpcodeVIsub: |
| 460 |
x, y, lane := instr.Arg2WithLane() |
| 461 |
arr := ssaLaneToArrangement(lane) |
| 462 |
m.lowerVecRRR(vecOpSub, x, y, instr.Return(), arr) |
| 463 |
case ssa.OpcodeVSsubSat: |
| 464 |
x, y, lane := instr.Arg2WithLane() |
| 465 |
arr := ssaLaneToArrangement(lane) |
| 466 |
m.lowerVecRRR(vecOpSqsub, x, y, instr.Return(), arr) |
| 467 |
case ssa.OpcodeVUsubSat: |
| 468 |
x, y, lane := instr.Arg2WithLane() |
| 469 |
arr := ssaLaneToArrangement(lane) |
| 470 |
m.lowerVecRRR(vecOpUqsub, x, y, instr.Return(), arr) |
| 471 |
case ssa.OpcodeVImin: |
| 472 |
x, y, lane := instr.Arg2WithLane() |
| 473 |
arr := ssaLaneToArrangement(lane) |
| 474 |
m.lowerVecRRR(vecOpSmin, x, y, instr.Return(), arr) |
| 475 |
case ssa.OpcodeVUmin: |
| 476 |
x, y, lane := instr.Arg2WithLane() |
| 477 |
arr := ssaLaneToArrangement(lane) |
| 478 |
m.lowerVecRRR(vecOpUmin, x, y, instr.Return(), arr) |
| 479 |
case ssa.OpcodeVImax: |
| 480 |
x, y, lane := instr.Arg2WithLane() |
| 481 |
arr := ssaLaneToArrangement(lane) |
| 482 |
m.lowerVecRRR(vecOpSmax, x, y, instr.Return(), arr) |
| 483 |
case ssa.OpcodeVUmax: |
| 484 |
x, y, lane := instr.Arg2WithLane() |
| 485 |
arr := ssaLaneToArrangement(lane) |
| 486 |
m.lowerVecRRR(vecOpUmax, x, y, instr.Return(), arr) |
| 487 |
case ssa.OpcodeVAvgRound: |
| 488 |
x, y, lane := instr.Arg2WithLane() |
| 489 |
arr := ssaLaneToArrangement(lane) |
| 490 |
m.lowerVecRRR(vecOpUrhadd, x, y, instr.Return(), arr) |
| 491 |
case ssa.OpcodeVImul: |
| 492 |
x, y, lane := instr.Arg2WithLane() |
| 493 |
arr := ssaLaneToArrangement(lane) |
| 494 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 495 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 496 |
rd := m.compiler.VRegOf(instr.Return()) |
| 497 |
m.lowerVIMul(rd, rn, rm, arr) |
| 498 |
case ssa.OpcodeVIabs: |
| 499 |
m.lowerVecMisc(vecOpAbs, instr) |
| 500 |
case ssa.OpcodeVIneg: |
| 501 |
m.lowerVecMisc(vecOpNeg, instr) |
| 502 |
case ssa.OpcodeVIpopcnt: |
| 503 |
m.lowerVecMisc(vecOpCnt, instr) |
| 504 |
case ssa.OpcodeVIshl, |
| 505 |
ssa.OpcodeVSshr, ssa.OpcodeVUshr: |
| 506 |
x, y, lane := instr.Arg2WithLane() |
| 507 |
arr := ssaLaneToArrangement(lane) |
| 508 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 509 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 510 |
rd := m.compiler.VRegOf(instr.Return()) |
| 511 |
m.lowerVShift(op, rd, rn, rm, arr) |
| 512 |
case ssa.OpcodeVSqrt: |
| 513 |
m.lowerVecMisc(vecOpFsqrt, instr) |
| 514 |
case ssa.OpcodeVFabs: |
| 515 |
m.lowerVecMisc(vecOpFabs, instr) |
| 516 |
case ssa.OpcodeVFneg: |
| 517 |
m.lowerVecMisc(vecOpFneg, instr) |
| 518 |
case ssa.OpcodeVFmin: |
| 519 |
x, y, lane := instr.Arg2WithLane() |
| 520 |
arr := ssaLaneToArrangement(lane) |
| 521 |
m.lowerVecRRR(vecOpFmin, x, y, instr.Return(), arr) |
| 522 |
case ssa.OpcodeVFmax: |
| 523 |
x, y, lane := instr.Arg2WithLane() |
| 524 |
arr := ssaLaneToArrangement(lane) |
| 525 |
m.lowerVecRRR(vecOpFmax, x, y, instr.Return(), arr) |
| 526 |
case ssa.OpcodeVFadd: |
| 527 |
x, y, lane := instr.Arg2WithLane() |
| 528 |
arr := ssaLaneToArrangement(lane) |
| 529 |
m.lowerVecRRR(vecOpFadd, x, y, instr.Return(), arr) |
| 530 |
case ssa.OpcodeVFsub: |
| 531 |
x, y, lane := instr.Arg2WithLane() |
| 532 |
arr := ssaLaneToArrangement(lane) |
| 533 |
m.lowerVecRRR(vecOpFsub, x, y, instr.Return(), arr) |
| 534 |
case ssa.OpcodeVFmul: |
| 535 |
x, y, lane := instr.Arg2WithLane() |
| 536 |
arr := ssaLaneToArrangement(lane) |
| 537 |
m.lowerVecRRR(vecOpFmul, x, y, instr.Return(), arr) |
| 538 |
case ssa.OpcodeSqmulRoundSat: |
| 539 |
x, y, lane := instr.Arg2WithLane() |
| 540 |
arr := ssaLaneToArrangement(lane) |
| 541 |
m.lowerVecRRR(vecOpSqrdmulh, x, y, instr.Return(), arr) |
| 542 |
case ssa.OpcodeVFdiv: |
| 543 |
x, y, lane := instr.Arg2WithLane() |
| 544 |
arr := ssaLaneToArrangement(lane) |
| 545 |
m.lowerVecRRR(vecOpFdiv, x, y, instr.Return(), arr) |
| 546 |
case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat: |
| 547 |
x, lane := instr.ArgWithLane() |
| 548 |
arr := ssaLaneToArrangement(lane) |
| 549 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 550 |
rd := m.compiler.VRegOf(instr.Return()) |
| 551 |
m.lowerVfpuToInt(rd, rn, arr, op == ssa.OpcodeVFcvtToSintSat) |
| 552 |
case ssa.OpcodeVFcvtFromSint, ssa.OpcodeVFcvtFromUint: |
| 553 |
x, lane := instr.ArgWithLane() |
| 554 |
arr := ssaLaneToArrangement(lane) |
| 555 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 556 |
rd := m.compiler.VRegOf(instr.Return()) |
| 557 |
m.lowerVfpuFromInt(rd, rn, arr, op == ssa.OpcodeVFcvtFromSint) |
| 558 |
case ssa.OpcodeSwidenLow, ssa.OpcodeUwidenLow: |
| 559 |
x, lane := instr.ArgWithLane() |
| 560 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 561 |
rd := m.compiler.VRegOf(instr.Return()) |
| 562 |
|
| 563 |
var arr vecArrangement |
| 564 |
switch lane { |
| 565 |
case ssa.VecLaneI8x16: |
| 566 |
arr = vecArrangement8B |
| 567 |
case ssa.VecLaneI16x8: |
| 568 |
arr = vecArrangement4H |
| 569 |
case ssa.VecLaneI32x4: |
| 570 |
arr = vecArrangement2S |
| 571 |
} |
| 572 |
|
| 573 |
shll := m.allocateInstr() |
| 574 |
if signed := op == ssa.OpcodeSwidenLow; signed { |
| 575 |
shll.asVecShiftImm(vecOpSshll, rd, rn, operandShiftImm(0), arr) |
| 576 |
} else { |
| 577 |
shll.asVecShiftImm(vecOpUshll, rd, rn, operandShiftImm(0), arr) |
| 578 |
} |
| 579 |
m.insert(shll) |
| 580 |
case ssa.OpcodeSwidenHigh, ssa.OpcodeUwidenHigh: |
| 581 |
x, lane := instr.ArgWithLane() |
| 582 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 583 |
rd := m.compiler.VRegOf(instr.Return()) |
| 584 |
|
| 585 |
arr := ssaLaneToArrangement(lane) |
| 586 |
|
| 587 |
shll := m.allocateInstr() |
| 588 |
if signed := op == ssa.OpcodeSwidenHigh; signed { |
| 589 |
shll.asVecShiftImm(vecOpSshll, rd, rn, operandShiftImm(0), arr) |
| 590 |
} else { |
| 591 |
shll.asVecShiftImm(vecOpUshll, rd, rn, operandShiftImm(0), arr) |
| 592 |
} |
| 593 |
m.insert(shll) |
| 594 |
|
| 595 |
case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow: |
| 596 |
x, y, lane := instr.Arg2WithLane() |
| 597 |
var arr, arr2 vecArrangement |
| 598 |
switch lane { |
| 599 |
case ssa.VecLaneI16x8: // I16x8 |
| 600 |
arr = vecArrangement8B |
| 601 |
arr2 = vecArrangement16B // Implies sqxtn2. |
| 602 |
case ssa.VecLaneI32x4: |
| 603 |
arr = vecArrangement4H |
| 604 |
arr2 = vecArrangement8H // Implies sqxtn2. |
| 605 |
default: |
| 606 |
panic("unsupported lane " + lane.String()) |
| 607 |
} |
| 608 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 609 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 610 |
rd := m.compiler.VRegOf(instr.Return()) |
| 611 |
|
| 612 |
tmp := m.compiler.AllocateVReg(ssa.TypeV128) |
| 613 |
|
| 614 |
loQxtn := m.allocateInstr() |
| 615 |
hiQxtn := m.allocateInstr() |
| 616 |
if signed := op == ssa.OpcodeSnarrow; signed { |
| 617 |
// Narrow lanes on rn and write them into lower-half of rd. |
| 618 |
loQxtn.asVecMisc(vecOpSqxtn, tmp, rn, arr) // low |
| 619 |
// Narrow lanes on rm and write them into higher-half of rd. |
| 620 |
hiQxtn.asVecMisc(vecOpSqxtn, tmp, rm, arr2) // high (sqxtn2) |
| 621 |
} else { |
| 622 |
// Narrow lanes on rn and write them into lower-half of rd. |
| 623 |
loQxtn.asVecMisc(vecOpSqxtun, tmp, rn, arr) // low |
| 624 |
// Narrow lanes on rm and write them into higher-half of rd. |
| 625 |
hiQxtn.asVecMisc(vecOpSqxtun, tmp, rm, arr2) // high (sqxtn2) |
| 626 |
} |
| 627 |
m.insert(loQxtn) |
| 628 |
m.insert(hiQxtn) |
| 629 |
|
| 630 |
mov := m.allocateInstr() |
| 631 |
mov.asFpuMov128(rd, tmp) |
| 632 |
m.insert(mov) |
| 633 |
case ssa.OpcodeFvpromoteLow: |
| 634 |
x, lane := instr.ArgWithLane() |
| 635 |
if lane != ssa.VecLaneF32x4 { |
| 636 |
panic("unsupported lane type " + lane.String()) |
| 637 |
} |
| 638 |
ins := m.allocateInstr() |
| 639 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 640 |
rd := m.compiler.VRegOf(instr.Return()) |
| 641 |
ins.asVecMisc(vecOpFcvtl, rd, rn, vecArrangement2S) |
| 642 |
m.insert(ins) |
| 643 |
case ssa.OpcodeFvdemote: |
| 644 |
x, lane := instr.ArgWithLane() |
| 645 |
if lane != ssa.VecLaneF64x2 { |
| 646 |
panic("unsupported lane type " + lane.String()) |
| 647 |
} |
| 648 |
ins := m.allocateInstr() |
| 649 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 650 |
rd := m.compiler.VRegOf(instr.Return()) |
| 651 |
ins.asVecMisc(vecOpFcvtn, rd, rn, vecArrangement2S) |
| 652 |
m.insert(ins) |
| 653 |
case ssa.OpcodeExtractlane: |
| 654 |
x, index, signed, lane := instr.ExtractlaneData() |
| 655 |
|
| 656 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 657 |
rd := m.compiler.VRegOf(instr.Return()) |
| 658 |
|
| 659 |
mov := m.allocateInstr() |
| 660 |
switch lane { |
| 661 |
case ssa.VecLaneI8x16: |
| 662 |
mov.asMovFromVec(rd, rn, vecArrangementB, vecIndex(index), signed) |
| 663 |
case ssa.VecLaneI16x8: |
| 664 |
mov.asMovFromVec(rd, rn, vecArrangementH, vecIndex(index), signed) |
| 665 |
case ssa.VecLaneI32x4: |
| 666 |
mov.asMovFromVec(rd, rn, vecArrangementS, vecIndex(index), signed) |
| 667 |
case ssa.VecLaneI64x2: |
| 668 |
mov.asMovFromVec(rd, rn, vecArrangementD, vecIndex(index), signed) |
| 669 |
case ssa.VecLaneF32x4: |
| 670 |
mov.asVecMovElement(rd, rn, vecArrangementS, vecIndex(0), vecIndex(index)) |
| 671 |
case ssa.VecLaneF64x2: |
| 672 |
mov.asVecMovElement(rd, rn, vecArrangementD, vecIndex(0), vecIndex(index)) |
| 673 |
default: |
| 674 |
panic("unsupported lane: " + lane.String()) |
| 675 |
} |
| 676 |
|
| 677 |
m.insert(mov) |
| 678 |
|
| 679 |
case ssa.OpcodeInsertlane: |
| 680 |
x, y, index, lane := instr.InsertlaneData() |
| 681 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 682 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 683 |
rd := m.compiler.VRegOf(instr.Return()) |
| 684 |
tmpReg := m.compiler.AllocateVReg(ssa.TypeV128) |
| 685 |
|
| 686 |
// Initially mov rn to tmp. |
| 687 |
mov1 := m.allocateInstr() |
| 688 |
mov1.asFpuMov128(tmpReg, rn.nr()) |
| 689 |
m.insert(mov1) |
| 690 |
|
| 691 |
// movToVec and vecMovElement do not clear the remaining bits to zero, |
| 692 |
// thus, we can mov rm in-place to tmp. |
| 693 |
mov2 := m.allocateInstr() |
| 694 |
switch lane { |
| 695 |
case ssa.VecLaneI8x16: |
| 696 |
mov2.asMovToVec(tmpReg, rm, vecArrangementB, vecIndex(index)) |
| 697 |
case ssa.VecLaneI16x8: |
| 698 |
mov2.asMovToVec(tmpReg, rm, vecArrangementH, vecIndex(index)) |
| 699 |
case ssa.VecLaneI32x4: |
| 700 |
mov2.asMovToVec(tmpReg, rm, vecArrangementS, vecIndex(index)) |
| 701 |
case ssa.VecLaneI64x2: |
| 702 |
mov2.asMovToVec(tmpReg, rm, vecArrangementD, vecIndex(index)) |
| 703 |
case ssa.VecLaneF32x4: |
| 704 |
mov2.asVecMovElement(tmpReg, rm, vecArrangementS, vecIndex(index), vecIndex(0)) |
| 705 |
case ssa.VecLaneF64x2: |
| 706 |
mov2.asVecMovElement(tmpReg, rm, vecArrangementD, vecIndex(index), vecIndex(0)) |
| 707 |
} |
| 708 |
m.insert(mov2) |
| 709 |
|
| 710 |
// Finally mov tmp to rd. |
| 711 |
mov3 := m.allocateInstr() |
| 712 |
mov3.asFpuMov128(rd, tmpReg) |
| 713 |
m.insert(mov3) |
| 714 |
|
| 715 |
case ssa.OpcodeSwizzle: |
| 716 |
x, y, lane := instr.Arg2WithLane() |
| 717 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 718 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 719 |
rd := m.compiler.VRegOf(instr.Return()) |
| 720 |
|
| 721 |
arr := ssaLaneToArrangement(lane) |
| 722 |
|
| 723 |
// tbl <rd>.<arr>, { <rn>.<arr> }, <rm>.<arr> |
| 724 |
tbl1 := m.allocateInstr() |
| 725 |
tbl1.asVecTbl(1, rd, rn, rm, arr) |
| 726 |
m.insert(tbl1) |
| 727 |
|
| 728 |
case ssa.OpcodeShuffle: |
| 729 |
x, y, lane1, lane2 := instr.ShuffleData() |
| 730 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 731 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 732 |
rd := m.compiler.VRegOf(instr.Return()) |
| 733 |
|
| 734 |
m.lowerShuffle(rd, rn, rm, lane1, lane2) |
| 735 |
|
| 736 |
case ssa.OpcodeSplat: |
| 737 |
x, lane := instr.ArgWithLane() |
| 738 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 739 |
rd := m.compiler.VRegOf(instr.Return()) |
| 740 |
|
| 741 |
dup := m.allocateInstr() |
| 742 |
switch lane { |
| 743 |
case ssa.VecLaneI8x16: |
| 744 |
dup.asVecDup(rd, rn, vecArrangement16B) |
| 745 |
case ssa.VecLaneI16x8: |
| 746 |
dup.asVecDup(rd, rn, vecArrangement8H) |
| 747 |
case ssa.VecLaneI32x4: |
| 748 |
dup.asVecDup(rd, rn, vecArrangement4S) |
| 749 |
case ssa.VecLaneI64x2: |
| 750 |
dup.asVecDup(rd, rn, vecArrangement2D) |
| 751 |
case ssa.VecLaneF32x4: |
| 752 |
dup.asVecDupElement(rd, rn, vecArrangementS, vecIndex(0)) |
| 753 |
case ssa.VecLaneF64x2: |
| 754 |
dup.asVecDupElement(rd, rn, vecArrangementD, vecIndex(0)) |
| 755 |
} |
| 756 |
m.insert(dup) |
| 757 |
|
| 758 |
case ssa.OpcodeWideningPairwiseDotProductS: |
| 759 |
x, y := instr.Arg2() |
| 760 |
xx, yy := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone), |
| 761 |
m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 762 |
tmp, tmp2 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 763 |
m.insert(m.allocateInstr().asVecRRR(vecOpSmull, tmp.nr(), xx, yy, vecArrangement8H)) |
| 764 |
m.insert(m.allocateInstr().asVecRRR(vecOpSmull2, tmp2.nr(), xx, yy, vecArrangement8H)) |
| 765 |
m.insert(m.allocateInstr().asVecRRR(vecOpAddp, tmp.nr(), tmp, tmp2, vecArrangement4S)) |
| 766 |
|
| 767 |
rd := m.compiler.VRegOf(instr.Return()) |
| 768 |
m.insert(m.allocateInstr().asFpuMov128(rd, tmp.nr())) |
| 769 |
|
| 770 |
case ssa.OpcodeLoadSplat: |
| 771 |
ptr, offset, lane := instr.LoadSplatData() |
| 772 |
m.lowerLoadSplat(ptr, offset, lane, instr.Return()) |
| 773 |
|
| 774 |
case ssa.OpcodeAtomicRmw: |
| 775 |
m.lowerAtomicRmw(instr) |
| 776 |
|
| 777 |
case ssa.OpcodeAtomicCas: |
| 778 |
m.lowerAtomicCas(instr) |
| 779 |
|
| 780 |
case ssa.OpcodeAtomicLoad: |
| 781 |
m.lowerAtomicLoad(instr) |
| 782 |
|
| 783 |
case ssa.OpcodeAtomicStore: |
| 784 |
m.lowerAtomicStore(instr) |
| 785 |
|
| 786 |
case ssa.OpcodeFence: |
| 787 |
instr := m.allocateInstr() |
| 788 |
instr.asDMB() |
| 789 |
m.insert(instr) |
| 790 |
|
| 791 |
case ssa.OpcodeTailCallReturnCall, ssa.OpcodeTailCallReturnCallIndirect: |
| 792 |
m.lowerTailCall(instr) |
| 793 |
|
| 794 |
default: |
| 795 |
panic("TODO: lowering " + op.String()) |
| 796 |
} |
| 797 |
m.FlushPendingInstructions() |
| 798 |
} |
| 799 |
|
| 800 |
func (m *machine) lowerShuffle(rd regalloc.VReg, rn, rm operand, lane1, lane2 uint64) { |
| 801 |
// `tbl2` requires 2 consecutive registers, so we arbitrarily pick v29, v30. |
| 802 |
vReg, wReg := v29VReg, v30VReg |
| 803 |
|
| 804 |
// Initialize v29, v30 to rn, rm. |
| 805 |
movv := m.allocateInstr() |
| 806 |
movv.asFpuMov128(vReg, rn.nr()) |
| 807 |
m.insert(movv) |
| 808 |
|
| 809 |
movw := m.allocateInstr() |
| 810 |
movw.asFpuMov128(wReg, rm.nr()) |
| 811 |
m.insert(movw) |
| 812 |
|
| 813 |
// `lane1`, `lane2` are already encoded as two u64s with the right layout: |
| 814 |
// lane1 := lane[7]<<56 | ... | lane[1]<<8 | lane[0] |
| 815 |
// lane2 := lane[15]<<56 | ... | lane[9]<<8 | lane[8] |
| 816 |
// Thus, we can use loadFpuConst128. |
| 817 |
tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 818 |
lfc := m.allocateInstr() |
| 819 |
lfc.asLoadFpuConst128(tmp.nr(), lane1, lane2) |
| 820 |
m.insert(lfc) |
| 821 |
|
| 822 |
// tbl <rd>.16b, { <vReg>.16B, <wReg>.16b }, <tmp>.16b |
| 823 |
tbl2 := m.allocateInstr() |
| 824 |
tbl2.asVecTbl(2, rd, operandNR(vReg), tmp, vecArrangement16B) |
| 825 |
m.insert(tbl2) |
| 826 |
} |
| 827 |
|
| 828 |
func (m *machine) lowerVShift(op ssa.Opcode, rd regalloc.VReg, rn, rm operand, arr vecArrangement) { |
| 829 |
var modulo byte |
| 830 |
switch arr { |
| 831 |
case vecArrangement16B: |
| 832 |
modulo = 0x7 // Modulo 8. |
| 833 |
case vecArrangement8H: |
| 834 |
modulo = 0xf // Modulo 16. |
| 835 |
case vecArrangement4S: |
| 836 |
modulo = 0x1f // Modulo 32. |
| 837 |
case vecArrangement2D: |
| 838 |
modulo = 0x3f // Modulo 64. |
| 839 |
default: |
| 840 |
panic("unsupported arrangment " + arr.String()) |
| 841 |
} |
| 842 |
|
| 843 |
rtmp := operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) |
| 844 |
vtmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 845 |
|
| 846 |
and := m.allocateInstr() |
| 847 |
and.asALUBitmaskImm(aluOpAnd, rtmp.nr(), rm.nr(), uint64(modulo), true) |
| 848 |
m.insert(and) |
| 849 |
|
| 850 |
if op != ssa.OpcodeVIshl { |
| 851 |
// Negate the amount to make this as right shift. |
| 852 |
neg := m.allocateInstr() |
| 853 |
neg.asALU(aluOpSub, rtmp.nr(), operandNR(xzrVReg), rtmp, true) |
| 854 |
m.insert(neg) |
| 855 |
} |
| 856 |
|
| 857 |
// Copy the shift amount into a vector register as sshl/ushl requires it to be there. |
| 858 |
dup := m.allocateInstr() |
| 859 |
dup.asVecDup(vtmp.nr(), rtmp, arr) |
| 860 |
m.insert(dup) |
| 861 |
|
| 862 |
if op == ssa.OpcodeVIshl || op == ssa.OpcodeVSshr { |
| 863 |
sshl := m.allocateInstr() |
| 864 |
sshl.asVecRRR(vecOpSshl, rd, rn, vtmp, arr) |
| 865 |
m.insert(sshl) |
| 866 |
} else { |
| 867 |
ushl := m.allocateInstr() |
| 868 |
ushl.asVecRRR(vecOpUshl, rd, rn, vtmp, arr) |
| 869 |
m.insert(ushl) |
| 870 |
} |
| 871 |
} |
| 872 |
|
| 873 |
func (m *machine) lowerVcheckTrue(op ssa.Opcode, rm operand, rd regalloc.VReg, arr vecArrangement) { |
| 874 |
tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 875 |
|
| 876 |
// Special case VallTrue for i64x2. |
| 877 |
if op == ssa.OpcodeVallTrue && arr == vecArrangement2D { |
| 878 |
// cmeq v3?.2d, v2?.2d, #0 |
| 879 |
// addp v3?.2d, v3?.2d, v3?.2d |
| 880 |
// fcmp v3?, v3? |
| 881 |
// cset dst, eq |
| 882 |
|
| 883 |
ins := m.allocateInstr() |
| 884 |
ins.asVecMisc(vecOpCmeq0, tmp.nr(), rm, vecArrangement2D) |
| 885 |
m.insert(ins) |
| 886 |
|
| 887 |
addp := m.allocateInstr() |
| 888 |
addp.asVecRRR(vecOpAddp, tmp.nr(), tmp, tmp, vecArrangement2D) |
| 889 |
m.insert(addp) |
| 890 |
|
| 891 |
fcmp := m.allocateInstr() |
| 892 |
fcmp.asFpuCmp(tmp, tmp, true) |
| 893 |
m.insert(fcmp) |
| 894 |
|
| 895 |
cset := m.allocateInstr() |
| 896 |
cset.asCSet(rd, false, eq) |
| 897 |
m.insert(cset) |
| 898 |
|
| 899 |
return |
| 900 |
} |
| 901 |
|
| 902 |
// Create a scalar value with umaxp or uminv, then compare it against zero. |
| 903 |
ins := m.allocateInstr() |
| 904 |
if op == ssa.OpcodeVanyTrue { |
| 905 |
// umaxp v4?.16b, v2?.16b, v2?.16b |
| 906 |
ins.asVecRRR(vecOpUmaxp, tmp.nr(), rm, rm, vecArrangement16B) |
| 907 |
} else { |
| 908 |
// uminv d4?, v2?.4s |
| 909 |
ins.asVecLanes(vecOpUminv, tmp.nr(), rm, arr) |
| 910 |
} |
| 911 |
m.insert(ins) |
| 912 |
|
| 913 |
// mov x3?, v4?.d[0] |
| 914 |
// ccmp x3?, #0x0, #0x0, al |
| 915 |
// cset x3?, ne |
| 916 |
// mov x0, x3? |
| 917 |
|
| 918 |
movv := m.allocateInstr() |
| 919 |
movv.asMovFromVec(rd, tmp, vecArrangementD, vecIndex(0), false) |
| 920 |
m.insert(movv) |
| 921 |
|
| 922 |
fc := m.allocateInstr() |
| 923 |
fc.asCCmpImm(operandNR(rd), uint64(0), al, 0, true) |
| 924 |
m.insert(fc) |
| 925 |
|
| 926 |
cset := m.allocateInstr() |
| 927 |
cset.asCSet(rd, false, ne) |
| 928 |
m.insert(cset) |
| 929 |
} |
| 930 |
|
| 931 |
func (m *machine) lowerVhighBits(rm operand, rd regalloc.VReg, arr vecArrangement) { |
| 932 |
r0 := operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) |
| 933 |
v0 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 934 |
v1 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) |
| 935 |
|
| 936 |
switch arr { |
| 937 |
case vecArrangement16B: |
| 938 |
// sshr v6?.16b, v2?.16b, #7 |
| 939 |
// movz x4?, #0x201, lsl 0 |
| 940 |
// movk x4?, #0x804, lsl 16 |
| 941 |
// movk x4?, #0x2010, lsl 32 |
| 942 |
// movk x4?, #0x8040, lsl 48 |
| 943 |
// dup v5?.2d, x4? |
| 944 |
// and v6?.16b, v6?.16b, v5?.16b |
| 945 |
// ext v5?.16b, v6?.16b, v6?.16b, #8 |
| 946 |
// zip1 v5?.16b, v6?.16b, v5?.16b |
| 947 |
// addv s5?, v5?.8h |
| 948 |
// umov s3?, v5?.h[0] |
| 949 |
|
| 950 |
// Right arithmetic shift on the original vector and store the result into v1. So we have: |
| 951 |
// v1[i] = 0xff if vi<0, 0 otherwise. |
| 952 |
sshr := m.allocateInstr() |
| 953 |
sshr.asVecShiftImm(vecOpSshr, v1.nr(), rm, operandShiftImm(7), vecArrangement16B) |
| 954 |
m.insert(sshr) |
| 955 |
|
| 956 |
// Load the bit mask into r0. |
| 957 |
m.insertMOVZ(r0.nr(), 0x0201, 0, true) |
| 958 |
m.insertMOVK(r0.nr(), 0x0804, 1, true) |
| 959 |
m.insertMOVK(r0.nr(), 0x2010, 2, true) |
| 960 |
m.insertMOVK(r0.nr(), 0x8040, 3, true) |
| 961 |
|
| 962 |
// dup r0 to v0. |
| 963 |
dup := m.allocateInstr() |
| 964 |
dup.asVecDup(v0.nr(), r0, vecArrangement2D) |
| 965 |
m.insert(dup) |
| 966 |
|
| 967 |
// Lane-wise logical AND with the bit mask, meaning that we have |
| 968 |
// v[i] = (1 << i) if vi<0, 0 otherwise. |
| 969 |
// |
| 970 |
// Below, we use the following notation: |
| 971 |
// wi := (1 << i) if vi<0, 0 otherwise. |
| 972 |
and := m.allocateInstr() |
| 973 |
and.asVecRRR(vecOpAnd, v1.nr(), v1, v0, vecArrangement16B) |
| 974 |
m.insert(and) |
| 975 |
|
| 976 |
// Swap the lower and higher 8 byte elements, and write it into v0, meaning that we have |
| 977 |
// v0[i] = w(i+8) if i < 8, w(i-8) otherwise. |
| 978 |
ext := m.allocateInstr() |
| 979 |
ext.asVecExtract(v0.nr(), v1, v1, vecArrangement16B, uint32(8)) |
| 980 |
m.insert(ext) |
| 981 |
|
| 982 |
// v = [w0, w8, ..., w7, w15] |
| 983 |
zip1 := m.allocateInstr() |
| 984 |
zip1.asVecPermute(vecOpZip1, v0.nr(), v1, v0, vecArrangement16B) |
| 985 |
m.insert(zip1) |
| 986 |
|
| 987 |
// v.h[0] = w0 + ... + w15 |
| 988 |
addv := m.allocateInstr() |
| 989 |
addv.asVecLanes(vecOpAddv, v0.nr(), v0, vecArrangement8H) |
| 990 |
m.insert(addv) |
| 991 |
|
| 992 |
// Extract the v.h[0] as the result. |
| 993 |
movfv := m.allocateInstr() |
| 994 |
movfv.asMovFromVec(rd, v0, vecArrangementH, vecIndex(0), false) |
| 995 |
m.insert(movfv) |
| 996 |
case vecArrangement8H: |
| 997 |
// sshr v6?.8h, v2?.8h, #15 |
| 998 |
// movz x4?, #0x1, lsl 0 |
| 999 |
// movk x4?, #0x2, lsl 16 |
| 1000 |
// movk x4?, #0x4, lsl 32 |
| 1001 |
// movk x4?, #0x8, lsl 48 |
| 1002 |
// dup v5?.2d, x4? |
| 1003 |
// lsl x4?, x4?, 0x4 |
| 1004 |
// ins v5?.d[1], x4? |
| 1005 |
// and v5?.16b, v6?.16b, v5?.16b |
| 1006 |
// addv s5?, v5?.8h |
| 1007 |
// umov s3?, v5?.h[0] |
| 1008 |
|
| 1009 |
// Right arithmetic shift on the original vector and store the result into v1. So we have: |
| 1010 |
// v[i] = 0xffff if vi<0, 0 otherwise. |
| 1011 |
sshr := m.allocateInstr() |
| 1012 |
sshr.asVecShiftImm(vecOpSshr, v1.nr(), rm, operandShiftImm(15), vecArrangement8H) |
| 1013 |
m.insert(sshr) |
| 1014 |
|
| 1015 |
// Load the bit mask into r0. |
| 1016 |
m.lowerConstantI64(r0.nr(), 0x0008000400020001) |
| 1017 |
|
| 1018 |
// dup r0 to vector v0. |
| 1019 |
dup := m.allocateInstr() |
| 1020 |
dup.asVecDup(v0.nr(), r0, vecArrangement2D) |
| 1021 |
m.insert(dup) |
| 1022 |
|
| 1023 |
lsl := m.allocateInstr() |
| 1024 |
lsl.asALUShift(aluOpLsl, r0.nr(), r0, operandShiftImm(4), true) |
| 1025 |
m.insert(lsl) |
| 1026 |
|
| 1027 |
movv := m.allocateInstr() |
| 1028 |
movv.asMovToVec(v0.nr(), r0, vecArrangementD, vecIndex(1)) |
| 1029 |
m.insert(movv) |
| 1030 |
|
| 1031 |
// Lane-wise logical AND with the bitmask, meaning that we have |
| 1032 |
// v[i] = (1 << i) if vi<0, 0 otherwise for i=0..3 |
| 1033 |
// = (1 << (i+4)) if vi<0, 0 otherwise for i=3..7 |
| 1034 |
and := m.allocateInstr() |
| 1035 |
and.asVecRRR(vecOpAnd, v0.nr(), v1, v0, vecArrangement16B) |
| 1036 |
m.insert(and) |
| 1037 |
|
| 1038 |
addv := m.allocateInstr() |
| 1039 |
addv.asVecLanes(vecOpAddv, v0.nr(), v0, vecArrangement8H) |
| 1040 |
m.insert(addv) |
| 1041 |
|
| 1042 |
movfv := m.allocateInstr() |
| 1043 |
movfv.asMovFromVec(rd, v0, vecArrangementH, vecIndex(0), false) |
| 1044 |
m.insert(movfv) |
| 1045 |
case vecArrangement4S: |
| 1046 |
// sshr v6?.8h, v2?.8h, #15 |
| 1047 |
// movz x4?, #0x1, lsl 0 |
| 1048 |
// movk x4?, #0x2, lsl 16 |
| 1049 |
// movk x4?, #0x4, lsl 32 |
| 1050 |
// movk x4?, #0x8, lsl 48 |
| 1051 |
// dup v5?.2d, x4? |
| 1052 |
// lsl x4?, x4?, 0x4 |
| 1053 |
// ins v5?.d[1], x4? |
| 1054 |
// and v5?.16b, v6?.16b, v5?.16b |
| 1055 |
// addv s5?, v5?.8h |
| 1056 |
// umov s3?, v5?.h[0] |
| 1057 |
|
| 1058 |
// Right arithmetic shift on the original vector and store the result into v1. So we have: |
| 1059 |
// v[i] = 0xffffffff if vi<0, 0 otherwise. |
| 1060 |
sshr := m.allocateInstr() |
| 1061 |
sshr.asVecShiftImm(vecOpSshr, v1.nr(), rm, operandShiftImm(31), vecArrangement4S) |
| 1062 |
m.insert(sshr) |
| 1063 |
|
| 1064 |
// Load the bit mask into r0. |
| 1065 |
m.lowerConstantI64(r0.nr(), 0x0000000200000001) |
| 1066 |
|
| 1067 |
// dup r0 to vector v0. |
| 1068 |
dup := m.allocateInstr() |
| 1069 |
dup.asVecDup(v0.nr(), r0, vecArrangement2D) |
| 1070 |
m.insert(dup) |
| 1071 |
|
| 1072 |
lsl := m.allocateInstr() |
| 1073 |
lsl.asALUShift(aluOpLsl, r0.nr(), r0, operandShiftImm(2), true) |
| 1074 |
m.insert(lsl) |
| 1075 |
|
| 1076 |
movv := m.allocateInstr() |
| 1077 |
movv.asMovToVec(v0.nr(), r0, vecArrangementD, vecIndex(1)) |
| 1078 |
m.insert(movv) |
| 1079 |
|
| 1080 |
// Lane-wise logical AND with the bitmask, meaning that we have |
| 1081 |
// v[i] = (1 << i) if vi<0, 0 otherwise for i in [0, 1] |
| 1082 |
// = (1 << (i+4)) if vi<0, 0 otherwise for i in [2, 3] |
| 1083 |
and := m.allocateInstr() |
| 1084 |
and.asVecRRR(vecOpAnd, v0.nr(), v1, v0, vecArrangement16B) |
| 1085 |
m.insert(and) |
| 1086 |
|
| 1087 |
addv := m.allocateInstr() |
| 1088 |
addv.asVecLanes(vecOpAddv, v0.nr(), v0, vecArrangement4S) |
| 1089 |
m.insert(addv) |
| 1090 |
|
| 1091 |
movfv := m.allocateInstr() |
| 1092 |
movfv.asMovFromVec(rd, v0, vecArrangementS, vecIndex(0), false) |
| 1093 |
m.insert(movfv) |
| 1094 |
case vecArrangement2D: |
| 1095 |
// mov d3?, v2?.d[0] |
| 1096 |
// mov x4?, v2?.d[1] |
| 1097 |
// lsr x4?, x4?, 0x3f |
| 1098 |
// lsr d3?, d3?, 0x3f |
| 1099 |
// add s3?, s3?, w4?, lsl #1 |
| 1100 |
|
| 1101 |
// Move the lower 64-bit int into result. |
| 1102 |
movv0 := m.allocateInstr() |
| 1103 |
movv0.asMovFromVec(rd, rm, vecArrangementD, vecIndex(0), false) |
| 1104 |
m.insert(movv0) |
| 1105 |
|
| 1106 |
// Move the higher 64-bit int into r0. |
| 1107 |
movv1 := m.allocateInstr() |
| 1108 |
movv1.asMovFromVec(r0.nr(), rm, vecArrangementD, vecIndex(1), false) |
| 1109 |
m.insert(movv1) |
| 1110 |
|
| 1111 |
// Move the sign bit into the least significant bit. |
| 1112 |
lsr1 := m.allocateInstr() |
| 1113 |
lsr1.asALUShift(aluOpLsr, r0.nr(), r0, operandShiftImm(63), true) |
| 1114 |
m.insert(lsr1) |
| 1115 |
|
| 1116 |
lsr2 := m.allocateInstr() |
| 1117 |
lsr2.asALUShift(aluOpLsr, rd, operandNR(rd), operandShiftImm(63), true) |
| 1118 |
m.insert(lsr2) |
| 1119 |
|
| 1120 |
// rd = (r0<<1) | rd |
| 1121 |
lsl := m.allocateInstr() |
| 1122 |
lsl.asALU(aluOpAdd, rd, operandNR(rd), operandSR(r0.nr(), 1, shiftOpLSL), false) |
| 1123 |
m.insert(lsl) |
| 1124 |
default: |
| 1125 |
panic("Unsupported " + arr.String()) |
| 1126 |
} |
| 1127 |
} |
| 1128 |
|
| 1129 |
func (m *machine) lowerVecMisc(op vecOp, instr *ssa.Instruction) { |
| 1130 |
x, lane := instr.ArgWithLane() |
| 1131 |
arr := ssaLaneToArrangement(lane) |
| 1132 |
ins := m.allocateInstr() |
| 1133 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1134 |
rd := m.compiler.VRegOf(instr.Return()) |
| 1135 |
ins.asVecMisc(op, rd, rn, arr) |
| 1136 |
m.insert(ins) |
| 1137 |
} |
| 1138 |
|
| 1139 |
func (m *machine) lowerVecRRR(op vecOp, x, y, ret ssa.Value, arr vecArrangement) { |
| 1140 |
ins := m.allocateInstr() |
| 1141 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1142 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1143 |
rd := m.compiler.VRegOf(ret) |
| 1144 |
ins.asVecRRR(op, rd, rn, rm, arr) |
| 1145 |
m.insert(ins) |
| 1146 |
} |
| 1147 |
|
| 1148 |
func (m *machine) lowerVIMul(rd regalloc.VReg, rn, rm operand, arr vecArrangement) { |
| 1149 |
if arr != vecArrangement2D { |
| 1150 |
mul := m.allocateInstr() |
| 1151 |
mul.asVecRRR(vecOpMul, rd, rn, rm, arr) |
| 1152 |
m.insert(mul) |
| 1153 |
} else { |
| 1154 |
tmp1 := m.compiler.AllocateVReg(ssa.TypeV128) |
| 1155 |
tmp2 := m.compiler.AllocateVReg(ssa.TypeV128) |
| 1156 |
tmp3 := m.compiler.AllocateVReg(ssa.TypeV128) |
| 1157 |
|
| 1158 |
tmpRes := m.compiler.AllocateVReg(ssa.TypeV128) |
| 1159 |
|
| 1160 |
// Following the algorithm in https://chromium-review.googlesource.com/c/v8/v8/+/1781696 |
| 1161 |
rev64 := m.allocateInstr() |
| 1162 |
rev64.asVecMisc(vecOpRev64, tmp2, rm, vecArrangement4S) |
| 1163 |
m.insert(rev64) |
| 1164 |
|
| 1165 |
mul := m.allocateInstr() |
| 1166 |
mul.asVecRRR(vecOpMul, tmp2, operandNR(tmp2), rn, vecArrangement4S) |
| 1167 |
m.insert(mul) |
| 1168 |
|
| 1169 |
xtn1 := m.allocateInstr() |
| 1170 |
xtn1.asVecMisc(vecOpXtn, tmp1, rn, vecArrangement2S) |
| 1171 |
m.insert(xtn1) |
| 1172 |
|
| 1173 |
addp := m.allocateInstr() |
| 1174 |
addp.asVecRRR(vecOpAddp, tmp2, operandNR(tmp2), operandNR(tmp2), vecArrangement4S) |
| 1175 |
m.insert(addp) |
| 1176 |
|
| 1177 |
xtn2 := m.allocateInstr() |
| 1178 |
xtn2.asVecMisc(vecOpXtn, tmp3, rm, vecArrangement2S) |
| 1179 |
m.insert(xtn2) |
| 1180 |
|
| 1181 |
// Note: do not write the result directly into result yet. This is the same reason as in bsl. |
| 1182 |
// In short, in UMLAL instruction, the result register is also one of the source register, and |
| 1183 |
// the value on the result register is significant. |
| 1184 |
shll := m.allocateInstr() |
| 1185 |
shll.asVecMisc(vecOpShll, tmpRes, operandNR(tmp2), vecArrangement2S) |
| 1186 |
m.insert(shll) |
| 1187 |
|
| 1188 |
umlal := m.allocateInstr() |
| 1189 |
umlal.asVecRRRRewrite(vecOpUmlal, tmpRes, operandNR(tmp3), operandNR(tmp1), vecArrangement2S) |
| 1190 |
m.insert(umlal) |
| 1191 |
|
| 1192 |
mov := m.allocateInstr() |
| 1193 |
mov.asFpuMov128(rd, tmpRes) |
| 1194 |
m.insert(mov) |
| 1195 |
} |
| 1196 |
} |
| 1197 |
|
| 1198 |
func (m *machine) lowerVMinMaxPseudo(instr *ssa.Instruction, max bool) { |
| 1199 |
x, y, lane := instr.Arg2WithLane() |
| 1200 |
arr := ssaLaneToArrangement(lane) |
| 1201 |
|
| 1202 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1203 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1204 |
|
| 1205 |
// Note: this usage of tmp is important. |
| 1206 |
// BSL modifies the destination register, so we need to use a temporary register so that |
| 1207 |
// the actual definition of the destination register happens *after* the BSL instruction. |
| 1208 |
// That way, we can force the spill instruction to be inserted after the BSL instruction. |
| 1209 |
tmp := m.compiler.AllocateVReg(ssa.TypeV128) |
| 1210 |
|
| 1211 |
fcmgt := m.allocateInstr() |
| 1212 |
if max { |
| 1213 |
fcmgt.asVecRRR(vecOpFcmgt, tmp, rm, rn, arr) |
| 1214 |
} else { |
| 1215 |
// If min, swap the args. |
| 1216 |
fcmgt.asVecRRR(vecOpFcmgt, tmp, rn, rm, arr) |
| 1217 |
} |
| 1218 |
m.insert(fcmgt) |
| 1219 |
|
| 1220 |
bsl := m.allocateInstr() |
| 1221 |
bsl.asVecRRRRewrite(vecOpBsl, tmp, rm, rn, vecArrangement16B) |
| 1222 |
m.insert(bsl) |
| 1223 |
|
| 1224 |
res := operandNR(m.compiler.VRegOf(instr.Return())) |
| 1225 |
mov2 := m.allocateInstr() |
| 1226 |
mov2.asFpuMov128(res.nr(), tmp) |
| 1227 |
m.insert(mov2) |
| 1228 |
} |
| 1229 |
|
| 1230 |
func (m *machine) lowerIRem(execCtxVReg regalloc.VReg, rd, rn regalloc.VReg, rm operand, _64bit, signed bool) { |
| 1231 |
div := m.allocateInstr() |
| 1232 |
|
| 1233 |
if signed { |
| 1234 |
div.asALU(aluOpSDiv, rd, operandNR(rn), rm, _64bit) |
| 1235 |
} else { |
| 1236 |
div.asALU(aluOpUDiv, rd, operandNR(rn), rm, _64bit) |
| 1237 |
} |
| 1238 |
m.insert(div) |
| 1239 |
|
| 1240 |
// Check if rm is zero: |
| 1241 |
m.exitIfNot(execCtxVReg, registerAsRegNotZeroCond(rm.nr()), _64bit, wazevoapi.ExitCodeIntegerDivisionByZero) |
| 1242 |
|
| 1243 |
// rd = rn-rd*rm by MSUB instruction. |
| 1244 |
msub := m.allocateInstr() |
| 1245 |
msub.asALURRRR(aluOpMSub, rd, operandNR(rd), rm, rn, _64bit) |
| 1246 |
m.insert(msub) |
| 1247 |
} |
| 1248 |
|
| 1249 |
func (m *machine) lowerIDiv(execCtxVReg, rd regalloc.VReg, rn, rm operand, _64bit, signed bool) { |
| 1250 |
div := m.allocateInstr() |
| 1251 |
|
| 1252 |
if signed { |
| 1253 |
div.asALU(aluOpSDiv, rd, rn, rm, _64bit) |
| 1254 |
} else { |
| 1255 |
div.asALU(aluOpUDiv, rd, rn, rm, _64bit) |
| 1256 |
} |
| 1257 |
m.insert(div) |
| 1258 |
|
| 1259 |
// Check if rm is zero: |
| 1260 |
m.exitIfNot(execCtxVReg, registerAsRegNotZeroCond(rm.nr()), _64bit, wazevoapi.ExitCodeIntegerDivisionByZero) |
| 1261 |
|
| 1262 |
if signed { |
| 1263 |
// We need to check the signed overflow which happens iff "math.MinInt{32,64} / -1" |
| 1264 |
minusOneCheck := m.allocateInstr() |
| 1265 |
// Sets eq condition if rm == -1. |
| 1266 |
minusOneCheck.asALU(aluOpAddS, xzrVReg, rm, operandImm12(1, 0), _64bit) |
| 1267 |
m.insert(minusOneCheck) |
| 1268 |
|
| 1269 |
ccmp := m.allocateInstr() |
| 1270 |
// If eq condition is set, sets the flag by the result based on "rn - 1", otherwise clears the flag. |
| 1271 |
ccmp.asCCmpImm(rn, 1, eq, 0, _64bit) |
| 1272 |
m.insert(ccmp) |
| 1273 |
|
| 1274 |
// Check the overflow flag. |
| 1275 |
m.exitIfNot(execCtxVReg, vs.invert().asCond(), false, wazevoapi.ExitCodeIntegerOverflow) |
| 1276 |
} |
| 1277 |
} |
| 1278 |
|
| 1279 |
// exitIfNot emits a conditional branch to exit if the condition is not met. |
| 1280 |
// If `c` (cond type) is a register, `cond64bit` must be chosen to indicate whether the register is 32-bit or 64-bit. |
| 1281 |
// Otherwise, `cond64bit` is ignored. |
| 1282 |
func (m *machine) exitIfNot(execCtxVReg regalloc.VReg, c cond, cond64bit bool, code wazevoapi.ExitCode) { |
| 1283 |
execCtxTmp := m.copyToTmp(execCtxVReg) |
| 1284 |
|
| 1285 |
cbr := m.allocateInstr() |
| 1286 |
m.insert(cbr) |
| 1287 |
m.lowerExitWithCode(execCtxTmp, code) |
| 1288 |
// Conditional branch target is after exit. |
| 1289 |
l := m.insertBrTargetLabel() |
| 1290 |
cbr.asCondBr(c, l, cond64bit) |
| 1291 |
} |
| 1292 |
|
| 1293 |
func (m *machine) lowerFcopysign(x, y, ret ssa.Value) { |
| 1294 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1295 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1296 |
var tmpI, tmpF regalloc.VReg |
| 1297 |
_64 := x.Type() == ssa.TypeF64 |
| 1298 |
if _64 { |
| 1299 |
tmpF = m.compiler.AllocateVReg(ssa.TypeF64) |
| 1300 |
tmpI = m.compiler.AllocateVReg(ssa.TypeI64) |
| 1301 |
} else { |
| 1302 |
tmpF = m.compiler.AllocateVReg(ssa.TypeF32) |
| 1303 |
tmpI = m.compiler.AllocateVReg(ssa.TypeI32) |
| 1304 |
} |
| 1305 |
rd := m.compiler.VRegOf(ret) |
| 1306 |
m.lowerFcopysignImpl(rd, rn, rm, tmpI, tmpF, _64) |
| 1307 |
} |
| 1308 |
|
| 1309 |
func (m *machine) lowerFcopysignImpl(rd regalloc.VReg, rn, rm operand, tmpI, tmpF regalloc.VReg, _64bit bool) { |
| 1310 |
// This is exactly the same code emitted by GCC for "__builtin_copysign": |
| 1311 |
// |
| 1312 |
// mov x0, -9223372036854775808 |
| 1313 |
// fmov d2, x0 |
| 1314 |
// vbit v0.8b, v1.8b, v2.8b |
| 1315 |
// |
| 1316 |
|
| 1317 |
setMSB := m.allocateInstr() |
| 1318 |
if _64bit { |
| 1319 |
m.lowerConstantI64(tmpI, math.MinInt64) |
| 1320 |
setMSB.asMovToVec(tmpF, operandNR(tmpI), vecArrangementD, vecIndex(0)) |
| 1321 |
} else { |
| 1322 |
m.lowerConstantI32(tmpI, math.MinInt32) |
| 1323 |
setMSB.asMovToVec(tmpF, operandNR(tmpI), vecArrangementS, vecIndex(0)) |
| 1324 |
} |
| 1325 |
m.insert(setMSB) |
| 1326 |
|
| 1327 |
tmpReg := m.compiler.AllocateVReg(ssa.TypeF64) |
| 1328 |
|
| 1329 |
mov := m.allocateInstr() |
| 1330 |
mov.asFpuMov64(tmpReg, rn.nr()) |
| 1331 |
m.insert(mov) |
| 1332 |
|
| 1333 |
vbit := m.allocateInstr() |
| 1334 |
vbit.asVecRRRRewrite(vecOpBit, tmpReg, rm, operandNR(tmpF), vecArrangement8B) |
| 1335 |
m.insert(vbit) |
| 1336 |
|
| 1337 |
movDst := m.allocateInstr() |
| 1338 |
movDst.asFpuMov64(rd, tmpReg) |
| 1339 |
m.insert(movDst) |
| 1340 |
} |
| 1341 |
|
| 1342 |
func (m *machine) lowerBitcast(instr *ssa.Instruction) { |
| 1343 |
v, dstType := instr.BitcastData() |
| 1344 |
srcType := v.Type() |
| 1345 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) |
| 1346 |
rd := m.compiler.VRegOf(instr.Return()) |
| 1347 |
srcInt := srcType.IsInt() |
| 1348 |
dstInt := dstType.IsInt() |
| 1349 |
switch { |
| 1350 |
case srcInt && !dstInt: // Int to Float: |
| 1351 |
mov := m.allocateInstr() |
| 1352 |
var arr vecArrangement |
| 1353 |
if srcType.Bits() == 64 { |
| 1354 |
arr = vecArrangementD |
| 1355 |
} else { |
| 1356 |
arr = vecArrangementS |
| 1357 |
} |
| 1358 |
mov.asMovToVec(rd, rn, arr, vecIndex(0)) |
| 1359 |
m.insert(mov) |
| 1360 |
case !srcInt && dstInt: // Float to Int: |
| 1361 |
mov := m.allocateInstr() |
| 1362 |
var arr vecArrangement |
| 1363 |
if dstType.Bits() == 64 { |
| 1364 |
arr = vecArrangementD |
| 1365 |
} else { |
| 1366 |
arr = vecArrangementS |
| 1367 |
} |
| 1368 |
mov.asMovFromVec(rd, rn, arr, vecIndex(0), false) |
| 1369 |
m.insert(mov) |
| 1370 |
default: |
| 1371 |
panic("TODO?BUG?") |
| 1372 |
} |
| 1373 |
} |
| 1374 |
|
| 1375 |
func (m *machine) lowerFpuUniOp(op fpuUniOp, in, out ssa.Value) { |
| 1376 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(in), extModeNone) |
| 1377 |
rd := m.compiler.VRegOf(out) |
| 1378 |
|
| 1379 |
neg := m.allocateInstr() |
| 1380 |
neg.asFpuRR(op, rd, rn, in.Type().Bits() == 64) |
| 1381 |
m.insert(neg) |
| 1382 |
} |
| 1383 |
|
| 1384 |
func (m *machine) lowerFpuToInt(rd regalloc.VReg, rn operand, ctx regalloc.VReg, signed, src64bit, dst64bit, nonTrapping bool) { |
| 1385 |
if !nonTrapping { |
| 1386 |
// First of all, we have to clear the FPU flags. |
| 1387 |
flagClear := m.allocateInstr() |
| 1388 |
flagClear.asMovToFPSR(xzrVReg) |
| 1389 |
m.insert(flagClear) |
| 1390 |
} |
| 1391 |
|
| 1392 |
// Then, do the conversion which doesn't trap inherently. |
| 1393 |
cvt := m.allocateInstr() |
| 1394 |
cvt.asFpuToInt(rd, rn, signed, src64bit, dst64bit) |
| 1395 |
m.insert(cvt) |
| 1396 |
|
| 1397 |
if !nonTrapping { |
| 1398 |
tmpReg := m.compiler.AllocateVReg(ssa.TypeI64) |
| 1399 |
|
| 1400 |
// After the conversion, check the FPU flags. |
| 1401 |
getFlag := m.allocateInstr() |
| 1402 |
getFlag.asMovFromFPSR(tmpReg) |
| 1403 |
m.insert(getFlag) |
| 1404 |
|
| 1405 |
execCtx := m.copyToTmp(ctx) |
| 1406 |
_rn := operandNR(m.copyToTmp(rn.nr())) |
| 1407 |
|
| 1408 |
// Check if the conversion was undefined by comparing the status with 1. |
| 1409 |
// See https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/FPSR--Floating-point-Status-Register |
| 1410 |
alu := m.allocateInstr() |
| 1411 |
alu.asALU(aluOpSubS, xzrVReg, operandNR(tmpReg), operandImm12(1, 0), true) |
| 1412 |
m.insert(alu) |
| 1413 |
|
| 1414 |
// If it is not undefined, we can return the result. |
| 1415 |
ok := m.allocateInstr() |
| 1416 |
m.insert(ok) |
| 1417 |
|
| 1418 |
// Otherwise, we have to choose the status depending on it is overflow or NaN conversion. |
| 1419 |
|
| 1420 |
// Comparing itself to check if it is a NaN. |
| 1421 |
fpuCmp := m.allocateInstr() |
| 1422 |
fpuCmp.asFpuCmp(_rn, _rn, src64bit) |
| 1423 |
m.insert(fpuCmp) |
| 1424 |
// If the VC flag is not set (== VS flag is set), it is a NaN. |
| 1425 |
m.exitIfNot(execCtx, vc.asCond(), false, wazevoapi.ExitCodeInvalidConversionToInteger) |
| 1426 |
// Otherwise, it is an overflow. |
| 1427 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) |
| 1428 |
|
| 1429 |
// Conditional branch target is after exit. |
| 1430 |
l := m.insertBrTargetLabel() |
| 1431 |
ok.asCondBr(ne.asCond(), l, false /* ignored */) |
| 1432 |
} |
| 1433 |
} |
| 1434 |
|
| 1435 |
func (m *machine) lowerIntToFpu(rd regalloc.VReg, rn operand, signed, src64bit, dst64bit bool) { |
| 1436 |
cvt := m.allocateInstr() |
| 1437 |
cvt.asIntToFpu(rd, rn, signed, src64bit, dst64bit) |
| 1438 |
m.insert(cvt) |
| 1439 |
} |
| 1440 |
|
| 1441 |
func (m *machine) lowerFpuBinOp(si *ssa.Instruction) { |
| 1442 |
instr := m.allocateInstr() |
| 1443 |
var op fpuBinOp |
| 1444 |
switch si.Opcode() { |
| 1445 |
case ssa.OpcodeFadd: |
| 1446 |
op = fpuBinOpAdd |
| 1447 |
case ssa.OpcodeFsub: |
| 1448 |
op = fpuBinOpSub |
| 1449 |
case ssa.OpcodeFmul: |
| 1450 |
op = fpuBinOpMul |
| 1451 |
case ssa.OpcodeFdiv: |
| 1452 |
op = fpuBinOpDiv |
| 1453 |
case ssa.OpcodeFmax: |
| 1454 |
op = fpuBinOpMax |
| 1455 |
case ssa.OpcodeFmin: |
| 1456 |
op = fpuBinOpMin |
| 1457 |
} |
| 1458 |
x, y := si.Arg2() |
| 1459 |
xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) |
| 1460 |
rn := m.getOperand_NR(xDef, extModeNone) |
| 1461 |
rm := m.getOperand_NR(yDef, extModeNone) |
| 1462 |
rd := m.compiler.VRegOf(si.Return()) |
| 1463 |
instr.asFpuRRR(op, rd, rn, rm, x.Type().Bits() == 64) |
| 1464 |
m.insert(instr) |
| 1465 |
} |
| 1466 |
|
| 1467 |
func (m *machine) lowerSubOrAdd(si *ssa.Instruction, add bool) { |
| 1468 |
x, y := si.Arg2() |
| 1469 |
if !x.Type().IsInt() { |
| 1470 |
panic("BUG?") |
| 1471 |
} |
| 1472 |
|
| 1473 |
xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) |
| 1474 |
rn := m.getOperand_NR(xDef, extModeNone) |
| 1475 |
rm, yNegated := m.getOperand_MaybeNegatedImm12_ER_SR_NR(yDef, extModeNone) |
| 1476 |
|
| 1477 |
var aop aluOp |
| 1478 |
switch { |
| 1479 |
case add && !yNegated: // rn+rm = x+y |
| 1480 |
aop = aluOpAdd |
| 1481 |
case add && yNegated: // rn-rm = x-(-y) = x+y |
| 1482 |
aop = aluOpSub |
| 1483 |
case !add && !yNegated: // rn-rm = x-y |
| 1484 |
aop = aluOpSub |
| 1485 |
case !add && yNegated: // rn+rm = x-(-y) = x-y |
| 1486 |
aop = aluOpAdd |
| 1487 |
} |
| 1488 |
rd := m.compiler.VRegOf(si.Return()) |
| 1489 |
alu := m.allocateInstr() |
| 1490 |
alu.asALU(aop, rd, rn, rm, x.Type().Bits() == 64) |
| 1491 |
m.insert(alu) |
| 1492 |
} |
| 1493 |
|
| 1494 |
// InsertMove implements backend.Machine. |
| 1495 |
func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) { |
| 1496 |
instr := m.allocateInstr() |
| 1497 |
switch typ { |
| 1498 |
case ssa.TypeI32, ssa.TypeI64: |
| 1499 |
instr.asMove64(dst, src) |
| 1500 |
case ssa.TypeF32, ssa.TypeF64: |
| 1501 |
instr.asFpuMov64(dst, src) |
| 1502 |
case ssa.TypeV128: |
| 1503 |
instr.asFpuMov128(dst, src) |
| 1504 |
default: |
| 1505 |
panic("TODO") |
| 1506 |
} |
| 1507 |
m.insert(instr) |
| 1508 |
} |
| 1509 |
|
| 1510 |
func (m *machine) lowerIcmp(si *ssa.Instruction) { |
| 1511 |
x, y, c := si.IcmpData() |
| 1512 |
flag := condFlagFromSSAIntegerCmpCond(c) |
| 1513 |
|
| 1514 |
in64bit := x.Type().Bits() == 64 |
| 1515 |
var ext extMode |
| 1516 |
if in64bit { |
| 1517 |
if c.Signed() { |
| 1518 |
ext = extModeSignExtend64 |
| 1519 |
} else { |
| 1520 |
ext = extModeZeroExtend64 |
| 1521 |
} |
| 1522 |
} else { |
| 1523 |
if c.Signed() { |
| 1524 |
ext = extModeSignExtend32 |
| 1525 |
} else { |
| 1526 |
ext = extModeZeroExtend32 |
| 1527 |
} |
| 1528 |
} |
| 1529 |
|
| 1530 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), ext) |
| 1531 |
rm := m.getOperand_Imm12_ER_SR_NR(m.compiler.ValueDefinition(y), ext) |
| 1532 |
alu := m.allocateInstr() |
| 1533 |
alu.asALU(aluOpSubS, xzrVReg, rn, rm, in64bit) |
| 1534 |
m.insert(alu) |
| 1535 |
|
| 1536 |
cset := m.allocateInstr() |
| 1537 |
cset.asCSet(m.compiler.VRegOf(si.Return()), false, flag) |
| 1538 |
m.insert(cset) |
| 1539 |
} |
| 1540 |
|
| 1541 |
func (m *machine) lowerVIcmp(si *ssa.Instruction) { |
| 1542 |
x, y, c, lane := si.VIcmpData() |
| 1543 |
flag := condFlagFromSSAIntegerCmpCond(c) |
| 1544 |
arr := ssaLaneToArrangement(lane) |
| 1545 |
|
| 1546 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1547 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1548 |
rd := m.compiler.VRegOf(si.Return()) |
| 1549 |
|
| 1550 |
switch flag { |
| 1551 |
case eq: |
| 1552 |
cmp := m.allocateInstr() |
| 1553 |
cmp.asVecRRR(vecOpCmeq, rd, rn, rm, arr) |
| 1554 |
m.insert(cmp) |
| 1555 |
case ne: |
| 1556 |
cmp := m.allocateInstr() |
| 1557 |
cmp.asVecRRR(vecOpCmeq, rd, rn, rm, arr) |
| 1558 |
m.insert(cmp) |
| 1559 |
not := m.allocateInstr() |
| 1560 |
not.asVecMisc(vecOpNot, rd, operandNR(rd), vecArrangement16B) |
| 1561 |
m.insert(not) |
| 1562 |
case ge: |
| 1563 |
cmp := m.allocateInstr() |
| 1564 |
cmp.asVecRRR(vecOpCmge, rd, rn, rm, arr) |
| 1565 |
m.insert(cmp) |
| 1566 |
case gt: |
| 1567 |
cmp := m.allocateInstr() |
| 1568 |
cmp.asVecRRR(vecOpCmgt, rd, rn, rm, arr) |
| 1569 |
m.insert(cmp) |
| 1570 |
case le: |
| 1571 |
cmp := m.allocateInstr() |
| 1572 |
cmp.asVecRRR(vecOpCmge, rd, rm, rn, arr) // rm, rn are swapped |
| 1573 |
m.insert(cmp) |
| 1574 |
case lt: |
| 1575 |
cmp := m.allocateInstr() |
| 1576 |
cmp.asVecRRR(vecOpCmgt, rd, rm, rn, arr) // rm, rn are swapped |
| 1577 |
m.insert(cmp) |
| 1578 |
case hs: |
| 1579 |
cmp := m.allocateInstr() |
| 1580 |
cmp.asVecRRR(vecOpCmhs, rd, rn, rm, arr) |
| 1581 |
m.insert(cmp) |
| 1582 |
case hi: |
| 1583 |
cmp := m.allocateInstr() |
| 1584 |
cmp.asVecRRR(vecOpCmhi, rd, rn, rm, arr) |
| 1585 |
m.insert(cmp) |
| 1586 |
case ls: |
| 1587 |
cmp := m.allocateInstr() |
| 1588 |
cmp.asVecRRR(vecOpCmhs, rd, rm, rn, arr) // rm, rn are swapped |
| 1589 |
m.insert(cmp) |
| 1590 |
case lo: |
| 1591 |
cmp := m.allocateInstr() |
| 1592 |
cmp.asVecRRR(vecOpCmhi, rd, rm, rn, arr) // rm, rn are swapped |
| 1593 |
m.insert(cmp) |
| 1594 |
} |
| 1595 |
} |
| 1596 |
|
| 1597 |
func (m *machine) lowerVFcmp(si *ssa.Instruction) { |
| 1598 |
x, y, c, lane := si.VFcmpData() |
| 1599 |
flag := condFlagFromSSAFloatCmpCond(c) |
| 1600 |
arr := ssaLaneToArrangement(lane) |
| 1601 |
|
| 1602 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1603 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1604 |
rd := m.compiler.VRegOf(si.Return()) |
| 1605 |
|
| 1606 |
switch flag { |
| 1607 |
case eq: |
| 1608 |
cmp := m.allocateInstr() |
| 1609 |
cmp.asVecRRR(vecOpFcmeq, rd, rn, rm, arr) |
| 1610 |
m.insert(cmp) |
| 1611 |
case ne: |
| 1612 |
cmp := m.allocateInstr() |
| 1613 |
cmp.asVecRRR(vecOpFcmeq, rd, rn, rm, arr) |
| 1614 |
m.insert(cmp) |
| 1615 |
not := m.allocateInstr() |
| 1616 |
not.asVecMisc(vecOpNot, rd, operandNR(rd), vecArrangement16B) |
| 1617 |
m.insert(not) |
| 1618 |
case ge: |
| 1619 |
cmp := m.allocateInstr() |
| 1620 |
cmp.asVecRRR(vecOpFcmge, rd, rn, rm, arr) |
| 1621 |
m.insert(cmp) |
| 1622 |
case gt: |
| 1623 |
cmp := m.allocateInstr() |
| 1624 |
cmp.asVecRRR(vecOpFcmgt, rd, rn, rm, arr) |
| 1625 |
m.insert(cmp) |
| 1626 |
case mi: |
| 1627 |
cmp := m.allocateInstr() |
| 1628 |
cmp.asVecRRR(vecOpFcmgt, rd, rm, rn, arr) // rm, rn are swapped |
| 1629 |
m.insert(cmp) |
| 1630 |
case ls: |
| 1631 |
cmp := m.allocateInstr() |
| 1632 |
cmp.asVecRRR(vecOpFcmge, rd, rm, rn, arr) // rm, rn are swapped |
| 1633 |
m.insert(cmp) |
| 1634 |
} |
| 1635 |
} |
| 1636 |
|
| 1637 |
func (m *machine) lowerVfpuToInt(rd regalloc.VReg, rn operand, arr vecArrangement, signed bool) { |
| 1638 |
cvt := m.allocateInstr() |
| 1639 |
if signed { |
| 1640 |
cvt.asVecMisc(vecOpFcvtzs, rd, rn, arr) |
| 1641 |
} else { |
| 1642 |
cvt.asVecMisc(vecOpFcvtzu, rd, rn, arr) |
| 1643 |
} |
| 1644 |
m.insert(cvt) |
| 1645 |
|
| 1646 |
if arr == vecArrangement2D { |
| 1647 |
narrow := m.allocateInstr() |
| 1648 |
if signed { |
| 1649 |
narrow.asVecMisc(vecOpSqxtn, rd, operandNR(rd), vecArrangement2S) |
| 1650 |
} else { |
| 1651 |
narrow.asVecMisc(vecOpUqxtn, rd, operandNR(rd), vecArrangement2S) |
| 1652 |
} |
| 1653 |
m.insert(narrow) |
| 1654 |
} |
| 1655 |
} |
| 1656 |
|
| 1657 |
func (m *machine) lowerVfpuFromInt(rd regalloc.VReg, rn operand, arr vecArrangement, signed bool) { |
| 1658 |
cvt := m.allocateInstr() |
| 1659 |
if signed { |
| 1660 |
cvt.asVecMisc(vecOpScvtf, rd, rn, arr) |
| 1661 |
} else { |
| 1662 |
cvt.asVecMisc(vecOpUcvtf, rd, rn, arr) |
| 1663 |
} |
| 1664 |
m.insert(cvt) |
| 1665 |
} |
| 1666 |
|
| 1667 |
func (m *machine) lowerShifts(si *ssa.Instruction, ext extMode, aluOp aluOp) { |
| 1668 |
x, amount := si.Arg2() |
| 1669 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), ext) |
| 1670 |
rm := m.getOperand_ShiftImm_NR(m.compiler.ValueDefinition(amount), ext, x.Type().Bits()) |
| 1671 |
rd := m.compiler.VRegOf(si.Return()) |
| 1672 |
|
| 1673 |
alu := m.allocateInstr() |
| 1674 |
alu.asALUShift(aluOp, rd, rn, rm, x.Type().Bits() == 64) |
| 1675 |
m.insert(alu) |
| 1676 |
} |
| 1677 |
|
| 1678 |
func (m *machine) lowerBitwiseAluOp(si *ssa.Instruction, op aluOp, ignoreResult bool) { |
| 1679 |
x, y := si.Arg2() |
| 1680 |
|
| 1681 |
xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) |
| 1682 |
rn := m.getOperand_NR(xDef, extModeNone) |
| 1683 |
|
| 1684 |
var rd regalloc.VReg |
| 1685 |
if ignoreResult { |
| 1686 |
rd = xzrVReg |
| 1687 |
} else { |
| 1688 |
rd = m.compiler.VRegOf(si.Return()) |
| 1689 |
} |
| 1690 |
|
| 1691 |
_64 := x.Type().Bits() == 64 |
| 1692 |
alu := m.allocateInstr() |
| 1693 |
if instr := yDef.Instr; instr != nil && instr.Constant() { |
| 1694 |
c := instr.ConstantVal() |
| 1695 |
if isBitMaskImmediate(c, _64) { |
| 1696 |
// Constant bit wise operations can be lowered to a single instruction. |
| 1697 |
alu.asALUBitmaskImm(op, rd, rn.nr(), c, _64) |
| 1698 |
m.insert(alu) |
| 1699 |
return |
| 1700 |
} |
| 1701 |
} |
| 1702 |
|
| 1703 |
rm := m.getOperand_SR_NR(yDef, extModeNone) |
| 1704 |
alu.asALU(op, rd, rn, rm, _64) |
| 1705 |
m.insert(alu) |
| 1706 |
} |
| 1707 |
|
| 1708 |
func (m *machine) lowerRotl(si *ssa.Instruction) { |
| 1709 |
x, y := si.Arg2() |
| 1710 |
r := si.Return() |
| 1711 |
_64 := r.Type().Bits() == 64 |
| 1712 |
|
| 1713 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1714 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1715 |
var tmp regalloc.VReg |
| 1716 |
if _64 { |
| 1717 |
tmp = m.compiler.AllocateVReg(ssa.TypeI64) |
| 1718 |
} else { |
| 1719 |
tmp = m.compiler.AllocateVReg(ssa.TypeI32) |
| 1720 |
} |
| 1721 |
rd := m.compiler.VRegOf(r) |
| 1722 |
|
| 1723 |
// Encode rotl as neg + rotr: neg is a sub against the zero-reg. |
| 1724 |
m.lowerRotlImpl(rd, rn, rm, tmp, _64) |
| 1725 |
} |
| 1726 |
|
| 1727 |
func (m *machine) lowerRotlImpl(rd regalloc.VReg, rn, rm operand, tmp regalloc.VReg, is64bit bool) { |
| 1728 |
// Encode rotl as neg + rotr: neg is a sub against the zero-reg. |
| 1729 |
neg := m.allocateInstr() |
| 1730 |
neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), rm, is64bit) |
| 1731 |
m.insert(neg) |
| 1732 |
alu := m.allocateInstr() |
| 1733 |
alu.asALU(aluOpRotR, rd, rn, operandNR(tmp), is64bit) |
| 1734 |
m.insert(alu) |
| 1735 |
} |
| 1736 |
|
| 1737 |
func (m *machine) lowerRotr(si *ssa.Instruction) { |
| 1738 |
x, y := si.Arg2() |
| 1739 |
|
| 1740 |
xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) |
| 1741 |
rn := m.getOperand_NR(xDef, extModeNone) |
| 1742 |
rm := m.getOperand_NR(yDef, extModeNone) |
| 1743 |
rd := m.compiler.VRegOf(si.Return()) |
| 1744 |
|
| 1745 |
alu := m.allocateInstr() |
| 1746 |
alu.asALU(aluOpRotR, rd, rn, rm, si.Return().Type().Bits() == 64) |
| 1747 |
m.insert(alu) |
| 1748 |
} |
| 1749 |
|
| 1750 |
func (m *machine) lowerExtend(arg, ret ssa.Value, from, to byte, signed bool) { |
| 1751 |
rd := m.compiler.VRegOf(ret) |
| 1752 |
def := m.compiler.ValueDefinition(arg) |
| 1753 |
|
| 1754 |
if instr := def.Instr; !signed && from == 32 && instr != nil { |
| 1755 |
// We can optimize out the unsigned extend because: |
| 1756 |
// Writes to the W register set bits [63:32] of the X register to zero |
| 1757 |
// https://developer.arm.com/documentation/den0024/a/An-Introduction-to-the-ARMv8-Instruction-Sets/The-ARMv8-instruction-sets/Distinguishing-between-32-bit-and-64-bit-A64-instructions |
| 1758 |
switch instr.Opcode() { |
| 1759 |
case |
| 1760 |
ssa.OpcodeIadd, ssa.OpcodeIsub, ssa.OpcodeLoad, |
| 1761 |
ssa.OpcodeBand, ssa.OpcodeBor, ssa.OpcodeBnot, |
| 1762 |
ssa.OpcodeIshl, ssa.OpcodeUshr, ssa.OpcodeSshr, |
| 1763 |
ssa.OpcodeRotl, ssa.OpcodeRotr, |
| 1764 |
ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32: |
| 1765 |
// So, if the argument is the result of a 32-bit operation, we can just copy the register. |
| 1766 |
// It is highly likely that this copy will be optimized out after register allocation. |
| 1767 |
rn := m.compiler.VRegOf(arg) |
| 1768 |
mov := m.allocateInstr() |
| 1769 |
// Note: do not use move32 as it will be lowered to a 32-bit move, which is not copy (that is actually the impl of UExtend). |
| 1770 |
mov.asMove64(rd, rn) |
| 1771 |
m.insert(mov) |
| 1772 |
return |
| 1773 |
default: |
| 1774 |
} |
| 1775 |
} |
| 1776 |
rn := m.getOperand_NR(def, extModeNone) |
| 1777 |
|
| 1778 |
ext := m.allocateInstr() |
| 1779 |
ext.asExtend(rd, rn.nr(), from, to, signed) |
| 1780 |
m.insert(ext) |
| 1781 |
} |
| 1782 |
|
| 1783 |
func (m *machine) lowerFcmp(x, y, result ssa.Value, c ssa.FloatCmpCond) { |
| 1784 |
rn, rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone), m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1785 |
|
| 1786 |
fc := m.allocateInstr() |
| 1787 |
fc.asFpuCmp(rn, rm, x.Type().Bits() == 64) |
| 1788 |
m.insert(fc) |
| 1789 |
|
| 1790 |
cset := m.allocateInstr() |
| 1791 |
cset.asCSet(m.compiler.VRegOf(result), false, condFlagFromSSAFloatCmpCond(c)) |
| 1792 |
m.insert(cset) |
| 1793 |
} |
| 1794 |
|
| 1795 |
func (m *machine) lowerImul(x, y, result ssa.Value) { |
| 1796 |
rd := m.compiler.VRegOf(result) |
| 1797 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1798 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1799 |
|
| 1800 |
// TODO: if this comes before Add/Sub, we could merge it by putting it into the place of xzrVReg. |
| 1801 |
|
| 1802 |
mul := m.allocateInstr() |
| 1803 |
mul.asALURRRR(aluOpMAdd, rd, rn, rm, xzrVReg, x.Type().Bits() == 64) |
| 1804 |
m.insert(mul) |
| 1805 |
} |
| 1806 |
|
| 1807 |
func (m *machine) lowerClz(x, result ssa.Value) { |
| 1808 |
rd := m.compiler.VRegOf(result) |
| 1809 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1810 |
clz := m.allocateInstr() |
| 1811 |
clz.asBitRR(bitOpClz, rd, rn.nr(), x.Type().Bits() == 64) |
| 1812 |
m.insert(clz) |
| 1813 |
} |
| 1814 |
|
| 1815 |
func (m *machine) lowerCtz(x, result ssa.Value) { |
| 1816 |
rd := m.compiler.VRegOf(result) |
| 1817 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1818 |
rbit := m.allocateInstr() |
| 1819 |
_64 := x.Type().Bits() == 64 |
| 1820 |
var tmpReg regalloc.VReg |
| 1821 |
if _64 { |
| 1822 |
tmpReg = m.compiler.AllocateVReg(ssa.TypeI64) |
| 1823 |
} else { |
| 1824 |
tmpReg = m.compiler.AllocateVReg(ssa.TypeI32) |
| 1825 |
} |
| 1826 |
rbit.asBitRR(bitOpRbit, tmpReg, rn.nr(), _64) |
| 1827 |
m.insert(rbit) |
| 1828 |
|
| 1829 |
clz := m.allocateInstr() |
| 1830 |
clz.asBitRR(bitOpClz, rd, tmpReg, _64) |
| 1831 |
m.insert(clz) |
| 1832 |
} |
| 1833 |
|
| 1834 |
func (m *machine) lowerPopcnt(x, result ssa.Value) { |
| 1835 |
// arm64 doesn't have an instruction for population count on scalar register, |
| 1836 |
// so we use the vector instruction `cnt`. |
| 1837 |
// This is exactly what the official Go implements bits.OneCount. |
| 1838 |
// For example, "func () int { return bits.OneCount(10) }" is compiled as |
| 1839 |
// |
| 1840 |
// MOVD $10, R0 ;; Load 10. |
| 1841 |
// FMOVD R0, F0 |
| 1842 |
// VCNT V0.B8, V0.B8 |
| 1843 |
// UADDLV V0.B8, V0 |
| 1844 |
// |
| 1845 |
// In aarch64 asm, FMOVD is encoded as `ins`, VCNT is `cnt`, |
| 1846 |
// and the registers may use different names. In our encoding we use the following |
| 1847 |
// instructions: |
| 1848 |
// |
| 1849 |
// ins v0.d[0], x0 ;; mov from GPR to vec (FMOV above) is encoded as INS |
| 1850 |
// cnt v0.16b, v0.16b ;; we use vec arrangement 16b |
| 1851 |
// uaddlv h0, v0.8b ;; h0 is still v0 with the dest width specifier 'H', implied when src arrangement is 8b |
| 1852 |
// mov x5, v0.d[0] ;; finally we mov the result back to a GPR |
| 1853 |
// |
| 1854 |
|
| 1855 |
rd := m.compiler.VRegOf(result) |
| 1856 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1857 |
|
| 1858 |
rf1 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) |
| 1859 |
ins := m.allocateInstr() |
| 1860 |
ins.asMovToVec(rf1.nr(), rn, vecArrangementD, vecIndex(0)) |
| 1861 |
m.insert(ins) |
| 1862 |
|
| 1863 |
rf2 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) |
| 1864 |
cnt := m.allocateInstr() |
| 1865 |
cnt.asVecMisc(vecOpCnt, rf2.nr(), rf1, vecArrangement16B) |
| 1866 |
m.insert(cnt) |
| 1867 |
|
| 1868 |
rf3 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) |
| 1869 |
uaddlv := m.allocateInstr() |
| 1870 |
uaddlv.asVecLanes(vecOpUaddlv, rf3.nr(), rf2, vecArrangement8B) |
| 1871 |
m.insert(uaddlv) |
| 1872 |
|
| 1873 |
mov := m.allocateInstr() |
| 1874 |
mov.asMovFromVec(rd, rf3, vecArrangementD, vecIndex(0), false) |
| 1875 |
m.insert(mov) |
| 1876 |
} |
| 1877 |
|
| 1878 |
// lowerExitWithCode lowers the lowerExitWithCode takes a context pointer as argument. |
| 1879 |
func (m *machine) lowerExitWithCode(execCtxVReg regalloc.VReg, code wazevoapi.ExitCode) { |
| 1880 |
tmpReg1 := m.compiler.AllocateVReg(ssa.TypeI32) |
| 1881 |
loadExitCodeConst := m.allocateInstr() |
| 1882 |
loadExitCodeConst.asMOVZ(tmpReg1, uint64(code), 0, true) |
| 1883 |
|
| 1884 |
setExitCode := m.allocateInstr() |
| 1885 |
mode := m.amodePool.Allocate() |
| 1886 |
*mode = addressMode{ |
| 1887 |
kind: addressModeKindRegUnsignedImm12, |
| 1888 |
rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetExitCodeOffset.I64(), |
| 1889 |
} |
| 1890 |
setExitCode.asStore(operandNR(tmpReg1), mode, 32) |
| 1891 |
|
| 1892 |
// In order to unwind the stack, we also need to push the current stack pointer: |
| 1893 |
tmp2 := m.compiler.AllocateVReg(ssa.TypeI64) |
| 1894 |
movSpToTmp := m.allocateInstr() |
| 1895 |
movSpToTmp.asMove64(tmp2, spVReg) |
| 1896 |
strSpToExecCtx := m.allocateInstr() |
| 1897 |
mode2 := m.amodePool.Allocate() |
| 1898 |
*mode2 = addressMode{ |
| 1899 |
kind: addressModeKindRegUnsignedImm12, |
| 1900 |
rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.I64(), |
| 1901 |
} |
| 1902 |
strSpToExecCtx.asStore(operandNR(tmp2), mode2, 64) |
| 1903 |
// Also the address of this exit. |
| 1904 |
tmp3 := m.compiler.AllocateVReg(ssa.TypeI64) |
| 1905 |
currentAddrToTmp := m.allocateInstr() |
| 1906 |
currentAddrToTmp.asAdr(tmp3, 0) |
| 1907 |
storeCurrentAddrToExecCtx := m.allocateInstr() |
| 1908 |
mode3 := m.amodePool.Allocate() |
| 1909 |
*mode3 = addressMode{ |
| 1910 |
kind: addressModeKindRegUnsignedImm12, |
| 1911 |
rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetGoCallReturnAddress.I64(), |
| 1912 |
} |
| 1913 |
storeCurrentAddrToExecCtx.asStore(operandNR(tmp3), mode3, 64) |
| 1914 |
|
| 1915 |
exitSeq := m.allocateInstr() |
| 1916 |
exitSeq.asExitSequence(execCtxVReg) |
| 1917 |
|
| 1918 |
m.insert(loadExitCodeConst) |
| 1919 |
m.insert(setExitCode) |
| 1920 |
m.insert(movSpToTmp) |
| 1921 |
m.insert(strSpToExecCtx) |
| 1922 |
m.insert(currentAddrToTmp) |
| 1923 |
m.insert(storeCurrentAddrToExecCtx) |
| 1924 |
m.insert(exitSeq) |
| 1925 |
} |
| 1926 |
|
| 1927 |
func (m *machine) lowerIcmpToFlag(x, y ssa.Value, signed bool) { |
| 1928 |
if x.Type() != y.Type() { |
| 1929 |
panic( |
| 1930 |
fmt.Sprintf("TODO(maybe): support icmp with different types: v%d=%s != v%d=%s", |
| 1931 |
x.ID(), x.Type(), y.ID(), y.Type())) |
| 1932 |
} |
| 1933 |
|
| 1934 |
extMod := extModeOf(x.Type(), signed) |
| 1935 |
|
| 1936 |
// First operand must be in pure register form. |
| 1937 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extMod) |
| 1938 |
// Second operand can be in any of Imm12, ER, SR, or NR form supported by the SUBS instructions. |
| 1939 |
rm := m.getOperand_Imm12_ER_SR_NR(m.compiler.ValueDefinition(y), extMod) |
| 1940 |
|
| 1941 |
alu := m.allocateInstr() |
| 1942 |
// subs zr, rn, rm |
| 1943 |
alu.asALU( |
| 1944 |
aluOpSubS, |
| 1945 |
// We don't need the result, just need to set flags. |
| 1946 |
xzrVReg, |
| 1947 |
rn, |
| 1948 |
rm, |
| 1949 |
x.Type().Bits() == 64, |
| 1950 |
) |
| 1951 |
m.insert(alu) |
| 1952 |
} |
| 1953 |
|
| 1954 |
func (m *machine) lowerFcmpToFlag(x, y ssa.Value) { |
| 1955 |
if x.Type() != y.Type() { |
| 1956 |
panic("TODO(maybe): support icmp with different types") |
| 1957 |
} |
| 1958 |
|
| 1959 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 1960 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 1961 |
cmp := m.allocateInstr() |
| 1962 |
cmp.asFpuCmp(rn, rm, x.Type().Bits() == 64) |
| 1963 |
m.insert(cmp) |
| 1964 |
} |
| 1965 |
|
| 1966 |
func (m *machine) lowerExitIfTrueWithCode(execCtxVReg regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) { |
| 1967 |
condDef := m.compiler.ValueDefinition(cond) |
| 1968 |
if !m.compiler.MatchInstr(condDef, ssa.OpcodeIcmp) { |
| 1969 |
panic("TODO: OpcodeExitIfTrueWithCode must come after Icmp at the moment: " + condDef.Instr.Opcode().String()) |
| 1970 |
} |
| 1971 |
condDef.Instr.MarkLowered() |
| 1972 |
|
| 1973 |
cvalInstr := condDef.Instr |
| 1974 |
x, y, c := cvalInstr.IcmpData() |
| 1975 |
signed := c.Signed() |
| 1976 |
|
| 1977 |
if !m.tryLowerBandToFlag(x, y) { |
| 1978 |
m.lowerIcmpToFlag(x, y, signed) |
| 1979 |
} |
| 1980 |
|
| 1981 |
// We need to copy the execution context to a temp register, because if it's spilled, |
| 1982 |
// it might end up being reloaded inside the exiting branch. |
| 1983 |
execCtxTmp := m.copyToTmp(execCtxVReg) |
| 1984 |
|
| 1985 |
// We have to skip the entire exit sequence if the condition is false. |
| 1986 |
cbr := m.allocateInstr() |
| 1987 |
m.insert(cbr) |
| 1988 |
m.lowerExitWithCode(execCtxTmp, code) |
| 1989 |
// conditional branch target is after exit. |
| 1990 |
l := m.insertBrTargetLabel() |
| 1991 |
cbr.asCondBr(condFlagFromSSAIntegerCmpCond(c).invert().asCond(), l, false /* ignored */) |
| 1992 |
} |
| 1993 |
|
| 1994 |
func (m *machine) lowerSelect(c, x, y, result ssa.Value) { |
| 1995 |
cvalDef := m.compiler.ValueDefinition(c) |
| 1996 |
|
| 1997 |
var cc condFlag |
| 1998 |
switch { |
| 1999 |
case m.compiler.MatchInstr(cvalDef, ssa.OpcodeIcmp): // This case, we can use the ALU flag set by SUBS instruction. |
| 2000 |
cvalInstr := cvalDef.Instr |
| 2001 |
x, y, c := cvalInstr.IcmpData() |
| 2002 |
cc = condFlagFromSSAIntegerCmpCond(c) |
| 2003 |
m.lowerIcmpToFlag(x, y, c.Signed()) |
| 2004 |
cvalDef.Instr.MarkLowered() |
| 2005 |
case m.compiler.MatchInstr(cvalDef, ssa.OpcodeFcmp): // This case we can use the Fpu flag directly. |
| 2006 |
cvalInstr := cvalDef.Instr |
| 2007 |
x, y, c := cvalInstr.FcmpData() |
| 2008 |
cc = condFlagFromSSAFloatCmpCond(c) |
| 2009 |
m.lowerFcmpToFlag(x, y) |
| 2010 |
cvalDef.Instr.MarkLowered() |
| 2011 |
default: |
| 2012 |
rn := m.getOperand_NR(cvalDef, extModeNone) |
| 2013 |
if c.Type() != ssa.TypeI32 && c.Type() != ssa.TypeI64 { |
| 2014 |
panic("TODO?BUG?: support select with non-integer condition") |
| 2015 |
} |
| 2016 |
alu := m.allocateInstr() |
| 2017 |
// subs zr, rn, zr |
| 2018 |
alu.asALU( |
| 2019 |
aluOpSubS, |
| 2020 |
// We don't need the result, just need to set flags. |
| 2021 |
xzrVReg, |
| 2022 |
rn, |
| 2023 |
operandNR(xzrVReg), |
| 2024 |
c.Type().Bits() == 64, |
| 2025 |
) |
| 2026 |
m.insert(alu) |
| 2027 |
cc = ne |
| 2028 |
} |
| 2029 |
|
| 2030 |
rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) |
| 2031 |
rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) |
| 2032 |
|
| 2033 |
rd := m.compiler.VRegOf(result) |
| 2034 |
switch x.Type() { |
| 2035 |
case ssa.TypeI32, ssa.TypeI64: |
| 2036 |
// csel rd, rn, rm, cc |
| 2037 |
csel := m.allocateInstr() |
| 2038 |
csel.asCSel(rd, rn, rm, cc, x.Type().Bits() == 64) |
| 2039 |
m.insert(csel) |
| 2040 |
case ssa.TypeF32, ssa.TypeF64: |
| 2041 |
// fcsel rd, rn, rm, cc |
| 2042 |
fcsel := m.allocateInstr() |
| 2043 |
fcsel.asFpuCSel(rd, rn, rm, cc, x.Type().Bits() == 64) |
| 2044 |
m.insert(fcsel) |
| 2045 |
default: |
| 2046 |
panic("BUG") |
| 2047 |
} |
| 2048 |
} |
| 2049 |
|
| 2050 |
func (m *machine) lowerSelectVec(rc, rn, rm operand, rd regalloc.VReg) { |
| 2051 |
// First check if `rc` is zero or not. |
| 2052 |
checkZero := m.allocateInstr() |
| 2053 |
checkZero.asALU(aluOpSubS, xzrVReg, rc, operandNR(xzrVReg), false) |
| 2054 |
m.insert(checkZero) |
| 2055 |
|
| 2056 |
// Then use CSETM to set all bits to one if `rc` is zero. |
| 2057 |
allOnesOrZero := m.compiler.AllocateVReg(ssa.TypeI64) |
| 2058 |
cset := m.allocateInstr() |
| 2059 |
cset.asCSet(allOnesOrZero, true, ne) |
| 2060 |
m.insert(cset) |
| 2061 |
|
| 2062 |
// Then move the bits to the result vector register. |
| 2063 |
tmp2 := m.compiler.AllocateVReg(ssa.TypeV128) |
| 2064 |
dup := m.allocateInstr() |
| 2065 |
dup.asVecDup(tmp2, operandNR(allOnesOrZero), vecArrangement2D) |
| 2066 |
m.insert(dup) |
| 2067 |
|
| 2068 |
// Now that `tmp2` has either all bits one or zero depending on `rc`, |
| 2069 |
// we can use bsl to select between `rn` and `rm`. |
| 2070 |
ins := m.allocateInstr() |
| 2071 |
ins.asVecRRRRewrite(vecOpBsl, tmp2, rn, rm, vecArrangement16B) |
| 2072 |
m.insert(ins) |
| 2073 |
|
| 2074 |
// Finally, move the result to the destination register. |
| 2075 |
mov2 := m.allocateInstr() |
| 2076 |
mov2.asFpuMov128(rd, tmp2) |
| 2077 |
m.insert(mov2) |
| 2078 |
} |
| 2079 |
|
| 2080 |
func (m *machine) lowerAtomicRmw(si *ssa.Instruction) { |
| 2081 |
ssaOp, size := si.AtomicRmwData() |
| 2082 |
|
| 2083 |
var op atomicRmwOp |
| 2084 |
var negateArg bool |
| 2085 |
var flipArg bool |
| 2086 |
switch ssaOp { |
| 2087 |
case ssa.AtomicRmwOpAdd: |
| 2088 |
op = atomicRmwOpAdd |
| 2089 |
case ssa.AtomicRmwOpSub: |
| 2090 |
op = atomicRmwOpAdd |
| 2091 |
negateArg = true |
| 2092 |
case ssa.AtomicRmwOpAnd: |
| 2093 |
op = atomicRmwOpClr |
| 2094 |
flipArg = true |
| 2095 |
case ssa.AtomicRmwOpOr: |
| 2096 |
op = atomicRmwOpSet |
| 2097 |
case ssa.AtomicRmwOpXor: |
| 2098 |
op = atomicRmwOpEor |
| 2099 |
case ssa.AtomicRmwOpXchg: |
| 2100 |
op = atomicRmwOpSwp |
| 2101 |
default: |
| 2102 |
panic(fmt.Sprintf("unknown ssa atomic rmw op: %s", ssaOp)) |
| 2103 |
} |
| 2104 |
|
| 2105 |
addr, val := si.Arg2() |
| 2106 |
addrDef, valDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(val) |
| 2107 |
rn := m.getOperand_NR(addrDef, extModeNone) |
| 2108 |
rt := m.compiler.VRegOf(si.Return()) |
| 2109 |
rs := m.getOperand_NR(valDef, extModeNone) |
| 2110 |
|
| 2111 |
_64 := si.Return().Type().Bits() == 64 |
| 2112 |
var tmp regalloc.VReg |
| 2113 |
if _64 { |
| 2114 |
tmp = m.compiler.AllocateVReg(ssa.TypeI64) |
| 2115 |
} else { |
| 2116 |
tmp = m.compiler.AllocateVReg(ssa.TypeI32) |
| 2117 |
} |
| 2118 |
m.lowerAtomicRmwImpl(op, rn.nr(), rs.nr(), rt, tmp, size, negateArg, flipArg, _64) |
| 2119 |
} |
| 2120 |
|
| 2121 |
func (m *machine) lowerAtomicRmwImpl(op atomicRmwOp, rn, rs, rt, tmp regalloc.VReg, size uint64, negateArg, flipArg, dst64bit bool) { |
| 2122 |
switch { |
| 2123 |
case negateArg: |
| 2124 |
neg := m.allocateInstr() |
| 2125 |
neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), operandNR(rs), dst64bit) |
| 2126 |
m.insert(neg) |
| 2127 |
case flipArg: |
| 2128 |
flip := m.allocateInstr() |
| 2129 |
flip.asALU(aluOpOrn, tmp, operandNR(xzrVReg), operandNR(rs), dst64bit) |
| 2130 |
m.insert(flip) |
| 2131 |
default: |
| 2132 |
tmp = rs |
| 2133 |
} |
| 2134 |
|
| 2135 |
rmw := m.allocateInstr() |
| 2136 |
rmw.asAtomicRmw(op, rn, tmp, rt, size) |
| 2137 |
m.insert(rmw) |
| 2138 |
} |
| 2139 |
|
| 2140 |
func (m *machine) lowerAtomicCas(si *ssa.Instruction) { |
| 2141 |
addr, exp, repl := si.Arg3() |
| 2142 |
size := si.AtomicTargetSize() |
| 2143 |
|
| 2144 |
addrDef, expDef, replDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(exp), m.compiler.ValueDefinition(repl) |
| 2145 |
rn := m.getOperand_NR(addrDef, extModeNone) |
| 2146 |
rt := m.getOperand_NR(replDef, extModeNone) |
| 2147 |
rs := m.getOperand_NR(expDef, extModeNone) |
| 2148 |
tmp := m.compiler.AllocateVReg(si.Return().Type()) |
| 2149 |
|
| 2150 |
_64 := si.Return().Type().Bits() == 64 |
| 2151 |
// rs is overwritten by CAS, so we need to move it to the result register before the instruction |
| 2152 |
// in case when it is used somewhere else. |
| 2153 |
mov := m.allocateInstr() |
| 2154 |
if _64 { |
| 2155 |
mov.asMove64(tmp, rs.nr()) |
| 2156 |
} else { |
| 2157 |
mov.asMove32(tmp, rs.nr()) |
| 2158 |
} |
| 2159 |
m.insert(mov) |
| 2160 |
|
| 2161 |
m.lowerAtomicCasImpl(rn.nr(), tmp, rt.nr(), size) |
| 2162 |
|
| 2163 |
mov2 := m.allocateInstr() |
| 2164 |
rd := m.compiler.VRegOf(si.Return()) |
| 2165 |
if _64 { |
| 2166 |
mov2.asMove64(rd, tmp) |
| 2167 |
} else { |
| 2168 |
mov2.asMove32(rd, tmp) |
| 2169 |
} |
| 2170 |
m.insert(mov2) |
| 2171 |
} |
| 2172 |
|
| 2173 |
func (m *machine) lowerAtomicCasImpl(rn, rs, rt regalloc.VReg, size uint64) { |
| 2174 |
cas := m.allocateInstr() |
| 2175 |
cas.asAtomicCas(rn, rs, rt, size) |
| 2176 |
m.insert(cas) |
| 2177 |
} |
| 2178 |
|
| 2179 |
func (m *machine) lowerAtomicLoad(si *ssa.Instruction) { |
| 2180 |
addr := si.Arg() |
| 2181 |
size := si.AtomicTargetSize() |
| 2182 |
|
| 2183 |
addrDef := m.compiler.ValueDefinition(addr) |
| 2184 |
rn := m.getOperand_NR(addrDef, extModeNone) |
| 2185 |
rt := m.compiler.VRegOf(si.Return()) |
| 2186 |
|
| 2187 |
m.lowerAtomicLoadImpl(rn.nr(), rt, size) |
| 2188 |
} |
| 2189 |
|
| 2190 |
func (m *machine) lowerAtomicLoadImpl(rn, rt regalloc.VReg, size uint64) { |
| 2191 |
ld := m.allocateInstr() |
| 2192 |
ld.asAtomicLoad(rn, rt, size) |
| 2193 |
m.insert(ld) |
| 2194 |
} |
| 2195 |
|
| 2196 |
func (m *machine) lowerAtomicStore(si *ssa.Instruction) { |
| 2197 |
addr, val := si.Arg2() |
| 2198 |
size := si.AtomicTargetSize() |
| 2199 |
|
| 2200 |
addrDef := m.compiler.ValueDefinition(addr) |
| 2201 |
valDef := m.compiler.ValueDefinition(val) |
| 2202 |
rn := m.getOperand_NR(addrDef, extModeNone) |
| 2203 |
rt := m.getOperand_NR(valDef, extModeNone) |
| 2204 |
|
| 2205 |
m.lowerAtomicStoreImpl(rn, rt, size) |
| 2206 |
} |
| 2207 |
|
| 2208 |
func (m *machine) lowerAtomicStoreImpl(rn, rt operand, size uint64) { |
| 2209 |
ld := m.allocateInstr() |
| 2210 |
ld.asAtomicStore(rn, rt, size) |
| 2211 |
m.insert(ld) |
| 2212 |
} |
| 2213 |
|
| 2214 |
// copyToTmp copies the given regalloc.VReg to a temporary register. This is called before cbr to avoid the regalloc issue |
| 2215 |
// e.g. reload happening in the middle of the exit sequence which is not the path the normal path executes |
| 2216 |
func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg { |
| 2217 |
typ := m.compiler.TypeOf(v) |
| 2218 |
mov := m.allocateInstr() |
| 2219 |
tmp := m.compiler.AllocateVReg(typ) |
| 2220 |
if typ.IsInt() { |
| 2221 |
mov.asMove64(tmp, v) |
| 2222 |
} else { |
| 2223 |
mov.asFpuMov128(tmp, v) |
| 2224 |
} |
| 2225 |
m.insert(mov) |
| 2226 |
return tmp |
| 2227 |
} |