Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/lower_instr.go

raw
1 package arm64
2
3 // Files prefixed as lower_instr** do the instruction selection, meaning that lowering SSA level instructions
4 // into machine specific instructions.
5 //
6 // Importantly, what the lower** functions does includes tree-matching; find the pattern from the given instruction tree,
7 // and merge the multiple instructions if possible. It can be considered as "N:1" instruction selection.
8
9 import (
10 "fmt"
11 "math"
12
13 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
14 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
15 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
16 )
17
18 // LowerSingleBranch implements backend.Machine.
19 func (m *machine) LowerSingleBranch(br *ssa.Instruction) {
20 switch br.Opcode() {
21 case ssa.OpcodeJump:
22 _, _, targetBlkID := br.BranchData()
23 if br.IsFallthroughJump() {
24 return
25 }
26 b := m.allocateInstr()
27 targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID)
28 if targetBlk.ReturnBlock() {
29 b.asRet()
30 } else {
31 b.asBr(ssaBlockLabel(targetBlk))
32 }
33 m.insert(b)
34 case ssa.OpcodeBrTable:
35 m.lowerBrTable(br)
36 default:
37 panic("BUG: unexpected branch opcode" + br.Opcode().String())
38 }
39 }
40
41 func (m *machine) lowerBrTable(i *ssa.Instruction) {
42 index, targetBlockIDs := i.BrTableData()
43 targetBlockCount := len(targetBlockIDs.View())
44 indexOperand := m.getOperand_NR(m.compiler.ValueDefinition(index), extModeNone)
45
46 // Firstly, we have to do the bounds check of the index, and
47 // set it to the default target (sitting at the end of the list) if it's out of bounds.
48
49 // mov maxIndexReg #maximum_index
50 // subs wzr, index, maxIndexReg
51 // csel adjustedIndex, maxIndexReg, index, hs ;; if index is higher or equal than maxIndexReg.
52 maxIndexReg := m.compiler.AllocateVReg(ssa.TypeI32)
53 m.lowerConstantI32(maxIndexReg, int32(targetBlockCount-1))
54 subs := m.allocateInstr()
55 subs.asALU(aluOpSubS, xzrVReg, indexOperand, operandNR(maxIndexReg), false)
56 m.insert(subs)
57 csel := m.allocateInstr()
58 adjustedIndex := m.compiler.AllocateVReg(ssa.TypeI32)
59 csel.asCSel(adjustedIndex, operandNR(maxIndexReg), indexOperand, hs, false)
60 m.insert(csel)
61
62 brSequence := m.allocateInstr()
63
64 tableIndex := m.addJmpTableTarget(targetBlockIDs)
65 brSequence.asBrTableSequence(adjustedIndex, tableIndex, targetBlockCount)
66 m.insert(brSequence)
67 }
68
69 // LowerConditionalBranch implements backend.Machine.
70 func (m *machine) LowerConditionalBranch(b *ssa.Instruction) {
71 cval, args, targetBlkID := b.BranchData()
72 if len(args) > 0 {
73 panic(fmt.Sprintf(
74 "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s",
75 m.currentLabelPos.sb,
76 targetBlkID,
77 ))
78 }
79
80 targetBlk := m.compiler.SSABuilder().BasicBlock(targetBlkID)
81 target := ssaBlockLabel(targetBlk)
82 cvalDef := m.compiler.ValueDefinition(cval)
83
84 switch {
85 case m.compiler.MatchInstr(cvalDef, ssa.OpcodeIcmp): // This case, we can use the ALU flag set by SUBS instruction.
86 cvalInstr := cvalDef.Instr
87 x, y, c := cvalInstr.IcmpData()
88 cc, signed := condFlagFromSSAIntegerCmpCond(c), c.Signed()
89 if b.Opcode() == ssa.OpcodeBrz {
90 cc = cc.invert()
91 }
92
93 if !m.tryLowerBandToFlag(x, y) {
94 m.lowerIcmpToFlag(x, y, signed)
95 }
96 cbr := m.allocateInstr()
97 cbr.asCondBr(cc.asCond(), target, false /* ignored */)
98 m.insert(cbr)
99 cvalDef.Instr.MarkLowered()
100 case m.compiler.MatchInstr(cvalDef, ssa.OpcodeFcmp): // This case we can use the Fpu flag directly.
101 cvalInstr := cvalDef.Instr
102 x, y, c := cvalInstr.FcmpData()
103 cc := condFlagFromSSAFloatCmpCond(c)
104 if b.Opcode() == ssa.OpcodeBrz {
105 cc = cc.invert()
106 }
107 m.lowerFcmpToFlag(x, y)
108 cbr := m.allocateInstr()
109 cbr.asCondBr(cc.asCond(), target, false /* ignored */)
110 m.insert(cbr)
111 cvalDef.Instr.MarkLowered()
112 default:
113 rn := m.getOperand_NR(cvalDef, extModeNone)
114 var c cond
115 if b.Opcode() == ssa.OpcodeBrz {
116 c = registerAsRegZeroCond(rn.nr())
117 } else {
118 c = registerAsRegNotZeroCond(rn.nr())
119 }
120 cbr := m.allocateInstr()
121 cbr.asCondBr(c, target, false)
122 m.insert(cbr)
123 }
124 }
125
126 func (m *machine) tryLowerBandToFlag(x, y ssa.Value) (ok bool) {
127 xx := m.compiler.ValueDefinition(x)
128 yy := m.compiler.ValueDefinition(y)
129 if xx.IsFromInstr() && xx.Instr.Constant() && xx.Instr.ConstantVal() == 0 {
130 if m.compiler.MatchInstr(yy, ssa.OpcodeBand) {
131 bandInstr := yy.Instr
132 m.lowerBitwiseAluOp(bandInstr, aluOpAnds, true)
133 ok = true
134 bandInstr.MarkLowered()
135 return
136 }
137 }
138
139 if yy.IsFromInstr() && yy.Instr.Constant() && yy.Instr.ConstantVal() == 0 {
140 if m.compiler.MatchInstr(xx, ssa.OpcodeBand) {
141 bandInstr := xx.Instr
142 m.lowerBitwiseAluOp(bandInstr, aluOpAnds, true)
143 ok = true
144 bandInstr.MarkLowered()
145 return
146 }
147 }
148 return
149 }
150
151 // LowerInstr implements backend.Machine.
152 func (m *machine) LowerInstr(instr *ssa.Instruction) {
153 if l := instr.SourceOffset(); l.Valid() {
154 info := m.allocateInstr().asEmitSourceOffsetInfo(l)
155 m.insert(info)
156 }
157
158 switch op := instr.Opcode(); op {
159 case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable:
160 panic("BUG: branching instructions are handled by LowerBranches")
161 case ssa.OpcodeReturn:
162 panic("BUG: return must be handled by backend.Compiler")
163 case ssa.OpcodeIadd, ssa.OpcodeIsub:
164 m.lowerSubOrAdd(instr, op == ssa.OpcodeIadd)
165 case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv, ssa.OpcodeFmax, ssa.OpcodeFmin:
166 m.lowerFpuBinOp(instr)
167 case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined.
168 case ssa.OpcodeExitWithCode:
169 execCtx, code := instr.ExitWithCodeData()
170 m.lowerExitWithCode(m.compiler.VRegOf(execCtx), code)
171 case ssa.OpcodeExitIfTrueWithCode:
172 execCtx, c, code := instr.ExitIfTrueWithCodeData()
173 m.lowerExitIfTrueWithCode(m.compiler.VRegOf(execCtx), c, code)
174 case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32:
175 m.lowerStore(instr)
176 case ssa.OpcodeLoad:
177 dst := instr.Return()
178 ptr, offset, typ := instr.LoadData()
179 m.lowerLoad(ptr, offset, typ, dst)
180 case ssa.OpcodeVZeroExtLoad:
181 dst := instr.Return()
182 ptr, offset, typ := instr.VZeroExtLoadData()
183 m.lowerLoad(ptr, offset, typ, dst)
184 case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32:
185 ptr, offset, _ := instr.LoadData()
186 ret := m.compiler.VRegOf(instr.Return())
187 m.lowerExtLoad(op, ptr, offset, ret)
188 case ssa.OpcodeCall, ssa.OpcodeCallIndirect:
189 m.lowerCall(instr)
190 case ssa.OpcodeIcmp:
191 m.lowerIcmp(instr)
192 case ssa.OpcodeVIcmp:
193 m.lowerVIcmp(instr)
194 case ssa.OpcodeVFcmp:
195 m.lowerVFcmp(instr)
196 case ssa.OpcodeVCeil:
197 m.lowerVecMisc(vecOpFrintp, instr)
198 case ssa.OpcodeVFloor:
199 m.lowerVecMisc(vecOpFrintm, instr)
200 case ssa.OpcodeVTrunc:
201 m.lowerVecMisc(vecOpFrintz, instr)
202 case ssa.OpcodeVNearest:
203 m.lowerVecMisc(vecOpFrintn, instr)
204 case ssa.OpcodeVMaxPseudo:
205 m.lowerVMinMaxPseudo(instr, true)
206 case ssa.OpcodeVMinPseudo:
207 m.lowerVMinMaxPseudo(instr, false)
208 case ssa.OpcodeBand:
209 m.lowerBitwiseAluOp(instr, aluOpAnd, false)
210 case ssa.OpcodeBor:
211 m.lowerBitwiseAluOp(instr, aluOpOrr, false)
212 case ssa.OpcodeBxor:
213 m.lowerBitwiseAluOp(instr, aluOpEor, false)
214 case ssa.OpcodeIshl:
215 m.lowerShifts(instr, extModeNone, aluOpLsl)
216 case ssa.OpcodeSshr:
217 if instr.Return().Type().Bits() == 64 {
218 m.lowerShifts(instr, extModeSignExtend64, aluOpAsr)
219 } else {
220 m.lowerShifts(instr, extModeSignExtend32, aluOpAsr)
221 }
222 case ssa.OpcodeUshr:
223 if instr.Return().Type().Bits() == 64 {
224 m.lowerShifts(instr, extModeZeroExtend64, aluOpLsr)
225 } else {
226 m.lowerShifts(instr, extModeZeroExtend32, aluOpLsr)
227 }
228 case ssa.OpcodeRotl:
229 m.lowerRotl(instr)
230 case ssa.OpcodeRotr:
231 m.lowerRotr(instr)
232 case ssa.OpcodeSExtend, ssa.OpcodeUExtend:
233 from, to, signed := instr.ExtendData()
234 m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed)
235 case ssa.OpcodeFcmp:
236 x, y, c := instr.FcmpData()
237 m.lowerFcmp(x, y, instr.Return(), c)
238 case ssa.OpcodeImul:
239 x, y := instr.Arg2()
240 result := instr.Return()
241 m.lowerImul(x, y, result)
242 case ssa.OpcodeUndefined:
243 undef := m.allocateInstr()
244 undef.asUDF()
245 m.insert(undef)
246 case ssa.OpcodeSelect:
247 c, x, y := instr.SelectData()
248 if x.Type() == ssa.TypeV128 {
249 rc := m.getOperand_NR(m.compiler.ValueDefinition(c), extModeNone)
250 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
251 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
252 rd := m.compiler.VRegOf(instr.Return())
253 m.lowerSelectVec(rc, rn, rm, rd)
254 } else {
255 m.lowerSelect(c, x, y, instr.Return())
256 }
257 case ssa.OpcodeClz:
258 x := instr.Arg()
259 result := instr.Return()
260 m.lowerClz(x, result)
261 case ssa.OpcodeCtz:
262 x := instr.Arg()
263 result := instr.Return()
264 m.lowerCtz(x, result)
265 case ssa.OpcodePopcnt:
266 x := instr.Arg()
267 result := instr.Return()
268 m.lowerPopcnt(x, result)
269 case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat:
270 x, ctx := instr.Arg2()
271 result := instr.Return()
272 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
273 rd := m.compiler.VRegOf(result)
274 ctxVReg := m.compiler.VRegOf(ctx)
275 m.lowerFpuToInt(rd, rn, ctxVReg, true, x.Type() == ssa.TypeF64,
276 result.Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat)
277 case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat:
278 x, ctx := instr.Arg2()
279 result := instr.Return()
280 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
281 rd := m.compiler.VRegOf(result)
282 ctxVReg := m.compiler.VRegOf(ctx)
283 m.lowerFpuToInt(rd, rn, ctxVReg, false, x.Type() == ssa.TypeF64,
284 result.Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat)
285 case ssa.OpcodeFcvtFromSint:
286 x := instr.Arg()
287 result := instr.Return()
288 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
289 rd := m.compiler.VRegOf(result)
290 m.lowerIntToFpu(rd, rn, true, x.Type() == ssa.TypeI64, result.Type().Bits() == 64)
291 case ssa.OpcodeFcvtFromUint:
292 x := instr.Arg()
293 result := instr.Return()
294 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
295 rd := m.compiler.VRegOf(result)
296 m.lowerIntToFpu(rd, rn, false, x.Type() == ssa.TypeI64, result.Type().Bits() == 64)
297 case ssa.OpcodeFdemote:
298 v := instr.Arg()
299 rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone)
300 rd := m.compiler.VRegOf(instr.Return())
301 cnt := m.allocateInstr()
302 cnt.asFpuRR(fpuUniOpCvt64To32, rd, rn, false)
303 m.insert(cnt)
304 case ssa.OpcodeFpromote:
305 v := instr.Arg()
306 rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone)
307 rd := m.compiler.VRegOf(instr.Return())
308 cnt := m.allocateInstr()
309 cnt.asFpuRR(fpuUniOpCvt32To64, rd, rn, true)
310 m.insert(cnt)
311 case ssa.OpcodeIreduce:
312 rn := m.getOperand_NR(m.compiler.ValueDefinition(instr.Arg()), extModeNone)
313 retVal := instr.Return()
314 rd := m.compiler.VRegOf(retVal)
315
316 if retVal.Type() != ssa.TypeI32 {
317 panic("TODO?: Ireduce to non-i32")
318 }
319 mov := m.allocateInstr()
320 mov.asMove32(rd, rn.reg())
321 m.insert(mov)
322 case ssa.OpcodeFneg:
323 m.lowerFpuUniOp(fpuUniOpNeg, instr.Arg(), instr.Return())
324 case ssa.OpcodeSqrt:
325 m.lowerFpuUniOp(fpuUniOpSqrt, instr.Arg(), instr.Return())
326 case ssa.OpcodeCeil:
327 m.lowerFpuUniOp(fpuUniOpRoundPlus, instr.Arg(), instr.Return())
328 case ssa.OpcodeFloor:
329 m.lowerFpuUniOp(fpuUniOpRoundMinus, instr.Arg(), instr.Return())
330 case ssa.OpcodeTrunc:
331 m.lowerFpuUniOp(fpuUniOpRoundZero, instr.Arg(), instr.Return())
332 case ssa.OpcodeNearest:
333 m.lowerFpuUniOp(fpuUniOpRoundNearest, instr.Arg(), instr.Return())
334 case ssa.OpcodeFabs:
335 m.lowerFpuUniOp(fpuUniOpAbs, instr.Arg(), instr.Return())
336 case ssa.OpcodeBitcast:
337 m.lowerBitcast(instr)
338 case ssa.OpcodeFcopysign:
339 x, y := instr.Arg2()
340 m.lowerFcopysign(x, y, instr.Return())
341 case ssa.OpcodeSdiv, ssa.OpcodeUdiv:
342 x, y, ctx := instr.Arg3()
343 ctxVReg := m.compiler.VRegOf(ctx)
344 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
345 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
346 rd := m.compiler.VRegOf(instr.Return())
347 m.lowerIDiv(ctxVReg, rd, rn, rm, x.Type() == ssa.TypeI64, op == ssa.OpcodeSdiv)
348 case ssa.OpcodeSrem, ssa.OpcodeUrem:
349 x, y, ctx := instr.Arg3()
350 ctxVReg := m.compiler.VRegOf(ctx)
351 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
352 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
353 rd := m.compiler.VRegOf(instr.Return())
354 m.lowerIRem(ctxVReg, rd, rn.nr(), rm, x.Type() == ssa.TypeI64, op == ssa.OpcodeSrem)
355 case ssa.OpcodeVconst:
356 result := m.compiler.VRegOf(instr.Return())
357 lo, hi := instr.VconstData()
358 v := m.allocateInstr()
359 v.asLoadFpuConst128(result, lo, hi)
360 m.insert(v)
361 case ssa.OpcodeVbnot:
362 x := instr.Arg()
363 ins := m.allocateInstr()
364 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
365 rd := m.compiler.VRegOf(instr.Return())
366 ins.asVecMisc(vecOpNot, rd, rn, vecArrangement16B)
367 m.insert(ins)
368 case ssa.OpcodeVbxor:
369 x, y := instr.Arg2()
370 m.lowerVecRRR(vecOpEOR, x, y, instr.Return(), vecArrangement16B)
371 case ssa.OpcodeVbor:
372 x, y := instr.Arg2()
373 m.lowerVecRRR(vecOpOrr, x, y, instr.Return(), vecArrangement16B)
374 case ssa.OpcodeVband:
375 x, y := instr.Arg2()
376 m.lowerVecRRR(vecOpAnd, x, y, instr.Return(), vecArrangement16B)
377 case ssa.OpcodeVbandnot:
378 x, y := instr.Arg2()
379 m.lowerVecRRR(vecOpBic, x, y, instr.Return(), vecArrangement16B)
380 case ssa.OpcodeVbitselect:
381 c, x, y := instr.SelectData()
382 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
383 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
384 creg := m.getOperand_NR(m.compiler.ValueDefinition(c), extModeNone)
385 tmp := m.compiler.AllocateVReg(ssa.TypeV128)
386
387 // creg is overwritten by BSL, so we need to move it to the result register before the instruction
388 // in case when it is used somewhere else.
389 mov := m.allocateInstr()
390 mov.asFpuMov128(tmp, creg.nr())
391 m.insert(mov)
392
393 ins := m.allocateInstr()
394 ins.asVecRRRRewrite(vecOpBsl, tmp, rn, rm, vecArrangement16B)
395 m.insert(ins)
396
397 mov2 := m.allocateInstr()
398 rd := m.compiler.VRegOf(instr.Return())
399 mov2.asFpuMov128(rd, tmp)
400 m.insert(mov2)
401 case ssa.OpcodeVanyTrue, ssa.OpcodeVallTrue:
402 x, lane := instr.ArgWithLane()
403 var arr vecArrangement
404 if op == ssa.OpcodeVallTrue {
405 arr = ssaLaneToArrangement(lane)
406 }
407 rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
408 rd := m.compiler.VRegOf(instr.Return())
409 m.lowerVcheckTrue(op, rm, rd, arr)
410 case ssa.OpcodeVhighBits:
411 x, lane := instr.ArgWithLane()
412 rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
413 rd := m.compiler.VRegOf(instr.Return())
414 arr := ssaLaneToArrangement(lane)
415 m.lowerVhighBits(rm, rd, arr)
416 case ssa.OpcodeVIadd:
417 x, y, lane := instr.Arg2WithLane()
418 arr := ssaLaneToArrangement(lane)
419 m.lowerVecRRR(vecOpAdd, x, y, instr.Return(), arr)
420 case ssa.OpcodeExtIaddPairwise:
421 v, lane, signed := instr.ExtIaddPairwiseData()
422 vv := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone)
423
424 tmpLo, tmpHi := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
425 var widen vecOp
426 if signed {
427 widen = vecOpSshll
428 } else {
429 widen = vecOpUshll
430 }
431
432 var loArr, hiArr, dstArr vecArrangement
433 switch lane {
434 case ssa.VecLaneI8x16:
435 loArr, hiArr, dstArr = vecArrangement8B, vecArrangement16B, vecArrangement8H
436 case ssa.VecLaneI16x8:
437 loArr, hiArr, dstArr = vecArrangement4H, vecArrangement8H, vecArrangement4S
438 case ssa.VecLaneI32x4:
439 loArr, hiArr, dstArr = vecArrangement2S, vecArrangement4S, vecArrangement2D
440 default:
441 panic("unsupported lane " + lane.String())
442 }
443
444 widenLo := m.allocateInstr().asVecShiftImm(widen, tmpLo.nr(), vv, operandShiftImm(0), loArr)
445 widenHi := m.allocateInstr().asVecShiftImm(widen, tmpHi.nr(), vv, operandShiftImm(0), hiArr)
446 addp := m.allocateInstr().asVecRRR(vecOpAddp, m.compiler.VRegOf(instr.Return()), tmpLo, tmpHi, dstArr)
447 m.insert(widenLo)
448 m.insert(widenHi)
449 m.insert(addp)
450
451 case ssa.OpcodeVSaddSat:
452 x, y, lane := instr.Arg2WithLane()
453 arr := ssaLaneToArrangement(lane)
454 m.lowerVecRRR(vecOpSqadd, x, y, instr.Return(), arr)
455 case ssa.OpcodeVUaddSat:
456 x, y, lane := instr.Arg2WithLane()
457 arr := ssaLaneToArrangement(lane)
458 m.lowerVecRRR(vecOpUqadd, x, y, instr.Return(), arr)
459 case ssa.OpcodeVIsub:
460 x, y, lane := instr.Arg2WithLane()
461 arr := ssaLaneToArrangement(lane)
462 m.lowerVecRRR(vecOpSub, x, y, instr.Return(), arr)
463 case ssa.OpcodeVSsubSat:
464 x, y, lane := instr.Arg2WithLane()
465 arr := ssaLaneToArrangement(lane)
466 m.lowerVecRRR(vecOpSqsub, x, y, instr.Return(), arr)
467 case ssa.OpcodeVUsubSat:
468 x, y, lane := instr.Arg2WithLane()
469 arr := ssaLaneToArrangement(lane)
470 m.lowerVecRRR(vecOpUqsub, x, y, instr.Return(), arr)
471 case ssa.OpcodeVImin:
472 x, y, lane := instr.Arg2WithLane()
473 arr := ssaLaneToArrangement(lane)
474 m.lowerVecRRR(vecOpSmin, x, y, instr.Return(), arr)
475 case ssa.OpcodeVUmin:
476 x, y, lane := instr.Arg2WithLane()
477 arr := ssaLaneToArrangement(lane)
478 m.lowerVecRRR(vecOpUmin, x, y, instr.Return(), arr)
479 case ssa.OpcodeVImax:
480 x, y, lane := instr.Arg2WithLane()
481 arr := ssaLaneToArrangement(lane)
482 m.lowerVecRRR(vecOpSmax, x, y, instr.Return(), arr)
483 case ssa.OpcodeVUmax:
484 x, y, lane := instr.Arg2WithLane()
485 arr := ssaLaneToArrangement(lane)
486 m.lowerVecRRR(vecOpUmax, x, y, instr.Return(), arr)
487 case ssa.OpcodeVAvgRound:
488 x, y, lane := instr.Arg2WithLane()
489 arr := ssaLaneToArrangement(lane)
490 m.lowerVecRRR(vecOpUrhadd, x, y, instr.Return(), arr)
491 case ssa.OpcodeVImul:
492 x, y, lane := instr.Arg2WithLane()
493 arr := ssaLaneToArrangement(lane)
494 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
495 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
496 rd := m.compiler.VRegOf(instr.Return())
497 m.lowerVIMul(rd, rn, rm, arr)
498 case ssa.OpcodeVIabs:
499 m.lowerVecMisc(vecOpAbs, instr)
500 case ssa.OpcodeVIneg:
501 m.lowerVecMisc(vecOpNeg, instr)
502 case ssa.OpcodeVIpopcnt:
503 m.lowerVecMisc(vecOpCnt, instr)
504 case ssa.OpcodeVIshl,
505 ssa.OpcodeVSshr, ssa.OpcodeVUshr:
506 x, y, lane := instr.Arg2WithLane()
507 arr := ssaLaneToArrangement(lane)
508 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
509 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
510 rd := m.compiler.VRegOf(instr.Return())
511 m.lowerVShift(op, rd, rn, rm, arr)
512 case ssa.OpcodeVSqrt:
513 m.lowerVecMisc(vecOpFsqrt, instr)
514 case ssa.OpcodeVFabs:
515 m.lowerVecMisc(vecOpFabs, instr)
516 case ssa.OpcodeVFneg:
517 m.lowerVecMisc(vecOpFneg, instr)
518 case ssa.OpcodeVFmin:
519 x, y, lane := instr.Arg2WithLane()
520 arr := ssaLaneToArrangement(lane)
521 m.lowerVecRRR(vecOpFmin, x, y, instr.Return(), arr)
522 case ssa.OpcodeVFmax:
523 x, y, lane := instr.Arg2WithLane()
524 arr := ssaLaneToArrangement(lane)
525 m.lowerVecRRR(vecOpFmax, x, y, instr.Return(), arr)
526 case ssa.OpcodeVFadd:
527 x, y, lane := instr.Arg2WithLane()
528 arr := ssaLaneToArrangement(lane)
529 m.lowerVecRRR(vecOpFadd, x, y, instr.Return(), arr)
530 case ssa.OpcodeVFsub:
531 x, y, lane := instr.Arg2WithLane()
532 arr := ssaLaneToArrangement(lane)
533 m.lowerVecRRR(vecOpFsub, x, y, instr.Return(), arr)
534 case ssa.OpcodeVFmul:
535 x, y, lane := instr.Arg2WithLane()
536 arr := ssaLaneToArrangement(lane)
537 m.lowerVecRRR(vecOpFmul, x, y, instr.Return(), arr)
538 case ssa.OpcodeSqmulRoundSat:
539 x, y, lane := instr.Arg2WithLane()
540 arr := ssaLaneToArrangement(lane)
541 m.lowerVecRRR(vecOpSqrdmulh, x, y, instr.Return(), arr)
542 case ssa.OpcodeVFdiv:
543 x, y, lane := instr.Arg2WithLane()
544 arr := ssaLaneToArrangement(lane)
545 m.lowerVecRRR(vecOpFdiv, x, y, instr.Return(), arr)
546 case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat:
547 x, lane := instr.ArgWithLane()
548 arr := ssaLaneToArrangement(lane)
549 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
550 rd := m.compiler.VRegOf(instr.Return())
551 m.lowerVfpuToInt(rd, rn, arr, op == ssa.OpcodeVFcvtToSintSat)
552 case ssa.OpcodeVFcvtFromSint, ssa.OpcodeVFcvtFromUint:
553 x, lane := instr.ArgWithLane()
554 arr := ssaLaneToArrangement(lane)
555 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
556 rd := m.compiler.VRegOf(instr.Return())
557 m.lowerVfpuFromInt(rd, rn, arr, op == ssa.OpcodeVFcvtFromSint)
558 case ssa.OpcodeSwidenLow, ssa.OpcodeUwidenLow:
559 x, lane := instr.ArgWithLane()
560 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
561 rd := m.compiler.VRegOf(instr.Return())
562
563 var arr vecArrangement
564 switch lane {
565 case ssa.VecLaneI8x16:
566 arr = vecArrangement8B
567 case ssa.VecLaneI16x8:
568 arr = vecArrangement4H
569 case ssa.VecLaneI32x4:
570 arr = vecArrangement2S
571 }
572
573 shll := m.allocateInstr()
574 if signed := op == ssa.OpcodeSwidenLow; signed {
575 shll.asVecShiftImm(vecOpSshll, rd, rn, operandShiftImm(0), arr)
576 } else {
577 shll.asVecShiftImm(vecOpUshll, rd, rn, operandShiftImm(0), arr)
578 }
579 m.insert(shll)
580 case ssa.OpcodeSwidenHigh, ssa.OpcodeUwidenHigh:
581 x, lane := instr.ArgWithLane()
582 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
583 rd := m.compiler.VRegOf(instr.Return())
584
585 arr := ssaLaneToArrangement(lane)
586
587 shll := m.allocateInstr()
588 if signed := op == ssa.OpcodeSwidenHigh; signed {
589 shll.asVecShiftImm(vecOpSshll, rd, rn, operandShiftImm(0), arr)
590 } else {
591 shll.asVecShiftImm(vecOpUshll, rd, rn, operandShiftImm(0), arr)
592 }
593 m.insert(shll)
594
595 case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow:
596 x, y, lane := instr.Arg2WithLane()
597 var arr, arr2 vecArrangement
598 switch lane {
599 case ssa.VecLaneI16x8: // I16x8
600 arr = vecArrangement8B
601 arr2 = vecArrangement16B // Implies sqxtn2.
602 case ssa.VecLaneI32x4:
603 arr = vecArrangement4H
604 arr2 = vecArrangement8H // Implies sqxtn2.
605 default:
606 panic("unsupported lane " + lane.String())
607 }
608 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
609 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
610 rd := m.compiler.VRegOf(instr.Return())
611
612 tmp := m.compiler.AllocateVReg(ssa.TypeV128)
613
614 loQxtn := m.allocateInstr()
615 hiQxtn := m.allocateInstr()
616 if signed := op == ssa.OpcodeSnarrow; signed {
617 // Narrow lanes on rn and write them into lower-half of rd.
618 loQxtn.asVecMisc(vecOpSqxtn, tmp, rn, arr) // low
619 // Narrow lanes on rm and write them into higher-half of rd.
620 hiQxtn.asVecMisc(vecOpSqxtn, tmp, rm, arr2) // high (sqxtn2)
621 } else {
622 // Narrow lanes on rn and write them into lower-half of rd.
623 loQxtn.asVecMisc(vecOpSqxtun, tmp, rn, arr) // low
624 // Narrow lanes on rm and write them into higher-half of rd.
625 hiQxtn.asVecMisc(vecOpSqxtun, tmp, rm, arr2) // high (sqxtn2)
626 }
627 m.insert(loQxtn)
628 m.insert(hiQxtn)
629
630 mov := m.allocateInstr()
631 mov.asFpuMov128(rd, tmp)
632 m.insert(mov)
633 case ssa.OpcodeFvpromoteLow:
634 x, lane := instr.ArgWithLane()
635 if lane != ssa.VecLaneF32x4 {
636 panic("unsupported lane type " + lane.String())
637 }
638 ins := m.allocateInstr()
639 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
640 rd := m.compiler.VRegOf(instr.Return())
641 ins.asVecMisc(vecOpFcvtl, rd, rn, vecArrangement2S)
642 m.insert(ins)
643 case ssa.OpcodeFvdemote:
644 x, lane := instr.ArgWithLane()
645 if lane != ssa.VecLaneF64x2 {
646 panic("unsupported lane type " + lane.String())
647 }
648 ins := m.allocateInstr()
649 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
650 rd := m.compiler.VRegOf(instr.Return())
651 ins.asVecMisc(vecOpFcvtn, rd, rn, vecArrangement2S)
652 m.insert(ins)
653 case ssa.OpcodeExtractlane:
654 x, index, signed, lane := instr.ExtractlaneData()
655
656 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
657 rd := m.compiler.VRegOf(instr.Return())
658
659 mov := m.allocateInstr()
660 switch lane {
661 case ssa.VecLaneI8x16:
662 mov.asMovFromVec(rd, rn, vecArrangementB, vecIndex(index), signed)
663 case ssa.VecLaneI16x8:
664 mov.asMovFromVec(rd, rn, vecArrangementH, vecIndex(index), signed)
665 case ssa.VecLaneI32x4:
666 mov.asMovFromVec(rd, rn, vecArrangementS, vecIndex(index), signed)
667 case ssa.VecLaneI64x2:
668 mov.asMovFromVec(rd, rn, vecArrangementD, vecIndex(index), signed)
669 case ssa.VecLaneF32x4:
670 mov.asVecMovElement(rd, rn, vecArrangementS, vecIndex(0), vecIndex(index))
671 case ssa.VecLaneF64x2:
672 mov.asVecMovElement(rd, rn, vecArrangementD, vecIndex(0), vecIndex(index))
673 default:
674 panic("unsupported lane: " + lane.String())
675 }
676
677 m.insert(mov)
678
679 case ssa.OpcodeInsertlane:
680 x, y, index, lane := instr.InsertlaneData()
681 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
682 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
683 rd := m.compiler.VRegOf(instr.Return())
684 tmpReg := m.compiler.AllocateVReg(ssa.TypeV128)
685
686 // Initially mov rn to tmp.
687 mov1 := m.allocateInstr()
688 mov1.asFpuMov128(tmpReg, rn.nr())
689 m.insert(mov1)
690
691 // movToVec and vecMovElement do not clear the remaining bits to zero,
692 // thus, we can mov rm in-place to tmp.
693 mov2 := m.allocateInstr()
694 switch lane {
695 case ssa.VecLaneI8x16:
696 mov2.asMovToVec(tmpReg, rm, vecArrangementB, vecIndex(index))
697 case ssa.VecLaneI16x8:
698 mov2.asMovToVec(tmpReg, rm, vecArrangementH, vecIndex(index))
699 case ssa.VecLaneI32x4:
700 mov2.asMovToVec(tmpReg, rm, vecArrangementS, vecIndex(index))
701 case ssa.VecLaneI64x2:
702 mov2.asMovToVec(tmpReg, rm, vecArrangementD, vecIndex(index))
703 case ssa.VecLaneF32x4:
704 mov2.asVecMovElement(tmpReg, rm, vecArrangementS, vecIndex(index), vecIndex(0))
705 case ssa.VecLaneF64x2:
706 mov2.asVecMovElement(tmpReg, rm, vecArrangementD, vecIndex(index), vecIndex(0))
707 }
708 m.insert(mov2)
709
710 // Finally mov tmp to rd.
711 mov3 := m.allocateInstr()
712 mov3.asFpuMov128(rd, tmpReg)
713 m.insert(mov3)
714
715 case ssa.OpcodeSwizzle:
716 x, y, lane := instr.Arg2WithLane()
717 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
718 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
719 rd := m.compiler.VRegOf(instr.Return())
720
721 arr := ssaLaneToArrangement(lane)
722
723 // tbl <rd>.<arr>, { <rn>.<arr> }, <rm>.<arr>
724 tbl1 := m.allocateInstr()
725 tbl1.asVecTbl(1, rd, rn, rm, arr)
726 m.insert(tbl1)
727
728 case ssa.OpcodeShuffle:
729 x, y, lane1, lane2 := instr.ShuffleData()
730 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
731 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
732 rd := m.compiler.VRegOf(instr.Return())
733
734 m.lowerShuffle(rd, rn, rm, lane1, lane2)
735
736 case ssa.OpcodeSplat:
737 x, lane := instr.ArgWithLane()
738 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
739 rd := m.compiler.VRegOf(instr.Return())
740
741 dup := m.allocateInstr()
742 switch lane {
743 case ssa.VecLaneI8x16:
744 dup.asVecDup(rd, rn, vecArrangement16B)
745 case ssa.VecLaneI16x8:
746 dup.asVecDup(rd, rn, vecArrangement8H)
747 case ssa.VecLaneI32x4:
748 dup.asVecDup(rd, rn, vecArrangement4S)
749 case ssa.VecLaneI64x2:
750 dup.asVecDup(rd, rn, vecArrangement2D)
751 case ssa.VecLaneF32x4:
752 dup.asVecDupElement(rd, rn, vecArrangementS, vecIndex(0))
753 case ssa.VecLaneF64x2:
754 dup.asVecDupElement(rd, rn, vecArrangementD, vecIndex(0))
755 }
756 m.insert(dup)
757
758 case ssa.OpcodeWideningPairwiseDotProductS:
759 x, y := instr.Arg2()
760 xx, yy := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone),
761 m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
762 tmp, tmp2 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
763 m.insert(m.allocateInstr().asVecRRR(vecOpSmull, tmp.nr(), xx, yy, vecArrangement8H))
764 m.insert(m.allocateInstr().asVecRRR(vecOpSmull2, tmp2.nr(), xx, yy, vecArrangement8H))
765 m.insert(m.allocateInstr().asVecRRR(vecOpAddp, tmp.nr(), tmp, tmp2, vecArrangement4S))
766
767 rd := m.compiler.VRegOf(instr.Return())
768 m.insert(m.allocateInstr().asFpuMov128(rd, tmp.nr()))
769
770 case ssa.OpcodeLoadSplat:
771 ptr, offset, lane := instr.LoadSplatData()
772 m.lowerLoadSplat(ptr, offset, lane, instr.Return())
773
774 case ssa.OpcodeAtomicRmw:
775 m.lowerAtomicRmw(instr)
776
777 case ssa.OpcodeAtomicCas:
778 m.lowerAtomicCas(instr)
779
780 case ssa.OpcodeAtomicLoad:
781 m.lowerAtomicLoad(instr)
782
783 case ssa.OpcodeAtomicStore:
784 m.lowerAtomicStore(instr)
785
786 case ssa.OpcodeFence:
787 instr := m.allocateInstr()
788 instr.asDMB()
789 m.insert(instr)
790
791 case ssa.OpcodeTailCallReturnCall, ssa.OpcodeTailCallReturnCallIndirect:
792 m.lowerTailCall(instr)
793
794 default:
795 panic("TODO: lowering " + op.String())
796 }
797 m.FlushPendingInstructions()
798 }
799
800 func (m *machine) lowerShuffle(rd regalloc.VReg, rn, rm operand, lane1, lane2 uint64) {
801 // `tbl2` requires 2 consecutive registers, so we arbitrarily pick v29, v30.
802 vReg, wReg := v29VReg, v30VReg
803
804 // Initialize v29, v30 to rn, rm.
805 movv := m.allocateInstr()
806 movv.asFpuMov128(vReg, rn.nr())
807 m.insert(movv)
808
809 movw := m.allocateInstr()
810 movw.asFpuMov128(wReg, rm.nr())
811 m.insert(movw)
812
813 // `lane1`, `lane2` are already encoded as two u64s with the right layout:
814 // lane1 := lane[7]<<56 | ... | lane[1]<<8 | lane[0]
815 // lane2 := lane[15]<<56 | ... | lane[9]<<8 | lane[8]
816 // Thus, we can use loadFpuConst128.
817 tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
818 lfc := m.allocateInstr()
819 lfc.asLoadFpuConst128(tmp.nr(), lane1, lane2)
820 m.insert(lfc)
821
822 // tbl <rd>.16b, { <vReg>.16B, <wReg>.16b }, <tmp>.16b
823 tbl2 := m.allocateInstr()
824 tbl2.asVecTbl(2, rd, operandNR(vReg), tmp, vecArrangement16B)
825 m.insert(tbl2)
826 }
827
828 func (m *machine) lowerVShift(op ssa.Opcode, rd regalloc.VReg, rn, rm operand, arr vecArrangement) {
829 var modulo byte
830 switch arr {
831 case vecArrangement16B:
832 modulo = 0x7 // Modulo 8.
833 case vecArrangement8H:
834 modulo = 0xf // Modulo 16.
835 case vecArrangement4S:
836 modulo = 0x1f // Modulo 32.
837 case vecArrangement2D:
838 modulo = 0x3f // Modulo 64.
839 default:
840 panic("unsupported arrangment " + arr.String())
841 }
842
843 rtmp := operandNR(m.compiler.AllocateVReg(ssa.TypeI64))
844 vtmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
845
846 and := m.allocateInstr()
847 and.asALUBitmaskImm(aluOpAnd, rtmp.nr(), rm.nr(), uint64(modulo), true)
848 m.insert(and)
849
850 if op != ssa.OpcodeVIshl {
851 // Negate the amount to make this as right shift.
852 neg := m.allocateInstr()
853 neg.asALU(aluOpSub, rtmp.nr(), operandNR(xzrVReg), rtmp, true)
854 m.insert(neg)
855 }
856
857 // Copy the shift amount into a vector register as sshl/ushl requires it to be there.
858 dup := m.allocateInstr()
859 dup.asVecDup(vtmp.nr(), rtmp, arr)
860 m.insert(dup)
861
862 if op == ssa.OpcodeVIshl || op == ssa.OpcodeVSshr {
863 sshl := m.allocateInstr()
864 sshl.asVecRRR(vecOpSshl, rd, rn, vtmp, arr)
865 m.insert(sshl)
866 } else {
867 ushl := m.allocateInstr()
868 ushl.asVecRRR(vecOpUshl, rd, rn, vtmp, arr)
869 m.insert(ushl)
870 }
871 }
872
873 func (m *machine) lowerVcheckTrue(op ssa.Opcode, rm operand, rd regalloc.VReg, arr vecArrangement) {
874 tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
875
876 // Special case VallTrue for i64x2.
877 if op == ssa.OpcodeVallTrue && arr == vecArrangement2D {
878 // cmeq v3?.2d, v2?.2d, #0
879 // addp v3?.2d, v3?.2d, v3?.2d
880 // fcmp v3?, v3?
881 // cset dst, eq
882
883 ins := m.allocateInstr()
884 ins.asVecMisc(vecOpCmeq0, tmp.nr(), rm, vecArrangement2D)
885 m.insert(ins)
886
887 addp := m.allocateInstr()
888 addp.asVecRRR(vecOpAddp, tmp.nr(), tmp, tmp, vecArrangement2D)
889 m.insert(addp)
890
891 fcmp := m.allocateInstr()
892 fcmp.asFpuCmp(tmp, tmp, true)
893 m.insert(fcmp)
894
895 cset := m.allocateInstr()
896 cset.asCSet(rd, false, eq)
897 m.insert(cset)
898
899 return
900 }
901
902 // Create a scalar value with umaxp or uminv, then compare it against zero.
903 ins := m.allocateInstr()
904 if op == ssa.OpcodeVanyTrue {
905 // umaxp v4?.16b, v2?.16b, v2?.16b
906 ins.asVecRRR(vecOpUmaxp, tmp.nr(), rm, rm, vecArrangement16B)
907 } else {
908 // uminv d4?, v2?.4s
909 ins.asVecLanes(vecOpUminv, tmp.nr(), rm, arr)
910 }
911 m.insert(ins)
912
913 // mov x3?, v4?.d[0]
914 // ccmp x3?, #0x0, #0x0, al
915 // cset x3?, ne
916 // mov x0, x3?
917
918 movv := m.allocateInstr()
919 movv.asMovFromVec(rd, tmp, vecArrangementD, vecIndex(0), false)
920 m.insert(movv)
921
922 fc := m.allocateInstr()
923 fc.asCCmpImm(operandNR(rd), uint64(0), al, 0, true)
924 m.insert(fc)
925
926 cset := m.allocateInstr()
927 cset.asCSet(rd, false, ne)
928 m.insert(cset)
929 }
930
931 func (m *machine) lowerVhighBits(rm operand, rd regalloc.VReg, arr vecArrangement) {
932 r0 := operandNR(m.compiler.AllocateVReg(ssa.TypeI64))
933 v0 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
934 v1 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
935
936 switch arr {
937 case vecArrangement16B:
938 // sshr v6?.16b, v2?.16b, #7
939 // movz x4?, #0x201, lsl 0
940 // movk x4?, #0x804, lsl 16
941 // movk x4?, #0x2010, lsl 32
942 // movk x4?, #0x8040, lsl 48
943 // dup v5?.2d, x4?
944 // and v6?.16b, v6?.16b, v5?.16b
945 // ext v5?.16b, v6?.16b, v6?.16b, #8
946 // zip1 v5?.16b, v6?.16b, v5?.16b
947 // addv s5?, v5?.8h
948 // umov s3?, v5?.h[0]
949
950 // Right arithmetic shift on the original vector and store the result into v1. So we have:
951 // v1[i] = 0xff if vi<0, 0 otherwise.
952 sshr := m.allocateInstr()
953 sshr.asVecShiftImm(vecOpSshr, v1.nr(), rm, operandShiftImm(7), vecArrangement16B)
954 m.insert(sshr)
955
956 // Load the bit mask into r0.
957 m.insertMOVZ(r0.nr(), 0x0201, 0, true)
958 m.insertMOVK(r0.nr(), 0x0804, 1, true)
959 m.insertMOVK(r0.nr(), 0x2010, 2, true)
960 m.insertMOVK(r0.nr(), 0x8040, 3, true)
961
962 // dup r0 to v0.
963 dup := m.allocateInstr()
964 dup.asVecDup(v0.nr(), r0, vecArrangement2D)
965 m.insert(dup)
966
967 // Lane-wise logical AND with the bit mask, meaning that we have
968 // v[i] = (1 << i) if vi<0, 0 otherwise.
969 //
970 // Below, we use the following notation:
971 // wi := (1 << i) if vi<0, 0 otherwise.
972 and := m.allocateInstr()
973 and.asVecRRR(vecOpAnd, v1.nr(), v1, v0, vecArrangement16B)
974 m.insert(and)
975
976 // Swap the lower and higher 8 byte elements, and write it into v0, meaning that we have
977 // v0[i] = w(i+8) if i < 8, w(i-8) otherwise.
978 ext := m.allocateInstr()
979 ext.asVecExtract(v0.nr(), v1, v1, vecArrangement16B, uint32(8))
980 m.insert(ext)
981
982 // v = [w0, w8, ..., w7, w15]
983 zip1 := m.allocateInstr()
984 zip1.asVecPermute(vecOpZip1, v0.nr(), v1, v0, vecArrangement16B)
985 m.insert(zip1)
986
987 // v.h[0] = w0 + ... + w15
988 addv := m.allocateInstr()
989 addv.asVecLanes(vecOpAddv, v0.nr(), v0, vecArrangement8H)
990 m.insert(addv)
991
992 // Extract the v.h[0] as the result.
993 movfv := m.allocateInstr()
994 movfv.asMovFromVec(rd, v0, vecArrangementH, vecIndex(0), false)
995 m.insert(movfv)
996 case vecArrangement8H:
997 // sshr v6?.8h, v2?.8h, #15
998 // movz x4?, #0x1, lsl 0
999 // movk x4?, #0x2, lsl 16
1000 // movk x4?, #0x4, lsl 32
1001 // movk x4?, #0x8, lsl 48
1002 // dup v5?.2d, x4?
1003 // lsl x4?, x4?, 0x4
1004 // ins v5?.d[1], x4?
1005 // and v5?.16b, v6?.16b, v5?.16b
1006 // addv s5?, v5?.8h
1007 // umov s3?, v5?.h[0]
1008
1009 // Right arithmetic shift on the original vector and store the result into v1. So we have:
1010 // v[i] = 0xffff if vi<0, 0 otherwise.
1011 sshr := m.allocateInstr()
1012 sshr.asVecShiftImm(vecOpSshr, v1.nr(), rm, operandShiftImm(15), vecArrangement8H)
1013 m.insert(sshr)
1014
1015 // Load the bit mask into r0.
1016 m.lowerConstantI64(r0.nr(), 0x0008000400020001)
1017
1018 // dup r0 to vector v0.
1019 dup := m.allocateInstr()
1020 dup.asVecDup(v0.nr(), r0, vecArrangement2D)
1021 m.insert(dup)
1022
1023 lsl := m.allocateInstr()
1024 lsl.asALUShift(aluOpLsl, r0.nr(), r0, operandShiftImm(4), true)
1025 m.insert(lsl)
1026
1027 movv := m.allocateInstr()
1028 movv.asMovToVec(v0.nr(), r0, vecArrangementD, vecIndex(1))
1029 m.insert(movv)
1030
1031 // Lane-wise logical AND with the bitmask, meaning that we have
1032 // v[i] = (1 << i) if vi<0, 0 otherwise for i=0..3
1033 // = (1 << (i+4)) if vi<0, 0 otherwise for i=3..7
1034 and := m.allocateInstr()
1035 and.asVecRRR(vecOpAnd, v0.nr(), v1, v0, vecArrangement16B)
1036 m.insert(and)
1037
1038 addv := m.allocateInstr()
1039 addv.asVecLanes(vecOpAddv, v0.nr(), v0, vecArrangement8H)
1040 m.insert(addv)
1041
1042 movfv := m.allocateInstr()
1043 movfv.asMovFromVec(rd, v0, vecArrangementH, vecIndex(0), false)
1044 m.insert(movfv)
1045 case vecArrangement4S:
1046 // sshr v6?.8h, v2?.8h, #15
1047 // movz x4?, #0x1, lsl 0
1048 // movk x4?, #0x2, lsl 16
1049 // movk x4?, #0x4, lsl 32
1050 // movk x4?, #0x8, lsl 48
1051 // dup v5?.2d, x4?
1052 // lsl x4?, x4?, 0x4
1053 // ins v5?.d[1], x4?
1054 // and v5?.16b, v6?.16b, v5?.16b
1055 // addv s5?, v5?.8h
1056 // umov s3?, v5?.h[0]
1057
1058 // Right arithmetic shift on the original vector and store the result into v1. So we have:
1059 // v[i] = 0xffffffff if vi<0, 0 otherwise.
1060 sshr := m.allocateInstr()
1061 sshr.asVecShiftImm(vecOpSshr, v1.nr(), rm, operandShiftImm(31), vecArrangement4S)
1062 m.insert(sshr)
1063
1064 // Load the bit mask into r0.
1065 m.lowerConstantI64(r0.nr(), 0x0000000200000001)
1066
1067 // dup r0 to vector v0.
1068 dup := m.allocateInstr()
1069 dup.asVecDup(v0.nr(), r0, vecArrangement2D)
1070 m.insert(dup)
1071
1072 lsl := m.allocateInstr()
1073 lsl.asALUShift(aluOpLsl, r0.nr(), r0, operandShiftImm(2), true)
1074 m.insert(lsl)
1075
1076 movv := m.allocateInstr()
1077 movv.asMovToVec(v0.nr(), r0, vecArrangementD, vecIndex(1))
1078 m.insert(movv)
1079
1080 // Lane-wise logical AND with the bitmask, meaning that we have
1081 // v[i] = (1 << i) if vi<0, 0 otherwise for i in [0, 1]
1082 // = (1 << (i+4)) if vi<0, 0 otherwise for i in [2, 3]
1083 and := m.allocateInstr()
1084 and.asVecRRR(vecOpAnd, v0.nr(), v1, v0, vecArrangement16B)
1085 m.insert(and)
1086
1087 addv := m.allocateInstr()
1088 addv.asVecLanes(vecOpAddv, v0.nr(), v0, vecArrangement4S)
1089 m.insert(addv)
1090
1091 movfv := m.allocateInstr()
1092 movfv.asMovFromVec(rd, v0, vecArrangementS, vecIndex(0), false)
1093 m.insert(movfv)
1094 case vecArrangement2D:
1095 // mov d3?, v2?.d[0]
1096 // mov x4?, v2?.d[1]
1097 // lsr x4?, x4?, 0x3f
1098 // lsr d3?, d3?, 0x3f
1099 // add s3?, s3?, w4?, lsl #1
1100
1101 // Move the lower 64-bit int into result.
1102 movv0 := m.allocateInstr()
1103 movv0.asMovFromVec(rd, rm, vecArrangementD, vecIndex(0), false)
1104 m.insert(movv0)
1105
1106 // Move the higher 64-bit int into r0.
1107 movv1 := m.allocateInstr()
1108 movv1.asMovFromVec(r0.nr(), rm, vecArrangementD, vecIndex(1), false)
1109 m.insert(movv1)
1110
1111 // Move the sign bit into the least significant bit.
1112 lsr1 := m.allocateInstr()
1113 lsr1.asALUShift(aluOpLsr, r0.nr(), r0, operandShiftImm(63), true)
1114 m.insert(lsr1)
1115
1116 lsr2 := m.allocateInstr()
1117 lsr2.asALUShift(aluOpLsr, rd, operandNR(rd), operandShiftImm(63), true)
1118 m.insert(lsr2)
1119
1120 // rd = (r0<<1) | rd
1121 lsl := m.allocateInstr()
1122 lsl.asALU(aluOpAdd, rd, operandNR(rd), operandSR(r0.nr(), 1, shiftOpLSL), false)
1123 m.insert(lsl)
1124 default:
1125 panic("Unsupported " + arr.String())
1126 }
1127 }
1128
1129 func (m *machine) lowerVecMisc(op vecOp, instr *ssa.Instruction) {
1130 x, lane := instr.ArgWithLane()
1131 arr := ssaLaneToArrangement(lane)
1132 ins := m.allocateInstr()
1133 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1134 rd := m.compiler.VRegOf(instr.Return())
1135 ins.asVecMisc(op, rd, rn, arr)
1136 m.insert(ins)
1137 }
1138
1139 func (m *machine) lowerVecRRR(op vecOp, x, y, ret ssa.Value, arr vecArrangement) {
1140 ins := m.allocateInstr()
1141 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1142 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1143 rd := m.compiler.VRegOf(ret)
1144 ins.asVecRRR(op, rd, rn, rm, arr)
1145 m.insert(ins)
1146 }
1147
1148 func (m *machine) lowerVIMul(rd regalloc.VReg, rn, rm operand, arr vecArrangement) {
1149 if arr != vecArrangement2D {
1150 mul := m.allocateInstr()
1151 mul.asVecRRR(vecOpMul, rd, rn, rm, arr)
1152 m.insert(mul)
1153 } else {
1154 tmp1 := m.compiler.AllocateVReg(ssa.TypeV128)
1155 tmp2 := m.compiler.AllocateVReg(ssa.TypeV128)
1156 tmp3 := m.compiler.AllocateVReg(ssa.TypeV128)
1157
1158 tmpRes := m.compiler.AllocateVReg(ssa.TypeV128)
1159
1160 // Following the algorithm in https://chromium-review.googlesource.com/c/v8/v8/+/1781696
1161 rev64 := m.allocateInstr()
1162 rev64.asVecMisc(vecOpRev64, tmp2, rm, vecArrangement4S)
1163 m.insert(rev64)
1164
1165 mul := m.allocateInstr()
1166 mul.asVecRRR(vecOpMul, tmp2, operandNR(tmp2), rn, vecArrangement4S)
1167 m.insert(mul)
1168
1169 xtn1 := m.allocateInstr()
1170 xtn1.asVecMisc(vecOpXtn, tmp1, rn, vecArrangement2S)
1171 m.insert(xtn1)
1172
1173 addp := m.allocateInstr()
1174 addp.asVecRRR(vecOpAddp, tmp2, operandNR(tmp2), operandNR(tmp2), vecArrangement4S)
1175 m.insert(addp)
1176
1177 xtn2 := m.allocateInstr()
1178 xtn2.asVecMisc(vecOpXtn, tmp3, rm, vecArrangement2S)
1179 m.insert(xtn2)
1180
1181 // Note: do not write the result directly into result yet. This is the same reason as in bsl.
1182 // In short, in UMLAL instruction, the result register is also one of the source register, and
1183 // the value on the result register is significant.
1184 shll := m.allocateInstr()
1185 shll.asVecMisc(vecOpShll, tmpRes, operandNR(tmp2), vecArrangement2S)
1186 m.insert(shll)
1187
1188 umlal := m.allocateInstr()
1189 umlal.asVecRRRRewrite(vecOpUmlal, tmpRes, operandNR(tmp3), operandNR(tmp1), vecArrangement2S)
1190 m.insert(umlal)
1191
1192 mov := m.allocateInstr()
1193 mov.asFpuMov128(rd, tmpRes)
1194 m.insert(mov)
1195 }
1196 }
1197
1198 func (m *machine) lowerVMinMaxPseudo(instr *ssa.Instruction, max bool) {
1199 x, y, lane := instr.Arg2WithLane()
1200 arr := ssaLaneToArrangement(lane)
1201
1202 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1203 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1204
1205 // Note: this usage of tmp is important.
1206 // BSL modifies the destination register, so we need to use a temporary register so that
1207 // the actual definition of the destination register happens *after* the BSL instruction.
1208 // That way, we can force the spill instruction to be inserted after the BSL instruction.
1209 tmp := m.compiler.AllocateVReg(ssa.TypeV128)
1210
1211 fcmgt := m.allocateInstr()
1212 if max {
1213 fcmgt.asVecRRR(vecOpFcmgt, tmp, rm, rn, arr)
1214 } else {
1215 // If min, swap the args.
1216 fcmgt.asVecRRR(vecOpFcmgt, tmp, rn, rm, arr)
1217 }
1218 m.insert(fcmgt)
1219
1220 bsl := m.allocateInstr()
1221 bsl.asVecRRRRewrite(vecOpBsl, tmp, rm, rn, vecArrangement16B)
1222 m.insert(bsl)
1223
1224 res := operandNR(m.compiler.VRegOf(instr.Return()))
1225 mov2 := m.allocateInstr()
1226 mov2.asFpuMov128(res.nr(), tmp)
1227 m.insert(mov2)
1228 }
1229
1230 func (m *machine) lowerIRem(execCtxVReg regalloc.VReg, rd, rn regalloc.VReg, rm operand, _64bit, signed bool) {
1231 div := m.allocateInstr()
1232
1233 if signed {
1234 div.asALU(aluOpSDiv, rd, operandNR(rn), rm, _64bit)
1235 } else {
1236 div.asALU(aluOpUDiv, rd, operandNR(rn), rm, _64bit)
1237 }
1238 m.insert(div)
1239
1240 // Check if rm is zero:
1241 m.exitIfNot(execCtxVReg, registerAsRegNotZeroCond(rm.nr()), _64bit, wazevoapi.ExitCodeIntegerDivisionByZero)
1242
1243 // rd = rn-rd*rm by MSUB instruction.
1244 msub := m.allocateInstr()
1245 msub.asALURRRR(aluOpMSub, rd, operandNR(rd), rm, rn, _64bit)
1246 m.insert(msub)
1247 }
1248
1249 func (m *machine) lowerIDiv(execCtxVReg, rd regalloc.VReg, rn, rm operand, _64bit, signed bool) {
1250 div := m.allocateInstr()
1251
1252 if signed {
1253 div.asALU(aluOpSDiv, rd, rn, rm, _64bit)
1254 } else {
1255 div.asALU(aluOpUDiv, rd, rn, rm, _64bit)
1256 }
1257 m.insert(div)
1258
1259 // Check if rm is zero:
1260 m.exitIfNot(execCtxVReg, registerAsRegNotZeroCond(rm.nr()), _64bit, wazevoapi.ExitCodeIntegerDivisionByZero)
1261
1262 if signed {
1263 // We need to check the signed overflow which happens iff "math.MinInt{32,64} / -1"
1264 minusOneCheck := m.allocateInstr()
1265 // Sets eq condition if rm == -1.
1266 minusOneCheck.asALU(aluOpAddS, xzrVReg, rm, operandImm12(1, 0), _64bit)
1267 m.insert(minusOneCheck)
1268
1269 ccmp := m.allocateInstr()
1270 // If eq condition is set, sets the flag by the result based on "rn - 1", otherwise clears the flag.
1271 ccmp.asCCmpImm(rn, 1, eq, 0, _64bit)
1272 m.insert(ccmp)
1273
1274 // Check the overflow flag.
1275 m.exitIfNot(execCtxVReg, vs.invert().asCond(), false, wazevoapi.ExitCodeIntegerOverflow)
1276 }
1277 }
1278
1279 // exitIfNot emits a conditional branch to exit if the condition is not met.
1280 // If `c` (cond type) is a register, `cond64bit` must be chosen to indicate whether the register is 32-bit or 64-bit.
1281 // Otherwise, `cond64bit` is ignored.
1282 func (m *machine) exitIfNot(execCtxVReg regalloc.VReg, c cond, cond64bit bool, code wazevoapi.ExitCode) {
1283 execCtxTmp := m.copyToTmp(execCtxVReg)
1284
1285 cbr := m.allocateInstr()
1286 m.insert(cbr)
1287 m.lowerExitWithCode(execCtxTmp, code)
1288 // Conditional branch target is after exit.
1289 l := m.insertBrTargetLabel()
1290 cbr.asCondBr(c, l, cond64bit)
1291 }
1292
1293 func (m *machine) lowerFcopysign(x, y, ret ssa.Value) {
1294 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1295 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1296 var tmpI, tmpF regalloc.VReg
1297 _64 := x.Type() == ssa.TypeF64
1298 if _64 {
1299 tmpF = m.compiler.AllocateVReg(ssa.TypeF64)
1300 tmpI = m.compiler.AllocateVReg(ssa.TypeI64)
1301 } else {
1302 tmpF = m.compiler.AllocateVReg(ssa.TypeF32)
1303 tmpI = m.compiler.AllocateVReg(ssa.TypeI32)
1304 }
1305 rd := m.compiler.VRegOf(ret)
1306 m.lowerFcopysignImpl(rd, rn, rm, tmpI, tmpF, _64)
1307 }
1308
1309 func (m *machine) lowerFcopysignImpl(rd regalloc.VReg, rn, rm operand, tmpI, tmpF regalloc.VReg, _64bit bool) {
1310 // This is exactly the same code emitted by GCC for "__builtin_copysign":
1311 //
1312 // mov x0, -9223372036854775808
1313 // fmov d2, x0
1314 // vbit v0.8b, v1.8b, v2.8b
1315 //
1316
1317 setMSB := m.allocateInstr()
1318 if _64bit {
1319 m.lowerConstantI64(tmpI, math.MinInt64)
1320 setMSB.asMovToVec(tmpF, operandNR(tmpI), vecArrangementD, vecIndex(0))
1321 } else {
1322 m.lowerConstantI32(tmpI, math.MinInt32)
1323 setMSB.asMovToVec(tmpF, operandNR(tmpI), vecArrangementS, vecIndex(0))
1324 }
1325 m.insert(setMSB)
1326
1327 tmpReg := m.compiler.AllocateVReg(ssa.TypeF64)
1328
1329 mov := m.allocateInstr()
1330 mov.asFpuMov64(tmpReg, rn.nr())
1331 m.insert(mov)
1332
1333 vbit := m.allocateInstr()
1334 vbit.asVecRRRRewrite(vecOpBit, tmpReg, rm, operandNR(tmpF), vecArrangement8B)
1335 m.insert(vbit)
1336
1337 movDst := m.allocateInstr()
1338 movDst.asFpuMov64(rd, tmpReg)
1339 m.insert(movDst)
1340 }
1341
1342 func (m *machine) lowerBitcast(instr *ssa.Instruction) {
1343 v, dstType := instr.BitcastData()
1344 srcType := v.Type()
1345 rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone)
1346 rd := m.compiler.VRegOf(instr.Return())
1347 srcInt := srcType.IsInt()
1348 dstInt := dstType.IsInt()
1349 switch {
1350 case srcInt && !dstInt: // Int to Float:
1351 mov := m.allocateInstr()
1352 var arr vecArrangement
1353 if srcType.Bits() == 64 {
1354 arr = vecArrangementD
1355 } else {
1356 arr = vecArrangementS
1357 }
1358 mov.asMovToVec(rd, rn, arr, vecIndex(0))
1359 m.insert(mov)
1360 case !srcInt && dstInt: // Float to Int:
1361 mov := m.allocateInstr()
1362 var arr vecArrangement
1363 if dstType.Bits() == 64 {
1364 arr = vecArrangementD
1365 } else {
1366 arr = vecArrangementS
1367 }
1368 mov.asMovFromVec(rd, rn, arr, vecIndex(0), false)
1369 m.insert(mov)
1370 default:
1371 panic("TODO?BUG?")
1372 }
1373 }
1374
1375 func (m *machine) lowerFpuUniOp(op fpuUniOp, in, out ssa.Value) {
1376 rn := m.getOperand_NR(m.compiler.ValueDefinition(in), extModeNone)
1377 rd := m.compiler.VRegOf(out)
1378
1379 neg := m.allocateInstr()
1380 neg.asFpuRR(op, rd, rn, in.Type().Bits() == 64)
1381 m.insert(neg)
1382 }
1383
1384 func (m *machine) lowerFpuToInt(rd regalloc.VReg, rn operand, ctx regalloc.VReg, signed, src64bit, dst64bit, nonTrapping bool) {
1385 if !nonTrapping {
1386 // First of all, we have to clear the FPU flags.
1387 flagClear := m.allocateInstr()
1388 flagClear.asMovToFPSR(xzrVReg)
1389 m.insert(flagClear)
1390 }
1391
1392 // Then, do the conversion which doesn't trap inherently.
1393 cvt := m.allocateInstr()
1394 cvt.asFpuToInt(rd, rn, signed, src64bit, dst64bit)
1395 m.insert(cvt)
1396
1397 if !nonTrapping {
1398 tmpReg := m.compiler.AllocateVReg(ssa.TypeI64)
1399
1400 // After the conversion, check the FPU flags.
1401 getFlag := m.allocateInstr()
1402 getFlag.asMovFromFPSR(tmpReg)
1403 m.insert(getFlag)
1404
1405 execCtx := m.copyToTmp(ctx)
1406 _rn := operandNR(m.copyToTmp(rn.nr()))
1407
1408 // Check if the conversion was undefined by comparing the status with 1.
1409 // See https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/FPSR--Floating-point-Status-Register
1410 alu := m.allocateInstr()
1411 alu.asALU(aluOpSubS, xzrVReg, operandNR(tmpReg), operandImm12(1, 0), true)
1412 m.insert(alu)
1413
1414 // If it is not undefined, we can return the result.
1415 ok := m.allocateInstr()
1416 m.insert(ok)
1417
1418 // Otherwise, we have to choose the status depending on it is overflow or NaN conversion.
1419
1420 // Comparing itself to check if it is a NaN.
1421 fpuCmp := m.allocateInstr()
1422 fpuCmp.asFpuCmp(_rn, _rn, src64bit)
1423 m.insert(fpuCmp)
1424 // If the VC flag is not set (== VS flag is set), it is a NaN.
1425 m.exitIfNot(execCtx, vc.asCond(), false, wazevoapi.ExitCodeInvalidConversionToInteger)
1426 // Otherwise, it is an overflow.
1427 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
1428
1429 // Conditional branch target is after exit.
1430 l := m.insertBrTargetLabel()
1431 ok.asCondBr(ne.asCond(), l, false /* ignored */)
1432 }
1433 }
1434
1435 func (m *machine) lowerIntToFpu(rd regalloc.VReg, rn operand, signed, src64bit, dst64bit bool) {
1436 cvt := m.allocateInstr()
1437 cvt.asIntToFpu(rd, rn, signed, src64bit, dst64bit)
1438 m.insert(cvt)
1439 }
1440
1441 func (m *machine) lowerFpuBinOp(si *ssa.Instruction) {
1442 instr := m.allocateInstr()
1443 var op fpuBinOp
1444 switch si.Opcode() {
1445 case ssa.OpcodeFadd:
1446 op = fpuBinOpAdd
1447 case ssa.OpcodeFsub:
1448 op = fpuBinOpSub
1449 case ssa.OpcodeFmul:
1450 op = fpuBinOpMul
1451 case ssa.OpcodeFdiv:
1452 op = fpuBinOpDiv
1453 case ssa.OpcodeFmax:
1454 op = fpuBinOpMax
1455 case ssa.OpcodeFmin:
1456 op = fpuBinOpMin
1457 }
1458 x, y := si.Arg2()
1459 xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y)
1460 rn := m.getOperand_NR(xDef, extModeNone)
1461 rm := m.getOperand_NR(yDef, extModeNone)
1462 rd := m.compiler.VRegOf(si.Return())
1463 instr.asFpuRRR(op, rd, rn, rm, x.Type().Bits() == 64)
1464 m.insert(instr)
1465 }
1466
1467 func (m *machine) lowerSubOrAdd(si *ssa.Instruction, add bool) {
1468 x, y := si.Arg2()
1469 if !x.Type().IsInt() {
1470 panic("BUG?")
1471 }
1472
1473 xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y)
1474 rn := m.getOperand_NR(xDef, extModeNone)
1475 rm, yNegated := m.getOperand_MaybeNegatedImm12_ER_SR_NR(yDef, extModeNone)
1476
1477 var aop aluOp
1478 switch {
1479 case add && !yNegated: // rn+rm = x+y
1480 aop = aluOpAdd
1481 case add && yNegated: // rn-rm = x-(-y) = x+y
1482 aop = aluOpSub
1483 case !add && !yNegated: // rn-rm = x-y
1484 aop = aluOpSub
1485 case !add && yNegated: // rn+rm = x-(-y) = x-y
1486 aop = aluOpAdd
1487 }
1488 rd := m.compiler.VRegOf(si.Return())
1489 alu := m.allocateInstr()
1490 alu.asALU(aop, rd, rn, rm, x.Type().Bits() == 64)
1491 m.insert(alu)
1492 }
1493
1494 // InsertMove implements backend.Machine.
1495 func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) {
1496 instr := m.allocateInstr()
1497 switch typ {
1498 case ssa.TypeI32, ssa.TypeI64:
1499 instr.asMove64(dst, src)
1500 case ssa.TypeF32, ssa.TypeF64:
1501 instr.asFpuMov64(dst, src)
1502 case ssa.TypeV128:
1503 instr.asFpuMov128(dst, src)
1504 default:
1505 panic("TODO")
1506 }
1507 m.insert(instr)
1508 }
1509
1510 func (m *machine) lowerIcmp(si *ssa.Instruction) {
1511 x, y, c := si.IcmpData()
1512 flag := condFlagFromSSAIntegerCmpCond(c)
1513
1514 in64bit := x.Type().Bits() == 64
1515 var ext extMode
1516 if in64bit {
1517 if c.Signed() {
1518 ext = extModeSignExtend64
1519 } else {
1520 ext = extModeZeroExtend64
1521 }
1522 } else {
1523 if c.Signed() {
1524 ext = extModeSignExtend32
1525 } else {
1526 ext = extModeZeroExtend32
1527 }
1528 }
1529
1530 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), ext)
1531 rm := m.getOperand_Imm12_ER_SR_NR(m.compiler.ValueDefinition(y), ext)
1532 alu := m.allocateInstr()
1533 alu.asALU(aluOpSubS, xzrVReg, rn, rm, in64bit)
1534 m.insert(alu)
1535
1536 cset := m.allocateInstr()
1537 cset.asCSet(m.compiler.VRegOf(si.Return()), false, flag)
1538 m.insert(cset)
1539 }
1540
1541 func (m *machine) lowerVIcmp(si *ssa.Instruction) {
1542 x, y, c, lane := si.VIcmpData()
1543 flag := condFlagFromSSAIntegerCmpCond(c)
1544 arr := ssaLaneToArrangement(lane)
1545
1546 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1547 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1548 rd := m.compiler.VRegOf(si.Return())
1549
1550 switch flag {
1551 case eq:
1552 cmp := m.allocateInstr()
1553 cmp.asVecRRR(vecOpCmeq, rd, rn, rm, arr)
1554 m.insert(cmp)
1555 case ne:
1556 cmp := m.allocateInstr()
1557 cmp.asVecRRR(vecOpCmeq, rd, rn, rm, arr)
1558 m.insert(cmp)
1559 not := m.allocateInstr()
1560 not.asVecMisc(vecOpNot, rd, operandNR(rd), vecArrangement16B)
1561 m.insert(not)
1562 case ge:
1563 cmp := m.allocateInstr()
1564 cmp.asVecRRR(vecOpCmge, rd, rn, rm, arr)
1565 m.insert(cmp)
1566 case gt:
1567 cmp := m.allocateInstr()
1568 cmp.asVecRRR(vecOpCmgt, rd, rn, rm, arr)
1569 m.insert(cmp)
1570 case le:
1571 cmp := m.allocateInstr()
1572 cmp.asVecRRR(vecOpCmge, rd, rm, rn, arr) // rm, rn are swapped
1573 m.insert(cmp)
1574 case lt:
1575 cmp := m.allocateInstr()
1576 cmp.asVecRRR(vecOpCmgt, rd, rm, rn, arr) // rm, rn are swapped
1577 m.insert(cmp)
1578 case hs:
1579 cmp := m.allocateInstr()
1580 cmp.asVecRRR(vecOpCmhs, rd, rn, rm, arr)
1581 m.insert(cmp)
1582 case hi:
1583 cmp := m.allocateInstr()
1584 cmp.asVecRRR(vecOpCmhi, rd, rn, rm, arr)
1585 m.insert(cmp)
1586 case ls:
1587 cmp := m.allocateInstr()
1588 cmp.asVecRRR(vecOpCmhs, rd, rm, rn, arr) // rm, rn are swapped
1589 m.insert(cmp)
1590 case lo:
1591 cmp := m.allocateInstr()
1592 cmp.asVecRRR(vecOpCmhi, rd, rm, rn, arr) // rm, rn are swapped
1593 m.insert(cmp)
1594 }
1595 }
1596
1597 func (m *machine) lowerVFcmp(si *ssa.Instruction) {
1598 x, y, c, lane := si.VFcmpData()
1599 flag := condFlagFromSSAFloatCmpCond(c)
1600 arr := ssaLaneToArrangement(lane)
1601
1602 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1603 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1604 rd := m.compiler.VRegOf(si.Return())
1605
1606 switch flag {
1607 case eq:
1608 cmp := m.allocateInstr()
1609 cmp.asVecRRR(vecOpFcmeq, rd, rn, rm, arr)
1610 m.insert(cmp)
1611 case ne:
1612 cmp := m.allocateInstr()
1613 cmp.asVecRRR(vecOpFcmeq, rd, rn, rm, arr)
1614 m.insert(cmp)
1615 not := m.allocateInstr()
1616 not.asVecMisc(vecOpNot, rd, operandNR(rd), vecArrangement16B)
1617 m.insert(not)
1618 case ge:
1619 cmp := m.allocateInstr()
1620 cmp.asVecRRR(vecOpFcmge, rd, rn, rm, arr)
1621 m.insert(cmp)
1622 case gt:
1623 cmp := m.allocateInstr()
1624 cmp.asVecRRR(vecOpFcmgt, rd, rn, rm, arr)
1625 m.insert(cmp)
1626 case mi:
1627 cmp := m.allocateInstr()
1628 cmp.asVecRRR(vecOpFcmgt, rd, rm, rn, arr) // rm, rn are swapped
1629 m.insert(cmp)
1630 case ls:
1631 cmp := m.allocateInstr()
1632 cmp.asVecRRR(vecOpFcmge, rd, rm, rn, arr) // rm, rn are swapped
1633 m.insert(cmp)
1634 }
1635 }
1636
1637 func (m *machine) lowerVfpuToInt(rd regalloc.VReg, rn operand, arr vecArrangement, signed bool) {
1638 cvt := m.allocateInstr()
1639 if signed {
1640 cvt.asVecMisc(vecOpFcvtzs, rd, rn, arr)
1641 } else {
1642 cvt.asVecMisc(vecOpFcvtzu, rd, rn, arr)
1643 }
1644 m.insert(cvt)
1645
1646 if arr == vecArrangement2D {
1647 narrow := m.allocateInstr()
1648 if signed {
1649 narrow.asVecMisc(vecOpSqxtn, rd, operandNR(rd), vecArrangement2S)
1650 } else {
1651 narrow.asVecMisc(vecOpUqxtn, rd, operandNR(rd), vecArrangement2S)
1652 }
1653 m.insert(narrow)
1654 }
1655 }
1656
1657 func (m *machine) lowerVfpuFromInt(rd regalloc.VReg, rn operand, arr vecArrangement, signed bool) {
1658 cvt := m.allocateInstr()
1659 if signed {
1660 cvt.asVecMisc(vecOpScvtf, rd, rn, arr)
1661 } else {
1662 cvt.asVecMisc(vecOpUcvtf, rd, rn, arr)
1663 }
1664 m.insert(cvt)
1665 }
1666
1667 func (m *machine) lowerShifts(si *ssa.Instruction, ext extMode, aluOp aluOp) {
1668 x, amount := si.Arg2()
1669 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), ext)
1670 rm := m.getOperand_ShiftImm_NR(m.compiler.ValueDefinition(amount), ext, x.Type().Bits())
1671 rd := m.compiler.VRegOf(si.Return())
1672
1673 alu := m.allocateInstr()
1674 alu.asALUShift(aluOp, rd, rn, rm, x.Type().Bits() == 64)
1675 m.insert(alu)
1676 }
1677
1678 func (m *machine) lowerBitwiseAluOp(si *ssa.Instruction, op aluOp, ignoreResult bool) {
1679 x, y := si.Arg2()
1680
1681 xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y)
1682 rn := m.getOperand_NR(xDef, extModeNone)
1683
1684 var rd regalloc.VReg
1685 if ignoreResult {
1686 rd = xzrVReg
1687 } else {
1688 rd = m.compiler.VRegOf(si.Return())
1689 }
1690
1691 _64 := x.Type().Bits() == 64
1692 alu := m.allocateInstr()
1693 if instr := yDef.Instr; instr != nil && instr.Constant() {
1694 c := instr.ConstantVal()
1695 if isBitMaskImmediate(c, _64) {
1696 // Constant bit wise operations can be lowered to a single instruction.
1697 alu.asALUBitmaskImm(op, rd, rn.nr(), c, _64)
1698 m.insert(alu)
1699 return
1700 }
1701 }
1702
1703 rm := m.getOperand_SR_NR(yDef, extModeNone)
1704 alu.asALU(op, rd, rn, rm, _64)
1705 m.insert(alu)
1706 }
1707
1708 func (m *machine) lowerRotl(si *ssa.Instruction) {
1709 x, y := si.Arg2()
1710 r := si.Return()
1711 _64 := r.Type().Bits() == 64
1712
1713 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1714 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1715 var tmp regalloc.VReg
1716 if _64 {
1717 tmp = m.compiler.AllocateVReg(ssa.TypeI64)
1718 } else {
1719 tmp = m.compiler.AllocateVReg(ssa.TypeI32)
1720 }
1721 rd := m.compiler.VRegOf(r)
1722
1723 // Encode rotl as neg + rotr: neg is a sub against the zero-reg.
1724 m.lowerRotlImpl(rd, rn, rm, tmp, _64)
1725 }
1726
1727 func (m *machine) lowerRotlImpl(rd regalloc.VReg, rn, rm operand, tmp regalloc.VReg, is64bit bool) {
1728 // Encode rotl as neg + rotr: neg is a sub against the zero-reg.
1729 neg := m.allocateInstr()
1730 neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), rm, is64bit)
1731 m.insert(neg)
1732 alu := m.allocateInstr()
1733 alu.asALU(aluOpRotR, rd, rn, operandNR(tmp), is64bit)
1734 m.insert(alu)
1735 }
1736
1737 func (m *machine) lowerRotr(si *ssa.Instruction) {
1738 x, y := si.Arg2()
1739
1740 xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y)
1741 rn := m.getOperand_NR(xDef, extModeNone)
1742 rm := m.getOperand_NR(yDef, extModeNone)
1743 rd := m.compiler.VRegOf(si.Return())
1744
1745 alu := m.allocateInstr()
1746 alu.asALU(aluOpRotR, rd, rn, rm, si.Return().Type().Bits() == 64)
1747 m.insert(alu)
1748 }
1749
1750 func (m *machine) lowerExtend(arg, ret ssa.Value, from, to byte, signed bool) {
1751 rd := m.compiler.VRegOf(ret)
1752 def := m.compiler.ValueDefinition(arg)
1753
1754 if instr := def.Instr; !signed && from == 32 && instr != nil {
1755 // We can optimize out the unsigned extend because:
1756 // Writes to the W register set bits [63:32] of the X register to zero
1757 // https://developer.arm.com/documentation/den0024/a/An-Introduction-to-the-ARMv8-Instruction-Sets/The-ARMv8-instruction-sets/Distinguishing-between-32-bit-and-64-bit-A64-instructions
1758 switch instr.Opcode() {
1759 case
1760 ssa.OpcodeIadd, ssa.OpcodeIsub, ssa.OpcodeLoad,
1761 ssa.OpcodeBand, ssa.OpcodeBor, ssa.OpcodeBnot,
1762 ssa.OpcodeIshl, ssa.OpcodeUshr, ssa.OpcodeSshr,
1763 ssa.OpcodeRotl, ssa.OpcodeRotr,
1764 ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32:
1765 // So, if the argument is the result of a 32-bit operation, we can just copy the register.
1766 // It is highly likely that this copy will be optimized out after register allocation.
1767 rn := m.compiler.VRegOf(arg)
1768 mov := m.allocateInstr()
1769 // Note: do not use move32 as it will be lowered to a 32-bit move, which is not copy (that is actually the impl of UExtend).
1770 mov.asMove64(rd, rn)
1771 m.insert(mov)
1772 return
1773 default:
1774 }
1775 }
1776 rn := m.getOperand_NR(def, extModeNone)
1777
1778 ext := m.allocateInstr()
1779 ext.asExtend(rd, rn.nr(), from, to, signed)
1780 m.insert(ext)
1781 }
1782
1783 func (m *machine) lowerFcmp(x, y, result ssa.Value, c ssa.FloatCmpCond) {
1784 rn, rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone), m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1785
1786 fc := m.allocateInstr()
1787 fc.asFpuCmp(rn, rm, x.Type().Bits() == 64)
1788 m.insert(fc)
1789
1790 cset := m.allocateInstr()
1791 cset.asCSet(m.compiler.VRegOf(result), false, condFlagFromSSAFloatCmpCond(c))
1792 m.insert(cset)
1793 }
1794
1795 func (m *machine) lowerImul(x, y, result ssa.Value) {
1796 rd := m.compiler.VRegOf(result)
1797 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1798 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1799
1800 // TODO: if this comes before Add/Sub, we could merge it by putting it into the place of xzrVReg.
1801
1802 mul := m.allocateInstr()
1803 mul.asALURRRR(aluOpMAdd, rd, rn, rm, xzrVReg, x.Type().Bits() == 64)
1804 m.insert(mul)
1805 }
1806
1807 func (m *machine) lowerClz(x, result ssa.Value) {
1808 rd := m.compiler.VRegOf(result)
1809 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1810 clz := m.allocateInstr()
1811 clz.asBitRR(bitOpClz, rd, rn.nr(), x.Type().Bits() == 64)
1812 m.insert(clz)
1813 }
1814
1815 func (m *machine) lowerCtz(x, result ssa.Value) {
1816 rd := m.compiler.VRegOf(result)
1817 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1818 rbit := m.allocateInstr()
1819 _64 := x.Type().Bits() == 64
1820 var tmpReg regalloc.VReg
1821 if _64 {
1822 tmpReg = m.compiler.AllocateVReg(ssa.TypeI64)
1823 } else {
1824 tmpReg = m.compiler.AllocateVReg(ssa.TypeI32)
1825 }
1826 rbit.asBitRR(bitOpRbit, tmpReg, rn.nr(), _64)
1827 m.insert(rbit)
1828
1829 clz := m.allocateInstr()
1830 clz.asBitRR(bitOpClz, rd, tmpReg, _64)
1831 m.insert(clz)
1832 }
1833
1834 func (m *machine) lowerPopcnt(x, result ssa.Value) {
1835 // arm64 doesn't have an instruction for population count on scalar register,
1836 // so we use the vector instruction `cnt`.
1837 // This is exactly what the official Go implements bits.OneCount.
1838 // For example, "func () int { return bits.OneCount(10) }" is compiled as
1839 //
1840 // MOVD $10, R0 ;; Load 10.
1841 // FMOVD R0, F0
1842 // VCNT V0.B8, V0.B8
1843 // UADDLV V0.B8, V0
1844 //
1845 // In aarch64 asm, FMOVD is encoded as `ins`, VCNT is `cnt`,
1846 // and the registers may use different names. In our encoding we use the following
1847 // instructions:
1848 //
1849 // ins v0.d[0], x0 ;; mov from GPR to vec (FMOV above) is encoded as INS
1850 // cnt v0.16b, v0.16b ;; we use vec arrangement 16b
1851 // uaddlv h0, v0.8b ;; h0 is still v0 with the dest width specifier 'H', implied when src arrangement is 8b
1852 // mov x5, v0.d[0] ;; finally we mov the result back to a GPR
1853 //
1854
1855 rd := m.compiler.VRegOf(result)
1856 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1857
1858 rf1 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64))
1859 ins := m.allocateInstr()
1860 ins.asMovToVec(rf1.nr(), rn, vecArrangementD, vecIndex(0))
1861 m.insert(ins)
1862
1863 rf2 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64))
1864 cnt := m.allocateInstr()
1865 cnt.asVecMisc(vecOpCnt, rf2.nr(), rf1, vecArrangement16B)
1866 m.insert(cnt)
1867
1868 rf3 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64))
1869 uaddlv := m.allocateInstr()
1870 uaddlv.asVecLanes(vecOpUaddlv, rf3.nr(), rf2, vecArrangement8B)
1871 m.insert(uaddlv)
1872
1873 mov := m.allocateInstr()
1874 mov.asMovFromVec(rd, rf3, vecArrangementD, vecIndex(0), false)
1875 m.insert(mov)
1876 }
1877
1878 // lowerExitWithCode lowers the lowerExitWithCode takes a context pointer as argument.
1879 func (m *machine) lowerExitWithCode(execCtxVReg regalloc.VReg, code wazevoapi.ExitCode) {
1880 tmpReg1 := m.compiler.AllocateVReg(ssa.TypeI32)
1881 loadExitCodeConst := m.allocateInstr()
1882 loadExitCodeConst.asMOVZ(tmpReg1, uint64(code), 0, true)
1883
1884 setExitCode := m.allocateInstr()
1885 mode := m.amodePool.Allocate()
1886 *mode = addressMode{
1887 kind: addressModeKindRegUnsignedImm12,
1888 rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetExitCodeOffset.I64(),
1889 }
1890 setExitCode.asStore(operandNR(tmpReg1), mode, 32)
1891
1892 // In order to unwind the stack, we also need to push the current stack pointer:
1893 tmp2 := m.compiler.AllocateVReg(ssa.TypeI64)
1894 movSpToTmp := m.allocateInstr()
1895 movSpToTmp.asMove64(tmp2, spVReg)
1896 strSpToExecCtx := m.allocateInstr()
1897 mode2 := m.amodePool.Allocate()
1898 *mode2 = addressMode{
1899 kind: addressModeKindRegUnsignedImm12,
1900 rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.I64(),
1901 }
1902 strSpToExecCtx.asStore(operandNR(tmp2), mode2, 64)
1903 // Also the address of this exit.
1904 tmp3 := m.compiler.AllocateVReg(ssa.TypeI64)
1905 currentAddrToTmp := m.allocateInstr()
1906 currentAddrToTmp.asAdr(tmp3, 0)
1907 storeCurrentAddrToExecCtx := m.allocateInstr()
1908 mode3 := m.amodePool.Allocate()
1909 *mode3 = addressMode{
1910 kind: addressModeKindRegUnsignedImm12,
1911 rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetGoCallReturnAddress.I64(),
1912 }
1913 storeCurrentAddrToExecCtx.asStore(operandNR(tmp3), mode3, 64)
1914
1915 exitSeq := m.allocateInstr()
1916 exitSeq.asExitSequence(execCtxVReg)
1917
1918 m.insert(loadExitCodeConst)
1919 m.insert(setExitCode)
1920 m.insert(movSpToTmp)
1921 m.insert(strSpToExecCtx)
1922 m.insert(currentAddrToTmp)
1923 m.insert(storeCurrentAddrToExecCtx)
1924 m.insert(exitSeq)
1925 }
1926
1927 func (m *machine) lowerIcmpToFlag(x, y ssa.Value, signed bool) {
1928 if x.Type() != y.Type() {
1929 panic(
1930 fmt.Sprintf("TODO(maybe): support icmp with different types: v%d=%s != v%d=%s",
1931 x.ID(), x.Type(), y.ID(), y.Type()))
1932 }
1933
1934 extMod := extModeOf(x.Type(), signed)
1935
1936 // First operand must be in pure register form.
1937 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extMod)
1938 // Second operand can be in any of Imm12, ER, SR, or NR form supported by the SUBS instructions.
1939 rm := m.getOperand_Imm12_ER_SR_NR(m.compiler.ValueDefinition(y), extMod)
1940
1941 alu := m.allocateInstr()
1942 // subs zr, rn, rm
1943 alu.asALU(
1944 aluOpSubS,
1945 // We don't need the result, just need to set flags.
1946 xzrVReg,
1947 rn,
1948 rm,
1949 x.Type().Bits() == 64,
1950 )
1951 m.insert(alu)
1952 }
1953
1954 func (m *machine) lowerFcmpToFlag(x, y ssa.Value) {
1955 if x.Type() != y.Type() {
1956 panic("TODO(maybe): support icmp with different types")
1957 }
1958
1959 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
1960 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
1961 cmp := m.allocateInstr()
1962 cmp.asFpuCmp(rn, rm, x.Type().Bits() == 64)
1963 m.insert(cmp)
1964 }
1965
1966 func (m *machine) lowerExitIfTrueWithCode(execCtxVReg regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) {
1967 condDef := m.compiler.ValueDefinition(cond)
1968 if !m.compiler.MatchInstr(condDef, ssa.OpcodeIcmp) {
1969 panic("TODO: OpcodeExitIfTrueWithCode must come after Icmp at the moment: " + condDef.Instr.Opcode().String())
1970 }
1971 condDef.Instr.MarkLowered()
1972
1973 cvalInstr := condDef.Instr
1974 x, y, c := cvalInstr.IcmpData()
1975 signed := c.Signed()
1976
1977 if !m.tryLowerBandToFlag(x, y) {
1978 m.lowerIcmpToFlag(x, y, signed)
1979 }
1980
1981 // We need to copy the execution context to a temp register, because if it's spilled,
1982 // it might end up being reloaded inside the exiting branch.
1983 execCtxTmp := m.copyToTmp(execCtxVReg)
1984
1985 // We have to skip the entire exit sequence if the condition is false.
1986 cbr := m.allocateInstr()
1987 m.insert(cbr)
1988 m.lowerExitWithCode(execCtxTmp, code)
1989 // conditional branch target is after exit.
1990 l := m.insertBrTargetLabel()
1991 cbr.asCondBr(condFlagFromSSAIntegerCmpCond(c).invert().asCond(), l, false /* ignored */)
1992 }
1993
1994 func (m *machine) lowerSelect(c, x, y, result ssa.Value) {
1995 cvalDef := m.compiler.ValueDefinition(c)
1996
1997 var cc condFlag
1998 switch {
1999 case m.compiler.MatchInstr(cvalDef, ssa.OpcodeIcmp): // This case, we can use the ALU flag set by SUBS instruction.
2000 cvalInstr := cvalDef.Instr
2001 x, y, c := cvalInstr.IcmpData()
2002 cc = condFlagFromSSAIntegerCmpCond(c)
2003 m.lowerIcmpToFlag(x, y, c.Signed())
2004 cvalDef.Instr.MarkLowered()
2005 case m.compiler.MatchInstr(cvalDef, ssa.OpcodeFcmp): // This case we can use the Fpu flag directly.
2006 cvalInstr := cvalDef.Instr
2007 x, y, c := cvalInstr.FcmpData()
2008 cc = condFlagFromSSAFloatCmpCond(c)
2009 m.lowerFcmpToFlag(x, y)
2010 cvalDef.Instr.MarkLowered()
2011 default:
2012 rn := m.getOperand_NR(cvalDef, extModeNone)
2013 if c.Type() != ssa.TypeI32 && c.Type() != ssa.TypeI64 {
2014 panic("TODO?BUG?: support select with non-integer condition")
2015 }
2016 alu := m.allocateInstr()
2017 // subs zr, rn, zr
2018 alu.asALU(
2019 aluOpSubS,
2020 // We don't need the result, just need to set flags.
2021 xzrVReg,
2022 rn,
2023 operandNR(xzrVReg),
2024 c.Type().Bits() == 64,
2025 )
2026 m.insert(alu)
2027 cc = ne
2028 }
2029
2030 rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone)
2031 rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
2032
2033 rd := m.compiler.VRegOf(result)
2034 switch x.Type() {
2035 case ssa.TypeI32, ssa.TypeI64:
2036 // csel rd, rn, rm, cc
2037 csel := m.allocateInstr()
2038 csel.asCSel(rd, rn, rm, cc, x.Type().Bits() == 64)
2039 m.insert(csel)
2040 case ssa.TypeF32, ssa.TypeF64:
2041 // fcsel rd, rn, rm, cc
2042 fcsel := m.allocateInstr()
2043 fcsel.asFpuCSel(rd, rn, rm, cc, x.Type().Bits() == 64)
2044 m.insert(fcsel)
2045 default:
2046 panic("BUG")
2047 }
2048 }
2049
2050 func (m *machine) lowerSelectVec(rc, rn, rm operand, rd regalloc.VReg) {
2051 // First check if `rc` is zero or not.
2052 checkZero := m.allocateInstr()
2053 checkZero.asALU(aluOpSubS, xzrVReg, rc, operandNR(xzrVReg), false)
2054 m.insert(checkZero)
2055
2056 // Then use CSETM to set all bits to one if `rc` is zero.
2057 allOnesOrZero := m.compiler.AllocateVReg(ssa.TypeI64)
2058 cset := m.allocateInstr()
2059 cset.asCSet(allOnesOrZero, true, ne)
2060 m.insert(cset)
2061
2062 // Then move the bits to the result vector register.
2063 tmp2 := m.compiler.AllocateVReg(ssa.TypeV128)
2064 dup := m.allocateInstr()
2065 dup.asVecDup(tmp2, operandNR(allOnesOrZero), vecArrangement2D)
2066 m.insert(dup)
2067
2068 // Now that `tmp2` has either all bits one or zero depending on `rc`,
2069 // we can use bsl to select between `rn` and `rm`.
2070 ins := m.allocateInstr()
2071 ins.asVecRRRRewrite(vecOpBsl, tmp2, rn, rm, vecArrangement16B)
2072 m.insert(ins)
2073
2074 // Finally, move the result to the destination register.
2075 mov2 := m.allocateInstr()
2076 mov2.asFpuMov128(rd, tmp2)
2077 m.insert(mov2)
2078 }
2079
2080 func (m *machine) lowerAtomicRmw(si *ssa.Instruction) {
2081 ssaOp, size := si.AtomicRmwData()
2082
2083 var op atomicRmwOp
2084 var negateArg bool
2085 var flipArg bool
2086 switch ssaOp {
2087 case ssa.AtomicRmwOpAdd:
2088 op = atomicRmwOpAdd
2089 case ssa.AtomicRmwOpSub:
2090 op = atomicRmwOpAdd
2091 negateArg = true
2092 case ssa.AtomicRmwOpAnd:
2093 op = atomicRmwOpClr
2094 flipArg = true
2095 case ssa.AtomicRmwOpOr:
2096 op = atomicRmwOpSet
2097 case ssa.AtomicRmwOpXor:
2098 op = atomicRmwOpEor
2099 case ssa.AtomicRmwOpXchg:
2100 op = atomicRmwOpSwp
2101 default:
2102 panic(fmt.Sprintf("unknown ssa atomic rmw op: %s", ssaOp))
2103 }
2104
2105 addr, val := si.Arg2()
2106 addrDef, valDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(val)
2107 rn := m.getOperand_NR(addrDef, extModeNone)
2108 rt := m.compiler.VRegOf(si.Return())
2109 rs := m.getOperand_NR(valDef, extModeNone)
2110
2111 _64 := si.Return().Type().Bits() == 64
2112 var tmp regalloc.VReg
2113 if _64 {
2114 tmp = m.compiler.AllocateVReg(ssa.TypeI64)
2115 } else {
2116 tmp = m.compiler.AllocateVReg(ssa.TypeI32)
2117 }
2118 m.lowerAtomicRmwImpl(op, rn.nr(), rs.nr(), rt, tmp, size, negateArg, flipArg, _64)
2119 }
2120
2121 func (m *machine) lowerAtomicRmwImpl(op atomicRmwOp, rn, rs, rt, tmp regalloc.VReg, size uint64, negateArg, flipArg, dst64bit bool) {
2122 switch {
2123 case negateArg:
2124 neg := m.allocateInstr()
2125 neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), operandNR(rs), dst64bit)
2126 m.insert(neg)
2127 case flipArg:
2128 flip := m.allocateInstr()
2129 flip.asALU(aluOpOrn, tmp, operandNR(xzrVReg), operandNR(rs), dst64bit)
2130 m.insert(flip)
2131 default:
2132 tmp = rs
2133 }
2134
2135 rmw := m.allocateInstr()
2136 rmw.asAtomicRmw(op, rn, tmp, rt, size)
2137 m.insert(rmw)
2138 }
2139
2140 func (m *machine) lowerAtomicCas(si *ssa.Instruction) {
2141 addr, exp, repl := si.Arg3()
2142 size := si.AtomicTargetSize()
2143
2144 addrDef, expDef, replDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(exp), m.compiler.ValueDefinition(repl)
2145 rn := m.getOperand_NR(addrDef, extModeNone)
2146 rt := m.getOperand_NR(replDef, extModeNone)
2147 rs := m.getOperand_NR(expDef, extModeNone)
2148 tmp := m.compiler.AllocateVReg(si.Return().Type())
2149
2150 _64 := si.Return().Type().Bits() == 64
2151 // rs is overwritten by CAS, so we need to move it to the result register before the instruction
2152 // in case when it is used somewhere else.
2153 mov := m.allocateInstr()
2154 if _64 {
2155 mov.asMove64(tmp, rs.nr())
2156 } else {
2157 mov.asMove32(tmp, rs.nr())
2158 }
2159 m.insert(mov)
2160
2161 m.lowerAtomicCasImpl(rn.nr(), tmp, rt.nr(), size)
2162
2163 mov2 := m.allocateInstr()
2164 rd := m.compiler.VRegOf(si.Return())
2165 if _64 {
2166 mov2.asMove64(rd, tmp)
2167 } else {
2168 mov2.asMove32(rd, tmp)
2169 }
2170 m.insert(mov2)
2171 }
2172
2173 func (m *machine) lowerAtomicCasImpl(rn, rs, rt regalloc.VReg, size uint64) {
2174 cas := m.allocateInstr()
2175 cas.asAtomicCas(rn, rs, rt, size)
2176 m.insert(cas)
2177 }
2178
2179 func (m *machine) lowerAtomicLoad(si *ssa.Instruction) {
2180 addr := si.Arg()
2181 size := si.AtomicTargetSize()
2182
2183 addrDef := m.compiler.ValueDefinition(addr)
2184 rn := m.getOperand_NR(addrDef, extModeNone)
2185 rt := m.compiler.VRegOf(si.Return())
2186
2187 m.lowerAtomicLoadImpl(rn.nr(), rt, size)
2188 }
2189
2190 func (m *machine) lowerAtomicLoadImpl(rn, rt regalloc.VReg, size uint64) {
2191 ld := m.allocateInstr()
2192 ld.asAtomicLoad(rn, rt, size)
2193 m.insert(ld)
2194 }
2195
2196 func (m *machine) lowerAtomicStore(si *ssa.Instruction) {
2197 addr, val := si.Arg2()
2198 size := si.AtomicTargetSize()
2199
2200 addrDef := m.compiler.ValueDefinition(addr)
2201 valDef := m.compiler.ValueDefinition(val)
2202 rn := m.getOperand_NR(addrDef, extModeNone)
2203 rt := m.getOperand_NR(valDef, extModeNone)
2204
2205 m.lowerAtomicStoreImpl(rn, rt, size)
2206 }
2207
2208 func (m *machine) lowerAtomicStoreImpl(rn, rt operand, size uint64) {
2209 ld := m.allocateInstr()
2210 ld.asAtomicStore(rn, rt, size)
2211 m.insert(ld)
2212 }
2213
2214 // copyToTmp copies the given regalloc.VReg to a temporary register. This is called before cbr to avoid the regalloc issue
2215 // e.g. reload happening in the middle of the exit sequence which is not the path the normal path executes
2216 func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg {
2217 typ := m.compiler.TypeOf(v)
2218 mov := m.allocateInstr()
2219 tmp := m.compiler.AllocateVReg(typ)
2220 if typ.IsInt() {
2221 mov.asMove64(tmp, v)
2222 } else {
2223 mov.asFpuMov128(tmp, v)
2224 }
2225 m.insert(mov)
2226 return tmp
2227 }