Debian Codegraph @

internal/engine/wazevo/backend/isa/amd64/machine.go

raw
1 package amd64
2
3 import (
4 "context"
5 "encoding/binary"
6 "fmt"
7 "math"
8 "strings"
9
10 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
11 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
12 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
13 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
14 "github.com/tetratelabs/wazero/internal/platform"
15 )
16
17 // NewBackend returns a new backend for arm64.
18 func NewBackend() backend.Machine {
19 m := &machine{
20 cpuFeatures: platform.CpuFeatures,
21 regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo),
22 spillSlots: map[regalloc.VRegID]int64{},
23 amodePool: wazevoapi.NewPool[amode](nil),
24 labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition),
25 instrPool: wazevoapi.NewPool[instruction](resetInstruction),
26 constSwizzleMaskConstIndex: -1,
27 constSqmulRoundSatIndex: -1,
28 constI8x16SHLMaskTableIndex: -1,
29 constI8x16LogicalSHRMaskTableIndex: -1,
30 constF64x2CvtFromIMaskIndex: -1,
31 constTwop52Index: -1,
32 constI32sMaxOnF64x2Index: -1,
33 constI32uMaxOnF64x2Index: -1,
34 constAllOnesI8x16Index: -1,
35 constAllOnesI16x8Index: -1,
36 constExtAddPairwiseI16x8uMask1Index: -1,
37 constExtAddPairwiseI16x8uMask2Index: -1,
38 }
39 m.regAllocFn.m = m
40 return m
41 }
42
43 type (
44 // machine implements backend.Machine for amd64.
45 machine struct {
46 c backend.Compiler
47 stackBoundsCheckDisabled bool
48
49 instrPool wazevoapi.Pool[instruction]
50 amodePool wazevoapi.Pool[amode]
51
52 cpuFeatures platform.CpuFeatureFlags
53
54 regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn]
55 regAllocFn regAllocFn
56 regAllocStarted bool
57
58 // labelPositionPool is the pool of labelPosition. The id is the label where
59 // if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID.
60 labelPositionPool wazevoapi.IDedPool[labelPosition]
61 // nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID
62 // so that we can have an identical label for the SSA block ID, which is useful for debugging.
63 nextLabel label
64 // rootInstr is the first instruction of the function.
65 rootInstr *instruction
66 // currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition.
67 currentLabelPos *labelPosition
68 // orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock.
69 orderedSSABlockLabelPos []*labelPosition
70 // returnLabelPos is the labelPosition for the return block.
71 returnLabelPos labelPosition
72 // perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock.
73 perBlockHead, perBlockEnd *instruction
74 // pendingInstructions are the instructions which are not yet emitted into the instruction list.
75 pendingInstructions []*instruction
76 // maxSSABlockID is the maximum ssa.BasicBlockID in the current function.
77 maxSSABlockID label
78
79 spillSlotSize int64
80 spillSlots map[regalloc.VRegID]int64
81 currentABI *backend.FunctionABI
82 clobberedRegs []regalloc.VReg
83
84 maxRequiredStackSizeForCalls int64
85
86 labelResolutionPends []labelResolutionPend
87
88 // jmpTableTargets holds the labels of the jump table targets.
89 jmpTableTargets [][]uint32
90 // jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table.
91 jmpTableTargetsNext int
92 consts []_const
93
94 constSwizzleMaskConstIndex, constSqmulRoundSatIndex,
95 constI8x16SHLMaskTableIndex, constI8x16LogicalSHRMaskTableIndex,
96 constF64x2CvtFromIMaskIndex, constTwop52Index,
97 constI32sMaxOnF64x2Index, constI32uMaxOnF64x2Index,
98 constAllOnesI8x16Index, constAllOnesI16x8Index,
99 constExtAddPairwiseI16x8uMask1Index, constExtAddPairwiseI16x8uMask2Index int
100 }
101
102 _const struct {
103 lo, hi uint64
104 _var []byte
105 label label
106 labelPos *labelPosition
107 }
108
109 labelResolutionPend struct {
110 instr *instruction
111 instrOffset int64
112 // imm32Offset is the offset of the last 4 bytes of the instruction.
113 imm32Offset int64
114 }
115 )
116
117 type (
118 // label represents a position in the generated code which is either
119 // a real instruction or the constant InstructionPool (e.g. jump tables).
120 //
121 // This is exactly the same as the traditional "label" in assembly code.
122 label uint32
123
124 // labelPosition represents the regions of the generated code which the label represents.
125 // This implements regalloc.Block.
126 labelPosition struct {
127 // sb is not nil if this corresponds to a ssa.BasicBlock.
128 sb ssa.BasicBlock
129 // cur is used to walk through the instructions in the block during the register allocation.
130 cur,
131 // begin and end are the first and last instructions of the block.
132 begin, end *instruction
133 // binaryOffset is the offset in the binary where the label is located.
134 binaryOffset int64
135 }
136 )
137
138 // String implements backend.Machine.
139 func (l label) String() string {
140 return fmt.Sprintf("L%d", l)
141 }
142
143 func resetLabelPosition(l *labelPosition) {
144 *l = labelPosition{}
145 }
146
147 const labelReturn = math.MaxUint32
148
149 func ssaBlockLabel(sb ssa.BasicBlock) label {
150 if sb.ReturnBlock() {
151 return labelReturn
152 }
153 return label(sb.ID())
154 }
155
156 // getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block.
157 func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition {
158 if sb.ReturnBlock() {
159 m.returnLabelPos.sb = sb
160 return &m.returnLabelPos
161 }
162
163 l := ssaBlockLabel(sb)
164 pos := m.labelPositionPool.GetOrAllocate(int(l))
165 pos.sb = sb
166 return pos
167 }
168
169 func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) label {
170 index := *i
171 if index == -1 {
172 l, pos := m.allocateLabel()
173 index = len(m.consts)
174 m.consts = append(m.consts, _const{
175 _var: _var,
176 label: l,
177 labelPos: pos,
178 })
179 *i = index
180 }
181 return m.consts[index].label
182 }
183
184 // Reset implements backend.Machine.
185 func (m *machine) Reset() {
186 m.consts = m.consts[:0]
187 m.clobberedRegs = m.clobberedRegs[:0]
188 for key := range m.spillSlots {
189 m.clobberedRegs = append(m.clobberedRegs, regalloc.VReg(key))
190 }
191 for _, key := range m.clobberedRegs {
192 delete(m.spillSlots, regalloc.VRegID(key))
193 }
194
195 m.stackBoundsCheckDisabled = false
196 m.regAlloc.Reset()
197 m.labelPositionPool.Reset()
198 m.instrPool.Reset()
199 m.regAllocStarted = false
200 m.clobberedRegs = m.clobberedRegs[:0]
201
202 m.spillSlotSize = 0
203 m.maxRequiredStackSizeForCalls = 0
204 m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil
205 m.pendingInstructions = m.pendingInstructions[:0]
206 m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0]
207
208 m.amodePool.Reset()
209 m.jmpTableTargetsNext = 0
210 m.constSwizzleMaskConstIndex = -1
211 m.constSqmulRoundSatIndex = -1
212 m.constI8x16SHLMaskTableIndex = -1
213 m.constI8x16LogicalSHRMaskTableIndex = -1
214 m.constF64x2CvtFromIMaskIndex = -1
215 m.constTwop52Index = -1
216 m.constI32sMaxOnF64x2Index = -1
217 m.constI32uMaxOnF64x2Index = -1
218 m.constAllOnesI8x16Index = -1
219 m.constAllOnesI16x8Index = -1
220 m.constExtAddPairwiseI16x8uMask1Index = -1
221 m.constExtAddPairwiseI16x8uMask2Index = -1
222 }
223
224 // StartLoweringFunction implements backend.Machine StartLoweringFunction.
225 func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) {
226 m.maxSSABlockID = label(maxBlockID)
227 m.nextLabel = label(maxBlockID) + 1
228 }
229
230 // LinkAdjacentBlocks implements backend.Machine.
231 func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) {
232 prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next)
233 prevPos.end.next = nextPos.begin
234 }
235
236 // StartBlock implements backend.Machine.
237 func (m *machine) StartBlock(blk ssa.BasicBlock) {
238 m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk)
239 labelPos := m.currentLabelPos
240 end := m.allocateNop()
241 m.perBlockHead, m.perBlockEnd = end, end
242 labelPos.begin, labelPos.end = end, end
243 m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos)
244 }
245
246 // EndBlock implements ExecutableContext.
247 func (m *machine) EndBlock() {
248 // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions.
249 m.insertAtPerBlockHead(m.allocateNop())
250
251 m.currentLabelPos.begin = m.perBlockHead
252
253 if m.currentLabelPos.sb.EntryBlock() {
254 m.rootInstr = m.perBlockHead
255 }
256 }
257
258 func (m *machine) insertAtPerBlockHead(i *instruction) {
259 if m.perBlockHead == nil {
260 m.perBlockHead = i
261 m.perBlockEnd = i
262 return
263 }
264
265 i.next = m.perBlockHead
266 m.perBlockHead.prev = i
267 m.perBlockHead = i
268 }
269
270 // FlushPendingInstructions implements backend.Machine.
271 func (m *machine) FlushPendingInstructions() {
272 l := len(m.pendingInstructions)
273 if l == 0 {
274 return
275 }
276 for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order.
277 m.insertAtPerBlockHead(m.pendingInstructions[i])
278 }
279 m.pendingInstructions = m.pendingInstructions[:0]
280 }
281
282 // DisableStackCheck implements backend.Machine.
283 func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true }
284
285 // SetCompiler implements backend.Machine.
286 func (m *machine) SetCompiler(c backend.Compiler) {
287 m.c = c
288 m.regAllocFn.ssaB = c.SSABuilder()
289 }
290
291 // SetCurrentABI implements backend.Machine.
292 func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { m.currentABI = abi }
293
294 // RegAlloc implements backend.Machine.
295 func (m *machine) RegAlloc() {
296 rf := m.regAllocFn
297 m.regAllocStarted = true
298 m.regAlloc.DoAllocation(&rf)
299 // Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes.
300 m.spillSlotSize = (m.spillSlotSize + 15) &^ 15
301 }
302
303 // InsertReturn implements backend.Machine.
304 func (m *machine) InsertReturn() {
305 i := m.allocateInstr().asRet()
306 m.insert(i)
307 }
308
309 // LowerSingleBranch implements backend.Machine.
310 func (m *machine) LowerSingleBranch(b *ssa.Instruction) {
311 switch b.Opcode() {
312 case ssa.OpcodeJump:
313 _, _, targetBlkID := b.BranchData()
314 if b.IsFallthroughJump() {
315 return
316 }
317 jmp := m.allocateInstr()
318 target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID))
319 if target == labelReturn {
320 jmp.asRet()
321 } else {
322 jmp.asJmp(newOperandLabel(target))
323 }
324 m.insert(jmp)
325 case ssa.OpcodeBrTable:
326 index, targetBlkIDs := b.BrTableData()
327 m.lowerBrTable(index, targetBlkIDs)
328 default:
329 panic("BUG: unexpected branch opcode" + b.Opcode().String())
330 }
331 }
332
333 func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) {
334 if m.jmpTableTargetsNext == len(m.jmpTableTargets) {
335 m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View())))
336 }
337
338 index = m.jmpTableTargetsNext
339 m.jmpTableTargetsNext++
340 m.jmpTableTargets[index] = m.jmpTableTargets[index][:0]
341 for _, targetBlockID := range targets.View() {
342 target := m.c.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID))
343 m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(ssaBlockLabel(target)))
344 }
345 return
346 }
347
348 var condBranchMatches = [...]ssa.Opcode{ssa.OpcodeIcmp, ssa.OpcodeFcmp}
349
350 func (m *machine) lowerBrTable(index ssa.Value, targets ssa.Values) {
351 _v := m.getOperand_Reg(m.c.ValueDefinition(index))
352 v := m.copyToTmp(_v.reg())
353
354 targetCount := len(targets.View())
355
356 // First, we need to do the bounds check.
357 maxIndex := m.c.AllocateVReg(ssa.TypeI32)
358 m.lowerIconst(maxIndex, uint64(targetCount-1), false)
359 cmp := m.allocateInstr().asCmpRmiR(true, newOperandReg(maxIndex), v, false)
360 m.insert(cmp)
361
362 // Then do the conditional move maxIndex to v if v > maxIndex.
363 cmov := m.allocateInstr().asCmove(condNB, newOperandReg(maxIndex), v, false)
364 m.insert(cmov)
365
366 // Now that v has the correct index. Load the address of the jump table into the addr.
367 addr := m.c.AllocateVReg(ssa.TypeI64)
368 leaJmpTableAddr := m.allocateInstr()
369 m.insert(leaJmpTableAddr)
370
371 // Then add the target's offset into jmpTableAddr.
372 loadTargetOffsetFromJmpTable := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd,
373 // Shift by 3 because each entry is 8 bytes.
374 newOperandMem(m.newAmodeRegRegShift(0, addr, v, 3)), addr, true)
375 m.insert(loadTargetOffsetFromJmpTable)
376
377 // Now ready to jump.
378 jmp := m.allocateInstr().asJmp(newOperandReg(addr))
379 m.insert(jmp)
380
381 jmpTableBegin, jmpTableBeginLabel := m.allocateBrTarget()
382 m.insert(jmpTableBegin)
383 leaJmpTableAddr.asLEA(newOperandLabel(jmpTableBeginLabel), addr)
384
385 jmpTable := m.allocateInstr()
386 targetSliceIndex := m.addJmpTableTarget(targets)
387 jmpTable.asJmpTableSequence(targetSliceIndex, targetCount)
388 m.insert(jmpTable)
389 }
390
391 // LowerConditionalBranch implements backend.Machine.
392 func (m *machine) LowerConditionalBranch(b *ssa.Instruction) {
393 cval, args, targetBlkID := b.BranchData()
394 if len(args) > 0 {
395 panic(fmt.Sprintf(
396 "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s",
397 m.currentLabelPos.sb,
398 targetBlkID,
399 ))
400 }
401
402 target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID))
403 cvalDef := m.c.ValueDefinition(cval)
404
405 switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) {
406 case ssa.OpcodeIcmp:
407 cvalInstr := cvalDef.Instr
408 x, y, c := cvalInstr.IcmpData()
409
410 cc := condFromSSAIntCmpCond(c)
411 if b.Opcode() == ssa.OpcodeBrz {
412 cc = cc.invert()
413 }
414
415 // First, perform the comparison and set the flag.
416 xd, yd := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
417 if !m.tryLowerBandToFlag(xd, yd) {
418 m.lowerIcmpToFlag(xd, yd, x.Type() == ssa.TypeI64)
419 }
420
421 // Then perform the conditional branch.
422 m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target)))
423 cvalDef.Instr.MarkLowered()
424 case ssa.OpcodeFcmp:
425 cvalInstr := cvalDef.Instr
426
427 f1, f2, and := m.lowerFcmpToFlags(cvalInstr)
428 isBrz := b.Opcode() == ssa.OpcodeBrz
429 if isBrz {
430 f1 = f1.invert()
431 }
432 if f2 == condInvalid {
433 m.insert(m.allocateInstr().asJmpIf(f1, newOperandLabel(target)))
434 } else {
435 if isBrz {
436 f2 = f2.invert()
437 and = !and
438 }
439 jmp1, jmp2 := m.allocateInstr(), m.allocateInstr()
440 m.insert(jmp1)
441 m.insert(jmp2)
442 notTaken, notTakenLabel := m.allocateBrTarget()
443 m.insert(notTaken)
444 if and {
445 jmp1.asJmpIf(f1.invert(), newOperandLabel(notTakenLabel))
446 jmp2.asJmpIf(f2, newOperandLabel(target))
447 } else {
448 jmp1.asJmpIf(f1, newOperandLabel(target))
449 jmp2.asJmpIf(f2, newOperandLabel(target))
450 }
451 }
452
453 cvalDef.Instr.MarkLowered()
454 default:
455 v := m.getOperand_Reg(cvalDef)
456
457 var cc cond
458 if b.Opcode() == ssa.OpcodeBrz {
459 cc = condZ
460 } else {
461 cc = condNZ
462 }
463
464 // Perform test %v, %v to set the flag.
465 cmp := m.allocateInstr().asCmpRmiR(false, v, v.reg(), false)
466 m.insert(cmp)
467 m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target)))
468 }
469 }
470
471 // LowerInstr implements backend.Machine.
472 func (m *machine) LowerInstr(instr *ssa.Instruction) {
473 if l := instr.SourceOffset(); l.Valid() {
474 info := m.allocateInstr().asEmitSourceOffsetInfo(l)
475 m.insert(info)
476 }
477
478 switch op := instr.Opcode(); op {
479 case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable:
480 panic("BUG: branching instructions are handled by LowerBranches")
481 case ssa.OpcodeReturn:
482 panic("BUG: return must be handled by backend.Compiler")
483 case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined.
484 case ssa.OpcodeCall, ssa.OpcodeCallIndirect:
485 m.lowerCall(instr)
486 case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32:
487 m.lowerStore(instr)
488 case ssa.OpcodeIadd:
489 m.lowerAluRmiROp(instr, aluRmiROpcodeAdd)
490 case ssa.OpcodeIsub:
491 m.lowerAluRmiROp(instr, aluRmiROpcodeSub)
492 case ssa.OpcodeImul:
493 m.lowerAluRmiROp(instr, aluRmiROpcodeMul)
494 case ssa.OpcodeSdiv, ssa.OpcodeUdiv, ssa.OpcodeSrem, ssa.OpcodeUrem:
495 isDiv := op == ssa.OpcodeSdiv || op == ssa.OpcodeUdiv
496 isSigned := op == ssa.OpcodeSdiv || op == ssa.OpcodeSrem
497 m.lowerIDivRem(instr, isDiv, isSigned)
498 case ssa.OpcodeBand:
499 m.lowerAluRmiROp(instr, aluRmiROpcodeAnd)
500 case ssa.OpcodeBor:
501 m.lowerAluRmiROp(instr, aluRmiROpcodeOr)
502 case ssa.OpcodeBxor:
503 m.lowerAluRmiROp(instr, aluRmiROpcodeXor)
504 case ssa.OpcodeIshl:
505 m.lowerShiftR(instr, shiftROpShiftLeft)
506 case ssa.OpcodeSshr:
507 m.lowerShiftR(instr, shiftROpShiftRightArithmetic)
508 case ssa.OpcodeUshr:
509 m.lowerShiftR(instr, shiftROpShiftRightLogical)
510 case ssa.OpcodeRotl:
511 m.lowerShiftR(instr, shiftROpRotateLeft)
512 case ssa.OpcodeRotr:
513 m.lowerShiftR(instr, shiftROpRotateRight)
514 case ssa.OpcodeClz:
515 m.lowerClz(instr)
516 case ssa.OpcodeCtz:
517 m.lowerCtz(instr)
518 case ssa.OpcodePopcnt:
519 m.lowerUnaryRmR(instr, unaryRmROpcodePopcnt)
520 case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv:
521 m.lowerXmmRmR(instr)
522 case ssa.OpcodeFabs:
523 m.lowerFabsFneg(instr)
524 case ssa.OpcodeFneg:
525 m.lowerFabsFneg(instr)
526 case ssa.OpcodeCeil:
527 m.lowerRound(instr, roundingModeUp)
528 case ssa.OpcodeFloor:
529 m.lowerRound(instr, roundingModeDown)
530 case ssa.OpcodeTrunc:
531 m.lowerRound(instr, roundingModeZero)
532 case ssa.OpcodeNearest:
533 m.lowerRound(instr, roundingModeNearest)
534 case ssa.OpcodeFmin, ssa.OpcodeFmax:
535 m.lowerFminFmax(instr)
536 case ssa.OpcodeFcopysign:
537 m.lowerFcopysign(instr)
538 case ssa.OpcodeBitcast:
539 m.lowerBitcast(instr)
540 case ssa.OpcodeSqrt:
541 m.lowerSqrt(instr)
542 case ssa.OpcodeFpromote:
543 v := instr.Arg()
544 rn := m.getOperand_Reg(m.c.ValueDefinition(v))
545 rd := m.c.VRegOf(instr.Return())
546 cnt := m.allocateInstr()
547 cnt.asXmmUnaryRmR(sseOpcodeCvtss2sd, rn, rd)
548 m.insert(cnt)
549 case ssa.OpcodeFdemote:
550 v := instr.Arg()
551 rn := m.getOperand_Reg(m.c.ValueDefinition(v))
552 rd := m.c.VRegOf(instr.Return())
553 cnt := m.allocateInstr()
554 cnt.asXmmUnaryRmR(sseOpcodeCvtsd2ss, rn, rd)
555 m.insert(cnt)
556 case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat:
557 x, ctx := instr.Arg2()
558 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
559 rd := m.c.VRegOf(instr.Return())
560 ctxVReg := m.c.VRegOf(ctx)
561 m.lowerFcvtToSint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64,
562 instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat)
563 case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat:
564 x, ctx := instr.Arg2()
565 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
566 rd := m.c.VRegOf(instr.Return())
567 ctxVReg := m.c.VRegOf(ctx)
568 m.lowerFcvtToUint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64,
569 instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat)
570 case ssa.OpcodeFcvtFromSint:
571 x := instr.Arg()
572 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
573 rd := newOperandReg(m.c.VRegOf(instr.Return()))
574 m.lowerFcvtFromSint(rn, rd,
575 x.Type() == ssa.TypeI64, instr.Return().Type().Bits() == 64)
576 case ssa.OpcodeFcvtFromUint:
577 x := instr.Arg()
578 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
579 rd := newOperandReg(m.c.VRegOf(instr.Return()))
580 m.lowerFcvtFromUint(rn, rd, x.Type() == ssa.TypeI64,
581 instr.Return().Type().Bits() == 64)
582 case ssa.OpcodeVanyTrue:
583 m.lowerVanyTrue(instr)
584 case ssa.OpcodeVallTrue:
585 m.lowerVallTrue(instr)
586 case ssa.OpcodeVhighBits:
587 m.lowerVhighBits(instr)
588 case ssa.OpcodeVbnot:
589 m.lowerVbnot(instr)
590 case ssa.OpcodeVband:
591 x, y := instr.Arg2()
592 m.lowerVbBinOp(sseOpcodePand, x, y, instr.Return())
593 case ssa.OpcodeVbor:
594 x, y := instr.Arg2()
595 m.lowerVbBinOp(sseOpcodePor, x, y, instr.Return())
596 case ssa.OpcodeVbxor:
597 x, y := instr.Arg2()
598 m.lowerVbBinOp(sseOpcodePxor, x, y, instr.Return())
599 case ssa.OpcodeVbandnot:
600 m.lowerVbandnot(instr, sseOpcodePandn)
601 case ssa.OpcodeVbitselect:
602 m.lowerVbitselect(instr)
603 case ssa.OpcodeVIadd:
604 x, y, lane := instr.Arg2WithLane()
605 var vecOp sseOpcode
606 switch lane {
607 case ssa.VecLaneI8x16:
608 vecOp = sseOpcodePaddb
609 case ssa.VecLaneI16x8:
610 vecOp = sseOpcodePaddw
611 case ssa.VecLaneI32x4:
612 vecOp = sseOpcodePaddd
613 case ssa.VecLaneI64x2:
614 vecOp = sseOpcodePaddq
615 }
616 m.lowerVbBinOp(vecOp, x, y, instr.Return())
617
618 case ssa.OpcodeVSaddSat:
619 x, y, lane := instr.Arg2WithLane()
620 var vecOp sseOpcode
621 switch lane {
622 case ssa.VecLaneI8x16:
623 vecOp = sseOpcodePaddsb
624 case ssa.VecLaneI16x8:
625 vecOp = sseOpcodePaddsw
626 }
627 m.lowerVbBinOp(vecOp, x, y, instr.Return())
628
629 case ssa.OpcodeVUaddSat:
630 x, y, lane := instr.Arg2WithLane()
631 var vecOp sseOpcode
632 switch lane {
633 case ssa.VecLaneI8x16:
634 vecOp = sseOpcodePaddusb
635 case ssa.VecLaneI16x8:
636 vecOp = sseOpcodePaddusw
637 }
638 m.lowerVbBinOp(vecOp, x, y, instr.Return())
639
640 case ssa.OpcodeVIsub:
641 x, y, lane := instr.Arg2WithLane()
642 var vecOp sseOpcode
643 switch lane {
644 case ssa.VecLaneI8x16:
645 vecOp = sseOpcodePsubb
646 case ssa.VecLaneI16x8:
647 vecOp = sseOpcodePsubw
648 case ssa.VecLaneI32x4:
649 vecOp = sseOpcodePsubd
650 case ssa.VecLaneI64x2:
651 vecOp = sseOpcodePsubq
652 }
653 m.lowerVbBinOp(vecOp, x, y, instr.Return())
654
655 case ssa.OpcodeVSsubSat:
656 x, y, lane := instr.Arg2WithLane()
657 var vecOp sseOpcode
658 switch lane {
659 case ssa.VecLaneI8x16:
660 vecOp = sseOpcodePsubsb
661 case ssa.VecLaneI16x8:
662 vecOp = sseOpcodePsubsw
663 }
664 m.lowerVbBinOp(vecOp, x, y, instr.Return())
665
666 case ssa.OpcodeVUsubSat:
667 x, y, lane := instr.Arg2WithLane()
668 var vecOp sseOpcode
669 switch lane {
670 case ssa.VecLaneI8x16:
671 vecOp = sseOpcodePsubusb
672 case ssa.VecLaneI16x8:
673 vecOp = sseOpcodePsubusw
674 }
675 m.lowerVbBinOp(vecOp, x, y, instr.Return())
676
677 case ssa.OpcodeVImul:
678 m.lowerVImul(instr)
679 case ssa.OpcodeVIneg:
680 x, lane := instr.ArgWithLane()
681 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
682 rd := m.c.VRegOf(instr.Return())
683 var vecOp sseOpcode
684 switch lane {
685 case ssa.VecLaneI8x16:
686 vecOp = sseOpcodePsubb
687 case ssa.VecLaneI16x8:
688 vecOp = sseOpcodePsubw
689 case ssa.VecLaneI32x4:
690 vecOp = sseOpcodePsubd
691 case ssa.VecLaneI64x2:
692 vecOp = sseOpcodePsubq
693 default:
694 panic("BUG")
695 }
696
697 tmp := m.c.AllocateVReg(ssa.TypeV128)
698 m.insert(m.allocateInstr().asZeros(tmp))
699
700 i := m.allocateInstr()
701 i.asXmmRmR(vecOp, rn, tmp)
702 m.insert(i)
703
704 m.copyTo(tmp, rd)
705 case ssa.OpcodeVFadd:
706 x, y, lane := instr.Arg2WithLane()
707 var vecOp sseOpcode
708 switch lane {
709 case ssa.VecLaneF32x4:
710 vecOp = sseOpcodeAddps
711 case ssa.VecLaneF64x2:
712 vecOp = sseOpcodeAddpd
713 }
714 m.lowerVbBinOp(vecOp, x, y, instr.Return())
715
716 case ssa.OpcodeVFsub:
717 x, y, lane := instr.Arg2WithLane()
718 var vecOp sseOpcode
719 switch lane {
720 case ssa.VecLaneF32x4:
721 vecOp = sseOpcodeSubps
722 case ssa.VecLaneF64x2:
723 vecOp = sseOpcodeSubpd
724 }
725 m.lowerVbBinOp(vecOp, x, y, instr.Return())
726
727 case ssa.OpcodeVFdiv:
728 x, y, lane := instr.Arg2WithLane()
729 var vecOp sseOpcode
730 switch lane {
731 case ssa.VecLaneF32x4:
732 vecOp = sseOpcodeDivps
733 case ssa.VecLaneF64x2:
734 vecOp = sseOpcodeDivpd
735 }
736 m.lowerVbBinOp(vecOp, x, y, instr.Return())
737
738 case ssa.OpcodeVFmul:
739 x, y, lane := instr.Arg2WithLane()
740 var vecOp sseOpcode
741 switch lane {
742 case ssa.VecLaneF32x4:
743 vecOp = sseOpcodeMulps
744 case ssa.VecLaneF64x2:
745 vecOp = sseOpcodeMulpd
746 }
747 m.lowerVbBinOp(vecOp, x, y, instr.Return())
748
749 case ssa.OpcodeVFneg:
750 x, lane := instr.ArgWithLane()
751 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
752 rd := m.c.VRegOf(instr.Return())
753
754 tmp := m.c.AllocateVReg(ssa.TypeV128)
755
756 var shiftOp, xorOp sseOpcode
757 var shiftAmt uint32
758 switch lane {
759 case ssa.VecLaneF32x4:
760 shiftOp, shiftAmt, xorOp = sseOpcodePslld, 31, sseOpcodeXorps
761 case ssa.VecLaneF64x2:
762 shiftOp, shiftAmt, xorOp = sseOpcodePsllq, 63, sseOpcodeXorpd
763 }
764
765 zero := m.allocateInstr()
766 zero.asZeros(tmp)
767 m.insert(zero)
768
769 // Set all bits on tmp by CMPPD with arg=0 (== pseudo CMPEQPD instruction).
770 // See https://www.felixcloutier.com/x86/cmpps
771 //
772 // Note: if we do not clear all the bits ^ with XORPS, this might end up not setting ones on some lane
773 // if the lane is NaN.
774 cmp := m.allocateInstr()
775 cmp.asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_UQ), newOperandReg(tmp), tmp)
776 m.insert(cmp)
777
778 // Do the left shift on each lane to set only the most significant bit in each.
779 i := m.allocateInstr()
780 i.asXmmRmiReg(shiftOp, newOperandImm32(shiftAmt), tmp)
781 m.insert(i)
782
783 // Get the negated result by XOR on each lane with tmp.
784 i = m.allocateInstr()
785 i.asXmmRmR(xorOp, rn, tmp)
786 m.insert(i)
787
788 m.copyTo(tmp, rd)
789
790 case ssa.OpcodeVSqrt:
791 x, lane := instr.ArgWithLane()
792 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
793 rd := m.c.VRegOf(instr.Return())
794
795 var vecOp sseOpcode
796 switch lane {
797 case ssa.VecLaneF32x4:
798 vecOp = sseOpcodeSqrtps
799 case ssa.VecLaneF64x2:
800 vecOp = sseOpcodeSqrtpd
801 }
802 i := m.allocateInstr()
803 i.asXmmUnaryRmR(vecOp, rn, rd)
804 m.insert(i)
805
806 case ssa.OpcodeVImin:
807 x, y, lane := instr.Arg2WithLane()
808 var vecOp sseOpcode
809 switch lane {
810 case ssa.VecLaneI8x16:
811 vecOp = sseOpcodePminsb
812 case ssa.VecLaneI16x8:
813 vecOp = sseOpcodePminsw
814 case ssa.VecLaneI32x4:
815 vecOp = sseOpcodePminsd
816 }
817 m.lowerVbBinOp(vecOp, x, y, instr.Return())
818
819 case ssa.OpcodeVUmin:
820 x, y, lane := instr.Arg2WithLane()
821 var vecOp sseOpcode
822 switch lane {
823 case ssa.VecLaneI8x16:
824 vecOp = sseOpcodePminub
825 case ssa.VecLaneI16x8:
826 vecOp = sseOpcodePminuw
827 case ssa.VecLaneI32x4:
828 vecOp = sseOpcodePminud
829 }
830 m.lowerVbBinOp(vecOp, x, y, instr.Return())
831
832 case ssa.OpcodeVImax:
833 x, y, lane := instr.Arg2WithLane()
834 var vecOp sseOpcode
835 switch lane {
836 case ssa.VecLaneI8x16:
837 vecOp = sseOpcodePmaxsb
838 case ssa.VecLaneI16x8:
839 vecOp = sseOpcodePmaxsw
840 case ssa.VecLaneI32x4:
841 vecOp = sseOpcodePmaxsd
842 }
843 m.lowerVbBinOp(vecOp, x, y, instr.Return())
844
845 case ssa.OpcodeVUmax:
846 x, y, lane := instr.Arg2WithLane()
847 var vecOp sseOpcode
848 switch lane {
849 case ssa.VecLaneI8x16:
850 vecOp = sseOpcodePmaxub
851 case ssa.VecLaneI16x8:
852 vecOp = sseOpcodePmaxuw
853 case ssa.VecLaneI32x4:
854 vecOp = sseOpcodePmaxud
855 }
856 m.lowerVbBinOp(vecOp, x, y, instr.Return())
857
858 case ssa.OpcodeVAvgRound:
859 x, y, lane := instr.Arg2WithLane()
860 var vecOp sseOpcode
861 switch lane {
862 case ssa.VecLaneI8x16:
863 vecOp = sseOpcodePavgb
864 case ssa.VecLaneI16x8:
865 vecOp = sseOpcodePavgw
866 }
867 m.lowerVbBinOp(vecOp, x, y, instr.Return())
868
869 case ssa.OpcodeVIcmp:
870 x, y, c, lane := instr.VIcmpData()
871 m.lowerVIcmp(x, y, c, instr.Return(), lane)
872
873 case ssa.OpcodeVFcmp:
874 x, y, c, lane := instr.VFcmpData()
875 m.lowerVFcmp(x, y, c, instr.Return(), lane)
876
877 case ssa.OpcodeExtractlane:
878 x, index, signed, lane := instr.ExtractlaneData()
879 m.lowerExtractLane(x, index, signed, instr.Return(), lane)
880
881 case ssa.OpcodeInsertlane:
882 x, y, index, lane := instr.InsertlaneData()
883 m.lowerInsertLane(x, y, index, instr.Return(), lane)
884
885 case ssa.OpcodeSwizzle:
886 x, y, _ := instr.Arg2WithLane()
887 m.lowerSwizzle(x, y, instr.Return())
888
889 case ssa.OpcodeShuffle:
890 x, y, lo, hi := instr.ShuffleData()
891 m.lowerShuffle(x, y, lo, hi, instr.Return())
892
893 case ssa.OpcodeSplat:
894 x, lane := instr.ArgWithLane()
895 m.lowerSplat(x, instr.Return(), lane)
896
897 case ssa.OpcodeSqmulRoundSat:
898 x, y := instr.Arg2()
899 m.lowerSqmulRoundSat(x, y, instr.Return())
900
901 case ssa.OpcodeVZeroExtLoad:
902 ptr, offset, typ := instr.VZeroExtLoadData()
903 var sseOp sseOpcode
904 // Both movss and movsd clears the higher bits of the destination register upt 128 bits.
905 // https://www.felixcloutier.com/x86/movss
906 // https://www.felixcloutier.com/x86/movsd
907 if typ == ssa.TypeF32 {
908 sseOp = sseOpcodeMovss
909 } else {
910 sseOp = sseOpcodeMovsd
911 }
912 mem := m.lowerToAddressMode(ptr, offset)
913 dst := m.c.VRegOf(instr.Return())
914 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandMem(mem), dst))
915
916 case ssa.OpcodeVMinPseudo:
917 x, y, lane := instr.Arg2WithLane()
918 var vecOp sseOpcode
919 switch lane {
920 case ssa.VecLaneF32x4:
921 vecOp = sseOpcodeMinps
922 case ssa.VecLaneF64x2:
923 vecOp = sseOpcodeMinpd
924 default:
925 panic("BUG: unexpected lane type")
926 }
927 m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return())
928
929 case ssa.OpcodeVMaxPseudo:
930 x, y, lane := instr.Arg2WithLane()
931 var vecOp sseOpcode
932 switch lane {
933 case ssa.VecLaneF32x4:
934 vecOp = sseOpcodeMaxps
935 case ssa.VecLaneF64x2:
936 vecOp = sseOpcodeMaxpd
937 default:
938 panic("BUG: unexpected lane type")
939 }
940 m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return())
941
942 case ssa.OpcodeVIshl:
943 x, y, lane := instr.Arg2WithLane()
944 m.lowerVIshl(x, y, instr.Return(), lane)
945
946 case ssa.OpcodeVSshr:
947 x, y, lane := instr.Arg2WithLane()
948 m.lowerVSshr(x, y, instr.Return(), lane)
949
950 case ssa.OpcodeVUshr:
951 x, y, lane := instr.Arg2WithLane()
952 m.lowerVUshr(x, y, instr.Return(), lane)
953
954 case ssa.OpcodeVCeil:
955 x, lane := instr.ArgWithLane()
956 m.lowerVRound(x, instr.Return(), 0x2, lane == ssa.VecLaneF64x2)
957
958 case ssa.OpcodeVFloor:
959 x, lane := instr.ArgWithLane()
960 m.lowerVRound(x, instr.Return(), 0x1, lane == ssa.VecLaneF64x2)
961
962 case ssa.OpcodeVTrunc:
963 x, lane := instr.ArgWithLane()
964 m.lowerVRound(x, instr.Return(), 0x3, lane == ssa.VecLaneF64x2)
965
966 case ssa.OpcodeVNearest:
967 x, lane := instr.ArgWithLane()
968 m.lowerVRound(x, instr.Return(), 0x0, lane == ssa.VecLaneF64x2)
969
970 case ssa.OpcodeExtIaddPairwise:
971 x, lane, signed := instr.ExtIaddPairwiseData()
972 m.lowerExtIaddPairwise(x, instr.Return(), lane, signed)
973
974 case ssa.OpcodeUwidenLow, ssa.OpcodeSwidenLow:
975 x, lane := instr.ArgWithLane()
976 m.lowerWidenLow(x, instr.Return(), lane, op == ssa.OpcodeSwidenLow)
977
978 case ssa.OpcodeUwidenHigh, ssa.OpcodeSwidenHigh:
979 x, lane := instr.ArgWithLane()
980 m.lowerWidenHigh(x, instr.Return(), lane, op == ssa.OpcodeSwidenHigh)
981
982 case ssa.OpcodeLoadSplat:
983 ptr, offset, lane := instr.LoadSplatData()
984 m.lowerLoadSplat(ptr, offset, instr.Return(), lane)
985
986 case ssa.OpcodeVFcvtFromUint, ssa.OpcodeVFcvtFromSint:
987 x, lane := instr.ArgWithLane()
988 m.lowerVFcvtFromInt(x, instr.Return(), lane, op == ssa.OpcodeVFcvtFromSint)
989
990 case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat:
991 x, lane := instr.ArgWithLane()
992 m.lowerVFcvtToIntSat(x, instr.Return(), lane, op == ssa.OpcodeVFcvtToSintSat)
993
994 case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow:
995 x, y, lane := instr.Arg2WithLane()
996 m.lowerNarrow(x, y, instr.Return(), lane, op == ssa.OpcodeSnarrow)
997
998 case ssa.OpcodeFvpromoteLow:
999 x := instr.Arg()
1000 src := m.getOperand_Reg(m.c.ValueDefinition(x))
1001 dst := m.c.VRegOf(instr.Return())
1002 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtps2pd, src, dst))
1003
1004 case ssa.OpcodeFvdemote:
1005 x := instr.Arg()
1006 src := m.getOperand_Reg(m.c.ValueDefinition(x))
1007 dst := m.c.VRegOf(instr.Return())
1008 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtpd2ps, src, dst))
1009
1010 case ssa.OpcodeWideningPairwiseDotProductS:
1011 x, y := instr.Arg2()
1012 m.lowerWideningPairwiseDotProductS(x, y, instr.Return())
1013
1014 case ssa.OpcodeVIabs:
1015 m.lowerVIabs(instr)
1016 case ssa.OpcodeVIpopcnt:
1017 m.lowerVIpopcnt(instr)
1018 case ssa.OpcodeVFmin:
1019 m.lowerVFmin(instr)
1020 case ssa.OpcodeVFmax:
1021 m.lowerVFmax(instr)
1022 case ssa.OpcodeVFabs:
1023 m.lowerVFabs(instr)
1024 case ssa.OpcodeUndefined:
1025 m.insert(m.allocateInstr().asUD2())
1026 case ssa.OpcodeExitWithCode:
1027 execCtx, code := instr.ExitWithCodeData()
1028 m.lowerExitWithCode(m.c.VRegOf(execCtx), code)
1029 case ssa.OpcodeExitIfTrueWithCode:
1030 execCtx, c, code := instr.ExitIfTrueWithCodeData()
1031 m.lowerExitIfTrueWithCode(m.c.VRegOf(execCtx), c, code)
1032 case ssa.OpcodeLoad:
1033 ptr, offset, typ := instr.LoadData()
1034 dst := m.c.VRegOf(instr.Return())
1035 m.lowerLoad(ptr, offset, typ, dst)
1036 case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32:
1037 ptr, offset, _ := instr.LoadData()
1038 ret := m.c.VRegOf(instr.Return())
1039 m.lowerExtLoad(op, ptr, offset, ret)
1040 case ssa.OpcodeVconst:
1041 result := m.c.VRegOf(instr.Return())
1042 lo, hi := instr.VconstData()
1043 m.lowerVconst(result, lo, hi)
1044 case ssa.OpcodeSExtend, ssa.OpcodeUExtend:
1045 from, to, signed := instr.ExtendData()
1046 m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed)
1047 case ssa.OpcodeIcmp:
1048 m.lowerIcmp(instr)
1049 case ssa.OpcodeFcmp:
1050 m.lowerFcmp(instr)
1051 case ssa.OpcodeSelect:
1052 cval, x, y := instr.SelectData()
1053 m.lowerSelect(x, y, cval, instr.Return())
1054 case ssa.OpcodeIreduce:
1055 rn := m.getOperand_Mem_Reg(m.c.ValueDefinition(instr.Arg()))
1056 retVal := instr.Return()
1057 rd := m.c.VRegOf(retVal)
1058
1059 if retVal.Type() != ssa.TypeI32 {
1060 panic("TODO?: Ireduce to non-i32")
1061 }
1062 m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, rd))
1063
1064 case ssa.OpcodeAtomicLoad:
1065 ptr := instr.Arg()
1066 size := instr.AtomicTargetSize()
1067 dst := m.c.VRegOf(instr.Return())
1068
1069 // At this point, the ptr is ensured to be aligned, so using a normal load is atomic.
1070 // https://github.com/golang/go/blob/go1.24.0/src/internal/runtime/atomic/atomic_amd64.go#L29
1071 mem := newOperandMem(m.lowerToAddressMode(ptr, 0))
1072 load := m.allocateInstr()
1073 switch size {
1074 case 8:
1075 load.asMov64MR(mem, dst)
1076 case 4:
1077 load.asMovzxRmR(extModeLQ, mem, dst)
1078 case 2:
1079 load.asMovzxRmR(extModeWQ, mem, dst)
1080 case 1:
1081 load.asMovzxRmR(extModeBQ, mem, dst)
1082 default:
1083 panic("BUG")
1084 }
1085 m.insert(load)
1086
1087 case ssa.OpcodeFence:
1088 m.insert(m.allocateInstr().asMFence())
1089
1090 case ssa.OpcodeAtomicStore:
1091 ptr, _val := instr.Arg2()
1092 size := instr.AtomicTargetSize()
1093
1094 val := m.getOperand_Reg(m.c.ValueDefinition(_val))
1095 // The content on the val register will be overwritten by xchg, so we need to copy it to a temporary register.
1096 copied := m.copyToTmp(val.reg())
1097
1098 mem := newOperandMem(m.lowerToAddressMode(ptr, 0))
1099 store := m.allocateInstr().asXCHG(copied, mem, byte(size))
1100 m.insert(store)
1101
1102 case ssa.OpcodeAtomicCas:
1103 addr, exp, repl := instr.Arg3()
1104 size := instr.AtomicTargetSize()
1105 m.lowerAtomicCas(addr, exp, repl, size, instr.Return())
1106
1107 case ssa.OpcodeAtomicRmw:
1108 addr, val := instr.Arg2()
1109 atomicOp, size := instr.AtomicRmwData()
1110 m.lowerAtomicRmw(atomicOp, addr, val, size, instr.Return())
1111
1112 case ssa.OpcodeTailCallReturnCall, ssa.OpcodeTailCallReturnCallIndirect:
1113 m.lowerTailCall(instr)
1114
1115 default:
1116 panic("TODO: lowering " + op.String())
1117 }
1118 }
1119
1120 func (m *machine) lowerAtomicRmw(op ssa.AtomicRmwOp, addr, val ssa.Value, size uint64, ret ssa.Value) {
1121 mem := m.lowerToAddressMode(addr, 0)
1122 _val := m.getOperand_Reg(m.c.ValueDefinition(val))
1123
1124 switch op {
1125 case ssa.AtomicRmwOpAdd, ssa.AtomicRmwOpSub:
1126 valCopied := m.copyToTmp(_val.reg())
1127 if op == ssa.AtomicRmwOpSub {
1128 // Negate the value.
1129 m.insert(m.allocateInstr().asNeg(newOperandReg(valCopied), true))
1130 }
1131 m.insert(m.allocateInstr().asLockXAdd(valCopied, mem, byte(size)))
1132 m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
1133 m.copyTo(valCopied, m.c.VRegOf(ret))
1134
1135 case ssa.AtomicRmwOpAnd, ssa.AtomicRmwOpOr, ssa.AtomicRmwOpXor:
1136 accumulator := raxVReg
1137 // Reserve rax for the accumulator to make regalloc happy.
1138 // Note: do this initialization before defining valCopied, because it might be the same register and
1139 // if that happens, the unnecessary load/store will be performed inside the loop.
1140 // This can be mitigated in any way once the register allocator is clever enough.
1141 m.insert(m.allocateInstr().asDefineUninitializedReg(accumulator))
1142
1143 // Copy the value to a temporary register.
1144 valCopied := m.copyToTmp(_val.reg())
1145 m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
1146
1147 memOp := newOperandMem(mem)
1148 tmp := m.c.AllocateVReg(ssa.TypeI64)
1149 beginLoop, beginLoopLabel := m.allocateBrTarget()
1150 {
1151 m.insert(beginLoop)
1152 // Reset the value on tmp by the original value.
1153 m.copyTo(valCopied, tmp)
1154 // Load the current value at the memory location into accumulator.
1155 switch size {
1156 case 1:
1157 m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, memOp, accumulator))
1158 case 2:
1159 m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, memOp, accumulator))
1160 case 4:
1161 m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, memOp, accumulator))
1162 case 8:
1163 m.insert(m.allocateInstr().asMov64MR(memOp, accumulator))
1164 default:
1165 panic("BUG")
1166 }
1167 // Then perform the logical operation on the accumulator and the value on tmp.
1168 switch op {
1169 case ssa.AtomicRmwOpAnd:
1170 m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, newOperandReg(accumulator), tmp, true))
1171 case ssa.AtomicRmwOpOr:
1172 m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeOr, newOperandReg(accumulator), tmp, true))
1173 case ssa.AtomicRmwOpXor:
1174 m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(accumulator), tmp, true))
1175 default:
1176 panic("BUG")
1177 }
1178 // Finally, try compare-exchange the value at the memory location with the tmp.
1179 m.insert(m.allocateInstr().asLockCmpXCHG(tmp, memOp.addressMode(), byte(size)))
1180 // If it succeeds, ZF will be set, and we can break the loop.
1181 m.insert(m.allocateInstr().asJmpIf(condNZ, newOperandLabel(beginLoopLabel)))
1182 }
1183
1184 // valCopied must be alive at the end of the loop.
1185 m.insert(m.allocateInstr().asNopUseReg(valCopied))
1186
1187 // At this point, accumulator contains the result.
1188 m.clearHigherBitsForAtomic(accumulator, size, ret.Type())
1189 m.copyTo(accumulator, m.c.VRegOf(ret))
1190
1191 case ssa.AtomicRmwOpXchg:
1192 valCopied := m.copyToTmp(_val.reg())
1193
1194 m.insert(m.allocateInstr().asXCHG(valCopied, newOperandMem(mem), byte(size)))
1195 m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
1196 m.copyTo(valCopied, m.c.VRegOf(ret))
1197
1198 default:
1199 panic("BUG")
1200 }
1201 }
1202
1203 func (m *machine) lowerAtomicCas(addr, exp, repl ssa.Value, size uint64, ret ssa.Value) {
1204 mem := m.lowerToAddressMode(addr, 0)
1205 expOp := m.getOperand_Reg(m.c.ValueDefinition(exp))
1206 replOp := m.getOperand_Reg(m.c.ValueDefinition(repl))
1207
1208 accumulator := raxVReg
1209 m.copyTo(expOp.reg(), accumulator)
1210 m.insert(m.allocateInstr().asLockCmpXCHG(replOp.reg(), mem, byte(size)))
1211 m.clearHigherBitsForAtomic(accumulator, size, ret.Type())
1212 m.copyTo(accumulator, m.c.VRegOf(ret))
1213 }
1214
1215 func (m *machine) clearHigherBitsForAtomic(r regalloc.VReg, valSize uint64, resultType ssa.Type) {
1216 switch resultType {
1217 case ssa.TypeI32:
1218 switch valSize {
1219 case 1:
1220 m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(r), r))
1221 case 2:
1222 m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(r), r))
1223 }
1224 case ssa.TypeI64:
1225 switch valSize {
1226 case 1:
1227 m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(r), r))
1228 case 2:
1229 m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, newOperandReg(r), r))
1230 case 4:
1231 m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, newOperandReg(r), r))
1232 }
1233 }
1234 }
1235
1236 func (m *machine) lowerFcmp(instr *ssa.Instruction) {
1237 f1, f2, and := m.lowerFcmpToFlags(instr)
1238 rd := m.c.VRegOf(instr.Return())
1239 if f2 == condInvalid {
1240 tmp := m.c.AllocateVReg(ssa.TypeI32)
1241 m.insert(m.allocateInstr().asSetcc(f1, tmp))
1242 // On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match
1243 // the semantics of Icmp that sets either 0 or 1.
1244 m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd))
1245 } else {
1246 tmp1, tmp2 := m.c.AllocateVReg(ssa.TypeI32), m.c.AllocateVReg(ssa.TypeI32)
1247 m.insert(m.allocateInstr().asSetcc(f1, tmp1))
1248 m.insert(m.allocateInstr().asSetcc(f2, tmp2))
1249 var op aluRmiROpcode
1250 if and {
1251 op = aluRmiROpcodeAnd
1252 } else {
1253 op = aluRmiROpcodeOr
1254 }
1255 m.insert(m.allocateInstr().asAluRmiR(op, newOperandReg(tmp1), tmp2, false))
1256 m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp2), rd))
1257 }
1258 }
1259
1260 func (m *machine) lowerIcmp(instr *ssa.Instruction) {
1261 x, y, c := instr.IcmpData()
1262 m.lowerIcmpToFlag(m.c.ValueDefinition(x), m.c.ValueDefinition(y), x.Type() == ssa.TypeI64)
1263 rd := m.c.VRegOf(instr.Return())
1264 tmp := m.c.AllocateVReg(ssa.TypeI32)
1265 m.insert(m.allocateInstr().asSetcc(condFromSSAIntCmpCond(c), tmp))
1266 // On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match
1267 // the semantics of Icmp that sets either 0 or 1.
1268 m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd))
1269 }
1270
1271 func (m *machine) lowerSelect(x, y, cval, ret ssa.Value) {
1272 xo, yo := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y))
1273 rd := m.c.VRegOf(ret)
1274
1275 var cond cond
1276 cvalDef := m.c.ValueDefinition(cval)
1277 switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) {
1278 case ssa.OpcodeIcmp:
1279 icmp := cvalDef.Instr
1280 xc, yc, cc := icmp.IcmpData()
1281 m.lowerIcmpToFlag(m.c.ValueDefinition(xc), m.c.ValueDefinition(yc), xc.Type() == ssa.TypeI64)
1282 cond = condFromSSAIntCmpCond(cc)
1283 icmp.Lowered()
1284 default: // TODO: match ssa.OpcodeFcmp for optimization, but seems a bit complex.
1285 cv := m.getOperand_Reg(cvalDef)
1286 test := m.allocateInstr().asCmpRmiR(false, cv, cv.reg(), false)
1287 m.insert(test)
1288 cond = condNZ
1289 }
1290
1291 if typ := x.Type(); typ.IsInt() {
1292 _64 := typ.Bits() == 64
1293 mov := m.allocateInstr()
1294 tmp := m.c.AllocateVReg(typ)
1295 switch yo.kind {
1296 case operandKindReg:
1297 mov.asMovRR(yo.reg(), tmp, _64)
1298 case operandKindMem:
1299 if _64 {
1300 mov.asMov64MR(yo, tmp)
1301 } else {
1302 mov.asMovzxRmR(extModeLQ, yo, tmp)
1303 }
1304 default:
1305 panic("BUG")
1306 }
1307 m.insert(mov)
1308 cmov := m.allocateInstr().asCmove(cond, xo, tmp, _64)
1309 m.insert(cmov)
1310 m.insert(m.allocateInstr().asMovRR(tmp, rd, _64))
1311 } else {
1312 mov := m.allocateInstr()
1313 tmp := m.c.AllocateVReg(typ)
1314 switch typ {
1315 case ssa.TypeF32:
1316 mov.asXmmUnaryRmR(sseOpcodeMovss, yo, tmp)
1317 case ssa.TypeF64:
1318 mov.asXmmUnaryRmR(sseOpcodeMovsd, yo, tmp)
1319 case ssa.TypeV128:
1320 mov.asXmmUnaryRmR(sseOpcodeMovdqu, yo, tmp)
1321 default:
1322 panic("BUG")
1323 }
1324 m.insert(mov)
1325
1326 cmov := m.allocateInstr().asXmmCMov(cond, xo, tmp, typ.Size())
1327 m.insert(cmov)
1328
1329 m.copyTo(tmp, rd)
1330 }
1331 }
1332
1333 func (m *machine) lowerXmmCmovAfterRegAlloc(i *instruction) {
1334 x := i.op1
1335 rd := i.op2.reg()
1336 cond := cond(i.u1)
1337
1338 jcc := m.allocateInstr()
1339 m.insert(jcc)
1340
1341 mov := m.allocateInstr()
1342 switch i.u2 {
1343 case 4:
1344 mov.asXmmUnaryRmR(sseOpcodeMovss, x, rd)
1345 case 8:
1346 mov.asXmmUnaryRmR(sseOpcodeMovsd, x, rd)
1347 case 16:
1348 mov.asXmmUnaryRmR(sseOpcodeMovdqu, x, rd)
1349 default:
1350 panic("BUG")
1351 }
1352 m.insert(mov)
1353
1354 nop, end := m.allocateBrTarget()
1355 m.insert(nop)
1356 jcc.asJmpIf(cond.invert(), newOperandLabel(end))
1357 }
1358
1359 func (m *machine) lowerExtend(_arg, ret ssa.Value, from, to byte, signed bool) {
1360 rd0 := m.c.VRegOf(ret)
1361 arg := m.getOperand_Mem_Reg(m.c.ValueDefinition(_arg))
1362
1363 rd := m.c.AllocateVReg(ret.Type())
1364
1365 ext := m.allocateInstr()
1366 switch {
1367 case from == 8 && to == 16 && signed:
1368 ext.asMovsxRmR(extModeBQ, arg, rd)
1369 case from == 8 && to == 16 && !signed:
1370 ext.asMovzxRmR(extModeBL, arg, rd)
1371 case from == 8 && to == 32 && signed:
1372 ext.asMovsxRmR(extModeBL, arg, rd)
1373 case from == 8 && to == 32 && !signed:
1374 ext.asMovzxRmR(extModeBQ, arg, rd)
1375 case from == 8 && to == 64 && signed:
1376 ext.asMovsxRmR(extModeBQ, arg, rd)
1377 case from == 8 && to == 64 && !signed:
1378 ext.asMovzxRmR(extModeBQ, arg, rd)
1379 case from == 16 && to == 32 && signed:
1380 ext.asMovsxRmR(extModeWL, arg, rd)
1381 case from == 16 && to == 32 && !signed:
1382 ext.asMovzxRmR(extModeWL, arg, rd)
1383 case from == 16 && to == 64 && signed:
1384 ext.asMovsxRmR(extModeWQ, arg, rd)
1385 case from == 16 && to == 64 && !signed:
1386 ext.asMovzxRmR(extModeWQ, arg, rd)
1387 case from == 32 && to == 64 && signed:
1388 ext.asMovsxRmR(extModeLQ, arg, rd)
1389 case from == 32 && to == 64 && !signed:
1390 ext.asMovzxRmR(extModeLQ, arg, rd)
1391 default:
1392 panic(fmt.Sprintf("BUG: unhandled extend: from=%d, to=%d, signed=%t", from, to, signed))
1393 }
1394 m.insert(ext)
1395
1396 m.copyTo(rd, rd0)
1397 }
1398
1399 func (m *machine) lowerVconst(dst regalloc.VReg, lo, hi uint64) {
1400 if lo == 0 && hi == 0 {
1401 m.insert(m.allocateInstr().asZeros(dst))
1402 return
1403 }
1404
1405 load := m.allocateInstr()
1406 l, pos := m.allocateLabel()
1407 m.consts = append(m.consts, _const{label: l, labelPos: pos, lo: lo, hi: hi})
1408 load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(l)), dst)
1409 m.insert(load)
1410 }
1411
1412 func (m *machine) lowerCtz(instr *ssa.Instruction) {
1413 if m.cpuFeatures.Has(platform.CpuFeatureAmd64BMI1) {
1414 m.lowerUnaryRmR(instr, unaryRmROpcodeTzcnt)
1415 } else {
1416 // On processors that do not support TZCNT, the BSF instruction is
1417 // executed instead. The key difference between TZCNT and BSF
1418 // instruction is that if source operand is zero, the content of
1419 // destination operand is undefined.
1420 // https://www.felixcloutier.com/x86/tzcnt.html
1421
1422 x := instr.Arg()
1423 if !x.Type().IsInt() {
1424 panic("BUG?")
1425 }
1426 _64 := x.Type().Bits() == 64
1427
1428 xDef := m.c.ValueDefinition(x)
1429 tmp := m.c.AllocateVReg(x.Type())
1430 rm := m.getOperand_Reg(xDef)
1431
1432 // First, we have to check if the target is non-zero.
1433 test := m.allocateInstr()
1434 test.asCmpRmiR(false, rm, rm.reg(), _64)
1435 m.insert(test)
1436
1437 jmpNz := m.allocateInstr()
1438 m.insert(jmpNz)
1439
1440 // If the value is zero, we just push the const value.
1441 m.lowerIconst(tmp, uint64(x.Type().Bits()), _64)
1442
1443 // Now jump right after the non-zero case.
1444 jmpAtEnd := m.allocateInstr()
1445 m.insert(jmpAtEnd)
1446
1447 // jmpNz target label is set here.
1448 nop, nz := m.allocateBrTarget()
1449 jmpNz.asJmpIf(condNZ, newOperandLabel(nz))
1450 m.insert(nop)
1451
1452 // Emit the non-zero case.
1453 bsr := m.allocateInstr()
1454 bsr.asUnaryRmR(unaryRmROpcodeBsf, rm, tmp, _64)
1455 m.insert(bsr)
1456
1457 // jmpAtEnd target label is set here.
1458 nopEnd, end := m.allocateBrTarget()
1459 jmpAtEnd.asJmp(newOperandLabel(end))
1460 m.insert(nopEnd)
1461
1462 m.copyTo(tmp, m.c.VRegOf(instr.Return()))
1463 }
1464 }
1465
1466 func (m *machine) lowerClz(instr *ssa.Instruction) {
1467 if m.cpuFeatures.Has(platform.CpuFeatureAmd64ABM) {
1468 m.lowerUnaryRmR(instr, unaryRmROpcodeLzcnt)
1469 } else {
1470 // On processors that do not support LZCNT, we combine BSR (calculating
1471 // most significant set bit) with XOR. This logic is described in
1472 // "Replace Raw Assembly Code with Builtin Intrinsics" section in:
1473 // https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code.
1474
1475 x := instr.Arg()
1476 if !x.Type().IsInt() {
1477 panic("BUG?")
1478 }
1479 _64 := x.Type().Bits() == 64
1480
1481 xDef := m.c.ValueDefinition(x)
1482 rm := m.getOperand_Reg(xDef)
1483 tmp := m.c.AllocateVReg(x.Type())
1484
1485 // First, we have to check if the rm is non-zero as BSR is undefined
1486 // on zero. See https://www.felixcloutier.com/x86/bsr.
1487 test := m.allocateInstr()
1488 test.asCmpRmiR(false, rm, rm.reg(), _64)
1489 m.insert(test)
1490
1491 jmpNz := m.allocateInstr()
1492 m.insert(jmpNz)
1493
1494 // If the value is zero, we just push the const value.
1495 m.lowerIconst(tmp, uint64(x.Type().Bits()), _64)
1496
1497 // Now jump right after the non-zero case.
1498 jmpAtEnd := m.allocateInstr()
1499 m.insert(jmpAtEnd)
1500
1501 // jmpNz target label is set here.
1502 nop, nz := m.allocateBrTarget()
1503 jmpNz.asJmpIf(condNZ, newOperandLabel(nz))
1504 m.insert(nop)
1505
1506 // Emit the non-zero case.
1507 bsr := m.allocateInstr()
1508 bsr.asUnaryRmR(unaryRmROpcodeBsr, rm, tmp, _64)
1509 m.insert(bsr)
1510
1511 // Now we XOR the value with the bit length minus one.
1512 xor := m.allocateInstr()
1513 xor.asAluRmiR(aluRmiROpcodeXor, newOperandImm32(uint32(x.Type().Bits()-1)), tmp, _64)
1514 m.insert(xor)
1515
1516 // jmpAtEnd target label is set here.
1517 nopEnd, end := m.allocateBrTarget()
1518 jmpAtEnd.asJmp(newOperandLabel(end))
1519 m.insert(nopEnd)
1520
1521 m.copyTo(tmp, m.c.VRegOf(instr.Return()))
1522 }
1523 }
1524
1525 func (m *machine) lowerUnaryRmR(si *ssa.Instruction, op unaryRmROpcode) {
1526 x := si.Arg()
1527 if !x.Type().IsInt() {
1528 panic("BUG?")
1529 }
1530 _64 := x.Type().Bits() == 64
1531
1532 xDef := m.c.ValueDefinition(x)
1533 rm := m.getOperand_Mem_Reg(xDef)
1534 rd := m.c.VRegOf(si.Return())
1535
1536 instr := m.allocateInstr()
1537 instr.asUnaryRmR(op, rm, rd, _64)
1538 m.insert(instr)
1539 }
1540
1541 func (m *machine) lowerLoad(ptr ssa.Value, offset uint32, typ ssa.Type, dst regalloc.VReg) {
1542 mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
1543 load := m.allocateInstr()
1544 switch typ {
1545 case ssa.TypeI32:
1546 load.asMovzxRmR(extModeLQ, mem, dst)
1547 case ssa.TypeI64:
1548 load.asMov64MR(mem, dst)
1549 case ssa.TypeF32:
1550 load.asXmmUnaryRmR(sseOpcodeMovss, mem, dst)
1551 case ssa.TypeF64:
1552 load.asXmmUnaryRmR(sseOpcodeMovsd, mem, dst)
1553 case ssa.TypeV128:
1554 load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, dst)
1555 default:
1556 panic("BUG")
1557 }
1558 m.insert(load)
1559 }
1560
1561 func (m *machine) lowerExtLoad(op ssa.Opcode, ptr ssa.Value, offset uint32, dst regalloc.VReg) {
1562 mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
1563 load := m.allocateInstr()
1564 switch op {
1565 case ssa.OpcodeUload8:
1566 load.asMovzxRmR(extModeBQ, mem, dst)
1567 case ssa.OpcodeUload16:
1568 load.asMovzxRmR(extModeWQ, mem, dst)
1569 case ssa.OpcodeUload32:
1570 load.asMovzxRmR(extModeLQ, mem, dst)
1571 case ssa.OpcodeSload8:
1572 load.asMovsxRmR(extModeBQ, mem, dst)
1573 case ssa.OpcodeSload16:
1574 load.asMovsxRmR(extModeWQ, mem, dst)
1575 case ssa.OpcodeSload32:
1576 load.asMovsxRmR(extModeLQ, mem, dst)
1577 default:
1578 panic("BUG")
1579 }
1580 m.insert(load)
1581 }
1582
1583 func (m *machine) lowerExitIfTrueWithCode(execCtx regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) {
1584 condDef := m.c.ValueDefinition(cond)
1585 if !m.c.MatchInstr(condDef, ssa.OpcodeIcmp) {
1586 panic("TODO: ExitIfTrue must come after Icmp at the moment: " + condDef.Instr.Opcode().String())
1587 }
1588 cvalInstr := condDef.Instr
1589 cvalInstr.MarkLowered()
1590
1591 // We need to copy the execution context to a temp register, because if it's spilled,
1592 // it might end up being reloaded inside the exiting branch.
1593 execCtxTmp := m.copyToTmp(execCtx)
1594
1595 x, y, c := cvalInstr.IcmpData()
1596 xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
1597 if !m.tryLowerBandToFlag(xx, yy) {
1598 m.lowerIcmpToFlag(xx, yy, x.Type() == ssa.TypeI64)
1599 }
1600
1601 jmpIf := m.allocateInstr()
1602 m.insert(jmpIf)
1603 l := m.lowerExitWithCode(execCtxTmp, code)
1604 jmpIf.asJmpIf(condFromSSAIntCmpCond(c).invert(), newOperandLabel(l))
1605 }
1606
1607 func (m *machine) tryLowerBandToFlag(x, y backend.SSAValueDefinition) (ok bool) {
1608 var target backend.SSAValueDefinition
1609 var got bool
1610 if x.IsFromInstr() && x.Instr.Constant() && x.Instr.ConstantVal() == 0 {
1611 if m.c.MatchInstr(y, ssa.OpcodeBand) {
1612 target = y
1613 got = true
1614 }
1615 }
1616
1617 if y.IsFromInstr() && y.Instr.Constant() && y.Instr.ConstantVal() == 0 {
1618 if m.c.MatchInstr(x, ssa.OpcodeBand) {
1619 target = x
1620 got = true
1621 }
1622 }
1623
1624 if !got {
1625 return false
1626 }
1627
1628 bandInstr := target.Instr
1629 bandX, bandY := bandInstr.Arg2()
1630
1631 xx := m.getOperand_Reg(m.c.ValueDefinition(bandX))
1632 yy := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(bandY))
1633 test := m.allocateInstr().asCmpRmiR(false, yy, xx.reg(), bandX.Type() == ssa.TypeI64)
1634 m.insert(test)
1635 bandInstr.MarkLowered()
1636 return true
1637 }
1638
1639 func (m *machine) allocateExitInstructions(execCtx, exitCodeReg regalloc.VReg) (saveRsp, saveRbp, setExitCode *instruction) {
1640 saveRsp = m.allocateInstr().asMovRM(
1641 rspVReg,
1642 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.U32(), execCtx)),
1643 8,
1644 )
1645
1646 saveRbp = m.allocateInstr().asMovRM(
1647 rbpVReg,
1648 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetFramePointerBeforeGoCall.U32(), execCtx)),
1649 8,
1650 )
1651 setExitCode = m.allocateInstr().asMovRM(
1652 exitCodeReg,
1653 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetExitCodeOffset.U32(), execCtx)),
1654 4,
1655 )
1656 return
1657 }
1658
1659 func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel label) {
1660 exitCodeReg := rbpVReg
1661 saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtx, exitCodeReg)
1662
1663 // Set save RSP, RBP, and write exit code.
1664 m.insert(saveRsp)
1665 m.insert(saveRbp)
1666 m.lowerIconst(exitCodeReg, uint64(code), false)
1667 m.insert(setExitCode)
1668
1669 ripReg := rbpVReg
1670
1671 // Next is to save the current address for stack unwinding.
1672 nop, currentAddrLabel := m.allocateBrTarget()
1673 m.insert(nop)
1674 readRip := m.allocateInstr().asLEA(newOperandLabel(currentAddrLabel), ripReg)
1675 m.insert(readRip)
1676 saveRip := m.allocateInstr().asMovRM(
1677 ripReg,
1678 newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)),
1679 8,
1680 )
1681 m.insert(saveRip)
1682
1683 // Finally exit.
1684 exitSq := m.allocateExitSeq(execCtx)
1685 m.insert(exitSq)
1686
1687 // Return the label for continuation.
1688 continuation, afterLabel := m.allocateBrTarget()
1689 m.insert(continuation)
1690 return afterLabel
1691 }
1692
1693 func (m *machine) lowerAluRmiROp(si *ssa.Instruction, op aluRmiROpcode) {
1694 x, y := si.Arg2()
1695 if !x.Type().IsInt() {
1696 panic("BUG?")
1697 }
1698
1699 _64 := x.Type().Bits() == 64
1700
1701 xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
1702
1703 // TODO: commutative args can be swapped if one of them is an immediate.
1704 rn := m.getOperand_Reg(xDef)
1705 rm := m.getOperand_Mem_Imm32_Reg(yDef)
1706 rd := m.c.VRegOf(si.Return())
1707
1708 // rn is being overwritten, so we first copy its value to a temp register,
1709 // in case it is referenced again later.
1710 tmp := m.copyToTmp(rn.reg())
1711
1712 alu := m.allocateInstr()
1713 alu.asAluRmiR(op, rm, tmp, _64)
1714 m.insert(alu)
1715
1716 // tmp now contains the result, we copy it to the dest register.
1717 m.copyTo(tmp, rd)
1718 }
1719
1720 func (m *machine) lowerShiftR(si *ssa.Instruction, op shiftROp) {
1721 x, amt := si.Arg2()
1722 if !x.Type().IsInt() {
1723 panic("BUG?")
1724 }
1725 _64 := x.Type().Bits() == 64
1726
1727 xDef, amtDef := m.c.ValueDefinition(x), m.c.ValueDefinition(amt)
1728
1729 opAmt := m.getOperand_Imm32_Reg(amtDef)
1730 rx := m.getOperand_Reg(xDef)
1731 rd := m.c.VRegOf(si.Return())
1732
1733 // rx is being overwritten, so we first copy its value to a temp register,
1734 // in case it is referenced again later.
1735 tmpDst := m.copyToTmp(rx.reg())
1736
1737 if opAmt.kind == operandKindReg {
1738 // If opAmt is a register we must copy its value to rcx,
1739 // because shiftR encoding mandates that the shift amount is in rcx.
1740 m.copyTo(opAmt.reg(), rcxVReg)
1741
1742 alu := m.allocateInstr()
1743 alu.asShiftR(op, newOperandReg(rcxVReg), tmpDst, _64)
1744 m.insert(alu)
1745
1746 } else {
1747 alu := m.allocateInstr()
1748 alu.asShiftR(op, opAmt, tmpDst, _64)
1749 m.insert(alu)
1750 }
1751
1752 // tmp now contains the result, we copy it to the dest register.
1753 m.copyTo(tmpDst, rd)
1754 }
1755
1756 func (m *machine) lowerXmmRmR(instr *ssa.Instruction) {
1757 x, y := instr.Arg2()
1758 if !x.Type().IsFloat() {
1759 panic("BUG?")
1760 }
1761 _64 := x.Type().Bits() == 64
1762
1763 var op sseOpcode
1764 if _64 {
1765 switch instr.Opcode() {
1766 case ssa.OpcodeFadd:
1767 op = sseOpcodeAddsd
1768 case ssa.OpcodeFsub:
1769 op = sseOpcodeSubsd
1770 case ssa.OpcodeFmul:
1771 op = sseOpcodeMulsd
1772 case ssa.OpcodeFdiv:
1773 op = sseOpcodeDivsd
1774 default:
1775 panic("BUG")
1776 }
1777 } else {
1778 switch instr.Opcode() {
1779 case ssa.OpcodeFadd:
1780 op = sseOpcodeAddss
1781 case ssa.OpcodeFsub:
1782 op = sseOpcodeSubss
1783 case ssa.OpcodeFmul:
1784 op = sseOpcodeMulss
1785 case ssa.OpcodeFdiv:
1786 op = sseOpcodeDivss
1787 default:
1788 panic("BUG")
1789 }
1790 }
1791
1792 xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
1793 rn := m.getOperand_Reg(yDef)
1794 rm := m.getOperand_Reg(xDef)
1795 rd := m.c.VRegOf(instr.Return())
1796
1797 // rm is being overwritten, so we first copy its value to a temp register,
1798 // in case it is referenced again later.
1799 tmp := m.copyToTmp(rm.reg())
1800
1801 xmm := m.allocateInstr().asXmmRmR(op, rn, tmp)
1802 m.insert(xmm)
1803
1804 m.copyTo(tmp, rd)
1805 }
1806
1807 func (m *machine) lowerSqrt(instr *ssa.Instruction) {
1808 x := instr.Arg()
1809 if !x.Type().IsFloat() {
1810 panic("BUG")
1811 }
1812 _64 := x.Type().Bits() == 64
1813 var op sseOpcode
1814 if _64 {
1815 op = sseOpcodeSqrtsd
1816 } else {
1817 op = sseOpcodeSqrtss
1818 }
1819
1820 xDef := m.c.ValueDefinition(x)
1821 rm := m.getOperand_Mem_Reg(xDef)
1822 rd := m.c.VRegOf(instr.Return())
1823
1824 xmm := m.allocateInstr().asXmmUnaryRmR(op, rm, rd)
1825 m.insert(xmm)
1826 }
1827
1828 func (m *machine) lowerFabsFneg(instr *ssa.Instruction) {
1829 x := instr.Arg()
1830 if !x.Type().IsFloat() {
1831 panic("BUG")
1832 }
1833 _64 := x.Type().Bits() == 64
1834 var op sseOpcode
1835 var mask uint64
1836 if _64 {
1837 switch instr.Opcode() {
1838 case ssa.OpcodeFabs:
1839 mask, op = 0x7fffffffffffffff, sseOpcodeAndpd
1840 case ssa.OpcodeFneg:
1841 mask, op = 0x8000000000000000, sseOpcodeXorpd
1842 }
1843 } else {
1844 switch instr.Opcode() {
1845 case ssa.OpcodeFabs:
1846 mask, op = 0x7fffffff, sseOpcodeAndps
1847 case ssa.OpcodeFneg:
1848 mask, op = 0x80000000, sseOpcodeXorps
1849 }
1850 }
1851
1852 tmp := m.c.AllocateVReg(x.Type())
1853
1854 xDef := m.c.ValueDefinition(x)
1855 rm := m.getOperand_Reg(xDef)
1856 rd := m.c.VRegOf(instr.Return())
1857
1858 m.lowerFconst(tmp, mask, _64)
1859
1860 xmm := m.allocateInstr().asXmmRmR(op, rm, tmp)
1861 m.insert(xmm)
1862
1863 m.copyTo(tmp, rd)
1864 }
1865
1866 func (m *machine) lowerStore(si *ssa.Instruction) {
1867 value, ptr, offset, storeSizeInBits := si.StoreData()
1868 rm := m.getOperand_Reg(m.c.ValueDefinition(value))
1869 mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
1870
1871 store := m.allocateInstr()
1872 switch value.Type() {
1873 case ssa.TypeI32:
1874 store.asMovRM(rm.reg(), mem, storeSizeInBits/8)
1875 case ssa.TypeI64:
1876 store.asMovRM(rm.reg(), mem, storeSizeInBits/8)
1877 case ssa.TypeF32:
1878 store.asXmmMovRM(sseOpcodeMovss, rm.reg(), mem)
1879 case ssa.TypeF64:
1880 store.asXmmMovRM(sseOpcodeMovsd, rm.reg(), mem)
1881 case ssa.TypeV128:
1882 store.asXmmMovRM(sseOpcodeMovdqu, rm.reg(), mem)
1883 default:
1884 panic("BUG")
1885 }
1886 m.insert(store)
1887 }
1888
1889 func (m *machine) lowerCall(si *ssa.Instruction) {
1890 isDirectCall := si.Opcode() == ssa.OpcodeCall
1891 indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
1892
1893 if isMemmove {
1894 // Go's memmove *might* use all xmm0-xmm15, so we need to release them.
1895 // https://github.com/golang/go/blob/go1.24.0/src/cmd/compile/abi-internal.md#architecture-specifics
1896 // https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L286-L301
1897 for i := regalloc.RealReg(0); i < 16; i++ {
1898 m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[xmm0+i]))
1899 }
1900 // Since Go 1.24 it may also use DX, which is not reserved for the function call's 3 args.
1901 // https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L123
1902 m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[rdx]))
1903 }
1904
1905 if isDirectCall {
1906 call := m.allocateInstr().asCall(directCallee, calleeABI)
1907 m.insert(call)
1908 } else {
1909 ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr))
1910 callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI)
1911 m.insert(callInd)
1912 }
1913
1914 if isMemmove {
1915 for i := regalloc.RealReg(0); i < 16; i++ {
1916 m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[xmm0+i]))
1917 }
1918 m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[rdx]))
1919 }
1920
1921 m.insertReturns(si, calleeABI, stackSlotSize)
1922 }
1923
1924 func (m *machine) prepareCall(si *ssa.Instruction, isDirectCall bool) (ssa.Value, ssa.FuncRef, bool, *backend.FunctionABI, int64) {
1925 var indirectCalleePtr ssa.Value
1926 var directCallee ssa.FuncRef
1927 var sigID ssa.SignatureID
1928 var args []ssa.Value
1929 var isMemmove bool
1930 if isDirectCall {
1931 directCallee, sigID, args = si.CallData()
1932 } else {
1933 indirectCalleePtr, sigID, args, isMemmove = si.CallIndirectData()
1934 }
1935 calleeABI := m.c.GetFunctionABI(m.c.SSABuilder().ResolveSignature(sigID))
1936
1937 stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize())
1938 if m.maxRequiredStackSizeForCalls < stackSlotSize+16 {
1939 m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // 16 == return address + RBP.
1940 }
1941
1942 // Note: See machine.SetupPrologue for the stack layout.
1943 // The stack pointer decrease/increase will be inserted later in the compilation.
1944
1945 for i, arg := range args {
1946 reg := m.c.VRegOf(arg)
1947 def := m.c.ValueDefinition(arg)
1948 m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize)
1949 }
1950 return indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize
1951 }
1952
1953 func (m *machine) insertReturns(si *ssa.Instruction, calleeABI *backend.FunctionABI, stackSlotSize int64) {
1954 var index int
1955 r1, rs := si.Returns()
1956 if r1.Valid() {
1957 m.callerGenFunctionReturnVReg(calleeABI, 0, m.c.VRegOf(r1), stackSlotSize)
1958 index++
1959 }
1960
1961 for _, r := range rs {
1962 m.callerGenFunctionReturnVReg(calleeABI, index, m.c.VRegOf(r), stackSlotSize)
1963 index++
1964 }
1965 }
1966
1967 func (m *machine) lowerTailCall(si *ssa.Instruction) {
1968 isDirectCall := si.Opcode() == ssa.OpcodeTailCallReturnCall
1969 indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
1970 if isMemmove {
1971 panic("memmove not supported in tail calls")
1972 }
1973
1974 isAllRegs := stackSlotSize == 0
1975
1976 switch {
1977 case isDirectCall && isAllRegs:
1978 call := m.allocateInstr().asTailCallReturnCall(directCallee, calleeABI)
1979 m.insert(call)
1980 case !isDirectCall && isAllRegs:
1981 // In a tail call we insert the epilogue before the jump instruction,
1982 // so an arbitrary register might be overwritten while restoring the stack.
1983 // So, as compared to a regular indirect call, we ensure the pointer is stored
1984 // in a caller-saved register (r11).
1985 // For details, see internal/engine/RATIONALE.md
1986 ptrOp := m.getOperand_Reg(m.c.ValueDefinition(indirectCalleePtr))
1987 tmpJmp := r11VReg
1988 m.InsertMove(tmpJmp, ptrOp.reg(), ssa.TypeI64)
1989 callInd := m.allocateInstr().asTailCallReturnCallIndirect(newOperandReg(tmpJmp), calleeABI)
1990 m.insert(callInd)
1991 case isDirectCall && !isAllRegs:
1992 call := m.allocateInstr().asCall(directCallee, calleeABI)
1993 m.insert(call)
1994 case !isDirectCall && !isAllRegs:
1995 ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr))
1996 callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI)
1997 m.insert(callInd)
1998 }
1999
2000 // If this is a proper tail call, returns will be cleared in the postRegAlloc phase.
2001 m.insertReturns(si, calleeABI, stackSlotSize)
2002 }
2003
2004 // callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the
2005 // caller side of the function call.
2006 func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, stackSlotSize int64) {
2007 arg := &a.Args[argIndex]
2008 if def.IsFromInstr() {
2009 // Constant instructions are inlined.
2010 if inst := def.Instr; inst.Constant() {
2011 m.insertLoadConstant(inst, reg)
2012 }
2013 }
2014 if arg.Kind == backend.ABIArgKindReg {
2015 m.InsertMove(arg.Reg, reg, arg.Type)
2016 } else {
2017 store := m.allocateInstr()
2018 mem := newOperandMem(m.newAmodeImmReg(
2019 // -stackSlotSize because the stack pointer is not yet decreased.
2020 uint32(arg.Offset-stackSlotSize), rspVReg))
2021 switch arg.Type {
2022 case ssa.TypeI32:
2023 store.asMovRM(reg, mem, 4)
2024 case ssa.TypeI64:
2025 store.asMovRM(reg, mem, 8)
2026 case ssa.TypeF32:
2027 store.asXmmMovRM(sseOpcodeMovss, reg, mem)
2028 case ssa.TypeF64:
2029 store.asXmmMovRM(sseOpcodeMovsd, reg, mem)
2030 case ssa.TypeV128:
2031 store.asXmmMovRM(sseOpcodeMovdqu, reg, mem)
2032 default:
2033 panic("BUG")
2034 }
2035 m.insert(store)
2036 }
2037 }
2038
2039 func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, stackSlotSize int64) {
2040 r := &a.Rets[retIndex]
2041 if r.Kind == backend.ABIArgKindReg {
2042 m.InsertMove(reg, r.Reg, r.Type)
2043 } else {
2044 load := m.allocateInstr()
2045 mem := newOperandMem(m.newAmodeImmReg(
2046 // -stackSlotSize because the stack pointer is not yet decreased.
2047 uint32(a.ArgStackSize+r.Offset-stackSlotSize), rspVReg))
2048 switch r.Type {
2049 case ssa.TypeI32:
2050 load.asMovzxRmR(extModeLQ, mem, reg)
2051 case ssa.TypeI64:
2052 load.asMov64MR(mem, reg)
2053 case ssa.TypeF32:
2054 load.asXmmUnaryRmR(sseOpcodeMovss, mem, reg)
2055 case ssa.TypeF64:
2056 load.asXmmUnaryRmR(sseOpcodeMovsd, mem, reg)
2057 case ssa.TypeV128:
2058 load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, reg)
2059 default:
2060 panic("BUG")
2061 }
2062 m.insert(load)
2063 }
2064 }
2065
2066 // InsertMove implements backend.Machine.
2067 func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) {
2068 switch typ {
2069 case ssa.TypeI32, ssa.TypeI64:
2070 i := m.allocateInstr().asMovRR(src, dst, typ.Bits() == 64)
2071 m.insert(i)
2072 case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
2073 var op sseOpcode
2074 switch typ {
2075 case ssa.TypeF32:
2076 op = sseOpcodeMovss
2077 case ssa.TypeF64:
2078 op = sseOpcodeMovsd
2079 case ssa.TypeV128:
2080 op = sseOpcodeMovdqa
2081 }
2082 i := m.allocateInstr().asXmmUnaryRmR(op, newOperandReg(src), dst)
2083 m.insert(i)
2084 default:
2085 panic("BUG")
2086 }
2087 }
2088
2089 // Format implements backend.Machine.
2090 func (m *machine) Format() string {
2091 begins := map[*instruction]label{}
2092 for l := label(0); l < m.nextLabel; l++ {
2093 pos := m.labelPositionPool.Get(int(l))
2094 if pos != nil {
2095 begins[pos.begin] = l
2096 }
2097 }
2098
2099 var lines []string
2100 for cur := m.rootInstr; cur != nil; cur = cur.next {
2101 if l, ok := begins[cur]; ok {
2102 var labelStr string
2103 if l <= m.maxSSABlockID {
2104 labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, l)
2105 } else {
2106 labelStr = fmt.Sprintf("%s:", l)
2107 }
2108 lines = append(lines, labelStr)
2109 }
2110 if cur.kind == nop0 {
2111 continue
2112 }
2113 lines = append(lines, "\t"+cur.String())
2114 }
2115 for _, vc := range m.consts {
2116 if vc._var == nil {
2117 lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label, vc.lo, vc.hi))
2118 } else {
2119 lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label, vc._var))
2120 }
2121 }
2122 return "\n" + strings.Join(lines, "\n") + "\n"
2123 }
2124
2125 func (m *machine) encodeWithoutSSA(root *instruction) {
2126 m.labelResolutionPends = m.labelResolutionPends[:0]
2127 bufPtr := m.c.BufPtr()
2128 for cur := root; cur != nil; cur = cur.next {
2129 offset := int64(len(*bufPtr))
2130 if cur.kind == nop0 {
2131 l := cur.nop0Label()
2132 pos := m.labelPositionPool.Get(int(l))
2133 if pos != nil {
2134 pos.binaryOffset = offset
2135 }
2136 }
2137
2138 needLabelResolution := cur.encode(m.c)
2139 if needLabelResolution {
2140 m.labelResolutionPends = append(m.labelResolutionPends,
2141 labelResolutionPend{instr: cur, imm32Offset: int64(len(*bufPtr)) - 4},
2142 )
2143 }
2144 }
2145
2146 for i := range m.labelResolutionPends {
2147 p := &m.labelResolutionPends[i]
2148 switch p.instr.kind {
2149 case jmp, jmpIf, lea:
2150 target := p.instr.jmpLabel()
2151 targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset
2152 imm32Offset := p.imm32Offset
2153 jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction.
2154 binary.LittleEndian.PutUint32((*bufPtr)[imm32Offset:], uint32(jmpOffset))
2155 default:
2156 panic("BUG")
2157 }
2158 }
2159 }
2160
2161 // Encode implements backend.Machine Encode.
2162 func (m *machine) Encode(ctx context.Context) (err error) {
2163 bufPtr := m.c.BufPtr()
2164
2165 var fn string
2166 var fnIndex int
2167 var labelPosToLabel map[*labelPosition]label
2168 if wazevoapi.PerfMapEnabled {
2169 fn = wazevoapi.GetCurrentFunctionName(ctx)
2170 labelPosToLabel = make(map[*labelPosition]label)
2171 for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ {
2172 pos := m.labelPositionPool.Get(i)
2173 labelPosToLabel[pos] = label(i)
2174 }
2175 fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx)
2176 }
2177
2178 m.labelResolutionPends = m.labelResolutionPends[:0]
2179 for _, pos := range m.orderedSSABlockLabelPos {
2180 offset := int64(len(*bufPtr))
2181 pos.binaryOffset = offset
2182 for cur := pos.begin; cur != pos.end.next; cur = cur.next {
2183 offset := int64(len(*bufPtr))
2184
2185 switch cur.kind {
2186 case nop0:
2187 l := cur.nop0Label()
2188 if pos := m.labelPositionPool.Get(int(l)); pos != nil {
2189 pos.binaryOffset = offset
2190 }
2191 case sourceOffsetInfo:
2192 m.c.AddSourceOffsetInfo(offset, cur.sourceOffsetInfo())
2193 }
2194
2195 needLabelResolution := cur.encode(m.c)
2196 if needLabelResolution {
2197 m.labelResolutionPends = append(m.labelResolutionPends,
2198 labelResolutionPend{instr: cur, instrOffset: offset, imm32Offset: int64(len(*bufPtr)) - 4},
2199 )
2200 }
2201 }
2202
2203 if wazevoapi.PerfMapEnabled {
2204 l := labelPosToLabel[pos]
2205 size := int64(len(*bufPtr)) - offset
2206 wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, l))
2207 }
2208 }
2209
2210 for i := range m.consts {
2211 offset := int64(len(*bufPtr))
2212 vc := &m.consts[i]
2213 vc.labelPos.binaryOffset = offset
2214 if vc._var == nil {
2215 lo, hi := vc.lo, vc.hi
2216 m.c.Emit8Bytes(lo)
2217 m.c.Emit8Bytes(hi)
2218 } else {
2219 for _, b := range vc._var {
2220 m.c.EmitByte(b)
2221 }
2222 }
2223 }
2224
2225 buf := *bufPtr
2226 for i := range m.labelResolutionPends {
2227 p := &m.labelResolutionPends[i]
2228 switch p.instr.kind {
2229 case jmp, jmpIf, lea, xmmUnaryRmR:
2230 target := p.instr.jmpLabel()
2231 targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset
2232 imm32Offset := p.imm32Offset
2233 jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction.
2234 binary.LittleEndian.PutUint32(buf[imm32Offset:], uint32(jmpOffset))
2235 case jmpTableIsland:
2236 tableBegin := p.instrOffset
2237 // Each entry is the offset from the beginning of the jmpTableIsland instruction in 8 bytes.
2238 targets := m.jmpTableTargets[p.instr.u1]
2239 for i, l := range targets {
2240 targetOffset := m.labelPositionPool.Get(int(l)).binaryOffset
2241 jmpOffset := targetOffset - tableBegin
2242 binary.LittleEndian.PutUint64(buf[tableBegin+int64(i)*8:], uint64(jmpOffset))
2243 }
2244 default:
2245 panic("BUG")
2246 }
2247 }
2248 return
2249 }
2250
2251 // ResolveRelocations implements backend.Machine.
2252 func (m *machine) ResolveRelocations(refToBinaryOffset []int, _ int, binary []byte, relocations []backend.RelocationInfo, _ []int) {
2253 for _, r := range relocations {
2254 offset := r.Offset
2255 calleeFnOffset := refToBinaryOffset[r.FuncRef]
2256 // offset is the offset of the last 4 bytes of the call instruction.
2257 callInstrOffsetBytes := binary[offset : offset+4]
2258 diff := int64(calleeFnOffset) - (offset + 4) // +4 because we want the offset of the next instruction (In x64, RIP always points to the next instruction).
2259 callInstrOffsetBytes[0] = byte(diff)
2260 callInstrOffsetBytes[1] = byte(diff >> 8)
2261 callInstrOffsetBytes[2] = byte(diff >> 16)
2262 callInstrOffsetBytes[3] = byte(diff >> 24)
2263 }
2264 }
2265
2266 // CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo.
2267 func (m *machine) CallTrampolineIslandInfo(_ int) (_, _ int, _ error) { return }
2268
2269 func (m *machine) lowerIcmpToFlag(xd, yd backend.SSAValueDefinition, _64 bool) {
2270 x := m.getOperand_Reg(xd)
2271 y := m.getOperand_Mem_Imm32_Reg(yd)
2272 cmp := m.allocateInstr().asCmpRmiR(true, y, x.reg(), _64)
2273 m.insert(cmp)
2274 }
2275
2276 func (m *machine) lowerFcmpToFlags(instr *ssa.Instruction) (f1, f2 cond, and bool) {
2277 x, y, c := instr.FcmpData()
2278 switch c {
2279 case ssa.FloatCmpCondEqual:
2280 f1, f2 = condNP, condZ
2281 and = true
2282 case ssa.FloatCmpCondNotEqual:
2283 f1, f2 = condP, condNZ
2284 case ssa.FloatCmpCondLessThan:
2285 f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThan)
2286 f2 = condInvalid
2287 x, y = y, x
2288 case ssa.FloatCmpCondLessThanOrEqual:
2289 f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThanOrEqual)
2290 f2 = condInvalid
2291 x, y = y, x
2292 default:
2293 f1 = condFromSSAFloatCmpCond(c)
2294 f2 = condInvalid
2295 }
2296
2297 var opc sseOpcode
2298 if x.Type() == ssa.TypeF32 {
2299 opc = sseOpcodeUcomiss
2300 } else {
2301 opc = sseOpcodeUcomisd
2302 }
2303
2304 xr := m.getOperand_Reg(m.c.ValueDefinition(x))
2305 yr := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
2306 m.insert(m.allocateInstr().asXmmCmpRmR(opc, yr, xr.reg()))
2307 return
2308 }
2309
2310 // allocateInstr allocates an instruction.
2311 func (m *machine) allocateInstr() *instruction {
2312 instr := m.instrPool.Allocate()
2313 if !m.regAllocStarted {
2314 instr.addedBeforeRegAlloc = true
2315 }
2316 return instr
2317 }
2318
2319 func (m *machine) allocateNop() *instruction {
2320 instr := m.allocateInstr()
2321 instr.kind = nop0
2322 return instr
2323 }
2324
2325 func (m *machine) insert(i *instruction) {
2326 m.pendingInstructions = append(m.pendingInstructions, i)
2327 }
2328
2329 func (m *machine) allocateBrTarget() (nop *instruction, l label) { //nolint
2330 l, pos := m.allocateLabel()
2331 nop = m.allocateInstr()
2332 nop.asNop0WithLabel(l)
2333 pos.begin, pos.end = nop, nop
2334 return
2335 }
2336
2337 func (m *machine) allocateLabel() (label, *labelPosition) {
2338 l := m.nextLabel
2339 pos := m.labelPositionPool.GetOrAllocate(int(l))
2340 m.nextLabel++
2341 return l, pos
2342 }
2343
2344 func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 {
2345 offset, ok := m.spillSlots[id]
2346 if !ok {
2347 offset = m.spillSlotSize
2348 m.spillSlots[id] = offset
2349 m.spillSlotSize += int64(size)
2350 }
2351 return offset
2352 }
2353
2354 func (m *machine) copyTo(src regalloc.VReg, dst regalloc.VReg) {
2355 mov := m.allocateInstr()
2356 if src.RegType() == regalloc.RegTypeInt {
2357 mov.asMovRR(src, dst, true)
2358 } else {
2359 mov.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), dst)
2360 }
2361 m.insert(mov)
2362 }
2363
2364 func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg {
2365 typ := m.c.TypeOf(v)
2366 tmp := m.c.AllocateVReg(typ)
2367 m.copyTo(v, tmp)
2368 return tmp
2369 }
2370
2371 func (m *machine) requiredStackSize() int64 {
2372 return m.maxRequiredStackSizeForCalls +
2373 m.frameSize() +
2374 16 + // Need for stack checking.
2375 16 // return address and the caller RBP.
2376 }
2377
2378 func (m *machine) frameSize() int64 {
2379 s := m.clobberedRegSlotSize() + m.spillSlotSize
2380 if s&0xf != 0 {
2381 panic(fmt.Errorf("BUG: frame size %d is not 16-byte aligned", s))
2382 }
2383 return s
2384 }
2385
2386 func (m *machine) clobberedRegSlotSize() int64 {
2387 return int64(len(m.clobberedRegs) * 16)
2388 }
2389
2390 func (m *machine) lowerIDivRem(si *ssa.Instruction, isDiv bool, signed bool) {
2391 x, y, execCtx := si.Arg3()
2392
2393 dividend := m.getOperand_Reg(m.c.ValueDefinition(x))
2394 divisor := m.getOperand_Reg(m.c.ValueDefinition(y))
2395 ctxVReg := m.c.VRegOf(execCtx)
2396 tmpGp := m.c.AllocateVReg(si.Return().Type())
2397
2398 m.copyTo(dividend.reg(), raxVReg)
2399 m.insert(m.allocateInstr().asDefineUninitializedReg(rdxVReg))
2400 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
2401 seq := m.allocateInstr().asIdivRemSequence(ctxVReg, divisor.reg(), tmpGp, isDiv, signed, x.Type().Bits() == 64)
2402 m.insert(seq)
2403 rd := m.c.VRegOf(si.Return())
2404 if isDiv {
2405 m.copyTo(raxVReg, rd)
2406 } else {
2407 m.copyTo(rdxVReg, rd)
2408 }
2409 }
2410
2411 func (m *machine) lowerIDivRemSequenceAfterRegAlloc(i *instruction) {
2412 execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData()
2413
2414 dividend := raxVReg
2415
2416 // Ensure yr is not zero.
2417 test := m.allocateInstr()
2418 test.asCmpRmiR(false, newOperandReg(divisor), divisor, _64)
2419 m.insert(test)
2420
2421 jnz := m.allocateInstr()
2422 m.insert(jnz)
2423
2424 nz := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerDivisionByZero)
2425
2426 // If not zero, we can proceed with the division.
2427 jnz.asJmpIf(condNZ, newOperandLabel(nz))
2428
2429 var ifRemNeg1 *instruction
2430 if signed {
2431 var neg1 uint64
2432 if _64 {
2433 neg1 = 0xffffffffffffffff
2434 } else {
2435 neg1 = 0xffffffff
2436 }
2437 m.lowerIconst(tmpGp, neg1, _64)
2438
2439 if isDiv {
2440 // For signed division, we have to have branches for "math.MinInt{32,64} / -1"
2441 // case which results in the floating point exception via division error as
2442 // the resulting value exceeds the maximum of signed int.
2443
2444 // First, we check if the divisor is -1.
2445 cmp := m.allocateInstr()
2446 cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64)
2447 m.insert(cmp)
2448
2449 ifNotNeg1 := m.allocateInstr()
2450 m.insert(ifNotNeg1)
2451
2452 var minInt uint64
2453 if _64 {
2454 minInt = 0x8000000000000000
2455 } else {
2456 minInt = 0x80000000
2457 }
2458 m.lowerIconst(tmpGp, minInt, _64)
2459
2460 // Next we check if the quotient is the most negative value for the signed integer, i.e.
2461 // if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1) respectively.
2462 cmp2 := m.allocateInstr()
2463 cmp2.asCmpRmiR(true, newOperandReg(tmpGp), dividend, _64)
2464 m.insert(cmp2)
2465
2466 ifNotMinInt := m.allocateInstr()
2467 m.insert(ifNotMinInt)
2468
2469 // Trap if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1),
2470 // as that is the overflow in division as the result becomes 2^31 which is larger than
2471 // the maximum of signed 32-bit int (2^31-1).
2472 end := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2473 ifNotNeg1.asJmpIf(condNZ, newOperandLabel(end))
2474 ifNotMinInt.asJmpIf(condNZ, newOperandLabel(end))
2475 } else {
2476 // If it is remainder, zeros DX register and compare the divisor to -1.
2477 xor := m.allocateInstr().asZeros(rdxVReg)
2478 m.insert(xor)
2479
2480 // We check if the divisor is -1.
2481 cmp := m.allocateInstr()
2482 cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64)
2483 m.insert(cmp)
2484
2485 ifRemNeg1 = m.allocateInstr()
2486 m.insert(ifRemNeg1)
2487 }
2488
2489 // Sign-extend DX register to have 2*x.Type().Bits() dividend over DX and AX registers.
2490 sed := m.allocateInstr()
2491 sed.asSignExtendData(_64)
2492 m.insert(sed)
2493 } else {
2494 // Zeros DX register to have 2*x.Type().Bits() dividend over DX and AX registers.
2495 zeros := m.allocateInstr().asZeros(rdxVReg)
2496 m.insert(zeros)
2497 }
2498
2499 div := m.allocateInstr()
2500 div.asDiv(newOperandReg(divisor), signed, _64)
2501 m.insert(div)
2502
2503 nop, end := m.allocateBrTarget()
2504 m.insert(nop)
2505 // If we are compiling a Rem instruction, when the divisor is -1 we land at the end of the function.
2506 if ifRemNeg1 != nil {
2507 ifRemNeg1.asJmpIf(condZ, newOperandLabel(end))
2508 }
2509 }
2510
2511 func (m *machine) lowerRound(instr *ssa.Instruction, imm roundingMode) {
2512 x := instr.Arg()
2513 if !x.Type().IsFloat() {
2514 panic("BUG?")
2515 }
2516 var op sseOpcode
2517 if x.Type().Bits() == 64 {
2518 op = sseOpcodeRoundsd
2519 } else {
2520 op = sseOpcodeRoundss
2521 }
2522
2523 xDef := m.c.ValueDefinition(x)
2524 rm := m.getOperand_Mem_Reg(xDef)
2525 rd := m.c.VRegOf(instr.Return())
2526
2527 xmm := m.allocateInstr().asXmmUnaryRmRImm(op, uint8(imm), rm, rd)
2528 m.insert(xmm)
2529 }
2530
2531 func (m *machine) lowerFminFmax(instr *ssa.Instruction) {
2532 x, y := instr.Arg2()
2533 if !x.Type().IsFloat() {
2534 panic("BUG?")
2535 }
2536
2537 _64 := x.Type().Bits() == 64
2538 isMin := instr.Opcode() == ssa.OpcodeFmin
2539 var minMaxOp sseOpcode
2540
2541 switch {
2542 case _64 && isMin:
2543 minMaxOp = sseOpcodeMinpd
2544 case _64 && !isMin:
2545 minMaxOp = sseOpcodeMaxpd
2546 case !_64 && isMin:
2547 minMaxOp = sseOpcodeMinps
2548 case !_64 && !isMin:
2549 minMaxOp = sseOpcodeMaxps
2550 }
2551
2552 xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
2553 rm := m.getOperand_Reg(xDef)
2554 // We cannot ensure that y is aligned to 16 bytes, so we have to use it on reg.
2555 rn := m.getOperand_Reg(yDef)
2556 rd := m.c.VRegOf(instr.Return())
2557
2558 tmp := m.copyToTmp(rm.reg())
2559
2560 // Check if this is (either x1 or x2 is NaN) or (x1 equals x2) case.
2561 cmp := m.allocateInstr()
2562 if _64 {
2563 cmp.asXmmCmpRmR(sseOpcodeUcomisd, rn, tmp)
2564 } else {
2565 cmp.asXmmCmpRmR(sseOpcodeUcomiss, rn, tmp)
2566 }
2567 m.insert(cmp)
2568
2569 // At this point, we have the three cases of conditional flags below
2570 // (See https://www.felixcloutier.com/x86/ucomiss#operation for detail.)
2571 //
2572 // 1) Two values are NaN-free and different: All flags are cleared.
2573 // 2) Two values are NaN-free and equal: Only ZF flags is set.
2574 // 3) One of Two values is NaN: ZF, PF and CF flags are set.
2575
2576 // Jump instruction to handle 1) case by checking the ZF flag
2577 // as ZF is only set for 2) and 3) cases.
2578 nanFreeOrDiffJump := m.allocateInstr()
2579 m.insert(nanFreeOrDiffJump)
2580
2581 // Start handling 2) and 3).
2582
2583 // Jump if one of two values is NaN by checking the parity flag (PF).
2584 ifIsNan := m.allocateInstr()
2585 m.insert(ifIsNan)
2586
2587 // Start handling 2) NaN-free and equal.
2588
2589 // Before we exit this case, we have to ensure that positive zero (or negative zero for min instruction) is
2590 // returned if two values are positive and negative zeros.
2591 var op sseOpcode
2592 switch {
2593 case !_64 && isMin:
2594 op = sseOpcodeOrps
2595 case _64 && isMin:
2596 op = sseOpcodeOrpd
2597 case !_64 && !isMin:
2598 op = sseOpcodeAndps
2599 case _64 && !isMin:
2600 op = sseOpcodeAndpd
2601 }
2602 orAnd := m.allocateInstr()
2603 orAnd.asXmmRmR(op, rn, tmp)
2604 m.insert(orAnd)
2605
2606 // Done, jump to end.
2607 sameExitJump := m.allocateInstr()
2608 m.insert(sameExitJump)
2609
2610 // Start handling 3) either is NaN.
2611 isNanTarget, isNan := m.allocateBrTarget()
2612 m.insert(isNanTarget)
2613 ifIsNan.asJmpIf(condP, newOperandLabel(isNan))
2614
2615 // We emit the ADD instruction to produce the NaN in tmp.
2616 add := m.allocateInstr()
2617 if _64 {
2618 add.asXmmRmR(sseOpcodeAddsd, rn, tmp)
2619 } else {
2620 add.asXmmRmR(sseOpcodeAddss, rn, tmp)
2621 }
2622 m.insert(add)
2623
2624 // Exit from the NaN case branch.
2625 nanExitJmp := m.allocateInstr()
2626 m.insert(nanExitJmp)
2627
2628 // Start handling 1).
2629 doMinMaxTarget, doMinMax := m.allocateBrTarget()
2630 m.insert(doMinMaxTarget)
2631 nanFreeOrDiffJump.asJmpIf(condNZ, newOperandLabel(doMinMax))
2632
2633 // Now handle the NaN-free and different values case.
2634 minMax := m.allocateInstr()
2635 minMax.asXmmRmR(minMaxOp, rn, tmp)
2636 m.insert(minMax)
2637
2638 endNop, end := m.allocateBrTarget()
2639 m.insert(endNop)
2640 nanExitJmp.asJmp(newOperandLabel(end))
2641 sameExitJump.asJmp(newOperandLabel(end))
2642
2643 m.copyTo(tmp, rd)
2644 }
2645
2646 func (m *machine) lowerFcopysign(instr *ssa.Instruction) {
2647 x, y := instr.Arg2()
2648 if !x.Type().IsFloat() {
2649 panic("BUG")
2650 }
2651
2652 _64 := x.Type().Bits() == 64
2653
2654 xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
2655 rm := m.getOperand_Reg(xDef)
2656 rn := m.getOperand_Reg(yDef)
2657 rd := m.c.VRegOf(instr.Return())
2658
2659 // Clear the non-sign bits of src via AND with the mask.
2660 var opAnd, opOr sseOpcode
2661 var signMask uint64
2662 if _64 {
2663 signMask, opAnd, opOr = 0x8000000000000000, sseOpcodeAndpd, sseOpcodeOrpd
2664 } else {
2665 signMask, opAnd, opOr = 0x80000000, sseOpcodeAndps, sseOpcodeOrps
2666 }
2667
2668 signBitReg := m.c.AllocateVReg(x.Type())
2669 m.lowerFconst(signBitReg, signMask, _64)
2670 nonSignBitReg := m.c.AllocateVReg(x.Type())
2671 m.lowerFconst(nonSignBitReg, ^signMask, _64)
2672
2673 // Extract the sign bits of rn.
2674 and := m.allocateInstr().asXmmRmR(opAnd, rn, signBitReg)
2675 m.insert(and)
2676
2677 // Clear the sign bit of dst via AND with the non-sign bit mask.
2678 xor := m.allocateInstr().asXmmRmR(opAnd, rm, nonSignBitReg)
2679 m.insert(xor)
2680
2681 // Copy the sign bits of src to dst via OR.
2682 or := m.allocateInstr().asXmmRmR(opOr, newOperandReg(signBitReg), nonSignBitReg)
2683 m.insert(or)
2684
2685 m.copyTo(nonSignBitReg, rd)
2686 }
2687
2688 func (m *machine) lowerBitcast(instr *ssa.Instruction) {
2689 x, dstTyp := instr.BitcastData()
2690 srcTyp := x.Type()
2691 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
2692 rd := m.c.VRegOf(instr.Return())
2693 switch {
2694 case srcTyp == ssa.TypeF32 && dstTyp == ssa.TypeI32:
2695 cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovd, rn.reg(), rd, false)
2696 m.insert(cvt)
2697 case srcTyp == ssa.TypeI32 && dstTyp == ssa.TypeF32:
2698 cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovd, rn, rd, false)
2699 m.insert(cvt)
2700 case srcTyp == ssa.TypeF64 && dstTyp == ssa.TypeI64:
2701 cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovq, rn.reg(), rd, true)
2702 m.insert(cvt)
2703 case srcTyp == ssa.TypeI64 && dstTyp == ssa.TypeF64:
2704 cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovq, rn, rd, true)
2705 m.insert(cvt)
2706 default:
2707 panic(fmt.Sprintf("invalid bitcast from %s to %s", srcTyp, dstTyp))
2708 }
2709 }
2710
2711 func (m *machine) lowerFcvtToSint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) {
2712 var tmpXmm regalloc.VReg
2713 if dst64 {
2714 tmpXmm = m.c.AllocateVReg(ssa.TypeF64)
2715 } else {
2716 tmpXmm = m.c.AllocateVReg(ssa.TypeF32)
2717 }
2718
2719 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm))
2720 tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64)
2721 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
2722 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2))
2723
2724 m.insert(m.allocateFcvtToSintSequence(ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat))
2725 m.copyTo(tmpGp, rd)
2726 }
2727
2728 func (m *machine) lowerFcvtToSintSequenceAfterRegalloc(i *instruction) {
2729 execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData()
2730 var cmpOp, truncOp sseOpcode
2731 if src64 {
2732 cmpOp, truncOp = sseOpcodeUcomisd, sseOpcodeCvttsd2si
2733 } else {
2734 cmpOp, truncOp = sseOpcodeUcomiss, sseOpcodeCvttss2si
2735 }
2736
2737 trunc := m.allocateInstr()
2738 trunc.asXmmToGpr(truncOp, src, tmpGp, dst64)
2739 m.insert(trunc)
2740
2741 // Check if the dst operand was INT_MIN, by checking it against 1.
2742 cmp1 := m.allocateInstr()
2743 cmp1.asCmpRmiR(true, newOperandImm32(1), tmpGp, dst64)
2744 m.insert(cmp1)
2745
2746 // If no overflow, then we are done.
2747 doneTarget, done := m.allocateBrTarget()
2748 ifNoOverflow := m.allocateInstr()
2749 ifNoOverflow.asJmpIf(condNO, newOperandLabel(done))
2750 m.insert(ifNoOverflow)
2751
2752 // Now, check for NaN.
2753 cmpNan := m.allocateInstr()
2754 cmpNan.asXmmCmpRmR(cmpOp, newOperandReg(src), src)
2755 m.insert(cmpNan)
2756
2757 // We allocate the "non-nan target" here, but we will insert it later.
2758 notNanTarget, notNaN := m.allocateBrTarget()
2759 ifNotNan := m.allocateInstr()
2760 ifNotNan.asJmpIf(condNP, newOperandLabel(notNaN))
2761 m.insert(ifNotNan)
2762
2763 if sat {
2764 // If NaN and saturating, return 0.
2765 zeroDst := m.allocateInstr().asZeros(tmpGp)
2766 m.insert(zeroDst)
2767
2768 jmpEnd := m.allocateInstr()
2769 jmpEnd.asJmp(newOperandLabel(done))
2770 m.insert(jmpEnd)
2771
2772 // Otherwise:
2773 m.insert(notNanTarget)
2774
2775 // Zero-out the tmp register.
2776 zero := m.allocateInstr().asZeros(tmpXmm)
2777 m.insert(zero)
2778
2779 cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
2780 m.insert(cmpXmm)
2781
2782 // if >= jump to end.
2783 jmpEnd2 := m.allocateInstr()
2784 jmpEnd2.asJmpIf(condB, newOperandLabel(done))
2785 m.insert(jmpEnd2)
2786
2787 // Otherwise, saturate to INT_MAX.
2788 if dst64 {
2789 m.lowerIconst(tmpGp, math.MaxInt64, dst64)
2790 } else {
2791 m.lowerIconst(tmpGp, math.MaxInt32, dst64)
2792 }
2793
2794 } else {
2795
2796 // If non-sat, NaN, trap.
2797 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger)
2798
2799 // Otherwise, we will jump here.
2800 m.insert(notNanTarget)
2801
2802 // jump over trap if src larger than threshold
2803 condAboveThreshold := condNB
2804
2805 // The magic constants are various combination of minInt for int[32|64] represented as float[32|64].
2806 var minInt uint64
2807 switch {
2808 case src64 && dst64:
2809 minInt = 0xc3e0000000000000
2810 case src64 && !dst64:
2811 condAboveThreshold = condNBE
2812 minInt = 0xC1E0_0000_0020_0000
2813 case !src64 && dst64:
2814 minInt = 0xDF00_0000
2815 case !src64 && !dst64:
2816 minInt = 0xCF00_0000
2817 }
2818
2819 loadToGP := m.allocateInstr().asImm(tmpGp2, minInt, src64)
2820 m.insert(loadToGP)
2821
2822 movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp2), tmpXmm, src64)
2823 m.insert(movToXmm)
2824
2825 cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
2826 m.insert(cmpXmm)
2827
2828 jmpIfLarger := m.allocateInstr()
2829 checkPositiveTarget, checkPositive := m.allocateBrTarget()
2830 jmpIfLarger.asJmpIf(condAboveThreshold, newOperandLabel(checkPositive))
2831 m.insert(jmpIfLarger)
2832
2833 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2834
2835 // If positive, it was a real overflow.
2836 m.insert(checkPositiveTarget)
2837
2838 // Zero out the temp register.
2839 xorpd := m.allocateInstr()
2840 xorpd.asXmmRmR(sseOpcodeXorpd, newOperandReg(tmpXmm), tmpXmm)
2841 m.insert(xorpd)
2842
2843 pos := m.allocateInstr()
2844 pos.asXmmCmpRmR(cmpOp, newOperandReg(src), tmpXmm)
2845 m.insert(pos)
2846
2847 // If >= jump to end.
2848 jmp := m.allocateInstr().asJmpIf(condNB, newOperandLabel(done))
2849 m.insert(jmp)
2850 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2851 }
2852
2853 m.insert(doneTarget)
2854 }
2855
2856 func (m *machine) lowerFcvtToUint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) {
2857 tmpXmm, tmpXmm2 := m.c.AllocateVReg(ssa.TypeF64), m.c.AllocateVReg(ssa.TypeF64)
2858 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm))
2859 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm2))
2860 tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64)
2861 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
2862 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2))
2863
2864 m.insert(m.allocateFcvtToUintSequence(
2865 ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat,
2866 ))
2867 m.copyTo(tmpGp, rd)
2868 }
2869
2870 func (m *machine) lowerFcvtToUintSequenceAfterRegalloc(i *instruction) {
2871 execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData()
2872
2873 var subOp, cmpOp, truncOp sseOpcode
2874 if src64 {
2875 subOp, cmpOp, truncOp = sseOpcodeSubsd, sseOpcodeUcomisd, sseOpcodeCvttsd2si
2876 } else {
2877 subOp, cmpOp, truncOp = sseOpcodeSubss, sseOpcodeUcomiss, sseOpcodeCvttss2si
2878 }
2879
2880 doneTarget, done := m.allocateBrTarget()
2881
2882 switch {
2883 case src64 && dst64:
2884 loadToGP := m.allocateInstr().asImm(tmpGp, 0x43e0000000000000, true)
2885 m.insert(loadToGP)
2886 movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true)
2887 m.insert(movToXmm)
2888 case src64 && !dst64:
2889 loadToGP := m.allocateInstr().asImm(tmpGp, 0x41e0000000000000, true)
2890 m.insert(loadToGP)
2891 movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true)
2892 m.insert(movToXmm)
2893 case !src64 && dst64:
2894 loadToGP := m.allocateInstr().asImm(tmpGp, 0x5f000000, false)
2895 m.insert(loadToGP)
2896 movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false)
2897 m.insert(movToXmm)
2898 case !src64 && !dst64:
2899 loadToGP := m.allocateInstr().asImm(tmpGp, 0x4f000000, false)
2900 m.insert(loadToGP)
2901 movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false)
2902 m.insert(movToXmm)
2903 }
2904
2905 cmp := m.allocateInstr()
2906 cmp.asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
2907 m.insert(cmp)
2908
2909 // If above `tmp` ("large threshold"), jump to `ifAboveThreshold`
2910 ifAboveThresholdTarget, ifAboveThreshold := m.allocateBrTarget()
2911 jmpIfAboveThreshold := m.allocateInstr()
2912 jmpIfAboveThreshold.asJmpIf(condNB, newOperandLabel(ifAboveThreshold))
2913 m.insert(jmpIfAboveThreshold)
2914
2915 ifNotNaNTarget, ifNotNaN := m.allocateBrTarget()
2916 jmpIfNotNaN := m.allocateInstr()
2917 jmpIfNotNaN.asJmpIf(condNP, newOperandLabel(ifNotNaN))
2918 m.insert(jmpIfNotNaN)
2919
2920 // If NaN, handle the error condition.
2921 if sat {
2922 // On NaN, saturating, we just return 0.
2923 zeros := m.allocateInstr().asZeros(tmpGp)
2924 m.insert(zeros)
2925
2926 jmpEnd := m.allocateInstr()
2927 jmpEnd.asJmp(newOperandLabel(done))
2928 m.insert(jmpEnd)
2929 } else {
2930 // On NaN, non-saturating, we trap.
2931 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger)
2932 }
2933
2934 // If not NaN, land here.
2935 m.insert(ifNotNaNTarget)
2936
2937 // Truncation happens here.
2938
2939 trunc := m.allocateInstr()
2940 trunc.asXmmToGpr(truncOp, src, tmpGp, dst64)
2941 m.insert(trunc)
2942
2943 // Check if the result is negative.
2944 cmpNeg := m.allocateInstr()
2945 cmpNeg.asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64)
2946 m.insert(cmpNeg)
2947
2948 // If non-neg, jump to end.
2949 jmpIfNonNeg := m.allocateInstr()
2950 jmpIfNonNeg.asJmpIf(condNL, newOperandLabel(done))
2951 m.insert(jmpIfNonNeg)
2952
2953 if sat {
2954 // If the input was "small" (< 2**(width -1)), the only way to get an integer
2955 // overflow is because the input was too small: saturate to the min value, i.e. 0.
2956 zeros := m.allocateInstr().asZeros(tmpGp)
2957 m.insert(zeros)
2958
2959 jmpEnd := m.allocateInstr()
2960 jmpEnd.asJmp(newOperandLabel(done))
2961 m.insert(jmpEnd)
2962 } else {
2963 // If not saturating, trap.
2964 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2965 }
2966
2967 // If above the threshold, land here.
2968 m.insert(ifAboveThresholdTarget)
2969
2970 // tmpDiff := threshold - rn.
2971 copySrc := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), tmpXmm2)
2972 m.insert(copySrc)
2973
2974 sub := m.allocateInstr()
2975 sub.asXmmRmR(subOp, newOperandReg(tmpXmm), tmpXmm2) // must be -0x8000000000000000
2976 m.insert(sub)
2977
2978 trunc2 := m.allocateInstr()
2979 trunc2.asXmmToGpr(truncOp, tmpXmm2, tmpGp, dst64)
2980 m.insert(trunc2)
2981
2982 // Check if the result is negative.
2983 cmpNeg2 := m.allocateInstr().asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64)
2984 m.insert(cmpNeg2)
2985
2986 ifNextLargeTarget, ifNextLarge := m.allocateBrTarget()
2987 jmpIfNextLarge := m.allocateInstr()
2988 jmpIfNextLarge.asJmpIf(condNL, newOperandLabel(ifNextLarge))
2989 m.insert(jmpIfNextLarge)
2990
2991 if sat {
2992 // The input was "large" (>= maxInt), so the only way to get an integer
2993 // overflow is because the input was too large: saturate to the max value.
2994 var maxInt uint64
2995 if dst64 {
2996 maxInt = math.MaxUint64
2997 } else {
2998 maxInt = math.MaxUint32
2999 }
3000 m.lowerIconst(tmpGp, maxInt, dst64)
3001
3002 jmpToEnd := m.allocateInstr()
3003 jmpToEnd.asJmp(newOperandLabel(done))
3004 m.insert(jmpToEnd)
3005 } else {
3006 // If not saturating, trap.
3007 m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
3008 }
3009
3010 m.insert(ifNextLargeTarget)
3011
3012 var op operand
3013 if dst64 {
3014 m.lowerIconst(tmpGp2, 0x8000000000000000, true)
3015 op = newOperandReg(tmpGp2)
3016 } else {
3017 op = newOperandImm32(0x80000000)
3018 }
3019
3020 add := m.allocateInstr()
3021 add.asAluRmiR(aluRmiROpcodeAdd, op, tmpGp, dst64)
3022 m.insert(add)
3023
3024 m.insert(doneTarget)
3025 }
3026
3027 func (m *machine) lowerFcvtFromSint(rn, rd operand, src64, dst64 bool) {
3028 var op sseOpcode
3029 if dst64 {
3030 op = sseOpcodeCvtsi2sd
3031 } else {
3032 op = sseOpcodeCvtsi2ss
3033 }
3034
3035 trunc := m.allocateInstr()
3036 trunc.asGprToXmm(op, rn, rd.reg(), src64)
3037 m.insert(trunc)
3038 }
3039
3040 func (m *machine) lowerFcvtFromUint(rn, rd operand, src64, dst64 bool) {
3041 var op sseOpcode
3042 if dst64 {
3043 op = sseOpcodeCvtsi2sd
3044 } else {
3045 op = sseOpcodeCvtsi2ss
3046 }
3047
3048 // Src is 32 bit, then we just perform the conversion with 64 bit width.
3049 //
3050 // See the following link for why we use 64bit conversion for unsigned 32bit integer sources:
3051 // https://stackoverflow.com/questions/41495498/fpu-operations-generated-by-gcc-during-casting-integer-to-float.
3052 //
3053 // Here's the summary:
3054 // >> CVTSI2SS is indeed designed for converting a signed integer to a scalar single-precision float,
3055 // >> not an unsigned integer like you have here. So what gives? Well, a 64-bit processor has 64-bit wide
3056 // >> registers available, so the unsigned 32-bit input values can be stored as signed 64-bit intermediate values,
3057 // >> which allows CVTSI2SS to be used after all.
3058 //
3059 if !src64 {
3060 // Before we convert, we have to clear the higher 32-bits of the 64-bit register
3061 // to get the correct result.
3062 tmp := m.c.AllocateVReg(ssa.TypeI32)
3063 m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, tmp))
3064 m.insert(m.allocateInstr().asGprToXmm(op, newOperandReg(tmp), rd.reg(), true))
3065 return
3066 }
3067
3068 // If uint64, we have to do a bit more work.
3069 endTarget, end := m.allocateBrTarget()
3070
3071 var tmpXmm regalloc.VReg
3072 if dst64 {
3073 tmpXmm = m.c.AllocateVReg(ssa.TypeF64)
3074 } else {
3075 tmpXmm = m.c.AllocateVReg(ssa.TypeF32)
3076 }
3077
3078 // Check if the most significant bit (sign bit) is set.
3079 test := m.allocateInstr()
3080 test.asCmpRmiR(false, rn, rn.reg(), src64)
3081 m.insert(test)
3082
3083 // Jump if the sign bit is set.
3084 ifSignTarget, ifSign := m.allocateBrTarget()
3085 jmpIfNeg := m.allocateInstr()
3086 jmpIfNeg.asJmpIf(condS, newOperandLabel(ifSign))
3087 m.insert(jmpIfNeg)
3088
3089 // If the sign bit is not set, we could fit the unsigned int into float32/float64.
3090 // So, we convert it to float and emit jump instruction to exit from this branch.
3091 cvt := m.allocateInstr()
3092 cvt.asGprToXmm(op, rn, tmpXmm, src64)
3093 m.insert(cvt)
3094
3095 // We are done, jump to end.
3096 jmpEnd := m.allocateInstr()
3097 jmpEnd.asJmp(newOperandLabel(end))
3098 m.insert(jmpEnd)
3099
3100 // Now handling the case where sign-bit is set.
3101 // We emit the following sequences:
3102 // mov %rn, %tmp
3103 // shr 1, %tmp
3104 // mov %rn, %tmp2
3105 // and 1, %tmp2
3106 // or %tmp2, %tmp
3107 // cvtsi2ss %tmp, %xmm0
3108 // addsd %xmm0, %xmm0
3109 m.insert(ifSignTarget)
3110
3111 tmp := m.copyToTmp(rn.reg())
3112 shr := m.allocateInstr()
3113 shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(1), tmp, src64)
3114 m.insert(shr)
3115
3116 tmp2 := m.copyToTmp(rn.reg())
3117 and := m.allocateInstr()
3118 and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, src64)
3119 m.insert(and)
3120
3121 or := m.allocateInstr()
3122 or.asAluRmiR(aluRmiROpcodeOr, newOperandReg(tmp2), tmp, src64)
3123 m.insert(or)
3124
3125 cvt2 := m.allocateInstr()
3126 cvt2.asGprToXmm(op, newOperandReg(tmp), tmpXmm, src64)
3127 m.insert(cvt2)
3128
3129 addsd := m.allocateInstr()
3130 if dst64 {
3131 addsd.asXmmRmR(sseOpcodeAddsd, newOperandReg(tmpXmm), tmpXmm)
3132 } else {
3133 addsd.asXmmRmR(sseOpcodeAddss, newOperandReg(tmpXmm), tmpXmm)
3134 }
3135 m.insert(addsd)
3136
3137 m.insert(endTarget)
3138 m.copyTo(tmpXmm, rd.reg())
3139 }
3140
3141 func (m *machine) lowerVanyTrue(instr *ssa.Instruction) {
3142 x := instr.Arg()
3143 rm := m.getOperand_Reg(m.c.ValueDefinition(x))
3144 rd := m.c.VRegOf(instr.Return())
3145
3146 tmp := m.c.AllocateVReg(ssa.TypeI32)
3147
3148 cmp := m.allocateInstr()
3149 cmp.asXmmCmpRmR(sseOpcodePtest, rm, rm.reg())
3150 m.insert(cmp)
3151
3152 setcc := m.allocateInstr()
3153 setcc.asSetcc(condNZ, tmp)
3154 m.insert(setcc)
3155
3156 // Clear the irrelevant bits.
3157 and := m.allocateInstr()
3158 and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp, false)
3159 m.insert(and)
3160
3161 m.copyTo(tmp, rd)
3162 }
3163
3164 func (m *machine) lowerVallTrue(instr *ssa.Instruction) {
3165 x, lane := instr.ArgWithLane()
3166 var op sseOpcode
3167 switch lane {
3168 case ssa.VecLaneI8x16:
3169 op = sseOpcodePcmpeqb
3170 case ssa.VecLaneI16x8:
3171 op = sseOpcodePcmpeqw
3172 case ssa.VecLaneI32x4:
3173 op = sseOpcodePcmpeqd
3174 case ssa.VecLaneI64x2:
3175 op = sseOpcodePcmpeqq
3176 }
3177 rm := m.getOperand_Reg(m.c.ValueDefinition(x))
3178 rd := m.c.VRegOf(instr.Return())
3179
3180 tmp := m.c.AllocateVReg(ssa.TypeV128)
3181
3182 zeros := m.allocateInstr()
3183 zeros.asZeros(tmp)
3184 m.insert(zeros)
3185
3186 pcmp := m.allocateInstr()
3187 pcmp.asXmmRmR(op, rm, tmp)
3188 m.insert(pcmp)
3189
3190 test := m.allocateInstr()
3191 test.asXmmCmpRmR(sseOpcodePtest, newOperandReg(tmp), tmp)
3192 m.insert(test)
3193
3194 tmp2 := m.c.AllocateVReg(ssa.TypeI32)
3195
3196 setcc := m.allocateInstr()
3197 setcc.asSetcc(condZ, tmp2)
3198 m.insert(setcc)
3199
3200 // Clear the irrelevant bits.
3201 and := m.allocateInstr()
3202 and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, false)
3203 m.insert(and)
3204
3205 m.copyTo(tmp2, rd)
3206 }
3207
3208 func (m *machine) lowerVhighBits(instr *ssa.Instruction) {
3209 x, lane := instr.ArgWithLane()
3210 rm := m.getOperand_Reg(m.c.ValueDefinition(x))
3211 rd := m.c.VRegOf(instr.Return())
3212 switch lane {
3213 case ssa.VecLaneI8x16:
3214 mov := m.allocateInstr()
3215 mov.asXmmToGpr(sseOpcodePmovmskb, rm.reg(), rd, false)
3216 m.insert(mov)
3217
3218 case ssa.VecLaneI16x8:
3219 // When we have:
3220 // R1 = [R1(w1), R1(w2), R1(w3), R1(w4), R1(w5), R1(w6), R1(w7), R1(v8)]
3221 // R2 = [R2(w1), R2(w2), R2(w3), R2(v4), R2(w5), R2(w6), R2(w7), R2(v8)]
3222 // where RX(wn) is n-th signed word (16-bit) of RX register,
3223 //
3224 // "PACKSSWB R1, R2" produces
3225 // R1 = [
3226 // byte_sat(R1(w1)), byte_sat(R1(w2)), byte_sat(R1(w3)), byte_sat(R1(w4)),
3227 // byte_sat(R1(w5)), byte_sat(R1(w6)), byte_sat(R1(w7)), byte_sat(R1(w8)),
3228 // byte_sat(R2(w1)), byte_sat(R2(w2)), byte_sat(R2(w3)), byte_sat(R2(w4)),
3229 // byte_sat(R2(w5)), byte_sat(R2(w6)), byte_sat(R2(w7)), byte_sat(R2(w8)),
3230 // ]
3231 // where R1 is the destination register, and
3232 // byte_sat(w) = int8(w) if w fits as signed 8-bit,
3233 // 0x80 if w is less than 0x80
3234 // 0x7F if w is greater than 0x7f
3235 //
3236 // See https://www.felixcloutier.com/x86/packsswb:packssdw for detail.
3237 //
3238 // Therefore, v.register ends up having i-th and (i+8)-th bit set if i-th lane is negative (for i in 0..8).
3239 tmp := m.copyToTmp(rm.reg())
3240 res := m.c.AllocateVReg(ssa.TypeI32)
3241
3242 pak := m.allocateInstr()
3243 pak.asXmmRmR(sseOpcodePacksswb, rm, tmp)
3244 m.insert(pak)
3245
3246 mov := m.allocateInstr()
3247 mov.asXmmToGpr(sseOpcodePmovmskb, tmp, res, false)
3248 m.insert(mov)
3249
3250 // Clear the higher bits than 8.
3251 shr := m.allocateInstr()
3252 shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(8), res, false)
3253 m.insert(shr)
3254
3255 m.copyTo(res, rd)
3256
3257 case ssa.VecLaneI32x4:
3258 mov := m.allocateInstr()
3259 mov.asXmmToGpr(sseOpcodeMovmskps, rm.reg(), rd, true)
3260 m.insert(mov)
3261
3262 case ssa.VecLaneI64x2:
3263 mov := m.allocateInstr()
3264 mov.asXmmToGpr(sseOpcodeMovmskpd, rm.reg(), rd, true)
3265 m.insert(mov)
3266 }
3267 }
3268
3269 func (m *machine) lowerVbnot(instr *ssa.Instruction) {
3270 x := instr.Arg()
3271 xDef := m.c.ValueDefinition(x)
3272 rm := m.getOperand_Reg(xDef)
3273 rd := m.c.VRegOf(instr.Return())
3274
3275 tmp := m.copyToTmp(rm.reg())
3276 tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3277
3278 // Ensure tmp2 is considered defined by regalloc.
3279 m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3280
3281 // Set all bits on tmp register.
3282 pak := m.allocateInstr()
3283 pak.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp2), tmp2)
3284 m.insert(pak)
3285
3286 // Then XOR with tmp to reverse all bits on v.register.
3287 xor := m.allocateInstr()
3288 xor.asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)
3289 m.insert(xor)
3290
3291 m.copyTo(tmp, rd)
3292 }
3293
3294 func (m *machine) lowerSplat(x, ret ssa.Value, lane ssa.VecLane) {
3295 tmpDst := m.c.AllocateVReg(ssa.TypeV128)
3296 m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst))
3297
3298 switch lane {
3299 case ssa.VecLaneI8x16:
3300 tmp := m.c.AllocateVReg(ssa.TypeV128)
3301 m.insert(m.allocateInstr().asDefineUninitializedReg(tmp))
3302 xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3303 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, xx, tmpDst))
3304 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp))
3305 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpDst))
3306 case ssa.VecLaneI16x8:
3307 xx := m.getOperand_Reg(m.c.ValueDefinition(x))
3308 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, xx, tmpDst))
3309 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, xx, tmpDst))
3310 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
3311 case ssa.VecLaneI32x4:
3312 xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3313 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, xx, tmpDst))
3314 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
3315 case ssa.VecLaneI64x2:
3316 xx := m.getOperand_Reg(m.c.ValueDefinition(x))
3317 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, xx, tmpDst))
3318 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, xx, tmpDst))
3319 case ssa.VecLaneF32x4:
3320 xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3321 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, 0, xx, tmpDst))
3322 m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
3323 case ssa.VecLaneF64x2:
3324 xx := m.getOperand_Reg(m.c.ValueDefinition(x))
3325 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst))
3326 m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, xx, tmpDst))
3327 default:
3328 panic(fmt.Sprintf("invalid lane type: %s", lane))
3329 }
3330
3331 m.copyTo(tmpDst, m.c.VRegOf(ret))
3332 }
3333
3334 func (m *machine) lowerShuffle(x, y ssa.Value, lo, hi uint64, ret ssa.Value) {
3335 var xMask, yMask [2]uint64
3336 for i := 0; i < 8; i++ {
3337 loLane := byte(lo >> (i * 8))
3338 if loLane < 16 {
3339 xMask[0] |= uint64(loLane) << (i * 8)
3340 yMask[0] |= uint64(0x80) << (i * 8)
3341 } else {
3342 xMask[0] |= uint64(0x80) << (i * 8)
3343 yMask[0] |= uint64(loLane-16) << (i * 8)
3344 }
3345 hiLane := byte(hi >> (i * 8))
3346 if hiLane < 16 {
3347 xMask[1] |= uint64(hiLane) << (i * 8)
3348 yMask[1] |= uint64(0x80) << (i * 8)
3349 } else {
3350 xMask[1] |= uint64(0x80) << (i * 8)
3351 yMask[1] |= uint64(hiLane-16) << (i * 8)
3352 }
3353 }
3354
3355 xl, xmaskPos := m.allocateLabel()
3356 m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xl, labelPos: xmaskPos})
3357 yl, ymaskPos := m.allocateLabel()
3358 m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: yl, labelPos: ymaskPos})
3359
3360 xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y))
3361 tmpX, tmpY := m.copyToTmp(xx.reg()), m.copyToTmp(yy.reg())
3362
3363 // Apply mask to X.
3364 tmp := m.c.AllocateVReg(ssa.TypeV128)
3365 loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xl)), tmp)
3366 m.insert(loadMaskLo)
3367 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpX))
3368
3369 // Apply mask to Y.
3370 loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(yl)), tmp)
3371 m.insert(loadMaskHi)
3372 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpY))
3373
3374 // Combine the results.
3375 m.insert(m.allocateInstr().asXmmRmR(sseOpcodeOrps, newOperandReg(tmpX), tmpY))
3376
3377 m.copyTo(tmpY, m.c.VRegOf(ret))
3378 }
3379
3380 func (m *machine) lowerVbBinOpUnaligned(op sseOpcode, x, y, ret ssa.Value) {
3381 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3382 rm := m.getOperand_Reg(m.c.ValueDefinition(y))
3383 rd := m.c.VRegOf(ret)
3384
3385 tmp := m.copyToTmp(rn.reg())
3386
3387 binOp := m.allocateInstr()
3388 binOp.asXmmRmR(op, rm, tmp)
3389 m.insert(binOp)
3390
3391 m.copyTo(tmp, rd)
3392 }
3393
3394 func (m *machine) lowerVbBinOp(op sseOpcode, x, y, ret ssa.Value) {
3395 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3396 rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3397 rd := m.c.VRegOf(ret)
3398
3399 tmp := m.copyToTmp(rn.reg())
3400
3401 binOp := m.allocateInstr()
3402 binOp.asXmmRmR(op, rm, tmp)
3403 m.insert(binOp)
3404
3405 m.copyTo(tmp, rd)
3406 }
3407
3408 func (m *machine) lowerVFcmp(x, y ssa.Value, c ssa.FloatCmpCond, ret ssa.Value, lane ssa.VecLane) {
3409 var cmpOp sseOpcode
3410 switch lane {
3411 case ssa.VecLaneF32x4:
3412 cmpOp = sseOpcodeCmpps
3413 case ssa.VecLaneF64x2:
3414 cmpOp = sseOpcodeCmppd
3415 default:
3416 panic(fmt.Sprintf("invalid lane type: %s", lane))
3417 }
3418
3419 xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
3420 var cmpImm cmpPred
3421 switch c {
3422 case ssa.FloatCmpCondGreaterThan:
3423 yy, xx = xx, yy
3424 cmpImm = cmpPredLT_OS
3425 case ssa.FloatCmpCondGreaterThanOrEqual:
3426 yy, xx = xx, yy
3427 cmpImm = cmpPredLE_OS
3428 case ssa.FloatCmpCondEqual:
3429 cmpImm = cmpPredEQ_OQ
3430 case ssa.FloatCmpCondNotEqual:
3431 cmpImm = cmpPredNEQ_UQ
3432 case ssa.FloatCmpCondLessThan:
3433 cmpImm = cmpPredLT_OS
3434 case ssa.FloatCmpCondLessThanOrEqual:
3435 cmpImm = cmpPredLE_OS
3436 default:
3437 panic(fmt.Sprintf("invalid float comparison condition: %s", c))
3438 }
3439
3440 tmp := m.c.AllocateVReg(ssa.TypeV128)
3441 xxx := m.getOperand_Mem_Reg(xx)
3442 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, xxx, tmp))
3443
3444 rm := m.getOperand_Mem_Reg(yy)
3445 m.insert(m.allocateInstr().asXmmRmRImm(cmpOp, byte(cmpImm), rm, tmp))
3446
3447 m.copyTo(tmp, m.c.VRegOf(ret))
3448 }
3449
3450 func (m *machine) lowerVIcmp(x, y ssa.Value, c ssa.IntegerCmpCond, ret ssa.Value, lane ssa.VecLane) {
3451 var eq, gt, maxu, minu, mins sseOpcode
3452 switch lane {
3453 case ssa.VecLaneI8x16:
3454 eq, gt, maxu, minu, mins = sseOpcodePcmpeqb, sseOpcodePcmpgtb, sseOpcodePmaxub, sseOpcodePminub, sseOpcodePminsb
3455 case ssa.VecLaneI16x8:
3456 eq, gt, maxu, minu, mins = sseOpcodePcmpeqw, sseOpcodePcmpgtw, sseOpcodePmaxuw, sseOpcodePminuw, sseOpcodePminsw
3457 case ssa.VecLaneI32x4:
3458 eq, gt, maxu, minu, mins = sseOpcodePcmpeqd, sseOpcodePcmpgtd, sseOpcodePmaxud, sseOpcodePminud, sseOpcodePminsd
3459 case ssa.VecLaneI64x2:
3460 eq, gt = sseOpcodePcmpeqq, sseOpcodePcmpgtq
3461 default:
3462 panic(fmt.Sprintf("invalid lane type: %s", lane))
3463 }
3464
3465 tmp := m.c.AllocateVReg(ssa.TypeV128)
3466 var op operand
3467 switch c {
3468 case ssa.IntegerCmpCondSignedLessThanOrEqual:
3469 if lane == ssa.VecLaneI64x2 {
3470 x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3471 // Copy x to tmp.
3472 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
3473 op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3474 } else {
3475 y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3476 // Copy y to tmp.
3477 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
3478 op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3479 }
3480 case ssa.IntegerCmpCondSignedGreaterThanOrEqual:
3481 if lane == ssa.VecLaneI64x2 {
3482 y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3483 // Copy y to tmp.
3484 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
3485 op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3486 } else {
3487 x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3488 // Copy x to tmp.
3489 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
3490 op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3491 }
3492 case ssa.IntegerCmpCondSignedLessThan, ssa.IntegerCmpCondUnsignedLessThan, ssa.IntegerCmpCondUnsignedLessThanOrEqual:
3493 y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3494 // Copy y to tmp.
3495 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
3496 op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3497 default:
3498 x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3499 // Copy x to tmp.
3500 m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
3501 op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3502 }
3503
3504 switch c {
3505 case ssa.IntegerCmpCondEqual:
3506 m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3507 case ssa.IntegerCmpCondNotEqual:
3508 // First we compare for equality.
3509 m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3510 // Then flip the bits. To do so, we set all bits on tmp2.
3511 tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3512 m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3513 m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
3514 // And then xor with tmp.
3515 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
3516 case ssa.IntegerCmpCondSignedGreaterThan, ssa.IntegerCmpCondSignedLessThan:
3517 m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp))
3518 case ssa.IntegerCmpCondSignedGreaterThanOrEqual, ssa.IntegerCmpCondSignedLessThanOrEqual:
3519 if lane == ssa.VecLaneI64x2 {
3520 m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp))
3521 // Then flip the bits. To do so, we set all bits on tmp2.
3522 tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3523 m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3524 m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
3525 // And then xor with tmp.
3526 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
3527 } else {
3528 // First take min of x and y.
3529 m.insert(m.allocateInstr().asXmmRmR(mins, op, tmp))
3530 // Then compare for equality.
3531 m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3532 }
3533 case ssa.IntegerCmpCondUnsignedGreaterThan, ssa.IntegerCmpCondUnsignedLessThan:
3534 // First maxu of x and y.
3535 m.insert(m.allocateInstr().asXmmRmR(maxu, op, tmp))
3536 // Then compare for equality.
3537 m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3538 // Then flip the bits. To do so, we set all bits on tmp2.
3539 tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3540 m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3541 m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
3542 // And then xor with tmp.
3543 m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
3544 case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual, ssa.IntegerCmpCondUnsignedLessThanOrEqual:
3545 m.insert(m.allocateInstr().asXmmRmR(minu, op, tmp))
3546 m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3547 default:
3548 panic("BUG")
3549 }
3550
3551 m.copyTo(tmp, m.c.VRegOf(ret))
3552 }
3553
3554 func (m *machine) lowerVbandnot(instr *ssa.Instruction, op sseOpcode) {
3555 x, y := instr.Arg2()
3556 xDef := m.c.ValueDefinition(x)
3557 yDef := m.c.ValueDefinition(y)
3558 rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
3559 rd := m.c.VRegOf(instr.Return())
3560
3561 tmp := m.copyToTmp(rn.reg())
3562
3563 // pandn between rn, rm.
3564 pand := m.allocateInstr()
3565 pand.asXmmRmR(sseOpcodePandn, rm, tmp)
3566 m.insert(pand)
3567
3568 m.copyTo(tmp, rd)
3569 }
3570
3571 func (m *machine) lowerVbitselect(instr *ssa.Instruction) {
3572 c, x, y := instr.SelectData()
3573 xDef := m.c.ValueDefinition(x)
3574 yDef := m.c.ValueDefinition(y)
3575 rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
3576 creg := m.getOperand_Reg(m.c.ValueDefinition(c))
3577 rd := m.c.VRegOf(instr.Return())
3578
3579 tmpC := m.copyToTmp(creg.reg())
3580 tmpX := m.copyToTmp(rm.reg())
3581
3582 // And between c, x (overwrites x).
3583 pand := m.allocateInstr()
3584 pand.asXmmRmR(sseOpcodePand, creg, tmpX)
3585 m.insert(pand)
3586
3587 // Andn between y, c (overwrites c).
3588 pandn := m.allocateInstr()
3589 pandn.asXmmRmR(sseOpcodePandn, rn, tmpC)
3590 m.insert(pandn)
3591
3592 por := m.allocateInstr()
3593 por.asXmmRmR(sseOpcodePor, newOperandReg(tmpC), tmpX)
3594 m.insert(por)
3595
3596 m.copyTo(tmpX, rd)
3597 }
3598
3599 func (m *machine) lowerVFmin(instr *ssa.Instruction) {
3600 x, y, lane := instr.Arg2WithLane()
3601 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3602 rm := m.getOperand_Reg(m.c.ValueDefinition(y))
3603 rd := m.c.VRegOf(instr.Return())
3604
3605 var min, cmp, andn, or, srl /* shift right logical */ sseOpcode
3606 var shiftNumToInverseNaN uint32
3607 if lane == ssa.VecLaneF32x4 {
3608 min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodePsrld, 0xa
3609 } else {
3610 min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodePsrlq, 0xd
3611 }
3612
3613 tmp1 := m.copyToTmp(rn.reg())
3614 tmp2 := m.copyToTmp(rm.reg())
3615
3616 // tmp1=min(rn, rm)
3617 minIns1 := m.allocateInstr()
3618 minIns1.asXmmRmR(min, rn, tmp2)
3619 m.insert(minIns1)
3620
3621 // tmp2=min(rm, rn)
3622 minIns2 := m.allocateInstr()
3623 minIns2.asXmmRmR(min, rm, tmp1)
3624 m.insert(minIns2)
3625
3626 // tmp3:=tmp1=min(rn, rm)
3627 tmp3 := m.copyToTmp(tmp1)
3628
3629 // tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
3630 // NaN if rn == NaN || rm == NaN
3631 // min(rm, rm) otherwise
3632 orIns := m.allocateInstr()
3633 orIns.asXmmRmR(or, newOperandReg(tmp2), tmp1)
3634 m.insert(orIns)
3635
3636 // tmp3 is originally min(rn,rm).
3637 // tmp3 = 0^ (set all bits) if rn == NaN || rm == NaN
3638 // 0 otherwise
3639 cmpIns := m.allocateInstr()
3640 cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp2), tmp3)
3641 m.insert(cmpIns)
3642
3643 // tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
3644 // ^0 if rn == NaN || rm == NaN
3645 // min(v1, v2) otherwise
3646 orIns2 := m.allocateInstr()
3647 orIns2.asXmmRmR(or, newOperandReg(tmp3), tmp1)
3648 m.insert(orIns2)
3649
3650 // tmp3 = set all bits on the mantissa bits
3651 // 0 otherwise
3652 shift := m.allocateInstr()
3653 shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp3)
3654 m.insert(shift)
3655
3656 // tmp3 = tmp1 and !tmp3
3657 // = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
3658 // set all bits on exponential and sign bit (== NaN) if rn == NaN || rm == NaN
3659 // min(rn, rm) otherwise
3660 andnIns := m.allocateInstr()
3661 andnIns.asXmmRmR(andn, newOperandReg(tmp1), tmp3)
3662 m.insert(andnIns)
3663
3664 m.copyTo(tmp3, rd)
3665 }
3666
3667 func (m *machine) lowerVFmax(instr *ssa.Instruction) {
3668 x, y, lane := instr.Arg2WithLane()
3669 rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3670 rm := m.getOperand_Reg(m.c.ValueDefinition(y))
3671 rd := m.c.VRegOf(instr.Return())
3672
3673 var max, cmp, andn, or, xor, sub, srl /* shift right logical */ sseOpcode
3674 var shiftNumToInverseNaN uint32
3675 if lane == ssa.VecLaneF32x4 {
3676 max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodeXorps, sseOpcodeSubps, sseOpcodePsrld, 0xa
3677 } else {
3678 max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodeXorpd, sseOpcodeSubpd, sseOpcodePsrlq, 0xd
3679 }
3680
3681 tmp0 := m.copyToTmp(rm.reg())
3682 tmp1 := m.copyToTmp(rn.reg())
3683
3684 // tmp0=max(rn, rm)
3685 maxIns1 := m.allocateInstr()
3686 maxIns1.asXmmRmR(max, rn, tmp0)
3687 m.insert(maxIns1)
3688
3689 // tmp1=max(rm, rn)
3690 maxIns2 := m.allocateInstr()
3691 maxIns2.asXmmRmR(max, rm, tmp1)
3692 m.insert(maxIns2)
3693
3694 // tmp2=max(rm, rn)
3695 tmp2 := m.copyToTmp(tmp1)
3696
3697 // tmp2 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0)
3698 // 0 if (rn == 0 && rm == 0)
3699 // -0 if (rn == -0 && rm == -0)
3700 // v1^v2 if rn == NaN || rm == NaN
3701 // 0 otherwise
3702 xorInstr := m.allocateInstr()
3703 xorInstr.asXmmRmR(xor, newOperandReg(tmp0), tmp2)
3704 m.insert(xorInstr)
3705 // tmp1 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0)
3706 // 0 if (rn == 0 && rm == 0)
3707 // -0 if (rn == -0 && rm == -0)
3708 // NaN if rn == NaN || rm == NaN
3709 // max(v1, v2) otherwise
3710 orInstr := m.allocateInstr()
3711 orInstr.asXmmRmR(or, newOperandReg(tmp2), tmp1)
3712 m.insert(orInstr)
3713
3714 tmp3 := m.copyToTmp(tmp1)
3715
3716 // tmp3 = 0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) || (rn == 0 && rm == 0)
3717 // -0 if (rn == -0 && rm == -0)
3718 // NaN if rn == NaN || rm == NaN
3719 // max(v1, v2) otherwise
3720 //
3721 // Note: -0 - (-0) = 0 (!= -0) in floating point operation.
3722 subIns := m.allocateInstr()
3723 subIns.asXmmRmR(sub, newOperandReg(tmp2), tmp3)
3724 m.insert(subIns)
3725
3726 // tmp1 = 0^ if rn == NaN || rm == NaN
3727 cmpIns := m.allocateInstr()
3728 cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp1), tmp1)
3729 m.insert(cmpIns)
3730
3731 // tmp1 = set all bits on the mantissa bits
3732 // 0 otherwise
3733 shift := m.allocateInstr()
3734 shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp1)
3735 m.insert(shift)
3736
3737 andnIns := m.allocateInstr()
3738 andnIns.asXmmRmR(andn, newOperandReg(tmp3), tmp1)
3739 m.insert(andnIns)
3740
3741 m.copyTo(tmp1, rd)
3742 }
3743
3744 func (m *machine) lowerVFabs(instr *ssa.Instruction) {
3745 x, lane := instr.ArgWithLane()
3746 rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3747 rd := m.c.VRegOf(instr.Return())
3748
3749 tmp := m.c.AllocateVReg(ssa.TypeV128)
3750
3751 def := m.allocateInstr()
3752 def.asDefineUninitializedReg(tmp)
3753 m.insert(def)
3754
3755 // Set all bits on tmp.
3756 pcmp := m.allocateInstr()
3757 pcmp.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp)
3758 m.insert(pcmp)
3759
3760 switch lane {
3761 case ssa.VecLaneF32x4:
3762 // Shift right packed single floats by 1 to clear the sign bits.
3763 shift := m.allocateInstr()
3764 shift.asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp)
3765 m.insert(shift)
3766 // Clear the sign bit of rm.
3767 andp := m.allocateInstr()
3768 andp.asXmmRmR(sseOpcodeAndpd, rm, tmp)
3769 m.insert(andp)
3770 case ssa.VecLaneF64x2:
3771 // Shift right packed single floats by 1 to clear the sign bits.
3772 shift := m.allocateInstr()
3773 shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(1), tmp)
3774 m.insert(shift)
3775 // Clear the sign bit of rm.
3776 andp := m.allocateInstr()
3777 andp.asXmmRmR(sseOpcodeAndps, rm, tmp)
3778 m.insert(andp)
3779 }
3780
3781 m.copyTo(tmp, rd)
3782 }