| 1 |
package amd64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"context" |
| 5 |
"encoding/binary" |
| 6 |
"fmt" |
| 7 |
"math" |
| 8 |
"strings" |
| 9 |
|
| 10 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 11 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 12 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 13 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" |
| 14 |
"github.com/tetratelabs/wazero/internal/platform" |
| 15 |
) |
| 16 |
|
| 17 |
// NewBackend returns a new backend for arm64. |
| 18 |
func NewBackend() backend.Machine { |
| 19 |
m := &machine{ |
| 20 |
cpuFeatures: platform.CpuFeatures, |
| 21 |
regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo), |
| 22 |
spillSlots: map[regalloc.VRegID]int64{}, |
| 23 |
amodePool: wazevoapi.NewPool[amode](nil), |
| 24 |
labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition), |
| 25 |
instrPool: wazevoapi.NewPool[instruction](resetInstruction), |
| 26 |
constSwizzleMaskConstIndex: -1, |
| 27 |
constSqmulRoundSatIndex: -1, |
| 28 |
constI8x16SHLMaskTableIndex: -1, |
| 29 |
constI8x16LogicalSHRMaskTableIndex: -1, |
| 30 |
constF64x2CvtFromIMaskIndex: -1, |
| 31 |
constTwop52Index: -1, |
| 32 |
constI32sMaxOnF64x2Index: -1, |
| 33 |
constI32uMaxOnF64x2Index: -1, |
| 34 |
constAllOnesI8x16Index: -1, |
| 35 |
constAllOnesI16x8Index: -1, |
| 36 |
constExtAddPairwiseI16x8uMask1Index: -1, |
| 37 |
constExtAddPairwiseI16x8uMask2Index: -1, |
| 38 |
} |
| 39 |
m.regAllocFn.m = m |
| 40 |
return m |
| 41 |
} |
| 42 |
|
| 43 |
type ( |
| 44 |
// machine implements backend.Machine for amd64. |
| 45 |
machine struct { |
| 46 |
c backend.Compiler |
| 47 |
stackBoundsCheckDisabled bool |
| 48 |
|
| 49 |
instrPool wazevoapi.Pool[instruction] |
| 50 |
amodePool wazevoapi.Pool[amode] |
| 51 |
|
| 52 |
cpuFeatures platform.CpuFeatureFlags |
| 53 |
|
| 54 |
regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn] |
| 55 |
regAllocFn regAllocFn |
| 56 |
regAllocStarted bool |
| 57 |
|
| 58 |
// labelPositionPool is the pool of labelPosition. The id is the label where |
| 59 |
// if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID. |
| 60 |
labelPositionPool wazevoapi.IDedPool[labelPosition] |
| 61 |
// nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID |
| 62 |
// so that we can have an identical label for the SSA block ID, which is useful for debugging. |
| 63 |
nextLabel label |
| 64 |
// rootInstr is the first instruction of the function. |
| 65 |
rootInstr *instruction |
| 66 |
// currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition. |
| 67 |
currentLabelPos *labelPosition |
| 68 |
// orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock. |
| 69 |
orderedSSABlockLabelPos []*labelPosition |
| 70 |
// returnLabelPos is the labelPosition for the return block. |
| 71 |
returnLabelPos labelPosition |
| 72 |
// perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock. |
| 73 |
perBlockHead, perBlockEnd *instruction |
| 74 |
// pendingInstructions are the instructions which are not yet emitted into the instruction list. |
| 75 |
pendingInstructions []*instruction |
| 76 |
// maxSSABlockID is the maximum ssa.BasicBlockID in the current function. |
| 77 |
maxSSABlockID label |
| 78 |
|
| 79 |
spillSlotSize int64 |
| 80 |
spillSlots map[regalloc.VRegID]int64 |
| 81 |
currentABI *backend.FunctionABI |
| 82 |
clobberedRegs []regalloc.VReg |
| 83 |
|
| 84 |
maxRequiredStackSizeForCalls int64 |
| 85 |
|
| 86 |
labelResolutionPends []labelResolutionPend |
| 87 |
|
| 88 |
// jmpTableTargets holds the labels of the jump table targets. |
| 89 |
jmpTableTargets [][]uint32 |
| 90 |
// jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table. |
| 91 |
jmpTableTargetsNext int |
| 92 |
consts []_const |
| 93 |
|
| 94 |
constSwizzleMaskConstIndex, constSqmulRoundSatIndex, |
| 95 |
constI8x16SHLMaskTableIndex, constI8x16LogicalSHRMaskTableIndex, |
| 96 |
constF64x2CvtFromIMaskIndex, constTwop52Index, |
| 97 |
constI32sMaxOnF64x2Index, constI32uMaxOnF64x2Index, |
| 98 |
constAllOnesI8x16Index, constAllOnesI16x8Index, |
| 99 |
constExtAddPairwiseI16x8uMask1Index, constExtAddPairwiseI16x8uMask2Index int |
| 100 |
} |
| 101 |
|
| 102 |
_const struct { |
| 103 |
lo, hi uint64 |
| 104 |
_var []byte |
| 105 |
label label |
| 106 |
labelPos *labelPosition |
| 107 |
} |
| 108 |
|
| 109 |
labelResolutionPend struct { |
| 110 |
instr *instruction |
| 111 |
instrOffset int64 |
| 112 |
// imm32Offset is the offset of the last 4 bytes of the instruction. |
| 113 |
imm32Offset int64 |
| 114 |
} |
| 115 |
) |
| 116 |
|
| 117 |
type ( |
| 118 |
// label represents a position in the generated code which is either |
| 119 |
// a real instruction or the constant InstructionPool (e.g. jump tables). |
| 120 |
// |
| 121 |
// This is exactly the same as the traditional "label" in assembly code. |
| 122 |
label uint32 |
| 123 |
|
| 124 |
// labelPosition represents the regions of the generated code which the label represents. |
| 125 |
// This implements regalloc.Block. |
| 126 |
labelPosition struct { |
| 127 |
// sb is not nil if this corresponds to a ssa.BasicBlock. |
| 128 |
sb ssa.BasicBlock |
| 129 |
// cur is used to walk through the instructions in the block during the register allocation. |
| 130 |
cur, |
| 131 |
// begin and end are the first and last instructions of the block. |
| 132 |
begin, end *instruction |
| 133 |
// binaryOffset is the offset in the binary where the label is located. |
| 134 |
binaryOffset int64 |
| 135 |
} |
| 136 |
) |
| 137 |
|
| 138 |
// String implements backend.Machine. |
| 139 |
func (l label) String() string { |
| 140 |
return fmt.Sprintf("L%d", l) |
| 141 |
} |
| 142 |
|
| 143 |
func resetLabelPosition(l *labelPosition) { |
| 144 |
*l = labelPosition{} |
| 145 |
} |
| 146 |
|
| 147 |
const labelReturn = math.MaxUint32 |
| 148 |
|
| 149 |
func ssaBlockLabel(sb ssa.BasicBlock) label { |
| 150 |
if sb.ReturnBlock() { |
| 151 |
return labelReturn |
| 152 |
} |
| 153 |
return label(sb.ID()) |
| 154 |
} |
| 155 |
|
| 156 |
// getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block. |
| 157 |
func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition { |
| 158 |
if sb.ReturnBlock() { |
| 159 |
m.returnLabelPos.sb = sb |
| 160 |
return &m.returnLabelPos |
| 161 |
} |
| 162 |
|
| 163 |
l := ssaBlockLabel(sb) |
| 164 |
pos := m.labelPositionPool.GetOrAllocate(int(l)) |
| 165 |
pos.sb = sb |
| 166 |
return pos |
| 167 |
} |
| 168 |
|
| 169 |
func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) label { |
| 170 |
index := *i |
| 171 |
if index == -1 { |
| 172 |
l, pos := m.allocateLabel() |
| 173 |
index = len(m.consts) |
| 174 |
m.consts = append(m.consts, _const{ |
| 175 |
_var: _var, |
| 176 |
label: l, |
| 177 |
labelPos: pos, |
| 178 |
}) |
| 179 |
*i = index |
| 180 |
} |
| 181 |
return m.consts[index].label |
| 182 |
} |
| 183 |
|
| 184 |
// Reset implements backend.Machine. |
| 185 |
func (m *machine) Reset() { |
| 186 |
m.consts = m.consts[:0] |
| 187 |
m.clobberedRegs = m.clobberedRegs[:0] |
| 188 |
for key := range m.spillSlots { |
| 189 |
m.clobberedRegs = append(m.clobberedRegs, regalloc.VReg(key)) |
| 190 |
} |
| 191 |
for _, key := range m.clobberedRegs { |
| 192 |
delete(m.spillSlots, regalloc.VRegID(key)) |
| 193 |
} |
| 194 |
|
| 195 |
m.stackBoundsCheckDisabled = false |
| 196 |
m.regAlloc.Reset() |
| 197 |
m.labelPositionPool.Reset() |
| 198 |
m.instrPool.Reset() |
| 199 |
m.regAllocStarted = false |
| 200 |
m.clobberedRegs = m.clobberedRegs[:0] |
| 201 |
|
| 202 |
m.spillSlotSize = 0 |
| 203 |
m.maxRequiredStackSizeForCalls = 0 |
| 204 |
m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil |
| 205 |
m.pendingInstructions = m.pendingInstructions[:0] |
| 206 |
m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0] |
| 207 |
|
| 208 |
m.amodePool.Reset() |
| 209 |
m.jmpTableTargetsNext = 0 |
| 210 |
m.constSwizzleMaskConstIndex = -1 |
| 211 |
m.constSqmulRoundSatIndex = -1 |
| 212 |
m.constI8x16SHLMaskTableIndex = -1 |
| 213 |
m.constI8x16LogicalSHRMaskTableIndex = -1 |
| 214 |
m.constF64x2CvtFromIMaskIndex = -1 |
| 215 |
m.constTwop52Index = -1 |
| 216 |
m.constI32sMaxOnF64x2Index = -1 |
| 217 |
m.constI32uMaxOnF64x2Index = -1 |
| 218 |
m.constAllOnesI8x16Index = -1 |
| 219 |
m.constAllOnesI16x8Index = -1 |
| 220 |
m.constExtAddPairwiseI16x8uMask1Index = -1 |
| 221 |
m.constExtAddPairwiseI16x8uMask2Index = -1 |
| 222 |
} |
| 223 |
|
| 224 |
// StartLoweringFunction implements backend.Machine StartLoweringFunction. |
| 225 |
func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) { |
| 226 |
m.maxSSABlockID = label(maxBlockID) |
| 227 |
m.nextLabel = label(maxBlockID) + 1 |
| 228 |
} |
| 229 |
|
| 230 |
// LinkAdjacentBlocks implements backend.Machine. |
| 231 |
func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { |
| 232 |
prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next) |
| 233 |
prevPos.end.next = nextPos.begin |
| 234 |
} |
| 235 |
|
| 236 |
// StartBlock implements backend.Machine. |
| 237 |
func (m *machine) StartBlock(blk ssa.BasicBlock) { |
| 238 |
m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk) |
| 239 |
labelPos := m.currentLabelPos |
| 240 |
end := m.allocateNop() |
| 241 |
m.perBlockHead, m.perBlockEnd = end, end |
| 242 |
labelPos.begin, labelPos.end = end, end |
| 243 |
m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos) |
| 244 |
} |
| 245 |
|
| 246 |
// EndBlock implements ExecutableContext. |
| 247 |
func (m *machine) EndBlock() { |
| 248 |
// Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions. |
| 249 |
m.insertAtPerBlockHead(m.allocateNop()) |
| 250 |
|
| 251 |
m.currentLabelPos.begin = m.perBlockHead |
| 252 |
|
| 253 |
if m.currentLabelPos.sb.EntryBlock() { |
| 254 |
m.rootInstr = m.perBlockHead |
| 255 |
} |
| 256 |
} |
| 257 |
|
| 258 |
func (m *machine) insertAtPerBlockHead(i *instruction) { |
| 259 |
if m.perBlockHead == nil { |
| 260 |
m.perBlockHead = i |
| 261 |
m.perBlockEnd = i |
| 262 |
return |
| 263 |
} |
| 264 |
|
| 265 |
i.next = m.perBlockHead |
| 266 |
m.perBlockHead.prev = i |
| 267 |
m.perBlockHead = i |
| 268 |
} |
| 269 |
|
| 270 |
// FlushPendingInstructions implements backend.Machine. |
| 271 |
func (m *machine) FlushPendingInstructions() { |
| 272 |
l := len(m.pendingInstructions) |
| 273 |
if l == 0 { |
| 274 |
return |
| 275 |
} |
| 276 |
for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order. |
| 277 |
m.insertAtPerBlockHead(m.pendingInstructions[i]) |
| 278 |
} |
| 279 |
m.pendingInstructions = m.pendingInstructions[:0] |
| 280 |
} |
| 281 |
|
| 282 |
// DisableStackCheck implements backend.Machine. |
| 283 |
func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true } |
| 284 |
|
| 285 |
// SetCompiler implements backend.Machine. |
| 286 |
func (m *machine) SetCompiler(c backend.Compiler) { |
| 287 |
m.c = c |
| 288 |
m.regAllocFn.ssaB = c.SSABuilder() |
| 289 |
} |
| 290 |
|
| 291 |
// SetCurrentABI implements backend.Machine. |
| 292 |
func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { m.currentABI = abi } |
| 293 |
|
| 294 |
// RegAlloc implements backend.Machine. |
| 295 |
func (m *machine) RegAlloc() { |
| 296 |
rf := m.regAllocFn |
| 297 |
m.regAllocStarted = true |
| 298 |
m.regAlloc.DoAllocation(&rf) |
| 299 |
// Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes. |
| 300 |
m.spillSlotSize = (m.spillSlotSize + 15) &^ 15 |
| 301 |
} |
| 302 |
|
| 303 |
// InsertReturn implements backend.Machine. |
| 304 |
func (m *machine) InsertReturn() { |
| 305 |
i := m.allocateInstr().asRet() |
| 306 |
m.insert(i) |
| 307 |
} |
| 308 |
|
| 309 |
// LowerSingleBranch implements backend.Machine. |
| 310 |
func (m *machine) LowerSingleBranch(b *ssa.Instruction) { |
| 311 |
switch b.Opcode() { |
| 312 |
case ssa.OpcodeJump: |
| 313 |
_, _, targetBlkID := b.BranchData() |
| 314 |
if b.IsFallthroughJump() { |
| 315 |
return |
| 316 |
} |
| 317 |
jmp := m.allocateInstr() |
| 318 |
target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID)) |
| 319 |
if target == labelReturn { |
| 320 |
jmp.asRet() |
| 321 |
} else { |
| 322 |
jmp.asJmp(newOperandLabel(target)) |
| 323 |
} |
| 324 |
m.insert(jmp) |
| 325 |
case ssa.OpcodeBrTable: |
| 326 |
index, targetBlkIDs := b.BrTableData() |
| 327 |
m.lowerBrTable(index, targetBlkIDs) |
| 328 |
default: |
| 329 |
panic("BUG: unexpected branch opcode" + b.Opcode().String()) |
| 330 |
} |
| 331 |
} |
| 332 |
|
| 333 |
func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) { |
| 334 |
if m.jmpTableTargetsNext == len(m.jmpTableTargets) { |
| 335 |
m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View()))) |
| 336 |
} |
| 337 |
|
| 338 |
index = m.jmpTableTargetsNext |
| 339 |
m.jmpTableTargetsNext++ |
| 340 |
m.jmpTableTargets[index] = m.jmpTableTargets[index][:0] |
| 341 |
for _, targetBlockID := range targets.View() { |
| 342 |
target := m.c.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID)) |
| 343 |
m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(ssaBlockLabel(target))) |
| 344 |
} |
| 345 |
return |
| 346 |
} |
| 347 |
|
| 348 |
var condBranchMatches = [...]ssa.Opcode{ssa.OpcodeIcmp, ssa.OpcodeFcmp} |
| 349 |
|
| 350 |
func (m *machine) lowerBrTable(index ssa.Value, targets ssa.Values) { |
| 351 |
_v := m.getOperand_Reg(m.c.ValueDefinition(index)) |
| 352 |
v := m.copyToTmp(_v.reg()) |
| 353 |
|
| 354 |
targetCount := len(targets.View()) |
| 355 |
|
| 356 |
// First, we need to do the bounds check. |
| 357 |
maxIndex := m.c.AllocateVReg(ssa.TypeI32) |
| 358 |
m.lowerIconst(maxIndex, uint64(targetCount-1), false) |
| 359 |
cmp := m.allocateInstr().asCmpRmiR(true, newOperandReg(maxIndex), v, false) |
| 360 |
m.insert(cmp) |
| 361 |
|
| 362 |
// Then do the conditional move maxIndex to v if v > maxIndex. |
| 363 |
cmov := m.allocateInstr().asCmove(condNB, newOperandReg(maxIndex), v, false) |
| 364 |
m.insert(cmov) |
| 365 |
|
| 366 |
// Now that v has the correct index. Load the address of the jump table into the addr. |
| 367 |
addr := m.c.AllocateVReg(ssa.TypeI64) |
| 368 |
leaJmpTableAddr := m.allocateInstr() |
| 369 |
m.insert(leaJmpTableAddr) |
| 370 |
|
| 371 |
// Then add the target's offset into jmpTableAddr. |
| 372 |
loadTargetOffsetFromJmpTable := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, |
| 373 |
// Shift by 3 because each entry is 8 bytes. |
| 374 |
newOperandMem(m.newAmodeRegRegShift(0, addr, v, 3)), addr, true) |
| 375 |
m.insert(loadTargetOffsetFromJmpTable) |
| 376 |
|
| 377 |
// Now ready to jump. |
| 378 |
jmp := m.allocateInstr().asJmp(newOperandReg(addr)) |
| 379 |
m.insert(jmp) |
| 380 |
|
| 381 |
jmpTableBegin, jmpTableBeginLabel := m.allocateBrTarget() |
| 382 |
m.insert(jmpTableBegin) |
| 383 |
leaJmpTableAddr.asLEA(newOperandLabel(jmpTableBeginLabel), addr) |
| 384 |
|
| 385 |
jmpTable := m.allocateInstr() |
| 386 |
targetSliceIndex := m.addJmpTableTarget(targets) |
| 387 |
jmpTable.asJmpTableSequence(targetSliceIndex, targetCount) |
| 388 |
m.insert(jmpTable) |
| 389 |
} |
| 390 |
|
| 391 |
// LowerConditionalBranch implements backend.Machine. |
| 392 |
func (m *machine) LowerConditionalBranch(b *ssa.Instruction) { |
| 393 |
cval, args, targetBlkID := b.BranchData() |
| 394 |
if len(args) > 0 { |
| 395 |
panic(fmt.Sprintf( |
| 396 |
"conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s", |
| 397 |
m.currentLabelPos.sb, |
| 398 |
targetBlkID, |
| 399 |
)) |
| 400 |
} |
| 401 |
|
| 402 |
target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID)) |
| 403 |
cvalDef := m.c.ValueDefinition(cval) |
| 404 |
|
| 405 |
switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) { |
| 406 |
case ssa.OpcodeIcmp: |
| 407 |
cvalInstr := cvalDef.Instr |
| 408 |
x, y, c := cvalInstr.IcmpData() |
| 409 |
|
| 410 |
cc := condFromSSAIntCmpCond(c) |
| 411 |
if b.Opcode() == ssa.OpcodeBrz { |
| 412 |
cc = cc.invert() |
| 413 |
} |
| 414 |
|
| 415 |
// First, perform the comparison and set the flag. |
| 416 |
xd, yd := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 417 |
if !m.tryLowerBandToFlag(xd, yd) { |
| 418 |
m.lowerIcmpToFlag(xd, yd, x.Type() == ssa.TypeI64) |
| 419 |
} |
| 420 |
|
| 421 |
// Then perform the conditional branch. |
| 422 |
m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target))) |
| 423 |
cvalDef.Instr.MarkLowered() |
| 424 |
case ssa.OpcodeFcmp: |
| 425 |
cvalInstr := cvalDef.Instr |
| 426 |
|
| 427 |
f1, f2, and := m.lowerFcmpToFlags(cvalInstr) |
| 428 |
isBrz := b.Opcode() == ssa.OpcodeBrz |
| 429 |
if isBrz { |
| 430 |
f1 = f1.invert() |
| 431 |
} |
| 432 |
if f2 == condInvalid { |
| 433 |
m.insert(m.allocateInstr().asJmpIf(f1, newOperandLabel(target))) |
| 434 |
} else { |
| 435 |
if isBrz { |
| 436 |
f2 = f2.invert() |
| 437 |
and = !and |
| 438 |
} |
| 439 |
jmp1, jmp2 := m.allocateInstr(), m.allocateInstr() |
| 440 |
m.insert(jmp1) |
| 441 |
m.insert(jmp2) |
| 442 |
notTaken, notTakenLabel := m.allocateBrTarget() |
| 443 |
m.insert(notTaken) |
| 444 |
if and { |
| 445 |
jmp1.asJmpIf(f1.invert(), newOperandLabel(notTakenLabel)) |
| 446 |
jmp2.asJmpIf(f2, newOperandLabel(target)) |
| 447 |
} else { |
| 448 |
jmp1.asJmpIf(f1, newOperandLabel(target)) |
| 449 |
jmp2.asJmpIf(f2, newOperandLabel(target)) |
| 450 |
} |
| 451 |
} |
| 452 |
|
| 453 |
cvalDef.Instr.MarkLowered() |
| 454 |
default: |
| 455 |
v := m.getOperand_Reg(cvalDef) |
| 456 |
|
| 457 |
var cc cond |
| 458 |
if b.Opcode() == ssa.OpcodeBrz { |
| 459 |
cc = condZ |
| 460 |
} else { |
| 461 |
cc = condNZ |
| 462 |
} |
| 463 |
|
| 464 |
// Perform test %v, %v to set the flag. |
| 465 |
cmp := m.allocateInstr().asCmpRmiR(false, v, v.reg(), false) |
| 466 |
m.insert(cmp) |
| 467 |
m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target))) |
| 468 |
} |
| 469 |
} |
| 470 |
|
| 471 |
// LowerInstr implements backend.Machine. |
| 472 |
func (m *machine) LowerInstr(instr *ssa.Instruction) { |
| 473 |
if l := instr.SourceOffset(); l.Valid() { |
| 474 |
info := m.allocateInstr().asEmitSourceOffsetInfo(l) |
| 475 |
m.insert(info) |
| 476 |
} |
| 477 |
|
| 478 |
switch op := instr.Opcode(); op { |
| 479 |
case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable: |
| 480 |
panic("BUG: branching instructions are handled by LowerBranches") |
| 481 |
case ssa.OpcodeReturn: |
| 482 |
panic("BUG: return must be handled by backend.Compiler") |
| 483 |
case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined. |
| 484 |
case ssa.OpcodeCall, ssa.OpcodeCallIndirect: |
| 485 |
m.lowerCall(instr) |
| 486 |
case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32: |
| 487 |
m.lowerStore(instr) |
| 488 |
case ssa.OpcodeIadd: |
| 489 |
m.lowerAluRmiROp(instr, aluRmiROpcodeAdd) |
| 490 |
case ssa.OpcodeIsub: |
| 491 |
m.lowerAluRmiROp(instr, aluRmiROpcodeSub) |
| 492 |
case ssa.OpcodeImul: |
| 493 |
m.lowerAluRmiROp(instr, aluRmiROpcodeMul) |
| 494 |
case ssa.OpcodeSdiv, ssa.OpcodeUdiv, ssa.OpcodeSrem, ssa.OpcodeUrem: |
| 495 |
isDiv := op == ssa.OpcodeSdiv || op == ssa.OpcodeUdiv |
| 496 |
isSigned := op == ssa.OpcodeSdiv || op == ssa.OpcodeSrem |
| 497 |
m.lowerIDivRem(instr, isDiv, isSigned) |
| 498 |
case ssa.OpcodeBand: |
| 499 |
m.lowerAluRmiROp(instr, aluRmiROpcodeAnd) |
| 500 |
case ssa.OpcodeBor: |
| 501 |
m.lowerAluRmiROp(instr, aluRmiROpcodeOr) |
| 502 |
case ssa.OpcodeBxor: |
| 503 |
m.lowerAluRmiROp(instr, aluRmiROpcodeXor) |
| 504 |
case ssa.OpcodeIshl: |
| 505 |
m.lowerShiftR(instr, shiftROpShiftLeft) |
| 506 |
case ssa.OpcodeSshr: |
| 507 |
m.lowerShiftR(instr, shiftROpShiftRightArithmetic) |
| 508 |
case ssa.OpcodeUshr: |
| 509 |
m.lowerShiftR(instr, shiftROpShiftRightLogical) |
| 510 |
case ssa.OpcodeRotl: |
| 511 |
m.lowerShiftR(instr, shiftROpRotateLeft) |
| 512 |
case ssa.OpcodeRotr: |
| 513 |
m.lowerShiftR(instr, shiftROpRotateRight) |
| 514 |
case ssa.OpcodeClz: |
| 515 |
m.lowerClz(instr) |
| 516 |
case ssa.OpcodeCtz: |
| 517 |
m.lowerCtz(instr) |
| 518 |
case ssa.OpcodePopcnt: |
| 519 |
m.lowerUnaryRmR(instr, unaryRmROpcodePopcnt) |
| 520 |
case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv: |
| 521 |
m.lowerXmmRmR(instr) |
| 522 |
case ssa.OpcodeFabs: |
| 523 |
m.lowerFabsFneg(instr) |
| 524 |
case ssa.OpcodeFneg: |
| 525 |
m.lowerFabsFneg(instr) |
| 526 |
case ssa.OpcodeCeil: |
| 527 |
m.lowerRound(instr, roundingModeUp) |
| 528 |
case ssa.OpcodeFloor: |
| 529 |
m.lowerRound(instr, roundingModeDown) |
| 530 |
case ssa.OpcodeTrunc: |
| 531 |
m.lowerRound(instr, roundingModeZero) |
| 532 |
case ssa.OpcodeNearest: |
| 533 |
m.lowerRound(instr, roundingModeNearest) |
| 534 |
case ssa.OpcodeFmin, ssa.OpcodeFmax: |
| 535 |
m.lowerFminFmax(instr) |
| 536 |
case ssa.OpcodeFcopysign: |
| 537 |
m.lowerFcopysign(instr) |
| 538 |
case ssa.OpcodeBitcast: |
| 539 |
m.lowerBitcast(instr) |
| 540 |
case ssa.OpcodeSqrt: |
| 541 |
m.lowerSqrt(instr) |
| 542 |
case ssa.OpcodeFpromote: |
| 543 |
v := instr.Arg() |
| 544 |
rn := m.getOperand_Reg(m.c.ValueDefinition(v)) |
| 545 |
rd := m.c.VRegOf(instr.Return()) |
| 546 |
cnt := m.allocateInstr() |
| 547 |
cnt.asXmmUnaryRmR(sseOpcodeCvtss2sd, rn, rd) |
| 548 |
m.insert(cnt) |
| 549 |
case ssa.OpcodeFdemote: |
| 550 |
v := instr.Arg() |
| 551 |
rn := m.getOperand_Reg(m.c.ValueDefinition(v)) |
| 552 |
rd := m.c.VRegOf(instr.Return()) |
| 553 |
cnt := m.allocateInstr() |
| 554 |
cnt.asXmmUnaryRmR(sseOpcodeCvtsd2ss, rn, rd) |
| 555 |
m.insert(cnt) |
| 556 |
case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat: |
| 557 |
x, ctx := instr.Arg2() |
| 558 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 559 |
rd := m.c.VRegOf(instr.Return()) |
| 560 |
ctxVReg := m.c.VRegOf(ctx) |
| 561 |
m.lowerFcvtToSint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64, |
| 562 |
instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat) |
| 563 |
case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat: |
| 564 |
x, ctx := instr.Arg2() |
| 565 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 566 |
rd := m.c.VRegOf(instr.Return()) |
| 567 |
ctxVReg := m.c.VRegOf(ctx) |
| 568 |
m.lowerFcvtToUint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64, |
| 569 |
instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat) |
| 570 |
case ssa.OpcodeFcvtFromSint: |
| 571 |
x := instr.Arg() |
| 572 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 573 |
rd := newOperandReg(m.c.VRegOf(instr.Return())) |
| 574 |
m.lowerFcvtFromSint(rn, rd, |
| 575 |
x.Type() == ssa.TypeI64, instr.Return().Type().Bits() == 64) |
| 576 |
case ssa.OpcodeFcvtFromUint: |
| 577 |
x := instr.Arg() |
| 578 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 579 |
rd := newOperandReg(m.c.VRegOf(instr.Return())) |
| 580 |
m.lowerFcvtFromUint(rn, rd, x.Type() == ssa.TypeI64, |
| 581 |
instr.Return().Type().Bits() == 64) |
| 582 |
case ssa.OpcodeVanyTrue: |
| 583 |
m.lowerVanyTrue(instr) |
| 584 |
case ssa.OpcodeVallTrue: |
| 585 |
m.lowerVallTrue(instr) |
| 586 |
case ssa.OpcodeVhighBits: |
| 587 |
m.lowerVhighBits(instr) |
| 588 |
case ssa.OpcodeVbnot: |
| 589 |
m.lowerVbnot(instr) |
| 590 |
case ssa.OpcodeVband: |
| 591 |
x, y := instr.Arg2() |
| 592 |
m.lowerVbBinOp(sseOpcodePand, x, y, instr.Return()) |
| 593 |
case ssa.OpcodeVbor: |
| 594 |
x, y := instr.Arg2() |
| 595 |
m.lowerVbBinOp(sseOpcodePor, x, y, instr.Return()) |
| 596 |
case ssa.OpcodeVbxor: |
| 597 |
x, y := instr.Arg2() |
| 598 |
m.lowerVbBinOp(sseOpcodePxor, x, y, instr.Return()) |
| 599 |
case ssa.OpcodeVbandnot: |
| 600 |
m.lowerVbandnot(instr, sseOpcodePandn) |
| 601 |
case ssa.OpcodeVbitselect: |
| 602 |
m.lowerVbitselect(instr) |
| 603 |
case ssa.OpcodeVIadd: |
| 604 |
x, y, lane := instr.Arg2WithLane() |
| 605 |
var vecOp sseOpcode |
| 606 |
switch lane { |
| 607 |
case ssa.VecLaneI8x16: |
| 608 |
vecOp = sseOpcodePaddb |
| 609 |
case ssa.VecLaneI16x8: |
| 610 |
vecOp = sseOpcodePaddw |
| 611 |
case ssa.VecLaneI32x4: |
| 612 |
vecOp = sseOpcodePaddd |
| 613 |
case ssa.VecLaneI64x2: |
| 614 |
vecOp = sseOpcodePaddq |
| 615 |
} |
| 616 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 617 |
|
| 618 |
case ssa.OpcodeVSaddSat: |
| 619 |
x, y, lane := instr.Arg2WithLane() |
| 620 |
var vecOp sseOpcode |
| 621 |
switch lane { |
| 622 |
case ssa.VecLaneI8x16: |
| 623 |
vecOp = sseOpcodePaddsb |
| 624 |
case ssa.VecLaneI16x8: |
| 625 |
vecOp = sseOpcodePaddsw |
| 626 |
} |
| 627 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 628 |
|
| 629 |
case ssa.OpcodeVUaddSat: |
| 630 |
x, y, lane := instr.Arg2WithLane() |
| 631 |
var vecOp sseOpcode |
| 632 |
switch lane { |
| 633 |
case ssa.VecLaneI8x16: |
| 634 |
vecOp = sseOpcodePaddusb |
| 635 |
case ssa.VecLaneI16x8: |
| 636 |
vecOp = sseOpcodePaddusw |
| 637 |
} |
| 638 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 639 |
|
| 640 |
case ssa.OpcodeVIsub: |
| 641 |
x, y, lane := instr.Arg2WithLane() |
| 642 |
var vecOp sseOpcode |
| 643 |
switch lane { |
| 644 |
case ssa.VecLaneI8x16: |
| 645 |
vecOp = sseOpcodePsubb |
| 646 |
case ssa.VecLaneI16x8: |
| 647 |
vecOp = sseOpcodePsubw |
| 648 |
case ssa.VecLaneI32x4: |
| 649 |
vecOp = sseOpcodePsubd |
| 650 |
case ssa.VecLaneI64x2: |
| 651 |
vecOp = sseOpcodePsubq |
| 652 |
} |
| 653 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 654 |
|
| 655 |
case ssa.OpcodeVSsubSat: |
| 656 |
x, y, lane := instr.Arg2WithLane() |
| 657 |
var vecOp sseOpcode |
| 658 |
switch lane { |
| 659 |
case ssa.VecLaneI8x16: |
| 660 |
vecOp = sseOpcodePsubsb |
| 661 |
case ssa.VecLaneI16x8: |
| 662 |
vecOp = sseOpcodePsubsw |
| 663 |
} |
| 664 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 665 |
|
| 666 |
case ssa.OpcodeVUsubSat: |
| 667 |
x, y, lane := instr.Arg2WithLane() |
| 668 |
var vecOp sseOpcode |
| 669 |
switch lane { |
| 670 |
case ssa.VecLaneI8x16: |
| 671 |
vecOp = sseOpcodePsubusb |
| 672 |
case ssa.VecLaneI16x8: |
| 673 |
vecOp = sseOpcodePsubusw |
| 674 |
} |
| 675 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 676 |
|
| 677 |
case ssa.OpcodeVImul: |
| 678 |
m.lowerVImul(instr) |
| 679 |
case ssa.OpcodeVIneg: |
| 680 |
x, lane := instr.ArgWithLane() |
| 681 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 682 |
rd := m.c.VRegOf(instr.Return()) |
| 683 |
var vecOp sseOpcode |
| 684 |
switch lane { |
| 685 |
case ssa.VecLaneI8x16: |
| 686 |
vecOp = sseOpcodePsubb |
| 687 |
case ssa.VecLaneI16x8: |
| 688 |
vecOp = sseOpcodePsubw |
| 689 |
case ssa.VecLaneI32x4: |
| 690 |
vecOp = sseOpcodePsubd |
| 691 |
case ssa.VecLaneI64x2: |
| 692 |
vecOp = sseOpcodePsubq |
| 693 |
default: |
| 694 |
panic("BUG") |
| 695 |
} |
| 696 |
|
| 697 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 698 |
m.insert(m.allocateInstr().asZeros(tmp)) |
| 699 |
|
| 700 |
i := m.allocateInstr() |
| 701 |
i.asXmmRmR(vecOp, rn, tmp) |
| 702 |
m.insert(i) |
| 703 |
|
| 704 |
m.copyTo(tmp, rd) |
| 705 |
case ssa.OpcodeVFadd: |
| 706 |
x, y, lane := instr.Arg2WithLane() |
| 707 |
var vecOp sseOpcode |
| 708 |
switch lane { |
| 709 |
case ssa.VecLaneF32x4: |
| 710 |
vecOp = sseOpcodeAddps |
| 711 |
case ssa.VecLaneF64x2: |
| 712 |
vecOp = sseOpcodeAddpd |
| 713 |
} |
| 714 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 715 |
|
| 716 |
case ssa.OpcodeVFsub: |
| 717 |
x, y, lane := instr.Arg2WithLane() |
| 718 |
var vecOp sseOpcode |
| 719 |
switch lane { |
| 720 |
case ssa.VecLaneF32x4: |
| 721 |
vecOp = sseOpcodeSubps |
| 722 |
case ssa.VecLaneF64x2: |
| 723 |
vecOp = sseOpcodeSubpd |
| 724 |
} |
| 725 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 726 |
|
| 727 |
case ssa.OpcodeVFdiv: |
| 728 |
x, y, lane := instr.Arg2WithLane() |
| 729 |
var vecOp sseOpcode |
| 730 |
switch lane { |
| 731 |
case ssa.VecLaneF32x4: |
| 732 |
vecOp = sseOpcodeDivps |
| 733 |
case ssa.VecLaneF64x2: |
| 734 |
vecOp = sseOpcodeDivpd |
| 735 |
} |
| 736 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 737 |
|
| 738 |
case ssa.OpcodeVFmul: |
| 739 |
x, y, lane := instr.Arg2WithLane() |
| 740 |
var vecOp sseOpcode |
| 741 |
switch lane { |
| 742 |
case ssa.VecLaneF32x4: |
| 743 |
vecOp = sseOpcodeMulps |
| 744 |
case ssa.VecLaneF64x2: |
| 745 |
vecOp = sseOpcodeMulpd |
| 746 |
} |
| 747 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 748 |
|
| 749 |
case ssa.OpcodeVFneg: |
| 750 |
x, lane := instr.ArgWithLane() |
| 751 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 752 |
rd := m.c.VRegOf(instr.Return()) |
| 753 |
|
| 754 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 755 |
|
| 756 |
var shiftOp, xorOp sseOpcode |
| 757 |
var shiftAmt uint32 |
| 758 |
switch lane { |
| 759 |
case ssa.VecLaneF32x4: |
| 760 |
shiftOp, shiftAmt, xorOp = sseOpcodePslld, 31, sseOpcodeXorps |
| 761 |
case ssa.VecLaneF64x2: |
| 762 |
shiftOp, shiftAmt, xorOp = sseOpcodePsllq, 63, sseOpcodeXorpd |
| 763 |
} |
| 764 |
|
| 765 |
zero := m.allocateInstr() |
| 766 |
zero.asZeros(tmp) |
| 767 |
m.insert(zero) |
| 768 |
|
| 769 |
// Set all bits on tmp by CMPPD with arg=0 (== pseudo CMPEQPD instruction). |
| 770 |
// See https://www.felixcloutier.com/x86/cmpps |
| 771 |
// |
| 772 |
// Note: if we do not clear all the bits ^ with XORPS, this might end up not setting ones on some lane |
| 773 |
// if the lane is NaN. |
| 774 |
cmp := m.allocateInstr() |
| 775 |
cmp.asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_UQ), newOperandReg(tmp), tmp) |
| 776 |
m.insert(cmp) |
| 777 |
|
| 778 |
// Do the left shift on each lane to set only the most significant bit in each. |
| 779 |
i := m.allocateInstr() |
| 780 |
i.asXmmRmiReg(shiftOp, newOperandImm32(shiftAmt), tmp) |
| 781 |
m.insert(i) |
| 782 |
|
| 783 |
// Get the negated result by XOR on each lane with tmp. |
| 784 |
i = m.allocateInstr() |
| 785 |
i.asXmmRmR(xorOp, rn, tmp) |
| 786 |
m.insert(i) |
| 787 |
|
| 788 |
m.copyTo(tmp, rd) |
| 789 |
|
| 790 |
case ssa.OpcodeVSqrt: |
| 791 |
x, lane := instr.ArgWithLane() |
| 792 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 793 |
rd := m.c.VRegOf(instr.Return()) |
| 794 |
|
| 795 |
var vecOp sseOpcode |
| 796 |
switch lane { |
| 797 |
case ssa.VecLaneF32x4: |
| 798 |
vecOp = sseOpcodeSqrtps |
| 799 |
case ssa.VecLaneF64x2: |
| 800 |
vecOp = sseOpcodeSqrtpd |
| 801 |
} |
| 802 |
i := m.allocateInstr() |
| 803 |
i.asXmmUnaryRmR(vecOp, rn, rd) |
| 804 |
m.insert(i) |
| 805 |
|
| 806 |
case ssa.OpcodeVImin: |
| 807 |
x, y, lane := instr.Arg2WithLane() |
| 808 |
var vecOp sseOpcode |
| 809 |
switch lane { |
| 810 |
case ssa.VecLaneI8x16: |
| 811 |
vecOp = sseOpcodePminsb |
| 812 |
case ssa.VecLaneI16x8: |
| 813 |
vecOp = sseOpcodePminsw |
| 814 |
case ssa.VecLaneI32x4: |
| 815 |
vecOp = sseOpcodePminsd |
| 816 |
} |
| 817 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 818 |
|
| 819 |
case ssa.OpcodeVUmin: |
| 820 |
x, y, lane := instr.Arg2WithLane() |
| 821 |
var vecOp sseOpcode |
| 822 |
switch lane { |
| 823 |
case ssa.VecLaneI8x16: |
| 824 |
vecOp = sseOpcodePminub |
| 825 |
case ssa.VecLaneI16x8: |
| 826 |
vecOp = sseOpcodePminuw |
| 827 |
case ssa.VecLaneI32x4: |
| 828 |
vecOp = sseOpcodePminud |
| 829 |
} |
| 830 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 831 |
|
| 832 |
case ssa.OpcodeVImax: |
| 833 |
x, y, lane := instr.Arg2WithLane() |
| 834 |
var vecOp sseOpcode |
| 835 |
switch lane { |
| 836 |
case ssa.VecLaneI8x16: |
| 837 |
vecOp = sseOpcodePmaxsb |
| 838 |
case ssa.VecLaneI16x8: |
| 839 |
vecOp = sseOpcodePmaxsw |
| 840 |
case ssa.VecLaneI32x4: |
| 841 |
vecOp = sseOpcodePmaxsd |
| 842 |
} |
| 843 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 844 |
|
| 845 |
case ssa.OpcodeVUmax: |
| 846 |
x, y, lane := instr.Arg2WithLane() |
| 847 |
var vecOp sseOpcode |
| 848 |
switch lane { |
| 849 |
case ssa.VecLaneI8x16: |
| 850 |
vecOp = sseOpcodePmaxub |
| 851 |
case ssa.VecLaneI16x8: |
| 852 |
vecOp = sseOpcodePmaxuw |
| 853 |
case ssa.VecLaneI32x4: |
| 854 |
vecOp = sseOpcodePmaxud |
| 855 |
} |
| 856 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 857 |
|
| 858 |
case ssa.OpcodeVAvgRound: |
| 859 |
x, y, lane := instr.Arg2WithLane() |
| 860 |
var vecOp sseOpcode |
| 861 |
switch lane { |
| 862 |
case ssa.VecLaneI8x16: |
| 863 |
vecOp = sseOpcodePavgb |
| 864 |
case ssa.VecLaneI16x8: |
| 865 |
vecOp = sseOpcodePavgw |
| 866 |
} |
| 867 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 868 |
|
| 869 |
case ssa.OpcodeVIcmp: |
| 870 |
x, y, c, lane := instr.VIcmpData() |
| 871 |
m.lowerVIcmp(x, y, c, instr.Return(), lane) |
| 872 |
|
| 873 |
case ssa.OpcodeVFcmp: |
| 874 |
x, y, c, lane := instr.VFcmpData() |
| 875 |
m.lowerVFcmp(x, y, c, instr.Return(), lane) |
| 876 |
|
| 877 |
case ssa.OpcodeExtractlane: |
| 878 |
x, index, signed, lane := instr.ExtractlaneData() |
| 879 |
m.lowerExtractLane(x, index, signed, instr.Return(), lane) |
| 880 |
|
| 881 |
case ssa.OpcodeInsertlane: |
| 882 |
x, y, index, lane := instr.InsertlaneData() |
| 883 |
m.lowerInsertLane(x, y, index, instr.Return(), lane) |
| 884 |
|
| 885 |
case ssa.OpcodeSwizzle: |
| 886 |
x, y, _ := instr.Arg2WithLane() |
| 887 |
m.lowerSwizzle(x, y, instr.Return()) |
| 888 |
|
| 889 |
case ssa.OpcodeShuffle: |
| 890 |
x, y, lo, hi := instr.ShuffleData() |
| 891 |
m.lowerShuffle(x, y, lo, hi, instr.Return()) |
| 892 |
|
| 893 |
case ssa.OpcodeSplat: |
| 894 |
x, lane := instr.ArgWithLane() |
| 895 |
m.lowerSplat(x, instr.Return(), lane) |
| 896 |
|
| 897 |
case ssa.OpcodeSqmulRoundSat: |
| 898 |
x, y := instr.Arg2() |
| 899 |
m.lowerSqmulRoundSat(x, y, instr.Return()) |
| 900 |
|
| 901 |
case ssa.OpcodeVZeroExtLoad: |
| 902 |
ptr, offset, typ := instr.VZeroExtLoadData() |
| 903 |
var sseOp sseOpcode |
| 904 |
// Both movss and movsd clears the higher bits of the destination register upt 128 bits. |
| 905 |
// https://www.felixcloutier.com/x86/movss |
| 906 |
// https://www.felixcloutier.com/x86/movsd |
| 907 |
if typ == ssa.TypeF32 { |
| 908 |
sseOp = sseOpcodeMovss |
| 909 |
} else { |
| 910 |
sseOp = sseOpcodeMovsd |
| 911 |
} |
| 912 |
mem := m.lowerToAddressMode(ptr, offset) |
| 913 |
dst := m.c.VRegOf(instr.Return()) |
| 914 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandMem(mem), dst)) |
| 915 |
|
| 916 |
case ssa.OpcodeVMinPseudo: |
| 917 |
x, y, lane := instr.Arg2WithLane() |
| 918 |
var vecOp sseOpcode |
| 919 |
switch lane { |
| 920 |
case ssa.VecLaneF32x4: |
| 921 |
vecOp = sseOpcodeMinps |
| 922 |
case ssa.VecLaneF64x2: |
| 923 |
vecOp = sseOpcodeMinpd |
| 924 |
default: |
| 925 |
panic("BUG: unexpected lane type") |
| 926 |
} |
| 927 |
m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return()) |
| 928 |
|
| 929 |
case ssa.OpcodeVMaxPseudo: |
| 930 |
x, y, lane := instr.Arg2WithLane() |
| 931 |
var vecOp sseOpcode |
| 932 |
switch lane { |
| 933 |
case ssa.VecLaneF32x4: |
| 934 |
vecOp = sseOpcodeMaxps |
| 935 |
case ssa.VecLaneF64x2: |
| 936 |
vecOp = sseOpcodeMaxpd |
| 937 |
default: |
| 938 |
panic("BUG: unexpected lane type") |
| 939 |
} |
| 940 |
m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return()) |
| 941 |
|
| 942 |
case ssa.OpcodeVIshl: |
| 943 |
x, y, lane := instr.Arg2WithLane() |
| 944 |
m.lowerVIshl(x, y, instr.Return(), lane) |
| 945 |
|
| 946 |
case ssa.OpcodeVSshr: |
| 947 |
x, y, lane := instr.Arg2WithLane() |
| 948 |
m.lowerVSshr(x, y, instr.Return(), lane) |
| 949 |
|
| 950 |
case ssa.OpcodeVUshr: |
| 951 |
x, y, lane := instr.Arg2WithLane() |
| 952 |
m.lowerVUshr(x, y, instr.Return(), lane) |
| 953 |
|
| 954 |
case ssa.OpcodeVCeil: |
| 955 |
x, lane := instr.ArgWithLane() |
| 956 |
m.lowerVRound(x, instr.Return(), 0x2, lane == ssa.VecLaneF64x2) |
| 957 |
|
| 958 |
case ssa.OpcodeVFloor: |
| 959 |
x, lane := instr.ArgWithLane() |
| 960 |
m.lowerVRound(x, instr.Return(), 0x1, lane == ssa.VecLaneF64x2) |
| 961 |
|
| 962 |
case ssa.OpcodeVTrunc: |
| 963 |
x, lane := instr.ArgWithLane() |
| 964 |
m.lowerVRound(x, instr.Return(), 0x3, lane == ssa.VecLaneF64x2) |
| 965 |
|
| 966 |
case ssa.OpcodeVNearest: |
| 967 |
x, lane := instr.ArgWithLane() |
| 968 |
m.lowerVRound(x, instr.Return(), 0x0, lane == ssa.VecLaneF64x2) |
| 969 |
|
| 970 |
case ssa.OpcodeExtIaddPairwise: |
| 971 |
x, lane, signed := instr.ExtIaddPairwiseData() |
| 972 |
m.lowerExtIaddPairwise(x, instr.Return(), lane, signed) |
| 973 |
|
| 974 |
case ssa.OpcodeUwidenLow, ssa.OpcodeSwidenLow: |
| 975 |
x, lane := instr.ArgWithLane() |
| 976 |
m.lowerWidenLow(x, instr.Return(), lane, op == ssa.OpcodeSwidenLow) |
| 977 |
|
| 978 |
case ssa.OpcodeUwidenHigh, ssa.OpcodeSwidenHigh: |
| 979 |
x, lane := instr.ArgWithLane() |
| 980 |
m.lowerWidenHigh(x, instr.Return(), lane, op == ssa.OpcodeSwidenHigh) |
| 981 |
|
| 982 |
case ssa.OpcodeLoadSplat: |
| 983 |
ptr, offset, lane := instr.LoadSplatData() |
| 984 |
m.lowerLoadSplat(ptr, offset, instr.Return(), lane) |
| 985 |
|
| 986 |
case ssa.OpcodeVFcvtFromUint, ssa.OpcodeVFcvtFromSint: |
| 987 |
x, lane := instr.ArgWithLane() |
| 988 |
m.lowerVFcvtFromInt(x, instr.Return(), lane, op == ssa.OpcodeVFcvtFromSint) |
| 989 |
|
| 990 |
case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat: |
| 991 |
x, lane := instr.ArgWithLane() |
| 992 |
m.lowerVFcvtToIntSat(x, instr.Return(), lane, op == ssa.OpcodeVFcvtToSintSat) |
| 993 |
|
| 994 |
case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow: |
| 995 |
x, y, lane := instr.Arg2WithLane() |
| 996 |
m.lowerNarrow(x, y, instr.Return(), lane, op == ssa.OpcodeSnarrow) |
| 997 |
|
| 998 |
case ssa.OpcodeFvpromoteLow: |
| 999 |
x := instr.Arg() |
| 1000 |
src := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 1001 |
dst := m.c.VRegOf(instr.Return()) |
| 1002 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtps2pd, src, dst)) |
| 1003 |
|
| 1004 |
case ssa.OpcodeFvdemote: |
| 1005 |
x := instr.Arg() |
| 1006 |
src := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 1007 |
dst := m.c.VRegOf(instr.Return()) |
| 1008 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtpd2ps, src, dst)) |
| 1009 |
|
| 1010 |
case ssa.OpcodeWideningPairwiseDotProductS: |
| 1011 |
x, y := instr.Arg2() |
| 1012 |
m.lowerWideningPairwiseDotProductS(x, y, instr.Return()) |
| 1013 |
|
| 1014 |
case ssa.OpcodeVIabs: |
| 1015 |
m.lowerVIabs(instr) |
| 1016 |
case ssa.OpcodeVIpopcnt: |
| 1017 |
m.lowerVIpopcnt(instr) |
| 1018 |
case ssa.OpcodeVFmin: |
| 1019 |
m.lowerVFmin(instr) |
| 1020 |
case ssa.OpcodeVFmax: |
| 1021 |
m.lowerVFmax(instr) |
| 1022 |
case ssa.OpcodeVFabs: |
| 1023 |
m.lowerVFabs(instr) |
| 1024 |
case ssa.OpcodeUndefined: |
| 1025 |
m.insert(m.allocateInstr().asUD2()) |
| 1026 |
case ssa.OpcodeExitWithCode: |
| 1027 |
execCtx, code := instr.ExitWithCodeData() |
| 1028 |
m.lowerExitWithCode(m.c.VRegOf(execCtx), code) |
| 1029 |
case ssa.OpcodeExitIfTrueWithCode: |
| 1030 |
execCtx, c, code := instr.ExitIfTrueWithCodeData() |
| 1031 |
m.lowerExitIfTrueWithCode(m.c.VRegOf(execCtx), c, code) |
| 1032 |
case ssa.OpcodeLoad: |
| 1033 |
ptr, offset, typ := instr.LoadData() |
| 1034 |
dst := m.c.VRegOf(instr.Return()) |
| 1035 |
m.lowerLoad(ptr, offset, typ, dst) |
| 1036 |
case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32: |
| 1037 |
ptr, offset, _ := instr.LoadData() |
| 1038 |
ret := m.c.VRegOf(instr.Return()) |
| 1039 |
m.lowerExtLoad(op, ptr, offset, ret) |
| 1040 |
case ssa.OpcodeVconst: |
| 1041 |
result := m.c.VRegOf(instr.Return()) |
| 1042 |
lo, hi := instr.VconstData() |
| 1043 |
m.lowerVconst(result, lo, hi) |
| 1044 |
case ssa.OpcodeSExtend, ssa.OpcodeUExtend: |
| 1045 |
from, to, signed := instr.ExtendData() |
| 1046 |
m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed) |
| 1047 |
case ssa.OpcodeIcmp: |
| 1048 |
m.lowerIcmp(instr) |
| 1049 |
case ssa.OpcodeFcmp: |
| 1050 |
m.lowerFcmp(instr) |
| 1051 |
case ssa.OpcodeSelect: |
| 1052 |
cval, x, y := instr.SelectData() |
| 1053 |
m.lowerSelect(x, y, cval, instr.Return()) |
| 1054 |
case ssa.OpcodeIreduce: |
| 1055 |
rn := m.getOperand_Mem_Reg(m.c.ValueDefinition(instr.Arg())) |
| 1056 |
retVal := instr.Return() |
| 1057 |
rd := m.c.VRegOf(retVal) |
| 1058 |
|
| 1059 |
if retVal.Type() != ssa.TypeI32 { |
| 1060 |
panic("TODO?: Ireduce to non-i32") |
| 1061 |
} |
| 1062 |
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, rd)) |
| 1063 |
|
| 1064 |
case ssa.OpcodeAtomicLoad: |
| 1065 |
ptr := instr.Arg() |
| 1066 |
size := instr.AtomicTargetSize() |
| 1067 |
dst := m.c.VRegOf(instr.Return()) |
| 1068 |
|
| 1069 |
// At this point, the ptr is ensured to be aligned, so using a normal load is atomic. |
| 1070 |
// https://github.com/golang/go/blob/go1.24.0/src/internal/runtime/atomic/atomic_amd64.go#L29 |
| 1071 |
mem := newOperandMem(m.lowerToAddressMode(ptr, 0)) |
| 1072 |
load := m.allocateInstr() |
| 1073 |
switch size { |
| 1074 |
case 8: |
| 1075 |
load.asMov64MR(mem, dst) |
| 1076 |
case 4: |
| 1077 |
load.asMovzxRmR(extModeLQ, mem, dst) |
| 1078 |
case 2: |
| 1079 |
load.asMovzxRmR(extModeWQ, mem, dst) |
| 1080 |
case 1: |
| 1081 |
load.asMovzxRmR(extModeBQ, mem, dst) |
| 1082 |
default: |
| 1083 |
panic("BUG") |
| 1084 |
} |
| 1085 |
m.insert(load) |
| 1086 |
|
| 1087 |
case ssa.OpcodeFence: |
| 1088 |
m.insert(m.allocateInstr().asMFence()) |
| 1089 |
|
| 1090 |
case ssa.OpcodeAtomicStore: |
| 1091 |
ptr, _val := instr.Arg2() |
| 1092 |
size := instr.AtomicTargetSize() |
| 1093 |
|
| 1094 |
val := m.getOperand_Reg(m.c.ValueDefinition(_val)) |
| 1095 |
// The content on the val register will be overwritten by xchg, so we need to copy it to a temporary register. |
| 1096 |
copied := m.copyToTmp(val.reg()) |
| 1097 |
|
| 1098 |
mem := newOperandMem(m.lowerToAddressMode(ptr, 0)) |
| 1099 |
store := m.allocateInstr().asXCHG(copied, mem, byte(size)) |
| 1100 |
m.insert(store) |
| 1101 |
|
| 1102 |
case ssa.OpcodeAtomicCas: |
| 1103 |
addr, exp, repl := instr.Arg3() |
| 1104 |
size := instr.AtomicTargetSize() |
| 1105 |
m.lowerAtomicCas(addr, exp, repl, size, instr.Return()) |
| 1106 |
|
| 1107 |
case ssa.OpcodeAtomicRmw: |
| 1108 |
addr, val := instr.Arg2() |
| 1109 |
atomicOp, size := instr.AtomicRmwData() |
| 1110 |
m.lowerAtomicRmw(atomicOp, addr, val, size, instr.Return()) |
| 1111 |
|
| 1112 |
case ssa.OpcodeTailCallReturnCall, ssa.OpcodeTailCallReturnCallIndirect: |
| 1113 |
m.lowerTailCall(instr) |
| 1114 |
|
| 1115 |
default: |
| 1116 |
panic("TODO: lowering " + op.String()) |
| 1117 |
} |
| 1118 |
} |
| 1119 |
|
| 1120 |
func (m *machine) lowerAtomicRmw(op ssa.AtomicRmwOp, addr, val ssa.Value, size uint64, ret ssa.Value) { |
| 1121 |
mem := m.lowerToAddressMode(addr, 0) |
| 1122 |
_val := m.getOperand_Reg(m.c.ValueDefinition(val)) |
| 1123 |
|
| 1124 |
switch op { |
| 1125 |
case ssa.AtomicRmwOpAdd, ssa.AtomicRmwOpSub: |
| 1126 |
valCopied := m.copyToTmp(_val.reg()) |
| 1127 |
if op == ssa.AtomicRmwOpSub { |
| 1128 |
// Negate the value. |
| 1129 |
m.insert(m.allocateInstr().asNeg(newOperandReg(valCopied), true)) |
| 1130 |
} |
| 1131 |
m.insert(m.allocateInstr().asLockXAdd(valCopied, mem, byte(size))) |
| 1132 |
m.clearHigherBitsForAtomic(valCopied, size, ret.Type()) |
| 1133 |
m.copyTo(valCopied, m.c.VRegOf(ret)) |
| 1134 |
|
| 1135 |
case ssa.AtomicRmwOpAnd, ssa.AtomicRmwOpOr, ssa.AtomicRmwOpXor: |
| 1136 |
accumulator := raxVReg |
| 1137 |
// Reserve rax for the accumulator to make regalloc happy. |
| 1138 |
// Note: do this initialization before defining valCopied, because it might be the same register and |
| 1139 |
// if that happens, the unnecessary load/store will be performed inside the loop. |
| 1140 |
// This can be mitigated in any way once the register allocator is clever enough. |
| 1141 |
m.insert(m.allocateInstr().asDefineUninitializedReg(accumulator)) |
| 1142 |
|
| 1143 |
// Copy the value to a temporary register. |
| 1144 |
valCopied := m.copyToTmp(_val.reg()) |
| 1145 |
m.clearHigherBitsForAtomic(valCopied, size, ret.Type()) |
| 1146 |
|
| 1147 |
memOp := newOperandMem(mem) |
| 1148 |
tmp := m.c.AllocateVReg(ssa.TypeI64) |
| 1149 |
beginLoop, beginLoopLabel := m.allocateBrTarget() |
| 1150 |
{ |
| 1151 |
m.insert(beginLoop) |
| 1152 |
// Reset the value on tmp by the original value. |
| 1153 |
m.copyTo(valCopied, tmp) |
| 1154 |
// Load the current value at the memory location into accumulator. |
| 1155 |
switch size { |
| 1156 |
case 1: |
| 1157 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, memOp, accumulator)) |
| 1158 |
case 2: |
| 1159 |
m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, memOp, accumulator)) |
| 1160 |
case 4: |
| 1161 |
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, memOp, accumulator)) |
| 1162 |
case 8: |
| 1163 |
m.insert(m.allocateInstr().asMov64MR(memOp, accumulator)) |
| 1164 |
default: |
| 1165 |
panic("BUG") |
| 1166 |
} |
| 1167 |
// Then perform the logical operation on the accumulator and the value on tmp. |
| 1168 |
switch op { |
| 1169 |
case ssa.AtomicRmwOpAnd: |
| 1170 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, newOperandReg(accumulator), tmp, true)) |
| 1171 |
case ssa.AtomicRmwOpOr: |
| 1172 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeOr, newOperandReg(accumulator), tmp, true)) |
| 1173 |
case ssa.AtomicRmwOpXor: |
| 1174 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(accumulator), tmp, true)) |
| 1175 |
default: |
| 1176 |
panic("BUG") |
| 1177 |
} |
| 1178 |
// Finally, try compare-exchange the value at the memory location with the tmp. |
| 1179 |
m.insert(m.allocateInstr().asLockCmpXCHG(tmp, memOp.addressMode(), byte(size))) |
| 1180 |
// If it succeeds, ZF will be set, and we can break the loop. |
| 1181 |
m.insert(m.allocateInstr().asJmpIf(condNZ, newOperandLabel(beginLoopLabel))) |
| 1182 |
} |
| 1183 |
|
| 1184 |
// valCopied must be alive at the end of the loop. |
| 1185 |
m.insert(m.allocateInstr().asNopUseReg(valCopied)) |
| 1186 |
|
| 1187 |
// At this point, accumulator contains the result. |
| 1188 |
m.clearHigherBitsForAtomic(accumulator, size, ret.Type()) |
| 1189 |
m.copyTo(accumulator, m.c.VRegOf(ret)) |
| 1190 |
|
| 1191 |
case ssa.AtomicRmwOpXchg: |
| 1192 |
valCopied := m.copyToTmp(_val.reg()) |
| 1193 |
|
| 1194 |
m.insert(m.allocateInstr().asXCHG(valCopied, newOperandMem(mem), byte(size))) |
| 1195 |
m.clearHigherBitsForAtomic(valCopied, size, ret.Type()) |
| 1196 |
m.copyTo(valCopied, m.c.VRegOf(ret)) |
| 1197 |
|
| 1198 |
default: |
| 1199 |
panic("BUG") |
| 1200 |
} |
| 1201 |
} |
| 1202 |
|
| 1203 |
func (m *machine) lowerAtomicCas(addr, exp, repl ssa.Value, size uint64, ret ssa.Value) { |
| 1204 |
mem := m.lowerToAddressMode(addr, 0) |
| 1205 |
expOp := m.getOperand_Reg(m.c.ValueDefinition(exp)) |
| 1206 |
replOp := m.getOperand_Reg(m.c.ValueDefinition(repl)) |
| 1207 |
|
| 1208 |
accumulator := raxVReg |
| 1209 |
m.copyTo(expOp.reg(), accumulator) |
| 1210 |
m.insert(m.allocateInstr().asLockCmpXCHG(replOp.reg(), mem, byte(size))) |
| 1211 |
m.clearHigherBitsForAtomic(accumulator, size, ret.Type()) |
| 1212 |
m.copyTo(accumulator, m.c.VRegOf(ret)) |
| 1213 |
} |
| 1214 |
|
| 1215 |
func (m *machine) clearHigherBitsForAtomic(r regalloc.VReg, valSize uint64, resultType ssa.Type) { |
| 1216 |
switch resultType { |
| 1217 |
case ssa.TypeI32: |
| 1218 |
switch valSize { |
| 1219 |
case 1: |
| 1220 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(r), r)) |
| 1221 |
case 2: |
| 1222 |
m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(r), r)) |
| 1223 |
} |
| 1224 |
case ssa.TypeI64: |
| 1225 |
switch valSize { |
| 1226 |
case 1: |
| 1227 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(r), r)) |
| 1228 |
case 2: |
| 1229 |
m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, newOperandReg(r), r)) |
| 1230 |
case 4: |
| 1231 |
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, newOperandReg(r), r)) |
| 1232 |
} |
| 1233 |
} |
| 1234 |
} |
| 1235 |
|
| 1236 |
func (m *machine) lowerFcmp(instr *ssa.Instruction) { |
| 1237 |
f1, f2, and := m.lowerFcmpToFlags(instr) |
| 1238 |
rd := m.c.VRegOf(instr.Return()) |
| 1239 |
if f2 == condInvalid { |
| 1240 |
tmp := m.c.AllocateVReg(ssa.TypeI32) |
| 1241 |
m.insert(m.allocateInstr().asSetcc(f1, tmp)) |
| 1242 |
// On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match |
| 1243 |
// the semantics of Icmp that sets either 0 or 1. |
| 1244 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd)) |
| 1245 |
} else { |
| 1246 |
tmp1, tmp2 := m.c.AllocateVReg(ssa.TypeI32), m.c.AllocateVReg(ssa.TypeI32) |
| 1247 |
m.insert(m.allocateInstr().asSetcc(f1, tmp1)) |
| 1248 |
m.insert(m.allocateInstr().asSetcc(f2, tmp2)) |
| 1249 |
var op aluRmiROpcode |
| 1250 |
if and { |
| 1251 |
op = aluRmiROpcodeAnd |
| 1252 |
} else { |
| 1253 |
op = aluRmiROpcodeOr |
| 1254 |
} |
| 1255 |
m.insert(m.allocateInstr().asAluRmiR(op, newOperandReg(tmp1), tmp2, false)) |
| 1256 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp2), rd)) |
| 1257 |
} |
| 1258 |
} |
| 1259 |
|
| 1260 |
func (m *machine) lowerIcmp(instr *ssa.Instruction) { |
| 1261 |
x, y, c := instr.IcmpData() |
| 1262 |
m.lowerIcmpToFlag(m.c.ValueDefinition(x), m.c.ValueDefinition(y), x.Type() == ssa.TypeI64) |
| 1263 |
rd := m.c.VRegOf(instr.Return()) |
| 1264 |
tmp := m.c.AllocateVReg(ssa.TypeI32) |
| 1265 |
m.insert(m.allocateInstr().asSetcc(condFromSSAIntCmpCond(c), tmp)) |
| 1266 |
// On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match |
| 1267 |
// the semantics of Icmp that sets either 0 or 1. |
| 1268 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd)) |
| 1269 |
} |
| 1270 |
|
| 1271 |
func (m *machine) lowerSelect(x, y, cval, ret ssa.Value) { |
| 1272 |
xo, yo := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 1273 |
rd := m.c.VRegOf(ret) |
| 1274 |
|
| 1275 |
var cond cond |
| 1276 |
cvalDef := m.c.ValueDefinition(cval) |
| 1277 |
switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) { |
| 1278 |
case ssa.OpcodeIcmp: |
| 1279 |
icmp := cvalDef.Instr |
| 1280 |
xc, yc, cc := icmp.IcmpData() |
| 1281 |
m.lowerIcmpToFlag(m.c.ValueDefinition(xc), m.c.ValueDefinition(yc), xc.Type() == ssa.TypeI64) |
| 1282 |
cond = condFromSSAIntCmpCond(cc) |
| 1283 |
icmp.Lowered() |
| 1284 |
default: // TODO: match ssa.OpcodeFcmp for optimization, but seems a bit complex. |
| 1285 |
cv := m.getOperand_Reg(cvalDef) |
| 1286 |
test := m.allocateInstr().asCmpRmiR(false, cv, cv.reg(), false) |
| 1287 |
m.insert(test) |
| 1288 |
cond = condNZ |
| 1289 |
} |
| 1290 |
|
| 1291 |
if typ := x.Type(); typ.IsInt() { |
| 1292 |
_64 := typ.Bits() == 64 |
| 1293 |
mov := m.allocateInstr() |
| 1294 |
tmp := m.c.AllocateVReg(typ) |
| 1295 |
switch yo.kind { |
| 1296 |
case operandKindReg: |
| 1297 |
mov.asMovRR(yo.reg(), tmp, _64) |
| 1298 |
case operandKindMem: |
| 1299 |
if _64 { |
| 1300 |
mov.asMov64MR(yo, tmp) |
| 1301 |
} else { |
| 1302 |
mov.asMovzxRmR(extModeLQ, yo, tmp) |
| 1303 |
} |
| 1304 |
default: |
| 1305 |
panic("BUG") |
| 1306 |
} |
| 1307 |
m.insert(mov) |
| 1308 |
cmov := m.allocateInstr().asCmove(cond, xo, tmp, _64) |
| 1309 |
m.insert(cmov) |
| 1310 |
m.insert(m.allocateInstr().asMovRR(tmp, rd, _64)) |
| 1311 |
} else { |
| 1312 |
mov := m.allocateInstr() |
| 1313 |
tmp := m.c.AllocateVReg(typ) |
| 1314 |
switch typ { |
| 1315 |
case ssa.TypeF32: |
| 1316 |
mov.asXmmUnaryRmR(sseOpcodeMovss, yo, tmp) |
| 1317 |
case ssa.TypeF64: |
| 1318 |
mov.asXmmUnaryRmR(sseOpcodeMovsd, yo, tmp) |
| 1319 |
case ssa.TypeV128: |
| 1320 |
mov.asXmmUnaryRmR(sseOpcodeMovdqu, yo, tmp) |
| 1321 |
default: |
| 1322 |
panic("BUG") |
| 1323 |
} |
| 1324 |
m.insert(mov) |
| 1325 |
|
| 1326 |
cmov := m.allocateInstr().asXmmCMov(cond, xo, tmp, typ.Size()) |
| 1327 |
m.insert(cmov) |
| 1328 |
|
| 1329 |
m.copyTo(tmp, rd) |
| 1330 |
} |
| 1331 |
} |
| 1332 |
|
| 1333 |
func (m *machine) lowerXmmCmovAfterRegAlloc(i *instruction) { |
| 1334 |
x := i.op1 |
| 1335 |
rd := i.op2.reg() |
| 1336 |
cond := cond(i.u1) |
| 1337 |
|
| 1338 |
jcc := m.allocateInstr() |
| 1339 |
m.insert(jcc) |
| 1340 |
|
| 1341 |
mov := m.allocateInstr() |
| 1342 |
switch i.u2 { |
| 1343 |
case 4: |
| 1344 |
mov.asXmmUnaryRmR(sseOpcodeMovss, x, rd) |
| 1345 |
case 8: |
| 1346 |
mov.asXmmUnaryRmR(sseOpcodeMovsd, x, rd) |
| 1347 |
case 16: |
| 1348 |
mov.asXmmUnaryRmR(sseOpcodeMovdqu, x, rd) |
| 1349 |
default: |
| 1350 |
panic("BUG") |
| 1351 |
} |
| 1352 |
m.insert(mov) |
| 1353 |
|
| 1354 |
nop, end := m.allocateBrTarget() |
| 1355 |
m.insert(nop) |
| 1356 |
jcc.asJmpIf(cond.invert(), newOperandLabel(end)) |
| 1357 |
} |
| 1358 |
|
| 1359 |
func (m *machine) lowerExtend(_arg, ret ssa.Value, from, to byte, signed bool) { |
| 1360 |
rd0 := m.c.VRegOf(ret) |
| 1361 |
arg := m.getOperand_Mem_Reg(m.c.ValueDefinition(_arg)) |
| 1362 |
|
| 1363 |
rd := m.c.AllocateVReg(ret.Type()) |
| 1364 |
|
| 1365 |
ext := m.allocateInstr() |
| 1366 |
switch { |
| 1367 |
case from == 8 && to == 16 && signed: |
| 1368 |
ext.asMovsxRmR(extModeBQ, arg, rd) |
| 1369 |
case from == 8 && to == 16 && !signed: |
| 1370 |
ext.asMovzxRmR(extModeBL, arg, rd) |
| 1371 |
case from == 8 && to == 32 && signed: |
| 1372 |
ext.asMovsxRmR(extModeBL, arg, rd) |
| 1373 |
case from == 8 && to == 32 && !signed: |
| 1374 |
ext.asMovzxRmR(extModeBQ, arg, rd) |
| 1375 |
case from == 8 && to == 64 && signed: |
| 1376 |
ext.asMovsxRmR(extModeBQ, arg, rd) |
| 1377 |
case from == 8 && to == 64 && !signed: |
| 1378 |
ext.asMovzxRmR(extModeBQ, arg, rd) |
| 1379 |
case from == 16 && to == 32 && signed: |
| 1380 |
ext.asMovsxRmR(extModeWL, arg, rd) |
| 1381 |
case from == 16 && to == 32 && !signed: |
| 1382 |
ext.asMovzxRmR(extModeWL, arg, rd) |
| 1383 |
case from == 16 && to == 64 && signed: |
| 1384 |
ext.asMovsxRmR(extModeWQ, arg, rd) |
| 1385 |
case from == 16 && to == 64 && !signed: |
| 1386 |
ext.asMovzxRmR(extModeWQ, arg, rd) |
| 1387 |
case from == 32 && to == 64 && signed: |
| 1388 |
ext.asMovsxRmR(extModeLQ, arg, rd) |
| 1389 |
case from == 32 && to == 64 && !signed: |
| 1390 |
ext.asMovzxRmR(extModeLQ, arg, rd) |
| 1391 |
default: |
| 1392 |
panic(fmt.Sprintf("BUG: unhandled extend: from=%d, to=%d, signed=%t", from, to, signed)) |
| 1393 |
} |
| 1394 |
m.insert(ext) |
| 1395 |
|
| 1396 |
m.copyTo(rd, rd0) |
| 1397 |
} |
| 1398 |
|
| 1399 |
func (m *machine) lowerVconst(dst regalloc.VReg, lo, hi uint64) { |
| 1400 |
if lo == 0 && hi == 0 { |
| 1401 |
m.insert(m.allocateInstr().asZeros(dst)) |
| 1402 |
return |
| 1403 |
} |
| 1404 |
|
| 1405 |
load := m.allocateInstr() |
| 1406 |
l, pos := m.allocateLabel() |
| 1407 |
m.consts = append(m.consts, _const{label: l, labelPos: pos, lo: lo, hi: hi}) |
| 1408 |
load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(l)), dst) |
| 1409 |
m.insert(load) |
| 1410 |
} |
| 1411 |
|
| 1412 |
func (m *machine) lowerCtz(instr *ssa.Instruction) { |
| 1413 |
if m.cpuFeatures.Has(platform.CpuFeatureAmd64BMI1) { |
| 1414 |
m.lowerUnaryRmR(instr, unaryRmROpcodeTzcnt) |
| 1415 |
} else { |
| 1416 |
// On processors that do not support TZCNT, the BSF instruction is |
| 1417 |
// executed instead. The key difference between TZCNT and BSF |
| 1418 |
// instruction is that if source operand is zero, the content of |
| 1419 |
// destination operand is undefined. |
| 1420 |
// https://www.felixcloutier.com/x86/tzcnt.html |
| 1421 |
|
| 1422 |
x := instr.Arg() |
| 1423 |
if !x.Type().IsInt() { |
| 1424 |
panic("BUG?") |
| 1425 |
} |
| 1426 |
_64 := x.Type().Bits() == 64 |
| 1427 |
|
| 1428 |
xDef := m.c.ValueDefinition(x) |
| 1429 |
tmp := m.c.AllocateVReg(x.Type()) |
| 1430 |
rm := m.getOperand_Reg(xDef) |
| 1431 |
|
| 1432 |
// First, we have to check if the target is non-zero. |
| 1433 |
test := m.allocateInstr() |
| 1434 |
test.asCmpRmiR(false, rm, rm.reg(), _64) |
| 1435 |
m.insert(test) |
| 1436 |
|
| 1437 |
jmpNz := m.allocateInstr() |
| 1438 |
m.insert(jmpNz) |
| 1439 |
|
| 1440 |
// If the value is zero, we just push the const value. |
| 1441 |
m.lowerIconst(tmp, uint64(x.Type().Bits()), _64) |
| 1442 |
|
| 1443 |
// Now jump right after the non-zero case. |
| 1444 |
jmpAtEnd := m.allocateInstr() |
| 1445 |
m.insert(jmpAtEnd) |
| 1446 |
|
| 1447 |
// jmpNz target label is set here. |
| 1448 |
nop, nz := m.allocateBrTarget() |
| 1449 |
jmpNz.asJmpIf(condNZ, newOperandLabel(nz)) |
| 1450 |
m.insert(nop) |
| 1451 |
|
| 1452 |
// Emit the non-zero case. |
| 1453 |
bsr := m.allocateInstr() |
| 1454 |
bsr.asUnaryRmR(unaryRmROpcodeBsf, rm, tmp, _64) |
| 1455 |
m.insert(bsr) |
| 1456 |
|
| 1457 |
// jmpAtEnd target label is set here. |
| 1458 |
nopEnd, end := m.allocateBrTarget() |
| 1459 |
jmpAtEnd.asJmp(newOperandLabel(end)) |
| 1460 |
m.insert(nopEnd) |
| 1461 |
|
| 1462 |
m.copyTo(tmp, m.c.VRegOf(instr.Return())) |
| 1463 |
} |
| 1464 |
} |
| 1465 |
|
| 1466 |
func (m *machine) lowerClz(instr *ssa.Instruction) { |
| 1467 |
if m.cpuFeatures.Has(platform.CpuFeatureAmd64ABM) { |
| 1468 |
m.lowerUnaryRmR(instr, unaryRmROpcodeLzcnt) |
| 1469 |
} else { |
| 1470 |
// On processors that do not support LZCNT, we combine BSR (calculating |
| 1471 |
// most significant set bit) with XOR. This logic is described in |
| 1472 |
// "Replace Raw Assembly Code with Builtin Intrinsics" section in: |
| 1473 |
// https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code. |
| 1474 |
|
| 1475 |
x := instr.Arg() |
| 1476 |
if !x.Type().IsInt() { |
| 1477 |
panic("BUG?") |
| 1478 |
} |
| 1479 |
_64 := x.Type().Bits() == 64 |
| 1480 |
|
| 1481 |
xDef := m.c.ValueDefinition(x) |
| 1482 |
rm := m.getOperand_Reg(xDef) |
| 1483 |
tmp := m.c.AllocateVReg(x.Type()) |
| 1484 |
|
| 1485 |
// First, we have to check if the rm is non-zero as BSR is undefined |
| 1486 |
// on zero. See https://www.felixcloutier.com/x86/bsr. |
| 1487 |
test := m.allocateInstr() |
| 1488 |
test.asCmpRmiR(false, rm, rm.reg(), _64) |
| 1489 |
m.insert(test) |
| 1490 |
|
| 1491 |
jmpNz := m.allocateInstr() |
| 1492 |
m.insert(jmpNz) |
| 1493 |
|
| 1494 |
// If the value is zero, we just push the const value. |
| 1495 |
m.lowerIconst(tmp, uint64(x.Type().Bits()), _64) |
| 1496 |
|
| 1497 |
// Now jump right after the non-zero case. |
| 1498 |
jmpAtEnd := m.allocateInstr() |
| 1499 |
m.insert(jmpAtEnd) |
| 1500 |
|
| 1501 |
// jmpNz target label is set here. |
| 1502 |
nop, nz := m.allocateBrTarget() |
| 1503 |
jmpNz.asJmpIf(condNZ, newOperandLabel(nz)) |
| 1504 |
m.insert(nop) |
| 1505 |
|
| 1506 |
// Emit the non-zero case. |
| 1507 |
bsr := m.allocateInstr() |
| 1508 |
bsr.asUnaryRmR(unaryRmROpcodeBsr, rm, tmp, _64) |
| 1509 |
m.insert(bsr) |
| 1510 |
|
| 1511 |
// Now we XOR the value with the bit length minus one. |
| 1512 |
xor := m.allocateInstr() |
| 1513 |
xor.asAluRmiR(aluRmiROpcodeXor, newOperandImm32(uint32(x.Type().Bits()-1)), tmp, _64) |
| 1514 |
m.insert(xor) |
| 1515 |
|
| 1516 |
// jmpAtEnd target label is set here. |
| 1517 |
nopEnd, end := m.allocateBrTarget() |
| 1518 |
jmpAtEnd.asJmp(newOperandLabel(end)) |
| 1519 |
m.insert(nopEnd) |
| 1520 |
|
| 1521 |
m.copyTo(tmp, m.c.VRegOf(instr.Return())) |
| 1522 |
} |
| 1523 |
} |
| 1524 |
|
| 1525 |
func (m *machine) lowerUnaryRmR(si *ssa.Instruction, op unaryRmROpcode) { |
| 1526 |
x := si.Arg() |
| 1527 |
if !x.Type().IsInt() { |
| 1528 |
panic("BUG?") |
| 1529 |
} |
| 1530 |
_64 := x.Type().Bits() == 64 |
| 1531 |
|
| 1532 |
xDef := m.c.ValueDefinition(x) |
| 1533 |
rm := m.getOperand_Mem_Reg(xDef) |
| 1534 |
rd := m.c.VRegOf(si.Return()) |
| 1535 |
|
| 1536 |
instr := m.allocateInstr() |
| 1537 |
instr.asUnaryRmR(op, rm, rd, _64) |
| 1538 |
m.insert(instr) |
| 1539 |
} |
| 1540 |
|
| 1541 |
func (m *machine) lowerLoad(ptr ssa.Value, offset uint32, typ ssa.Type, dst regalloc.VReg) { |
| 1542 |
mem := newOperandMem(m.lowerToAddressMode(ptr, offset)) |
| 1543 |
load := m.allocateInstr() |
| 1544 |
switch typ { |
| 1545 |
case ssa.TypeI32: |
| 1546 |
load.asMovzxRmR(extModeLQ, mem, dst) |
| 1547 |
case ssa.TypeI64: |
| 1548 |
load.asMov64MR(mem, dst) |
| 1549 |
case ssa.TypeF32: |
| 1550 |
load.asXmmUnaryRmR(sseOpcodeMovss, mem, dst) |
| 1551 |
case ssa.TypeF64: |
| 1552 |
load.asXmmUnaryRmR(sseOpcodeMovsd, mem, dst) |
| 1553 |
case ssa.TypeV128: |
| 1554 |
load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, dst) |
| 1555 |
default: |
| 1556 |
panic("BUG") |
| 1557 |
} |
| 1558 |
m.insert(load) |
| 1559 |
} |
| 1560 |
|
| 1561 |
func (m *machine) lowerExtLoad(op ssa.Opcode, ptr ssa.Value, offset uint32, dst regalloc.VReg) { |
| 1562 |
mem := newOperandMem(m.lowerToAddressMode(ptr, offset)) |
| 1563 |
load := m.allocateInstr() |
| 1564 |
switch op { |
| 1565 |
case ssa.OpcodeUload8: |
| 1566 |
load.asMovzxRmR(extModeBQ, mem, dst) |
| 1567 |
case ssa.OpcodeUload16: |
| 1568 |
load.asMovzxRmR(extModeWQ, mem, dst) |
| 1569 |
case ssa.OpcodeUload32: |
| 1570 |
load.asMovzxRmR(extModeLQ, mem, dst) |
| 1571 |
case ssa.OpcodeSload8: |
| 1572 |
load.asMovsxRmR(extModeBQ, mem, dst) |
| 1573 |
case ssa.OpcodeSload16: |
| 1574 |
load.asMovsxRmR(extModeWQ, mem, dst) |
| 1575 |
case ssa.OpcodeSload32: |
| 1576 |
load.asMovsxRmR(extModeLQ, mem, dst) |
| 1577 |
default: |
| 1578 |
panic("BUG") |
| 1579 |
} |
| 1580 |
m.insert(load) |
| 1581 |
} |
| 1582 |
|
| 1583 |
func (m *machine) lowerExitIfTrueWithCode(execCtx regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) { |
| 1584 |
condDef := m.c.ValueDefinition(cond) |
| 1585 |
if !m.c.MatchInstr(condDef, ssa.OpcodeIcmp) { |
| 1586 |
panic("TODO: ExitIfTrue must come after Icmp at the moment: " + condDef.Instr.Opcode().String()) |
| 1587 |
} |
| 1588 |
cvalInstr := condDef.Instr |
| 1589 |
cvalInstr.MarkLowered() |
| 1590 |
|
| 1591 |
// We need to copy the execution context to a temp register, because if it's spilled, |
| 1592 |
// it might end up being reloaded inside the exiting branch. |
| 1593 |
execCtxTmp := m.copyToTmp(execCtx) |
| 1594 |
|
| 1595 |
x, y, c := cvalInstr.IcmpData() |
| 1596 |
xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 1597 |
if !m.tryLowerBandToFlag(xx, yy) { |
| 1598 |
m.lowerIcmpToFlag(xx, yy, x.Type() == ssa.TypeI64) |
| 1599 |
} |
| 1600 |
|
| 1601 |
jmpIf := m.allocateInstr() |
| 1602 |
m.insert(jmpIf) |
| 1603 |
l := m.lowerExitWithCode(execCtxTmp, code) |
| 1604 |
jmpIf.asJmpIf(condFromSSAIntCmpCond(c).invert(), newOperandLabel(l)) |
| 1605 |
} |
| 1606 |
|
| 1607 |
func (m *machine) tryLowerBandToFlag(x, y backend.SSAValueDefinition) (ok bool) { |
| 1608 |
var target backend.SSAValueDefinition |
| 1609 |
var got bool |
| 1610 |
if x.IsFromInstr() && x.Instr.Constant() && x.Instr.ConstantVal() == 0 { |
| 1611 |
if m.c.MatchInstr(y, ssa.OpcodeBand) { |
| 1612 |
target = y |
| 1613 |
got = true |
| 1614 |
} |
| 1615 |
} |
| 1616 |
|
| 1617 |
if y.IsFromInstr() && y.Instr.Constant() && y.Instr.ConstantVal() == 0 { |
| 1618 |
if m.c.MatchInstr(x, ssa.OpcodeBand) { |
| 1619 |
target = x |
| 1620 |
got = true |
| 1621 |
} |
| 1622 |
} |
| 1623 |
|
| 1624 |
if !got { |
| 1625 |
return false |
| 1626 |
} |
| 1627 |
|
| 1628 |
bandInstr := target.Instr |
| 1629 |
bandX, bandY := bandInstr.Arg2() |
| 1630 |
|
| 1631 |
xx := m.getOperand_Reg(m.c.ValueDefinition(bandX)) |
| 1632 |
yy := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(bandY)) |
| 1633 |
test := m.allocateInstr().asCmpRmiR(false, yy, xx.reg(), bandX.Type() == ssa.TypeI64) |
| 1634 |
m.insert(test) |
| 1635 |
bandInstr.MarkLowered() |
| 1636 |
return true |
| 1637 |
} |
| 1638 |
|
| 1639 |
func (m *machine) allocateExitInstructions(execCtx, exitCodeReg regalloc.VReg) (saveRsp, saveRbp, setExitCode *instruction) { |
| 1640 |
saveRsp = m.allocateInstr().asMovRM( |
| 1641 |
rspVReg, |
| 1642 |
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.U32(), execCtx)), |
| 1643 |
8, |
| 1644 |
) |
| 1645 |
|
| 1646 |
saveRbp = m.allocateInstr().asMovRM( |
| 1647 |
rbpVReg, |
| 1648 |
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetFramePointerBeforeGoCall.U32(), execCtx)), |
| 1649 |
8, |
| 1650 |
) |
| 1651 |
setExitCode = m.allocateInstr().asMovRM( |
| 1652 |
exitCodeReg, |
| 1653 |
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetExitCodeOffset.U32(), execCtx)), |
| 1654 |
4, |
| 1655 |
) |
| 1656 |
return |
| 1657 |
} |
| 1658 |
|
| 1659 |
func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel label) { |
| 1660 |
exitCodeReg := rbpVReg |
| 1661 |
saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtx, exitCodeReg) |
| 1662 |
|
| 1663 |
// Set save RSP, RBP, and write exit code. |
| 1664 |
m.insert(saveRsp) |
| 1665 |
m.insert(saveRbp) |
| 1666 |
m.lowerIconst(exitCodeReg, uint64(code), false) |
| 1667 |
m.insert(setExitCode) |
| 1668 |
|
| 1669 |
ripReg := rbpVReg |
| 1670 |
|
| 1671 |
// Next is to save the current address for stack unwinding. |
| 1672 |
nop, currentAddrLabel := m.allocateBrTarget() |
| 1673 |
m.insert(nop) |
| 1674 |
readRip := m.allocateInstr().asLEA(newOperandLabel(currentAddrLabel), ripReg) |
| 1675 |
m.insert(readRip) |
| 1676 |
saveRip := m.allocateInstr().asMovRM( |
| 1677 |
ripReg, |
| 1678 |
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)), |
| 1679 |
8, |
| 1680 |
) |
| 1681 |
m.insert(saveRip) |
| 1682 |
|
| 1683 |
// Finally exit. |
| 1684 |
exitSq := m.allocateExitSeq(execCtx) |
| 1685 |
m.insert(exitSq) |
| 1686 |
|
| 1687 |
// Return the label for continuation. |
| 1688 |
continuation, afterLabel := m.allocateBrTarget() |
| 1689 |
m.insert(continuation) |
| 1690 |
return afterLabel |
| 1691 |
} |
| 1692 |
|
| 1693 |
func (m *machine) lowerAluRmiROp(si *ssa.Instruction, op aluRmiROpcode) { |
| 1694 |
x, y := si.Arg2() |
| 1695 |
if !x.Type().IsInt() { |
| 1696 |
panic("BUG?") |
| 1697 |
} |
| 1698 |
|
| 1699 |
_64 := x.Type().Bits() == 64 |
| 1700 |
|
| 1701 |
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 1702 |
|
| 1703 |
// TODO: commutative args can be swapped if one of them is an immediate. |
| 1704 |
rn := m.getOperand_Reg(xDef) |
| 1705 |
rm := m.getOperand_Mem_Imm32_Reg(yDef) |
| 1706 |
rd := m.c.VRegOf(si.Return()) |
| 1707 |
|
| 1708 |
// rn is being overwritten, so we first copy its value to a temp register, |
| 1709 |
// in case it is referenced again later. |
| 1710 |
tmp := m.copyToTmp(rn.reg()) |
| 1711 |
|
| 1712 |
alu := m.allocateInstr() |
| 1713 |
alu.asAluRmiR(op, rm, tmp, _64) |
| 1714 |
m.insert(alu) |
| 1715 |
|
| 1716 |
// tmp now contains the result, we copy it to the dest register. |
| 1717 |
m.copyTo(tmp, rd) |
| 1718 |
} |
| 1719 |
|
| 1720 |
func (m *machine) lowerShiftR(si *ssa.Instruction, op shiftROp) { |
| 1721 |
x, amt := si.Arg2() |
| 1722 |
if !x.Type().IsInt() { |
| 1723 |
panic("BUG?") |
| 1724 |
} |
| 1725 |
_64 := x.Type().Bits() == 64 |
| 1726 |
|
| 1727 |
xDef, amtDef := m.c.ValueDefinition(x), m.c.ValueDefinition(amt) |
| 1728 |
|
| 1729 |
opAmt := m.getOperand_Imm32_Reg(amtDef) |
| 1730 |
rx := m.getOperand_Reg(xDef) |
| 1731 |
rd := m.c.VRegOf(si.Return()) |
| 1732 |
|
| 1733 |
// rx is being overwritten, so we first copy its value to a temp register, |
| 1734 |
// in case it is referenced again later. |
| 1735 |
tmpDst := m.copyToTmp(rx.reg()) |
| 1736 |
|
| 1737 |
if opAmt.kind == operandKindReg { |
| 1738 |
// If opAmt is a register we must copy its value to rcx, |
| 1739 |
// because shiftR encoding mandates that the shift amount is in rcx. |
| 1740 |
m.copyTo(opAmt.reg(), rcxVReg) |
| 1741 |
|
| 1742 |
alu := m.allocateInstr() |
| 1743 |
alu.asShiftR(op, newOperandReg(rcxVReg), tmpDst, _64) |
| 1744 |
m.insert(alu) |
| 1745 |
|
| 1746 |
} else { |
| 1747 |
alu := m.allocateInstr() |
| 1748 |
alu.asShiftR(op, opAmt, tmpDst, _64) |
| 1749 |
m.insert(alu) |
| 1750 |
} |
| 1751 |
|
| 1752 |
// tmp now contains the result, we copy it to the dest register. |
| 1753 |
m.copyTo(tmpDst, rd) |
| 1754 |
} |
| 1755 |
|
| 1756 |
func (m *machine) lowerXmmRmR(instr *ssa.Instruction) { |
| 1757 |
x, y := instr.Arg2() |
| 1758 |
if !x.Type().IsFloat() { |
| 1759 |
panic("BUG?") |
| 1760 |
} |
| 1761 |
_64 := x.Type().Bits() == 64 |
| 1762 |
|
| 1763 |
var op sseOpcode |
| 1764 |
if _64 { |
| 1765 |
switch instr.Opcode() { |
| 1766 |
case ssa.OpcodeFadd: |
| 1767 |
op = sseOpcodeAddsd |
| 1768 |
case ssa.OpcodeFsub: |
| 1769 |
op = sseOpcodeSubsd |
| 1770 |
case ssa.OpcodeFmul: |
| 1771 |
op = sseOpcodeMulsd |
| 1772 |
case ssa.OpcodeFdiv: |
| 1773 |
op = sseOpcodeDivsd |
| 1774 |
default: |
| 1775 |
panic("BUG") |
| 1776 |
} |
| 1777 |
} else { |
| 1778 |
switch instr.Opcode() { |
| 1779 |
case ssa.OpcodeFadd: |
| 1780 |
op = sseOpcodeAddss |
| 1781 |
case ssa.OpcodeFsub: |
| 1782 |
op = sseOpcodeSubss |
| 1783 |
case ssa.OpcodeFmul: |
| 1784 |
op = sseOpcodeMulss |
| 1785 |
case ssa.OpcodeFdiv: |
| 1786 |
op = sseOpcodeDivss |
| 1787 |
default: |
| 1788 |
panic("BUG") |
| 1789 |
} |
| 1790 |
} |
| 1791 |
|
| 1792 |
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 1793 |
rn := m.getOperand_Reg(yDef) |
| 1794 |
rm := m.getOperand_Reg(xDef) |
| 1795 |
rd := m.c.VRegOf(instr.Return()) |
| 1796 |
|
| 1797 |
// rm is being overwritten, so we first copy its value to a temp register, |
| 1798 |
// in case it is referenced again later. |
| 1799 |
tmp := m.copyToTmp(rm.reg()) |
| 1800 |
|
| 1801 |
xmm := m.allocateInstr().asXmmRmR(op, rn, tmp) |
| 1802 |
m.insert(xmm) |
| 1803 |
|
| 1804 |
m.copyTo(tmp, rd) |
| 1805 |
} |
| 1806 |
|
| 1807 |
func (m *machine) lowerSqrt(instr *ssa.Instruction) { |
| 1808 |
x := instr.Arg() |
| 1809 |
if !x.Type().IsFloat() { |
| 1810 |
panic("BUG") |
| 1811 |
} |
| 1812 |
_64 := x.Type().Bits() == 64 |
| 1813 |
var op sseOpcode |
| 1814 |
if _64 { |
| 1815 |
op = sseOpcodeSqrtsd |
| 1816 |
} else { |
| 1817 |
op = sseOpcodeSqrtss |
| 1818 |
} |
| 1819 |
|
| 1820 |
xDef := m.c.ValueDefinition(x) |
| 1821 |
rm := m.getOperand_Mem_Reg(xDef) |
| 1822 |
rd := m.c.VRegOf(instr.Return()) |
| 1823 |
|
| 1824 |
xmm := m.allocateInstr().asXmmUnaryRmR(op, rm, rd) |
| 1825 |
m.insert(xmm) |
| 1826 |
} |
| 1827 |
|
| 1828 |
func (m *machine) lowerFabsFneg(instr *ssa.Instruction) { |
| 1829 |
x := instr.Arg() |
| 1830 |
if !x.Type().IsFloat() { |
| 1831 |
panic("BUG") |
| 1832 |
} |
| 1833 |
_64 := x.Type().Bits() == 64 |
| 1834 |
var op sseOpcode |
| 1835 |
var mask uint64 |
| 1836 |
if _64 { |
| 1837 |
switch instr.Opcode() { |
| 1838 |
case ssa.OpcodeFabs: |
| 1839 |
mask, op = 0x7fffffffffffffff, sseOpcodeAndpd |
| 1840 |
case ssa.OpcodeFneg: |
| 1841 |
mask, op = 0x8000000000000000, sseOpcodeXorpd |
| 1842 |
} |
| 1843 |
} else { |
| 1844 |
switch instr.Opcode() { |
| 1845 |
case ssa.OpcodeFabs: |
| 1846 |
mask, op = 0x7fffffff, sseOpcodeAndps |
| 1847 |
case ssa.OpcodeFneg: |
| 1848 |
mask, op = 0x80000000, sseOpcodeXorps |
| 1849 |
} |
| 1850 |
} |
| 1851 |
|
| 1852 |
tmp := m.c.AllocateVReg(x.Type()) |
| 1853 |
|
| 1854 |
xDef := m.c.ValueDefinition(x) |
| 1855 |
rm := m.getOperand_Reg(xDef) |
| 1856 |
rd := m.c.VRegOf(instr.Return()) |
| 1857 |
|
| 1858 |
m.lowerFconst(tmp, mask, _64) |
| 1859 |
|
| 1860 |
xmm := m.allocateInstr().asXmmRmR(op, rm, tmp) |
| 1861 |
m.insert(xmm) |
| 1862 |
|
| 1863 |
m.copyTo(tmp, rd) |
| 1864 |
} |
| 1865 |
|
| 1866 |
func (m *machine) lowerStore(si *ssa.Instruction) { |
| 1867 |
value, ptr, offset, storeSizeInBits := si.StoreData() |
| 1868 |
rm := m.getOperand_Reg(m.c.ValueDefinition(value)) |
| 1869 |
mem := newOperandMem(m.lowerToAddressMode(ptr, offset)) |
| 1870 |
|
| 1871 |
store := m.allocateInstr() |
| 1872 |
switch value.Type() { |
| 1873 |
case ssa.TypeI32: |
| 1874 |
store.asMovRM(rm.reg(), mem, storeSizeInBits/8) |
| 1875 |
case ssa.TypeI64: |
| 1876 |
store.asMovRM(rm.reg(), mem, storeSizeInBits/8) |
| 1877 |
case ssa.TypeF32: |
| 1878 |
store.asXmmMovRM(sseOpcodeMovss, rm.reg(), mem) |
| 1879 |
case ssa.TypeF64: |
| 1880 |
store.asXmmMovRM(sseOpcodeMovsd, rm.reg(), mem) |
| 1881 |
case ssa.TypeV128: |
| 1882 |
store.asXmmMovRM(sseOpcodeMovdqu, rm.reg(), mem) |
| 1883 |
default: |
| 1884 |
panic("BUG") |
| 1885 |
} |
| 1886 |
m.insert(store) |
| 1887 |
} |
| 1888 |
|
| 1889 |
func (m *machine) lowerCall(si *ssa.Instruction) { |
| 1890 |
isDirectCall := si.Opcode() == ssa.OpcodeCall |
| 1891 |
indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall) |
| 1892 |
|
| 1893 |
if isMemmove { |
| 1894 |
// Go's memmove *might* use all xmm0-xmm15, so we need to release them. |
| 1895 |
// https://github.com/golang/go/blob/go1.24.0/src/cmd/compile/abi-internal.md#architecture-specifics |
| 1896 |
// https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L286-L301 |
| 1897 |
for i := regalloc.RealReg(0); i < 16; i++ { |
| 1898 |
m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[xmm0+i])) |
| 1899 |
} |
| 1900 |
// Since Go 1.24 it may also use DX, which is not reserved for the function call's 3 args. |
| 1901 |
// https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L123 |
| 1902 |
m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[rdx])) |
| 1903 |
} |
| 1904 |
|
| 1905 |
if isDirectCall { |
| 1906 |
call := m.allocateInstr().asCall(directCallee, calleeABI) |
| 1907 |
m.insert(call) |
| 1908 |
} else { |
| 1909 |
ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr)) |
| 1910 |
callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI) |
| 1911 |
m.insert(callInd) |
| 1912 |
} |
| 1913 |
|
| 1914 |
if isMemmove { |
| 1915 |
for i := regalloc.RealReg(0); i < 16; i++ { |
| 1916 |
m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[xmm0+i])) |
| 1917 |
} |
| 1918 |
m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[rdx])) |
| 1919 |
} |
| 1920 |
|
| 1921 |
m.insertReturns(si, calleeABI, stackSlotSize) |
| 1922 |
} |
| 1923 |
|
| 1924 |
func (m *machine) prepareCall(si *ssa.Instruction, isDirectCall bool) (ssa.Value, ssa.FuncRef, bool, *backend.FunctionABI, int64) { |
| 1925 |
var indirectCalleePtr ssa.Value |
| 1926 |
var directCallee ssa.FuncRef |
| 1927 |
var sigID ssa.SignatureID |
| 1928 |
var args []ssa.Value |
| 1929 |
var isMemmove bool |
| 1930 |
if isDirectCall { |
| 1931 |
directCallee, sigID, args = si.CallData() |
| 1932 |
} else { |
| 1933 |
indirectCalleePtr, sigID, args, isMemmove = si.CallIndirectData() |
| 1934 |
} |
| 1935 |
calleeABI := m.c.GetFunctionABI(m.c.SSABuilder().ResolveSignature(sigID)) |
| 1936 |
|
| 1937 |
stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize()) |
| 1938 |
if m.maxRequiredStackSizeForCalls < stackSlotSize+16 { |
| 1939 |
m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // 16 == return address + RBP. |
| 1940 |
} |
| 1941 |
|
| 1942 |
// Note: See machine.SetupPrologue for the stack layout. |
| 1943 |
// The stack pointer decrease/increase will be inserted later in the compilation. |
| 1944 |
|
| 1945 |
for i, arg := range args { |
| 1946 |
reg := m.c.VRegOf(arg) |
| 1947 |
def := m.c.ValueDefinition(arg) |
| 1948 |
m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize) |
| 1949 |
} |
| 1950 |
return indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize |
| 1951 |
} |
| 1952 |
|
| 1953 |
func (m *machine) insertReturns(si *ssa.Instruction, calleeABI *backend.FunctionABI, stackSlotSize int64) { |
| 1954 |
var index int |
| 1955 |
r1, rs := si.Returns() |
| 1956 |
if r1.Valid() { |
| 1957 |
m.callerGenFunctionReturnVReg(calleeABI, 0, m.c.VRegOf(r1), stackSlotSize) |
| 1958 |
index++ |
| 1959 |
} |
| 1960 |
|
| 1961 |
for _, r := range rs { |
| 1962 |
m.callerGenFunctionReturnVReg(calleeABI, index, m.c.VRegOf(r), stackSlotSize) |
| 1963 |
index++ |
| 1964 |
} |
| 1965 |
} |
| 1966 |
|
| 1967 |
func (m *machine) lowerTailCall(si *ssa.Instruction) { |
| 1968 |
isDirectCall := si.Opcode() == ssa.OpcodeTailCallReturnCall |
| 1969 |
indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall) |
| 1970 |
if isMemmove { |
| 1971 |
panic("memmove not supported in tail calls") |
| 1972 |
} |
| 1973 |
|
| 1974 |
isAllRegs := stackSlotSize == 0 |
| 1975 |
|
| 1976 |
switch { |
| 1977 |
case isDirectCall && isAllRegs: |
| 1978 |
call := m.allocateInstr().asTailCallReturnCall(directCallee, calleeABI) |
| 1979 |
m.insert(call) |
| 1980 |
case !isDirectCall && isAllRegs: |
| 1981 |
// In a tail call we insert the epilogue before the jump instruction, |
| 1982 |
// so an arbitrary register might be overwritten while restoring the stack. |
| 1983 |
// So, as compared to a regular indirect call, we ensure the pointer is stored |
| 1984 |
// in a caller-saved register (r11). |
| 1985 |
// For details, see internal/engine/RATIONALE.md |
| 1986 |
ptrOp := m.getOperand_Reg(m.c.ValueDefinition(indirectCalleePtr)) |
| 1987 |
tmpJmp := r11VReg |
| 1988 |
m.InsertMove(tmpJmp, ptrOp.reg(), ssa.TypeI64) |
| 1989 |
callInd := m.allocateInstr().asTailCallReturnCallIndirect(newOperandReg(tmpJmp), calleeABI) |
| 1990 |
m.insert(callInd) |
| 1991 |
case isDirectCall && !isAllRegs: |
| 1992 |
call := m.allocateInstr().asCall(directCallee, calleeABI) |
| 1993 |
m.insert(call) |
| 1994 |
case !isDirectCall && !isAllRegs: |
| 1995 |
ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr)) |
| 1996 |
callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI) |
| 1997 |
m.insert(callInd) |
| 1998 |
} |
| 1999 |
|
| 2000 |
// If this is a proper tail call, returns will be cleared in the postRegAlloc phase. |
| 2001 |
m.insertReturns(si, calleeABI, stackSlotSize) |
| 2002 |
} |
| 2003 |
|
| 2004 |
// callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the |
| 2005 |
// caller side of the function call. |
| 2006 |
func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, stackSlotSize int64) { |
| 2007 |
arg := &a.Args[argIndex] |
| 2008 |
if def.IsFromInstr() { |
| 2009 |
// Constant instructions are inlined. |
| 2010 |
if inst := def.Instr; inst.Constant() { |
| 2011 |
m.insertLoadConstant(inst, reg) |
| 2012 |
} |
| 2013 |
} |
| 2014 |
if arg.Kind == backend.ABIArgKindReg { |
| 2015 |
m.InsertMove(arg.Reg, reg, arg.Type) |
| 2016 |
} else { |
| 2017 |
store := m.allocateInstr() |
| 2018 |
mem := newOperandMem(m.newAmodeImmReg( |
| 2019 |
// -stackSlotSize because the stack pointer is not yet decreased. |
| 2020 |
uint32(arg.Offset-stackSlotSize), rspVReg)) |
| 2021 |
switch arg.Type { |
| 2022 |
case ssa.TypeI32: |
| 2023 |
store.asMovRM(reg, mem, 4) |
| 2024 |
case ssa.TypeI64: |
| 2025 |
store.asMovRM(reg, mem, 8) |
| 2026 |
case ssa.TypeF32: |
| 2027 |
store.asXmmMovRM(sseOpcodeMovss, reg, mem) |
| 2028 |
case ssa.TypeF64: |
| 2029 |
store.asXmmMovRM(sseOpcodeMovsd, reg, mem) |
| 2030 |
case ssa.TypeV128: |
| 2031 |
store.asXmmMovRM(sseOpcodeMovdqu, reg, mem) |
| 2032 |
default: |
| 2033 |
panic("BUG") |
| 2034 |
} |
| 2035 |
m.insert(store) |
| 2036 |
} |
| 2037 |
} |
| 2038 |
|
| 2039 |
func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, stackSlotSize int64) { |
| 2040 |
r := &a.Rets[retIndex] |
| 2041 |
if r.Kind == backend.ABIArgKindReg { |
| 2042 |
m.InsertMove(reg, r.Reg, r.Type) |
| 2043 |
} else { |
| 2044 |
load := m.allocateInstr() |
| 2045 |
mem := newOperandMem(m.newAmodeImmReg( |
| 2046 |
// -stackSlotSize because the stack pointer is not yet decreased. |
| 2047 |
uint32(a.ArgStackSize+r.Offset-stackSlotSize), rspVReg)) |
| 2048 |
switch r.Type { |
| 2049 |
case ssa.TypeI32: |
| 2050 |
load.asMovzxRmR(extModeLQ, mem, reg) |
| 2051 |
case ssa.TypeI64: |
| 2052 |
load.asMov64MR(mem, reg) |
| 2053 |
case ssa.TypeF32: |
| 2054 |
load.asXmmUnaryRmR(sseOpcodeMovss, mem, reg) |
| 2055 |
case ssa.TypeF64: |
| 2056 |
load.asXmmUnaryRmR(sseOpcodeMovsd, mem, reg) |
| 2057 |
case ssa.TypeV128: |
| 2058 |
load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, reg) |
| 2059 |
default: |
| 2060 |
panic("BUG") |
| 2061 |
} |
| 2062 |
m.insert(load) |
| 2063 |
} |
| 2064 |
} |
| 2065 |
|
| 2066 |
// InsertMove implements backend.Machine. |
| 2067 |
func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) { |
| 2068 |
switch typ { |
| 2069 |
case ssa.TypeI32, ssa.TypeI64: |
| 2070 |
i := m.allocateInstr().asMovRR(src, dst, typ.Bits() == 64) |
| 2071 |
m.insert(i) |
| 2072 |
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: |
| 2073 |
var op sseOpcode |
| 2074 |
switch typ { |
| 2075 |
case ssa.TypeF32: |
| 2076 |
op = sseOpcodeMovss |
| 2077 |
case ssa.TypeF64: |
| 2078 |
op = sseOpcodeMovsd |
| 2079 |
case ssa.TypeV128: |
| 2080 |
op = sseOpcodeMovdqa |
| 2081 |
} |
| 2082 |
i := m.allocateInstr().asXmmUnaryRmR(op, newOperandReg(src), dst) |
| 2083 |
m.insert(i) |
| 2084 |
default: |
| 2085 |
panic("BUG") |
| 2086 |
} |
| 2087 |
} |
| 2088 |
|
| 2089 |
// Format implements backend.Machine. |
| 2090 |
func (m *machine) Format() string { |
| 2091 |
begins := map[*instruction]label{} |
| 2092 |
for l := label(0); l < m.nextLabel; l++ { |
| 2093 |
pos := m.labelPositionPool.Get(int(l)) |
| 2094 |
if pos != nil { |
| 2095 |
begins[pos.begin] = l |
| 2096 |
} |
| 2097 |
} |
| 2098 |
|
| 2099 |
var lines []string |
| 2100 |
for cur := m.rootInstr; cur != nil; cur = cur.next { |
| 2101 |
if l, ok := begins[cur]; ok { |
| 2102 |
var labelStr string |
| 2103 |
if l <= m.maxSSABlockID { |
| 2104 |
labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, l) |
| 2105 |
} else { |
| 2106 |
labelStr = fmt.Sprintf("%s:", l) |
| 2107 |
} |
| 2108 |
lines = append(lines, labelStr) |
| 2109 |
} |
| 2110 |
if cur.kind == nop0 { |
| 2111 |
continue |
| 2112 |
} |
| 2113 |
lines = append(lines, "\t"+cur.String()) |
| 2114 |
} |
| 2115 |
for _, vc := range m.consts { |
| 2116 |
if vc._var == nil { |
| 2117 |
lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label, vc.lo, vc.hi)) |
| 2118 |
} else { |
| 2119 |
lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label, vc._var)) |
| 2120 |
} |
| 2121 |
} |
| 2122 |
return "\n" + strings.Join(lines, "\n") + "\n" |
| 2123 |
} |
| 2124 |
|
| 2125 |
func (m *machine) encodeWithoutSSA(root *instruction) { |
| 2126 |
m.labelResolutionPends = m.labelResolutionPends[:0] |
| 2127 |
bufPtr := m.c.BufPtr() |
| 2128 |
for cur := root; cur != nil; cur = cur.next { |
| 2129 |
offset := int64(len(*bufPtr)) |
| 2130 |
if cur.kind == nop0 { |
| 2131 |
l := cur.nop0Label() |
| 2132 |
pos := m.labelPositionPool.Get(int(l)) |
| 2133 |
if pos != nil { |
| 2134 |
pos.binaryOffset = offset |
| 2135 |
} |
| 2136 |
} |
| 2137 |
|
| 2138 |
needLabelResolution := cur.encode(m.c) |
| 2139 |
if needLabelResolution { |
| 2140 |
m.labelResolutionPends = append(m.labelResolutionPends, |
| 2141 |
labelResolutionPend{instr: cur, imm32Offset: int64(len(*bufPtr)) - 4}, |
| 2142 |
) |
| 2143 |
} |
| 2144 |
} |
| 2145 |
|
| 2146 |
for i := range m.labelResolutionPends { |
| 2147 |
p := &m.labelResolutionPends[i] |
| 2148 |
switch p.instr.kind { |
| 2149 |
case jmp, jmpIf, lea: |
| 2150 |
target := p.instr.jmpLabel() |
| 2151 |
targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset |
| 2152 |
imm32Offset := p.imm32Offset |
| 2153 |
jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction. |
| 2154 |
binary.LittleEndian.PutUint32((*bufPtr)[imm32Offset:], uint32(jmpOffset)) |
| 2155 |
default: |
| 2156 |
panic("BUG") |
| 2157 |
} |
| 2158 |
} |
| 2159 |
} |
| 2160 |
|
| 2161 |
// Encode implements backend.Machine Encode. |
| 2162 |
func (m *machine) Encode(ctx context.Context) (err error) { |
| 2163 |
bufPtr := m.c.BufPtr() |
| 2164 |
|
| 2165 |
var fn string |
| 2166 |
var fnIndex int |
| 2167 |
var labelPosToLabel map[*labelPosition]label |
| 2168 |
if wazevoapi.PerfMapEnabled { |
| 2169 |
fn = wazevoapi.GetCurrentFunctionName(ctx) |
| 2170 |
labelPosToLabel = make(map[*labelPosition]label) |
| 2171 |
for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ { |
| 2172 |
pos := m.labelPositionPool.Get(i) |
| 2173 |
labelPosToLabel[pos] = label(i) |
| 2174 |
} |
| 2175 |
fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx) |
| 2176 |
} |
| 2177 |
|
| 2178 |
m.labelResolutionPends = m.labelResolutionPends[:0] |
| 2179 |
for _, pos := range m.orderedSSABlockLabelPos { |
| 2180 |
offset := int64(len(*bufPtr)) |
| 2181 |
pos.binaryOffset = offset |
| 2182 |
for cur := pos.begin; cur != pos.end.next; cur = cur.next { |
| 2183 |
offset := int64(len(*bufPtr)) |
| 2184 |
|
| 2185 |
switch cur.kind { |
| 2186 |
case nop0: |
| 2187 |
l := cur.nop0Label() |
| 2188 |
if pos := m.labelPositionPool.Get(int(l)); pos != nil { |
| 2189 |
pos.binaryOffset = offset |
| 2190 |
} |
| 2191 |
case sourceOffsetInfo: |
| 2192 |
m.c.AddSourceOffsetInfo(offset, cur.sourceOffsetInfo()) |
| 2193 |
} |
| 2194 |
|
| 2195 |
needLabelResolution := cur.encode(m.c) |
| 2196 |
if needLabelResolution { |
| 2197 |
m.labelResolutionPends = append(m.labelResolutionPends, |
| 2198 |
labelResolutionPend{instr: cur, instrOffset: offset, imm32Offset: int64(len(*bufPtr)) - 4}, |
| 2199 |
) |
| 2200 |
} |
| 2201 |
} |
| 2202 |
|
| 2203 |
if wazevoapi.PerfMapEnabled { |
| 2204 |
l := labelPosToLabel[pos] |
| 2205 |
size := int64(len(*bufPtr)) - offset |
| 2206 |
wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, l)) |
| 2207 |
} |
| 2208 |
} |
| 2209 |
|
| 2210 |
for i := range m.consts { |
| 2211 |
offset := int64(len(*bufPtr)) |
| 2212 |
vc := &m.consts[i] |
| 2213 |
vc.labelPos.binaryOffset = offset |
| 2214 |
if vc._var == nil { |
| 2215 |
lo, hi := vc.lo, vc.hi |
| 2216 |
m.c.Emit8Bytes(lo) |
| 2217 |
m.c.Emit8Bytes(hi) |
| 2218 |
} else { |
| 2219 |
for _, b := range vc._var { |
| 2220 |
m.c.EmitByte(b) |
| 2221 |
} |
| 2222 |
} |
| 2223 |
} |
| 2224 |
|
| 2225 |
buf := *bufPtr |
| 2226 |
for i := range m.labelResolutionPends { |
| 2227 |
p := &m.labelResolutionPends[i] |
| 2228 |
switch p.instr.kind { |
| 2229 |
case jmp, jmpIf, lea, xmmUnaryRmR: |
| 2230 |
target := p.instr.jmpLabel() |
| 2231 |
targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset |
| 2232 |
imm32Offset := p.imm32Offset |
| 2233 |
jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction. |
| 2234 |
binary.LittleEndian.PutUint32(buf[imm32Offset:], uint32(jmpOffset)) |
| 2235 |
case jmpTableIsland: |
| 2236 |
tableBegin := p.instrOffset |
| 2237 |
// Each entry is the offset from the beginning of the jmpTableIsland instruction in 8 bytes. |
| 2238 |
targets := m.jmpTableTargets[p.instr.u1] |
| 2239 |
for i, l := range targets { |
| 2240 |
targetOffset := m.labelPositionPool.Get(int(l)).binaryOffset |
| 2241 |
jmpOffset := targetOffset - tableBegin |
| 2242 |
binary.LittleEndian.PutUint64(buf[tableBegin+int64(i)*8:], uint64(jmpOffset)) |
| 2243 |
} |
| 2244 |
default: |
| 2245 |
panic("BUG") |
| 2246 |
} |
| 2247 |
} |
| 2248 |
return |
| 2249 |
} |
| 2250 |
|
| 2251 |
// ResolveRelocations implements backend.Machine. |
| 2252 |
func (m *machine) ResolveRelocations(refToBinaryOffset []int, _ int, binary []byte, relocations []backend.RelocationInfo, _ []int) { |
| 2253 |
for _, r := range relocations { |
| 2254 |
offset := r.Offset |
| 2255 |
calleeFnOffset := refToBinaryOffset[r.FuncRef] |
| 2256 |
// offset is the offset of the last 4 bytes of the call instruction. |
| 2257 |
callInstrOffsetBytes := binary[offset : offset+4] |
| 2258 |
diff := int64(calleeFnOffset) - (offset + 4) // +4 because we want the offset of the next instruction (In x64, RIP always points to the next instruction). |
| 2259 |
callInstrOffsetBytes[0] = byte(diff) |
| 2260 |
callInstrOffsetBytes[1] = byte(diff >> 8) |
| 2261 |
callInstrOffsetBytes[2] = byte(diff >> 16) |
| 2262 |
callInstrOffsetBytes[3] = byte(diff >> 24) |
| 2263 |
} |
| 2264 |
} |
| 2265 |
|
| 2266 |
// CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo. |
| 2267 |
func (m *machine) CallTrampolineIslandInfo(_ int) (_, _ int, _ error) { return } |
| 2268 |
|
| 2269 |
func (m *machine) lowerIcmpToFlag(xd, yd backend.SSAValueDefinition, _64 bool) { |
| 2270 |
x := m.getOperand_Reg(xd) |
| 2271 |
y := m.getOperand_Mem_Imm32_Reg(yd) |
| 2272 |
cmp := m.allocateInstr().asCmpRmiR(true, y, x.reg(), _64) |
| 2273 |
m.insert(cmp) |
| 2274 |
} |
| 2275 |
|
| 2276 |
func (m *machine) lowerFcmpToFlags(instr *ssa.Instruction) (f1, f2 cond, and bool) { |
| 2277 |
x, y, c := instr.FcmpData() |
| 2278 |
switch c { |
| 2279 |
case ssa.FloatCmpCondEqual: |
| 2280 |
f1, f2 = condNP, condZ |
| 2281 |
and = true |
| 2282 |
case ssa.FloatCmpCondNotEqual: |
| 2283 |
f1, f2 = condP, condNZ |
| 2284 |
case ssa.FloatCmpCondLessThan: |
| 2285 |
f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThan) |
| 2286 |
f2 = condInvalid |
| 2287 |
x, y = y, x |
| 2288 |
case ssa.FloatCmpCondLessThanOrEqual: |
| 2289 |
f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThanOrEqual) |
| 2290 |
f2 = condInvalid |
| 2291 |
x, y = y, x |
| 2292 |
default: |
| 2293 |
f1 = condFromSSAFloatCmpCond(c) |
| 2294 |
f2 = condInvalid |
| 2295 |
} |
| 2296 |
|
| 2297 |
var opc sseOpcode |
| 2298 |
if x.Type() == ssa.TypeF32 { |
| 2299 |
opc = sseOpcodeUcomiss |
| 2300 |
} else { |
| 2301 |
opc = sseOpcodeUcomisd |
| 2302 |
} |
| 2303 |
|
| 2304 |
xr := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 2305 |
yr := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 2306 |
m.insert(m.allocateInstr().asXmmCmpRmR(opc, yr, xr.reg())) |
| 2307 |
return |
| 2308 |
} |
| 2309 |
|
| 2310 |
// allocateInstr allocates an instruction. |
| 2311 |
func (m *machine) allocateInstr() *instruction { |
| 2312 |
instr := m.instrPool.Allocate() |
| 2313 |
if !m.regAllocStarted { |
| 2314 |
instr.addedBeforeRegAlloc = true |
| 2315 |
} |
| 2316 |
return instr |
| 2317 |
} |
| 2318 |
|
| 2319 |
func (m *machine) allocateNop() *instruction { |
| 2320 |
instr := m.allocateInstr() |
| 2321 |
instr.kind = nop0 |
| 2322 |
return instr |
| 2323 |
} |
| 2324 |
|
| 2325 |
func (m *machine) insert(i *instruction) { |
| 2326 |
m.pendingInstructions = append(m.pendingInstructions, i) |
| 2327 |
} |
| 2328 |
|
| 2329 |
func (m *machine) allocateBrTarget() (nop *instruction, l label) { //nolint |
| 2330 |
l, pos := m.allocateLabel() |
| 2331 |
nop = m.allocateInstr() |
| 2332 |
nop.asNop0WithLabel(l) |
| 2333 |
pos.begin, pos.end = nop, nop |
| 2334 |
return |
| 2335 |
} |
| 2336 |
|
| 2337 |
func (m *machine) allocateLabel() (label, *labelPosition) { |
| 2338 |
l := m.nextLabel |
| 2339 |
pos := m.labelPositionPool.GetOrAllocate(int(l)) |
| 2340 |
m.nextLabel++ |
| 2341 |
return l, pos |
| 2342 |
} |
| 2343 |
|
| 2344 |
func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 { |
| 2345 |
offset, ok := m.spillSlots[id] |
| 2346 |
if !ok { |
| 2347 |
offset = m.spillSlotSize |
| 2348 |
m.spillSlots[id] = offset |
| 2349 |
m.spillSlotSize += int64(size) |
| 2350 |
} |
| 2351 |
return offset |
| 2352 |
} |
| 2353 |
|
| 2354 |
func (m *machine) copyTo(src regalloc.VReg, dst regalloc.VReg) { |
| 2355 |
mov := m.allocateInstr() |
| 2356 |
if src.RegType() == regalloc.RegTypeInt { |
| 2357 |
mov.asMovRR(src, dst, true) |
| 2358 |
} else { |
| 2359 |
mov.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), dst) |
| 2360 |
} |
| 2361 |
m.insert(mov) |
| 2362 |
} |
| 2363 |
|
| 2364 |
func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg { |
| 2365 |
typ := m.c.TypeOf(v) |
| 2366 |
tmp := m.c.AllocateVReg(typ) |
| 2367 |
m.copyTo(v, tmp) |
| 2368 |
return tmp |
| 2369 |
} |
| 2370 |
|
| 2371 |
func (m *machine) requiredStackSize() int64 { |
| 2372 |
return m.maxRequiredStackSizeForCalls + |
| 2373 |
m.frameSize() + |
| 2374 |
16 + // Need for stack checking. |
| 2375 |
16 // return address and the caller RBP. |
| 2376 |
} |
| 2377 |
|
| 2378 |
func (m *machine) frameSize() int64 { |
| 2379 |
s := m.clobberedRegSlotSize() + m.spillSlotSize |
| 2380 |
if s&0xf != 0 { |
| 2381 |
panic(fmt.Errorf("BUG: frame size %d is not 16-byte aligned", s)) |
| 2382 |
} |
| 2383 |
return s |
| 2384 |
} |
| 2385 |
|
| 2386 |
func (m *machine) clobberedRegSlotSize() int64 { |
| 2387 |
return int64(len(m.clobberedRegs) * 16) |
| 2388 |
} |
| 2389 |
|
| 2390 |
func (m *machine) lowerIDivRem(si *ssa.Instruction, isDiv bool, signed bool) { |
| 2391 |
x, y, execCtx := si.Arg3() |
| 2392 |
|
| 2393 |
dividend := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 2394 |
divisor := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 2395 |
ctxVReg := m.c.VRegOf(execCtx) |
| 2396 |
tmpGp := m.c.AllocateVReg(si.Return().Type()) |
| 2397 |
|
| 2398 |
m.copyTo(dividend.reg(), raxVReg) |
| 2399 |
m.insert(m.allocateInstr().asDefineUninitializedReg(rdxVReg)) |
| 2400 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) |
| 2401 |
seq := m.allocateInstr().asIdivRemSequence(ctxVReg, divisor.reg(), tmpGp, isDiv, signed, x.Type().Bits() == 64) |
| 2402 |
m.insert(seq) |
| 2403 |
rd := m.c.VRegOf(si.Return()) |
| 2404 |
if isDiv { |
| 2405 |
m.copyTo(raxVReg, rd) |
| 2406 |
} else { |
| 2407 |
m.copyTo(rdxVReg, rd) |
| 2408 |
} |
| 2409 |
} |
| 2410 |
|
| 2411 |
func (m *machine) lowerIDivRemSequenceAfterRegAlloc(i *instruction) { |
| 2412 |
execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData() |
| 2413 |
|
| 2414 |
dividend := raxVReg |
| 2415 |
|
| 2416 |
// Ensure yr is not zero. |
| 2417 |
test := m.allocateInstr() |
| 2418 |
test.asCmpRmiR(false, newOperandReg(divisor), divisor, _64) |
| 2419 |
m.insert(test) |
| 2420 |
|
| 2421 |
jnz := m.allocateInstr() |
| 2422 |
m.insert(jnz) |
| 2423 |
|
| 2424 |
nz := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerDivisionByZero) |
| 2425 |
|
| 2426 |
// If not zero, we can proceed with the division. |
| 2427 |
jnz.asJmpIf(condNZ, newOperandLabel(nz)) |
| 2428 |
|
| 2429 |
var ifRemNeg1 *instruction |
| 2430 |
if signed { |
| 2431 |
var neg1 uint64 |
| 2432 |
if _64 { |
| 2433 |
neg1 = 0xffffffffffffffff |
| 2434 |
} else { |
| 2435 |
neg1 = 0xffffffff |
| 2436 |
} |
| 2437 |
m.lowerIconst(tmpGp, neg1, _64) |
| 2438 |
|
| 2439 |
if isDiv { |
| 2440 |
// For signed division, we have to have branches for "math.MinInt{32,64} / -1" |
| 2441 |
// case which results in the floating point exception via division error as |
| 2442 |
// the resulting value exceeds the maximum of signed int. |
| 2443 |
|
| 2444 |
// First, we check if the divisor is -1. |
| 2445 |
cmp := m.allocateInstr() |
| 2446 |
cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64) |
| 2447 |
m.insert(cmp) |
| 2448 |
|
| 2449 |
ifNotNeg1 := m.allocateInstr() |
| 2450 |
m.insert(ifNotNeg1) |
| 2451 |
|
| 2452 |
var minInt uint64 |
| 2453 |
if _64 { |
| 2454 |
minInt = 0x8000000000000000 |
| 2455 |
} else { |
| 2456 |
minInt = 0x80000000 |
| 2457 |
} |
| 2458 |
m.lowerIconst(tmpGp, minInt, _64) |
| 2459 |
|
| 2460 |
// Next we check if the quotient is the most negative value for the signed integer, i.e. |
| 2461 |
// if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1) respectively. |
| 2462 |
cmp2 := m.allocateInstr() |
| 2463 |
cmp2.asCmpRmiR(true, newOperandReg(tmpGp), dividend, _64) |
| 2464 |
m.insert(cmp2) |
| 2465 |
|
| 2466 |
ifNotMinInt := m.allocateInstr() |
| 2467 |
m.insert(ifNotMinInt) |
| 2468 |
|
| 2469 |
// Trap if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1), |
| 2470 |
// as that is the overflow in division as the result becomes 2^31 which is larger than |
| 2471 |
// the maximum of signed 32-bit int (2^31-1). |
| 2472 |
end := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) |
| 2473 |
ifNotNeg1.asJmpIf(condNZ, newOperandLabel(end)) |
| 2474 |
ifNotMinInt.asJmpIf(condNZ, newOperandLabel(end)) |
| 2475 |
} else { |
| 2476 |
// If it is remainder, zeros DX register and compare the divisor to -1. |
| 2477 |
xor := m.allocateInstr().asZeros(rdxVReg) |
| 2478 |
m.insert(xor) |
| 2479 |
|
| 2480 |
// We check if the divisor is -1. |
| 2481 |
cmp := m.allocateInstr() |
| 2482 |
cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64) |
| 2483 |
m.insert(cmp) |
| 2484 |
|
| 2485 |
ifRemNeg1 = m.allocateInstr() |
| 2486 |
m.insert(ifRemNeg1) |
| 2487 |
} |
| 2488 |
|
| 2489 |
// Sign-extend DX register to have 2*x.Type().Bits() dividend over DX and AX registers. |
| 2490 |
sed := m.allocateInstr() |
| 2491 |
sed.asSignExtendData(_64) |
| 2492 |
m.insert(sed) |
| 2493 |
} else { |
| 2494 |
// Zeros DX register to have 2*x.Type().Bits() dividend over DX and AX registers. |
| 2495 |
zeros := m.allocateInstr().asZeros(rdxVReg) |
| 2496 |
m.insert(zeros) |
| 2497 |
} |
| 2498 |
|
| 2499 |
div := m.allocateInstr() |
| 2500 |
div.asDiv(newOperandReg(divisor), signed, _64) |
| 2501 |
m.insert(div) |
| 2502 |
|
| 2503 |
nop, end := m.allocateBrTarget() |
| 2504 |
m.insert(nop) |
| 2505 |
// If we are compiling a Rem instruction, when the divisor is -1 we land at the end of the function. |
| 2506 |
if ifRemNeg1 != nil { |
| 2507 |
ifRemNeg1.asJmpIf(condZ, newOperandLabel(end)) |
| 2508 |
} |
| 2509 |
} |
| 2510 |
|
| 2511 |
func (m *machine) lowerRound(instr *ssa.Instruction, imm roundingMode) { |
| 2512 |
x := instr.Arg() |
| 2513 |
if !x.Type().IsFloat() { |
| 2514 |
panic("BUG?") |
| 2515 |
} |
| 2516 |
var op sseOpcode |
| 2517 |
if x.Type().Bits() == 64 { |
| 2518 |
op = sseOpcodeRoundsd |
| 2519 |
} else { |
| 2520 |
op = sseOpcodeRoundss |
| 2521 |
} |
| 2522 |
|
| 2523 |
xDef := m.c.ValueDefinition(x) |
| 2524 |
rm := m.getOperand_Mem_Reg(xDef) |
| 2525 |
rd := m.c.VRegOf(instr.Return()) |
| 2526 |
|
| 2527 |
xmm := m.allocateInstr().asXmmUnaryRmRImm(op, uint8(imm), rm, rd) |
| 2528 |
m.insert(xmm) |
| 2529 |
} |
| 2530 |
|
| 2531 |
func (m *machine) lowerFminFmax(instr *ssa.Instruction) { |
| 2532 |
x, y := instr.Arg2() |
| 2533 |
if !x.Type().IsFloat() { |
| 2534 |
panic("BUG?") |
| 2535 |
} |
| 2536 |
|
| 2537 |
_64 := x.Type().Bits() == 64 |
| 2538 |
isMin := instr.Opcode() == ssa.OpcodeFmin |
| 2539 |
var minMaxOp sseOpcode |
| 2540 |
|
| 2541 |
switch { |
| 2542 |
case _64 && isMin: |
| 2543 |
minMaxOp = sseOpcodeMinpd |
| 2544 |
case _64 && !isMin: |
| 2545 |
minMaxOp = sseOpcodeMaxpd |
| 2546 |
case !_64 && isMin: |
| 2547 |
minMaxOp = sseOpcodeMinps |
| 2548 |
case !_64 && !isMin: |
| 2549 |
minMaxOp = sseOpcodeMaxps |
| 2550 |
} |
| 2551 |
|
| 2552 |
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 2553 |
rm := m.getOperand_Reg(xDef) |
| 2554 |
// We cannot ensure that y is aligned to 16 bytes, so we have to use it on reg. |
| 2555 |
rn := m.getOperand_Reg(yDef) |
| 2556 |
rd := m.c.VRegOf(instr.Return()) |
| 2557 |
|
| 2558 |
tmp := m.copyToTmp(rm.reg()) |
| 2559 |
|
| 2560 |
// Check if this is (either x1 or x2 is NaN) or (x1 equals x2) case. |
| 2561 |
cmp := m.allocateInstr() |
| 2562 |
if _64 { |
| 2563 |
cmp.asXmmCmpRmR(sseOpcodeUcomisd, rn, tmp) |
| 2564 |
} else { |
| 2565 |
cmp.asXmmCmpRmR(sseOpcodeUcomiss, rn, tmp) |
| 2566 |
} |
| 2567 |
m.insert(cmp) |
| 2568 |
|
| 2569 |
// At this point, we have the three cases of conditional flags below |
| 2570 |
// (See https://www.felixcloutier.com/x86/ucomiss#operation for detail.) |
| 2571 |
// |
| 2572 |
// 1) Two values are NaN-free and different: All flags are cleared. |
| 2573 |
// 2) Two values are NaN-free and equal: Only ZF flags is set. |
| 2574 |
// 3) One of Two values is NaN: ZF, PF and CF flags are set. |
| 2575 |
|
| 2576 |
// Jump instruction to handle 1) case by checking the ZF flag |
| 2577 |
// as ZF is only set for 2) and 3) cases. |
| 2578 |
nanFreeOrDiffJump := m.allocateInstr() |
| 2579 |
m.insert(nanFreeOrDiffJump) |
| 2580 |
|
| 2581 |
// Start handling 2) and 3). |
| 2582 |
|
| 2583 |
// Jump if one of two values is NaN by checking the parity flag (PF). |
| 2584 |
ifIsNan := m.allocateInstr() |
| 2585 |
m.insert(ifIsNan) |
| 2586 |
|
| 2587 |
// Start handling 2) NaN-free and equal. |
| 2588 |
|
| 2589 |
// Before we exit this case, we have to ensure that positive zero (or negative zero for min instruction) is |
| 2590 |
// returned if two values are positive and negative zeros. |
| 2591 |
var op sseOpcode |
| 2592 |
switch { |
| 2593 |
case !_64 && isMin: |
| 2594 |
op = sseOpcodeOrps |
| 2595 |
case _64 && isMin: |
| 2596 |
op = sseOpcodeOrpd |
| 2597 |
case !_64 && !isMin: |
| 2598 |
op = sseOpcodeAndps |
| 2599 |
case _64 && !isMin: |
| 2600 |
op = sseOpcodeAndpd |
| 2601 |
} |
| 2602 |
orAnd := m.allocateInstr() |
| 2603 |
orAnd.asXmmRmR(op, rn, tmp) |
| 2604 |
m.insert(orAnd) |
| 2605 |
|
| 2606 |
// Done, jump to end. |
| 2607 |
sameExitJump := m.allocateInstr() |
| 2608 |
m.insert(sameExitJump) |
| 2609 |
|
| 2610 |
// Start handling 3) either is NaN. |
| 2611 |
isNanTarget, isNan := m.allocateBrTarget() |
| 2612 |
m.insert(isNanTarget) |
| 2613 |
ifIsNan.asJmpIf(condP, newOperandLabel(isNan)) |
| 2614 |
|
| 2615 |
// We emit the ADD instruction to produce the NaN in tmp. |
| 2616 |
add := m.allocateInstr() |
| 2617 |
if _64 { |
| 2618 |
add.asXmmRmR(sseOpcodeAddsd, rn, tmp) |
| 2619 |
} else { |
| 2620 |
add.asXmmRmR(sseOpcodeAddss, rn, tmp) |
| 2621 |
} |
| 2622 |
m.insert(add) |
| 2623 |
|
| 2624 |
// Exit from the NaN case branch. |
| 2625 |
nanExitJmp := m.allocateInstr() |
| 2626 |
m.insert(nanExitJmp) |
| 2627 |
|
| 2628 |
// Start handling 1). |
| 2629 |
doMinMaxTarget, doMinMax := m.allocateBrTarget() |
| 2630 |
m.insert(doMinMaxTarget) |
| 2631 |
nanFreeOrDiffJump.asJmpIf(condNZ, newOperandLabel(doMinMax)) |
| 2632 |
|
| 2633 |
// Now handle the NaN-free and different values case. |
| 2634 |
minMax := m.allocateInstr() |
| 2635 |
minMax.asXmmRmR(minMaxOp, rn, tmp) |
| 2636 |
m.insert(minMax) |
| 2637 |
|
| 2638 |
endNop, end := m.allocateBrTarget() |
| 2639 |
m.insert(endNop) |
| 2640 |
nanExitJmp.asJmp(newOperandLabel(end)) |
| 2641 |
sameExitJump.asJmp(newOperandLabel(end)) |
| 2642 |
|
| 2643 |
m.copyTo(tmp, rd) |
| 2644 |
} |
| 2645 |
|
| 2646 |
func (m *machine) lowerFcopysign(instr *ssa.Instruction) { |
| 2647 |
x, y := instr.Arg2() |
| 2648 |
if !x.Type().IsFloat() { |
| 2649 |
panic("BUG") |
| 2650 |
} |
| 2651 |
|
| 2652 |
_64 := x.Type().Bits() == 64 |
| 2653 |
|
| 2654 |
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 2655 |
rm := m.getOperand_Reg(xDef) |
| 2656 |
rn := m.getOperand_Reg(yDef) |
| 2657 |
rd := m.c.VRegOf(instr.Return()) |
| 2658 |
|
| 2659 |
// Clear the non-sign bits of src via AND with the mask. |
| 2660 |
var opAnd, opOr sseOpcode |
| 2661 |
var signMask uint64 |
| 2662 |
if _64 { |
| 2663 |
signMask, opAnd, opOr = 0x8000000000000000, sseOpcodeAndpd, sseOpcodeOrpd |
| 2664 |
} else { |
| 2665 |
signMask, opAnd, opOr = 0x80000000, sseOpcodeAndps, sseOpcodeOrps |
| 2666 |
} |
| 2667 |
|
| 2668 |
signBitReg := m.c.AllocateVReg(x.Type()) |
| 2669 |
m.lowerFconst(signBitReg, signMask, _64) |
| 2670 |
nonSignBitReg := m.c.AllocateVReg(x.Type()) |
| 2671 |
m.lowerFconst(nonSignBitReg, ^signMask, _64) |
| 2672 |
|
| 2673 |
// Extract the sign bits of rn. |
| 2674 |
and := m.allocateInstr().asXmmRmR(opAnd, rn, signBitReg) |
| 2675 |
m.insert(and) |
| 2676 |
|
| 2677 |
// Clear the sign bit of dst via AND with the non-sign bit mask. |
| 2678 |
xor := m.allocateInstr().asXmmRmR(opAnd, rm, nonSignBitReg) |
| 2679 |
m.insert(xor) |
| 2680 |
|
| 2681 |
// Copy the sign bits of src to dst via OR. |
| 2682 |
or := m.allocateInstr().asXmmRmR(opOr, newOperandReg(signBitReg), nonSignBitReg) |
| 2683 |
m.insert(or) |
| 2684 |
|
| 2685 |
m.copyTo(nonSignBitReg, rd) |
| 2686 |
} |
| 2687 |
|
| 2688 |
func (m *machine) lowerBitcast(instr *ssa.Instruction) { |
| 2689 |
x, dstTyp := instr.BitcastData() |
| 2690 |
srcTyp := x.Type() |
| 2691 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 2692 |
rd := m.c.VRegOf(instr.Return()) |
| 2693 |
switch { |
| 2694 |
case srcTyp == ssa.TypeF32 && dstTyp == ssa.TypeI32: |
| 2695 |
cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovd, rn.reg(), rd, false) |
| 2696 |
m.insert(cvt) |
| 2697 |
case srcTyp == ssa.TypeI32 && dstTyp == ssa.TypeF32: |
| 2698 |
cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovd, rn, rd, false) |
| 2699 |
m.insert(cvt) |
| 2700 |
case srcTyp == ssa.TypeF64 && dstTyp == ssa.TypeI64: |
| 2701 |
cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovq, rn.reg(), rd, true) |
| 2702 |
m.insert(cvt) |
| 2703 |
case srcTyp == ssa.TypeI64 && dstTyp == ssa.TypeF64: |
| 2704 |
cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovq, rn, rd, true) |
| 2705 |
m.insert(cvt) |
| 2706 |
default: |
| 2707 |
panic(fmt.Sprintf("invalid bitcast from %s to %s", srcTyp, dstTyp)) |
| 2708 |
} |
| 2709 |
} |
| 2710 |
|
| 2711 |
func (m *machine) lowerFcvtToSint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) { |
| 2712 |
var tmpXmm regalloc.VReg |
| 2713 |
if dst64 { |
| 2714 |
tmpXmm = m.c.AllocateVReg(ssa.TypeF64) |
| 2715 |
} else { |
| 2716 |
tmpXmm = m.c.AllocateVReg(ssa.TypeF32) |
| 2717 |
} |
| 2718 |
|
| 2719 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm)) |
| 2720 |
tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64) |
| 2721 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) |
| 2722 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2)) |
| 2723 |
|
| 2724 |
m.insert(m.allocateFcvtToSintSequence(ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat)) |
| 2725 |
m.copyTo(tmpGp, rd) |
| 2726 |
} |
| 2727 |
|
| 2728 |
func (m *machine) lowerFcvtToSintSequenceAfterRegalloc(i *instruction) { |
| 2729 |
execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData() |
| 2730 |
var cmpOp, truncOp sseOpcode |
| 2731 |
if src64 { |
| 2732 |
cmpOp, truncOp = sseOpcodeUcomisd, sseOpcodeCvttsd2si |
| 2733 |
} else { |
| 2734 |
cmpOp, truncOp = sseOpcodeUcomiss, sseOpcodeCvttss2si |
| 2735 |
} |
| 2736 |
|
| 2737 |
trunc := m.allocateInstr() |
| 2738 |
trunc.asXmmToGpr(truncOp, src, tmpGp, dst64) |
| 2739 |
m.insert(trunc) |
| 2740 |
|
| 2741 |
// Check if the dst operand was INT_MIN, by checking it against 1. |
| 2742 |
cmp1 := m.allocateInstr() |
| 2743 |
cmp1.asCmpRmiR(true, newOperandImm32(1), tmpGp, dst64) |
| 2744 |
m.insert(cmp1) |
| 2745 |
|
| 2746 |
// If no overflow, then we are done. |
| 2747 |
doneTarget, done := m.allocateBrTarget() |
| 2748 |
ifNoOverflow := m.allocateInstr() |
| 2749 |
ifNoOverflow.asJmpIf(condNO, newOperandLabel(done)) |
| 2750 |
m.insert(ifNoOverflow) |
| 2751 |
|
| 2752 |
// Now, check for NaN. |
| 2753 |
cmpNan := m.allocateInstr() |
| 2754 |
cmpNan.asXmmCmpRmR(cmpOp, newOperandReg(src), src) |
| 2755 |
m.insert(cmpNan) |
| 2756 |
|
| 2757 |
// We allocate the "non-nan target" here, but we will insert it later. |
| 2758 |
notNanTarget, notNaN := m.allocateBrTarget() |
| 2759 |
ifNotNan := m.allocateInstr() |
| 2760 |
ifNotNan.asJmpIf(condNP, newOperandLabel(notNaN)) |
| 2761 |
m.insert(ifNotNan) |
| 2762 |
|
| 2763 |
if sat { |
| 2764 |
// If NaN and saturating, return 0. |
| 2765 |
zeroDst := m.allocateInstr().asZeros(tmpGp) |
| 2766 |
m.insert(zeroDst) |
| 2767 |
|
| 2768 |
jmpEnd := m.allocateInstr() |
| 2769 |
jmpEnd.asJmp(newOperandLabel(done)) |
| 2770 |
m.insert(jmpEnd) |
| 2771 |
|
| 2772 |
// Otherwise: |
| 2773 |
m.insert(notNanTarget) |
| 2774 |
|
| 2775 |
// Zero-out the tmp register. |
| 2776 |
zero := m.allocateInstr().asZeros(tmpXmm) |
| 2777 |
m.insert(zero) |
| 2778 |
|
| 2779 |
cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src) |
| 2780 |
m.insert(cmpXmm) |
| 2781 |
|
| 2782 |
// if >= jump to end. |
| 2783 |
jmpEnd2 := m.allocateInstr() |
| 2784 |
jmpEnd2.asJmpIf(condB, newOperandLabel(done)) |
| 2785 |
m.insert(jmpEnd2) |
| 2786 |
|
| 2787 |
// Otherwise, saturate to INT_MAX. |
| 2788 |
if dst64 { |
| 2789 |
m.lowerIconst(tmpGp, math.MaxInt64, dst64) |
| 2790 |
} else { |
| 2791 |
m.lowerIconst(tmpGp, math.MaxInt32, dst64) |
| 2792 |
} |
| 2793 |
|
| 2794 |
} else { |
| 2795 |
|
| 2796 |
// If non-sat, NaN, trap. |
| 2797 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger) |
| 2798 |
|
| 2799 |
// Otherwise, we will jump here. |
| 2800 |
m.insert(notNanTarget) |
| 2801 |
|
| 2802 |
// jump over trap if src larger than threshold |
| 2803 |
condAboveThreshold := condNB |
| 2804 |
|
| 2805 |
// The magic constants are various combination of minInt for int[32|64] represented as float[32|64]. |
| 2806 |
var minInt uint64 |
| 2807 |
switch { |
| 2808 |
case src64 && dst64: |
| 2809 |
minInt = 0xc3e0000000000000 |
| 2810 |
case src64 && !dst64: |
| 2811 |
condAboveThreshold = condNBE |
| 2812 |
minInt = 0xC1E0_0000_0020_0000 |
| 2813 |
case !src64 && dst64: |
| 2814 |
minInt = 0xDF00_0000 |
| 2815 |
case !src64 && !dst64: |
| 2816 |
minInt = 0xCF00_0000 |
| 2817 |
} |
| 2818 |
|
| 2819 |
loadToGP := m.allocateInstr().asImm(tmpGp2, minInt, src64) |
| 2820 |
m.insert(loadToGP) |
| 2821 |
|
| 2822 |
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp2), tmpXmm, src64) |
| 2823 |
m.insert(movToXmm) |
| 2824 |
|
| 2825 |
cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src) |
| 2826 |
m.insert(cmpXmm) |
| 2827 |
|
| 2828 |
jmpIfLarger := m.allocateInstr() |
| 2829 |
checkPositiveTarget, checkPositive := m.allocateBrTarget() |
| 2830 |
jmpIfLarger.asJmpIf(condAboveThreshold, newOperandLabel(checkPositive)) |
| 2831 |
m.insert(jmpIfLarger) |
| 2832 |
|
| 2833 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) |
| 2834 |
|
| 2835 |
// If positive, it was a real overflow. |
| 2836 |
m.insert(checkPositiveTarget) |
| 2837 |
|
| 2838 |
// Zero out the temp register. |
| 2839 |
xorpd := m.allocateInstr() |
| 2840 |
xorpd.asXmmRmR(sseOpcodeXorpd, newOperandReg(tmpXmm), tmpXmm) |
| 2841 |
m.insert(xorpd) |
| 2842 |
|
| 2843 |
pos := m.allocateInstr() |
| 2844 |
pos.asXmmCmpRmR(cmpOp, newOperandReg(src), tmpXmm) |
| 2845 |
m.insert(pos) |
| 2846 |
|
| 2847 |
// If >= jump to end. |
| 2848 |
jmp := m.allocateInstr().asJmpIf(condNB, newOperandLabel(done)) |
| 2849 |
m.insert(jmp) |
| 2850 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) |
| 2851 |
} |
| 2852 |
|
| 2853 |
m.insert(doneTarget) |
| 2854 |
} |
| 2855 |
|
| 2856 |
func (m *machine) lowerFcvtToUint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) { |
| 2857 |
tmpXmm, tmpXmm2 := m.c.AllocateVReg(ssa.TypeF64), m.c.AllocateVReg(ssa.TypeF64) |
| 2858 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm)) |
| 2859 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm2)) |
| 2860 |
tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64) |
| 2861 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) |
| 2862 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2)) |
| 2863 |
|
| 2864 |
m.insert(m.allocateFcvtToUintSequence( |
| 2865 |
ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat, |
| 2866 |
)) |
| 2867 |
m.copyTo(tmpGp, rd) |
| 2868 |
} |
| 2869 |
|
| 2870 |
func (m *machine) lowerFcvtToUintSequenceAfterRegalloc(i *instruction) { |
| 2871 |
execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData() |
| 2872 |
|
| 2873 |
var subOp, cmpOp, truncOp sseOpcode |
| 2874 |
if src64 { |
| 2875 |
subOp, cmpOp, truncOp = sseOpcodeSubsd, sseOpcodeUcomisd, sseOpcodeCvttsd2si |
| 2876 |
} else { |
| 2877 |
subOp, cmpOp, truncOp = sseOpcodeSubss, sseOpcodeUcomiss, sseOpcodeCvttss2si |
| 2878 |
} |
| 2879 |
|
| 2880 |
doneTarget, done := m.allocateBrTarget() |
| 2881 |
|
| 2882 |
switch { |
| 2883 |
case src64 && dst64: |
| 2884 |
loadToGP := m.allocateInstr().asImm(tmpGp, 0x43e0000000000000, true) |
| 2885 |
m.insert(loadToGP) |
| 2886 |
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true) |
| 2887 |
m.insert(movToXmm) |
| 2888 |
case src64 && !dst64: |
| 2889 |
loadToGP := m.allocateInstr().asImm(tmpGp, 0x41e0000000000000, true) |
| 2890 |
m.insert(loadToGP) |
| 2891 |
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true) |
| 2892 |
m.insert(movToXmm) |
| 2893 |
case !src64 && dst64: |
| 2894 |
loadToGP := m.allocateInstr().asImm(tmpGp, 0x5f000000, false) |
| 2895 |
m.insert(loadToGP) |
| 2896 |
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false) |
| 2897 |
m.insert(movToXmm) |
| 2898 |
case !src64 && !dst64: |
| 2899 |
loadToGP := m.allocateInstr().asImm(tmpGp, 0x4f000000, false) |
| 2900 |
m.insert(loadToGP) |
| 2901 |
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false) |
| 2902 |
m.insert(movToXmm) |
| 2903 |
} |
| 2904 |
|
| 2905 |
cmp := m.allocateInstr() |
| 2906 |
cmp.asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src) |
| 2907 |
m.insert(cmp) |
| 2908 |
|
| 2909 |
// If above `tmp` ("large threshold"), jump to `ifAboveThreshold` |
| 2910 |
ifAboveThresholdTarget, ifAboveThreshold := m.allocateBrTarget() |
| 2911 |
jmpIfAboveThreshold := m.allocateInstr() |
| 2912 |
jmpIfAboveThreshold.asJmpIf(condNB, newOperandLabel(ifAboveThreshold)) |
| 2913 |
m.insert(jmpIfAboveThreshold) |
| 2914 |
|
| 2915 |
ifNotNaNTarget, ifNotNaN := m.allocateBrTarget() |
| 2916 |
jmpIfNotNaN := m.allocateInstr() |
| 2917 |
jmpIfNotNaN.asJmpIf(condNP, newOperandLabel(ifNotNaN)) |
| 2918 |
m.insert(jmpIfNotNaN) |
| 2919 |
|
| 2920 |
// If NaN, handle the error condition. |
| 2921 |
if sat { |
| 2922 |
// On NaN, saturating, we just return 0. |
| 2923 |
zeros := m.allocateInstr().asZeros(tmpGp) |
| 2924 |
m.insert(zeros) |
| 2925 |
|
| 2926 |
jmpEnd := m.allocateInstr() |
| 2927 |
jmpEnd.asJmp(newOperandLabel(done)) |
| 2928 |
m.insert(jmpEnd) |
| 2929 |
} else { |
| 2930 |
// On NaN, non-saturating, we trap. |
| 2931 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger) |
| 2932 |
} |
| 2933 |
|
| 2934 |
// If not NaN, land here. |
| 2935 |
m.insert(ifNotNaNTarget) |
| 2936 |
|
| 2937 |
// Truncation happens here. |
| 2938 |
|
| 2939 |
trunc := m.allocateInstr() |
| 2940 |
trunc.asXmmToGpr(truncOp, src, tmpGp, dst64) |
| 2941 |
m.insert(trunc) |
| 2942 |
|
| 2943 |
// Check if the result is negative. |
| 2944 |
cmpNeg := m.allocateInstr() |
| 2945 |
cmpNeg.asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64) |
| 2946 |
m.insert(cmpNeg) |
| 2947 |
|
| 2948 |
// If non-neg, jump to end. |
| 2949 |
jmpIfNonNeg := m.allocateInstr() |
| 2950 |
jmpIfNonNeg.asJmpIf(condNL, newOperandLabel(done)) |
| 2951 |
m.insert(jmpIfNonNeg) |
| 2952 |
|
| 2953 |
if sat { |
| 2954 |
// If the input was "small" (< 2**(width -1)), the only way to get an integer |
| 2955 |
// overflow is because the input was too small: saturate to the min value, i.e. 0. |
| 2956 |
zeros := m.allocateInstr().asZeros(tmpGp) |
| 2957 |
m.insert(zeros) |
| 2958 |
|
| 2959 |
jmpEnd := m.allocateInstr() |
| 2960 |
jmpEnd.asJmp(newOperandLabel(done)) |
| 2961 |
m.insert(jmpEnd) |
| 2962 |
} else { |
| 2963 |
// If not saturating, trap. |
| 2964 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) |
| 2965 |
} |
| 2966 |
|
| 2967 |
// If above the threshold, land here. |
| 2968 |
m.insert(ifAboveThresholdTarget) |
| 2969 |
|
| 2970 |
// tmpDiff := threshold - rn. |
| 2971 |
copySrc := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), tmpXmm2) |
| 2972 |
m.insert(copySrc) |
| 2973 |
|
| 2974 |
sub := m.allocateInstr() |
| 2975 |
sub.asXmmRmR(subOp, newOperandReg(tmpXmm), tmpXmm2) // must be -0x8000000000000000 |
| 2976 |
m.insert(sub) |
| 2977 |
|
| 2978 |
trunc2 := m.allocateInstr() |
| 2979 |
trunc2.asXmmToGpr(truncOp, tmpXmm2, tmpGp, dst64) |
| 2980 |
m.insert(trunc2) |
| 2981 |
|
| 2982 |
// Check if the result is negative. |
| 2983 |
cmpNeg2 := m.allocateInstr().asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64) |
| 2984 |
m.insert(cmpNeg2) |
| 2985 |
|
| 2986 |
ifNextLargeTarget, ifNextLarge := m.allocateBrTarget() |
| 2987 |
jmpIfNextLarge := m.allocateInstr() |
| 2988 |
jmpIfNextLarge.asJmpIf(condNL, newOperandLabel(ifNextLarge)) |
| 2989 |
m.insert(jmpIfNextLarge) |
| 2990 |
|
| 2991 |
if sat { |
| 2992 |
// The input was "large" (>= maxInt), so the only way to get an integer |
| 2993 |
// overflow is because the input was too large: saturate to the max value. |
| 2994 |
var maxInt uint64 |
| 2995 |
if dst64 { |
| 2996 |
maxInt = math.MaxUint64 |
| 2997 |
} else { |
| 2998 |
maxInt = math.MaxUint32 |
| 2999 |
} |
| 3000 |
m.lowerIconst(tmpGp, maxInt, dst64) |
| 3001 |
|
| 3002 |
jmpToEnd := m.allocateInstr() |
| 3003 |
jmpToEnd.asJmp(newOperandLabel(done)) |
| 3004 |
m.insert(jmpToEnd) |
| 3005 |
} else { |
| 3006 |
// If not saturating, trap. |
| 3007 |
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) |
| 3008 |
} |
| 3009 |
|
| 3010 |
m.insert(ifNextLargeTarget) |
| 3011 |
|
| 3012 |
var op operand |
| 3013 |
if dst64 { |
| 3014 |
m.lowerIconst(tmpGp2, 0x8000000000000000, true) |
| 3015 |
op = newOperandReg(tmpGp2) |
| 3016 |
} else { |
| 3017 |
op = newOperandImm32(0x80000000) |
| 3018 |
} |
| 3019 |
|
| 3020 |
add := m.allocateInstr() |
| 3021 |
add.asAluRmiR(aluRmiROpcodeAdd, op, tmpGp, dst64) |
| 3022 |
m.insert(add) |
| 3023 |
|
| 3024 |
m.insert(doneTarget) |
| 3025 |
} |
| 3026 |
|
| 3027 |
func (m *machine) lowerFcvtFromSint(rn, rd operand, src64, dst64 bool) { |
| 3028 |
var op sseOpcode |
| 3029 |
if dst64 { |
| 3030 |
op = sseOpcodeCvtsi2sd |
| 3031 |
} else { |
| 3032 |
op = sseOpcodeCvtsi2ss |
| 3033 |
} |
| 3034 |
|
| 3035 |
trunc := m.allocateInstr() |
| 3036 |
trunc.asGprToXmm(op, rn, rd.reg(), src64) |
| 3037 |
m.insert(trunc) |
| 3038 |
} |
| 3039 |
|
| 3040 |
func (m *machine) lowerFcvtFromUint(rn, rd operand, src64, dst64 bool) { |
| 3041 |
var op sseOpcode |
| 3042 |
if dst64 { |
| 3043 |
op = sseOpcodeCvtsi2sd |
| 3044 |
} else { |
| 3045 |
op = sseOpcodeCvtsi2ss |
| 3046 |
} |
| 3047 |
|
| 3048 |
// Src is 32 bit, then we just perform the conversion with 64 bit width. |
| 3049 |
// |
| 3050 |
// See the following link for why we use 64bit conversion for unsigned 32bit integer sources: |
| 3051 |
// https://stackoverflow.com/questions/41495498/fpu-operations-generated-by-gcc-during-casting-integer-to-float. |
| 3052 |
// |
| 3053 |
// Here's the summary: |
| 3054 |
// >> CVTSI2SS is indeed designed for converting a signed integer to a scalar single-precision float, |
| 3055 |
// >> not an unsigned integer like you have here. So what gives? Well, a 64-bit processor has 64-bit wide |
| 3056 |
// >> registers available, so the unsigned 32-bit input values can be stored as signed 64-bit intermediate values, |
| 3057 |
// >> which allows CVTSI2SS to be used after all. |
| 3058 |
// |
| 3059 |
if !src64 { |
| 3060 |
// Before we convert, we have to clear the higher 32-bits of the 64-bit register |
| 3061 |
// to get the correct result. |
| 3062 |
tmp := m.c.AllocateVReg(ssa.TypeI32) |
| 3063 |
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, tmp)) |
| 3064 |
m.insert(m.allocateInstr().asGprToXmm(op, newOperandReg(tmp), rd.reg(), true)) |
| 3065 |
return |
| 3066 |
} |
| 3067 |
|
| 3068 |
// If uint64, we have to do a bit more work. |
| 3069 |
endTarget, end := m.allocateBrTarget() |
| 3070 |
|
| 3071 |
var tmpXmm regalloc.VReg |
| 3072 |
if dst64 { |
| 3073 |
tmpXmm = m.c.AllocateVReg(ssa.TypeF64) |
| 3074 |
} else { |
| 3075 |
tmpXmm = m.c.AllocateVReg(ssa.TypeF32) |
| 3076 |
} |
| 3077 |
|
| 3078 |
// Check if the most significant bit (sign bit) is set. |
| 3079 |
test := m.allocateInstr() |
| 3080 |
test.asCmpRmiR(false, rn, rn.reg(), src64) |
| 3081 |
m.insert(test) |
| 3082 |
|
| 3083 |
// Jump if the sign bit is set. |
| 3084 |
ifSignTarget, ifSign := m.allocateBrTarget() |
| 3085 |
jmpIfNeg := m.allocateInstr() |
| 3086 |
jmpIfNeg.asJmpIf(condS, newOperandLabel(ifSign)) |
| 3087 |
m.insert(jmpIfNeg) |
| 3088 |
|
| 3089 |
// If the sign bit is not set, we could fit the unsigned int into float32/float64. |
| 3090 |
// So, we convert it to float and emit jump instruction to exit from this branch. |
| 3091 |
cvt := m.allocateInstr() |
| 3092 |
cvt.asGprToXmm(op, rn, tmpXmm, src64) |
| 3093 |
m.insert(cvt) |
| 3094 |
|
| 3095 |
// We are done, jump to end. |
| 3096 |
jmpEnd := m.allocateInstr() |
| 3097 |
jmpEnd.asJmp(newOperandLabel(end)) |
| 3098 |
m.insert(jmpEnd) |
| 3099 |
|
| 3100 |
// Now handling the case where sign-bit is set. |
| 3101 |
// We emit the following sequences: |
| 3102 |
// mov %rn, %tmp |
| 3103 |
// shr 1, %tmp |
| 3104 |
// mov %rn, %tmp2 |
| 3105 |
// and 1, %tmp2 |
| 3106 |
// or %tmp2, %tmp |
| 3107 |
// cvtsi2ss %tmp, %xmm0 |
| 3108 |
// addsd %xmm0, %xmm0 |
| 3109 |
m.insert(ifSignTarget) |
| 3110 |
|
| 3111 |
tmp := m.copyToTmp(rn.reg()) |
| 3112 |
shr := m.allocateInstr() |
| 3113 |
shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(1), tmp, src64) |
| 3114 |
m.insert(shr) |
| 3115 |
|
| 3116 |
tmp2 := m.copyToTmp(rn.reg()) |
| 3117 |
and := m.allocateInstr() |
| 3118 |
and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, src64) |
| 3119 |
m.insert(and) |
| 3120 |
|
| 3121 |
or := m.allocateInstr() |
| 3122 |
or.asAluRmiR(aluRmiROpcodeOr, newOperandReg(tmp2), tmp, src64) |
| 3123 |
m.insert(or) |
| 3124 |
|
| 3125 |
cvt2 := m.allocateInstr() |
| 3126 |
cvt2.asGprToXmm(op, newOperandReg(tmp), tmpXmm, src64) |
| 3127 |
m.insert(cvt2) |
| 3128 |
|
| 3129 |
addsd := m.allocateInstr() |
| 3130 |
if dst64 { |
| 3131 |
addsd.asXmmRmR(sseOpcodeAddsd, newOperandReg(tmpXmm), tmpXmm) |
| 3132 |
} else { |
| 3133 |
addsd.asXmmRmR(sseOpcodeAddss, newOperandReg(tmpXmm), tmpXmm) |
| 3134 |
} |
| 3135 |
m.insert(addsd) |
| 3136 |
|
| 3137 |
m.insert(endTarget) |
| 3138 |
m.copyTo(tmpXmm, rd.reg()) |
| 3139 |
} |
| 3140 |
|
| 3141 |
func (m *machine) lowerVanyTrue(instr *ssa.Instruction) { |
| 3142 |
x := instr.Arg() |
| 3143 |
rm := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3144 |
rd := m.c.VRegOf(instr.Return()) |
| 3145 |
|
| 3146 |
tmp := m.c.AllocateVReg(ssa.TypeI32) |
| 3147 |
|
| 3148 |
cmp := m.allocateInstr() |
| 3149 |
cmp.asXmmCmpRmR(sseOpcodePtest, rm, rm.reg()) |
| 3150 |
m.insert(cmp) |
| 3151 |
|
| 3152 |
setcc := m.allocateInstr() |
| 3153 |
setcc.asSetcc(condNZ, tmp) |
| 3154 |
m.insert(setcc) |
| 3155 |
|
| 3156 |
// Clear the irrelevant bits. |
| 3157 |
and := m.allocateInstr() |
| 3158 |
and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp, false) |
| 3159 |
m.insert(and) |
| 3160 |
|
| 3161 |
m.copyTo(tmp, rd) |
| 3162 |
} |
| 3163 |
|
| 3164 |
func (m *machine) lowerVallTrue(instr *ssa.Instruction) { |
| 3165 |
x, lane := instr.ArgWithLane() |
| 3166 |
var op sseOpcode |
| 3167 |
switch lane { |
| 3168 |
case ssa.VecLaneI8x16: |
| 3169 |
op = sseOpcodePcmpeqb |
| 3170 |
case ssa.VecLaneI16x8: |
| 3171 |
op = sseOpcodePcmpeqw |
| 3172 |
case ssa.VecLaneI32x4: |
| 3173 |
op = sseOpcodePcmpeqd |
| 3174 |
case ssa.VecLaneI64x2: |
| 3175 |
op = sseOpcodePcmpeqq |
| 3176 |
} |
| 3177 |
rm := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3178 |
rd := m.c.VRegOf(instr.Return()) |
| 3179 |
|
| 3180 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 3181 |
|
| 3182 |
zeros := m.allocateInstr() |
| 3183 |
zeros.asZeros(tmp) |
| 3184 |
m.insert(zeros) |
| 3185 |
|
| 3186 |
pcmp := m.allocateInstr() |
| 3187 |
pcmp.asXmmRmR(op, rm, tmp) |
| 3188 |
m.insert(pcmp) |
| 3189 |
|
| 3190 |
test := m.allocateInstr() |
| 3191 |
test.asXmmCmpRmR(sseOpcodePtest, newOperandReg(tmp), tmp) |
| 3192 |
m.insert(test) |
| 3193 |
|
| 3194 |
tmp2 := m.c.AllocateVReg(ssa.TypeI32) |
| 3195 |
|
| 3196 |
setcc := m.allocateInstr() |
| 3197 |
setcc.asSetcc(condZ, tmp2) |
| 3198 |
m.insert(setcc) |
| 3199 |
|
| 3200 |
// Clear the irrelevant bits. |
| 3201 |
and := m.allocateInstr() |
| 3202 |
and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, false) |
| 3203 |
m.insert(and) |
| 3204 |
|
| 3205 |
m.copyTo(tmp2, rd) |
| 3206 |
} |
| 3207 |
|
| 3208 |
func (m *machine) lowerVhighBits(instr *ssa.Instruction) { |
| 3209 |
x, lane := instr.ArgWithLane() |
| 3210 |
rm := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3211 |
rd := m.c.VRegOf(instr.Return()) |
| 3212 |
switch lane { |
| 3213 |
case ssa.VecLaneI8x16: |
| 3214 |
mov := m.allocateInstr() |
| 3215 |
mov.asXmmToGpr(sseOpcodePmovmskb, rm.reg(), rd, false) |
| 3216 |
m.insert(mov) |
| 3217 |
|
| 3218 |
case ssa.VecLaneI16x8: |
| 3219 |
// When we have: |
| 3220 |
// R1 = [R1(w1), R1(w2), R1(w3), R1(w4), R1(w5), R1(w6), R1(w7), R1(v8)] |
| 3221 |
// R2 = [R2(w1), R2(w2), R2(w3), R2(v4), R2(w5), R2(w6), R2(w7), R2(v8)] |
| 3222 |
// where RX(wn) is n-th signed word (16-bit) of RX register, |
| 3223 |
// |
| 3224 |
// "PACKSSWB R1, R2" produces |
| 3225 |
// R1 = [ |
| 3226 |
// byte_sat(R1(w1)), byte_sat(R1(w2)), byte_sat(R1(w3)), byte_sat(R1(w4)), |
| 3227 |
// byte_sat(R1(w5)), byte_sat(R1(w6)), byte_sat(R1(w7)), byte_sat(R1(w8)), |
| 3228 |
// byte_sat(R2(w1)), byte_sat(R2(w2)), byte_sat(R2(w3)), byte_sat(R2(w4)), |
| 3229 |
// byte_sat(R2(w5)), byte_sat(R2(w6)), byte_sat(R2(w7)), byte_sat(R2(w8)), |
| 3230 |
// ] |
| 3231 |
// where R1 is the destination register, and |
| 3232 |
// byte_sat(w) = int8(w) if w fits as signed 8-bit, |
| 3233 |
// 0x80 if w is less than 0x80 |
| 3234 |
// 0x7F if w is greater than 0x7f |
| 3235 |
// |
| 3236 |
// See https://www.felixcloutier.com/x86/packsswb:packssdw for detail. |
| 3237 |
// |
| 3238 |
// Therefore, v.register ends up having i-th and (i+8)-th bit set if i-th lane is negative (for i in 0..8). |
| 3239 |
tmp := m.copyToTmp(rm.reg()) |
| 3240 |
res := m.c.AllocateVReg(ssa.TypeI32) |
| 3241 |
|
| 3242 |
pak := m.allocateInstr() |
| 3243 |
pak.asXmmRmR(sseOpcodePacksswb, rm, tmp) |
| 3244 |
m.insert(pak) |
| 3245 |
|
| 3246 |
mov := m.allocateInstr() |
| 3247 |
mov.asXmmToGpr(sseOpcodePmovmskb, tmp, res, false) |
| 3248 |
m.insert(mov) |
| 3249 |
|
| 3250 |
// Clear the higher bits than 8. |
| 3251 |
shr := m.allocateInstr() |
| 3252 |
shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(8), res, false) |
| 3253 |
m.insert(shr) |
| 3254 |
|
| 3255 |
m.copyTo(res, rd) |
| 3256 |
|
| 3257 |
case ssa.VecLaneI32x4: |
| 3258 |
mov := m.allocateInstr() |
| 3259 |
mov.asXmmToGpr(sseOpcodeMovmskps, rm.reg(), rd, true) |
| 3260 |
m.insert(mov) |
| 3261 |
|
| 3262 |
case ssa.VecLaneI64x2: |
| 3263 |
mov := m.allocateInstr() |
| 3264 |
mov.asXmmToGpr(sseOpcodeMovmskpd, rm.reg(), rd, true) |
| 3265 |
m.insert(mov) |
| 3266 |
} |
| 3267 |
} |
| 3268 |
|
| 3269 |
func (m *machine) lowerVbnot(instr *ssa.Instruction) { |
| 3270 |
x := instr.Arg() |
| 3271 |
xDef := m.c.ValueDefinition(x) |
| 3272 |
rm := m.getOperand_Reg(xDef) |
| 3273 |
rd := m.c.VRegOf(instr.Return()) |
| 3274 |
|
| 3275 |
tmp := m.copyToTmp(rm.reg()) |
| 3276 |
tmp2 := m.c.AllocateVReg(ssa.TypeV128) |
| 3277 |
|
| 3278 |
// Ensure tmp2 is considered defined by regalloc. |
| 3279 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) |
| 3280 |
|
| 3281 |
// Set all bits on tmp register. |
| 3282 |
pak := m.allocateInstr() |
| 3283 |
pak.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp2), tmp2) |
| 3284 |
m.insert(pak) |
| 3285 |
|
| 3286 |
// Then XOR with tmp to reverse all bits on v.register. |
| 3287 |
xor := m.allocateInstr() |
| 3288 |
xor.asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp) |
| 3289 |
m.insert(xor) |
| 3290 |
|
| 3291 |
m.copyTo(tmp, rd) |
| 3292 |
} |
| 3293 |
|
| 3294 |
func (m *machine) lowerSplat(x, ret ssa.Value, lane ssa.VecLane) { |
| 3295 |
tmpDst := m.c.AllocateVReg(ssa.TypeV128) |
| 3296 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst)) |
| 3297 |
|
| 3298 |
switch lane { |
| 3299 |
case ssa.VecLaneI8x16: |
| 3300 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 3301 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp)) |
| 3302 |
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3303 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, xx, tmpDst)) |
| 3304 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp)) |
| 3305 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpDst)) |
| 3306 |
case ssa.VecLaneI16x8: |
| 3307 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3308 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, xx, tmpDst)) |
| 3309 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, xx, tmpDst)) |
| 3310 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) |
| 3311 |
case ssa.VecLaneI32x4: |
| 3312 |
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3313 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, xx, tmpDst)) |
| 3314 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) |
| 3315 |
case ssa.VecLaneI64x2: |
| 3316 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3317 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, xx, tmpDst)) |
| 3318 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, xx, tmpDst)) |
| 3319 |
case ssa.VecLaneF32x4: |
| 3320 |
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3321 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, 0, xx, tmpDst)) |
| 3322 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) |
| 3323 |
case ssa.VecLaneF64x2: |
| 3324 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3325 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst)) |
| 3326 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, xx, tmpDst)) |
| 3327 |
default: |
| 3328 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 3329 |
} |
| 3330 |
|
| 3331 |
m.copyTo(tmpDst, m.c.VRegOf(ret)) |
| 3332 |
} |
| 3333 |
|
| 3334 |
func (m *machine) lowerShuffle(x, y ssa.Value, lo, hi uint64, ret ssa.Value) { |
| 3335 |
var xMask, yMask [2]uint64 |
| 3336 |
for i := 0; i < 8; i++ { |
| 3337 |
loLane := byte(lo >> (i * 8)) |
| 3338 |
if loLane < 16 { |
| 3339 |
xMask[0] |= uint64(loLane) << (i * 8) |
| 3340 |
yMask[0] |= uint64(0x80) << (i * 8) |
| 3341 |
} else { |
| 3342 |
xMask[0] |= uint64(0x80) << (i * 8) |
| 3343 |
yMask[0] |= uint64(loLane-16) << (i * 8) |
| 3344 |
} |
| 3345 |
hiLane := byte(hi >> (i * 8)) |
| 3346 |
if hiLane < 16 { |
| 3347 |
xMask[1] |= uint64(hiLane) << (i * 8) |
| 3348 |
yMask[1] |= uint64(0x80) << (i * 8) |
| 3349 |
} else { |
| 3350 |
xMask[1] |= uint64(0x80) << (i * 8) |
| 3351 |
yMask[1] |= uint64(hiLane-16) << (i * 8) |
| 3352 |
} |
| 3353 |
} |
| 3354 |
|
| 3355 |
xl, xmaskPos := m.allocateLabel() |
| 3356 |
m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xl, labelPos: xmaskPos}) |
| 3357 |
yl, ymaskPos := m.allocateLabel() |
| 3358 |
m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: yl, labelPos: ymaskPos}) |
| 3359 |
|
| 3360 |
xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 3361 |
tmpX, tmpY := m.copyToTmp(xx.reg()), m.copyToTmp(yy.reg()) |
| 3362 |
|
| 3363 |
// Apply mask to X. |
| 3364 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 3365 |
loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xl)), tmp) |
| 3366 |
m.insert(loadMaskLo) |
| 3367 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpX)) |
| 3368 |
|
| 3369 |
// Apply mask to Y. |
| 3370 |
loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(yl)), tmp) |
| 3371 |
m.insert(loadMaskHi) |
| 3372 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpY)) |
| 3373 |
|
| 3374 |
// Combine the results. |
| 3375 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeOrps, newOperandReg(tmpX), tmpY)) |
| 3376 |
|
| 3377 |
m.copyTo(tmpY, m.c.VRegOf(ret)) |
| 3378 |
} |
| 3379 |
|
| 3380 |
func (m *machine) lowerVbBinOpUnaligned(op sseOpcode, x, y, ret ssa.Value) { |
| 3381 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3382 |
rm := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 3383 |
rd := m.c.VRegOf(ret) |
| 3384 |
|
| 3385 |
tmp := m.copyToTmp(rn.reg()) |
| 3386 |
|
| 3387 |
binOp := m.allocateInstr() |
| 3388 |
binOp.asXmmRmR(op, rm, tmp) |
| 3389 |
m.insert(binOp) |
| 3390 |
|
| 3391 |
m.copyTo(tmp, rd) |
| 3392 |
} |
| 3393 |
|
| 3394 |
func (m *machine) lowerVbBinOp(op sseOpcode, x, y, ret ssa.Value) { |
| 3395 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3396 |
rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3397 |
rd := m.c.VRegOf(ret) |
| 3398 |
|
| 3399 |
tmp := m.copyToTmp(rn.reg()) |
| 3400 |
|
| 3401 |
binOp := m.allocateInstr() |
| 3402 |
binOp.asXmmRmR(op, rm, tmp) |
| 3403 |
m.insert(binOp) |
| 3404 |
|
| 3405 |
m.copyTo(tmp, rd) |
| 3406 |
} |
| 3407 |
|
| 3408 |
func (m *machine) lowerVFcmp(x, y ssa.Value, c ssa.FloatCmpCond, ret ssa.Value, lane ssa.VecLane) { |
| 3409 |
var cmpOp sseOpcode |
| 3410 |
switch lane { |
| 3411 |
case ssa.VecLaneF32x4: |
| 3412 |
cmpOp = sseOpcodeCmpps |
| 3413 |
case ssa.VecLaneF64x2: |
| 3414 |
cmpOp = sseOpcodeCmppd |
| 3415 |
default: |
| 3416 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 3417 |
} |
| 3418 |
|
| 3419 |
xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y) |
| 3420 |
var cmpImm cmpPred |
| 3421 |
switch c { |
| 3422 |
case ssa.FloatCmpCondGreaterThan: |
| 3423 |
yy, xx = xx, yy |
| 3424 |
cmpImm = cmpPredLT_OS |
| 3425 |
case ssa.FloatCmpCondGreaterThanOrEqual: |
| 3426 |
yy, xx = xx, yy |
| 3427 |
cmpImm = cmpPredLE_OS |
| 3428 |
case ssa.FloatCmpCondEqual: |
| 3429 |
cmpImm = cmpPredEQ_OQ |
| 3430 |
case ssa.FloatCmpCondNotEqual: |
| 3431 |
cmpImm = cmpPredNEQ_UQ |
| 3432 |
case ssa.FloatCmpCondLessThan: |
| 3433 |
cmpImm = cmpPredLT_OS |
| 3434 |
case ssa.FloatCmpCondLessThanOrEqual: |
| 3435 |
cmpImm = cmpPredLE_OS |
| 3436 |
default: |
| 3437 |
panic(fmt.Sprintf("invalid float comparison condition: %s", c)) |
| 3438 |
} |
| 3439 |
|
| 3440 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 3441 |
xxx := m.getOperand_Mem_Reg(xx) |
| 3442 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, xxx, tmp)) |
| 3443 |
|
| 3444 |
rm := m.getOperand_Mem_Reg(yy) |
| 3445 |
m.insert(m.allocateInstr().asXmmRmRImm(cmpOp, byte(cmpImm), rm, tmp)) |
| 3446 |
|
| 3447 |
m.copyTo(tmp, m.c.VRegOf(ret)) |
| 3448 |
} |
| 3449 |
|
| 3450 |
func (m *machine) lowerVIcmp(x, y ssa.Value, c ssa.IntegerCmpCond, ret ssa.Value, lane ssa.VecLane) { |
| 3451 |
var eq, gt, maxu, minu, mins sseOpcode |
| 3452 |
switch lane { |
| 3453 |
case ssa.VecLaneI8x16: |
| 3454 |
eq, gt, maxu, minu, mins = sseOpcodePcmpeqb, sseOpcodePcmpgtb, sseOpcodePmaxub, sseOpcodePminub, sseOpcodePminsb |
| 3455 |
case ssa.VecLaneI16x8: |
| 3456 |
eq, gt, maxu, minu, mins = sseOpcodePcmpeqw, sseOpcodePcmpgtw, sseOpcodePmaxuw, sseOpcodePminuw, sseOpcodePminsw |
| 3457 |
case ssa.VecLaneI32x4: |
| 3458 |
eq, gt, maxu, minu, mins = sseOpcodePcmpeqd, sseOpcodePcmpgtd, sseOpcodePmaxud, sseOpcodePminud, sseOpcodePminsd |
| 3459 |
case ssa.VecLaneI64x2: |
| 3460 |
eq, gt = sseOpcodePcmpeqq, sseOpcodePcmpgtq |
| 3461 |
default: |
| 3462 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 3463 |
} |
| 3464 |
|
| 3465 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 3466 |
var op operand |
| 3467 |
switch c { |
| 3468 |
case ssa.IntegerCmpCondSignedLessThanOrEqual: |
| 3469 |
if lane == ssa.VecLaneI64x2 { |
| 3470 |
x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3471 |
// Copy x to tmp. |
| 3472 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp)) |
| 3473 |
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3474 |
} else { |
| 3475 |
y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3476 |
// Copy y to tmp. |
| 3477 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp)) |
| 3478 |
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3479 |
} |
| 3480 |
case ssa.IntegerCmpCondSignedGreaterThanOrEqual: |
| 3481 |
if lane == ssa.VecLaneI64x2 { |
| 3482 |
y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3483 |
// Copy y to tmp. |
| 3484 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp)) |
| 3485 |
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3486 |
} else { |
| 3487 |
x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3488 |
// Copy x to tmp. |
| 3489 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp)) |
| 3490 |
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3491 |
} |
| 3492 |
case ssa.IntegerCmpCondSignedLessThan, ssa.IntegerCmpCondUnsignedLessThan, ssa.IntegerCmpCondUnsignedLessThanOrEqual: |
| 3493 |
y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3494 |
// Copy y to tmp. |
| 3495 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp)) |
| 3496 |
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3497 |
default: |
| 3498 |
x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3499 |
// Copy x to tmp. |
| 3500 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp)) |
| 3501 |
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 3502 |
} |
| 3503 |
|
| 3504 |
switch c { |
| 3505 |
case ssa.IntegerCmpCondEqual: |
| 3506 |
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) |
| 3507 |
case ssa.IntegerCmpCondNotEqual: |
| 3508 |
// First we compare for equality. |
| 3509 |
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) |
| 3510 |
// Then flip the bits. To do so, we set all bits on tmp2. |
| 3511 |
tmp2 := m.c.AllocateVReg(ssa.TypeV128) |
| 3512 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) |
| 3513 |
m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2)) |
| 3514 |
// And then xor with tmp. |
| 3515 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)) |
| 3516 |
case ssa.IntegerCmpCondSignedGreaterThan, ssa.IntegerCmpCondSignedLessThan: |
| 3517 |
m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp)) |
| 3518 |
case ssa.IntegerCmpCondSignedGreaterThanOrEqual, ssa.IntegerCmpCondSignedLessThanOrEqual: |
| 3519 |
if lane == ssa.VecLaneI64x2 { |
| 3520 |
m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp)) |
| 3521 |
// Then flip the bits. To do so, we set all bits on tmp2. |
| 3522 |
tmp2 := m.c.AllocateVReg(ssa.TypeV128) |
| 3523 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) |
| 3524 |
m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2)) |
| 3525 |
// And then xor with tmp. |
| 3526 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)) |
| 3527 |
} else { |
| 3528 |
// First take min of x and y. |
| 3529 |
m.insert(m.allocateInstr().asXmmRmR(mins, op, tmp)) |
| 3530 |
// Then compare for equality. |
| 3531 |
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) |
| 3532 |
} |
| 3533 |
case ssa.IntegerCmpCondUnsignedGreaterThan, ssa.IntegerCmpCondUnsignedLessThan: |
| 3534 |
// First maxu of x and y. |
| 3535 |
m.insert(m.allocateInstr().asXmmRmR(maxu, op, tmp)) |
| 3536 |
// Then compare for equality. |
| 3537 |
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) |
| 3538 |
// Then flip the bits. To do so, we set all bits on tmp2. |
| 3539 |
tmp2 := m.c.AllocateVReg(ssa.TypeV128) |
| 3540 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) |
| 3541 |
m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2)) |
| 3542 |
// And then xor with tmp. |
| 3543 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)) |
| 3544 |
case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual, ssa.IntegerCmpCondUnsignedLessThanOrEqual: |
| 3545 |
m.insert(m.allocateInstr().asXmmRmR(minu, op, tmp)) |
| 3546 |
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) |
| 3547 |
default: |
| 3548 |
panic("BUG") |
| 3549 |
} |
| 3550 |
|
| 3551 |
m.copyTo(tmp, m.c.VRegOf(ret)) |
| 3552 |
} |
| 3553 |
|
| 3554 |
func (m *machine) lowerVbandnot(instr *ssa.Instruction, op sseOpcode) { |
| 3555 |
x, y := instr.Arg2() |
| 3556 |
xDef := m.c.ValueDefinition(x) |
| 3557 |
yDef := m.c.ValueDefinition(y) |
| 3558 |
rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef) |
| 3559 |
rd := m.c.VRegOf(instr.Return()) |
| 3560 |
|
| 3561 |
tmp := m.copyToTmp(rn.reg()) |
| 3562 |
|
| 3563 |
// pandn between rn, rm. |
| 3564 |
pand := m.allocateInstr() |
| 3565 |
pand.asXmmRmR(sseOpcodePandn, rm, tmp) |
| 3566 |
m.insert(pand) |
| 3567 |
|
| 3568 |
m.copyTo(tmp, rd) |
| 3569 |
} |
| 3570 |
|
| 3571 |
func (m *machine) lowerVbitselect(instr *ssa.Instruction) { |
| 3572 |
c, x, y := instr.SelectData() |
| 3573 |
xDef := m.c.ValueDefinition(x) |
| 3574 |
yDef := m.c.ValueDefinition(y) |
| 3575 |
rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef) |
| 3576 |
creg := m.getOperand_Reg(m.c.ValueDefinition(c)) |
| 3577 |
rd := m.c.VRegOf(instr.Return()) |
| 3578 |
|
| 3579 |
tmpC := m.copyToTmp(creg.reg()) |
| 3580 |
tmpX := m.copyToTmp(rm.reg()) |
| 3581 |
|
| 3582 |
// And between c, x (overwrites x). |
| 3583 |
pand := m.allocateInstr() |
| 3584 |
pand.asXmmRmR(sseOpcodePand, creg, tmpX) |
| 3585 |
m.insert(pand) |
| 3586 |
|
| 3587 |
// Andn between y, c (overwrites c). |
| 3588 |
pandn := m.allocateInstr() |
| 3589 |
pandn.asXmmRmR(sseOpcodePandn, rn, tmpC) |
| 3590 |
m.insert(pandn) |
| 3591 |
|
| 3592 |
por := m.allocateInstr() |
| 3593 |
por.asXmmRmR(sseOpcodePor, newOperandReg(tmpC), tmpX) |
| 3594 |
m.insert(por) |
| 3595 |
|
| 3596 |
m.copyTo(tmpX, rd) |
| 3597 |
} |
| 3598 |
|
| 3599 |
func (m *machine) lowerVFmin(instr *ssa.Instruction) { |
| 3600 |
x, y, lane := instr.Arg2WithLane() |
| 3601 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3602 |
rm := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 3603 |
rd := m.c.VRegOf(instr.Return()) |
| 3604 |
|
| 3605 |
var min, cmp, andn, or, srl /* shift right logical */ sseOpcode |
| 3606 |
var shiftNumToInverseNaN uint32 |
| 3607 |
if lane == ssa.VecLaneF32x4 { |
| 3608 |
min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodePsrld, 0xa |
| 3609 |
} else { |
| 3610 |
min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodePsrlq, 0xd |
| 3611 |
} |
| 3612 |
|
| 3613 |
tmp1 := m.copyToTmp(rn.reg()) |
| 3614 |
tmp2 := m.copyToTmp(rm.reg()) |
| 3615 |
|
| 3616 |
// tmp1=min(rn, rm) |
| 3617 |
minIns1 := m.allocateInstr() |
| 3618 |
minIns1.asXmmRmR(min, rn, tmp2) |
| 3619 |
m.insert(minIns1) |
| 3620 |
|
| 3621 |
// tmp2=min(rm, rn) |
| 3622 |
minIns2 := m.allocateInstr() |
| 3623 |
minIns2.asXmmRmR(min, rm, tmp1) |
| 3624 |
m.insert(minIns2) |
| 3625 |
|
| 3626 |
// tmp3:=tmp1=min(rn, rm) |
| 3627 |
tmp3 := m.copyToTmp(tmp1) |
| 3628 |
|
| 3629 |
// tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN |
| 3630 |
// NaN if rn == NaN || rm == NaN |
| 3631 |
// min(rm, rm) otherwise |
| 3632 |
orIns := m.allocateInstr() |
| 3633 |
orIns.asXmmRmR(or, newOperandReg(tmp2), tmp1) |
| 3634 |
m.insert(orIns) |
| 3635 |
|
| 3636 |
// tmp3 is originally min(rn,rm). |
| 3637 |
// tmp3 = 0^ (set all bits) if rn == NaN || rm == NaN |
| 3638 |
// 0 otherwise |
| 3639 |
cmpIns := m.allocateInstr() |
| 3640 |
cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp2), tmp3) |
| 3641 |
m.insert(cmpIns) |
| 3642 |
|
| 3643 |
// tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN |
| 3644 |
// ^0 if rn == NaN || rm == NaN |
| 3645 |
// min(v1, v2) otherwise |
| 3646 |
orIns2 := m.allocateInstr() |
| 3647 |
orIns2.asXmmRmR(or, newOperandReg(tmp3), tmp1) |
| 3648 |
m.insert(orIns2) |
| 3649 |
|
| 3650 |
// tmp3 = set all bits on the mantissa bits |
| 3651 |
// 0 otherwise |
| 3652 |
shift := m.allocateInstr() |
| 3653 |
shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp3) |
| 3654 |
m.insert(shift) |
| 3655 |
|
| 3656 |
// tmp3 = tmp1 and !tmp3 |
| 3657 |
// = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN |
| 3658 |
// set all bits on exponential and sign bit (== NaN) if rn == NaN || rm == NaN |
| 3659 |
// min(rn, rm) otherwise |
| 3660 |
andnIns := m.allocateInstr() |
| 3661 |
andnIns.asXmmRmR(andn, newOperandReg(tmp1), tmp3) |
| 3662 |
m.insert(andnIns) |
| 3663 |
|
| 3664 |
m.copyTo(tmp3, rd) |
| 3665 |
} |
| 3666 |
|
| 3667 |
func (m *machine) lowerVFmax(instr *ssa.Instruction) { |
| 3668 |
x, y, lane := instr.Arg2WithLane() |
| 3669 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 3670 |
rm := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 3671 |
rd := m.c.VRegOf(instr.Return()) |
| 3672 |
|
| 3673 |
var max, cmp, andn, or, xor, sub, srl /* shift right logical */ sseOpcode |
| 3674 |
var shiftNumToInverseNaN uint32 |
| 3675 |
if lane == ssa.VecLaneF32x4 { |
| 3676 |
max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodeXorps, sseOpcodeSubps, sseOpcodePsrld, 0xa |
| 3677 |
} else { |
| 3678 |
max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodeXorpd, sseOpcodeSubpd, sseOpcodePsrlq, 0xd |
| 3679 |
} |
| 3680 |
|
| 3681 |
tmp0 := m.copyToTmp(rm.reg()) |
| 3682 |
tmp1 := m.copyToTmp(rn.reg()) |
| 3683 |
|
| 3684 |
// tmp0=max(rn, rm) |
| 3685 |
maxIns1 := m.allocateInstr() |
| 3686 |
maxIns1.asXmmRmR(max, rn, tmp0) |
| 3687 |
m.insert(maxIns1) |
| 3688 |
|
| 3689 |
// tmp1=max(rm, rn) |
| 3690 |
maxIns2 := m.allocateInstr() |
| 3691 |
maxIns2.asXmmRmR(max, rm, tmp1) |
| 3692 |
m.insert(maxIns2) |
| 3693 |
|
| 3694 |
// tmp2=max(rm, rn) |
| 3695 |
tmp2 := m.copyToTmp(tmp1) |
| 3696 |
|
| 3697 |
// tmp2 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) |
| 3698 |
// 0 if (rn == 0 && rm == 0) |
| 3699 |
// -0 if (rn == -0 && rm == -0) |
| 3700 |
// v1^v2 if rn == NaN || rm == NaN |
| 3701 |
// 0 otherwise |
| 3702 |
xorInstr := m.allocateInstr() |
| 3703 |
xorInstr.asXmmRmR(xor, newOperandReg(tmp0), tmp2) |
| 3704 |
m.insert(xorInstr) |
| 3705 |
// tmp1 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) |
| 3706 |
// 0 if (rn == 0 && rm == 0) |
| 3707 |
// -0 if (rn == -0 && rm == -0) |
| 3708 |
// NaN if rn == NaN || rm == NaN |
| 3709 |
// max(v1, v2) otherwise |
| 3710 |
orInstr := m.allocateInstr() |
| 3711 |
orInstr.asXmmRmR(or, newOperandReg(tmp2), tmp1) |
| 3712 |
m.insert(orInstr) |
| 3713 |
|
| 3714 |
tmp3 := m.copyToTmp(tmp1) |
| 3715 |
|
| 3716 |
// tmp3 = 0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) || (rn == 0 && rm == 0) |
| 3717 |
// -0 if (rn == -0 && rm == -0) |
| 3718 |
// NaN if rn == NaN || rm == NaN |
| 3719 |
// max(v1, v2) otherwise |
| 3720 |
// |
| 3721 |
// Note: -0 - (-0) = 0 (!= -0) in floating point operation. |
| 3722 |
subIns := m.allocateInstr() |
| 3723 |
subIns.asXmmRmR(sub, newOperandReg(tmp2), tmp3) |
| 3724 |
m.insert(subIns) |
| 3725 |
|
| 3726 |
// tmp1 = 0^ if rn == NaN || rm == NaN |
| 3727 |
cmpIns := m.allocateInstr() |
| 3728 |
cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp1), tmp1) |
| 3729 |
m.insert(cmpIns) |
| 3730 |
|
| 3731 |
// tmp1 = set all bits on the mantissa bits |
| 3732 |
// 0 otherwise |
| 3733 |
shift := m.allocateInstr() |
| 3734 |
shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp1) |
| 3735 |
m.insert(shift) |
| 3736 |
|
| 3737 |
andnIns := m.allocateInstr() |
| 3738 |
andnIns.asXmmRmR(andn, newOperandReg(tmp3), tmp1) |
| 3739 |
m.insert(andnIns) |
| 3740 |
|
| 3741 |
m.copyTo(tmp1, rd) |
| 3742 |
} |
| 3743 |
|
| 3744 |
func (m *machine) lowerVFabs(instr *ssa.Instruction) { |
| 3745 |
x, lane := instr.ArgWithLane() |
| 3746 |
rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 3747 |
rd := m.c.VRegOf(instr.Return()) |
| 3748 |
|
| 3749 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 3750 |
|
| 3751 |
def := m.allocateInstr() |
| 3752 |
def.asDefineUninitializedReg(tmp) |
| 3753 |
m.insert(def) |
| 3754 |
|
| 3755 |
// Set all bits on tmp. |
| 3756 |
pcmp := m.allocateInstr() |
| 3757 |
pcmp.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp) |
| 3758 |
m.insert(pcmp) |
| 3759 |
|
| 3760 |
switch lane { |
| 3761 |
case ssa.VecLaneF32x4: |
| 3762 |
// Shift right packed single floats by 1 to clear the sign bits. |
| 3763 |
shift := m.allocateInstr() |
| 3764 |
shift.asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp) |
| 3765 |
m.insert(shift) |
| 3766 |
// Clear the sign bit of rm. |
| 3767 |
andp := m.allocateInstr() |
| 3768 |
andp.asXmmRmR(sseOpcodeAndpd, rm, tmp) |
| 3769 |
m.insert(andp) |
| 3770 |
case ssa.VecLaneF64x2: |
| 3771 |
// Shift right packed single floats by 1 to clear the sign bits. |
| 3772 |
shift := m.allocateInstr() |
| 3773 |
shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(1), tmp) |
| 3774 |
m.insert(shift) |
| 3775 |
// Clear the sign bit of rm. |
| 3776 |
andp := m.allocateInstr() |
| 3777 |
andp.asXmmRmR(sseOpcodeAndps, rm, tmp) |
| 3778 |
m.insert(andp) |
| 3779 |
} |
| 3780 |
|
| 3781 |
m.copyTo(tmp, rd) |
| 3782 |
} |