| 1 |
package amd64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"fmt" |
| 5 |
|
| 6 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 7 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" |
| 8 |
) |
| 9 |
|
| 10 |
var swizzleMask = [16]byte{ |
| 11 |
0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, |
| 12 |
0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, |
| 13 |
} |
| 14 |
|
| 15 |
func (m *machine) lowerSwizzle(x, y ssa.Value, ret ssa.Value) { |
| 16 |
masklabel := m.getOrAllocateConstLabel(&m.constSwizzleMaskConstIndex, swizzleMask[:]) |
| 17 |
|
| 18 |
// Load mask to maskReg. |
| 19 |
maskReg := m.c.AllocateVReg(ssa.TypeV128) |
| 20 |
loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(masklabel)), maskReg) |
| 21 |
m.insert(loadMask) |
| 22 |
|
| 23 |
// Copy x and y to tmp registers. |
| 24 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 25 |
tmpDst := m.copyToTmp(xx.reg()) |
| 26 |
yy := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 27 |
tmpX := m.copyToTmp(yy.reg()) |
| 28 |
|
| 29 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePaddusb, newOperandReg(maskReg), tmpX)) |
| 30 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmpX), tmpDst)) |
| 31 |
|
| 32 |
// Copy the result to the destination register. |
| 33 |
m.copyTo(tmpDst, m.c.VRegOf(ret)) |
| 34 |
} |
| 35 |
|
| 36 |
func (m *machine) lowerInsertLane(x, y ssa.Value, index byte, ret ssa.Value, lane ssa.VecLane) { |
| 37 |
// Copy x to tmp. |
| 38 |
tmpDst := m.c.AllocateVReg(ssa.TypeV128) |
| 39 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), tmpDst)) |
| 40 |
|
| 41 |
yy := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 42 |
switch lane { |
| 43 |
case ssa.VecLaneI8x16: |
| 44 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, index, yy, tmpDst)) |
| 45 |
case ssa.VecLaneI16x8: |
| 46 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, index, yy, tmpDst)) |
| 47 |
case ssa.VecLaneI32x4: |
| 48 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, index, yy, tmpDst)) |
| 49 |
case ssa.VecLaneI64x2: |
| 50 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, index, yy, tmpDst)) |
| 51 |
case ssa.VecLaneF32x4: |
| 52 |
// In INSERTPS instruction, the destination index is encoded at 4 and 5 bits of the argument. |
| 53 |
// See https://www.felixcloutier.com/x86/insertps |
| 54 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, index<<4, yy, tmpDst)) |
| 55 |
case ssa.VecLaneF64x2: |
| 56 |
if index == 0 { |
| 57 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, yy, tmpDst)) |
| 58 |
} else { |
| 59 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, yy, tmpDst)) |
| 60 |
} |
| 61 |
default: |
| 62 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 63 |
} |
| 64 |
|
| 65 |
m.copyTo(tmpDst, m.c.VRegOf(ret)) |
| 66 |
} |
| 67 |
|
| 68 |
func (m *machine) lowerExtractLane(x ssa.Value, index byte, signed bool, ret ssa.Value, lane ssa.VecLane) { |
| 69 |
// Pextr variants are used to extract a lane from a vector register. |
| 70 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 71 |
|
| 72 |
tmpDst := m.c.AllocateVReg(ret.Type()) |
| 73 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst)) |
| 74 |
switch lane { |
| 75 |
case ssa.VecLaneI8x16: |
| 76 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrb, index, xx, tmpDst)) |
| 77 |
if signed { |
| 78 |
m.insert(m.allocateInstr().asMovsxRmR(extModeBL, newOperandReg(tmpDst), tmpDst)) |
| 79 |
} else { |
| 80 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(tmpDst), tmpDst)) |
| 81 |
} |
| 82 |
case ssa.VecLaneI16x8: |
| 83 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrw, index, xx, tmpDst)) |
| 84 |
if signed { |
| 85 |
m.insert(m.allocateInstr().asMovsxRmR(extModeWL, newOperandReg(tmpDst), tmpDst)) |
| 86 |
} else { |
| 87 |
m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(tmpDst), tmpDst)) |
| 88 |
} |
| 89 |
case ssa.VecLaneI32x4: |
| 90 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrd, index, xx, tmpDst)) |
| 91 |
case ssa.VecLaneI64x2: |
| 92 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrq, index, xx, tmpDst)) |
| 93 |
case ssa.VecLaneF32x4: |
| 94 |
if index == 0 { |
| 95 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovss, xx, tmpDst)) |
| 96 |
} else { |
| 97 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, index, xx, tmpDst)) |
| 98 |
} |
| 99 |
case ssa.VecLaneF64x2: |
| 100 |
if index == 0 { |
| 101 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst)) |
| 102 |
} else { |
| 103 |
m.copyTo(xx.reg(), tmpDst) |
| 104 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0b00_00_11_10, newOperandReg(tmpDst), tmpDst)) |
| 105 |
} |
| 106 |
default: |
| 107 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 108 |
} |
| 109 |
|
| 110 |
m.copyTo(tmpDst, m.c.VRegOf(ret)) |
| 111 |
} |
| 112 |
|
| 113 |
var sqmulRoundSat = [16]byte{ |
| 114 |
0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, |
| 115 |
0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, |
| 116 |
} |
| 117 |
|
| 118 |
func (m *machine) lowerSqmulRoundSat(x, y, ret ssa.Value) { |
| 119 |
// See https://github.com/WebAssembly/simd/pull/365 for the following logic. |
| 120 |
maskLabel := m.getOrAllocateConstLabel(&m.constSqmulRoundSatIndex, sqmulRoundSat[:]) |
| 121 |
|
| 122 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 123 |
loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), tmp) |
| 124 |
m.insert(loadMask) |
| 125 |
|
| 126 |
xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 127 |
tmpX := m.copyToTmp(xx.reg()) |
| 128 |
|
| 129 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmulhrsw, yy, tmpX)) |
| 130 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePcmpeqw, newOperandReg(tmpX), tmp)) |
| 131 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmpX)) |
| 132 |
|
| 133 |
m.copyTo(tmpX, m.c.VRegOf(ret)) |
| 134 |
} |
| 135 |
|
| 136 |
func (m *machine) lowerVUshr(x, y, ret ssa.Value, lane ssa.VecLane) { |
| 137 |
switch lane { |
| 138 |
case ssa.VecLaneI8x16: |
| 139 |
m.lowerVUshri8x16(x, y, ret) |
| 140 |
case ssa.VecLaneI16x8, ssa.VecLaneI32x4, ssa.VecLaneI64x2: |
| 141 |
m.lowerShr(x, y, ret, lane, false) |
| 142 |
default: |
| 143 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 144 |
} |
| 145 |
} |
| 146 |
|
| 147 |
// i8x16LogicalSHRMaskTable is necessary for emulating non-existent packed bytes logical right shifts on amd64. |
| 148 |
// The mask is applied after performing packed word shifts on the value to clear out the unnecessary bits. |
| 149 |
var i8x16LogicalSHRMaskTable = [8 * 16]byte{ // (the number of possible shift amount 0, 1, ..., 7.) * 16 bytes. |
| 150 |
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // for 0 shift |
| 151 |
0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, // for 1 shift |
| 152 |
0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, // for 2 shift |
| 153 |
0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, // for 3 shift |
| 154 |
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, // for 4 shift |
| 155 |
0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, // for 5 shift |
| 156 |
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // for 6 shift |
| 157 |
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // for 7 shift |
| 158 |
} |
| 159 |
|
| 160 |
func (m *machine) lowerVUshri8x16(x, y, ret ssa.Value) { |
| 161 |
tmpGpReg := m.c.AllocateVReg(ssa.TypeI32) |
| 162 |
// Load the modulo 8 mask to tmpReg. |
| 163 |
m.lowerIconst(tmpGpReg, 0x7, false) |
| 164 |
// Take the modulo 8 of the shift amount. |
| 165 |
shiftAmt := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)) |
| 166 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, shiftAmt, tmpGpReg, false)) |
| 167 |
|
| 168 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 169 |
xx := m.copyToTmp(_xx.reg()) |
| 170 |
|
| 171 |
vecTmp := m.c.AllocateVReg(ssa.TypeV128) |
| 172 |
m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(tmpGpReg), vecTmp, false)) |
| 173 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrlw, newOperandReg(vecTmp), xx)) |
| 174 |
|
| 175 |
maskTableLabel := m.getOrAllocateConstLabel(&m.constI8x16LogicalSHRMaskTableIndex, i8x16LogicalSHRMaskTable[:]) |
| 176 |
base := m.c.AllocateVReg(ssa.TypeI64) |
| 177 |
lea := m.allocateInstr().asLEA(newOperandLabel(maskTableLabel), base) |
| 178 |
m.insert(lea) |
| 179 |
|
| 180 |
// Shift tmpGpReg by 4 to multiply the shift amount by 16. |
| 181 |
m.insert(m.allocateInstr().asShiftR(shiftROpShiftLeft, newOperandImm32(4), tmpGpReg, false)) |
| 182 |
|
| 183 |
mem := m.newAmodeRegRegShift(0, base, tmpGpReg, 0) |
| 184 |
loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(mem), vecTmp) |
| 185 |
m.insert(loadMask) |
| 186 |
|
| 187 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePand, newOperandReg(vecTmp), xx)) |
| 188 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 189 |
} |
| 190 |
|
| 191 |
func (m *machine) lowerVSshr(x, y, ret ssa.Value, lane ssa.VecLane) { |
| 192 |
switch lane { |
| 193 |
case ssa.VecLaneI8x16: |
| 194 |
m.lowerVSshri8x16(x, y, ret) |
| 195 |
case ssa.VecLaneI16x8, ssa.VecLaneI32x4: |
| 196 |
m.lowerShr(x, y, ret, lane, true) |
| 197 |
case ssa.VecLaneI64x2: |
| 198 |
m.lowerVSshri64x2(x, y, ret) |
| 199 |
default: |
| 200 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 201 |
} |
| 202 |
} |
| 203 |
|
| 204 |
func (m *machine) lowerVSshri8x16(x, y, ret ssa.Value) { |
| 205 |
shiftAmtReg := m.c.AllocateVReg(ssa.TypeI32) |
| 206 |
// Load the modulo 8 mask to tmpReg. |
| 207 |
m.lowerIconst(shiftAmtReg, 0x7, false) |
| 208 |
// Take the modulo 8 of the shift amount. |
| 209 |
shiftAmt := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)) |
| 210 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, shiftAmt, shiftAmtReg, false)) |
| 211 |
|
| 212 |
// Copy the x value to two temporary registers. |
| 213 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 214 |
xx := m.copyToTmp(_xx.reg()) |
| 215 |
vecTmp := m.c.AllocateVReg(ssa.TypeV128) |
| 216 |
m.copyTo(xx, vecTmp) |
| 217 |
|
| 218 |
// Assuming that we have |
| 219 |
// xx = [b1, ..., b16] |
| 220 |
// vecTmp = [b1, ..., b16] |
| 221 |
// at this point, then we use PUNPCKLBW and PUNPCKHBW to produce: |
| 222 |
// xx = [b1, b1, b2, b2, ..., b8, b8] |
| 223 |
// vecTmp = [b9, b9, b10, b10, ..., b16, b16] |
| 224 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePunpcklbw, newOperandReg(xx), xx)) |
| 225 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePunpckhbw, newOperandReg(vecTmp), vecTmp)) |
| 226 |
|
| 227 |
// Adding 8 to the shift amount, and then move the amount to vecTmp2. |
| 228 |
vecTmp2 := m.c.AllocateVReg(ssa.TypeV128) |
| 229 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, newOperandImm32(8), shiftAmtReg, false)) |
| 230 |
m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(shiftAmtReg), vecTmp2, false)) |
| 231 |
|
| 232 |
// Perform the word packed arithmetic right shifts on vreg and vecTmp. |
| 233 |
// This changes these two registers as: |
| 234 |
// xx = [xxx, b1 >> s, xxx, b2 >> s, ..., xxx, b8 >> s] |
| 235 |
// vecTmp = [xxx, b9 >> s, xxx, b10 >> s, ..., xxx, b16 >> s] |
| 236 |
// where xxx is 1 or 0 depending on each byte's sign, and ">>" is the arithmetic shift on a byte. |
| 237 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsraw, newOperandReg(vecTmp2), xx)) |
| 238 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsraw, newOperandReg(vecTmp2), vecTmp)) |
| 239 |
|
| 240 |
// Finally, we can get the result by packing these two word vectors. |
| 241 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePacksswb, newOperandReg(vecTmp), xx)) |
| 242 |
|
| 243 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 244 |
} |
| 245 |
|
| 246 |
func (m *machine) lowerVSshri64x2(x, y, ret ssa.Value) { |
| 247 |
// Load the shift amount to RCX. |
| 248 |
shiftAmt := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 249 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, shiftAmt, rcxVReg)) |
| 250 |
|
| 251 |
tmpGp := m.c.AllocateVReg(ssa.TypeI64) |
| 252 |
|
| 253 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 254 |
xxReg := m.copyToTmp(_xx.reg()) |
| 255 |
|
| 256 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) |
| 257 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrq, 0, newOperandReg(xxReg), tmpGp)) |
| 258 |
m.insert(m.allocateInstr().asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), tmpGp, true)) |
| 259 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, newOperandReg(tmpGp), xxReg)) |
| 260 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrq, 1, newOperandReg(xxReg), tmpGp)) |
| 261 |
m.insert(m.allocateInstr().asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), tmpGp, true)) |
| 262 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, newOperandReg(tmpGp), xxReg)) |
| 263 |
|
| 264 |
m.copyTo(xxReg, m.c.VRegOf(ret)) |
| 265 |
} |
| 266 |
|
| 267 |
func (m *machine) lowerShr(x, y, ret ssa.Value, lane ssa.VecLane, signed bool) { |
| 268 |
var modulo uint64 |
| 269 |
var shiftOp sseOpcode |
| 270 |
switch lane { |
| 271 |
case ssa.VecLaneI16x8: |
| 272 |
modulo = 0xf |
| 273 |
if signed { |
| 274 |
shiftOp = sseOpcodePsraw |
| 275 |
} else { |
| 276 |
shiftOp = sseOpcodePsrlw |
| 277 |
} |
| 278 |
case ssa.VecLaneI32x4: |
| 279 |
modulo = 0x1f |
| 280 |
if signed { |
| 281 |
shiftOp = sseOpcodePsrad |
| 282 |
} else { |
| 283 |
shiftOp = sseOpcodePsrld |
| 284 |
} |
| 285 |
case ssa.VecLaneI64x2: |
| 286 |
modulo = 0x3f |
| 287 |
if signed { |
| 288 |
panic("BUG") |
| 289 |
} |
| 290 |
shiftOp = sseOpcodePsrlq |
| 291 |
default: |
| 292 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 293 |
} |
| 294 |
|
| 295 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 296 |
xx := m.copyToTmp(_xx.reg()) |
| 297 |
|
| 298 |
tmpGpReg := m.c.AllocateVReg(ssa.TypeI32) |
| 299 |
// Load the modulo 8 mask to tmpReg. |
| 300 |
m.lowerIconst(tmpGpReg, modulo, false) |
| 301 |
// Take the modulo 8 of the shift amount. |
| 302 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, |
| 303 |
m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)), tmpGpReg, false)) |
| 304 |
// And move it to a xmm register. |
| 305 |
tmpVec := m.c.AllocateVReg(ssa.TypeV128) |
| 306 |
m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(tmpGpReg), tmpVec, false)) |
| 307 |
|
| 308 |
// Then do the actual shift. |
| 309 |
m.insert(m.allocateInstr().asXmmRmiReg(shiftOp, newOperandReg(tmpVec), xx)) |
| 310 |
|
| 311 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 312 |
} |
| 313 |
|
| 314 |
func (m *machine) lowerVIshl(x, y, ret ssa.Value, lane ssa.VecLane) { |
| 315 |
var modulo uint64 |
| 316 |
var shiftOp sseOpcode |
| 317 |
var isI8x16 bool |
| 318 |
switch lane { |
| 319 |
case ssa.VecLaneI8x16: |
| 320 |
isI8x16 = true |
| 321 |
modulo = 0x7 |
| 322 |
shiftOp = sseOpcodePsllw |
| 323 |
case ssa.VecLaneI16x8: |
| 324 |
modulo = 0xf |
| 325 |
shiftOp = sseOpcodePsllw |
| 326 |
case ssa.VecLaneI32x4: |
| 327 |
modulo = 0x1f |
| 328 |
shiftOp = sseOpcodePslld |
| 329 |
case ssa.VecLaneI64x2: |
| 330 |
modulo = 0x3f |
| 331 |
shiftOp = sseOpcodePsllq |
| 332 |
default: |
| 333 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 334 |
} |
| 335 |
|
| 336 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 337 |
xx := m.copyToTmp(_xx.reg()) |
| 338 |
|
| 339 |
tmpGpReg := m.c.AllocateVReg(ssa.TypeI32) |
| 340 |
// Load the modulo 8 mask to tmpReg. |
| 341 |
m.lowerIconst(tmpGpReg, modulo, false) |
| 342 |
// Take the modulo 8 of the shift amount. |
| 343 |
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, |
| 344 |
m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)), tmpGpReg, false)) |
| 345 |
// And move it to a xmm register. |
| 346 |
tmpVec := m.c.AllocateVReg(ssa.TypeV128) |
| 347 |
m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(tmpGpReg), tmpVec, false)) |
| 348 |
|
| 349 |
// Then do the actual shift. |
| 350 |
m.insert(m.allocateInstr().asXmmRmiReg(shiftOp, newOperandReg(tmpVec), xx)) |
| 351 |
|
| 352 |
if isI8x16 { |
| 353 |
maskTableLabel := m.getOrAllocateConstLabel(&m.constI8x16SHLMaskTableIndex, i8x16SHLMaskTable[:]) |
| 354 |
base := m.c.AllocateVReg(ssa.TypeI64) |
| 355 |
lea := m.allocateInstr().asLEA(newOperandLabel(maskTableLabel), base) |
| 356 |
m.insert(lea) |
| 357 |
|
| 358 |
// Shift tmpGpReg by 4 to multiply the shift amount by 16. |
| 359 |
m.insert(m.allocateInstr().asShiftR(shiftROpShiftLeft, newOperandImm32(4), tmpGpReg, false)) |
| 360 |
|
| 361 |
mem := m.newAmodeRegRegShift(0, base, tmpGpReg, 0) |
| 362 |
loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(mem), tmpVec) |
| 363 |
m.insert(loadMask) |
| 364 |
|
| 365 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePand, newOperandReg(tmpVec), xx)) |
| 366 |
} |
| 367 |
|
| 368 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 369 |
} |
| 370 |
|
| 371 |
// i8x16SHLMaskTable is necessary for emulating non-existent packed bytes left shifts on amd64. |
| 372 |
// The mask is applied after performing packed word shifts on the value to clear out the unnecessary bits. |
| 373 |
var i8x16SHLMaskTable = [8 * 16]byte{ // (the number of possible shift amount 0, 1, ..., 7.) * 16 bytes. |
| 374 |
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // for 0 shift |
| 375 |
0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, // for 1 shift |
| 376 |
0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, // for 2 shift |
| 377 |
0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, // for 3 shift |
| 378 |
0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, // for 4 shift |
| 379 |
0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, // for 5 shift |
| 380 |
0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, // for 6 shift |
| 381 |
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // for 7 shift |
| 382 |
} |
| 383 |
|
| 384 |
func (m *machine) lowerVRound(x, ret ssa.Value, imm byte, _64 bool) { |
| 385 |
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 386 |
var round sseOpcode |
| 387 |
if _64 { |
| 388 |
round = sseOpcodeRoundpd |
| 389 |
} else { |
| 390 |
round = sseOpcodeRoundps |
| 391 |
} |
| 392 |
m.insert(m.allocateInstr().asXmmUnaryRmRImm(round, imm, xx, m.c.VRegOf(ret))) |
| 393 |
} |
| 394 |
|
| 395 |
var ( |
| 396 |
allOnesI8x16 = [16]byte{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1} |
| 397 |
allOnesI16x8 = [16]byte{0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0} |
| 398 |
extAddPairwiseI16x8uMask1 = [16]byte{0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80} |
| 399 |
extAddPairwiseI16x8uMask2 = [16]byte{0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00} |
| 400 |
) |
| 401 |
|
| 402 |
func (m *machine) lowerExtIaddPairwise(x, ret ssa.Value, srcLane ssa.VecLane, signed bool) { |
| 403 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 404 |
xx := m.copyToTmp(_xx.reg()) |
| 405 |
switch srcLane { |
| 406 |
case ssa.VecLaneI8x16: |
| 407 |
allOneReg := m.c.AllocateVReg(ssa.TypeV128) |
| 408 |
mask := m.getOrAllocateConstLabel(&m.constAllOnesI8x16Index, allOnesI8x16[:]) |
| 409 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), allOneReg)) |
| 410 |
|
| 411 |
var resultReg regalloc.VReg |
| 412 |
if signed { |
| 413 |
resultReg = allOneReg |
| 414 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddubsw, newOperandReg(xx), resultReg)) |
| 415 |
} else { |
| 416 |
// Interpreter tmp (all ones) as signed byte meaning that all the multiply-add is unsigned. |
| 417 |
resultReg = xx |
| 418 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddubsw, newOperandReg(allOneReg), resultReg)) |
| 419 |
} |
| 420 |
m.copyTo(resultReg, m.c.VRegOf(ret)) |
| 421 |
|
| 422 |
case ssa.VecLaneI16x8: |
| 423 |
if signed { |
| 424 |
allOnesReg := m.c.AllocateVReg(ssa.TypeV128) |
| 425 |
mask := m.getOrAllocateConstLabel(&m.constAllOnesI16x8Index, allOnesI16x8[:]) |
| 426 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), allOnesReg)) |
| 427 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, newOperandReg(allOnesReg), xx)) |
| 428 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 429 |
} else { |
| 430 |
maskReg := m.c.AllocateVReg(ssa.TypeV128) |
| 431 |
mask := m.getOrAllocateConstLabel(&m.constExtAddPairwiseI16x8uMask1Index, extAddPairwiseI16x8uMask1[:]) |
| 432 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), maskReg)) |
| 433 |
|
| 434 |
// Flip the sign bits on xx. |
| 435 |
// |
| 436 |
// Assuming that xx = [w1, ..., w8], now we have, |
| 437 |
// xx[i] = int8(-w1) for i = 0...8 |
| 438 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(maskReg), xx)) |
| 439 |
|
| 440 |
mask = m.getOrAllocateConstLabel(&m.constAllOnesI16x8Index, allOnesI16x8[:]) |
| 441 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), maskReg)) |
| 442 |
|
| 443 |
// For i = 0,..4 (as this results in i32x4 lanes), now we have |
| 444 |
// xx[i] = int32(-wn + -w(n+1)) = int32(-(wn + w(n+1))) |
| 445 |
// c.assembler.CompileRegisterToRegister(amd64.PMADDWD, tmp, vr) |
| 446 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, newOperandReg(maskReg), xx)) |
| 447 |
|
| 448 |
mask = m.getOrAllocateConstLabel(&m.constExtAddPairwiseI16x8uMask2Index, extAddPairwiseI16x8uMask2[:]) |
| 449 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), maskReg)) |
| 450 |
|
| 451 |
// vr[i] = int32(-(wn + w(n+1))) + int32(math.MaxInt16+1) = int32((wn + w(n+1))) = uint32(wn + w(n+1)). |
| 452 |
// c.assembler.CompileRegisterToRegister(amd64.PADDD, tmp, vr) |
| 453 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePaddd, newOperandReg(maskReg), xx)) |
| 454 |
|
| 455 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 456 |
} |
| 457 |
default: |
| 458 |
panic(fmt.Sprintf("invalid lane type: %s", srcLane)) |
| 459 |
} |
| 460 |
} |
| 461 |
|
| 462 |
func (m *machine) lowerWidenLow(x, ret ssa.Value, lane ssa.VecLane, signed bool) { |
| 463 |
var sseOp sseOpcode |
| 464 |
switch lane { |
| 465 |
case ssa.VecLaneI8x16: |
| 466 |
if signed { |
| 467 |
sseOp = sseOpcodePmovsxbw |
| 468 |
} else { |
| 469 |
sseOp = sseOpcodePmovzxbw |
| 470 |
} |
| 471 |
case ssa.VecLaneI16x8: |
| 472 |
if signed { |
| 473 |
sseOp = sseOpcodePmovsxwd |
| 474 |
} else { |
| 475 |
sseOp = sseOpcodePmovzxwd |
| 476 |
} |
| 477 |
case ssa.VecLaneI32x4: |
| 478 |
if signed { |
| 479 |
sseOp = sseOpcodePmovsxdq |
| 480 |
} else { |
| 481 |
sseOp = sseOpcodePmovzxdq |
| 482 |
} |
| 483 |
default: |
| 484 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 485 |
} |
| 486 |
|
| 487 |
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 488 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, xx, m.c.VRegOf(ret))) |
| 489 |
} |
| 490 |
|
| 491 |
func (m *machine) lowerWidenHigh(x, ret ssa.Value, lane ssa.VecLane, signed bool) { |
| 492 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 493 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 494 |
m.copyTo(xx.reg(), tmp) |
| 495 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePalignr, 8, newOperandReg(tmp), tmp)) |
| 496 |
|
| 497 |
var sseOp sseOpcode |
| 498 |
switch lane { |
| 499 |
case ssa.VecLaneI8x16: |
| 500 |
if signed { |
| 501 |
sseOp = sseOpcodePmovsxbw |
| 502 |
} else { |
| 503 |
sseOp = sseOpcodePmovzxbw |
| 504 |
} |
| 505 |
case ssa.VecLaneI16x8: |
| 506 |
if signed { |
| 507 |
sseOp = sseOpcodePmovsxwd |
| 508 |
} else { |
| 509 |
sseOp = sseOpcodePmovzxwd |
| 510 |
} |
| 511 |
case ssa.VecLaneI32x4: |
| 512 |
if signed { |
| 513 |
sseOp = sseOpcodePmovsxdq |
| 514 |
} else { |
| 515 |
sseOp = sseOpcodePmovzxdq |
| 516 |
} |
| 517 |
default: |
| 518 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 519 |
} |
| 520 |
|
| 521 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandReg(tmp), m.c.VRegOf(ret))) |
| 522 |
} |
| 523 |
|
| 524 |
func (m *machine) lowerLoadSplat(ptr ssa.Value, offset uint32, ret ssa.Value, lane ssa.VecLane) { |
| 525 |
tmpDst, tmpGp := m.c.AllocateVReg(ssa.TypeV128), m.c.AllocateVReg(ssa.TypeI64) |
| 526 |
am := newOperandMem(m.lowerToAddressMode(ptr, offset)) |
| 527 |
|
| 528 |
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst)) |
| 529 |
switch lane { |
| 530 |
case ssa.VecLaneI8x16: |
| 531 |
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, am, tmpGp)) |
| 532 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, newOperandReg(tmpGp), tmpDst)) |
| 533 |
tmpZeroVec := m.c.AllocateVReg(ssa.TypeV128) |
| 534 |
m.insert(m.allocateInstr().asZeros(tmpZeroVec)) |
| 535 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmpZeroVec), tmpDst)) |
| 536 |
case ssa.VecLaneI16x8: |
| 537 |
m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, am, tmpGp)) |
| 538 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, newOperandReg(tmpGp), tmpDst)) |
| 539 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, newOperandReg(tmpGp), tmpDst)) |
| 540 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) |
| 541 |
case ssa.VecLaneI32x4: |
| 542 |
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, am, tmpGp)) |
| 543 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, newOperandReg(tmpGp), tmpDst)) |
| 544 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) |
| 545 |
case ssa.VecLaneI64x2: |
| 546 |
m.insert(m.allocateInstr().asMov64MR(am, tmpGp)) |
| 547 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, newOperandReg(tmpGp), tmpDst)) |
| 548 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, newOperandReg(tmpGp), tmpDst)) |
| 549 |
default: |
| 550 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 551 |
} |
| 552 |
|
| 553 |
m.copyTo(tmpDst, m.c.VRegOf(ret)) |
| 554 |
} |
| 555 |
|
| 556 |
var f64x2CvtFromIMask = [16]byte{ |
| 557 |
0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
| 558 |
} |
| 559 |
|
| 560 |
func (m *machine) lowerVFcvtFromInt(x, ret ssa.Value, lane ssa.VecLane, signed bool) { |
| 561 |
switch lane { |
| 562 |
case ssa.VecLaneF32x4: |
| 563 |
if signed { |
| 564 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 565 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, xx, m.c.VRegOf(ret))) |
| 566 |
} else { |
| 567 |
xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 568 |
// Copy the value to two temporary registers. |
| 569 |
tmp := m.copyToTmp(xx.reg()) |
| 570 |
tmp2 := m.copyToTmp(xx.reg()) |
| 571 |
|
| 572 |
// Clear the higher 16 bits of each 32-bit element. |
| 573 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePslld, newOperandImm32(0xa), tmp)) |
| 574 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrld, newOperandImm32(0xa), tmp)) |
| 575 |
|
| 576 |
// Subtract the higher 16-bits from tmp2: clear the lower 16-bits of tmp2. |
| 577 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePsubd, newOperandReg(tmp), tmp2)) |
| 578 |
|
| 579 |
// Convert the lower 16-bits in tmp. |
| 580 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, newOperandReg(tmp), tmp)) |
| 581 |
|
| 582 |
// Left shift by one and convert tmp2, meaning that halved conversion result of higher 16-bits in tmp2. |
| 583 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp2)) |
| 584 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, newOperandReg(tmp2), tmp2)) |
| 585 |
|
| 586 |
// Double the converted halved higher 16bits. |
| 587 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAddps, newOperandReg(tmp2), tmp2)) |
| 588 |
|
| 589 |
// Get the conversion result by add tmp (holding lower 16-bit conversion) into tmp2. |
| 590 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAddps, newOperandReg(tmp), tmp2)) |
| 591 |
|
| 592 |
m.copyTo(tmp2, m.c.VRegOf(ret)) |
| 593 |
} |
| 594 |
case ssa.VecLaneF64x2: |
| 595 |
if signed { |
| 596 |
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) |
| 597 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2pd, xx, m.c.VRegOf(ret))) |
| 598 |
} else { |
| 599 |
maskReg := m.c.AllocateVReg(ssa.TypeV128) |
| 600 |
maskLabel := m.getOrAllocateConstLabel(&m.constF64x2CvtFromIMaskIndex, f64x2CvtFromIMask[:]) |
| 601 |
// maskReg = [0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] |
| 602 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), maskReg)) |
| 603 |
|
| 604 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 605 |
xx := m.copyToTmp(_xx.reg()) |
| 606 |
|
| 607 |
// Given that we have xx = [d1, d2, d3, d4], this results in |
| 608 |
// xx = [d1, [0x00, 0x00, 0x30, 0x43], d2, [0x00, 0x00, 0x30, 0x43]] |
| 609 |
// = [float64(uint32(d1)) + 0x1.0p52, float64(uint32(d2)) + 0x1.0p52] |
| 610 |
// ^See https://stackoverflow.com/questions/13269523/can-all-32-bit-ints-be-exactly-represented-as-a-double |
| 611 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeUnpcklps, newOperandReg(maskReg), xx)) |
| 612 |
|
| 613 |
// maskReg = [float64(0x1.0p52), float64(0x1.0p52)] |
| 614 |
maskLabel = m.getOrAllocateConstLabel(&m.constTwop52Index, twop52[:]) |
| 615 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), maskReg)) |
| 616 |
|
| 617 |
// Now, we get the result as |
| 618 |
// xx = [float64(uint32(d1)), float64(uint32(d2))] |
| 619 |
// because the following equality always satisfies: |
| 620 |
// float64(0x1.0p52 + float64(uint32(x))) - float64(0x1.0p52 + float64(uint32(y))) = float64(uint32(x)) - float64(uint32(y)) |
| 621 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeSubpd, newOperandReg(maskReg), xx)) |
| 622 |
|
| 623 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 624 |
} |
| 625 |
default: |
| 626 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 627 |
} |
| 628 |
} |
| 629 |
|
| 630 |
var ( |
| 631 |
// i32sMaxOnF64x2 holds math.MaxInt32(=2147483647.0) on two f64 lanes. |
| 632 |
i32sMaxOnF64x2 = [16]byte{ |
| 633 |
0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, 0xdf, 0x41, // float64(2147483647.0) |
| 634 |
0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, 0xdf, 0x41, // float64(2147483647.0) |
| 635 |
} |
| 636 |
|
| 637 |
// i32sMaxOnF64x2 holds math.MaxUint32(=4294967295.0) on two f64 lanes. |
| 638 |
i32uMaxOnF64x2 = [16]byte{ |
| 639 |
0x00, 0x00, 0xe0, 0xff, 0xff, 0xff, 0xef, 0x41, // float64(4294967295.0) |
| 640 |
0x00, 0x00, 0xe0, 0xff, 0xff, 0xff, 0xef, 0x41, // float64(4294967295.0) |
| 641 |
} |
| 642 |
|
| 643 |
// twop52 holds two float64(0x1.0p52) on two f64 lanes. 0x1.0p52 is special in the sense that |
| 644 |
// with this exponent, the mantissa represents a corresponding uint32 number, and arithmetics, |
| 645 |
// like addition or subtraction, the resulted floating point holds exactly the same |
| 646 |
// bit representations in 32-bit integer on its mantissa. |
| 647 |
// |
| 648 |
// Note: the name twop52 is common across various compiler ecosystem. |
| 649 |
// E.g. https://github.com/llvm/llvm-project/blob/92ab024f81e5b64e258b7c3baaf213c7c26fcf40/compiler-rt/lib/builtins/floatdidf.c#L28 |
| 650 |
// E.g. https://opensource.apple.com/source/clang/clang-425.0.24/src/projects/compiler-rt/lib/floatdidf.c.auto.html |
| 651 |
twop52 = [16]byte{ |
| 652 |
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, // float64(0x1.0p52) |
| 653 |
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, // float64(0x1.0p52) |
| 654 |
} |
| 655 |
) |
| 656 |
|
| 657 |
func (m *machine) lowerVFcvtToIntSat(x, ret ssa.Value, lane ssa.VecLane, signed bool) { |
| 658 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 659 |
xx := m.copyToTmp(_xx.reg()) |
| 660 |
|
| 661 |
switch lane { |
| 662 |
case ssa.VecLaneF32x4: |
| 663 |
if signed { |
| 664 |
tmp := m.copyToTmp(xx) |
| 665 |
|
| 666 |
// Assuming we have xx = [v1, v2, v3, v4]. |
| 667 |
// |
| 668 |
// Set all bits if lane is not NaN on tmp. |
| 669 |
// tmp[i] = 0xffffffff if vi != NaN |
| 670 |
// = 0 if vi == NaN |
| 671 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeCmpps, uint8(cmpPredEQ_OQ), newOperandReg(tmp), tmp)) |
| 672 |
|
| 673 |
// Clear NaN lanes on xx, meaning that |
| 674 |
// xx[i] = vi if vi != NaN |
| 675 |
// 0 if vi == NaN |
| 676 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAndps, newOperandReg(tmp), xx)) |
| 677 |
|
| 678 |
// tmp[i] = ^vi if vi != NaN |
| 679 |
// = 0xffffffff if vi == NaN |
| 680 |
// which means that tmp[i] & 0x80000000 != 0 if and only if vi is negative. |
| 681 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeXorps, newOperandReg(xx), tmp)) |
| 682 |
|
| 683 |
// xx[i] = int32(vi) if vi != NaN and xx is not overflowing. |
| 684 |
// = 0x80000000 if vi != NaN and xx is overflowing (See https://www.felixcloutier.com/x86/cvttps2dq) |
| 685 |
// = 0 if vi == NaN |
| 686 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttps2dq, newOperandReg(xx), xx)) |
| 687 |
|
| 688 |
// Below, we have to convert 0x80000000 into 0x7FFFFFFF for positive overflowing lane. |
| 689 |
// |
| 690 |
// tmp[i] = 0x80000000 if vi is positive |
| 691 |
// = any satisfying any&0x80000000 = 0 if vi is negative or zero. |
| 692 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAndps, newOperandReg(xx), tmp)) |
| 693 |
|
| 694 |
// Arithmetic right shifting tmp by 31, meaning that we have |
| 695 |
// tmp[i] = 0xffffffff if vi is positive, 0 otherwise. |
| 696 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrad, newOperandImm32(0x1f), tmp)) |
| 697 |
|
| 698 |
// Flipping 0x80000000 if vi is positive, otherwise keep intact. |
| 699 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), xx)) |
| 700 |
} else { |
| 701 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 702 |
m.insert(m.allocateInstr().asZeros(tmp)) |
| 703 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMaxps, newOperandReg(tmp), xx)) |
| 704 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp)) |
| 705 |
m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrld, newOperandImm32(0x1), tmp)) |
| 706 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, newOperandReg(tmp), tmp)) |
| 707 |
tmp2 := m.copyToTmp(xx) |
| 708 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttps2dq, newOperandReg(xx), xx)) |
| 709 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeSubps, newOperandReg(tmp), tmp2)) |
| 710 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeCmpps, uint8(cmpPredLE_OS), newOperandReg(tmp2), tmp)) |
| 711 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttps2dq, newOperandReg(tmp2), tmp2)) |
| 712 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp2)) |
| 713 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp)) |
| 714 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaxsd, newOperandReg(tmp), tmp2)) |
| 715 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePaddd, newOperandReg(tmp2), xx)) |
| 716 |
} |
| 717 |
|
| 718 |
case ssa.VecLaneF64x2: |
| 719 |
tmp2 := m.c.AllocateVReg(ssa.TypeV128) |
| 720 |
if signed { |
| 721 |
tmp := m.copyToTmp(xx) |
| 722 |
|
| 723 |
// Set all bits for non-NaN lanes, zeros otherwise. |
| 724 |
// I.e. tmp[i] = 0xffffffff_ffffffff if vi != NaN, 0 otherwise. |
| 725 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_OQ), newOperandReg(tmp), tmp)) |
| 726 |
|
| 727 |
maskLabel := m.getOrAllocateConstLabel(&m.constI32sMaxOnF64x2Index, i32sMaxOnF64x2[:]) |
| 728 |
// Load the 2147483647 into tmp2's each lane. |
| 729 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), tmp2)) |
| 730 |
|
| 731 |
// tmp[i] = 2147483647 if vi != NaN, 0 otherwise. |
| 732 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAndps, newOperandReg(tmp2), tmp)) |
| 733 |
|
| 734 |
// MINPD returns the source register's value as-is, so we have |
| 735 |
// xx[i] = vi if vi != NaN |
| 736 |
// = 0 if vi == NaN |
| 737 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMinpd, newOperandReg(tmp), xx)) |
| 738 |
|
| 739 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttpd2dq, newOperandReg(xx), xx)) |
| 740 |
} else { |
| 741 |
tmp := m.c.AllocateVReg(ssa.TypeV128) |
| 742 |
m.insert(m.allocateInstr().asZeros(tmp)) |
| 743 |
|
| 744 |
// xx[i] = vi if vi != NaN && vi > 0 |
| 745 |
// = 0 if vi == NaN || vi <= 0 |
| 746 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMaxpd, newOperandReg(tmp), xx)) |
| 747 |
|
| 748 |
// tmp2[i] = float64(math.MaxUint32) = math.MaxUint32 |
| 749 |
maskIndex := m.getOrAllocateConstLabel(&m.constI32uMaxOnF64x2Index, i32uMaxOnF64x2[:]) |
| 750 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskIndex)), tmp2)) |
| 751 |
|
| 752 |
// xx[i] = vi if vi != NaN && vi > 0 && vi <= math.MaxUint32 |
| 753 |
// = 0 otherwise |
| 754 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMinpd, newOperandReg(tmp2), xx)) |
| 755 |
|
| 756 |
// Round the floating points into integer. |
| 757 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeRoundpd, 0x3, newOperandReg(xx), xx)) |
| 758 |
|
| 759 |
// tmp2[i] = float64(0x1.0p52) |
| 760 |
maskIndex = m.getOrAllocateConstLabel(&m.constTwop52Index, twop52[:]) |
| 761 |
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskIndex)), tmp2)) |
| 762 |
|
| 763 |
// xx[i] = float64(0x1.0p52) + float64(uint32(vi)) if vi != NaN && vi > 0 && vi <= math.MaxUint32 |
| 764 |
// = 0 otherwise |
| 765 |
// |
| 766 |
// This means that xx[i] holds exactly the same bit of uint32(vi) in its lower 32-bits. |
| 767 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAddpd, newOperandReg(tmp2), xx)) |
| 768 |
|
| 769 |
// At this point, we have |
| 770 |
// xx = [uint32(v0), float64(0x1.0p52), uint32(v1), float64(0x1.0p52)] |
| 771 |
// tmp = [0, 0, 0, 0] |
| 772 |
// as 32x4 lanes. Therefore, SHUFPS with 0b00_00_10_00 results in |
| 773 |
// xx = [xx[00], xx[10], tmp[00], tmp[00]] = [xx[00], xx[10], 0, 0] |
| 774 |
// meaning that for i = 0 and 1, we have |
| 775 |
// xx[i] = uint32(vi) if vi != NaN && vi > 0 && vi <= math.MaxUint32 |
| 776 |
// = 0 otherwise. |
| 777 |
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeShufps, 0b00_00_10_00, newOperandReg(tmp), xx)) |
| 778 |
} |
| 779 |
default: |
| 780 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 781 |
} |
| 782 |
|
| 783 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 784 |
} |
| 785 |
|
| 786 |
func (m *machine) lowerNarrow(x, y, ret ssa.Value, lane ssa.VecLane, signed bool) { |
| 787 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 788 |
xx := m.copyToTmp(_xx.reg()) |
| 789 |
yy := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 790 |
|
| 791 |
var sseOp sseOpcode |
| 792 |
switch lane { |
| 793 |
case ssa.VecLaneI16x8: |
| 794 |
if signed { |
| 795 |
sseOp = sseOpcodePacksswb |
| 796 |
} else { |
| 797 |
sseOp = sseOpcodePackuswb |
| 798 |
} |
| 799 |
case ssa.VecLaneI32x4: |
| 800 |
if signed { |
| 801 |
sseOp = sseOpcodePackssdw |
| 802 |
} else { |
| 803 |
sseOp = sseOpcodePackusdw |
| 804 |
} |
| 805 |
default: |
| 806 |
panic(fmt.Sprintf("invalid lane type: %s", lane)) |
| 807 |
} |
| 808 |
m.insert(m.allocateInstr().asXmmRmR(sseOp, yy, xx)) |
| 809 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 810 |
} |
| 811 |
|
| 812 |
func (m *machine) lowerWideningPairwiseDotProductS(x, y, ret ssa.Value) { |
| 813 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 814 |
xx := m.copyToTmp(_xx.reg()) |
| 815 |
yy := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) |
| 816 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, yy, xx)) |
| 817 |
m.copyTo(xx, m.c.VRegOf(ret)) |
| 818 |
} |
| 819 |
|
| 820 |
func (m *machine) lowerVIabs(instr *ssa.Instruction) { |
| 821 |
x, lane := instr.ArgWithLane() |
| 822 |
rd := m.c.VRegOf(instr.Return()) |
| 823 |
|
| 824 |
if lane == ssa.VecLaneI64x2 { |
| 825 |
_xx := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 826 |
|
| 827 |
blendReg := xmm0VReg |
| 828 |
m.insert(m.allocateInstr().asDefineUninitializedReg(blendReg)) |
| 829 |
|
| 830 |
tmp := m.copyToTmp(_xx.reg()) |
| 831 |
xx := m.copyToTmp(_xx.reg()) |
| 832 |
|
| 833 |
// Clear all bits on blendReg. |
| 834 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(blendReg), blendReg)) |
| 835 |
// Subtract xx from blendMaskReg. |
| 836 |
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePsubq, newOperandReg(xx), blendReg)) |
| 837 |
// Copy the subtracted value ^^ back into tmp. |
| 838 |
m.copyTo(blendReg, xx) |
| 839 |
|
| 840 |
m.insert(m.allocateInstr().asBlendvpd(newOperandReg(tmp), xx)) |
| 841 |
|
| 842 |
m.copyTo(xx, rd) |
| 843 |
} else { |
| 844 |
var vecOp sseOpcode |
| 845 |
switch lane { |
| 846 |
case ssa.VecLaneI8x16: |
| 847 |
vecOp = sseOpcodePabsb |
| 848 |
case ssa.VecLaneI16x8: |
| 849 |
vecOp = sseOpcodePabsw |
| 850 |
case ssa.VecLaneI32x4: |
| 851 |
vecOp = sseOpcodePabsd |
| 852 |
} |
| 853 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 854 |
|
| 855 |
i := m.allocateInstr() |
| 856 |
i.asXmmUnaryRmR(vecOp, rn, rd) |
| 857 |
m.insert(i) |
| 858 |
} |
| 859 |
} |
| 860 |
|
| 861 |
func (m *machine) lowerVIpopcnt(instr *ssa.Instruction) { |
| 862 |
x := instr.Arg() |
| 863 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 864 |
rd := m.c.VRegOf(instr.Return()) |
| 865 |
|
| 866 |
tmp1 := m.c.AllocateVReg(ssa.TypeV128) |
| 867 |
m.lowerVconst(tmp1, 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f) |
| 868 |
|
| 869 |
// Copy input into tmp2. |
| 870 |
tmp2 := m.copyToTmp(rn.reg()) |
| 871 |
|
| 872 |
// Given that we have: |
| 873 |
// rm = [b1, ..., b16] where bn = hn:ln and hn and ln are higher and lower 4-bits of bn. |
| 874 |
// |
| 875 |
// Take PAND on tmp1 and tmp2, so that we mask out all the higher bits. |
| 876 |
// tmp2 = [l1, ..., l16]. |
| 877 |
pand := m.allocateInstr() |
| 878 |
pand.asXmmRmR(sseOpcodePand, newOperandReg(tmp1), tmp2) |
| 879 |
m.insert(pand) |
| 880 |
|
| 881 |
// Do logical (packed word) right shift by 4 on rm and PAND against the mask (tmp1); meaning that we have |
| 882 |
// tmp3 = [h1, ...., h16]. |
| 883 |
tmp3 := m.copyToTmp(rn.reg()) |
| 884 |
psrlw := m.allocateInstr() |
| 885 |
psrlw.asXmmRmiReg(sseOpcodePsrlw, newOperandImm32(4), tmp3) |
| 886 |
m.insert(psrlw) |
| 887 |
|
| 888 |
pand2 := m.allocateInstr() |
| 889 |
pand2.asXmmRmR(sseOpcodePand, newOperandReg(tmp1), tmp3) |
| 890 |
m.insert(pand2) |
| 891 |
|
| 892 |
// Read the popcntTable into tmp4, and we have |
| 893 |
// tmp4 = [0x00, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x03, 0x01, 0x02, 0x02, 0x03, 0x02, 0x03, 0x03, 0x04] |
| 894 |
tmp4 := m.c.AllocateVReg(ssa.TypeV128) |
| 895 |
m.lowerVconst(tmp4, 0x03_02_02_01_02_01_01_00, 0x04_03_03_02_03_02_02_01) |
| 896 |
|
| 897 |
// Make a copy for later. |
| 898 |
tmp5 := m.copyToTmp(tmp4) |
| 899 |
|
| 900 |
// tmp4 = [popcnt(l1), ..., popcnt(l16)]. |
| 901 |
pshufb := m.allocateInstr() |
| 902 |
pshufb.asXmmRmR(sseOpcodePshufb, newOperandReg(tmp2), tmp4) |
| 903 |
m.insert(pshufb) |
| 904 |
|
| 905 |
pshufb2 := m.allocateInstr() |
| 906 |
pshufb2.asXmmRmR(sseOpcodePshufb, newOperandReg(tmp3), tmp5) |
| 907 |
m.insert(pshufb2) |
| 908 |
|
| 909 |
// tmp4 + tmp5 is the result. |
| 910 |
paddb := m.allocateInstr() |
| 911 |
paddb.asXmmRmR(sseOpcodePaddb, newOperandReg(tmp4), tmp5) |
| 912 |
m.insert(paddb) |
| 913 |
|
| 914 |
m.copyTo(tmp5, rd) |
| 915 |
} |
| 916 |
|
| 917 |
func (m *machine) lowerVImul(instr *ssa.Instruction) { |
| 918 |
x, y, lane := instr.Arg2WithLane() |
| 919 |
rd := m.c.VRegOf(instr.Return()) |
| 920 |
if lane == ssa.VecLaneI64x2 { |
| 921 |
rn := m.getOperand_Reg(m.c.ValueDefinition(x)) |
| 922 |
rm := m.getOperand_Reg(m.c.ValueDefinition(y)) |
| 923 |
// Assuming that we have |
| 924 |
// rm = [p1, p2] = [p1_lo, p1_hi, p2_lo, p2_high] |
| 925 |
// rn = [q1, q2] = [q1_lo, q1_hi, q2_lo, q2_high] |
| 926 |
// where pN and qN are 64-bit (quad word) lane, and pN_lo, pN_hi, qN_lo and qN_hi are 32-bit (double word) lane. |
| 927 |
|
| 928 |
// Copy rn into tmp1. |
| 929 |
tmp1 := m.copyToTmp(rn.reg()) |
| 930 |
|
| 931 |
// And do the logical right shift by 32-bit on tmp1, which makes tmp1 = [0, p1_high, 0, p2_high] |
| 932 |
shift := m.allocateInstr() |
| 933 |
shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(32), tmp1) |
| 934 |
m.insert(shift) |
| 935 |
|
| 936 |
// Execute "pmuludq rm,tmp1", which makes tmp1 = [p1_high*q1_lo, p2_high*q2_lo] where each lane is 64-bit. |
| 937 |
mul := m.allocateInstr() |
| 938 |
mul.asXmmRmR(sseOpcodePmuludq, rm, tmp1) |
| 939 |
m.insert(mul) |
| 940 |
|
| 941 |
// Copy rm value into tmp2. |
| 942 |
tmp2 := m.copyToTmp(rm.reg()) |
| 943 |
|
| 944 |
// And do the logical right shift by 32-bit on tmp2, which makes tmp2 = [0, q1_high, 0, q2_high] |
| 945 |
shift2 := m.allocateInstr() |
| 946 |
shift2.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(32), tmp2) |
| 947 |
m.insert(shift2) |
| 948 |
|
| 949 |
// Execute "pmuludq rm,tmp2", which makes tmp2 = [p1_lo*q1_high, p2_lo*q2_high] where each lane is 64-bit. |
| 950 |
mul2 := m.allocateInstr() |
| 951 |
mul2.asXmmRmR(sseOpcodePmuludq, rn, tmp2) |
| 952 |
m.insert(mul2) |
| 953 |
|
| 954 |
// Adds tmp1 and tmp2 and do the logical left shift by 32-bit, |
| 955 |
// which makes tmp1 = [(p1_lo*q1_high+p1_high*q1_lo)<<32, (p2_lo*q2_high+p2_high*q2_lo)<<32] |
| 956 |
add := m.allocateInstr() |
| 957 |
add.asXmmRmR(sseOpcodePaddq, newOperandReg(tmp2), tmp1) |
| 958 |
m.insert(add) |
| 959 |
|
| 960 |
shift3 := m.allocateInstr() |
| 961 |
shift3.asXmmRmiReg(sseOpcodePsllq, newOperandImm32(32), tmp1) |
| 962 |
m.insert(shift3) |
| 963 |
|
| 964 |
// Copy rm value into tmp3. |
| 965 |
tmp3 := m.copyToTmp(rm.reg()) |
| 966 |
|
| 967 |
// "pmuludq rm,tmp3" makes tmp3 = [p1_lo*q1_lo, p2_lo*q2_lo] where each lane is 64-bit. |
| 968 |
mul3 := m.allocateInstr() |
| 969 |
mul3.asXmmRmR(sseOpcodePmuludq, rn, tmp3) |
| 970 |
m.insert(mul3) |
| 971 |
|
| 972 |
// Finally, we get the result by computing tmp1 + tmp3, |
| 973 |
// which makes tmp1 = [(p1_lo*q1_high+p1_high*q1_lo)<<32+p1_lo*q1_lo, (p2_lo*q2_high+p2_high*q2_lo)<<32+p2_lo*q2_lo] |
| 974 |
add2 := m.allocateInstr() |
| 975 |
add2.asXmmRmR(sseOpcodePaddq, newOperandReg(tmp3), tmp1) |
| 976 |
m.insert(add2) |
| 977 |
|
| 978 |
m.copyTo(tmp1, rd) |
| 979 |
|
| 980 |
} else { |
| 981 |
var vecOp sseOpcode |
| 982 |
switch lane { |
| 983 |
case ssa.VecLaneI16x8: |
| 984 |
vecOp = sseOpcodePmullw |
| 985 |
case ssa.VecLaneI32x4: |
| 986 |
vecOp = sseOpcodePmulld |
| 987 |
default: |
| 988 |
panic("unsupported: " + lane.String()) |
| 989 |
} |
| 990 |
m.lowerVbBinOp(vecOp, x, y, instr.Return()) |
| 991 |
} |
| 992 |
} |