| 1 |
package arm64 |
| 2 |
|
| 3 |
import ( |
| 4 |
"context" |
| 5 |
"fmt" |
| 6 |
|
| 7 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend" |
| 8 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" |
| 9 |
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" |
| 10 |
) |
| 11 |
|
| 12 |
// Encode implements backend.Machine Encode. |
| 13 |
func (m *machine) Encode(ctx context.Context) error { |
| 14 |
m.resolveRelativeAddresses(ctx) |
| 15 |
m.encode(m.rootInstr) |
| 16 |
if l := len(m.compiler.Buf()); l > maxFunctionExecutableSize { |
| 17 |
return fmt.Errorf("function size exceeds the limit: %d > %d", l, maxFunctionExecutableSize) |
| 18 |
} |
| 19 |
return nil |
| 20 |
} |
| 21 |
|
| 22 |
func (m *machine) encode(root *instruction) { |
| 23 |
for cur := root; cur != nil; cur = cur.next { |
| 24 |
cur.encode(m) |
| 25 |
} |
| 26 |
} |
| 27 |
|
| 28 |
func (i *instruction) encode(m *machine) { |
| 29 |
c := m.compiler |
| 30 |
switch kind := i.kind; kind { |
| 31 |
case nop0, emitSourceOffsetInfo, loadConstBlockArg: |
| 32 |
case exitSequence: |
| 33 |
encodeExitSequence(c, i.rn.reg()) |
| 34 |
case ret: |
| 35 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en |
| 36 |
c.Emit4Bytes(encodeRet()) |
| 37 |
case br: |
| 38 |
imm := i.brOffset() |
| 39 |
c.Emit4Bytes(encodeUnconditionalBranch(false, imm)) |
| 40 |
case call: |
| 41 |
// We still don't know the exact address of the function to call, so we emit a placeholder. |
| 42 |
c.AddRelocationInfo(i.callFuncRef(), false) |
| 43 |
c.Emit4Bytes(encodeUnconditionalBranch(true, 0)) // 0 = placeholder |
| 44 |
case callInd: |
| 45 |
c.Emit4Bytes(encodeUnconditionalBranchReg(regNumberInEncoding[i.rn.realReg()], true)) |
| 46 |
case store8, store16, store32, store64, fpuStore32, fpuStore64, fpuStore128: |
| 47 |
c.Emit4Bytes(encodeLoadOrStore(i.kind, regNumberInEncoding[i.rn.realReg()], *i.getAmode())) |
| 48 |
case uLoad8, uLoad16, uLoad32, uLoad64, sLoad8, sLoad16, sLoad32, fpuLoad32, fpuLoad64, fpuLoad128: |
| 49 |
c.Emit4Bytes(encodeLoadOrStore(i.kind, regNumberInEncoding[i.rd.RealReg()], *i.getAmode())) |
| 50 |
case vecLoad1R: |
| 51 |
c.Emit4Bytes(encodeVecLoad1R( |
| 52 |
regNumberInEncoding[i.rd.RealReg()], |
| 53 |
regNumberInEncoding[i.rn.realReg()], |
| 54 |
vecArrangement(i.u1))) |
| 55 |
case condBr: |
| 56 |
imm19 := i.condBrOffset() |
| 57 |
if imm19%4 != 0 { |
| 58 |
panic("imm26 for branch must be a multiple of 4") |
| 59 |
} |
| 60 |
|
| 61 |
imm19U32 := uint32(imm19/4) & 0b111_11111111_11111111 |
| 62 |
brCond := i.condBrCond() |
| 63 |
switch brCond.kind() { |
| 64 |
case condKindRegisterZero: |
| 65 |
rt := regNumberInEncoding[brCond.register().RealReg()] |
| 66 |
c.Emit4Bytes(encodeCBZCBNZ(rt, false, imm19U32, i.condBr64bit())) |
| 67 |
case condKindRegisterNotZero: |
| 68 |
rt := regNumberInEncoding[brCond.register().RealReg()] |
| 69 |
c.Emit4Bytes(encodeCBZCBNZ(rt, true, imm19U32, i.condBr64bit())) |
| 70 |
case condKindCondFlagSet: |
| 71 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B-cond--Branch-conditionally- |
| 72 |
fl := brCond.flag() |
| 73 |
c.Emit4Bytes(0b01010100<<24 | (imm19U32 << 5) | uint32(fl)) |
| 74 |
default: |
| 75 |
panic("BUG") |
| 76 |
} |
| 77 |
case movN: |
| 78 |
c.Emit4Bytes(encodeMoveWideImmediate(0b00, regNumberInEncoding[i.rd.RealReg()], i.u1, uint32(i.u2), uint32(i.u2>>32))) |
| 79 |
case movZ: |
| 80 |
c.Emit4Bytes(encodeMoveWideImmediate(0b10, regNumberInEncoding[i.rd.RealReg()], i.u1, uint32(i.u2), uint32(i.u2>>32))) |
| 81 |
case movK: |
| 82 |
c.Emit4Bytes(encodeMoveWideImmediate(0b11, regNumberInEncoding[i.rd.RealReg()], i.u1, uint32(i.u2), uint32(i.u2>>32))) |
| 83 |
case mov32: |
| 84 |
to, from := i.rd.RealReg(), i.rn.realReg() |
| 85 |
c.Emit4Bytes(encodeAsMov32(regNumberInEncoding[from], regNumberInEncoding[to])) |
| 86 |
case mov64: |
| 87 |
to, from := i.rd.RealReg(), i.rn.realReg() |
| 88 |
toIsSp := to == sp |
| 89 |
fromIsSp := from == sp |
| 90 |
c.Emit4Bytes(encodeMov64(regNumberInEncoding[to], regNumberInEncoding[from], toIsSp, fromIsSp)) |
| 91 |
case loadP64, storeP64: |
| 92 |
rt, rt2 := regNumberInEncoding[i.rn.realReg()], regNumberInEncoding[i.rm.realReg()] |
| 93 |
amode := i.getAmode() |
| 94 |
rn := regNumberInEncoding[amode.rn.RealReg()] |
| 95 |
var pre bool |
| 96 |
switch amode.kind { |
| 97 |
case addressModeKindPostIndex: |
| 98 |
case addressModeKindPreIndex: |
| 99 |
pre = true |
| 100 |
default: |
| 101 |
panic("BUG") |
| 102 |
} |
| 103 |
c.Emit4Bytes(encodePreOrPostIndexLoadStorePair64(pre, kind == loadP64, rn, rt, rt2, amode.imm)) |
| 104 |
case loadFpuConst32: |
| 105 |
rd := regNumberInEncoding[i.rd.RealReg()] |
| 106 |
if i.u1 == 0 { |
| 107 |
c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement8B)) |
| 108 |
} else { |
| 109 |
encodeLoadFpuConst32(c, rd, i.u1) |
| 110 |
} |
| 111 |
case loadFpuConst64: |
| 112 |
rd := regNumberInEncoding[i.rd.RealReg()] |
| 113 |
if i.u1 == 0 { |
| 114 |
c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement8B)) |
| 115 |
} else { |
| 116 |
encodeLoadFpuConst64(c, regNumberInEncoding[i.rd.RealReg()], i.u1) |
| 117 |
} |
| 118 |
case loadFpuConst128: |
| 119 |
rd := regNumberInEncoding[i.rd.RealReg()] |
| 120 |
lo, hi := i.u1, i.u2 |
| 121 |
if lo == 0 && hi == 0 { |
| 122 |
c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement16B)) |
| 123 |
} else { |
| 124 |
encodeLoadFpuConst128(c, rd, lo, hi) |
| 125 |
} |
| 126 |
case aluRRRR: |
| 127 |
c.Emit4Bytes(encodeAluRRRR( |
| 128 |
aluOp(i.u1), |
| 129 |
regNumberInEncoding[i.rd.RealReg()], |
| 130 |
regNumberInEncoding[i.rn.realReg()], |
| 131 |
regNumberInEncoding[i.rm.realReg()], |
| 132 |
regNumberInEncoding[regalloc.VReg(i.u2).RealReg()], |
| 133 |
uint32(i.u1>>32), |
| 134 |
)) |
| 135 |
case aluRRImmShift: |
| 136 |
c.Emit4Bytes(encodeAluRRImm( |
| 137 |
aluOp(i.u1), |
| 138 |
regNumberInEncoding[i.rd.RealReg()], |
| 139 |
regNumberInEncoding[i.rn.realReg()], |
| 140 |
uint32(i.rm.shiftImm()), |
| 141 |
uint32(i.u2>>32), |
| 142 |
)) |
| 143 |
case aluRRR: |
| 144 |
rn := i.rn.realReg() |
| 145 |
c.Emit4Bytes(encodeAluRRR( |
| 146 |
aluOp(i.u1), |
| 147 |
regNumberInEncoding[i.rd.RealReg()], |
| 148 |
regNumberInEncoding[rn], |
| 149 |
regNumberInEncoding[i.rm.realReg()], |
| 150 |
i.u2>>32 == 1, |
| 151 |
rn == sp, |
| 152 |
)) |
| 153 |
case aluRRRExtend: |
| 154 |
rm, exo, to := i.rm.er() |
| 155 |
c.Emit4Bytes(encodeAluRRRExtend( |
| 156 |
aluOp(i.u1), |
| 157 |
regNumberInEncoding[i.rd.RealReg()], |
| 158 |
regNumberInEncoding[i.rn.realReg()], |
| 159 |
regNumberInEncoding[rm.RealReg()], |
| 160 |
exo, |
| 161 |
to, |
| 162 |
)) |
| 163 |
case aluRRRShift: |
| 164 |
r, amt, sop := i.rm.sr() |
| 165 |
c.Emit4Bytes(encodeAluRRRShift( |
| 166 |
aluOp(i.u1), |
| 167 |
regNumberInEncoding[i.rd.RealReg()], |
| 168 |
regNumberInEncoding[i.rn.realReg()], |
| 169 |
regNumberInEncoding[r.RealReg()], |
| 170 |
uint32(amt), |
| 171 |
sop, |
| 172 |
i.u2>>32 == 1, |
| 173 |
)) |
| 174 |
case aluRRBitmaskImm: |
| 175 |
c.Emit4Bytes(encodeAluBitmaskImmediate( |
| 176 |
aluOp(i.u1), |
| 177 |
regNumberInEncoding[i.rd.RealReg()], |
| 178 |
regNumberInEncoding[i.rn.realReg()], |
| 179 |
i.u2, |
| 180 |
i.u1>>32 == 1, |
| 181 |
)) |
| 182 |
case bitRR: |
| 183 |
c.Emit4Bytes(encodeBitRR( |
| 184 |
bitOp(i.u1), |
| 185 |
regNumberInEncoding[i.rd.RealReg()], |
| 186 |
regNumberInEncoding[i.rn.realReg()], |
| 187 |
uint32(i.u2)), |
| 188 |
) |
| 189 |
case aluRRImm12: |
| 190 |
imm12, shift := i.rm.imm12() |
| 191 |
c.Emit4Bytes(encodeAluRRImm12( |
| 192 |
aluOp(i.u1), |
| 193 |
regNumberInEncoding[i.rd.RealReg()], |
| 194 |
regNumberInEncoding[i.rn.realReg()], |
| 195 |
imm12, shift, |
| 196 |
i.u2>>32 == 1, |
| 197 |
)) |
| 198 |
case fpuRRR: |
| 199 |
c.Emit4Bytes(encodeFpuRRR( |
| 200 |
fpuBinOp(i.u1), |
| 201 |
regNumberInEncoding[i.rd.RealReg()], |
| 202 |
regNumberInEncoding[i.rn.realReg()], |
| 203 |
regNumberInEncoding[i.rm.realReg()], |
| 204 |
i.u2 == 1, |
| 205 |
)) |
| 206 |
case fpuMov64, fpuMov128: |
| 207 |
// https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/MOV--vector---Move-vector--an-alias-of-ORR--vector--register-- |
| 208 |
rd := regNumberInEncoding[i.rd.RealReg()] |
| 209 |
rn := regNumberInEncoding[i.rn.realReg()] |
| 210 |
var q uint32 |
| 211 |
if kind == fpuMov128 { |
| 212 |
q = 0b1 |
| 213 |
} |
| 214 |
c.Emit4Bytes(q<<30 | 0b1110101<<21 | rn<<16 | 0b000111<<10 | rn<<5 | rd) |
| 215 |
case cSet: |
| 216 |
rd := regNumberInEncoding[i.rd.RealReg()] |
| 217 |
cf := condFlag(i.u1) |
| 218 |
if i.u2 == 1 { |
| 219 |
// https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/CSETM--Conditional-Set-Mask--an-alias-of-CSINV- |
| 220 |
// Note that we set 64bit version here. |
| 221 |
c.Emit4Bytes(0b1101101010011111<<16 | uint32(cf.invert())<<12 | 0b011111<<5 | rd) |
| 222 |
} else { |
| 223 |
// https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/CSET--Conditional-Set--an-alias-of-CSINC- |
| 224 |
// Note that we set 64bit version here. |
| 225 |
c.Emit4Bytes(0b1001101010011111<<16 | uint32(cf.invert())<<12 | 0b111111<<5 | rd) |
| 226 |
} |
| 227 |
case extend: |
| 228 |
c.Emit4Bytes(encodeExtend((i.u2>>32) == 1, byte(i.u1), byte(i.u2), regNumberInEncoding[i.rd.RealReg()], regNumberInEncoding[i.rn.realReg()])) |
| 229 |
case fpuCmp: |
| 230 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/FCMP--Floating-point-quiet-Compare--scalar--?lang=en |
| 231 |
rn, rm := regNumberInEncoding[i.rn.realReg()], regNumberInEncoding[i.rm.realReg()] |
| 232 |
var ftype uint32 |
| 233 |
if i.u1 == 1 { |
| 234 |
ftype = 0b01 // double precision. |
| 235 |
} |
| 236 |
c.Emit4Bytes(0b1111<<25 | ftype<<22 | 1<<21 | rm<<16 | 0b1<<13 | rn<<5) |
| 237 |
case udf: |
| 238 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UDF--Permanently-Undefined-?lang=en |
| 239 |
if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { |
| 240 |
c.Emit4Bytes(dummyInstruction) |
| 241 |
} else { |
| 242 |
c.Emit4Bytes(0) |
| 243 |
} |
| 244 |
case adr: |
| 245 |
c.Emit4Bytes(encodeAdr(regNumberInEncoding[i.rd.RealReg()], uint32(i.u1))) |
| 246 |
case cSel: |
| 247 |
c.Emit4Bytes(encodeConditionalSelect( |
| 248 |
kind, |
| 249 |
regNumberInEncoding[i.rd.RealReg()], |
| 250 |
regNumberInEncoding[i.rn.realReg()], |
| 251 |
regNumberInEncoding[i.rm.realReg()], |
| 252 |
condFlag(i.u1), |
| 253 |
i.u2 == 1, |
| 254 |
)) |
| 255 |
case fpuCSel: |
| 256 |
c.Emit4Bytes(encodeFpuCSel( |
| 257 |
regNumberInEncoding[i.rd.RealReg()], |
| 258 |
regNumberInEncoding[i.rn.realReg()], |
| 259 |
regNumberInEncoding[i.rm.realReg()], |
| 260 |
condFlag(i.u1), |
| 261 |
i.u2 == 1, |
| 262 |
)) |
| 263 |
case movToVec: |
| 264 |
c.Emit4Bytes(encodeMoveToVec( |
| 265 |
regNumberInEncoding[i.rd.RealReg()], |
| 266 |
regNumberInEncoding[i.rn.realReg()], |
| 267 |
vecArrangement(byte(i.u1)), |
| 268 |
vecIndex(i.u2), |
| 269 |
)) |
| 270 |
case movFromVec, movFromVecSigned: |
| 271 |
c.Emit4Bytes(encodeMoveFromVec( |
| 272 |
regNumberInEncoding[i.rd.RealReg()], |
| 273 |
regNumberInEncoding[i.rn.realReg()], |
| 274 |
vecArrangement(byte(i.u1)), |
| 275 |
vecIndex(i.u2), |
| 276 |
i.kind == movFromVecSigned, |
| 277 |
)) |
| 278 |
case vecDup: |
| 279 |
c.Emit4Bytes(encodeVecDup( |
| 280 |
regNumberInEncoding[i.rd.RealReg()], |
| 281 |
regNumberInEncoding[i.rn.realReg()], |
| 282 |
vecArrangement(byte(i.u1)))) |
| 283 |
case vecDupElement: |
| 284 |
c.Emit4Bytes(encodeVecDupElement( |
| 285 |
regNumberInEncoding[i.rd.RealReg()], |
| 286 |
regNumberInEncoding[i.rn.realReg()], |
| 287 |
vecArrangement(byte(i.u1)), |
| 288 |
vecIndex(i.u2))) |
| 289 |
case vecExtract: |
| 290 |
c.Emit4Bytes(encodeVecExtract( |
| 291 |
regNumberInEncoding[i.rd.RealReg()], |
| 292 |
regNumberInEncoding[i.rn.realReg()], |
| 293 |
regNumberInEncoding[i.rm.realReg()], |
| 294 |
vecArrangement(byte(i.u1)), |
| 295 |
uint32(i.u2))) |
| 296 |
case vecPermute: |
| 297 |
c.Emit4Bytes(encodeVecPermute( |
| 298 |
vecOp(i.u1), |
| 299 |
regNumberInEncoding[i.rd.RealReg()], |
| 300 |
regNumberInEncoding[i.rn.realReg()], |
| 301 |
regNumberInEncoding[i.rm.realReg()], |
| 302 |
vecArrangement(byte(i.u2)))) |
| 303 |
case vecMovElement: |
| 304 |
c.Emit4Bytes(encodeVecMovElement( |
| 305 |
regNumberInEncoding[i.rd.RealReg()], |
| 306 |
regNumberInEncoding[i.rn.realReg()], |
| 307 |
vecArrangement(i.u1), |
| 308 |
uint32(i.u2), uint32(i.u2>>32), |
| 309 |
)) |
| 310 |
case vecMisc: |
| 311 |
c.Emit4Bytes(encodeAdvancedSIMDTwoMisc( |
| 312 |
vecOp(i.u1), |
| 313 |
regNumberInEncoding[i.rd.RealReg()], |
| 314 |
regNumberInEncoding[i.rn.realReg()], |
| 315 |
vecArrangement(i.u2), |
| 316 |
)) |
| 317 |
case vecLanes: |
| 318 |
c.Emit4Bytes(encodeVecLanes( |
| 319 |
vecOp(i.u1), |
| 320 |
regNumberInEncoding[i.rd.RealReg()], |
| 321 |
regNumberInEncoding[i.rn.realReg()], |
| 322 |
vecArrangement(i.u2), |
| 323 |
)) |
| 324 |
case vecShiftImm: |
| 325 |
c.Emit4Bytes(encodeVecShiftImm( |
| 326 |
vecOp(i.u1), |
| 327 |
regNumberInEncoding[i.rd.RealReg()], |
| 328 |
regNumberInEncoding[i.rn.realReg()], |
| 329 |
uint32(i.rm.shiftImm()), |
| 330 |
vecArrangement(i.u2), |
| 331 |
)) |
| 332 |
case vecTbl: |
| 333 |
c.Emit4Bytes(encodeVecTbl( |
| 334 |
1, |
| 335 |
regNumberInEncoding[i.rd.RealReg()], |
| 336 |
regNumberInEncoding[i.rn.realReg()], |
| 337 |
regNumberInEncoding[i.rm.realReg()], |
| 338 |
vecArrangement(i.u2)), |
| 339 |
) |
| 340 |
case vecTbl2: |
| 341 |
c.Emit4Bytes(encodeVecTbl( |
| 342 |
2, |
| 343 |
regNumberInEncoding[i.rd.RealReg()], |
| 344 |
regNumberInEncoding[i.rn.realReg()], |
| 345 |
regNumberInEncoding[i.rm.realReg()], |
| 346 |
vecArrangement(i.u2)), |
| 347 |
) |
| 348 |
case brTableSequence: |
| 349 |
targets := m.jmpTableTargets[i.u1] |
| 350 |
encodeBrTableSequence(c, i.rn.reg(), targets) |
| 351 |
case fpuToInt, intToFpu: |
| 352 |
c.Emit4Bytes(encodeCnvBetweenFloatInt(i)) |
| 353 |
case fpuRR: |
| 354 |
c.Emit4Bytes(encodeFloatDataOneSource( |
| 355 |
fpuUniOp(i.u1), |
| 356 |
regNumberInEncoding[i.rd.RealReg()], |
| 357 |
regNumberInEncoding[i.rn.realReg()], |
| 358 |
i.u2 == 1, |
| 359 |
)) |
| 360 |
case vecRRR: |
| 361 |
if op := vecOp(i.u1); op == vecOpBsl || op == vecOpBit || op == vecOpUmlal { |
| 362 |
panic(fmt.Sprintf("vecOp %s must use vecRRRRewrite instead of vecRRR", op.String())) |
| 363 |
} |
| 364 |
fallthrough |
| 365 |
case vecRRRRewrite: |
| 366 |
c.Emit4Bytes(encodeVecRRR( |
| 367 |
vecOp(i.u1), |
| 368 |
regNumberInEncoding[i.rd.RealReg()], |
| 369 |
regNumberInEncoding[i.rn.realReg()], |
| 370 |
regNumberInEncoding[i.rm.realReg()], |
| 371 |
vecArrangement(i.u2), |
| 372 |
)) |
| 373 |
case cCmpImm: |
| 374 |
// Conditional compare (immediate) in https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en |
| 375 |
sf := uint32((i.u2 >> 32) & 0b1) |
| 376 |
nzcv := uint32(i.u2 & 0b1111) |
| 377 |
cond := uint32(condFlag(i.u1)) |
| 378 |
imm := uint32(i.rm.data & 0b11111) |
| 379 |
rn := regNumberInEncoding[i.rn.realReg()] |
| 380 |
c.Emit4Bytes( |
| 381 |
sf<<31 | 0b111101001<<22 | imm<<16 | cond<<12 | 0b1<<11 | rn<<5 | nzcv, |
| 382 |
) |
| 383 |
case movFromFPSR: |
| 384 |
rt := regNumberInEncoding[i.rd.RealReg()] |
| 385 |
c.Emit4Bytes(encodeSystemRegisterMove(rt, true)) |
| 386 |
case movToFPSR: |
| 387 |
rt := regNumberInEncoding[i.rn.realReg()] |
| 388 |
c.Emit4Bytes(encodeSystemRegisterMove(rt, false)) |
| 389 |
case atomicRmw: |
| 390 |
c.Emit4Bytes(encodeAtomicRmw( |
| 391 |
atomicRmwOp(i.u1), |
| 392 |
regNumberInEncoding[i.rm.realReg()], |
| 393 |
regNumberInEncoding[i.rd.RealReg()], |
| 394 |
regNumberInEncoding[i.rn.realReg()], |
| 395 |
uint32(i.u2), |
| 396 |
)) |
| 397 |
case atomicCas: |
| 398 |
c.Emit4Bytes(encodeAtomicCas( |
| 399 |
regNumberInEncoding[i.rd.RealReg()], |
| 400 |
regNumberInEncoding[i.rm.realReg()], |
| 401 |
regNumberInEncoding[i.rn.realReg()], |
| 402 |
uint32(i.u2), |
| 403 |
)) |
| 404 |
case atomicLoad: |
| 405 |
c.Emit4Bytes(encodeAtomicLoadStore( |
| 406 |
regNumberInEncoding[i.rn.realReg()], |
| 407 |
regNumberInEncoding[i.rd.RealReg()], |
| 408 |
uint32(i.u2), |
| 409 |
1, |
| 410 |
)) |
| 411 |
case atomicStore: |
| 412 |
c.Emit4Bytes(encodeAtomicLoadStore( |
| 413 |
regNumberInEncoding[i.rn.realReg()], |
| 414 |
regNumberInEncoding[i.rm.realReg()], |
| 415 |
uint32(i.u2), |
| 416 |
0, |
| 417 |
)) |
| 418 |
case dmb: |
| 419 |
c.Emit4Bytes(encodeDMB()) |
| 420 |
case tailCall: |
| 421 |
// We still don't know the exact address of the function to call, so we emit a placeholder. |
| 422 |
c.AddRelocationInfo(i.callFuncRef(), true) // true = IsTailCall |
| 423 |
c.Emit4Bytes(encodeUnconditionalBranch(false, 0)) // 0 = placeholder |
| 424 |
case tailCallInd: |
| 425 |
c.Emit4Bytes(encodeUnconditionalBranchReg(regNumberInEncoding[i.rn.realReg()], false)) |
| 426 |
default: |
| 427 |
panic(i.String()) |
| 428 |
} |
| 429 |
} |
| 430 |
|
| 431 |
func encodeMov64(rd, rn uint32, toIsSp, fromIsSp bool) uint32 { |
| 432 |
if toIsSp || fromIsSp { |
| 433 |
// This is an alias of ADD (immediate): |
| 434 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--to-from-SP---Move-between-register-and-stack-pointer--an-alias-of-ADD--immediate-- |
| 435 |
return encodeAddSubtractImmediate(0b100, 0, 0, rn, rd) |
| 436 |
} else { |
| 437 |
// This is an alias of ORR (shifted register): |
| 438 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register-- |
| 439 |
return encodeLogicalShiftedRegister(0b101, 0, rn, 0, regNumberInEncoding[xzr], rd) |
| 440 |
} |
| 441 |
} |
| 442 |
|
| 443 |
// encodeSystemRegisterMove encodes as "System register move" in |
| 444 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en |
| 445 |
// |
| 446 |
// Note that currently we only supports read/write of FPSR. |
| 447 |
func encodeSystemRegisterMove(rt uint32, fromSystem bool) uint32 { |
| 448 |
ret := 0b11010101<<24 | 0b11011<<16 | 0b01000100<<8 | 0b001<<5 | rt |
| 449 |
if fromSystem { |
| 450 |
ret |= 0b1 << 21 |
| 451 |
} |
| 452 |
return ret |
| 453 |
} |
| 454 |
|
| 455 |
// encodeVecRRR encodes as either "Advanced SIMD three *" in |
| 456 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 457 |
func encodeVecRRR(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 { |
| 458 |
switch op { |
| 459 |
case vecOpBit: |
| 460 |
_, q := arrToSizeQEncoded(arr) |
| 461 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b10 /* always has size 0b10 */, 0b1, q) |
| 462 |
case vecOpBic: |
| 463 |
if arr > vecArrangement16B { |
| 464 |
panic("unsupported arrangement: " + arr.String()) |
| 465 |
} |
| 466 |
_, q := arrToSizeQEncoded(arr) |
| 467 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b01 /* always has size 0b01 */, 0b0, q) |
| 468 |
case vecOpBsl: |
| 469 |
if arr > vecArrangement16B { |
| 470 |
panic("unsupported arrangement: " + arr.String()) |
| 471 |
} |
| 472 |
_, q := arrToSizeQEncoded(arr) |
| 473 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b01 /* always has size 0b01 */, 0b1, q) |
| 474 |
case vecOpAnd: |
| 475 |
if arr > vecArrangement16B { |
| 476 |
panic("unsupported arrangement: " + arr.String()) |
| 477 |
} |
| 478 |
_, q := arrToSizeQEncoded(arr) |
| 479 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b00 /* always has size 0b00 */, 0b0, q) |
| 480 |
case vecOpOrr: |
| 481 |
_, q := arrToSizeQEncoded(arr) |
| 482 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b10 /* always has size 0b10 */, 0b0, q) |
| 483 |
case vecOpEOR: |
| 484 |
size, q := arrToSizeQEncoded(arr) |
| 485 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, size, 0b1, q) |
| 486 |
case vecOpCmeq: |
| 487 |
size, q := arrToSizeQEncoded(arr) |
| 488 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10001, size, 0b1, q) |
| 489 |
case vecOpCmgt: |
| 490 |
size, q := arrToSizeQEncoded(arr) |
| 491 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00110, size, 0b0, q) |
| 492 |
case vecOpCmhi: |
| 493 |
size, q := arrToSizeQEncoded(arr) |
| 494 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00110, size, 0b1, q) |
| 495 |
case vecOpCmge: |
| 496 |
size, q := arrToSizeQEncoded(arr) |
| 497 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00111, size, 0b0, q) |
| 498 |
case vecOpCmhs: |
| 499 |
size, q := arrToSizeQEncoded(arr) |
| 500 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00111, size, 0b1, q) |
| 501 |
case vecOpFcmeq: |
| 502 |
var size, q uint32 |
| 503 |
switch arr { |
| 504 |
case vecArrangement4S: |
| 505 |
size, q = 0b00, 0b1 |
| 506 |
case vecArrangement2S: |
| 507 |
size, q = 0b00, 0b0 |
| 508 |
case vecArrangement2D: |
| 509 |
size, q = 0b01, 0b1 |
| 510 |
default: |
| 511 |
panic("unsupported arrangement: " + arr.String()) |
| 512 |
} |
| 513 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b0, q) |
| 514 |
case vecOpFcmgt: |
| 515 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 516 |
panic("unsupported arrangement: " + arr.String()) |
| 517 |
} |
| 518 |
size, q := arrToSizeQEncoded(arr) |
| 519 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b1, q) |
| 520 |
case vecOpFcmge: |
| 521 |
var size, q uint32 |
| 522 |
switch arr { |
| 523 |
case vecArrangement4S: |
| 524 |
size, q = 0b00, 0b1 |
| 525 |
case vecArrangement2S: |
| 526 |
size, q = 0b00, 0b0 |
| 527 |
case vecArrangement2D: |
| 528 |
size, q = 0b01, 0b1 |
| 529 |
default: |
| 530 |
panic("unsupported arrangement: " + arr.String()) |
| 531 |
} |
| 532 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b1, q) |
| 533 |
case vecOpAdd: |
| 534 |
if arr == vecArrangement1D { |
| 535 |
panic("unsupported arrangement: " + arr.String()) |
| 536 |
} |
| 537 |
size, q := arrToSizeQEncoded(arr) |
| 538 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10000, size, 0b0, q) |
| 539 |
case vecOpSqadd: |
| 540 |
if arr == vecArrangement1D { |
| 541 |
panic("unsupported arrangement: " + arr.String()) |
| 542 |
} |
| 543 |
size, q := arrToSizeQEncoded(arr) |
| 544 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00001, size, 0b0, q) |
| 545 |
case vecOpUqadd: |
| 546 |
if arr == vecArrangement1D { |
| 547 |
panic("unsupported arrangement: " + arr.String()) |
| 548 |
} |
| 549 |
size, q := arrToSizeQEncoded(arr) |
| 550 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00001, size, 0b1, q) |
| 551 |
case vecOpAddp: |
| 552 |
if arr == vecArrangement1D { |
| 553 |
panic("unsupported arrangement: " + arr.String()) |
| 554 |
} |
| 555 |
size, q := arrToSizeQEncoded(arr) |
| 556 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10111, size, 0b0, q) |
| 557 |
case vecOpSqsub: |
| 558 |
if arr == vecArrangement1D { |
| 559 |
panic("unsupported arrangement: " + arr.String()) |
| 560 |
} |
| 561 |
size, q := arrToSizeQEncoded(arr) |
| 562 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00101, size, 0b0, q) |
| 563 |
case vecOpUqsub: |
| 564 |
if arr == vecArrangement1D { |
| 565 |
panic("unsupported arrangement: " + arr.String()) |
| 566 |
} |
| 567 |
size, q := arrToSizeQEncoded(arr) |
| 568 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00101, size, 0b1, q) |
| 569 |
case vecOpSub: |
| 570 |
if arr == vecArrangement1D { |
| 571 |
panic("unsupported arrangement: " + arr.String()) |
| 572 |
} |
| 573 |
size, q := arrToSizeQEncoded(arr) |
| 574 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10000, size, 0b1, q) |
| 575 |
case vecOpFmin: |
| 576 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 577 |
panic("unsupported arrangement: " + arr.String()) |
| 578 |
} |
| 579 |
size, q := arrToSizeQEncoded(arr) |
| 580 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11110, size, 0b0, q) |
| 581 |
case vecOpSmin: |
| 582 |
if arr > vecArrangement4S { |
| 583 |
panic("unsupported arrangement: " + arr.String()) |
| 584 |
} |
| 585 |
size, q := arrToSizeQEncoded(arr) |
| 586 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01101, size, 0b0, q) |
| 587 |
case vecOpUmin: |
| 588 |
if arr > vecArrangement4S { |
| 589 |
panic("unsupported arrangement: " + arr.String()) |
| 590 |
} |
| 591 |
size, q := arrToSizeQEncoded(arr) |
| 592 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01101, size, 0b1, q) |
| 593 |
case vecOpFmax: |
| 594 |
var size, q uint32 |
| 595 |
switch arr { |
| 596 |
case vecArrangement4S: |
| 597 |
size, q = 0b00, 0b1 |
| 598 |
case vecArrangement2S: |
| 599 |
size, q = 0b00, 0b0 |
| 600 |
case vecArrangement2D: |
| 601 |
size, q = 0b01, 0b1 |
| 602 |
default: |
| 603 |
panic("unsupported arrangement: " + arr.String()) |
| 604 |
} |
| 605 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11110, size, 0b0, q) |
| 606 |
case vecOpFadd: |
| 607 |
var size, q uint32 |
| 608 |
switch arr { |
| 609 |
case vecArrangement4S: |
| 610 |
size, q = 0b00, 0b1 |
| 611 |
case vecArrangement2S: |
| 612 |
size, q = 0b00, 0b0 |
| 613 |
case vecArrangement2D: |
| 614 |
size, q = 0b01, 0b1 |
| 615 |
default: |
| 616 |
panic("unsupported arrangement: " + arr.String()) |
| 617 |
} |
| 618 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11010, size, 0b0, q) |
| 619 |
case vecOpFsub: |
| 620 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 621 |
panic("unsupported arrangement: " + arr.String()) |
| 622 |
} |
| 623 |
size, q := arrToSizeQEncoded(arr) |
| 624 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11010, size, 0b0, q) |
| 625 |
case vecOpFmul: |
| 626 |
var size, q uint32 |
| 627 |
switch arr { |
| 628 |
case vecArrangement4S: |
| 629 |
size, q = 0b00, 0b1 |
| 630 |
case vecArrangement2S: |
| 631 |
size, q = 0b00, 0b0 |
| 632 |
case vecArrangement2D: |
| 633 |
size, q = 0b01, 0b1 |
| 634 |
default: |
| 635 |
panic("unsupported arrangement: " + arr.String()) |
| 636 |
} |
| 637 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11011, size, 0b1, q) |
| 638 |
case vecOpSqrdmulh: |
| 639 |
if arr < vecArrangement4H || arr > vecArrangement4S { |
| 640 |
panic("unsupported arrangement: " + arr.String()) |
| 641 |
} |
| 642 |
size, q := arrToSizeQEncoded(arr) |
| 643 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10110, size, 0b1, q) |
| 644 |
case vecOpFdiv: |
| 645 |
var size, q uint32 |
| 646 |
switch arr { |
| 647 |
case vecArrangement4S: |
| 648 |
size, q = 0b00, 0b1 |
| 649 |
case vecArrangement2S: |
| 650 |
size, q = 0b00, 0b0 |
| 651 |
case vecArrangement2D: |
| 652 |
size, q = 0b01, 0b1 |
| 653 |
default: |
| 654 |
panic("unsupported arrangement: " + arr.String()) |
| 655 |
} |
| 656 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11111, size, 0b1, q) |
| 657 |
case vecOpSmax: |
| 658 |
if arr > vecArrangement4S { |
| 659 |
panic("unsupported arrangement: " + arr.String()) |
| 660 |
} |
| 661 |
size, q := arrToSizeQEncoded(arr) |
| 662 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01100, size, 0b0, q) |
| 663 |
case vecOpUmax: |
| 664 |
if arr > vecArrangement4S { |
| 665 |
panic("unsupported arrangement: " + arr.String()) |
| 666 |
} |
| 667 |
size, q := arrToSizeQEncoded(arr) |
| 668 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01100, size, 0b1, q) |
| 669 |
case vecOpUmaxp: |
| 670 |
if arr > vecArrangement4S { |
| 671 |
panic("unsupported arrangement: " + arr.String()) |
| 672 |
} |
| 673 |
size, q := arrToSizeQEncoded(arr) |
| 674 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10100, size, 0b1, q) |
| 675 |
case vecOpUrhadd: |
| 676 |
if arr > vecArrangement4S { |
| 677 |
panic("unsupported arrangement: " + arr.String()) |
| 678 |
} |
| 679 |
size, q := arrToSizeQEncoded(arr) |
| 680 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00010, size, 0b1, q) |
| 681 |
case vecOpMul: |
| 682 |
if arr > vecArrangement4S { |
| 683 |
panic("unsupported arrangement: " + arr.String()) |
| 684 |
} |
| 685 |
size, q := arrToSizeQEncoded(arr) |
| 686 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10011, size, 0b0, q) |
| 687 |
case vecOpUmlal: |
| 688 |
if arr > vecArrangement4S { |
| 689 |
panic("unsupported arrangement: " + arr.String()) |
| 690 |
} |
| 691 |
size, q := arrToSizeQEncoded(arr) |
| 692 |
return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1000, size, 0b1, q) |
| 693 |
case vecOpSshl: |
| 694 |
if arr == vecArrangement1D { |
| 695 |
panic("unsupported arrangement: " + arr.String()) |
| 696 |
} |
| 697 |
size, q := arrToSizeQEncoded(arr) |
| 698 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b0, q) |
| 699 |
case vecOpUshl: |
| 700 |
if arr == vecArrangement1D { |
| 701 |
panic("unsupported arrangement: " + arr.String()) |
| 702 |
} |
| 703 |
size, q := arrToSizeQEncoded(arr) |
| 704 |
return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b1, q) |
| 705 |
|
| 706 |
case vecOpSmull: |
| 707 |
if arr > vecArrangement4S { |
| 708 |
panic("unsupported arrangement: " + arr.String()) |
| 709 |
} |
| 710 |
size, _ := arrToSizeQEncoded(arr) |
| 711 |
return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b0) |
| 712 |
|
| 713 |
case vecOpSmull2: |
| 714 |
if arr > vecArrangement4S { |
| 715 |
panic("unsupported arrangement: " + arr.String()) |
| 716 |
} |
| 717 |
size, _ := arrToSizeQEncoded(arr) |
| 718 |
return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b1) |
| 719 |
|
| 720 |
default: |
| 721 |
panic("TODO: " + op.String()) |
| 722 |
} |
| 723 |
} |
| 724 |
|
| 725 |
func arrToSizeQEncoded(arr vecArrangement) (size, q uint32) { |
| 726 |
switch arr { |
| 727 |
case vecArrangement16B: |
| 728 |
q = 0b1 |
| 729 |
fallthrough |
| 730 |
case vecArrangement8B: |
| 731 |
size = 0b00 |
| 732 |
case vecArrangement8H: |
| 733 |
q = 0b1 |
| 734 |
fallthrough |
| 735 |
case vecArrangement4H: |
| 736 |
size = 0b01 |
| 737 |
case vecArrangement4S: |
| 738 |
q = 0b1 |
| 739 |
fallthrough |
| 740 |
case vecArrangement2S: |
| 741 |
size = 0b10 |
| 742 |
case vecArrangement2D: |
| 743 |
q = 0b1 |
| 744 |
fallthrough |
| 745 |
case vecArrangement1D: |
| 746 |
size = 0b11 |
| 747 |
default: |
| 748 |
panic("BUG") |
| 749 |
} |
| 750 |
return |
| 751 |
} |
| 752 |
|
| 753 |
// encodeAdvancedSIMDThreeSame encodes as "Advanced SIMD three same" in |
| 754 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 755 |
func encodeAdvancedSIMDThreeSame(rd, rn, rm, opcode, size, U, Q uint32) uint32 { |
| 756 |
return Q<<30 | U<<29 | 0b111<<25 | size<<22 | 0b1<<21 | rm<<16 | opcode<<11 | 0b1<<10 | rn<<5 | rd |
| 757 |
} |
| 758 |
|
| 759 |
// encodeAdvancedSIMDThreeDifferent encodes as "Advanced SIMD three different" in |
| 760 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 761 |
func encodeAdvancedSIMDThreeDifferent(rd, rn, rm, opcode, size, U, Q uint32) uint32 { |
| 762 |
return Q<<30 | U<<29 | 0b111<<25 | size<<22 | 0b1<<21 | rm<<16 | opcode<<12 | rn<<5 | rd |
| 763 |
} |
| 764 |
|
| 765 |
// encodeFloatDataOneSource encodes as "Floating-point data-processing (1 source)" in |
| 766 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp |
| 767 |
func encodeFloatDataOneSource(op fpuUniOp, rd, rn uint32, dst64bit bool) uint32 { |
| 768 |
var opcode, ptype uint32 |
| 769 |
switch op { |
| 770 |
case fpuUniOpCvt32To64: |
| 771 |
opcode = 0b000101 |
| 772 |
case fpuUniOpCvt64To32: |
| 773 |
opcode = 0b000100 |
| 774 |
ptype = 0b01 |
| 775 |
case fpuUniOpNeg: |
| 776 |
opcode = 0b000010 |
| 777 |
if dst64bit { |
| 778 |
ptype = 0b01 |
| 779 |
} |
| 780 |
case fpuUniOpSqrt: |
| 781 |
opcode = 0b000011 |
| 782 |
if dst64bit { |
| 783 |
ptype = 0b01 |
| 784 |
} |
| 785 |
case fpuUniOpRoundPlus: |
| 786 |
opcode = 0b001001 |
| 787 |
if dst64bit { |
| 788 |
ptype = 0b01 |
| 789 |
} |
| 790 |
case fpuUniOpRoundMinus: |
| 791 |
opcode = 0b001010 |
| 792 |
if dst64bit { |
| 793 |
ptype = 0b01 |
| 794 |
} |
| 795 |
case fpuUniOpRoundZero: |
| 796 |
opcode = 0b001011 |
| 797 |
if dst64bit { |
| 798 |
ptype = 0b01 |
| 799 |
} |
| 800 |
case fpuUniOpRoundNearest: |
| 801 |
opcode = 0b001000 |
| 802 |
if dst64bit { |
| 803 |
ptype = 0b01 |
| 804 |
} |
| 805 |
case fpuUniOpAbs: |
| 806 |
opcode = 0b000001 |
| 807 |
if dst64bit { |
| 808 |
ptype = 0b01 |
| 809 |
} |
| 810 |
default: |
| 811 |
panic("BUG") |
| 812 |
} |
| 813 |
return 0b1111<<25 | ptype<<22 | 0b1<<21 | opcode<<15 | 0b1<<14 | rn<<5 | rd |
| 814 |
} |
| 815 |
|
| 816 |
// encodeCnvBetweenFloatInt encodes as "Conversion between floating-point and integer" in |
| 817 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 818 |
func encodeCnvBetweenFloatInt(i *instruction) uint32 { |
| 819 |
rd := regNumberInEncoding[i.rd.RealReg()] |
| 820 |
rn := regNumberInEncoding[i.rn.realReg()] |
| 821 |
|
| 822 |
var opcode uint32 |
| 823 |
var rmode uint32 |
| 824 |
var ptype uint32 |
| 825 |
var sf uint32 |
| 826 |
switch i.kind { |
| 827 |
case intToFpu: // Either UCVTF or SCVTF. |
| 828 |
rmode = 0b00 |
| 829 |
|
| 830 |
signed := i.u1 == 1 |
| 831 |
src64bit := i.u2&1 != 0 |
| 832 |
dst64bit := i.u2&2 != 0 |
| 833 |
if signed { |
| 834 |
opcode = 0b010 |
| 835 |
} else { |
| 836 |
opcode = 0b011 |
| 837 |
} |
| 838 |
if src64bit { |
| 839 |
sf = 0b1 |
| 840 |
} |
| 841 |
if dst64bit { |
| 842 |
ptype = 0b01 |
| 843 |
} else { |
| 844 |
ptype = 0b00 |
| 845 |
} |
| 846 |
case fpuToInt: // Either FCVTZU or FCVTZS. |
| 847 |
rmode = 0b11 |
| 848 |
|
| 849 |
signed := i.u1 == 1 |
| 850 |
src64bit := i.u2&1 != 0 |
| 851 |
dst64bit := i.u2&2 != 0 |
| 852 |
|
| 853 |
if signed { |
| 854 |
opcode = 0b000 |
| 855 |
} else { |
| 856 |
opcode = 0b001 |
| 857 |
} |
| 858 |
if dst64bit { |
| 859 |
sf = 0b1 |
| 860 |
} |
| 861 |
if src64bit { |
| 862 |
ptype = 0b01 |
| 863 |
} else { |
| 864 |
ptype = 0b00 |
| 865 |
} |
| 866 |
} |
| 867 |
return sf<<31 | 0b1111<<25 | ptype<<22 | 0b1<<21 | rmode<<19 | opcode<<16 | rn<<5 | rd |
| 868 |
} |
| 869 |
|
| 870 |
// encodeAdr encodes a PC-relative ADR instruction. |
| 871 |
// https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/ADR--Form-PC-relative-address- |
| 872 |
func encodeAdr(rd uint32, offset uint32) uint32 { |
| 873 |
if offset >= 1<<20 { |
| 874 |
panic("BUG: too large adr instruction") |
| 875 |
} |
| 876 |
return offset&0b11<<29 | 0b1<<28 | offset&0b1111111111_1111111100<<3 | rd |
| 877 |
} |
| 878 |
|
| 879 |
// encodeFpuCSel encodes as "Floating-point conditional select" in |
| 880 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 881 |
func encodeFpuCSel(rd, rn, rm uint32, c condFlag, _64bit bool) uint32 { |
| 882 |
var ftype uint32 |
| 883 |
if _64bit { |
| 884 |
ftype = 0b01 // double precision. |
| 885 |
} |
| 886 |
return 0b1111<<25 | ftype<<22 | 0b1<<21 | rm<<16 | uint32(c)<<12 | 0b11<<10 | rn<<5 | rd |
| 887 |
} |
| 888 |
|
| 889 |
// encodeMoveToVec encodes as "Move general-purpose register to a vector element" (represented as `ins`) in |
| 890 |
// https://developer.arm.com/documentation/dui0801/g/A64-SIMD-Vector-Instructions/MOV--vector--from-general- |
| 891 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--from-general---Move-general-purpose-register-to-a-vector-element--an-alias-of-INS--general--?lang=en |
| 892 |
func encodeMoveToVec(rd, rn uint32, arr vecArrangement, index vecIndex) uint32 { |
| 893 |
var imm5 uint32 |
| 894 |
switch arr { |
| 895 |
case vecArrangementB: |
| 896 |
imm5 |= 0b1 |
| 897 |
imm5 |= uint32(index) << 1 |
| 898 |
if index > 0b1111 { |
| 899 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", index)) |
| 900 |
} |
| 901 |
case vecArrangementH: |
| 902 |
imm5 |= 0b10 |
| 903 |
imm5 |= uint32(index) << 2 |
| 904 |
if index > 0b111 { |
| 905 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", index)) |
| 906 |
} |
| 907 |
case vecArrangementS: |
| 908 |
imm5 |= 0b100 |
| 909 |
imm5 |= uint32(index) << 3 |
| 910 |
if index > 0b11 { |
| 911 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", index)) |
| 912 |
} |
| 913 |
case vecArrangementD: |
| 914 |
imm5 |= 0b1000 |
| 915 |
imm5 |= uint32(index) << 4 |
| 916 |
if index > 0b1 { |
| 917 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", index)) |
| 918 |
} |
| 919 |
default: |
| 920 |
panic("Unsupported arrangement " + arr.String()) |
| 921 |
} |
| 922 |
|
| 923 |
return 0b01001110000<<21 | imm5<<16 | 0b000111<<10 | rn<<5 | rd |
| 924 |
} |
| 925 |
|
| 926 |
// encodeMoveToVec encodes as "Move vector element to another vector element, mov (element)" (represented as `ins`) in |
| 927 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--element---Move-vector-element-to-another-vector-element--an-alias-of-INS--element--?lang=en |
| 928 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/INS--element---Insert-vector-element-from-another-vector-element-?lang=en |
| 929 |
func encodeVecMovElement(rd, rn uint32, arr vecArrangement, srcIndex, dstIndex uint32) uint32 { |
| 930 |
var imm4, imm5 uint32 |
| 931 |
switch arr { |
| 932 |
case vecArrangementB: |
| 933 |
imm5 |= 0b1 |
| 934 |
imm5 |= srcIndex << 1 |
| 935 |
imm4 = dstIndex |
| 936 |
if srcIndex > 0b1111 { |
| 937 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", srcIndex)) |
| 938 |
} |
| 939 |
case vecArrangementH: |
| 940 |
imm5 |= 0b10 |
| 941 |
imm5 |= srcIndex << 2 |
| 942 |
imm4 = dstIndex << 1 |
| 943 |
if srcIndex > 0b111 { |
| 944 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", srcIndex)) |
| 945 |
} |
| 946 |
case vecArrangementS: |
| 947 |
imm5 |= 0b100 |
| 948 |
imm5 |= srcIndex << 3 |
| 949 |
imm4 = dstIndex << 2 |
| 950 |
if srcIndex > 0b11 { |
| 951 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", srcIndex)) |
| 952 |
} |
| 953 |
case vecArrangementD: |
| 954 |
imm5 |= 0b1000 |
| 955 |
imm5 |= srcIndex << 4 |
| 956 |
imm4 = dstIndex << 3 |
| 957 |
if srcIndex > 0b1 { |
| 958 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", srcIndex)) |
| 959 |
} |
| 960 |
default: |
| 961 |
panic("Unsupported arrangement " + arr.String()) |
| 962 |
} |
| 963 |
|
| 964 |
return 0b01101110000<<21 | imm5<<16 | imm4<<11 | 0b1<<10 | rn<<5 | rd |
| 965 |
} |
| 966 |
|
| 967 |
// encodeUnconditionalBranchReg encodes as "Unconditional branch (register)" in: |
| 968 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en |
| 969 |
func encodeUnconditionalBranchReg(rn uint32, link bool) uint32 { |
| 970 |
var opc uint32 |
| 971 |
if link { |
| 972 |
opc = 0b0001 |
| 973 |
} |
| 974 |
return 0b1101011<<25 | opc<<21 | 0b11111<<16 | rn<<5 |
| 975 |
} |
| 976 |
|
| 977 |
// encodeMoveFromVec encodes as "Move vector element to a general-purpose register" |
| 978 |
// (represented as `umov` when dest is 32-bit, `umov` otherwise) in |
| 979 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/UMOV--Unsigned-Move-vector-element-to-general-purpose-register-?lang=en |
| 980 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--to-general---Move-vector-element-to-general-purpose-register--an-alias-of-UMOV-?lang=en |
| 981 |
func encodeMoveFromVec(rd, rn uint32, arr vecArrangement, index vecIndex, signed bool) uint32 { |
| 982 |
var op, imm4, q, imm5 uint32 |
| 983 |
switch { |
| 984 |
case arr == vecArrangementB: |
| 985 |
imm5 |= 0b1 |
| 986 |
imm5 |= uint32(index) << 1 |
| 987 |
if index > 0b1111 { |
| 988 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", index)) |
| 989 |
} |
| 990 |
case arr == vecArrangementH: |
| 991 |
imm5 |= 0b10 |
| 992 |
imm5 |= uint32(index) << 2 |
| 993 |
if index > 0b111 { |
| 994 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", index)) |
| 995 |
} |
| 996 |
case arr == vecArrangementS && signed: |
| 997 |
q = 0b1 |
| 998 |
fallthrough |
| 999 |
case arr == vecArrangementS: |
| 1000 |
imm5 |= 0b100 |
| 1001 |
imm5 |= uint32(index) << 3 |
| 1002 |
if index > 0b11 { |
| 1003 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", index)) |
| 1004 |
} |
| 1005 |
case arr == vecArrangementD && !signed: |
| 1006 |
imm5 |= 0b1000 |
| 1007 |
imm5 |= uint32(index) << 4 |
| 1008 |
q = 0b1 |
| 1009 |
if index > 0b1 { |
| 1010 |
panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", index)) |
| 1011 |
} |
| 1012 |
default: |
| 1013 |
panic("Unsupported arrangement " + arr.String()) |
| 1014 |
} |
| 1015 |
if signed { |
| 1016 |
op, imm4 = 0, 0b0101 |
| 1017 |
} else { |
| 1018 |
op, imm4 = 0, 0b0111 |
| 1019 |
} |
| 1020 |
return op<<29 | 0b01110000<<21 | q<<30 | imm5<<16 | imm4<<11 | 1<<10 | rn<<5 | rd |
| 1021 |
} |
| 1022 |
|
| 1023 |
// encodeVecDup encodes as "Duplicate general-purpose register to vector" DUP (general) |
| 1024 |
// (represented as `dup`) |
| 1025 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/DUP--general---Duplicate-general-purpose-register-to-vector-?lang=en |
| 1026 |
func encodeVecDup(rd, rn uint32, arr vecArrangement) uint32 { |
| 1027 |
var q, imm5 uint32 |
| 1028 |
switch arr { |
| 1029 |
case vecArrangement8B: |
| 1030 |
q, imm5 = 0b0, 0b1 |
| 1031 |
case vecArrangement16B: |
| 1032 |
q, imm5 = 0b1, 0b1 |
| 1033 |
case vecArrangement4H: |
| 1034 |
q, imm5 = 0b0, 0b10 |
| 1035 |
case vecArrangement8H: |
| 1036 |
q, imm5 = 0b1, 0b10 |
| 1037 |
case vecArrangement2S: |
| 1038 |
q, imm5 = 0b0, 0b100 |
| 1039 |
case vecArrangement4S: |
| 1040 |
q, imm5 = 0b1, 0b100 |
| 1041 |
case vecArrangement2D: |
| 1042 |
q, imm5 = 0b1, 0b1000 |
| 1043 |
default: |
| 1044 |
panic("Unsupported arrangement " + arr.String()) |
| 1045 |
} |
| 1046 |
return q<<30 | 0b001110000<<21 | imm5<<16 | 0b000011<<10 | rn<<5 | rd |
| 1047 |
} |
| 1048 |
|
| 1049 |
// encodeVecDup encodes as "Duplicate vector element to vector or scalar" DUP (element). |
| 1050 |
// (represented as `dup`) |
| 1051 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/DUP--element---Duplicate-vector-element-to-vector-or-scalar- |
| 1052 |
func encodeVecDupElement(rd, rn uint32, arr vecArrangement, srcIndex vecIndex) uint32 { |
| 1053 |
var q, imm5 uint32 |
| 1054 |
q = 0b1 |
| 1055 |
switch arr { |
| 1056 |
case vecArrangementB: |
| 1057 |
imm5 |= 0b1 |
| 1058 |
imm5 |= uint32(srcIndex) << 1 |
| 1059 |
case vecArrangementH: |
| 1060 |
imm5 |= 0b10 |
| 1061 |
imm5 |= uint32(srcIndex) << 2 |
| 1062 |
case vecArrangementS: |
| 1063 |
imm5 |= 0b100 |
| 1064 |
imm5 |= uint32(srcIndex) << 3 |
| 1065 |
case vecArrangementD: |
| 1066 |
imm5 |= 0b1000 |
| 1067 |
imm5 |= uint32(srcIndex) << 4 |
| 1068 |
default: |
| 1069 |
panic("unsupported arrangement" + arr.String()) |
| 1070 |
} |
| 1071 |
|
| 1072 |
return q<<30 | 0b001110000<<21 | imm5<<16 | 0b1<<10 | rn<<5 | rd |
| 1073 |
} |
| 1074 |
|
| 1075 |
// encodeVecExtract encodes as "Advanced SIMD extract." |
| 1076 |
// Currently only `ext` is defined. |
| 1077 |
// https://developer.arm.com/documentation/ddi0602/2023-06/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp |
| 1078 |
// https://developer.arm.com/documentation/ddi0602/2023-06/SIMD-FP-Instructions/EXT--Extract-vector-from-pair-of-vectors-?lang=en |
| 1079 |
func encodeVecExtract(rd, rn, rm uint32, arr vecArrangement, index uint32) uint32 { |
| 1080 |
var q, imm4 uint32 |
| 1081 |
switch arr { |
| 1082 |
case vecArrangement8B: |
| 1083 |
q, imm4 = 0, 0b0111&uint32(index) |
| 1084 |
case vecArrangement16B: |
| 1085 |
q, imm4 = 1, 0b1111&uint32(index) |
| 1086 |
default: |
| 1087 |
panic("Unsupported arrangement " + arr.String()) |
| 1088 |
} |
| 1089 |
return q<<30 | 0b101110000<<21 | rm<<16 | imm4<<11 | rn<<5 | rd |
| 1090 |
} |
| 1091 |
|
| 1092 |
// encodeVecPermute encodes as "Advanced SIMD permute." |
| 1093 |
// https://developer.arm.com/documentation/ddi0602/2023-06/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp |
| 1094 |
func encodeVecPermute(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 { |
| 1095 |
var q, size, opcode uint32 |
| 1096 |
switch op { |
| 1097 |
case vecOpZip1: |
| 1098 |
opcode = 0b011 |
| 1099 |
if arr == vecArrangement1D { |
| 1100 |
panic("unsupported arrangement: " + arr.String()) |
| 1101 |
} |
| 1102 |
size, q = arrToSizeQEncoded(arr) |
| 1103 |
default: |
| 1104 |
panic("TODO: " + op.String()) |
| 1105 |
} |
| 1106 |
return q<<30 | 0b001110<<24 | size<<22 | rm<<16 | opcode<<12 | 0b10<<10 | rn<<5 | rd |
| 1107 |
} |
| 1108 |
|
| 1109 |
// encodeConditionalSelect encodes as "Conditional select" in |
| 1110 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#condsel |
| 1111 |
func encodeConditionalSelect(kind instructionKind, rd, rn, rm uint32, c condFlag, _64bit bool) uint32 { |
| 1112 |
if kind != cSel { |
| 1113 |
panic("TODO: support other conditional select") |
| 1114 |
} |
| 1115 |
|
| 1116 |
ret := 0b110101<<23 | rm<<16 | uint32(c)<<12 | rn<<5 | rd |
| 1117 |
if _64bit { |
| 1118 |
ret |= 0b1 << 31 |
| 1119 |
} |
| 1120 |
return ret |
| 1121 |
} |
| 1122 |
|
| 1123 |
const dummyInstruction uint32 = 0x14000000 // "b 0" |
| 1124 |
|
| 1125 |
// encodeLoadFpuConst32 encodes the following three instructions: |
| 1126 |
// |
| 1127 |
// ldr s8, #8 ;; literal load of data.f32 |
| 1128 |
// b 8 ;; skip the data |
| 1129 |
// data.f32 xxxxxxx |
| 1130 |
func encodeLoadFpuConst32(c backend.Compiler, rd uint32, rawF32 uint64) { |
| 1131 |
c.Emit4Bytes( |
| 1132 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en |
| 1133 |
0b111<<26 | (0x8/4)<<5 | rd, |
| 1134 |
) |
| 1135 |
c.Emit4Bytes(encodeUnconditionalBranch(false, 8)) // b 8 |
| 1136 |
if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { |
| 1137 |
// Inlined data.f32 cannot be disassembled, so we add a dummy instruction here. |
| 1138 |
c.Emit4Bytes(dummyInstruction) |
| 1139 |
} else { |
| 1140 |
c.Emit4Bytes(uint32(rawF32)) // data.f32 xxxxxxx |
| 1141 |
} |
| 1142 |
} |
| 1143 |
|
| 1144 |
// encodeLoadFpuConst64 encodes the following three instructions: |
| 1145 |
// |
| 1146 |
// ldr d8, #8 ;; literal load of data.f64 |
| 1147 |
// b 12 ;; skip the data |
| 1148 |
// data.f64 xxxxxxx |
| 1149 |
func encodeLoadFpuConst64(c backend.Compiler, rd uint32, rawF64 uint64) { |
| 1150 |
c.Emit4Bytes( |
| 1151 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en |
| 1152 |
0b1<<30 | 0b111<<26 | (0x8/4)<<5 | rd, |
| 1153 |
) |
| 1154 |
c.Emit4Bytes(encodeUnconditionalBranch(false, 12)) // b 12 |
| 1155 |
if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { |
| 1156 |
// Inlined data.f64 cannot be disassembled, so we add dummy instructions here. |
| 1157 |
c.Emit4Bytes(dummyInstruction) |
| 1158 |
c.Emit4Bytes(dummyInstruction) |
| 1159 |
} else { |
| 1160 |
// data.f64 xxxxxxx |
| 1161 |
c.Emit4Bytes(uint32(rawF64)) |
| 1162 |
c.Emit4Bytes(uint32(rawF64 >> 32)) |
| 1163 |
} |
| 1164 |
} |
| 1165 |
|
| 1166 |
// encodeLoadFpuConst128 encodes the following three instructions: |
| 1167 |
// |
| 1168 |
// ldr v8, #8 ;; literal load of data.f64 |
| 1169 |
// b 20 ;; skip the data |
| 1170 |
// data.v128 xxxxxxx |
| 1171 |
func encodeLoadFpuConst128(c backend.Compiler, rd uint32, lo, hi uint64) { |
| 1172 |
c.Emit4Bytes( |
| 1173 |
// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en |
| 1174 |
0b1<<31 | 0b111<<26 | (0x8/4)<<5 | rd, |
| 1175 |
) |
| 1176 |
c.Emit4Bytes(encodeUnconditionalBranch(false, 20)) // b 20 |
| 1177 |
if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { |
| 1178 |
// Inlined data.v128 cannot be disassembled, so we add dummy instructions here. |
| 1179 |
c.Emit4Bytes(dummyInstruction) |
| 1180 |
c.Emit4Bytes(dummyInstruction) |
| 1181 |
c.Emit4Bytes(dummyInstruction) |
| 1182 |
c.Emit4Bytes(dummyInstruction) |
| 1183 |
} else { |
| 1184 |
// data.v128 xxxxxxx |
| 1185 |
c.Emit4Bytes(uint32(lo)) |
| 1186 |
c.Emit4Bytes(uint32(lo >> 32)) |
| 1187 |
c.Emit4Bytes(uint32(hi)) |
| 1188 |
c.Emit4Bytes(uint32(hi >> 32)) |
| 1189 |
} |
| 1190 |
} |
| 1191 |
|
| 1192 |
// encodeAluRRRR encodes as Data-processing (3 source) in |
| 1193 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en |
| 1194 |
func encodeAluRRRR(op aluOp, rd, rn, rm, ra, _64bit uint32) uint32 { |
| 1195 |
var oO, op31 uint32 |
| 1196 |
switch op { |
| 1197 |
case aluOpMAdd: |
| 1198 |
op31, oO = 0b000, 0b0 |
| 1199 |
case aluOpMSub: |
| 1200 |
op31, oO = 0b000, 0b1 |
| 1201 |
default: |
| 1202 |
panic("TODO/BUG") |
| 1203 |
} |
| 1204 |
return _64bit<<31 | 0b11011<<24 | op31<<21 | rm<<16 | oO<<15 | ra<<10 | rn<<5 | rd |
| 1205 |
} |
| 1206 |
|
| 1207 |
// encodeBitRR encodes as Data-processing (1 source) in |
| 1208 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en |
| 1209 |
func encodeBitRR(op bitOp, rd, rn, _64bit uint32) uint32 { |
| 1210 |
var opcode2, opcode uint32 |
| 1211 |
switch op { |
| 1212 |
case bitOpRbit: |
| 1213 |
opcode2, opcode = 0b00000, 0b000000 |
| 1214 |
case bitOpClz: |
| 1215 |
opcode2, opcode = 0b00000, 0b000100 |
| 1216 |
default: |
| 1217 |
panic("TODO/BUG") |
| 1218 |
} |
| 1219 |
return _64bit<<31 | 0b1_0_11010110<<21 | opcode2<<15 | opcode<<10 | rn<<5 | rd |
| 1220 |
} |
| 1221 |
|
| 1222 |
func encodeAsMov32(rn, rd uint32) uint32 { |
| 1223 |
// This is an alias of ORR (shifted register): |
| 1224 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register-- |
| 1225 |
return encodeLogicalShiftedRegister(0b001, 0, rn, 0, regNumberInEncoding[xzr], rd) |
| 1226 |
} |
| 1227 |
|
| 1228 |
// encodeExtend encodes extension instructions. |
| 1229 |
func encodeExtend(signed bool, from, to byte, rd, rn uint32) uint32 { |
| 1230 |
// UTXB: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTB--Unsigned-Extend-Byte--an-alias-of-UBFM-?lang=en |
| 1231 |
// UTXH: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTH--Unsigned-Extend-Halfword--an-alias-of-UBFM-?lang=en |
| 1232 |
// STXB: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTB--Signed-Extend-Byte--an-alias-of-SBFM- |
| 1233 |
// STXH: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTH--Sign-Extend-Halfword--an-alias-of-SBFM- |
| 1234 |
// STXW: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTW--Sign-Extend-Word--an-alias-of-SBFM- |
| 1235 |
var _31to10 uint32 |
| 1236 |
switch { |
| 1237 |
case !signed && from == 8 && to == 32: |
| 1238 |
// 32-bit UXTB |
| 1239 |
_31to10 = 0b0101001100000000000111 |
| 1240 |
case !signed && from == 16 && to == 32: |
| 1241 |
// 32-bit UXTH |
| 1242 |
_31to10 = 0b0101001100000000001111 |
| 1243 |
case !signed && from == 8 && to == 64: |
| 1244 |
// 64-bit UXTB |
| 1245 |
_31to10 = 0b0101001100000000000111 |
| 1246 |
case !signed && from == 16 && to == 64: |
| 1247 |
// 64-bit UXTH |
| 1248 |
_31to10 = 0b0101001100000000001111 |
| 1249 |
case !signed && from == 32 && to == 64: |
| 1250 |
return encodeAsMov32(rn, rd) |
| 1251 |
case signed && from == 8 && to == 32: |
| 1252 |
// 32-bit SXTB |
| 1253 |
_31to10 = 0b0001001100000000000111 |
| 1254 |
case signed && from == 16 && to == 32: |
| 1255 |
// 32-bit SXTH |
| 1256 |
_31to10 = 0b0001001100000000001111 |
| 1257 |
case signed && from == 8 && to == 64: |
| 1258 |
// 64-bit SXTB |
| 1259 |
_31to10 = 0b1001001101000000000111 |
| 1260 |
case signed && from == 16 && to == 64: |
| 1261 |
// 64-bit SXTH |
| 1262 |
_31to10 = 0b1001001101000000001111 |
| 1263 |
case signed && from == 32 && to == 64: |
| 1264 |
// SXTW |
| 1265 |
_31to10 = 0b1001001101000000011111 |
| 1266 |
default: |
| 1267 |
panic("BUG") |
| 1268 |
} |
| 1269 |
return _31to10<<10 | rn<<5 | rd |
| 1270 |
} |
| 1271 |
|
| 1272 |
func encodeLoadOrStore(kind instructionKind, rt uint32, amode addressMode) uint32 { |
| 1273 |
var _22to31 uint32 |
| 1274 |
var bits int64 |
| 1275 |
switch kind { |
| 1276 |
case uLoad8: |
| 1277 |
_22to31 = 0b0011100001 |
| 1278 |
bits = 8 |
| 1279 |
case sLoad8: |
| 1280 |
_22to31 = 0b0011100010 |
| 1281 |
bits = 8 |
| 1282 |
case uLoad16: |
| 1283 |
_22to31 = 0b0111100001 |
| 1284 |
bits = 16 |
| 1285 |
case sLoad16: |
| 1286 |
_22to31 = 0b0111100010 |
| 1287 |
bits = 16 |
| 1288 |
case uLoad32: |
| 1289 |
_22to31 = 0b1011100001 |
| 1290 |
bits = 32 |
| 1291 |
case sLoad32: |
| 1292 |
_22to31 = 0b1011100010 |
| 1293 |
bits = 32 |
| 1294 |
case uLoad64: |
| 1295 |
_22to31 = 0b1111100001 |
| 1296 |
bits = 64 |
| 1297 |
case fpuLoad32: |
| 1298 |
_22to31 = 0b1011110001 |
| 1299 |
bits = 32 |
| 1300 |
case fpuLoad64: |
| 1301 |
_22to31 = 0b1111110001 |
| 1302 |
bits = 64 |
| 1303 |
case fpuLoad128: |
| 1304 |
_22to31 = 0b0011110011 |
| 1305 |
bits = 128 |
| 1306 |
case store8: |
| 1307 |
_22to31 = 0b0011100000 |
| 1308 |
bits = 8 |
| 1309 |
case store16: |
| 1310 |
_22to31 = 0b0111100000 |
| 1311 |
bits = 16 |
| 1312 |
case store32: |
| 1313 |
_22to31 = 0b1011100000 |
| 1314 |
bits = 32 |
| 1315 |
case store64: |
| 1316 |
_22to31 = 0b1111100000 |
| 1317 |
bits = 64 |
| 1318 |
case fpuStore32: |
| 1319 |
_22to31 = 0b1011110000 |
| 1320 |
bits = 32 |
| 1321 |
case fpuStore64: |
| 1322 |
_22to31 = 0b1111110000 |
| 1323 |
bits = 64 |
| 1324 |
case fpuStore128: |
| 1325 |
_22to31 = 0b0011110010 |
| 1326 |
bits = 128 |
| 1327 |
default: |
| 1328 |
panic("BUG") |
| 1329 |
} |
| 1330 |
|
| 1331 |
switch amode.kind { |
| 1332 |
case addressModeKindRegScaledExtended: |
| 1333 |
return encodeLoadOrStoreExtended(_22to31, |
| 1334 |
regNumberInEncoding[amode.rn.RealReg()], |
| 1335 |
regNumberInEncoding[amode.rm.RealReg()], |
| 1336 |
rt, true, amode.extOp) |
| 1337 |
case addressModeKindRegScaled: |
| 1338 |
return encodeLoadOrStoreExtended(_22to31, |
| 1339 |
regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()], |
| 1340 |
rt, true, extendOpNone) |
| 1341 |
case addressModeKindRegExtended: |
| 1342 |
return encodeLoadOrStoreExtended(_22to31, |
| 1343 |
regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()], |
| 1344 |
rt, false, amode.extOp) |
| 1345 |
case addressModeKindRegReg: |
| 1346 |
return encodeLoadOrStoreExtended(_22to31, |
| 1347 |
regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()], |
| 1348 |
rt, false, extendOpNone) |
| 1349 |
case addressModeKindRegSignedImm9: |
| 1350 |
// e.g. https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDUR--Load-Register--unscaled-- |
| 1351 |
return encodeLoadOrStoreSIMM9(_22to31, 0b00 /* unscaled */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm) |
| 1352 |
case addressModeKindPostIndex: |
| 1353 |
return encodeLoadOrStoreSIMM9(_22to31, 0b01 /* post index */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm) |
| 1354 |
case addressModeKindPreIndex: |
| 1355 |
return encodeLoadOrStoreSIMM9(_22to31, 0b11 /* pre index */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm) |
| 1356 |
case addressModeKindRegUnsignedImm12: |
| 1357 |
// "unsigned immediate" in https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en |
| 1358 |
rn := regNumberInEncoding[amode.rn.RealReg()] |
| 1359 |
imm := amode.imm |
| 1360 |
div := bits / 8 |
| 1361 |
if imm != 0 && !offsetFitsInAddressModeKindRegUnsignedImm12(byte(bits), imm) { |
| 1362 |
panic("BUG") |
| 1363 |
} |
| 1364 |
imm /= div |
| 1365 |
return _22to31<<22 | 0b1<<24 | uint32(imm&0b111111111111)<<10 | rn<<5 | rt |
| 1366 |
default: |
| 1367 |
panic("BUG") |
| 1368 |
} |
| 1369 |
} |
| 1370 |
|
| 1371 |
// encodeVecLoad1R encodes as Load one single-element structure and Replicate to all lanes (of one register) in |
| 1372 |
// https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/LD1R--Load-one-single-element-structure-and-Replicate-to-all-lanes--of-one-register--?lang=en#sa_imm |
| 1373 |
func encodeVecLoad1R(rt, rn uint32, arr vecArrangement) uint32 { |
| 1374 |
size, q := arrToSizeQEncoded(arr) |
| 1375 |
return q<<30 | 0b001101010000001100<<12 | size<<10 | rn<<5 | rt |
| 1376 |
} |
| 1377 |
|
| 1378 |
// encodeAluBitmaskImmediate encodes as Logical (immediate) in |
| 1379 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en |
| 1380 |
func encodeAluBitmaskImmediate(op aluOp, rd, rn uint32, imm uint64, _64bit bool) uint32 { |
| 1381 |
var _31to23 uint32 |
| 1382 |
switch op { |
| 1383 |
case aluOpAnd: |
| 1384 |
_31to23 = 0b00_100100 |
| 1385 |
case aluOpOrr: |
| 1386 |
_31to23 = 0b01_100100 |
| 1387 |
case aluOpEor: |
| 1388 |
_31to23 = 0b10_100100 |
| 1389 |
case aluOpAnds: |
| 1390 |
_31to23 = 0b11_100100 |
| 1391 |
default: |
| 1392 |
panic("BUG") |
| 1393 |
} |
| 1394 |
if _64bit { |
| 1395 |
_31to23 |= 0b1 << 8 |
| 1396 |
} |
| 1397 |
immr, imms, N := bitmaskImmediate(imm, _64bit) |
| 1398 |
return _31to23<<23 | uint32(N)<<22 | uint32(immr)<<16 | uint32(imms)<<10 | rn<<5 | rd |
| 1399 |
} |
| 1400 |
|
| 1401 |
func bitmaskImmediate(c uint64, is64bit bool) (immr, imms, N byte) { |
| 1402 |
var size uint32 |
| 1403 |
switch { |
| 1404 |
case c != c>>32|c<<32: |
| 1405 |
size = 64 |
| 1406 |
case c != c>>16|c<<48: |
| 1407 |
size = 32 |
| 1408 |
c = uint64(int32(c)) |
| 1409 |
case c != c>>8|c<<56: |
| 1410 |
size = 16 |
| 1411 |
c = uint64(int16(c)) |
| 1412 |
case c != c>>4|c<<60: |
| 1413 |
size = 8 |
| 1414 |
c = uint64(int8(c)) |
| 1415 |
case c != c>>2|c<<62: |
| 1416 |
size = 4 |
| 1417 |
c = uint64(int64(c<<60) >> 60) |
| 1418 |
default: |
| 1419 |
size = 2 |
| 1420 |
c = uint64(int64(c<<62) >> 62) |
| 1421 |
} |
| 1422 |
|
| 1423 |
neg := false |
| 1424 |
if int64(c) < 0 { |
| 1425 |
c = ^c |
| 1426 |
neg = true |
| 1427 |
} |
| 1428 |
|
| 1429 |
onesSize, nonZeroPos := getOnesSequenceSize(c) |
| 1430 |
if neg { |
| 1431 |
nonZeroPos = onesSize + nonZeroPos |
| 1432 |
onesSize = size - onesSize |
| 1433 |
} |
| 1434 |
|
| 1435 |
var mode byte = 32 |
| 1436 |
if is64bit && size == 64 { |
| 1437 |
N, mode = 0b1, 64 |
| 1438 |
} |
| 1439 |
|
| 1440 |
immr = byte((size - nonZeroPos) & (size - 1) & uint32(mode-1)) |
| 1441 |
imms = byte((onesSize - 1) | 63&^(size<<1-1)) |
| 1442 |
return |
| 1443 |
} |
| 1444 |
|
| 1445 |
func getOnesSequenceSize(x uint64) (size, nonZeroPos uint32) { |
| 1446 |
// Take 0b00111000 for example: |
| 1447 |
y := getLowestBit(x) // = 0b0000100 |
| 1448 |
nonZeroPos = setBitPos(y) // = 2 |
| 1449 |
size = setBitPos(x+y) - nonZeroPos // = setBitPos(0b0100000) - 2 = 5 - 2 = 3 |
| 1450 |
return |
| 1451 |
} |
| 1452 |
|
| 1453 |
func setBitPos(x uint64) (ret uint32) { |
| 1454 |
for ; ; ret++ { |
| 1455 |
if x == 0b1 { |
| 1456 |
break |
| 1457 |
} |
| 1458 |
x = x >> 1 |
| 1459 |
} |
| 1460 |
return |
| 1461 |
} |
| 1462 |
|
| 1463 |
// encodeLoadOrStoreExtended encodes store/load instruction as "extended register offset" in Load/store register (register offset): |
| 1464 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en |
| 1465 |
func encodeLoadOrStoreExtended(_22to32 uint32, rn, rm, rt uint32, scaled bool, extOp extendOp) uint32 { |
| 1466 |
var option uint32 |
| 1467 |
switch extOp { |
| 1468 |
case extendOpUXTW: |
| 1469 |
option = 0b010 |
| 1470 |
case extendOpSXTW: |
| 1471 |
option = 0b110 |
| 1472 |
case extendOpNone: |
| 1473 |
option = 0b111 |
| 1474 |
default: |
| 1475 |
panic("BUG") |
| 1476 |
} |
| 1477 |
var s uint32 |
| 1478 |
if scaled { |
| 1479 |
s = 0b1 |
| 1480 |
} |
| 1481 |
return _22to32<<22 | 0b1<<21 | rm<<16 | option<<13 | s<<12 | 0b10<<10 | rn<<5 | rt |
| 1482 |
} |
| 1483 |
|
| 1484 |
// encodeLoadOrStoreSIMM9 encodes store/load instruction as one of post-index, pre-index or unscaled immediate as in |
| 1485 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en |
| 1486 |
func encodeLoadOrStoreSIMM9(_22to32, _1011 uint32, rn, rt uint32, imm9 int64) uint32 { |
| 1487 |
return _22to32<<22 | (uint32(imm9)&0b111111111)<<12 | _1011<<10 | rn<<5 | rt |
| 1488 |
} |
| 1489 |
|
| 1490 |
// encodeFpuRRR encodes as single or double precision (depending on `_64bit`) of Floating-point data-processing (2 source) in |
| 1491 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 1492 |
func encodeFpuRRR(op fpuBinOp, rd, rn, rm uint32, _64bit bool) (ret uint32) { |
| 1493 |
// https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/ADD--vector--Add-vectors--scalar--floating-point-and-integer- |
| 1494 |
var opcode uint32 |
| 1495 |
switch op { |
| 1496 |
case fpuBinOpAdd: |
| 1497 |
opcode = 0b0010 |
| 1498 |
case fpuBinOpSub: |
| 1499 |
opcode = 0b0011 |
| 1500 |
case fpuBinOpMul: |
| 1501 |
opcode = 0b0000 |
| 1502 |
case fpuBinOpDiv: |
| 1503 |
opcode = 0b0001 |
| 1504 |
case fpuBinOpMax: |
| 1505 |
opcode = 0b0100 |
| 1506 |
case fpuBinOpMin: |
| 1507 |
opcode = 0b0101 |
| 1508 |
default: |
| 1509 |
panic("BUG") |
| 1510 |
} |
| 1511 |
var ptype uint32 |
| 1512 |
if _64bit { |
| 1513 |
ptype = 0b01 |
| 1514 |
} |
| 1515 |
return 0b1111<<25 | ptype<<22 | 0b1<<21 | rm<<16 | opcode<<12 | 0b1<<11 | rn<<5 | rd |
| 1516 |
} |
| 1517 |
|
| 1518 |
// encodeAluRRImm12 encodes as Add/subtract (immediate) in |
| 1519 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en |
| 1520 |
func encodeAluRRImm12(op aluOp, rd, rn uint32, imm12 uint16, shiftBit byte, _64bit bool) uint32 { |
| 1521 |
var _31to24 uint32 |
| 1522 |
switch op { |
| 1523 |
case aluOpAdd: |
| 1524 |
_31to24 = 0b00_10001 |
| 1525 |
case aluOpAddS: |
| 1526 |
_31to24 = 0b01_10001 |
| 1527 |
case aluOpSub: |
| 1528 |
_31to24 = 0b10_10001 |
| 1529 |
case aluOpSubS: |
| 1530 |
_31to24 = 0b11_10001 |
| 1531 |
default: |
| 1532 |
panic("BUG") |
| 1533 |
} |
| 1534 |
if _64bit { |
| 1535 |
_31to24 |= 0b1 << 7 |
| 1536 |
} |
| 1537 |
return _31to24<<24 | uint32(shiftBit)<<22 | uint32(imm12&0b111111111111)<<10 | rn<<5 | rd |
| 1538 |
} |
| 1539 |
|
| 1540 |
// encodeAluRRR encodes as Data Processing (shifted register), depending on aluOp. |
| 1541 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#addsub_shift |
| 1542 |
func encodeAluRRRShift(op aluOp, rd, rn, rm, amount uint32, shiftOp shiftOp, _64bit bool) uint32 { |
| 1543 |
var _31to24 uint32 |
| 1544 |
var opc, n uint32 |
| 1545 |
switch op { |
| 1546 |
case aluOpAdd: |
| 1547 |
_31to24 = 0b00001011 |
| 1548 |
case aluOpAddS: |
| 1549 |
_31to24 = 0b00101011 |
| 1550 |
case aluOpSub: |
| 1551 |
_31to24 = 0b01001011 |
| 1552 |
case aluOpSubS: |
| 1553 |
_31to24 = 0b01101011 |
| 1554 |
case aluOpAnd, aluOpOrr, aluOpEor, aluOpAnds: |
| 1555 |
// "Logical (shifted register)". |
| 1556 |
switch op { |
| 1557 |
case aluOpAnd: |
| 1558 |
// all zeros |
| 1559 |
case aluOpOrr: |
| 1560 |
opc = 0b01 |
| 1561 |
case aluOpEor: |
| 1562 |
opc = 0b10 |
| 1563 |
case aluOpAnds: |
| 1564 |
opc = 0b11 |
| 1565 |
} |
| 1566 |
_31to24 = 0b000_01010 |
| 1567 |
default: |
| 1568 |
panic(op.String()) |
| 1569 |
} |
| 1570 |
|
| 1571 |
if _64bit { |
| 1572 |
_31to24 |= 0b1 << 7 |
| 1573 |
} |
| 1574 |
|
| 1575 |
var shift uint32 |
| 1576 |
switch shiftOp { |
| 1577 |
case shiftOpLSL: |
| 1578 |
shift = 0b00 |
| 1579 |
case shiftOpLSR: |
| 1580 |
shift = 0b01 |
| 1581 |
case shiftOpASR: |
| 1582 |
shift = 0b10 |
| 1583 |
default: |
| 1584 |
panic(shiftOp.String()) |
| 1585 |
} |
| 1586 |
return opc<<29 | n<<21 | _31to24<<24 | shift<<22 | rm<<16 | (amount << 10) | (rn << 5) | rd |
| 1587 |
} |
| 1588 |
|
| 1589 |
// "Add/subtract (extended register)" in |
| 1590 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#addsub_ext |
| 1591 |
func encodeAluRRRExtend(ao aluOp, rd, rn, rm uint32, extOp extendOp, to byte) uint32 { |
| 1592 |
var s, op uint32 |
| 1593 |
switch ao { |
| 1594 |
case aluOpAdd: |
| 1595 |
op = 0b0 |
| 1596 |
case aluOpAddS: |
| 1597 |
op, s = 0b0, 0b1 |
| 1598 |
case aluOpSub: |
| 1599 |
op = 0b1 |
| 1600 |
case aluOpSubS: |
| 1601 |
op, s = 0b1, 0b1 |
| 1602 |
default: |
| 1603 |
panic("BUG: extended register operand can be used only for add/sub") |
| 1604 |
} |
| 1605 |
|
| 1606 |
var sf uint32 |
| 1607 |
if to == 64 { |
| 1608 |
sf = 0b1 |
| 1609 |
} |
| 1610 |
|
| 1611 |
var option uint32 |
| 1612 |
switch extOp { |
| 1613 |
case extendOpUXTB: |
| 1614 |
option = 0b000 |
| 1615 |
case extendOpUXTH: |
| 1616 |
option = 0b001 |
| 1617 |
case extendOpUXTW: |
| 1618 |
option = 0b010 |
| 1619 |
case extendOpSXTB: |
| 1620 |
option = 0b100 |
| 1621 |
case extendOpSXTH: |
| 1622 |
option = 0b101 |
| 1623 |
case extendOpSXTW: |
| 1624 |
option = 0b110 |
| 1625 |
case extendOpSXTX, extendOpUXTX: |
| 1626 |
panic(fmt.Sprintf("%s is essentially noop, and should be handled much earlier than encoding", extOp.String())) |
| 1627 |
} |
| 1628 |
return sf<<31 | op<<30 | s<<29 | 0b1011001<<21 | rm<<16 | option<<13 | rn<<5 | rd |
| 1629 |
} |
| 1630 |
|
| 1631 |
// encodeAluRRR encodes as Data Processing (register), depending on aluOp. |
| 1632 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en |
| 1633 |
func encodeAluRRR(op aluOp, rd, rn, rm uint32, _64bit, isRnSp bool) uint32 { |
| 1634 |
var _31to21, _15to10 uint32 |
| 1635 |
switch op { |
| 1636 |
case aluOpAdd: |
| 1637 |
if isRnSp { |
| 1638 |
// "Extended register" with UXTW. |
| 1639 |
_31to21 = 0b00001011_001 |
| 1640 |
_15to10 = 0b011000 |
| 1641 |
} else { |
| 1642 |
// "Shifted register" with shift = 0 |
| 1643 |
_31to21 = 0b00001011_000 |
| 1644 |
} |
| 1645 |
case aluOpAddS: |
| 1646 |
if isRnSp { |
| 1647 |
panic("TODO") |
| 1648 |
} |
| 1649 |
// "Shifted register" with shift = 0 |
| 1650 |
_31to21 = 0b00101011_000 |
| 1651 |
case aluOpSub: |
| 1652 |
if isRnSp { |
| 1653 |
// "Extended register" with UXTW. |
| 1654 |
_31to21 = 0b01001011_001 |
| 1655 |
_15to10 = 0b011000 |
| 1656 |
} else { |
| 1657 |
// "Shifted register" with shift = 0 |
| 1658 |
_31to21 = 0b01001011_000 |
| 1659 |
} |
| 1660 |
case aluOpSubS: |
| 1661 |
if isRnSp { |
| 1662 |
panic("TODO") |
| 1663 |
} |
| 1664 |
// "Shifted register" with shift = 0 |
| 1665 |
_31to21 = 0b01101011_000 |
| 1666 |
case aluOpAnd, aluOpOrr, aluOpOrn, aluOpEor, aluOpAnds: |
| 1667 |
// "Logical (shifted register)". |
| 1668 |
var opc, n uint32 |
| 1669 |
switch op { |
| 1670 |
case aluOpAnd: |
| 1671 |
// all zeros |
| 1672 |
case aluOpOrr: |
| 1673 |
opc = 0b01 |
| 1674 |
case aluOpOrn: |
| 1675 |
opc = 0b01 |
| 1676 |
n = 1 |
| 1677 |
case aluOpEor: |
| 1678 |
opc = 0b10 |
| 1679 |
case aluOpAnds: |
| 1680 |
opc = 0b11 |
| 1681 |
} |
| 1682 |
_31to21 = 0b000_01010_000 | opc<<8 | n |
| 1683 |
case aluOpLsl, aluOpAsr, aluOpLsr, aluOpRotR: |
| 1684 |
// "Data-processing (2 source)". |
| 1685 |
_31to21 = 0b00011010_110 |
| 1686 |
switch op { |
| 1687 |
case aluOpLsl: |
| 1688 |
_15to10 = 0b001000 |
| 1689 |
case aluOpLsr: |
| 1690 |
_15to10 = 0b001001 |
| 1691 |
case aluOpAsr: |
| 1692 |
_15to10 = 0b001010 |
| 1693 |
case aluOpRotR: |
| 1694 |
_15to10 = 0b001011 |
| 1695 |
} |
| 1696 |
case aluOpSDiv: |
| 1697 |
// "Data-processing (2 source)". |
| 1698 |
_31to21 = 0b11010110 |
| 1699 |
_15to10 = 0b000011 |
| 1700 |
case aluOpUDiv: |
| 1701 |
// "Data-processing (2 source)". |
| 1702 |
_31to21 = 0b11010110 |
| 1703 |
_15to10 = 0b000010 |
| 1704 |
default: |
| 1705 |
panic(op.String()) |
| 1706 |
} |
| 1707 |
if _64bit { |
| 1708 |
_31to21 |= 0b1 << 10 |
| 1709 |
} |
| 1710 |
return _31to21<<21 | rm<<16 | (_15to10 << 10) | (rn << 5) | rd |
| 1711 |
} |
| 1712 |
|
| 1713 |
// encodeLogicalShiftedRegister encodes as Logical (shifted register) in |
| 1714 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en |
| 1715 |
func encodeLogicalShiftedRegister(sf_opc uint32, shift_N uint32, rm uint32, imm6 uint32, rn, rd uint32) (ret uint32) { |
| 1716 |
ret = sf_opc << 29 |
| 1717 |
ret |= 0b01010 << 24 |
| 1718 |
ret |= shift_N << 21 |
| 1719 |
ret |= rm << 16 |
| 1720 |
ret |= imm6 << 10 |
| 1721 |
ret |= rn << 5 |
| 1722 |
ret |= rd |
| 1723 |
return |
| 1724 |
} |
| 1725 |
|
| 1726 |
// encodeAddSubtractImmediate encodes as Add/subtract (immediate) in |
| 1727 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en |
| 1728 |
func encodeAddSubtractImmediate(sf_op_s uint32, sh uint32, imm12 uint32, rn, rd uint32) (ret uint32) { |
| 1729 |
ret = sf_op_s << 29 |
| 1730 |
ret |= 0b100010 << 23 |
| 1731 |
ret |= sh << 22 |
| 1732 |
ret |= imm12 << 10 |
| 1733 |
ret |= rn << 5 |
| 1734 |
ret |= rd |
| 1735 |
return |
| 1736 |
} |
| 1737 |
|
| 1738 |
// encodePreOrPostIndexLoadStorePair64 encodes as Load/store pair (pre/post-indexed) in |
| 1739 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers- |
| 1740 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers- |
| 1741 |
func encodePreOrPostIndexLoadStorePair64(pre bool, load bool, rn, rt, rt2 uint32, imm7 int64) (ret uint32) { |
| 1742 |
if imm7%8 != 0 { |
| 1743 |
panic("imm7 for pair load/store must be a multiple of 8") |
| 1744 |
} |
| 1745 |
imm7 /= 8 |
| 1746 |
ret = rt |
| 1747 |
ret |= rn << 5 |
| 1748 |
ret |= rt2 << 10 |
| 1749 |
ret |= (uint32(imm7) & 0b1111111) << 15 |
| 1750 |
if load { |
| 1751 |
ret |= 0b1 << 22 |
| 1752 |
} |
| 1753 |
ret |= 0b101010001 << 23 |
| 1754 |
if pre { |
| 1755 |
ret |= 0b1 << 24 |
| 1756 |
} |
| 1757 |
return |
| 1758 |
} |
| 1759 |
|
| 1760 |
// encodeUnconditionalBranch encodes as B or BL instructions: |
| 1761 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B--Branch- |
| 1762 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/BL--Branch-with-Link- |
| 1763 |
func encodeUnconditionalBranch(link bool, imm26 int64) (ret uint32) { |
| 1764 |
if imm26%4 != 0 { |
| 1765 |
panic("imm26 for branch must be a multiple of 4") |
| 1766 |
} |
| 1767 |
imm26 /= 4 |
| 1768 |
ret = uint32(imm26 & 0b11_11111111_11111111_11111111) |
| 1769 |
ret |= 0b101 << 26 |
| 1770 |
if link { |
| 1771 |
ret |= 0b1 << 31 |
| 1772 |
} |
| 1773 |
return |
| 1774 |
} |
| 1775 |
|
| 1776 |
// encodeCBZCBNZ encodes as either CBZ or CBNZ: |
| 1777 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CBZ--Compare-and-Branch-on-Zero- |
| 1778 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CBNZ--Compare-and-Branch-on-Nonzero- |
| 1779 |
func encodeCBZCBNZ(rt uint32, nz bool, imm19 uint32, _64bit bool) (ret uint32) { |
| 1780 |
ret = rt |
| 1781 |
ret |= imm19 << 5 |
| 1782 |
if nz { |
| 1783 |
ret |= 1 << 24 |
| 1784 |
} |
| 1785 |
ret |= 0b11010 << 25 |
| 1786 |
if _64bit { |
| 1787 |
ret |= 1 << 31 |
| 1788 |
} |
| 1789 |
return |
| 1790 |
} |
| 1791 |
|
| 1792 |
// encodeMoveWideImmediate encodes as either MOVZ, MOVN or MOVK, as Move wide (immediate) in |
| 1793 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en |
| 1794 |
// |
| 1795 |
// "shift" must have been divided by 16 at this point. |
| 1796 |
func encodeMoveWideImmediate(opc uint32, rd uint32, imm uint64, shift, _64bit uint32) (ret uint32) { |
| 1797 |
ret = rd |
| 1798 |
ret |= uint32(imm&0xffff) << 5 |
| 1799 |
ret |= (shift) << 21 |
| 1800 |
ret |= 0b100101 << 23 |
| 1801 |
ret |= opc << 29 |
| 1802 |
ret |= _64bit << 31 |
| 1803 |
return |
| 1804 |
} |
| 1805 |
|
| 1806 |
// encodeAluRRImm encodes as "Bitfield" in |
| 1807 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en#log_imm |
| 1808 |
func encodeAluRRImm(op aluOp, rd, rn, amount, _64bit uint32) uint32 { |
| 1809 |
var opc uint32 |
| 1810 |
var immr, imms uint32 |
| 1811 |
switch op { |
| 1812 |
case aluOpLsl: |
| 1813 |
// LSL (immediate) is an alias for UBFM. |
| 1814 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/UBFM--Unsigned-Bitfield-Move-?lang=en |
| 1815 |
opc = 0b10 |
| 1816 |
if amount == 0 { |
| 1817 |
// This can be encoded as NOP, but we don't do it for consistency: lsr xn, xm, #0 |
| 1818 |
immr = 0 |
| 1819 |
if _64bit == 1 { |
| 1820 |
imms = 0b111111 |
| 1821 |
} else { |
| 1822 |
imms = 0b11111 |
| 1823 |
} |
| 1824 |
} else { |
| 1825 |
if _64bit == 1 { |
| 1826 |
immr = 64 - amount |
| 1827 |
} else { |
| 1828 |
immr = (32 - amount) & 0b11111 |
| 1829 |
} |
| 1830 |
imms = immr - 1 |
| 1831 |
} |
| 1832 |
case aluOpLsr: |
| 1833 |
// LSR (immediate) is an alias for UBFM. |
| 1834 |
// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LSR--immediate---Logical-Shift-Right--immediate---an-alias-of-UBFM-?lang=en |
| 1835 |
opc = 0b10 |
| 1836 |
imms, immr = 0b011111|_64bit<<5, amount |
| 1837 |
case aluOpAsr: |
| 1838 |
// ASR (immediate) is an alias for SBFM. |
| 1839 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SBFM--Signed-Bitfield-Move-?lang=en |
| 1840 |
opc = 0b00 |
| 1841 |
imms, immr = 0b011111|_64bit<<5, amount |
| 1842 |
default: |
| 1843 |
panic(op.String()) |
| 1844 |
} |
| 1845 |
return _64bit<<31 | opc<<29 | 0b100110<<23 | _64bit<<22 | immr<<16 | imms<<10 | rn<<5 | rd |
| 1846 |
} |
| 1847 |
|
| 1848 |
// encodeVecLanes encodes as Data Processing (Advanced SIMD across lanes) depending on vecOp in |
| 1849 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 1850 |
func encodeVecLanes(op vecOp, rd uint32, rn uint32, arr vecArrangement) uint32 { |
| 1851 |
var u, q, size, opcode uint32 |
| 1852 |
switch arr { |
| 1853 |
case vecArrangement8B: |
| 1854 |
q, size = 0b0, 0b00 |
| 1855 |
case vecArrangement16B: |
| 1856 |
q, size = 0b1, 0b00 |
| 1857 |
case vecArrangement4H: |
| 1858 |
q, size = 0, 0b01 |
| 1859 |
case vecArrangement8H: |
| 1860 |
q, size = 1, 0b01 |
| 1861 |
case vecArrangement4S: |
| 1862 |
q, size = 1, 0b10 |
| 1863 |
default: |
| 1864 |
panic("unsupported arrangement: " + arr.String()) |
| 1865 |
} |
| 1866 |
switch op { |
| 1867 |
case vecOpUaddlv: |
| 1868 |
u, opcode = 1, 0b00011 |
| 1869 |
case vecOpUminv: |
| 1870 |
u, opcode = 1, 0b11010 |
| 1871 |
case vecOpAddv: |
| 1872 |
u, opcode = 0, 0b11011 |
| 1873 |
default: |
| 1874 |
panic("unsupported or illegal vecOp: " + op.String()) |
| 1875 |
} |
| 1876 |
return q<<30 | u<<29 | 0b1110<<24 | size<<22 | 0b11000<<17 | opcode<<12 | 0b10<<10 | rn<<5 | rd |
| 1877 |
} |
| 1878 |
|
| 1879 |
// encodeVecLanes encodes as Data Processing (Advanced SIMD scalar shift by immediate) depending on vecOp in |
| 1880 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en |
| 1881 |
func encodeVecShiftImm(op vecOp, rd uint32, rn, amount uint32, arr vecArrangement) uint32 { |
| 1882 |
var u, q, immh, immb, opcode uint32 |
| 1883 |
switch op { |
| 1884 |
case vecOpSshll: |
| 1885 |
u, opcode = 0b0, 0b10100 |
| 1886 |
case vecOpUshll: |
| 1887 |
u, opcode = 0b1, 0b10100 |
| 1888 |
case vecOpSshr: |
| 1889 |
u, opcode = 0, 0b00000 |
| 1890 |
default: |
| 1891 |
panic("unsupported or illegal vecOp: " + op.String()) |
| 1892 |
} |
| 1893 |
switch arr { |
| 1894 |
case vecArrangement16B: |
| 1895 |
q = 0b1 |
| 1896 |
fallthrough |
| 1897 |
case vecArrangement8B: |
| 1898 |
immh = 0b0001 |
| 1899 |
immb = 8 - uint32(amount&0b111) |
| 1900 |
case vecArrangement8H: |
| 1901 |
q = 0b1 |
| 1902 |
fallthrough |
| 1903 |
case vecArrangement4H: |
| 1904 |
v := 16 - uint32(amount&0b1111) |
| 1905 |
immb = v & 0b111 |
| 1906 |
immh = 0b0010 | (v >> 3) |
| 1907 |
case vecArrangement4S: |
| 1908 |
q = 0b1 |
| 1909 |
fallthrough |
| 1910 |
case vecArrangement2S: |
| 1911 |
v := 32 - uint32(amount&0b11111) |
| 1912 |
immb = v & 0b111 |
| 1913 |
immh = 0b0100 | (v >> 3) |
| 1914 |
case vecArrangement2D: |
| 1915 |
q = 0b1 |
| 1916 |
v := 64 - uint32(amount&0b111111) |
| 1917 |
immb = v & 0b111 |
| 1918 |
immh = 0b1000 | (v >> 3) |
| 1919 |
default: |
| 1920 |
panic("unsupported arrangement: " + arr.String()) |
| 1921 |
} |
| 1922 |
return q<<30 | u<<29 | 0b011110<<23 | immh<<19 | immb<<16 | 0b000001<<10 | opcode<<11 | 0b1<<10 | rn<<5 | rd |
| 1923 |
} |
| 1924 |
|
| 1925 |
// encodeVecTbl encodes as Data Processing (Advanced SIMD table lookup) in |
| 1926 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp |
| 1927 |
// |
| 1928 |
// Note: tblOp may encode tbl1, tbl2... in the future. Currently, it is ignored. |
| 1929 |
func encodeVecTbl(nregs, rd, rn, rm uint32, arr vecArrangement) uint32 { |
| 1930 |
var q, op2, len, op uint32 |
| 1931 |
|
| 1932 |
switch nregs { |
| 1933 |
case 1: |
| 1934 |
// tbl: single-register |
| 1935 |
len = 0b00 |
| 1936 |
case 2: |
| 1937 |
// tbl2: 2-register table |
| 1938 |
len = 0b01 |
| 1939 |
default: |
| 1940 |
panic(fmt.Sprintf("unsupported number or registers %d", nregs)) |
| 1941 |
} |
| 1942 |
switch arr { |
| 1943 |
case vecArrangement8B: |
| 1944 |
q = 0b0 |
| 1945 |
case vecArrangement16B: |
| 1946 |
q = 0b1 |
| 1947 |
default: |
| 1948 |
panic("unsupported arrangement: " + arr.String()) |
| 1949 |
} |
| 1950 |
|
| 1951 |
return q<<30 | 0b001110<<24 | op2<<22 | rm<<16 | len<<13 | op<<12 | rn<<5 | rd |
| 1952 |
} |
| 1953 |
|
| 1954 |
// encodeVecMisc encodes as Data Processing (Advanced SIMD two-register miscellaneous) depending on vecOp in |
| 1955 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp |
| 1956 |
func encodeAdvancedSIMDTwoMisc(op vecOp, rd, rn uint32, arr vecArrangement) uint32 { |
| 1957 |
var q, u, size, opcode uint32 |
| 1958 |
switch op { |
| 1959 |
case vecOpCnt: |
| 1960 |
opcode = 0b00101 |
| 1961 |
switch arr { |
| 1962 |
case vecArrangement8B: |
| 1963 |
q, size = 0b0, 0b00 |
| 1964 |
case vecArrangement16B: |
| 1965 |
q, size = 0b1, 0b00 |
| 1966 |
default: |
| 1967 |
panic("unsupported arrangement: " + arr.String()) |
| 1968 |
} |
| 1969 |
case vecOpCmeq0: |
| 1970 |
if arr == vecArrangement1D { |
| 1971 |
panic("unsupported arrangement: " + arr.String()) |
| 1972 |
} |
| 1973 |
opcode = 0b01001 |
| 1974 |
size, q = arrToSizeQEncoded(arr) |
| 1975 |
case vecOpNot: |
| 1976 |
u = 1 |
| 1977 |
opcode = 0b00101 |
| 1978 |
switch arr { |
| 1979 |
case vecArrangement8B: |
| 1980 |
q, size = 0b0, 0b00 |
| 1981 |
case vecArrangement16B: |
| 1982 |
q, size = 0b1, 0b00 |
| 1983 |
default: |
| 1984 |
panic("unsupported arrangement: " + arr.String()) |
| 1985 |
} |
| 1986 |
case vecOpAbs: |
| 1987 |
if arr == vecArrangement1D { |
| 1988 |
panic("unsupported arrangement: " + arr.String()) |
| 1989 |
} |
| 1990 |
opcode = 0b01011 |
| 1991 |
u = 0b0 |
| 1992 |
size, q = arrToSizeQEncoded(arr) |
| 1993 |
case vecOpNeg: |
| 1994 |
if arr == vecArrangement1D { |
| 1995 |
panic("unsupported arrangement: " + arr.String()) |
| 1996 |
} |
| 1997 |
opcode = 0b01011 |
| 1998 |
u = 0b1 |
| 1999 |
size, q = arrToSizeQEncoded(arr) |
| 2000 |
case vecOpFabs: |
| 2001 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 2002 |
panic("unsupported arrangement: " + arr.String()) |
| 2003 |
} |
| 2004 |
opcode = 0b01111 |
| 2005 |
u = 0b0 |
| 2006 |
size, q = arrToSizeQEncoded(arr) |
| 2007 |
case vecOpFneg: |
| 2008 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 2009 |
panic("unsupported arrangement: " + arr.String()) |
| 2010 |
} |
| 2011 |
opcode = 0b01111 |
| 2012 |
u = 0b1 |
| 2013 |
size, q = arrToSizeQEncoded(arr) |
| 2014 |
case vecOpFrintm: |
| 2015 |
u = 0b0 |
| 2016 |
opcode = 0b11001 |
| 2017 |
switch arr { |
| 2018 |
case vecArrangement2S: |
| 2019 |
q, size = 0b0, 0b00 |
| 2020 |
case vecArrangement4S: |
| 2021 |
q, size = 0b1, 0b00 |
| 2022 |
case vecArrangement2D: |
| 2023 |
q, size = 0b1, 0b01 |
| 2024 |
default: |
| 2025 |
panic("unsupported arrangement: " + arr.String()) |
| 2026 |
} |
| 2027 |
case vecOpFrintn: |
| 2028 |
u = 0b0 |
| 2029 |
opcode = 0b11000 |
| 2030 |
switch arr { |
| 2031 |
case vecArrangement2S: |
| 2032 |
q, size = 0b0, 0b00 |
| 2033 |
case vecArrangement4S: |
| 2034 |
q, size = 0b1, 0b00 |
| 2035 |
case vecArrangement2D: |
| 2036 |
q, size = 0b1, 0b01 |
| 2037 |
default: |
| 2038 |
panic("unsupported arrangement: " + arr.String()) |
| 2039 |
} |
| 2040 |
case vecOpFrintp: |
| 2041 |
u = 0b0 |
| 2042 |
opcode = 0b11000 |
| 2043 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 2044 |
panic("unsupported arrangement: " + arr.String()) |
| 2045 |
} |
| 2046 |
size, q = arrToSizeQEncoded(arr) |
| 2047 |
case vecOpFrintz: |
| 2048 |
u = 0b0 |
| 2049 |
opcode = 0b11001 |
| 2050 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 2051 |
panic("unsupported arrangement: " + arr.String()) |
| 2052 |
} |
| 2053 |
size, q = arrToSizeQEncoded(arr) |
| 2054 |
case vecOpFsqrt: |
| 2055 |
if arr < vecArrangement2S || arr == vecArrangement1D { |
| 2056 |
panic("unsupported arrangement: " + arr.String()) |
| 2057 |
} |
| 2058 |
opcode = 0b11111 |
| 2059 |
u = 0b1 |
| 2060 |
size, q = arrToSizeQEncoded(arr) |
| 2061 |
case vecOpFcvtl: |
| 2062 |
opcode = 0b10111 |
| 2063 |
u = 0b0 |
| 2064 |
switch arr { |
| 2065 |
case vecArrangement2S: |
| 2066 |
size, q = 0b01, 0b0 |
| 2067 |
case vecArrangement4H: |
| 2068 |
size, q = 0b00, 0b0 |
| 2069 |
default: |
| 2070 |
panic("unsupported arrangement: " + arr.String()) |
| 2071 |
} |
| 2072 |
case vecOpFcvtn: |
| 2073 |
opcode = 0b10110 |
| 2074 |
u = 0b0 |
| 2075 |
switch arr { |
| 2076 |
case vecArrangement2S: |
| 2077 |
size, q = 0b01, 0b0 |
| 2078 |
case vecArrangement4H: |
| 2079 |
size, q = 0b00, 0b0 |
| 2080 |
default: |
| 2081 |
panic("unsupported arrangement: " + arr.String()) |
| 2082 |
} |
| 2083 |
case vecOpFcvtzs: |
| 2084 |
opcode = 0b11011 |
| 2085 |
u = 0b0 |
| 2086 |
switch arr { |
| 2087 |
case vecArrangement2S: |
| 2088 |
q, size = 0b0, 0b10 |
| 2089 |
case vecArrangement4S: |
| 2090 |
q, size = 0b1, 0b10 |
| 2091 |
case vecArrangement2D: |
| 2092 |
q, size = 0b1, 0b11 |
| 2093 |
default: |
| 2094 |
panic("unsupported arrangement: " + arr.String()) |
| 2095 |
} |
| 2096 |
case vecOpFcvtzu: |
| 2097 |
opcode = 0b11011 |
| 2098 |
u = 0b1 |
| 2099 |
switch arr { |
| 2100 |
case vecArrangement2S: |
| 2101 |
q, size = 0b0, 0b10 |
| 2102 |
case vecArrangement4S: |
| 2103 |
q, size = 0b1, 0b10 |
| 2104 |
case vecArrangement2D: |
| 2105 |
q, size = 0b1, 0b11 |
| 2106 |
default: |
| 2107 |
panic("unsupported arrangement: " + arr.String()) |
| 2108 |
} |
| 2109 |
case vecOpScvtf: |
| 2110 |
opcode = 0b11101 |
| 2111 |
u = 0b0 |
| 2112 |
switch arr { |
| 2113 |
case vecArrangement4S: |
| 2114 |
q, size = 0b1, 0b00 |
| 2115 |
case vecArrangement2S: |
| 2116 |
q, size = 0b0, 0b00 |
| 2117 |
case vecArrangement2D: |
| 2118 |
q, size = 0b1, 0b01 |
| 2119 |
default: |
| 2120 |
panic("unsupported arrangement: " + arr.String()) |
| 2121 |
} |
| 2122 |
case vecOpUcvtf: |
| 2123 |
opcode = 0b11101 |
| 2124 |
u = 0b1 |
| 2125 |
switch arr { |
| 2126 |
case vecArrangement4S: |
| 2127 |
q, size = 0b1, 0b00 |
| 2128 |
case vecArrangement2S: |
| 2129 |
q, size = 0b0, 0b00 |
| 2130 |
case vecArrangement2D: |
| 2131 |
q, size = 0b1, 0b01 |
| 2132 |
default: |
| 2133 |
panic("unsupported arrangement: " + arr.String()) |
| 2134 |
} |
| 2135 |
case vecOpSqxtn: |
| 2136 |
// When q == 1 it encodes sqxtn2 (operates on upper 64 bits). |
| 2137 |
opcode = 0b10100 |
| 2138 |
u = 0b0 |
| 2139 |
if arr > vecArrangement4S { |
| 2140 |
panic("unsupported arrangement: " + arr.String()) |
| 2141 |
} |
| 2142 |
size, q = arrToSizeQEncoded(arr) |
| 2143 |
case vecOpUqxtn: |
| 2144 |
// When q == 1 it encodes uqxtn2 (operates on upper 64 bits). |
| 2145 |
opcode = 0b10100 |
| 2146 |
u = 0b1 |
| 2147 |
if arr > vecArrangement4S { |
| 2148 |
panic("unsupported arrangement: " + arr.String()) |
| 2149 |
} |
| 2150 |
size, q = arrToSizeQEncoded(arr) |
| 2151 |
case vecOpSqxtun: |
| 2152 |
// When q == 1 it encodes sqxtun2 (operates on upper 64 bits). |
| 2153 |
opcode = 0b10010 // 0b10100 |
| 2154 |
u = 0b1 |
| 2155 |
if arr > vecArrangement4S { |
| 2156 |
panic("unsupported arrangement: " + arr.String()) |
| 2157 |
} |
| 2158 |
size, q = arrToSizeQEncoded(arr) |
| 2159 |
case vecOpRev64: |
| 2160 |
opcode = 0b00000 |
| 2161 |
size, q = arrToSizeQEncoded(arr) |
| 2162 |
case vecOpXtn: |
| 2163 |
u = 0b0 |
| 2164 |
opcode = 0b10010 |
| 2165 |
size, q = arrToSizeQEncoded(arr) |
| 2166 |
case vecOpShll: |
| 2167 |
u = 0b1 |
| 2168 |
opcode = 0b10011 |
| 2169 |
switch arr { |
| 2170 |
case vecArrangement8B: |
| 2171 |
q, size = 0b0, 0b00 |
| 2172 |
case vecArrangement4H: |
| 2173 |
q, size = 0b0, 0b01 |
| 2174 |
case vecArrangement2S: |
| 2175 |
q, size = 0b0, 0b10 |
| 2176 |
default: |
| 2177 |
panic("unsupported arrangement: " + arr.String()) |
| 2178 |
} |
| 2179 |
default: |
| 2180 |
panic("unsupported or illegal vecOp: " + op.String()) |
| 2181 |
} |
| 2182 |
return q<<30 | u<<29 | 0b01110<<24 | size<<22 | 0b10000<<17 | opcode<<12 | 0b10<<10 | rn<<5 | rd |
| 2183 |
} |
| 2184 |
|
| 2185 |
// brTableSequenceOffsetTableBegin is the offset inside the brTableSequence where the table begins after 4 instructions |
| 2186 |
const brTableSequenceOffsetTableBegin = 16 |
| 2187 |
|
| 2188 |
func encodeBrTableSequence(c backend.Compiler, index regalloc.VReg, targets []uint32) { |
| 2189 |
tmpRegNumber := regNumberInEncoding[tmp] |
| 2190 |
indexNumber := regNumberInEncoding[index.RealReg()] |
| 2191 |
|
| 2192 |
// adr tmpReg, PC+16 (PC+16 is the address of the first label offset) |
| 2193 |
// ldrsw index, [tmpReg, index, UXTW 2] ;; index = int64(*(tmpReg + index*8)) |
| 2194 |
// add tmpReg, tmpReg, index |
| 2195 |
// br tmpReg |
| 2196 |
// [offset_to_l1, offset_to_l2, ..., offset_to_lN] |
| 2197 |
c.Emit4Bytes(encodeAdr(tmpRegNumber, 16)) |
| 2198 |
c.Emit4Bytes(encodeLoadOrStore(sLoad32, indexNumber, |
| 2199 |
addressMode{kind: addressModeKindRegScaledExtended, rn: tmpRegVReg, rm: index, extOp: extendOpUXTW}, |
| 2200 |
)) |
| 2201 |
c.Emit4Bytes(encodeAluRRR(aluOpAdd, tmpRegNumber, tmpRegNumber, indexNumber, true, false)) |
| 2202 |
c.Emit4Bytes(encodeUnconditionalBranchReg(tmpRegNumber, false)) |
| 2203 |
|
| 2204 |
// Offsets are resolved in ResolveRelativeAddress phase. |
| 2205 |
for _, offset := range targets { |
| 2206 |
if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { |
| 2207 |
// Inlined offset tables cannot be disassembled properly, so pad dummy instructions to make the debugging easier. |
| 2208 |
c.Emit4Bytes(dummyInstruction) |
| 2209 |
} else { |
| 2210 |
c.Emit4Bytes(offset) |
| 2211 |
} |
| 2212 |
} |
| 2213 |
} |
| 2214 |
|
| 2215 |
// encodeExitSequence matches the implementation detail of functionABI.emitGoEntryPreamble. |
| 2216 |
func encodeExitSequence(c backend.Compiler, ctxReg regalloc.VReg) { |
| 2217 |
// Restore the FP, SP and LR, and return to the Go code: |
| 2218 |
// ldr lr, [ctxReg, #GoReturnAddress] |
| 2219 |
// ldr fp, [ctxReg, #OriginalFramePointer] |
| 2220 |
// ldr tmp, [ctxReg, #OriginalStackPointer] |
| 2221 |
// mov sp, tmp ;; sp cannot be str'ed directly. |
| 2222 |
// ret ;; --> return to the Go code |
| 2223 |
|
| 2224 |
var ctxEvicted bool |
| 2225 |
if ctx := ctxReg.RealReg(); ctx == fp || ctx == lr { |
| 2226 |
// In order to avoid overwriting the context register, we move ctxReg to tmp. |
| 2227 |
c.Emit4Bytes(encodeMov64(regNumberInEncoding[tmp], regNumberInEncoding[ctx], false, false)) |
| 2228 |
ctxReg = tmpRegVReg |
| 2229 |
ctxEvicted = true |
| 2230 |
} |
| 2231 |
|
| 2232 |
restoreLr := encodeLoadOrStore( |
| 2233 |
uLoad64, |
| 2234 |
regNumberInEncoding[lr], |
| 2235 |
addressMode{ |
| 2236 |
kind: addressModeKindRegUnsignedImm12, |
| 2237 |
rn: ctxReg, |
| 2238 |
imm: wazevoapi.ExecutionContextOffsetGoReturnAddress.I64(), |
| 2239 |
}, |
| 2240 |
) |
| 2241 |
|
| 2242 |
restoreFp := encodeLoadOrStore( |
| 2243 |
uLoad64, |
| 2244 |
regNumberInEncoding[fp], |
| 2245 |
addressMode{ |
| 2246 |
kind: addressModeKindRegUnsignedImm12, |
| 2247 |
rn: ctxReg, |
| 2248 |
imm: wazevoapi.ExecutionContextOffsetOriginalFramePointer.I64(), |
| 2249 |
}, |
| 2250 |
) |
| 2251 |
|
| 2252 |
restoreSpToTmp := encodeLoadOrStore( |
| 2253 |
uLoad64, |
| 2254 |
regNumberInEncoding[tmp], |
| 2255 |
addressMode{ |
| 2256 |
kind: addressModeKindRegUnsignedImm12, |
| 2257 |
rn: ctxReg, |
| 2258 |
imm: wazevoapi.ExecutionContextOffsetOriginalStackPointer.I64(), |
| 2259 |
}, |
| 2260 |
) |
| 2261 |
|
| 2262 |
movTmpToSp := encodeAddSubtractImmediate(0b100, 0, 0, |
| 2263 |
regNumberInEncoding[tmp], regNumberInEncoding[sp]) |
| 2264 |
|
| 2265 |
c.Emit4Bytes(restoreFp) |
| 2266 |
c.Emit4Bytes(restoreLr) |
| 2267 |
c.Emit4Bytes(restoreSpToTmp) |
| 2268 |
c.Emit4Bytes(movTmpToSp) |
| 2269 |
c.Emit4Bytes(encodeRet()) |
| 2270 |
if !ctxEvicted { |
| 2271 |
// In order to have the fixed-length exit sequence, we need to padd the binary. |
| 2272 |
// Since this will never be reached, we insert a dummy instruction. |
| 2273 |
c.Emit4Bytes(dummyInstruction) |
| 2274 |
} |
| 2275 |
} |
| 2276 |
|
| 2277 |
func encodeRet() uint32 { |
| 2278 |
// https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en |
| 2279 |
return 0b1101011001011111<<16 | regNumberInEncoding[lr]<<5 |
| 2280 |
} |
| 2281 |
|
| 2282 |
func encodeAtomicRmw(op atomicRmwOp, rs, rt, rn uint32, size uint32) uint32 { |
| 2283 |
var _31to21, _15to10, sz uint32 |
| 2284 |
|
| 2285 |
switch size { |
| 2286 |
case 8: |
| 2287 |
sz = 0b11 |
| 2288 |
case 4: |
| 2289 |
sz = 0b10 |
| 2290 |
case 2: |
| 2291 |
sz = 0b01 |
| 2292 |
case 1: |
| 2293 |
sz = 0b00 |
| 2294 |
} |
| 2295 |
|
| 2296 |
_31to21 = 0b00111000_111 | sz<<9 |
| 2297 |
|
| 2298 |
switch op { |
| 2299 |
case atomicRmwOpAdd: |
| 2300 |
_15to10 = 0b000000 |
| 2301 |
case atomicRmwOpClr: |
| 2302 |
_15to10 = 0b000100 |
| 2303 |
case atomicRmwOpSet: |
| 2304 |
_15to10 = 0b001100 |
| 2305 |
case atomicRmwOpEor: |
| 2306 |
_15to10 = 0b001000 |
| 2307 |
case atomicRmwOpSwp: |
| 2308 |
_15to10 = 0b100000 |
| 2309 |
} |
| 2310 |
|
| 2311 |
return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt |
| 2312 |
} |
| 2313 |
|
| 2314 |
func encodeAtomicCas(rs, rt, rn uint32, size uint32) uint32 { |
| 2315 |
var _31to21, _15to10, sz uint32 |
| 2316 |
|
| 2317 |
switch size { |
| 2318 |
case 8: |
| 2319 |
sz = 0b11 |
| 2320 |
case 4: |
| 2321 |
sz = 0b10 |
| 2322 |
case 2: |
| 2323 |
sz = 0b01 |
| 2324 |
case 1: |
| 2325 |
sz = 0b00 |
| 2326 |
} |
| 2327 |
|
| 2328 |
_31to21 = 0b00001000_111 | sz<<9 |
| 2329 |
_15to10 = 0b111111 |
| 2330 |
|
| 2331 |
return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt |
| 2332 |
} |
| 2333 |
|
| 2334 |
func encodeAtomicLoadStore(rn, rt, size, l uint32) uint32 { |
| 2335 |
var _31to21, _20to16, _15to10, sz uint32 |
| 2336 |
|
| 2337 |
switch size { |
| 2338 |
case 8: |
| 2339 |
sz = 0b11 |
| 2340 |
case 4: |
| 2341 |
sz = 0b10 |
| 2342 |
case 2: |
| 2343 |
sz = 0b01 |
| 2344 |
case 1: |
| 2345 |
sz = 0b00 |
| 2346 |
} |
| 2347 |
|
| 2348 |
_31to21 = 0b00001000_100 | sz<<9 | l<<1 |
| 2349 |
_20to16 = 0b11111 |
| 2350 |
_15to10 = 0b111111 |
| 2351 |
|
| 2352 |
return _31to21<<21 | _20to16<<16 | _15to10<<10 | rn<<5 | rt |
| 2353 |
} |
| 2354 |
|
| 2355 |
func encodeDMB() uint32 { |
| 2356 |
return 0b11010101000000110011101110111111 |
| 2357 |
} |