Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/instr_encoding.go

raw
1 package arm64
2
3 import (
4 "context"
5 "fmt"
6
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
8 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
9 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
10 )
11
12 // Encode implements backend.Machine Encode.
13 func (m *machine) Encode(ctx context.Context) error {
14 m.resolveRelativeAddresses(ctx)
15 m.encode(m.rootInstr)
16 if l := len(m.compiler.Buf()); l > maxFunctionExecutableSize {
17 return fmt.Errorf("function size exceeds the limit: %d > %d", l, maxFunctionExecutableSize)
18 }
19 return nil
20 }
21
22 func (m *machine) encode(root *instruction) {
23 for cur := root; cur != nil; cur = cur.next {
24 cur.encode(m)
25 }
26 }
27
28 func (i *instruction) encode(m *machine) {
29 c := m.compiler
30 switch kind := i.kind; kind {
31 case nop0, emitSourceOffsetInfo, loadConstBlockArg:
32 case exitSequence:
33 encodeExitSequence(c, i.rn.reg())
34 case ret:
35 // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en
36 c.Emit4Bytes(encodeRet())
37 case br:
38 imm := i.brOffset()
39 c.Emit4Bytes(encodeUnconditionalBranch(false, imm))
40 case call:
41 // We still don't know the exact address of the function to call, so we emit a placeholder.
42 c.AddRelocationInfo(i.callFuncRef(), false)
43 c.Emit4Bytes(encodeUnconditionalBranch(true, 0)) // 0 = placeholder
44 case callInd:
45 c.Emit4Bytes(encodeUnconditionalBranchReg(regNumberInEncoding[i.rn.realReg()], true))
46 case store8, store16, store32, store64, fpuStore32, fpuStore64, fpuStore128:
47 c.Emit4Bytes(encodeLoadOrStore(i.kind, regNumberInEncoding[i.rn.realReg()], *i.getAmode()))
48 case uLoad8, uLoad16, uLoad32, uLoad64, sLoad8, sLoad16, sLoad32, fpuLoad32, fpuLoad64, fpuLoad128:
49 c.Emit4Bytes(encodeLoadOrStore(i.kind, regNumberInEncoding[i.rd.RealReg()], *i.getAmode()))
50 case vecLoad1R:
51 c.Emit4Bytes(encodeVecLoad1R(
52 regNumberInEncoding[i.rd.RealReg()],
53 regNumberInEncoding[i.rn.realReg()],
54 vecArrangement(i.u1)))
55 case condBr:
56 imm19 := i.condBrOffset()
57 if imm19%4 != 0 {
58 panic("imm26 for branch must be a multiple of 4")
59 }
60
61 imm19U32 := uint32(imm19/4) & 0b111_11111111_11111111
62 brCond := i.condBrCond()
63 switch brCond.kind() {
64 case condKindRegisterZero:
65 rt := regNumberInEncoding[brCond.register().RealReg()]
66 c.Emit4Bytes(encodeCBZCBNZ(rt, false, imm19U32, i.condBr64bit()))
67 case condKindRegisterNotZero:
68 rt := regNumberInEncoding[brCond.register().RealReg()]
69 c.Emit4Bytes(encodeCBZCBNZ(rt, true, imm19U32, i.condBr64bit()))
70 case condKindCondFlagSet:
71 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B-cond--Branch-conditionally-
72 fl := brCond.flag()
73 c.Emit4Bytes(0b01010100<<24 | (imm19U32 << 5) | uint32(fl))
74 default:
75 panic("BUG")
76 }
77 case movN:
78 c.Emit4Bytes(encodeMoveWideImmediate(0b00, regNumberInEncoding[i.rd.RealReg()], i.u1, uint32(i.u2), uint32(i.u2>>32)))
79 case movZ:
80 c.Emit4Bytes(encodeMoveWideImmediate(0b10, regNumberInEncoding[i.rd.RealReg()], i.u1, uint32(i.u2), uint32(i.u2>>32)))
81 case movK:
82 c.Emit4Bytes(encodeMoveWideImmediate(0b11, regNumberInEncoding[i.rd.RealReg()], i.u1, uint32(i.u2), uint32(i.u2>>32)))
83 case mov32:
84 to, from := i.rd.RealReg(), i.rn.realReg()
85 c.Emit4Bytes(encodeAsMov32(regNumberInEncoding[from], regNumberInEncoding[to]))
86 case mov64:
87 to, from := i.rd.RealReg(), i.rn.realReg()
88 toIsSp := to == sp
89 fromIsSp := from == sp
90 c.Emit4Bytes(encodeMov64(regNumberInEncoding[to], regNumberInEncoding[from], toIsSp, fromIsSp))
91 case loadP64, storeP64:
92 rt, rt2 := regNumberInEncoding[i.rn.realReg()], regNumberInEncoding[i.rm.realReg()]
93 amode := i.getAmode()
94 rn := regNumberInEncoding[amode.rn.RealReg()]
95 var pre bool
96 switch amode.kind {
97 case addressModeKindPostIndex:
98 case addressModeKindPreIndex:
99 pre = true
100 default:
101 panic("BUG")
102 }
103 c.Emit4Bytes(encodePreOrPostIndexLoadStorePair64(pre, kind == loadP64, rn, rt, rt2, amode.imm))
104 case loadFpuConst32:
105 rd := regNumberInEncoding[i.rd.RealReg()]
106 if i.u1 == 0 {
107 c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement8B))
108 } else {
109 encodeLoadFpuConst32(c, rd, i.u1)
110 }
111 case loadFpuConst64:
112 rd := regNumberInEncoding[i.rd.RealReg()]
113 if i.u1 == 0 {
114 c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement8B))
115 } else {
116 encodeLoadFpuConst64(c, regNumberInEncoding[i.rd.RealReg()], i.u1)
117 }
118 case loadFpuConst128:
119 rd := regNumberInEncoding[i.rd.RealReg()]
120 lo, hi := i.u1, i.u2
121 if lo == 0 && hi == 0 {
122 c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement16B))
123 } else {
124 encodeLoadFpuConst128(c, rd, lo, hi)
125 }
126 case aluRRRR:
127 c.Emit4Bytes(encodeAluRRRR(
128 aluOp(i.u1),
129 regNumberInEncoding[i.rd.RealReg()],
130 regNumberInEncoding[i.rn.realReg()],
131 regNumberInEncoding[i.rm.realReg()],
132 regNumberInEncoding[regalloc.VReg(i.u2).RealReg()],
133 uint32(i.u1>>32),
134 ))
135 case aluRRImmShift:
136 c.Emit4Bytes(encodeAluRRImm(
137 aluOp(i.u1),
138 regNumberInEncoding[i.rd.RealReg()],
139 regNumberInEncoding[i.rn.realReg()],
140 uint32(i.rm.shiftImm()),
141 uint32(i.u2>>32),
142 ))
143 case aluRRR:
144 rn := i.rn.realReg()
145 c.Emit4Bytes(encodeAluRRR(
146 aluOp(i.u1),
147 regNumberInEncoding[i.rd.RealReg()],
148 regNumberInEncoding[rn],
149 regNumberInEncoding[i.rm.realReg()],
150 i.u2>>32 == 1,
151 rn == sp,
152 ))
153 case aluRRRExtend:
154 rm, exo, to := i.rm.er()
155 c.Emit4Bytes(encodeAluRRRExtend(
156 aluOp(i.u1),
157 regNumberInEncoding[i.rd.RealReg()],
158 regNumberInEncoding[i.rn.realReg()],
159 regNumberInEncoding[rm.RealReg()],
160 exo,
161 to,
162 ))
163 case aluRRRShift:
164 r, amt, sop := i.rm.sr()
165 c.Emit4Bytes(encodeAluRRRShift(
166 aluOp(i.u1),
167 regNumberInEncoding[i.rd.RealReg()],
168 regNumberInEncoding[i.rn.realReg()],
169 regNumberInEncoding[r.RealReg()],
170 uint32(amt),
171 sop,
172 i.u2>>32 == 1,
173 ))
174 case aluRRBitmaskImm:
175 c.Emit4Bytes(encodeAluBitmaskImmediate(
176 aluOp(i.u1),
177 regNumberInEncoding[i.rd.RealReg()],
178 regNumberInEncoding[i.rn.realReg()],
179 i.u2,
180 i.u1>>32 == 1,
181 ))
182 case bitRR:
183 c.Emit4Bytes(encodeBitRR(
184 bitOp(i.u1),
185 regNumberInEncoding[i.rd.RealReg()],
186 regNumberInEncoding[i.rn.realReg()],
187 uint32(i.u2)),
188 )
189 case aluRRImm12:
190 imm12, shift := i.rm.imm12()
191 c.Emit4Bytes(encodeAluRRImm12(
192 aluOp(i.u1),
193 regNumberInEncoding[i.rd.RealReg()],
194 regNumberInEncoding[i.rn.realReg()],
195 imm12, shift,
196 i.u2>>32 == 1,
197 ))
198 case fpuRRR:
199 c.Emit4Bytes(encodeFpuRRR(
200 fpuBinOp(i.u1),
201 regNumberInEncoding[i.rd.RealReg()],
202 regNumberInEncoding[i.rn.realReg()],
203 regNumberInEncoding[i.rm.realReg()],
204 i.u2 == 1,
205 ))
206 case fpuMov64, fpuMov128:
207 // https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/MOV--vector---Move-vector--an-alias-of-ORR--vector--register--
208 rd := regNumberInEncoding[i.rd.RealReg()]
209 rn := regNumberInEncoding[i.rn.realReg()]
210 var q uint32
211 if kind == fpuMov128 {
212 q = 0b1
213 }
214 c.Emit4Bytes(q<<30 | 0b1110101<<21 | rn<<16 | 0b000111<<10 | rn<<5 | rd)
215 case cSet:
216 rd := regNumberInEncoding[i.rd.RealReg()]
217 cf := condFlag(i.u1)
218 if i.u2 == 1 {
219 // https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/CSETM--Conditional-Set-Mask--an-alias-of-CSINV-
220 // Note that we set 64bit version here.
221 c.Emit4Bytes(0b1101101010011111<<16 | uint32(cf.invert())<<12 | 0b011111<<5 | rd)
222 } else {
223 // https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/CSET--Conditional-Set--an-alias-of-CSINC-
224 // Note that we set 64bit version here.
225 c.Emit4Bytes(0b1001101010011111<<16 | uint32(cf.invert())<<12 | 0b111111<<5 | rd)
226 }
227 case extend:
228 c.Emit4Bytes(encodeExtend((i.u2>>32) == 1, byte(i.u1), byte(i.u2), regNumberInEncoding[i.rd.RealReg()], regNumberInEncoding[i.rn.realReg()]))
229 case fpuCmp:
230 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/FCMP--Floating-point-quiet-Compare--scalar--?lang=en
231 rn, rm := regNumberInEncoding[i.rn.realReg()], regNumberInEncoding[i.rm.realReg()]
232 var ftype uint32
233 if i.u1 == 1 {
234 ftype = 0b01 // double precision.
235 }
236 c.Emit4Bytes(0b1111<<25 | ftype<<22 | 1<<21 | rm<<16 | 0b1<<13 | rn<<5)
237 case udf:
238 // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UDF--Permanently-Undefined-?lang=en
239 if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable {
240 c.Emit4Bytes(dummyInstruction)
241 } else {
242 c.Emit4Bytes(0)
243 }
244 case adr:
245 c.Emit4Bytes(encodeAdr(regNumberInEncoding[i.rd.RealReg()], uint32(i.u1)))
246 case cSel:
247 c.Emit4Bytes(encodeConditionalSelect(
248 kind,
249 regNumberInEncoding[i.rd.RealReg()],
250 regNumberInEncoding[i.rn.realReg()],
251 regNumberInEncoding[i.rm.realReg()],
252 condFlag(i.u1),
253 i.u2 == 1,
254 ))
255 case fpuCSel:
256 c.Emit4Bytes(encodeFpuCSel(
257 regNumberInEncoding[i.rd.RealReg()],
258 regNumberInEncoding[i.rn.realReg()],
259 regNumberInEncoding[i.rm.realReg()],
260 condFlag(i.u1),
261 i.u2 == 1,
262 ))
263 case movToVec:
264 c.Emit4Bytes(encodeMoveToVec(
265 regNumberInEncoding[i.rd.RealReg()],
266 regNumberInEncoding[i.rn.realReg()],
267 vecArrangement(byte(i.u1)),
268 vecIndex(i.u2),
269 ))
270 case movFromVec, movFromVecSigned:
271 c.Emit4Bytes(encodeMoveFromVec(
272 regNumberInEncoding[i.rd.RealReg()],
273 regNumberInEncoding[i.rn.realReg()],
274 vecArrangement(byte(i.u1)),
275 vecIndex(i.u2),
276 i.kind == movFromVecSigned,
277 ))
278 case vecDup:
279 c.Emit4Bytes(encodeVecDup(
280 regNumberInEncoding[i.rd.RealReg()],
281 regNumberInEncoding[i.rn.realReg()],
282 vecArrangement(byte(i.u1))))
283 case vecDupElement:
284 c.Emit4Bytes(encodeVecDupElement(
285 regNumberInEncoding[i.rd.RealReg()],
286 regNumberInEncoding[i.rn.realReg()],
287 vecArrangement(byte(i.u1)),
288 vecIndex(i.u2)))
289 case vecExtract:
290 c.Emit4Bytes(encodeVecExtract(
291 regNumberInEncoding[i.rd.RealReg()],
292 regNumberInEncoding[i.rn.realReg()],
293 regNumberInEncoding[i.rm.realReg()],
294 vecArrangement(byte(i.u1)),
295 uint32(i.u2)))
296 case vecPermute:
297 c.Emit4Bytes(encodeVecPermute(
298 vecOp(i.u1),
299 regNumberInEncoding[i.rd.RealReg()],
300 regNumberInEncoding[i.rn.realReg()],
301 regNumberInEncoding[i.rm.realReg()],
302 vecArrangement(byte(i.u2))))
303 case vecMovElement:
304 c.Emit4Bytes(encodeVecMovElement(
305 regNumberInEncoding[i.rd.RealReg()],
306 regNumberInEncoding[i.rn.realReg()],
307 vecArrangement(i.u1),
308 uint32(i.u2), uint32(i.u2>>32),
309 ))
310 case vecMisc:
311 c.Emit4Bytes(encodeAdvancedSIMDTwoMisc(
312 vecOp(i.u1),
313 regNumberInEncoding[i.rd.RealReg()],
314 regNumberInEncoding[i.rn.realReg()],
315 vecArrangement(i.u2),
316 ))
317 case vecLanes:
318 c.Emit4Bytes(encodeVecLanes(
319 vecOp(i.u1),
320 regNumberInEncoding[i.rd.RealReg()],
321 regNumberInEncoding[i.rn.realReg()],
322 vecArrangement(i.u2),
323 ))
324 case vecShiftImm:
325 c.Emit4Bytes(encodeVecShiftImm(
326 vecOp(i.u1),
327 regNumberInEncoding[i.rd.RealReg()],
328 regNumberInEncoding[i.rn.realReg()],
329 uint32(i.rm.shiftImm()),
330 vecArrangement(i.u2),
331 ))
332 case vecTbl:
333 c.Emit4Bytes(encodeVecTbl(
334 1,
335 regNumberInEncoding[i.rd.RealReg()],
336 regNumberInEncoding[i.rn.realReg()],
337 regNumberInEncoding[i.rm.realReg()],
338 vecArrangement(i.u2)),
339 )
340 case vecTbl2:
341 c.Emit4Bytes(encodeVecTbl(
342 2,
343 regNumberInEncoding[i.rd.RealReg()],
344 regNumberInEncoding[i.rn.realReg()],
345 regNumberInEncoding[i.rm.realReg()],
346 vecArrangement(i.u2)),
347 )
348 case brTableSequence:
349 targets := m.jmpTableTargets[i.u1]
350 encodeBrTableSequence(c, i.rn.reg(), targets)
351 case fpuToInt, intToFpu:
352 c.Emit4Bytes(encodeCnvBetweenFloatInt(i))
353 case fpuRR:
354 c.Emit4Bytes(encodeFloatDataOneSource(
355 fpuUniOp(i.u1),
356 regNumberInEncoding[i.rd.RealReg()],
357 regNumberInEncoding[i.rn.realReg()],
358 i.u2 == 1,
359 ))
360 case vecRRR:
361 if op := vecOp(i.u1); op == vecOpBsl || op == vecOpBit || op == vecOpUmlal {
362 panic(fmt.Sprintf("vecOp %s must use vecRRRRewrite instead of vecRRR", op.String()))
363 }
364 fallthrough
365 case vecRRRRewrite:
366 c.Emit4Bytes(encodeVecRRR(
367 vecOp(i.u1),
368 regNumberInEncoding[i.rd.RealReg()],
369 regNumberInEncoding[i.rn.realReg()],
370 regNumberInEncoding[i.rm.realReg()],
371 vecArrangement(i.u2),
372 ))
373 case cCmpImm:
374 // Conditional compare (immediate) in https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en
375 sf := uint32((i.u2 >> 32) & 0b1)
376 nzcv := uint32(i.u2 & 0b1111)
377 cond := uint32(condFlag(i.u1))
378 imm := uint32(i.rm.data & 0b11111)
379 rn := regNumberInEncoding[i.rn.realReg()]
380 c.Emit4Bytes(
381 sf<<31 | 0b111101001<<22 | imm<<16 | cond<<12 | 0b1<<11 | rn<<5 | nzcv,
382 )
383 case movFromFPSR:
384 rt := regNumberInEncoding[i.rd.RealReg()]
385 c.Emit4Bytes(encodeSystemRegisterMove(rt, true))
386 case movToFPSR:
387 rt := regNumberInEncoding[i.rn.realReg()]
388 c.Emit4Bytes(encodeSystemRegisterMove(rt, false))
389 case atomicRmw:
390 c.Emit4Bytes(encodeAtomicRmw(
391 atomicRmwOp(i.u1),
392 regNumberInEncoding[i.rm.realReg()],
393 regNumberInEncoding[i.rd.RealReg()],
394 regNumberInEncoding[i.rn.realReg()],
395 uint32(i.u2),
396 ))
397 case atomicCas:
398 c.Emit4Bytes(encodeAtomicCas(
399 regNumberInEncoding[i.rd.RealReg()],
400 regNumberInEncoding[i.rm.realReg()],
401 regNumberInEncoding[i.rn.realReg()],
402 uint32(i.u2),
403 ))
404 case atomicLoad:
405 c.Emit4Bytes(encodeAtomicLoadStore(
406 regNumberInEncoding[i.rn.realReg()],
407 regNumberInEncoding[i.rd.RealReg()],
408 uint32(i.u2),
409 1,
410 ))
411 case atomicStore:
412 c.Emit4Bytes(encodeAtomicLoadStore(
413 regNumberInEncoding[i.rn.realReg()],
414 regNumberInEncoding[i.rm.realReg()],
415 uint32(i.u2),
416 0,
417 ))
418 case dmb:
419 c.Emit4Bytes(encodeDMB())
420 case tailCall:
421 // We still don't know the exact address of the function to call, so we emit a placeholder.
422 c.AddRelocationInfo(i.callFuncRef(), true) // true = IsTailCall
423 c.Emit4Bytes(encodeUnconditionalBranch(false, 0)) // 0 = placeholder
424 case tailCallInd:
425 c.Emit4Bytes(encodeUnconditionalBranchReg(regNumberInEncoding[i.rn.realReg()], false))
426 default:
427 panic(i.String())
428 }
429 }
430
431 func encodeMov64(rd, rn uint32, toIsSp, fromIsSp bool) uint32 {
432 if toIsSp || fromIsSp {
433 // This is an alias of ADD (immediate):
434 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--to-from-SP---Move-between-register-and-stack-pointer--an-alias-of-ADD--immediate--
435 return encodeAddSubtractImmediate(0b100, 0, 0, rn, rd)
436 } else {
437 // This is an alias of ORR (shifted register):
438 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register--
439 return encodeLogicalShiftedRegister(0b101, 0, rn, 0, regNumberInEncoding[xzr], rd)
440 }
441 }
442
443 // encodeSystemRegisterMove encodes as "System register move" in
444 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en
445 //
446 // Note that currently we only supports read/write of FPSR.
447 func encodeSystemRegisterMove(rt uint32, fromSystem bool) uint32 {
448 ret := 0b11010101<<24 | 0b11011<<16 | 0b01000100<<8 | 0b001<<5 | rt
449 if fromSystem {
450 ret |= 0b1 << 21
451 }
452 return ret
453 }
454
455 // encodeVecRRR encodes as either "Advanced SIMD three *" in
456 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
457 func encodeVecRRR(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 {
458 switch op {
459 case vecOpBit:
460 _, q := arrToSizeQEncoded(arr)
461 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b10 /* always has size 0b10 */, 0b1, q)
462 case vecOpBic:
463 if arr > vecArrangement16B {
464 panic("unsupported arrangement: " + arr.String())
465 }
466 _, q := arrToSizeQEncoded(arr)
467 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b01 /* always has size 0b01 */, 0b0, q)
468 case vecOpBsl:
469 if arr > vecArrangement16B {
470 panic("unsupported arrangement: " + arr.String())
471 }
472 _, q := arrToSizeQEncoded(arr)
473 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b01 /* always has size 0b01 */, 0b1, q)
474 case vecOpAnd:
475 if arr > vecArrangement16B {
476 panic("unsupported arrangement: " + arr.String())
477 }
478 _, q := arrToSizeQEncoded(arr)
479 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b00 /* always has size 0b00 */, 0b0, q)
480 case vecOpOrr:
481 _, q := arrToSizeQEncoded(arr)
482 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b10 /* always has size 0b10 */, 0b0, q)
483 case vecOpEOR:
484 size, q := arrToSizeQEncoded(arr)
485 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, size, 0b1, q)
486 case vecOpCmeq:
487 size, q := arrToSizeQEncoded(arr)
488 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10001, size, 0b1, q)
489 case vecOpCmgt:
490 size, q := arrToSizeQEncoded(arr)
491 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00110, size, 0b0, q)
492 case vecOpCmhi:
493 size, q := arrToSizeQEncoded(arr)
494 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00110, size, 0b1, q)
495 case vecOpCmge:
496 size, q := arrToSizeQEncoded(arr)
497 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00111, size, 0b0, q)
498 case vecOpCmhs:
499 size, q := arrToSizeQEncoded(arr)
500 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00111, size, 0b1, q)
501 case vecOpFcmeq:
502 var size, q uint32
503 switch arr {
504 case vecArrangement4S:
505 size, q = 0b00, 0b1
506 case vecArrangement2S:
507 size, q = 0b00, 0b0
508 case vecArrangement2D:
509 size, q = 0b01, 0b1
510 default:
511 panic("unsupported arrangement: " + arr.String())
512 }
513 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b0, q)
514 case vecOpFcmgt:
515 if arr < vecArrangement2S || arr == vecArrangement1D {
516 panic("unsupported arrangement: " + arr.String())
517 }
518 size, q := arrToSizeQEncoded(arr)
519 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b1, q)
520 case vecOpFcmge:
521 var size, q uint32
522 switch arr {
523 case vecArrangement4S:
524 size, q = 0b00, 0b1
525 case vecArrangement2S:
526 size, q = 0b00, 0b0
527 case vecArrangement2D:
528 size, q = 0b01, 0b1
529 default:
530 panic("unsupported arrangement: " + arr.String())
531 }
532 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b1, q)
533 case vecOpAdd:
534 if arr == vecArrangement1D {
535 panic("unsupported arrangement: " + arr.String())
536 }
537 size, q := arrToSizeQEncoded(arr)
538 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10000, size, 0b0, q)
539 case vecOpSqadd:
540 if arr == vecArrangement1D {
541 panic("unsupported arrangement: " + arr.String())
542 }
543 size, q := arrToSizeQEncoded(arr)
544 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00001, size, 0b0, q)
545 case vecOpUqadd:
546 if arr == vecArrangement1D {
547 panic("unsupported arrangement: " + arr.String())
548 }
549 size, q := arrToSizeQEncoded(arr)
550 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00001, size, 0b1, q)
551 case vecOpAddp:
552 if arr == vecArrangement1D {
553 panic("unsupported arrangement: " + arr.String())
554 }
555 size, q := arrToSizeQEncoded(arr)
556 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10111, size, 0b0, q)
557 case vecOpSqsub:
558 if arr == vecArrangement1D {
559 panic("unsupported arrangement: " + arr.String())
560 }
561 size, q := arrToSizeQEncoded(arr)
562 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00101, size, 0b0, q)
563 case vecOpUqsub:
564 if arr == vecArrangement1D {
565 panic("unsupported arrangement: " + arr.String())
566 }
567 size, q := arrToSizeQEncoded(arr)
568 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00101, size, 0b1, q)
569 case vecOpSub:
570 if arr == vecArrangement1D {
571 panic("unsupported arrangement: " + arr.String())
572 }
573 size, q := arrToSizeQEncoded(arr)
574 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10000, size, 0b1, q)
575 case vecOpFmin:
576 if arr < vecArrangement2S || arr == vecArrangement1D {
577 panic("unsupported arrangement: " + arr.String())
578 }
579 size, q := arrToSizeQEncoded(arr)
580 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11110, size, 0b0, q)
581 case vecOpSmin:
582 if arr > vecArrangement4S {
583 panic("unsupported arrangement: " + arr.String())
584 }
585 size, q := arrToSizeQEncoded(arr)
586 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01101, size, 0b0, q)
587 case vecOpUmin:
588 if arr > vecArrangement4S {
589 panic("unsupported arrangement: " + arr.String())
590 }
591 size, q := arrToSizeQEncoded(arr)
592 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01101, size, 0b1, q)
593 case vecOpFmax:
594 var size, q uint32
595 switch arr {
596 case vecArrangement4S:
597 size, q = 0b00, 0b1
598 case vecArrangement2S:
599 size, q = 0b00, 0b0
600 case vecArrangement2D:
601 size, q = 0b01, 0b1
602 default:
603 panic("unsupported arrangement: " + arr.String())
604 }
605 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11110, size, 0b0, q)
606 case vecOpFadd:
607 var size, q uint32
608 switch arr {
609 case vecArrangement4S:
610 size, q = 0b00, 0b1
611 case vecArrangement2S:
612 size, q = 0b00, 0b0
613 case vecArrangement2D:
614 size, q = 0b01, 0b1
615 default:
616 panic("unsupported arrangement: " + arr.String())
617 }
618 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11010, size, 0b0, q)
619 case vecOpFsub:
620 if arr < vecArrangement2S || arr == vecArrangement1D {
621 panic("unsupported arrangement: " + arr.String())
622 }
623 size, q := arrToSizeQEncoded(arr)
624 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11010, size, 0b0, q)
625 case vecOpFmul:
626 var size, q uint32
627 switch arr {
628 case vecArrangement4S:
629 size, q = 0b00, 0b1
630 case vecArrangement2S:
631 size, q = 0b00, 0b0
632 case vecArrangement2D:
633 size, q = 0b01, 0b1
634 default:
635 panic("unsupported arrangement: " + arr.String())
636 }
637 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11011, size, 0b1, q)
638 case vecOpSqrdmulh:
639 if arr < vecArrangement4H || arr > vecArrangement4S {
640 panic("unsupported arrangement: " + arr.String())
641 }
642 size, q := arrToSizeQEncoded(arr)
643 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10110, size, 0b1, q)
644 case vecOpFdiv:
645 var size, q uint32
646 switch arr {
647 case vecArrangement4S:
648 size, q = 0b00, 0b1
649 case vecArrangement2S:
650 size, q = 0b00, 0b0
651 case vecArrangement2D:
652 size, q = 0b01, 0b1
653 default:
654 panic("unsupported arrangement: " + arr.String())
655 }
656 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11111, size, 0b1, q)
657 case vecOpSmax:
658 if arr > vecArrangement4S {
659 panic("unsupported arrangement: " + arr.String())
660 }
661 size, q := arrToSizeQEncoded(arr)
662 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01100, size, 0b0, q)
663 case vecOpUmax:
664 if arr > vecArrangement4S {
665 panic("unsupported arrangement: " + arr.String())
666 }
667 size, q := arrToSizeQEncoded(arr)
668 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01100, size, 0b1, q)
669 case vecOpUmaxp:
670 if arr > vecArrangement4S {
671 panic("unsupported arrangement: " + arr.String())
672 }
673 size, q := arrToSizeQEncoded(arr)
674 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10100, size, 0b1, q)
675 case vecOpUrhadd:
676 if arr > vecArrangement4S {
677 panic("unsupported arrangement: " + arr.String())
678 }
679 size, q := arrToSizeQEncoded(arr)
680 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00010, size, 0b1, q)
681 case vecOpMul:
682 if arr > vecArrangement4S {
683 panic("unsupported arrangement: " + arr.String())
684 }
685 size, q := arrToSizeQEncoded(arr)
686 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10011, size, 0b0, q)
687 case vecOpUmlal:
688 if arr > vecArrangement4S {
689 panic("unsupported arrangement: " + arr.String())
690 }
691 size, q := arrToSizeQEncoded(arr)
692 return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1000, size, 0b1, q)
693 case vecOpSshl:
694 if arr == vecArrangement1D {
695 panic("unsupported arrangement: " + arr.String())
696 }
697 size, q := arrToSizeQEncoded(arr)
698 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b0, q)
699 case vecOpUshl:
700 if arr == vecArrangement1D {
701 panic("unsupported arrangement: " + arr.String())
702 }
703 size, q := arrToSizeQEncoded(arr)
704 return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b1, q)
705
706 case vecOpSmull:
707 if arr > vecArrangement4S {
708 panic("unsupported arrangement: " + arr.String())
709 }
710 size, _ := arrToSizeQEncoded(arr)
711 return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b0)
712
713 case vecOpSmull2:
714 if arr > vecArrangement4S {
715 panic("unsupported arrangement: " + arr.String())
716 }
717 size, _ := arrToSizeQEncoded(arr)
718 return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b1)
719
720 default:
721 panic("TODO: " + op.String())
722 }
723 }
724
725 func arrToSizeQEncoded(arr vecArrangement) (size, q uint32) {
726 switch arr {
727 case vecArrangement16B:
728 q = 0b1
729 fallthrough
730 case vecArrangement8B:
731 size = 0b00
732 case vecArrangement8H:
733 q = 0b1
734 fallthrough
735 case vecArrangement4H:
736 size = 0b01
737 case vecArrangement4S:
738 q = 0b1
739 fallthrough
740 case vecArrangement2S:
741 size = 0b10
742 case vecArrangement2D:
743 q = 0b1
744 fallthrough
745 case vecArrangement1D:
746 size = 0b11
747 default:
748 panic("BUG")
749 }
750 return
751 }
752
753 // encodeAdvancedSIMDThreeSame encodes as "Advanced SIMD three same" in
754 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
755 func encodeAdvancedSIMDThreeSame(rd, rn, rm, opcode, size, U, Q uint32) uint32 {
756 return Q<<30 | U<<29 | 0b111<<25 | size<<22 | 0b1<<21 | rm<<16 | opcode<<11 | 0b1<<10 | rn<<5 | rd
757 }
758
759 // encodeAdvancedSIMDThreeDifferent encodes as "Advanced SIMD three different" in
760 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
761 func encodeAdvancedSIMDThreeDifferent(rd, rn, rm, opcode, size, U, Q uint32) uint32 {
762 return Q<<30 | U<<29 | 0b111<<25 | size<<22 | 0b1<<21 | rm<<16 | opcode<<12 | rn<<5 | rd
763 }
764
765 // encodeFloatDataOneSource encodes as "Floating-point data-processing (1 source)" in
766 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp
767 func encodeFloatDataOneSource(op fpuUniOp, rd, rn uint32, dst64bit bool) uint32 {
768 var opcode, ptype uint32
769 switch op {
770 case fpuUniOpCvt32To64:
771 opcode = 0b000101
772 case fpuUniOpCvt64To32:
773 opcode = 0b000100
774 ptype = 0b01
775 case fpuUniOpNeg:
776 opcode = 0b000010
777 if dst64bit {
778 ptype = 0b01
779 }
780 case fpuUniOpSqrt:
781 opcode = 0b000011
782 if dst64bit {
783 ptype = 0b01
784 }
785 case fpuUniOpRoundPlus:
786 opcode = 0b001001
787 if dst64bit {
788 ptype = 0b01
789 }
790 case fpuUniOpRoundMinus:
791 opcode = 0b001010
792 if dst64bit {
793 ptype = 0b01
794 }
795 case fpuUniOpRoundZero:
796 opcode = 0b001011
797 if dst64bit {
798 ptype = 0b01
799 }
800 case fpuUniOpRoundNearest:
801 opcode = 0b001000
802 if dst64bit {
803 ptype = 0b01
804 }
805 case fpuUniOpAbs:
806 opcode = 0b000001
807 if dst64bit {
808 ptype = 0b01
809 }
810 default:
811 panic("BUG")
812 }
813 return 0b1111<<25 | ptype<<22 | 0b1<<21 | opcode<<15 | 0b1<<14 | rn<<5 | rd
814 }
815
816 // encodeCnvBetweenFloatInt encodes as "Conversion between floating-point and integer" in
817 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
818 func encodeCnvBetweenFloatInt(i *instruction) uint32 {
819 rd := regNumberInEncoding[i.rd.RealReg()]
820 rn := regNumberInEncoding[i.rn.realReg()]
821
822 var opcode uint32
823 var rmode uint32
824 var ptype uint32
825 var sf uint32
826 switch i.kind {
827 case intToFpu: // Either UCVTF or SCVTF.
828 rmode = 0b00
829
830 signed := i.u1 == 1
831 src64bit := i.u2&1 != 0
832 dst64bit := i.u2&2 != 0
833 if signed {
834 opcode = 0b010
835 } else {
836 opcode = 0b011
837 }
838 if src64bit {
839 sf = 0b1
840 }
841 if dst64bit {
842 ptype = 0b01
843 } else {
844 ptype = 0b00
845 }
846 case fpuToInt: // Either FCVTZU or FCVTZS.
847 rmode = 0b11
848
849 signed := i.u1 == 1
850 src64bit := i.u2&1 != 0
851 dst64bit := i.u2&2 != 0
852
853 if signed {
854 opcode = 0b000
855 } else {
856 opcode = 0b001
857 }
858 if dst64bit {
859 sf = 0b1
860 }
861 if src64bit {
862 ptype = 0b01
863 } else {
864 ptype = 0b00
865 }
866 }
867 return sf<<31 | 0b1111<<25 | ptype<<22 | 0b1<<21 | rmode<<19 | opcode<<16 | rn<<5 | rd
868 }
869
870 // encodeAdr encodes a PC-relative ADR instruction.
871 // https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/ADR--Form-PC-relative-address-
872 func encodeAdr(rd uint32, offset uint32) uint32 {
873 if offset >= 1<<20 {
874 panic("BUG: too large adr instruction")
875 }
876 return offset&0b11<<29 | 0b1<<28 | offset&0b1111111111_1111111100<<3 | rd
877 }
878
879 // encodeFpuCSel encodes as "Floating-point conditional select" in
880 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
881 func encodeFpuCSel(rd, rn, rm uint32, c condFlag, _64bit bool) uint32 {
882 var ftype uint32
883 if _64bit {
884 ftype = 0b01 // double precision.
885 }
886 return 0b1111<<25 | ftype<<22 | 0b1<<21 | rm<<16 | uint32(c)<<12 | 0b11<<10 | rn<<5 | rd
887 }
888
889 // encodeMoveToVec encodes as "Move general-purpose register to a vector element" (represented as `ins`) in
890 // https://developer.arm.com/documentation/dui0801/g/A64-SIMD-Vector-Instructions/MOV--vector--from-general-
891 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--from-general---Move-general-purpose-register-to-a-vector-element--an-alias-of-INS--general--?lang=en
892 func encodeMoveToVec(rd, rn uint32, arr vecArrangement, index vecIndex) uint32 {
893 var imm5 uint32
894 switch arr {
895 case vecArrangementB:
896 imm5 |= 0b1
897 imm5 |= uint32(index) << 1
898 if index > 0b1111 {
899 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", index))
900 }
901 case vecArrangementH:
902 imm5 |= 0b10
903 imm5 |= uint32(index) << 2
904 if index > 0b111 {
905 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", index))
906 }
907 case vecArrangementS:
908 imm5 |= 0b100
909 imm5 |= uint32(index) << 3
910 if index > 0b11 {
911 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", index))
912 }
913 case vecArrangementD:
914 imm5 |= 0b1000
915 imm5 |= uint32(index) << 4
916 if index > 0b1 {
917 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", index))
918 }
919 default:
920 panic("Unsupported arrangement " + arr.String())
921 }
922
923 return 0b01001110000<<21 | imm5<<16 | 0b000111<<10 | rn<<5 | rd
924 }
925
926 // encodeMoveToVec encodes as "Move vector element to another vector element, mov (element)" (represented as `ins`) in
927 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--element---Move-vector-element-to-another-vector-element--an-alias-of-INS--element--?lang=en
928 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/INS--element---Insert-vector-element-from-another-vector-element-?lang=en
929 func encodeVecMovElement(rd, rn uint32, arr vecArrangement, srcIndex, dstIndex uint32) uint32 {
930 var imm4, imm5 uint32
931 switch arr {
932 case vecArrangementB:
933 imm5 |= 0b1
934 imm5 |= srcIndex << 1
935 imm4 = dstIndex
936 if srcIndex > 0b1111 {
937 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", srcIndex))
938 }
939 case vecArrangementH:
940 imm5 |= 0b10
941 imm5 |= srcIndex << 2
942 imm4 = dstIndex << 1
943 if srcIndex > 0b111 {
944 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", srcIndex))
945 }
946 case vecArrangementS:
947 imm5 |= 0b100
948 imm5 |= srcIndex << 3
949 imm4 = dstIndex << 2
950 if srcIndex > 0b11 {
951 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", srcIndex))
952 }
953 case vecArrangementD:
954 imm5 |= 0b1000
955 imm5 |= srcIndex << 4
956 imm4 = dstIndex << 3
957 if srcIndex > 0b1 {
958 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", srcIndex))
959 }
960 default:
961 panic("Unsupported arrangement " + arr.String())
962 }
963
964 return 0b01101110000<<21 | imm5<<16 | imm4<<11 | 0b1<<10 | rn<<5 | rd
965 }
966
967 // encodeUnconditionalBranchReg encodes as "Unconditional branch (register)" in:
968 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en
969 func encodeUnconditionalBranchReg(rn uint32, link bool) uint32 {
970 var opc uint32
971 if link {
972 opc = 0b0001
973 }
974 return 0b1101011<<25 | opc<<21 | 0b11111<<16 | rn<<5
975 }
976
977 // encodeMoveFromVec encodes as "Move vector element to a general-purpose register"
978 // (represented as `umov` when dest is 32-bit, `umov` otherwise) in
979 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/UMOV--Unsigned-Move-vector-element-to-general-purpose-register-?lang=en
980 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--to-general---Move-vector-element-to-general-purpose-register--an-alias-of-UMOV-?lang=en
981 func encodeMoveFromVec(rd, rn uint32, arr vecArrangement, index vecIndex, signed bool) uint32 {
982 var op, imm4, q, imm5 uint32
983 switch {
984 case arr == vecArrangementB:
985 imm5 |= 0b1
986 imm5 |= uint32(index) << 1
987 if index > 0b1111 {
988 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", index))
989 }
990 case arr == vecArrangementH:
991 imm5 |= 0b10
992 imm5 |= uint32(index) << 2
993 if index > 0b111 {
994 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", index))
995 }
996 case arr == vecArrangementS && signed:
997 q = 0b1
998 fallthrough
999 case arr == vecArrangementS:
1000 imm5 |= 0b100
1001 imm5 |= uint32(index) << 3
1002 if index > 0b11 {
1003 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", index))
1004 }
1005 case arr == vecArrangementD && !signed:
1006 imm5 |= 0b1000
1007 imm5 |= uint32(index) << 4
1008 q = 0b1
1009 if index > 0b1 {
1010 panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", index))
1011 }
1012 default:
1013 panic("Unsupported arrangement " + arr.String())
1014 }
1015 if signed {
1016 op, imm4 = 0, 0b0101
1017 } else {
1018 op, imm4 = 0, 0b0111
1019 }
1020 return op<<29 | 0b01110000<<21 | q<<30 | imm5<<16 | imm4<<11 | 1<<10 | rn<<5 | rd
1021 }
1022
1023 // encodeVecDup encodes as "Duplicate general-purpose register to vector" DUP (general)
1024 // (represented as `dup`)
1025 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/DUP--general---Duplicate-general-purpose-register-to-vector-?lang=en
1026 func encodeVecDup(rd, rn uint32, arr vecArrangement) uint32 {
1027 var q, imm5 uint32
1028 switch arr {
1029 case vecArrangement8B:
1030 q, imm5 = 0b0, 0b1
1031 case vecArrangement16B:
1032 q, imm5 = 0b1, 0b1
1033 case vecArrangement4H:
1034 q, imm5 = 0b0, 0b10
1035 case vecArrangement8H:
1036 q, imm5 = 0b1, 0b10
1037 case vecArrangement2S:
1038 q, imm5 = 0b0, 0b100
1039 case vecArrangement4S:
1040 q, imm5 = 0b1, 0b100
1041 case vecArrangement2D:
1042 q, imm5 = 0b1, 0b1000
1043 default:
1044 panic("Unsupported arrangement " + arr.String())
1045 }
1046 return q<<30 | 0b001110000<<21 | imm5<<16 | 0b000011<<10 | rn<<5 | rd
1047 }
1048
1049 // encodeVecDup encodes as "Duplicate vector element to vector or scalar" DUP (element).
1050 // (represented as `dup`)
1051 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/DUP--element---Duplicate-vector-element-to-vector-or-scalar-
1052 func encodeVecDupElement(rd, rn uint32, arr vecArrangement, srcIndex vecIndex) uint32 {
1053 var q, imm5 uint32
1054 q = 0b1
1055 switch arr {
1056 case vecArrangementB:
1057 imm5 |= 0b1
1058 imm5 |= uint32(srcIndex) << 1
1059 case vecArrangementH:
1060 imm5 |= 0b10
1061 imm5 |= uint32(srcIndex) << 2
1062 case vecArrangementS:
1063 imm5 |= 0b100
1064 imm5 |= uint32(srcIndex) << 3
1065 case vecArrangementD:
1066 imm5 |= 0b1000
1067 imm5 |= uint32(srcIndex) << 4
1068 default:
1069 panic("unsupported arrangement" + arr.String())
1070 }
1071
1072 return q<<30 | 0b001110000<<21 | imm5<<16 | 0b1<<10 | rn<<5 | rd
1073 }
1074
1075 // encodeVecExtract encodes as "Advanced SIMD extract."
1076 // Currently only `ext` is defined.
1077 // https://developer.arm.com/documentation/ddi0602/2023-06/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp
1078 // https://developer.arm.com/documentation/ddi0602/2023-06/SIMD-FP-Instructions/EXT--Extract-vector-from-pair-of-vectors-?lang=en
1079 func encodeVecExtract(rd, rn, rm uint32, arr vecArrangement, index uint32) uint32 {
1080 var q, imm4 uint32
1081 switch arr {
1082 case vecArrangement8B:
1083 q, imm4 = 0, 0b0111&uint32(index)
1084 case vecArrangement16B:
1085 q, imm4 = 1, 0b1111&uint32(index)
1086 default:
1087 panic("Unsupported arrangement " + arr.String())
1088 }
1089 return q<<30 | 0b101110000<<21 | rm<<16 | imm4<<11 | rn<<5 | rd
1090 }
1091
1092 // encodeVecPermute encodes as "Advanced SIMD permute."
1093 // https://developer.arm.com/documentation/ddi0602/2023-06/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp
1094 func encodeVecPermute(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 {
1095 var q, size, opcode uint32
1096 switch op {
1097 case vecOpZip1:
1098 opcode = 0b011
1099 if arr == vecArrangement1D {
1100 panic("unsupported arrangement: " + arr.String())
1101 }
1102 size, q = arrToSizeQEncoded(arr)
1103 default:
1104 panic("TODO: " + op.String())
1105 }
1106 return q<<30 | 0b001110<<24 | size<<22 | rm<<16 | opcode<<12 | 0b10<<10 | rn<<5 | rd
1107 }
1108
1109 // encodeConditionalSelect encodes as "Conditional select" in
1110 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#condsel
1111 func encodeConditionalSelect(kind instructionKind, rd, rn, rm uint32, c condFlag, _64bit bool) uint32 {
1112 if kind != cSel {
1113 panic("TODO: support other conditional select")
1114 }
1115
1116 ret := 0b110101<<23 | rm<<16 | uint32(c)<<12 | rn<<5 | rd
1117 if _64bit {
1118 ret |= 0b1 << 31
1119 }
1120 return ret
1121 }
1122
1123 const dummyInstruction uint32 = 0x14000000 // "b 0"
1124
1125 // encodeLoadFpuConst32 encodes the following three instructions:
1126 //
1127 // ldr s8, #8 ;; literal load of data.f32
1128 // b 8 ;; skip the data
1129 // data.f32 xxxxxxx
1130 func encodeLoadFpuConst32(c backend.Compiler, rd uint32, rawF32 uint64) {
1131 c.Emit4Bytes(
1132 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en
1133 0b111<<26 | (0x8/4)<<5 | rd,
1134 )
1135 c.Emit4Bytes(encodeUnconditionalBranch(false, 8)) // b 8
1136 if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable {
1137 // Inlined data.f32 cannot be disassembled, so we add a dummy instruction here.
1138 c.Emit4Bytes(dummyInstruction)
1139 } else {
1140 c.Emit4Bytes(uint32(rawF32)) // data.f32 xxxxxxx
1141 }
1142 }
1143
1144 // encodeLoadFpuConst64 encodes the following three instructions:
1145 //
1146 // ldr d8, #8 ;; literal load of data.f64
1147 // b 12 ;; skip the data
1148 // data.f64 xxxxxxx
1149 func encodeLoadFpuConst64(c backend.Compiler, rd uint32, rawF64 uint64) {
1150 c.Emit4Bytes(
1151 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en
1152 0b1<<30 | 0b111<<26 | (0x8/4)<<5 | rd,
1153 )
1154 c.Emit4Bytes(encodeUnconditionalBranch(false, 12)) // b 12
1155 if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable {
1156 // Inlined data.f64 cannot be disassembled, so we add dummy instructions here.
1157 c.Emit4Bytes(dummyInstruction)
1158 c.Emit4Bytes(dummyInstruction)
1159 } else {
1160 // data.f64 xxxxxxx
1161 c.Emit4Bytes(uint32(rawF64))
1162 c.Emit4Bytes(uint32(rawF64 >> 32))
1163 }
1164 }
1165
1166 // encodeLoadFpuConst128 encodes the following three instructions:
1167 //
1168 // ldr v8, #8 ;; literal load of data.f64
1169 // b 20 ;; skip the data
1170 // data.v128 xxxxxxx
1171 func encodeLoadFpuConst128(c backend.Compiler, rd uint32, lo, hi uint64) {
1172 c.Emit4Bytes(
1173 // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en
1174 0b1<<31 | 0b111<<26 | (0x8/4)<<5 | rd,
1175 )
1176 c.Emit4Bytes(encodeUnconditionalBranch(false, 20)) // b 20
1177 if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable {
1178 // Inlined data.v128 cannot be disassembled, so we add dummy instructions here.
1179 c.Emit4Bytes(dummyInstruction)
1180 c.Emit4Bytes(dummyInstruction)
1181 c.Emit4Bytes(dummyInstruction)
1182 c.Emit4Bytes(dummyInstruction)
1183 } else {
1184 // data.v128 xxxxxxx
1185 c.Emit4Bytes(uint32(lo))
1186 c.Emit4Bytes(uint32(lo >> 32))
1187 c.Emit4Bytes(uint32(hi))
1188 c.Emit4Bytes(uint32(hi >> 32))
1189 }
1190 }
1191
1192 // encodeAluRRRR encodes as Data-processing (3 source) in
1193 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en
1194 func encodeAluRRRR(op aluOp, rd, rn, rm, ra, _64bit uint32) uint32 {
1195 var oO, op31 uint32
1196 switch op {
1197 case aluOpMAdd:
1198 op31, oO = 0b000, 0b0
1199 case aluOpMSub:
1200 op31, oO = 0b000, 0b1
1201 default:
1202 panic("TODO/BUG")
1203 }
1204 return _64bit<<31 | 0b11011<<24 | op31<<21 | rm<<16 | oO<<15 | ra<<10 | rn<<5 | rd
1205 }
1206
1207 // encodeBitRR encodes as Data-processing (1 source) in
1208 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en
1209 func encodeBitRR(op bitOp, rd, rn, _64bit uint32) uint32 {
1210 var opcode2, opcode uint32
1211 switch op {
1212 case bitOpRbit:
1213 opcode2, opcode = 0b00000, 0b000000
1214 case bitOpClz:
1215 opcode2, opcode = 0b00000, 0b000100
1216 default:
1217 panic("TODO/BUG")
1218 }
1219 return _64bit<<31 | 0b1_0_11010110<<21 | opcode2<<15 | opcode<<10 | rn<<5 | rd
1220 }
1221
1222 func encodeAsMov32(rn, rd uint32) uint32 {
1223 // This is an alias of ORR (shifted register):
1224 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register--
1225 return encodeLogicalShiftedRegister(0b001, 0, rn, 0, regNumberInEncoding[xzr], rd)
1226 }
1227
1228 // encodeExtend encodes extension instructions.
1229 func encodeExtend(signed bool, from, to byte, rd, rn uint32) uint32 {
1230 // UTXB: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTB--Unsigned-Extend-Byte--an-alias-of-UBFM-?lang=en
1231 // UTXH: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTH--Unsigned-Extend-Halfword--an-alias-of-UBFM-?lang=en
1232 // STXB: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTB--Signed-Extend-Byte--an-alias-of-SBFM-
1233 // STXH: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTH--Sign-Extend-Halfword--an-alias-of-SBFM-
1234 // STXW: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTW--Sign-Extend-Word--an-alias-of-SBFM-
1235 var _31to10 uint32
1236 switch {
1237 case !signed && from == 8 && to == 32:
1238 // 32-bit UXTB
1239 _31to10 = 0b0101001100000000000111
1240 case !signed && from == 16 && to == 32:
1241 // 32-bit UXTH
1242 _31to10 = 0b0101001100000000001111
1243 case !signed && from == 8 && to == 64:
1244 // 64-bit UXTB
1245 _31to10 = 0b0101001100000000000111
1246 case !signed && from == 16 && to == 64:
1247 // 64-bit UXTH
1248 _31to10 = 0b0101001100000000001111
1249 case !signed && from == 32 && to == 64:
1250 return encodeAsMov32(rn, rd)
1251 case signed && from == 8 && to == 32:
1252 // 32-bit SXTB
1253 _31to10 = 0b0001001100000000000111
1254 case signed && from == 16 && to == 32:
1255 // 32-bit SXTH
1256 _31to10 = 0b0001001100000000001111
1257 case signed && from == 8 && to == 64:
1258 // 64-bit SXTB
1259 _31to10 = 0b1001001101000000000111
1260 case signed && from == 16 && to == 64:
1261 // 64-bit SXTH
1262 _31to10 = 0b1001001101000000001111
1263 case signed && from == 32 && to == 64:
1264 // SXTW
1265 _31to10 = 0b1001001101000000011111
1266 default:
1267 panic("BUG")
1268 }
1269 return _31to10<<10 | rn<<5 | rd
1270 }
1271
1272 func encodeLoadOrStore(kind instructionKind, rt uint32, amode addressMode) uint32 {
1273 var _22to31 uint32
1274 var bits int64
1275 switch kind {
1276 case uLoad8:
1277 _22to31 = 0b0011100001
1278 bits = 8
1279 case sLoad8:
1280 _22to31 = 0b0011100010
1281 bits = 8
1282 case uLoad16:
1283 _22to31 = 0b0111100001
1284 bits = 16
1285 case sLoad16:
1286 _22to31 = 0b0111100010
1287 bits = 16
1288 case uLoad32:
1289 _22to31 = 0b1011100001
1290 bits = 32
1291 case sLoad32:
1292 _22to31 = 0b1011100010
1293 bits = 32
1294 case uLoad64:
1295 _22to31 = 0b1111100001
1296 bits = 64
1297 case fpuLoad32:
1298 _22to31 = 0b1011110001
1299 bits = 32
1300 case fpuLoad64:
1301 _22to31 = 0b1111110001
1302 bits = 64
1303 case fpuLoad128:
1304 _22to31 = 0b0011110011
1305 bits = 128
1306 case store8:
1307 _22to31 = 0b0011100000
1308 bits = 8
1309 case store16:
1310 _22to31 = 0b0111100000
1311 bits = 16
1312 case store32:
1313 _22to31 = 0b1011100000
1314 bits = 32
1315 case store64:
1316 _22to31 = 0b1111100000
1317 bits = 64
1318 case fpuStore32:
1319 _22to31 = 0b1011110000
1320 bits = 32
1321 case fpuStore64:
1322 _22to31 = 0b1111110000
1323 bits = 64
1324 case fpuStore128:
1325 _22to31 = 0b0011110010
1326 bits = 128
1327 default:
1328 panic("BUG")
1329 }
1330
1331 switch amode.kind {
1332 case addressModeKindRegScaledExtended:
1333 return encodeLoadOrStoreExtended(_22to31,
1334 regNumberInEncoding[amode.rn.RealReg()],
1335 regNumberInEncoding[amode.rm.RealReg()],
1336 rt, true, amode.extOp)
1337 case addressModeKindRegScaled:
1338 return encodeLoadOrStoreExtended(_22to31,
1339 regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()],
1340 rt, true, extendOpNone)
1341 case addressModeKindRegExtended:
1342 return encodeLoadOrStoreExtended(_22to31,
1343 regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()],
1344 rt, false, amode.extOp)
1345 case addressModeKindRegReg:
1346 return encodeLoadOrStoreExtended(_22to31,
1347 regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()],
1348 rt, false, extendOpNone)
1349 case addressModeKindRegSignedImm9:
1350 // e.g. https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDUR--Load-Register--unscaled--
1351 return encodeLoadOrStoreSIMM9(_22to31, 0b00 /* unscaled */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm)
1352 case addressModeKindPostIndex:
1353 return encodeLoadOrStoreSIMM9(_22to31, 0b01 /* post index */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm)
1354 case addressModeKindPreIndex:
1355 return encodeLoadOrStoreSIMM9(_22to31, 0b11 /* pre index */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm)
1356 case addressModeKindRegUnsignedImm12:
1357 // "unsigned immediate" in https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en
1358 rn := regNumberInEncoding[amode.rn.RealReg()]
1359 imm := amode.imm
1360 div := bits / 8
1361 if imm != 0 && !offsetFitsInAddressModeKindRegUnsignedImm12(byte(bits), imm) {
1362 panic("BUG")
1363 }
1364 imm /= div
1365 return _22to31<<22 | 0b1<<24 | uint32(imm&0b111111111111)<<10 | rn<<5 | rt
1366 default:
1367 panic("BUG")
1368 }
1369 }
1370
1371 // encodeVecLoad1R encodes as Load one single-element structure and Replicate to all lanes (of one register) in
1372 // https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/LD1R--Load-one-single-element-structure-and-Replicate-to-all-lanes--of-one-register--?lang=en#sa_imm
1373 func encodeVecLoad1R(rt, rn uint32, arr vecArrangement) uint32 {
1374 size, q := arrToSizeQEncoded(arr)
1375 return q<<30 | 0b001101010000001100<<12 | size<<10 | rn<<5 | rt
1376 }
1377
1378 // encodeAluBitmaskImmediate encodes as Logical (immediate) in
1379 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en
1380 func encodeAluBitmaskImmediate(op aluOp, rd, rn uint32, imm uint64, _64bit bool) uint32 {
1381 var _31to23 uint32
1382 switch op {
1383 case aluOpAnd:
1384 _31to23 = 0b00_100100
1385 case aluOpOrr:
1386 _31to23 = 0b01_100100
1387 case aluOpEor:
1388 _31to23 = 0b10_100100
1389 case aluOpAnds:
1390 _31to23 = 0b11_100100
1391 default:
1392 panic("BUG")
1393 }
1394 if _64bit {
1395 _31to23 |= 0b1 << 8
1396 }
1397 immr, imms, N := bitmaskImmediate(imm, _64bit)
1398 return _31to23<<23 | uint32(N)<<22 | uint32(immr)<<16 | uint32(imms)<<10 | rn<<5 | rd
1399 }
1400
1401 func bitmaskImmediate(c uint64, is64bit bool) (immr, imms, N byte) {
1402 var size uint32
1403 switch {
1404 case c != c>>32|c<<32:
1405 size = 64
1406 case c != c>>16|c<<48:
1407 size = 32
1408 c = uint64(int32(c))
1409 case c != c>>8|c<<56:
1410 size = 16
1411 c = uint64(int16(c))
1412 case c != c>>4|c<<60:
1413 size = 8
1414 c = uint64(int8(c))
1415 case c != c>>2|c<<62:
1416 size = 4
1417 c = uint64(int64(c<<60) >> 60)
1418 default:
1419 size = 2
1420 c = uint64(int64(c<<62) >> 62)
1421 }
1422
1423 neg := false
1424 if int64(c) < 0 {
1425 c = ^c
1426 neg = true
1427 }
1428
1429 onesSize, nonZeroPos := getOnesSequenceSize(c)
1430 if neg {
1431 nonZeroPos = onesSize + nonZeroPos
1432 onesSize = size - onesSize
1433 }
1434
1435 var mode byte = 32
1436 if is64bit && size == 64 {
1437 N, mode = 0b1, 64
1438 }
1439
1440 immr = byte((size - nonZeroPos) & (size - 1) & uint32(mode-1))
1441 imms = byte((onesSize - 1) | 63&^(size<<1-1))
1442 return
1443 }
1444
1445 func getOnesSequenceSize(x uint64) (size, nonZeroPos uint32) {
1446 // Take 0b00111000 for example:
1447 y := getLowestBit(x) // = 0b0000100
1448 nonZeroPos = setBitPos(y) // = 2
1449 size = setBitPos(x+y) - nonZeroPos // = setBitPos(0b0100000) - 2 = 5 - 2 = 3
1450 return
1451 }
1452
1453 func setBitPos(x uint64) (ret uint32) {
1454 for ; ; ret++ {
1455 if x == 0b1 {
1456 break
1457 }
1458 x = x >> 1
1459 }
1460 return
1461 }
1462
1463 // encodeLoadOrStoreExtended encodes store/load instruction as "extended register offset" in Load/store register (register offset):
1464 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en
1465 func encodeLoadOrStoreExtended(_22to32 uint32, rn, rm, rt uint32, scaled bool, extOp extendOp) uint32 {
1466 var option uint32
1467 switch extOp {
1468 case extendOpUXTW:
1469 option = 0b010
1470 case extendOpSXTW:
1471 option = 0b110
1472 case extendOpNone:
1473 option = 0b111
1474 default:
1475 panic("BUG")
1476 }
1477 var s uint32
1478 if scaled {
1479 s = 0b1
1480 }
1481 return _22to32<<22 | 0b1<<21 | rm<<16 | option<<13 | s<<12 | 0b10<<10 | rn<<5 | rt
1482 }
1483
1484 // encodeLoadOrStoreSIMM9 encodes store/load instruction as one of post-index, pre-index or unscaled immediate as in
1485 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en
1486 func encodeLoadOrStoreSIMM9(_22to32, _1011 uint32, rn, rt uint32, imm9 int64) uint32 {
1487 return _22to32<<22 | (uint32(imm9)&0b111111111)<<12 | _1011<<10 | rn<<5 | rt
1488 }
1489
1490 // encodeFpuRRR encodes as single or double precision (depending on `_64bit`) of Floating-point data-processing (2 source) in
1491 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
1492 func encodeFpuRRR(op fpuBinOp, rd, rn, rm uint32, _64bit bool) (ret uint32) {
1493 // https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/ADD--vector--Add-vectors--scalar--floating-point-and-integer-
1494 var opcode uint32
1495 switch op {
1496 case fpuBinOpAdd:
1497 opcode = 0b0010
1498 case fpuBinOpSub:
1499 opcode = 0b0011
1500 case fpuBinOpMul:
1501 opcode = 0b0000
1502 case fpuBinOpDiv:
1503 opcode = 0b0001
1504 case fpuBinOpMax:
1505 opcode = 0b0100
1506 case fpuBinOpMin:
1507 opcode = 0b0101
1508 default:
1509 panic("BUG")
1510 }
1511 var ptype uint32
1512 if _64bit {
1513 ptype = 0b01
1514 }
1515 return 0b1111<<25 | ptype<<22 | 0b1<<21 | rm<<16 | opcode<<12 | 0b1<<11 | rn<<5 | rd
1516 }
1517
1518 // encodeAluRRImm12 encodes as Add/subtract (immediate) in
1519 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en
1520 func encodeAluRRImm12(op aluOp, rd, rn uint32, imm12 uint16, shiftBit byte, _64bit bool) uint32 {
1521 var _31to24 uint32
1522 switch op {
1523 case aluOpAdd:
1524 _31to24 = 0b00_10001
1525 case aluOpAddS:
1526 _31to24 = 0b01_10001
1527 case aluOpSub:
1528 _31to24 = 0b10_10001
1529 case aluOpSubS:
1530 _31to24 = 0b11_10001
1531 default:
1532 panic("BUG")
1533 }
1534 if _64bit {
1535 _31to24 |= 0b1 << 7
1536 }
1537 return _31to24<<24 | uint32(shiftBit)<<22 | uint32(imm12&0b111111111111)<<10 | rn<<5 | rd
1538 }
1539
1540 // encodeAluRRR encodes as Data Processing (shifted register), depending on aluOp.
1541 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#addsub_shift
1542 func encodeAluRRRShift(op aluOp, rd, rn, rm, amount uint32, shiftOp shiftOp, _64bit bool) uint32 {
1543 var _31to24 uint32
1544 var opc, n uint32
1545 switch op {
1546 case aluOpAdd:
1547 _31to24 = 0b00001011
1548 case aluOpAddS:
1549 _31to24 = 0b00101011
1550 case aluOpSub:
1551 _31to24 = 0b01001011
1552 case aluOpSubS:
1553 _31to24 = 0b01101011
1554 case aluOpAnd, aluOpOrr, aluOpEor, aluOpAnds:
1555 // "Logical (shifted register)".
1556 switch op {
1557 case aluOpAnd:
1558 // all zeros
1559 case aluOpOrr:
1560 opc = 0b01
1561 case aluOpEor:
1562 opc = 0b10
1563 case aluOpAnds:
1564 opc = 0b11
1565 }
1566 _31to24 = 0b000_01010
1567 default:
1568 panic(op.String())
1569 }
1570
1571 if _64bit {
1572 _31to24 |= 0b1 << 7
1573 }
1574
1575 var shift uint32
1576 switch shiftOp {
1577 case shiftOpLSL:
1578 shift = 0b00
1579 case shiftOpLSR:
1580 shift = 0b01
1581 case shiftOpASR:
1582 shift = 0b10
1583 default:
1584 panic(shiftOp.String())
1585 }
1586 return opc<<29 | n<<21 | _31to24<<24 | shift<<22 | rm<<16 | (amount << 10) | (rn << 5) | rd
1587 }
1588
1589 // "Add/subtract (extended register)" in
1590 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#addsub_ext
1591 func encodeAluRRRExtend(ao aluOp, rd, rn, rm uint32, extOp extendOp, to byte) uint32 {
1592 var s, op uint32
1593 switch ao {
1594 case aluOpAdd:
1595 op = 0b0
1596 case aluOpAddS:
1597 op, s = 0b0, 0b1
1598 case aluOpSub:
1599 op = 0b1
1600 case aluOpSubS:
1601 op, s = 0b1, 0b1
1602 default:
1603 panic("BUG: extended register operand can be used only for add/sub")
1604 }
1605
1606 var sf uint32
1607 if to == 64 {
1608 sf = 0b1
1609 }
1610
1611 var option uint32
1612 switch extOp {
1613 case extendOpUXTB:
1614 option = 0b000
1615 case extendOpUXTH:
1616 option = 0b001
1617 case extendOpUXTW:
1618 option = 0b010
1619 case extendOpSXTB:
1620 option = 0b100
1621 case extendOpSXTH:
1622 option = 0b101
1623 case extendOpSXTW:
1624 option = 0b110
1625 case extendOpSXTX, extendOpUXTX:
1626 panic(fmt.Sprintf("%s is essentially noop, and should be handled much earlier than encoding", extOp.String()))
1627 }
1628 return sf<<31 | op<<30 | s<<29 | 0b1011001<<21 | rm<<16 | option<<13 | rn<<5 | rd
1629 }
1630
1631 // encodeAluRRR encodes as Data Processing (register), depending on aluOp.
1632 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en
1633 func encodeAluRRR(op aluOp, rd, rn, rm uint32, _64bit, isRnSp bool) uint32 {
1634 var _31to21, _15to10 uint32
1635 switch op {
1636 case aluOpAdd:
1637 if isRnSp {
1638 // "Extended register" with UXTW.
1639 _31to21 = 0b00001011_001
1640 _15to10 = 0b011000
1641 } else {
1642 // "Shifted register" with shift = 0
1643 _31to21 = 0b00001011_000
1644 }
1645 case aluOpAddS:
1646 if isRnSp {
1647 panic("TODO")
1648 }
1649 // "Shifted register" with shift = 0
1650 _31to21 = 0b00101011_000
1651 case aluOpSub:
1652 if isRnSp {
1653 // "Extended register" with UXTW.
1654 _31to21 = 0b01001011_001
1655 _15to10 = 0b011000
1656 } else {
1657 // "Shifted register" with shift = 0
1658 _31to21 = 0b01001011_000
1659 }
1660 case aluOpSubS:
1661 if isRnSp {
1662 panic("TODO")
1663 }
1664 // "Shifted register" with shift = 0
1665 _31to21 = 0b01101011_000
1666 case aluOpAnd, aluOpOrr, aluOpOrn, aluOpEor, aluOpAnds:
1667 // "Logical (shifted register)".
1668 var opc, n uint32
1669 switch op {
1670 case aluOpAnd:
1671 // all zeros
1672 case aluOpOrr:
1673 opc = 0b01
1674 case aluOpOrn:
1675 opc = 0b01
1676 n = 1
1677 case aluOpEor:
1678 opc = 0b10
1679 case aluOpAnds:
1680 opc = 0b11
1681 }
1682 _31to21 = 0b000_01010_000 | opc<<8 | n
1683 case aluOpLsl, aluOpAsr, aluOpLsr, aluOpRotR:
1684 // "Data-processing (2 source)".
1685 _31to21 = 0b00011010_110
1686 switch op {
1687 case aluOpLsl:
1688 _15to10 = 0b001000
1689 case aluOpLsr:
1690 _15to10 = 0b001001
1691 case aluOpAsr:
1692 _15to10 = 0b001010
1693 case aluOpRotR:
1694 _15to10 = 0b001011
1695 }
1696 case aluOpSDiv:
1697 // "Data-processing (2 source)".
1698 _31to21 = 0b11010110
1699 _15to10 = 0b000011
1700 case aluOpUDiv:
1701 // "Data-processing (2 source)".
1702 _31to21 = 0b11010110
1703 _15to10 = 0b000010
1704 default:
1705 panic(op.String())
1706 }
1707 if _64bit {
1708 _31to21 |= 0b1 << 10
1709 }
1710 return _31to21<<21 | rm<<16 | (_15to10 << 10) | (rn << 5) | rd
1711 }
1712
1713 // encodeLogicalShiftedRegister encodes as Logical (shifted register) in
1714 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en
1715 func encodeLogicalShiftedRegister(sf_opc uint32, shift_N uint32, rm uint32, imm6 uint32, rn, rd uint32) (ret uint32) {
1716 ret = sf_opc << 29
1717 ret |= 0b01010 << 24
1718 ret |= shift_N << 21
1719 ret |= rm << 16
1720 ret |= imm6 << 10
1721 ret |= rn << 5
1722 ret |= rd
1723 return
1724 }
1725
1726 // encodeAddSubtractImmediate encodes as Add/subtract (immediate) in
1727 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en
1728 func encodeAddSubtractImmediate(sf_op_s uint32, sh uint32, imm12 uint32, rn, rd uint32) (ret uint32) {
1729 ret = sf_op_s << 29
1730 ret |= 0b100010 << 23
1731 ret |= sh << 22
1732 ret |= imm12 << 10
1733 ret |= rn << 5
1734 ret |= rd
1735 return
1736 }
1737
1738 // encodePreOrPostIndexLoadStorePair64 encodes as Load/store pair (pre/post-indexed) in
1739 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers-
1740 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers-
1741 func encodePreOrPostIndexLoadStorePair64(pre bool, load bool, rn, rt, rt2 uint32, imm7 int64) (ret uint32) {
1742 if imm7%8 != 0 {
1743 panic("imm7 for pair load/store must be a multiple of 8")
1744 }
1745 imm7 /= 8
1746 ret = rt
1747 ret |= rn << 5
1748 ret |= rt2 << 10
1749 ret |= (uint32(imm7) & 0b1111111) << 15
1750 if load {
1751 ret |= 0b1 << 22
1752 }
1753 ret |= 0b101010001 << 23
1754 if pre {
1755 ret |= 0b1 << 24
1756 }
1757 return
1758 }
1759
1760 // encodeUnconditionalBranch encodes as B or BL instructions:
1761 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B--Branch-
1762 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/BL--Branch-with-Link-
1763 func encodeUnconditionalBranch(link bool, imm26 int64) (ret uint32) {
1764 if imm26%4 != 0 {
1765 panic("imm26 for branch must be a multiple of 4")
1766 }
1767 imm26 /= 4
1768 ret = uint32(imm26 & 0b11_11111111_11111111_11111111)
1769 ret |= 0b101 << 26
1770 if link {
1771 ret |= 0b1 << 31
1772 }
1773 return
1774 }
1775
1776 // encodeCBZCBNZ encodes as either CBZ or CBNZ:
1777 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CBZ--Compare-and-Branch-on-Zero-
1778 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CBNZ--Compare-and-Branch-on-Nonzero-
1779 func encodeCBZCBNZ(rt uint32, nz bool, imm19 uint32, _64bit bool) (ret uint32) {
1780 ret = rt
1781 ret |= imm19 << 5
1782 if nz {
1783 ret |= 1 << 24
1784 }
1785 ret |= 0b11010 << 25
1786 if _64bit {
1787 ret |= 1 << 31
1788 }
1789 return
1790 }
1791
1792 // encodeMoveWideImmediate encodes as either MOVZ, MOVN or MOVK, as Move wide (immediate) in
1793 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en
1794 //
1795 // "shift" must have been divided by 16 at this point.
1796 func encodeMoveWideImmediate(opc uint32, rd uint32, imm uint64, shift, _64bit uint32) (ret uint32) {
1797 ret = rd
1798 ret |= uint32(imm&0xffff) << 5
1799 ret |= (shift) << 21
1800 ret |= 0b100101 << 23
1801 ret |= opc << 29
1802 ret |= _64bit << 31
1803 return
1804 }
1805
1806 // encodeAluRRImm encodes as "Bitfield" in
1807 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en#log_imm
1808 func encodeAluRRImm(op aluOp, rd, rn, amount, _64bit uint32) uint32 {
1809 var opc uint32
1810 var immr, imms uint32
1811 switch op {
1812 case aluOpLsl:
1813 // LSL (immediate) is an alias for UBFM.
1814 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/UBFM--Unsigned-Bitfield-Move-?lang=en
1815 opc = 0b10
1816 if amount == 0 {
1817 // This can be encoded as NOP, but we don't do it for consistency: lsr xn, xm, #0
1818 immr = 0
1819 if _64bit == 1 {
1820 imms = 0b111111
1821 } else {
1822 imms = 0b11111
1823 }
1824 } else {
1825 if _64bit == 1 {
1826 immr = 64 - amount
1827 } else {
1828 immr = (32 - amount) & 0b11111
1829 }
1830 imms = immr - 1
1831 }
1832 case aluOpLsr:
1833 // LSR (immediate) is an alias for UBFM.
1834 // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LSR--immediate---Logical-Shift-Right--immediate---an-alias-of-UBFM-?lang=en
1835 opc = 0b10
1836 imms, immr = 0b011111|_64bit<<5, amount
1837 case aluOpAsr:
1838 // ASR (immediate) is an alias for SBFM.
1839 // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SBFM--Signed-Bitfield-Move-?lang=en
1840 opc = 0b00
1841 imms, immr = 0b011111|_64bit<<5, amount
1842 default:
1843 panic(op.String())
1844 }
1845 return _64bit<<31 | opc<<29 | 0b100110<<23 | _64bit<<22 | immr<<16 | imms<<10 | rn<<5 | rd
1846 }
1847
1848 // encodeVecLanes encodes as Data Processing (Advanced SIMD across lanes) depending on vecOp in
1849 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
1850 func encodeVecLanes(op vecOp, rd uint32, rn uint32, arr vecArrangement) uint32 {
1851 var u, q, size, opcode uint32
1852 switch arr {
1853 case vecArrangement8B:
1854 q, size = 0b0, 0b00
1855 case vecArrangement16B:
1856 q, size = 0b1, 0b00
1857 case vecArrangement4H:
1858 q, size = 0, 0b01
1859 case vecArrangement8H:
1860 q, size = 1, 0b01
1861 case vecArrangement4S:
1862 q, size = 1, 0b10
1863 default:
1864 panic("unsupported arrangement: " + arr.String())
1865 }
1866 switch op {
1867 case vecOpUaddlv:
1868 u, opcode = 1, 0b00011
1869 case vecOpUminv:
1870 u, opcode = 1, 0b11010
1871 case vecOpAddv:
1872 u, opcode = 0, 0b11011
1873 default:
1874 panic("unsupported or illegal vecOp: " + op.String())
1875 }
1876 return q<<30 | u<<29 | 0b1110<<24 | size<<22 | 0b11000<<17 | opcode<<12 | 0b10<<10 | rn<<5 | rd
1877 }
1878
1879 // encodeVecLanes encodes as Data Processing (Advanced SIMD scalar shift by immediate) depending on vecOp in
1880 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en
1881 func encodeVecShiftImm(op vecOp, rd uint32, rn, amount uint32, arr vecArrangement) uint32 {
1882 var u, q, immh, immb, opcode uint32
1883 switch op {
1884 case vecOpSshll:
1885 u, opcode = 0b0, 0b10100
1886 case vecOpUshll:
1887 u, opcode = 0b1, 0b10100
1888 case vecOpSshr:
1889 u, opcode = 0, 0b00000
1890 default:
1891 panic("unsupported or illegal vecOp: " + op.String())
1892 }
1893 switch arr {
1894 case vecArrangement16B:
1895 q = 0b1
1896 fallthrough
1897 case vecArrangement8B:
1898 immh = 0b0001
1899 immb = 8 - uint32(amount&0b111)
1900 case vecArrangement8H:
1901 q = 0b1
1902 fallthrough
1903 case vecArrangement4H:
1904 v := 16 - uint32(amount&0b1111)
1905 immb = v & 0b111
1906 immh = 0b0010 | (v >> 3)
1907 case vecArrangement4S:
1908 q = 0b1
1909 fallthrough
1910 case vecArrangement2S:
1911 v := 32 - uint32(amount&0b11111)
1912 immb = v & 0b111
1913 immh = 0b0100 | (v >> 3)
1914 case vecArrangement2D:
1915 q = 0b1
1916 v := 64 - uint32(amount&0b111111)
1917 immb = v & 0b111
1918 immh = 0b1000 | (v >> 3)
1919 default:
1920 panic("unsupported arrangement: " + arr.String())
1921 }
1922 return q<<30 | u<<29 | 0b011110<<23 | immh<<19 | immb<<16 | 0b000001<<10 | opcode<<11 | 0b1<<10 | rn<<5 | rd
1923 }
1924
1925 // encodeVecTbl encodes as Data Processing (Advanced SIMD table lookup) in
1926 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp
1927 //
1928 // Note: tblOp may encode tbl1, tbl2... in the future. Currently, it is ignored.
1929 func encodeVecTbl(nregs, rd, rn, rm uint32, arr vecArrangement) uint32 {
1930 var q, op2, len, op uint32
1931
1932 switch nregs {
1933 case 1:
1934 // tbl: single-register
1935 len = 0b00
1936 case 2:
1937 // tbl2: 2-register table
1938 len = 0b01
1939 default:
1940 panic(fmt.Sprintf("unsupported number or registers %d", nregs))
1941 }
1942 switch arr {
1943 case vecArrangement8B:
1944 q = 0b0
1945 case vecArrangement16B:
1946 q = 0b1
1947 default:
1948 panic("unsupported arrangement: " + arr.String())
1949 }
1950
1951 return q<<30 | 0b001110<<24 | op2<<22 | rm<<16 | len<<13 | op<<12 | rn<<5 | rd
1952 }
1953
1954 // encodeVecMisc encodes as Data Processing (Advanced SIMD two-register miscellaneous) depending on vecOp in
1955 // https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp
1956 func encodeAdvancedSIMDTwoMisc(op vecOp, rd, rn uint32, arr vecArrangement) uint32 {
1957 var q, u, size, opcode uint32
1958 switch op {
1959 case vecOpCnt:
1960 opcode = 0b00101
1961 switch arr {
1962 case vecArrangement8B:
1963 q, size = 0b0, 0b00
1964 case vecArrangement16B:
1965 q, size = 0b1, 0b00
1966 default:
1967 panic("unsupported arrangement: " + arr.String())
1968 }
1969 case vecOpCmeq0:
1970 if arr == vecArrangement1D {
1971 panic("unsupported arrangement: " + arr.String())
1972 }
1973 opcode = 0b01001
1974 size, q = arrToSizeQEncoded(arr)
1975 case vecOpNot:
1976 u = 1
1977 opcode = 0b00101
1978 switch arr {
1979 case vecArrangement8B:
1980 q, size = 0b0, 0b00
1981 case vecArrangement16B:
1982 q, size = 0b1, 0b00
1983 default:
1984 panic("unsupported arrangement: " + arr.String())
1985 }
1986 case vecOpAbs:
1987 if arr == vecArrangement1D {
1988 panic("unsupported arrangement: " + arr.String())
1989 }
1990 opcode = 0b01011
1991 u = 0b0
1992 size, q = arrToSizeQEncoded(arr)
1993 case vecOpNeg:
1994 if arr == vecArrangement1D {
1995 panic("unsupported arrangement: " + arr.String())
1996 }
1997 opcode = 0b01011
1998 u = 0b1
1999 size, q = arrToSizeQEncoded(arr)
2000 case vecOpFabs:
2001 if arr < vecArrangement2S || arr == vecArrangement1D {
2002 panic("unsupported arrangement: " + arr.String())
2003 }
2004 opcode = 0b01111
2005 u = 0b0
2006 size, q = arrToSizeQEncoded(arr)
2007 case vecOpFneg:
2008 if arr < vecArrangement2S || arr == vecArrangement1D {
2009 panic("unsupported arrangement: " + arr.String())
2010 }
2011 opcode = 0b01111
2012 u = 0b1
2013 size, q = arrToSizeQEncoded(arr)
2014 case vecOpFrintm:
2015 u = 0b0
2016 opcode = 0b11001
2017 switch arr {
2018 case vecArrangement2S:
2019 q, size = 0b0, 0b00
2020 case vecArrangement4S:
2021 q, size = 0b1, 0b00
2022 case vecArrangement2D:
2023 q, size = 0b1, 0b01
2024 default:
2025 panic("unsupported arrangement: " + arr.String())
2026 }
2027 case vecOpFrintn:
2028 u = 0b0
2029 opcode = 0b11000
2030 switch arr {
2031 case vecArrangement2S:
2032 q, size = 0b0, 0b00
2033 case vecArrangement4S:
2034 q, size = 0b1, 0b00
2035 case vecArrangement2D:
2036 q, size = 0b1, 0b01
2037 default:
2038 panic("unsupported arrangement: " + arr.String())
2039 }
2040 case vecOpFrintp:
2041 u = 0b0
2042 opcode = 0b11000
2043 if arr < vecArrangement2S || arr == vecArrangement1D {
2044 panic("unsupported arrangement: " + arr.String())
2045 }
2046 size, q = arrToSizeQEncoded(arr)
2047 case vecOpFrintz:
2048 u = 0b0
2049 opcode = 0b11001
2050 if arr < vecArrangement2S || arr == vecArrangement1D {
2051 panic("unsupported arrangement: " + arr.String())
2052 }
2053 size, q = arrToSizeQEncoded(arr)
2054 case vecOpFsqrt:
2055 if arr < vecArrangement2S || arr == vecArrangement1D {
2056 panic("unsupported arrangement: " + arr.String())
2057 }
2058 opcode = 0b11111
2059 u = 0b1
2060 size, q = arrToSizeQEncoded(arr)
2061 case vecOpFcvtl:
2062 opcode = 0b10111
2063 u = 0b0
2064 switch arr {
2065 case vecArrangement2S:
2066 size, q = 0b01, 0b0
2067 case vecArrangement4H:
2068 size, q = 0b00, 0b0
2069 default:
2070 panic("unsupported arrangement: " + arr.String())
2071 }
2072 case vecOpFcvtn:
2073 opcode = 0b10110
2074 u = 0b0
2075 switch arr {
2076 case vecArrangement2S:
2077 size, q = 0b01, 0b0
2078 case vecArrangement4H:
2079 size, q = 0b00, 0b0
2080 default:
2081 panic("unsupported arrangement: " + arr.String())
2082 }
2083 case vecOpFcvtzs:
2084 opcode = 0b11011
2085 u = 0b0
2086 switch arr {
2087 case vecArrangement2S:
2088 q, size = 0b0, 0b10
2089 case vecArrangement4S:
2090 q, size = 0b1, 0b10
2091 case vecArrangement2D:
2092 q, size = 0b1, 0b11
2093 default:
2094 panic("unsupported arrangement: " + arr.String())
2095 }
2096 case vecOpFcvtzu:
2097 opcode = 0b11011
2098 u = 0b1
2099 switch arr {
2100 case vecArrangement2S:
2101 q, size = 0b0, 0b10
2102 case vecArrangement4S:
2103 q, size = 0b1, 0b10
2104 case vecArrangement2D:
2105 q, size = 0b1, 0b11
2106 default:
2107 panic("unsupported arrangement: " + arr.String())
2108 }
2109 case vecOpScvtf:
2110 opcode = 0b11101
2111 u = 0b0
2112 switch arr {
2113 case vecArrangement4S:
2114 q, size = 0b1, 0b00
2115 case vecArrangement2S:
2116 q, size = 0b0, 0b00
2117 case vecArrangement2D:
2118 q, size = 0b1, 0b01
2119 default:
2120 panic("unsupported arrangement: " + arr.String())
2121 }
2122 case vecOpUcvtf:
2123 opcode = 0b11101
2124 u = 0b1
2125 switch arr {
2126 case vecArrangement4S:
2127 q, size = 0b1, 0b00
2128 case vecArrangement2S:
2129 q, size = 0b0, 0b00
2130 case vecArrangement2D:
2131 q, size = 0b1, 0b01
2132 default:
2133 panic("unsupported arrangement: " + arr.String())
2134 }
2135 case vecOpSqxtn:
2136 // When q == 1 it encodes sqxtn2 (operates on upper 64 bits).
2137 opcode = 0b10100
2138 u = 0b0
2139 if arr > vecArrangement4S {
2140 panic("unsupported arrangement: " + arr.String())
2141 }
2142 size, q = arrToSizeQEncoded(arr)
2143 case vecOpUqxtn:
2144 // When q == 1 it encodes uqxtn2 (operates on upper 64 bits).
2145 opcode = 0b10100
2146 u = 0b1
2147 if arr > vecArrangement4S {
2148 panic("unsupported arrangement: " + arr.String())
2149 }
2150 size, q = arrToSizeQEncoded(arr)
2151 case vecOpSqxtun:
2152 // When q == 1 it encodes sqxtun2 (operates on upper 64 bits).
2153 opcode = 0b10010 // 0b10100
2154 u = 0b1
2155 if arr > vecArrangement4S {
2156 panic("unsupported arrangement: " + arr.String())
2157 }
2158 size, q = arrToSizeQEncoded(arr)
2159 case vecOpRev64:
2160 opcode = 0b00000
2161 size, q = arrToSizeQEncoded(arr)
2162 case vecOpXtn:
2163 u = 0b0
2164 opcode = 0b10010
2165 size, q = arrToSizeQEncoded(arr)
2166 case vecOpShll:
2167 u = 0b1
2168 opcode = 0b10011
2169 switch arr {
2170 case vecArrangement8B:
2171 q, size = 0b0, 0b00
2172 case vecArrangement4H:
2173 q, size = 0b0, 0b01
2174 case vecArrangement2S:
2175 q, size = 0b0, 0b10
2176 default:
2177 panic("unsupported arrangement: " + arr.String())
2178 }
2179 default:
2180 panic("unsupported or illegal vecOp: " + op.String())
2181 }
2182 return q<<30 | u<<29 | 0b01110<<24 | size<<22 | 0b10000<<17 | opcode<<12 | 0b10<<10 | rn<<5 | rd
2183 }
2184
2185 // brTableSequenceOffsetTableBegin is the offset inside the brTableSequence where the table begins after 4 instructions
2186 const brTableSequenceOffsetTableBegin = 16
2187
2188 func encodeBrTableSequence(c backend.Compiler, index regalloc.VReg, targets []uint32) {
2189 tmpRegNumber := regNumberInEncoding[tmp]
2190 indexNumber := regNumberInEncoding[index.RealReg()]
2191
2192 // adr tmpReg, PC+16 (PC+16 is the address of the first label offset)
2193 // ldrsw index, [tmpReg, index, UXTW 2] ;; index = int64(*(tmpReg + index*8))
2194 // add tmpReg, tmpReg, index
2195 // br tmpReg
2196 // [offset_to_l1, offset_to_l2, ..., offset_to_lN]
2197 c.Emit4Bytes(encodeAdr(tmpRegNumber, 16))
2198 c.Emit4Bytes(encodeLoadOrStore(sLoad32, indexNumber,
2199 addressMode{kind: addressModeKindRegScaledExtended, rn: tmpRegVReg, rm: index, extOp: extendOpUXTW},
2200 ))
2201 c.Emit4Bytes(encodeAluRRR(aluOpAdd, tmpRegNumber, tmpRegNumber, indexNumber, true, false))
2202 c.Emit4Bytes(encodeUnconditionalBranchReg(tmpRegNumber, false))
2203
2204 // Offsets are resolved in ResolveRelativeAddress phase.
2205 for _, offset := range targets {
2206 if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable {
2207 // Inlined offset tables cannot be disassembled properly, so pad dummy instructions to make the debugging easier.
2208 c.Emit4Bytes(dummyInstruction)
2209 } else {
2210 c.Emit4Bytes(offset)
2211 }
2212 }
2213 }
2214
2215 // encodeExitSequence matches the implementation detail of functionABI.emitGoEntryPreamble.
2216 func encodeExitSequence(c backend.Compiler, ctxReg regalloc.VReg) {
2217 // Restore the FP, SP and LR, and return to the Go code:
2218 // ldr lr, [ctxReg, #GoReturnAddress]
2219 // ldr fp, [ctxReg, #OriginalFramePointer]
2220 // ldr tmp, [ctxReg, #OriginalStackPointer]
2221 // mov sp, tmp ;; sp cannot be str'ed directly.
2222 // ret ;; --> return to the Go code
2223
2224 var ctxEvicted bool
2225 if ctx := ctxReg.RealReg(); ctx == fp || ctx == lr {
2226 // In order to avoid overwriting the context register, we move ctxReg to tmp.
2227 c.Emit4Bytes(encodeMov64(regNumberInEncoding[tmp], regNumberInEncoding[ctx], false, false))
2228 ctxReg = tmpRegVReg
2229 ctxEvicted = true
2230 }
2231
2232 restoreLr := encodeLoadOrStore(
2233 uLoad64,
2234 regNumberInEncoding[lr],
2235 addressMode{
2236 kind: addressModeKindRegUnsignedImm12,
2237 rn: ctxReg,
2238 imm: wazevoapi.ExecutionContextOffsetGoReturnAddress.I64(),
2239 },
2240 )
2241
2242 restoreFp := encodeLoadOrStore(
2243 uLoad64,
2244 regNumberInEncoding[fp],
2245 addressMode{
2246 kind: addressModeKindRegUnsignedImm12,
2247 rn: ctxReg,
2248 imm: wazevoapi.ExecutionContextOffsetOriginalFramePointer.I64(),
2249 },
2250 )
2251
2252 restoreSpToTmp := encodeLoadOrStore(
2253 uLoad64,
2254 regNumberInEncoding[tmp],
2255 addressMode{
2256 kind: addressModeKindRegUnsignedImm12,
2257 rn: ctxReg,
2258 imm: wazevoapi.ExecutionContextOffsetOriginalStackPointer.I64(),
2259 },
2260 )
2261
2262 movTmpToSp := encodeAddSubtractImmediate(0b100, 0, 0,
2263 regNumberInEncoding[tmp], regNumberInEncoding[sp])
2264
2265 c.Emit4Bytes(restoreFp)
2266 c.Emit4Bytes(restoreLr)
2267 c.Emit4Bytes(restoreSpToTmp)
2268 c.Emit4Bytes(movTmpToSp)
2269 c.Emit4Bytes(encodeRet())
2270 if !ctxEvicted {
2271 // In order to have the fixed-length exit sequence, we need to padd the binary.
2272 // Since this will never be reached, we insert a dummy instruction.
2273 c.Emit4Bytes(dummyInstruction)
2274 }
2275 }
2276
2277 func encodeRet() uint32 {
2278 // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en
2279 return 0b1101011001011111<<16 | regNumberInEncoding[lr]<<5
2280 }
2281
2282 func encodeAtomicRmw(op atomicRmwOp, rs, rt, rn uint32, size uint32) uint32 {
2283 var _31to21, _15to10, sz uint32
2284
2285 switch size {
2286 case 8:
2287 sz = 0b11
2288 case 4:
2289 sz = 0b10
2290 case 2:
2291 sz = 0b01
2292 case 1:
2293 sz = 0b00
2294 }
2295
2296 _31to21 = 0b00111000_111 | sz<<9
2297
2298 switch op {
2299 case atomicRmwOpAdd:
2300 _15to10 = 0b000000
2301 case atomicRmwOpClr:
2302 _15to10 = 0b000100
2303 case atomicRmwOpSet:
2304 _15to10 = 0b001100
2305 case atomicRmwOpEor:
2306 _15to10 = 0b001000
2307 case atomicRmwOpSwp:
2308 _15to10 = 0b100000
2309 }
2310
2311 return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt
2312 }
2313
2314 func encodeAtomicCas(rs, rt, rn uint32, size uint32) uint32 {
2315 var _31to21, _15to10, sz uint32
2316
2317 switch size {
2318 case 8:
2319 sz = 0b11
2320 case 4:
2321 sz = 0b10
2322 case 2:
2323 sz = 0b01
2324 case 1:
2325 sz = 0b00
2326 }
2327
2328 _31to21 = 0b00001000_111 | sz<<9
2329 _15to10 = 0b111111
2330
2331 return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt
2332 }
2333
2334 func encodeAtomicLoadStore(rn, rt, size, l uint32) uint32 {
2335 var _31to21, _20to16, _15to10, sz uint32
2336
2337 switch size {
2338 case 8:
2339 sz = 0b11
2340 case 4:
2341 sz = 0b10
2342 case 2:
2343 sz = 0b01
2344 case 1:
2345 sz = 0b00
2346 }
2347
2348 _31to21 = 0b00001000_100 | sz<<9 | l<<1
2349 _20to16 = 0b11111
2350 _15to10 = 0b111111
2351
2352 return _31to21<<21 | _20to16<<16 | _15to10<<10 | rn<<5 | rt
2353 }
2354
2355 func encodeDMB() uint32 {
2356 return 0b11010101000000110011101110111111
2357 }