Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/instr.go

raw
1 package arm64
2
3 import (
4 "fmt"
5 "math"
6 "unsafe"
7
8 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
9 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
10 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
11 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
12 )
13
14 type (
15 // instruction represents either a real instruction in arm64, or the meta instructions
16 // that are convenient for code generation. For example, inline constants are also treated
17 // as instructions.
18 //
19 // Basically, each instruction knows how to get encoded in binaries. Hence, the final output of compilation
20 // can be considered equivalent to the sequence of such instructions.
21 //
22 // Each field is interpreted depending on the kind.
23 //
24 // TODO: optimize the layout later once the impl settles.
25 instruction struct {
26 prev, next *instruction
27 u1, u2 uint64
28 rd regalloc.VReg
29 rm, rn operand
30 kind instructionKind
31 addedBeforeRegAlloc bool
32 }
33
34 // instructionKind represents the kind of instruction.
35 // This controls how the instruction struct is interpreted.
36 instructionKind byte
37 )
38
39 // IsCall implements regalloc.Instr IsCall.
40 func (i *instruction) IsCall() bool {
41 return i.kind == call
42 }
43
44 // IsIndirectCall implements regalloc.Instr IsIndirectCall.
45 func (i *instruction) IsIndirectCall() bool {
46 return i.kind == callInd
47 }
48
49 // IsReturn implements regalloc.Instr IsReturn.
50 func (i *instruction) IsReturn() bool {
51 return i.kind == ret
52 }
53
54 type defKind byte
55
56 const (
57 defKindNone defKind = iota + 1
58 defKindRD
59 defKindCall
60 )
61
62 var defKinds = [numInstructionKinds]defKind{
63 adr: defKindRD,
64 aluRRR: defKindRD,
65 aluRRRR: defKindRD,
66 aluRRImm12: defKindRD,
67 aluRRBitmaskImm: defKindRD,
68 aluRRRShift: defKindRD,
69 aluRRImmShift: defKindRD,
70 aluRRRExtend: defKindRD,
71 bitRR: defKindRD,
72 movZ: defKindRD,
73 movK: defKindRD,
74 movN: defKindRD,
75 mov32: defKindRD,
76 mov64: defKindRD,
77 fpuMov64: defKindRD,
78 fpuMov128: defKindRD,
79 fpuRR: defKindRD,
80 fpuRRR: defKindRD,
81 nop0: defKindNone,
82 call: defKindCall,
83 callInd: defKindCall,
84 ret: defKindNone,
85 store8: defKindNone,
86 store16: defKindNone,
87 store32: defKindNone,
88 store64: defKindNone,
89 exitSequence: defKindNone,
90 condBr: defKindNone,
91 br: defKindNone,
92 brTableSequence: defKindNone,
93 cSet: defKindRD,
94 extend: defKindRD,
95 fpuCmp: defKindNone,
96 uLoad8: defKindRD,
97 uLoad16: defKindRD,
98 uLoad32: defKindRD,
99 sLoad8: defKindRD,
100 sLoad16: defKindRD,
101 sLoad32: defKindRD,
102 uLoad64: defKindRD,
103 fpuLoad32: defKindRD,
104 fpuLoad64: defKindRD,
105 fpuLoad128: defKindRD,
106 vecLoad1R: defKindRD,
107 loadFpuConst32: defKindRD,
108 loadFpuConst64: defKindRD,
109 loadFpuConst128: defKindRD,
110 fpuStore32: defKindNone,
111 fpuStore64: defKindNone,
112 fpuStore128: defKindNone,
113 udf: defKindNone,
114 cSel: defKindRD,
115 fpuCSel: defKindRD,
116 movToVec: defKindRD,
117 movFromVec: defKindRD,
118 movFromVecSigned: defKindRD,
119 vecDup: defKindRD,
120 vecDupElement: defKindRD,
121 vecExtract: defKindRD,
122 vecMisc: defKindRD,
123 vecMovElement: defKindRD,
124 vecLanes: defKindRD,
125 vecShiftImm: defKindRD,
126 vecTbl: defKindRD,
127 vecTbl2: defKindRD,
128 vecPermute: defKindRD,
129 vecRRR: defKindRD,
130 vecRRRRewrite: defKindNone,
131 fpuToInt: defKindRD,
132 intToFpu: defKindRD,
133 cCmpImm: defKindNone,
134 movToFPSR: defKindNone,
135 movFromFPSR: defKindRD,
136 emitSourceOffsetInfo: defKindNone,
137 atomicRmw: defKindRD,
138 atomicCas: defKindNone,
139 atomicLoad: defKindRD,
140 atomicStore: defKindNone,
141 dmb: defKindNone,
142 loadConstBlockArg: defKindRD,
143 tailCall: defKindCall,
144 tailCallInd: defKindCall,
145 }
146
147 // Defs returns the list of regalloc.VReg that are defined by the instruction.
148 // In order to reduce the number of allocations, the caller can pass the slice to be used.
149 func (i *instruction) Defs(regs *[]regalloc.VReg) []regalloc.VReg {
150 *regs = (*regs)[:0]
151 switch defKinds[i.kind] {
152 case defKindNone:
153 case defKindRD:
154 *regs = append(*regs, i.rd)
155 case defKindCall:
156 _, _, retIntRealRegs, retFloatRealRegs, _ := backend.ABIInfoFromUint64(i.u2)
157 for i := byte(0); i < retIntRealRegs; i++ {
158 *regs = append(*regs, regInfo.RealRegToVReg[intParamResultRegs[i]])
159 }
160 for i := byte(0); i < retFloatRealRegs; i++ {
161 *regs = append(*regs, regInfo.RealRegToVReg[floatParamResultRegs[i]])
162 }
163 default:
164 panic(fmt.Sprintf("defKind for %v not defined", i))
165 }
166 return *regs
167 }
168
169 // AssignDef implements regalloc.Instr AssignDef.
170 func (i *instruction) AssignDef(reg regalloc.VReg) {
171 switch defKinds[i.kind] {
172 case defKindNone:
173 case defKindRD:
174 i.rd = reg
175 case defKindCall:
176 panic("BUG: call instructions shouldn't be assigned")
177 default:
178 panic(fmt.Sprintf("defKind for %v not defined", i))
179 }
180 }
181
182 type useKind byte
183
184 const (
185 useKindNone useKind = iota + 1
186 useKindRN
187 useKindRNRM
188 useKindRNRMRA
189 useKindRNRN1RM
190 useKindCall
191 useKindCallInd
192 useKindAMode
193 useKindRNAMode
194 useKindCond
195 // useKindRDRewrite indicates an instruction where RD is used both as a source and destination.
196 // A temporary register for RD must be allocated explicitly with the source copied to this
197 // register before the instruction and the value copied from this register to the instruction
198 // return register.
199 useKindRDRewrite
200 )
201
202 var useKinds = [numInstructionKinds]useKind{
203 udf: useKindNone,
204 aluRRR: useKindRNRM,
205 aluRRRR: useKindRNRMRA,
206 aluRRImm12: useKindRN,
207 aluRRBitmaskImm: useKindRN,
208 aluRRRShift: useKindRNRM,
209 aluRRImmShift: useKindRN,
210 aluRRRExtend: useKindRNRM,
211 bitRR: useKindRN,
212 movZ: useKindNone,
213 movK: useKindNone,
214 movN: useKindNone,
215 mov32: useKindRN,
216 mov64: useKindRN,
217 fpuMov64: useKindRN,
218 fpuMov128: useKindRN,
219 fpuRR: useKindRN,
220 fpuRRR: useKindRNRM,
221 nop0: useKindNone,
222 call: useKindCall,
223 callInd: useKindCallInd,
224 ret: useKindNone,
225 store8: useKindRNAMode,
226 store16: useKindRNAMode,
227 store32: useKindRNAMode,
228 store64: useKindRNAMode,
229 exitSequence: useKindRN,
230 condBr: useKindCond,
231 br: useKindNone,
232 brTableSequence: useKindRN,
233 cSet: useKindNone,
234 extend: useKindRN,
235 fpuCmp: useKindRNRM,
236 uLoad8: useKindAMode,
237 uLoad16: useKindAMode,
238 uLoad32: useKindAMode,
239 sLoad8: useKindAMode,
240 sLoad16: useKindAMode,
241 sLoad32: useKindAMode,
242 uLoad64: useKindAMode,
243 fpuLoad32: useKindAMode,
244 fpuLoad64: useKindAMode,
245 fpuLoad128: useKindAMode,
246 fpuStore32: useKindRNAMode,
247 fpuStore64: useKindRNAMode,
248 fpuStore128: useKindRNAMode,
249 loadFpuConst32: useKindNone,
250 loadFpuConst64: useKindNone,
251 loadFpuConst128: useKindNone,
252 vecLoad1R: useKindRN,
253 cSel: useKindRNRM,
254 fpuCSel: useKindRNRM,
255 movToVec: useKindRN,
256 movFromVec: useKindRN,
257 movFromVecSigned: useKindRN,
258 vecDup: useKindRN,
259 vecDupElement: useKindRN,
260 vecExtract: useKindRNRM,
261 cCmpImm: useKindRN,
262 vecMisc: useKindRN,
263 vecMovElement: useKindRN,
264 vecLanes: useKindRN,
265 vecShiftImm: useKindRN,
266 vecTbl: useKindRNRM,
267 vecTbl2: useKindRNRN1RM,
268 vecRRR: useKindRNRM,
269 vecRRRRewrite: useKindRDRewrite,
270 vecPermute: useKindRNRM,
271 fpuToInt: useKindRN,
272 intToFpu: useKindRN,
273 movToFPSR: useKindRN,
274 movFromFPSR: useKindNone,
275 adr: useKindNone,
276 emitSourceOffsetInfo: useKindNone,
277 atomicRmw: useKindRNRM,
278 atomicCas: useKindRDRewrite,
279 atomicLoad: useKindRN,
280 atomicStore: useKindRNRM,
281 loadConstBlockArg: useKindNone,
282 dmb: useKindNone,
283 tailCall: useKindCall,
284 tailCallInd: useKindCallInd,
285 }
286
287 // Uses returns the list of regalloc.VReg that are used by the instruction.
288 // In order to reduce the number of allocations, the caller can pass the slice to be used.
289 func (i *instruction) Uses(regs *[]regalloc.VReg) []regalloc.VReg {
290 *regs = (*regs)[:0]
291 switch useKinds[i.kind] {
292 case useKindNone:
293 case useKindRN:
294 if rn := i.rn.reg(); rn.Valid() {
295 *regs = append(*regs, rn)
296 }
297 case useKindRNRM:
298 if rn := i.rn.reg(); rn.Valid() {
299 *regs = append(*regs, rn)
300 }
301 if rm := i.rm.reg(); rm.Valid() {
302 *regs = append(*regs, rm)
303 }
304 case useKindRNRMRA:
305 if rn := i.rn.reg(); rn.Valid() {
306 *regs = append(*regs, rn)
307 }
308 if rm := i.rm.reg(); rm.Valid() {
309 *regs = append(*regs, rm)
310 }
311 if ra := regalloc.VReg(i.u2); ra.Valid() {
312 *regs = append(*regs, ra)
313 }
314 case useKindRNRN1RM:
315 if rn := i.rn.reg(); rn.Valid() && rn.IsRealReg() {
316 rn1 := regalloc.FromRealReg(rn.RealReg()+1, rn.RegType())
317 *regs = append(*regs, rn, rn1)
318 }
319 if rm := i.rm.reg(); rm.Valid() {
320 *regs = append(*regs, rm)
321 }
322 case useKindAMode:
323 amode := i.getAmode()
324 if amodeRN := amode.rn; amodeRN.Valid() {
325 *regs = append(*regs, amodeRN)
326 }
327 if amodeRM := amode.rm; amodeRM.Valid() {
328 *regs = append(*regs, amodeRM)
329 }
330 case useKindRNAMode:
331 *regs = append(*regs, i.rn.reg())
332 amode := i.getAmode()
333 if amodeRN := amode.rn; amodeRN.Valid() {
334 *regs = append(*regs, amodeRN)
335 }
336 if amodeRM := amode.rm; amodeRM.Valid() {
337 *regs = append(*regs, amodeRM)
338 }
339 case useKindCond:
340 cnd := cond(i.u1)
341 if cnd.kind() != condKindCondFlagSet {
342 *regs = append(*regs, cnd.register())
343 }
344 case useKindCallInd:
345 *regs = append(*regs, i.rn.nr())
346 fallthrough
347 case useKindCall:
348 argIntRealRegs, argFloatRealRegs, _, _, _ := backend.ABIInfoFromUint64(i.u2)
349 for i := byte(0); i < argIntRealRegs; i++ {
350 *regs = append(*regs, regInfo.RealRegToVReg[intParamResultRegs[i]])
351 }
352 for i := byte(0); i < argFloatRealRegs; i++ {
353 *regs = append(*regs, regInfo.RealRegToVReg[floatParamResultRegs[i]])
354 }
355 case useKindRDRewrite:
356 *regs = append(*regs, i.rn.reg())
357 *regs = append(*regs, i.rm.reg())
358 *regs = append(*regs, i.rd)
359 default:
360 panic(fmt.Sprintf("useKind for %v not defined", i))
361 }
362 return *regs
363 }
364
365 func (i *instruction) AssignUse(index int, reg regalloc.VReg) {
366 switch useKinds[i.kind] {
367 case useKindNone:
368 case useKindRN:
369 if rn := i.rn.reg(); rn.Valid() {
370 i.rn = i.rn.assignReg(reg)
371 }
372 case useKindRNRM:
373 if index == 0 {
374 if rn := i.rn.reg(); rn.Valid() {
375 i.rn = i.rn.assignReg(reg)
376 }
377 } else {
378 if rm := i.rm.reg(); rm.Valid() {
379 i.rm = i.rm.assignReg(reg)
380 }
381 }
382 case useKindRDRewrite:
383 if index == 0 {
384 if rn := i.rn.reg(); rn.Valid() {
385 i.rn = i.rn.assignReg(reg)
386 }
387 } else if index == 1 {
388 if rm := i.rm.reg(); rm.Valid() {
389 i.rm = i.rm.assignReg(reg)
390 }
391 } else {
392 if rd := i.rd; rd.Valid() {
393 i.rd = reg
394 }
395 }
396 case useKindRNRN1RM:
397 if index == 0 {
398 if rn := i.rn.reg(); rn.Valid() {
399 i.rn = i.rn.assignReg(reg)
400 }
401 if rn1 := i.rn.reg() + 1; rn1.Valid() {
402 i.rm = i.rm.assignReg(reg + 1)
403 }
404 } else {
405 if rm := i.rm.reg(); rm.Valid() {
406 i.rm = i.rm.assignReg(reg)
407 }
408 }
409 case useKindRNRMRA:
410 if index == 0 {
411 if rn := i.rn.reg(); rn.Valid() {
412 i.rn = i.rn.assignReg(reg)
413 }
414 } else if index == 1 {
415 if rm := i.rm.reg(); rm.Valid() {
416 i.rm = i.rm.assignReg(reg)
417 }
418 } else {
419 if ra := regalloc.VReg(i.u2); ra.Valid() {
420 i.u2 = uint64(reg)
421 }
422 }
423 case useKindAMode:
424 if index == 0 {
425 amode := i.getAmode()
426 if amodeRN := amode.rn; amodeRN.Valid() {
427 amode.rn = reg
428 }
429 } else {
430 amode := i.getAmode()
431 if amodeRM := amode.rm; amodeRM.Valid() {
432 amode.rm = reg
433 }
434 }
435 case useKindRNAMode:
436 if index == 0 {
437 i.rn = i.rn.assignReg(reg)
438 } else if index == 1 {
439 amode := i.getAmode()
440 if amodeRN := amode.rn; amodeRN.Valid() {
441 amode.rn = reg
442 } else {
443 panic("BUG")
444 }
445 } else {
446 amode := i.getAmode()
447 if amodeRM := amode.rm; amodeRM.Valid() {
448 amode.rm = reg
449 } else {
450 panic("BUG")
451 }
452 }
453 case useKindCond:
454 c := cond(i.u1)
455 switch c.kind() {
456 case condKindRegisterZero:
457 i.u1 = uint64(registerAsRegZeroCond(reg))
458 case condKindRegisterNotZero:
459 i.u1 = uint64(registerAsRegNotZeroCond(reg))
460 }
461 case useKindCall:
462 panic("BUG: call instructions shouldn't be assigned")
463 case useKindCallInd:
464 i.rn = i.rn.assignReg(reg)
465 default:
466 panic(fmt.Sprintf("useKind for %v not defined", i))
467 }
468 }
469
470 func (i *instruction) asCall(ref ssa.FuncRef, abi *backend.FunctionABI) {
471 i.kind = call
472 i.u1 = uint64(ref)
473 if abi != nil {
474 i.u2 = abi.ABIInfoAsUint64()
475 }
476 }
477
478 func (i *instruction) asCallIndirect(ptr regalloc.VReg, abi *backend.FunctionABI) {
479 i.kind = callInd
480 i.rn = operandNR(ptr)
481 if abi != nil {
482 i.u2 = abi.ABIInfoAsUint64()
483 }
484 }
485
486 func (i *instruction) callFuncRef() ssa.FuncRef {
487 return ssa.FuncRef(i.u1)
488 }
489
490 // shift must be divided by 16 and must be in range 0-3 (if dst64bit is true) or 0-1 (if dst64bit is false)
491 func (i *instruction) asMOVZ(dst regalloc.VReg, imm uint64, shift uint32, dst64bit bool) {
492 i.kind = movZ
493 i.rd = dst
494 i.u1 = imm
495 i.u2 = uint64(shift)
496 if dst64bit {
497 i.u2 |= 1 << 32
498 }
499 }
500
501 // shift must be divided by 16 and must be in range 0-3 (if dst64bit is true) or 0-1 (if dst64bit is false)
502 func (i *instruction) asMOVK(dst regalloc.VReg, imm uint64, shift uint32, dst64bit bool) {
503 i.kind = movK
504 i.rd = dst
505 i.u1 = imm
506 i.u2 = uint64(shift)
507 if dst64bit {
508 i.u2 |= 1 << 32
509 }
510 }
511
512 // shift must be divided by 16 and must be in range 0-3 (if dst64bit is true) or 0-1 (if dst64bit is false)
513 func (i *instruction) asMOVN(dst regalloc.VReg, imm uint64, shift uint32, dst64bit bool) {
514 i.kind = movN
515 i.rd = dst
516 i.u1 = imm
517 i.u2 = uint64(shift)
518 if dst64bit {
519 i.u2 |= 1 << 32
520 }
521 }
522
523 func (i *instruction) asNop0() *instruction {
524 i.kind = nop0
525 return i
526 }
527
528 func (i *instruction) asNop0WithLabel(l label) {
529 i.kind = nop0
530 i.u1 = uint64(l)
531 }
532
533 func (i *instruction) nop0Label() label {
534 return label(i.u1)
535 }
536
537 func (i *instruction) asRet() {
538 i.kind = ret
539 }
540
541 func (i *instruction) asStorePair64(src1, src2 regalloc.VReg, amode *addressMode) {
542 i.kind = storeP64
543 i.rn = operandNR(src1)
544 i.rm = operandNR(src2)
545 i.setAmode(amode)
546 }
547
548 func (i *instruction) asLoadPair64(src1, src2 regalloc.VReg, amode *addressMode) {
549 i.kind = loadP64
550 i.rn = operandNR(src1)
551 i.rm = operandNR(src2)
552 i.setAmode(amode)
553 }
554
555 func (i *instruction) asStore(src operand, amode *addressMode, sizeInBits byte) {
556 switch sizeInBits {
557 case 8:
558 i.kind = store8
559 case 16:
560 i.kind = store16
561 case 32:
562 if src.reg().RegType() == regalloc.RegTypeInt {
563 i.kind = store32
564 } else {
565 i.kind = fpuStore32
566 }
567 case 64:
568 if src.reg().RegType() == regalloc.RegTypeInt {
569 i.kind = store64
570 } else {
571 i.kind = fpuStore64
572 }
573 case 128:
574 i.kind = fpuStore128
575 }
576 i.rn = src
577 i.setAmode(amode)
578 }
579
580 func (i *instruction) asSLoad(dst regalloc.VReg, amode *addressMode, sizeInBits byte) {
581 switch sizeInBits {
582 case 8:
583 i.kind = sLoad8
584 case 16:
585 i.kind = sLoad16
586 case 32:
587 i.kind = sLoad32
588 default:
589 panic("BUG")
590 }
591 i.rd = dst
592 i.setAmode(amode)
593 }
594
595 func (i *instruction) asULoad(dst regalloc.VReg, amode *addressMode, sizeInBits byte) {
596 switch sizeInBits {
597 case 8:
598 i.kind = uLoad8
599 case 16:
600 i.kind = uLoad16
601 case 32:
602 i.kind = uLoad32
603 case 64:
604 i.kind = uLoad64
605 }
606 i.rd = dst
607 i.setAmode(amode)
608 }
609
610 func (i *instruction) asFpuLoad(dst regalloc.VReg, amode *addressMode, sizeInBits byte) {
611 switch sizeInBits {
612 case 32:
613 i.kind = fpuLoad32
614 case 64:
615 i.kind = fpuLoad64
616 case 128:
617 i.kind = fpuLoad128
618 }
619 i.rd = dst
620 i.setAmode(amode)
621 }
622
623 func (i *instruction) getAmode() *addressMode {
624 return wazevoapi.PtrFromUintptr[addressMode](uintptr(i.u1))
625 }
626
627 func (i *instruction) setAmode(a *addressMode) {
628 i.u1 = uint64(uintptr(unsafe.Pointer(a)))
629 }
630
631 func (i *instruction) asVecLoad1R(rd regalloc.VReg, rn operand, arr vecArrangement) {
632 // NOTE: currently only has support for no-offset loads, though it is suspicious that
633 // we would need to support offset load (that is only available for post-index).
634 i.kind = vecLoad1R
635 i.rd = rd
636 i.rn = rn
637 i.u1 = uint64(arr)
638 }
639
640 func (i *instruction) asCSet(rd regalloc.VReg, mask bool, c condFlag) {
641 i.kind = cSet
642 i.rd = rd
643 i.u1 = uint64(c)
644 if mask {
645 i.u2 = 1
646 }
647 }
648
649 func (i *instruction) asCSel(rd regalloc.VReg, rn, rm operand, c condFlag, _64bit bool) {
650 i.kind = cSel
651 i.rd = rd
652 i.rn = rn
653 i.rm = rm
654 i.u1 = uint64(c)
655 if _64bit {
656 i.u2 = 1
657 }
658 }
659
660 func (i *instruction) asFpuCSel(rd regalloc.VReg, rn, rm operand, c condFlag, _64bit bool) {
661 i.kind = fpuCSel
662 i.rd = rd
663 i.rn = rn
664 i.rm = rm
665 i.u1 = uint64(c)
666 if _64bit {
667 i.u2 = 1
668 }
669 }
670
671 func (i *instruction) asBr(target label) {
672 if target == labelReturn {
673 panic("BUG: call site should special case for returnLabel")
674 }
675 i.kind = br
676 i.u1 = uint64(target)
677 }
678
679 func (i *instruction) asBrTableSequence(indexReg regalloc.VReg, targetIndex, targetCounts int) {
680 i.kind = brTableSequence
681 i.rn = operandNR(indexReg)
682 i.u1 = uint64(targetIndex)
683 i.u2 = uint64(targetCounts)
684 }
685
686 func (i *instruction) brTableSequenceOffsetsResolved() {
687 i.rm.data = 1 // indicate that the offsets are resolved, for debugging.
688 }
689
690 func (i *instruction) brLabel() label {
691 return label(i.u1)
692 }
693
694 // brOffsetResolved is called when the target label is resolved.
695 func (i *instruction) brOffsetResolve(offset int64) {
696 i.u2 = uint64(offset)
697 i.rm.data = 1 // indicate that the offset is resolved, for debugging.
698 }
699
700 func (i *instruction) brOffset() int64 {
701 return int64(i.u2)
702 }
703
704 // asCondBr encodes a conditional branch instruction. is64bit is only needed when cond is not flag.
705 func (i *instruction) asCondBr(c cond, target label, is64bit bool) {
706 i.kind = condBr
707 i.u1 = c.asUint64()
708 i.u2 = uint64(target)
709 if is64bit {
710 i.u2 |= 1 << 32
711 }
712 }
713
714 func (i *instruction) setCondBrTargets(target label) {
715 i.u2 = uint64(target)
716 }
717
718 func (i *instruction) condBrLabel() label {
719 return label(i.u2)
720 }
721
722 // condBrOffsetResolve is called when the target label is resolved.
723 func (i *instruction) condBrOffsetResolve(offset int64) {
724 i.rn.data = uint64(offset)
725 i.rn.data2 = 1 // indicate that the offset is resolved, for debugging.
726 }
727
728 // condBrOffsetResolved returns true if condBrOffsetResolve is already called.
729 func (i *instruction) condBrOffsetResolved() bool {
730 return i.rn.data2 == 1
731 }
732
733 func (i *instruction) condBrOffset() int64 {
734 return int64(i.rn.data)
735 }
736
737 func (i *instruction) condBrCond() cond {
738 return cond(i.u1)
739 }
740
741 func (i *instruction) condBr64bit() bool {
742 return i.u2&(1<<32) != 0
743 }
744
745 func (i *instruction) asLoadFpuConst32(rd regalloc.VReg, raw uint64) {
746 i.kind = loadFpuConst32
747 i.u1 = raw
748 i.rd = rd
749 }
750
751 func (i *instruction) asLoadFpuConst64(rd regalloc.VReg, raw uint64) {
752 i.kind = loadFpuConst64
753 i.u1 = raw
754 i.rd = rd
755 }
756
757 func (i *instruction) asLoadFpuConst128(rd regalloc.VReg, lo, hi uint64) {
758 i.kind = loadFpuConst128
759 i.u1 = lo
760 i.u2 = hi
761 i.rd = rd
762 }
763
764 func (i *instruction) asFpuCmp(rn, rm operand, is64bit bool) {
765 i.kind = fpuCmp
766 i.rn, i.rm = rn, rm
767 if is64bit {
768 i.u1 = 1
769 }
770 }
771
772 func (i *instruction) asCCmpImm(rn operand, imm uint64, c condFlag, flag byte, is64bit bool) {
773 i.kind = cCmpImm
774 i.rn = rn
775 i.rm.data = imm
776 i.u1 = uint64(c)
777 i.u2 = uint64(flag)
778 if is64bit {
779 i.u2 |= 1 << 32
780 }
781 }
782
783 // asALU setups a basic ALU instruction.
784 func (i *instruction) asALU(aluOp aluOp, rd regalloc.VReg, rn, rm operand, dst64bit bool) {
785 switch rm.kind {
786 case operandKindNR:
787 i.kind = aluRRR
788 case operandKindSR:
789 i.kind = aluRRRShift
790 case operandKindER:
791 i.kind = aluRRRExtend
792 case operandKindImm12:
793 i.kind = aluRRImm12
794 default:
795 panic("BUG")
796 }
797 i.u1 = uint64(aluOp)
798 i.rd, i.rn, i.rm = rd, rn, rm
799 if dst64bit {
800 i.u2 |= 1 << 32
801 }
802 }
803
804 // asALU setups a basic ALU instruction.
805 func (i *instruction) asALURRRR(aluOp aluOp, rd regalloc.VReg, rn, rm operand, ra regalloc.VReg, dst64bit bool) {
806 i.kind = aluRRRR
807 i.u1 = uint64(aluOp)
808 i.rd, i.rn, i.rm, i.u2 = rd, rn, rm, uint64(ra)
809 if dst64bit {
810 i.u1 |= 1 << 32
811 }
812 }
813
814 // asALUShift setups a shift based ALU instruction.
815 func (i *instruction) asALUShift(aluOp aluOp, rd regalloc.VReg, rn, rm operand, dst64bit bool) {
816 switch rm.kind {
817 case operandKindNR:
818 i.kind = aluRRR // If the shift amount op is a register, then the instruction is encoded as a normal ALU instruction with two register operands.
819 case operandKindShiftImm:
820 i.kind = aluRRImmShift
821 default:
822 panic("BUG")
823 }
824 i.u1 = uint64(aluOp)
825 i.rd, i.rn, i.rm = rd, rn, rm
826 if dst64bit {
827 i.u2 |= 1 << 32
828 }
829 }
830
831 func (i *instruction) asALUBitmaskImm(aluOp aluOp, rd, rn regalloc.VReg, imm uint64, dst64bit bool) {
832 i.kind = aluRRBitmaskImm
833 i.u1 = uint64(aluOp)
834 i.rn, i.rd = operandNR(rn), rd
835 i.u2 = imm
836 if dst64bit {
837 i.u1 |= 1 << 32
838 }
839 }
840
841 func (i *instruction) asMovToFPSR(rn regalloc.VReg) {
842 i.kind = movToFPSR
843 i.rn = operandNR(rn)
844 }
845
846 func (i *instruction) asMovFromFPSR(rd regalloc.VReg) {
847 i.kind = movFromFPSR
848 i.rd = rd
849 }
850
851 func (i *instruction) asBitRR(bitOp bitOp, rd, rn regalloc.VReg, is64bit bool) {
852 i.kind = bitRR
853 i.rn, i.rd = operandNR(rn), rd
854 i.u1 = uint64(bitOp)
855 if is64bit {
856 i.u2 = 1
857 }
858 }
859
860 func (i *instruction) asFpuRRR(op fpuBinOp, rd regalloc.VReg, rn, rm operand, dst64bit bool) {
861 i.kind = fpuRRR
862 i.u1 = uint64(op)
863 i.rd, i.rn, i.rm = rd, rn, rm
864 if dst64bit {
865 i.u2 = 1
866 }
867 }
868
869 func (i *instruction) asFpuRR(op fpuUniOp, rd regalloc.VReg, rn operand, dst64bit bool) {
870 i.kind = fpuRR
871 i.u1 = uint64(op)
872 i.rd, i.rn = rd, rn
873 if dst64bit {
874 i.u2 = 1
875 }
876 }
877
878 func (i *instruction) asExtend(rd, rn regalloc.VReg, fromBits, toBits byte, signed bool) {
879 i.kind = extend
880 i.rn, i.rd = operandNR(rn), rd
881 i.u1 = uint64(fromBits)
882 i.u2 = uint64(toBits)
883 if signed {
884 i.u2 |= 1 << 32
885 }
886 }
887
888 func (i *instruction) asMove32(rd, rn regalloc.VReg) {
889 i.kind = mov32
890 i.rn, i.rd = operandNR(rn), rd
891 }
892
893 func (i *instruction) asMove64(rd, rn regalloc.VReg) *instruction {
894 i.kind = mov64
895 i.rn, i.rd = operandNR(rn), rd
896 return i
897 }
898
899 func (i *instruction) asFpuMov64(rd, rn regalloc.VReg) {
900 i.kind = fpuMov64
901 i.rn, i.rd = operandNR(rn), rd
902 }
903
904 func (i *instruction) asFpuMov128(rd, rn regalloc.VReg) *instruction {
905 i.kind = fpuMov128
906 i.rn, i.rd = operandNR(rn), rd
907 return i
908 }
909
910 func (i *instruction) asMovToVec(rd regalloc.VReg, rn operand, arr vecArrangement, index vecIndex) {
911 i.kind = movToVec
912 i.rd = rd
913 i.rn = rn
914 i.u1, i.u2 = uint64(arr), uint64(index)
915 }
916
917 func (i *instruction) asMovFromVec(rd regalloc.VReg, rn operand, arr vecArrangement, index vecIndex, signed bool) {
918 if signed {
919 i.kind = movFromVecSigned
920 } else {
921 i.kind = movFromVec
922 }
923 i.rd = rd
924 i.rn = rn
925 i.u1, i.u2 = uint64(arr), uint64(index)
926 }
927
928 func (i *instruction) asVecDup(rd regalloc.VReg, rn operand, arr vecArrangement) {
929 i.kind = vecDup
930 i.u1 = uint64(arr)
931 i.rn, i.rd = rn, rd
932 }
933
934 func (i *instruction) asVecDupElement(rd regalloc.VReg, rn operand, arr vecArrangement, index vecIndex) {
935 i.kind = vecDupElement
936 i.u1 = uint64(arr)
937 i.rn, i.rd = rn, rd
938 i.u2 = uint64(index)
939 }
940
941 func (i *instruction) asVecExtract(rd regalloc.VReg, rn, rm operand, arr vecArrangement, index uint32) {
942 i.kind = vecExtract
943 i.u1 = uint64(arr)
944 i.rn, i.rm, i.rd = rn, rm, rd
945 i.u2 = uint64(index)
946 }
947
948 func (i *instruction) asVecMovElement(rd regalloc.VReg, rn operand, arr vecArrangement, rdIndex, rnIndex vecIndex) {
949 i.kind = vecMovElement
950 i.u1 = uint64(arr)
951 i.u2 = uint64(rdIndex) | uint64(rnIndex)<<32
952 i.rn, i.rd = rn, rd
953 }
954
955 func (i *instruction) asVecMisc(op vecOp, rd regalloc.VReg, rn operand, arr vecArrangement) {
956 i.kind = vecMisc
957 i.u1 = uint64(op)
958 i.rn, i.rd = rn, rd
959 i.u2 = uint64(arr)
960 }
961
962 func (i *instruction) asVecLanes(op vecOp, rd regalloc.VReg, rn operand, arr vecArrangement) {
963 i.kind = vecLanes
964 i.u1 = uint64(op)
965 i.rn, i.rd = rn, rd
966 i.u2 = uint64(arr)
967 }
968
969 func (i *instruction) asVecShiftImm(op vecOp, rd regalloc.VReg, rn, rm operand, arr vecArrangement) *instruction {
970 i.kind = vecShiftImm
971 i.u1 = uint64(op)
972 i.rn, i.rm, i.rd = rn, rm, rd
973 i.u2 = uint64(arr)
974 return i
975 }
976
977 func (i *instruction) asVecTbl(nregs byte, rd regalloc.VReg, rn, rm operand, arr vecArrangement) {
978 switch nregs {
979 case 0, 1:
980 i.kind = vecTbl
981 case 2:
982 i.kind = vecTbl2
983 if !rn.reg().IsRealReg() {
984 panic("rn is not a RealReg")
985 }
986 if rn.realReg() == v31 {
987 panic("rn cannot be v31")
988 }
989 default:
990 panic(fmt.Sprintf("unsupported number of registers %d", nregs))
991 }
992 i.rn, i.rm, i.rd = rn, rm, rd
993 i.u2 = uint64(arr)
994 }
995
996 func (i *instruction) asVecPermute(op vecOp, rd regalloc.VReg, rn, rm operand, arr vecArrangement) {
997 i.kind = vecPermute
998 i.u1 = uint64(op)
999 i.rn, i.rm, i.rd = rn, rm, rd
1000 i.u2 = uint64(arr)
1001 }
1002
1003 func (i *instruction) asVecRRR(op vecOp, rd regalloc.VReg, rn, rm operand, arr vecArrangement) *instruction {
1004 i.kind = vecRRR
1005 i.u1 = uint64(op)
1006 i.rn, i.rd, i.rm = rn, rd, rm
1007 i.u2 = uint64(arr)
1008 return i
1009 }
1010
1011 // asVecRRRRewrite encodes a vector instruction that rewrites the destination register.
1012 // IMPORTANT: the destination register must be already defined before this instruction.
1013 func (i *instruction) asVecRRRRewrite(op vecOp, rd regalloc.VReg, rn, rm operand, arr vecArrangement) {
1014 i.kind = vecRRRRewrite
1015 i.u1 = uint64(op)
1016 i.rn, i.rd, i.rm = rn, rd, rm
1017 i.u2 = uint64(arr)
1018 }
1019
1020 func (i *instruction) IsCopy() bool {
1021 op := i.kind
1022 // We do not include mov32 as it is not a copy instruction in the sense that it does not preserve the upper 32 bits,
1023 // and it is only used in the translation of IReduce, not the actual copy indeed.
1024 return op == mov64 || op == fpuMov64 || op == fpuMov128
1025 }
1026
1027 // String implements fmt.Stringer.
1028 func (i *instruction) String() (str string) {
1029 is64SizeBitToSize := func(v uint64) byte {
1030 if v == 0 {
1031 return 32
1032 }
1033 return 64
1034 }
1035
1036 switch i.kind {
1037 case nop0:
1038 if i.u1 != 0 {
1039 l := label(i.u1)
1040 str = fmt.Sprintf("%s:", l)
1041 } else {
1042 str = "nop0"
1043 }
1044 case aluRRR:
1045 size := is64SizeBitToSize(i.u2 >> 32)
1046 str = fmt.Sprintf("%s %s, %s, %s", aluOp(i.u1).String(),
1047 formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), size),
1048 i.rm.format(size))
1049 case aluRRRR:
1050 size := is64SizeBitToSize(i.u1 >> 32)
1051 str = fmt.Sprintf("%s %s, %s, %s, %s", aluOp(i.u1).String(),
1052 formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), size), formatVRegSized(i.rm.nr(), size), formatVRegSized(regalloc.VReg(i.u2), size))
1053 case aluRRImm12:
1054 size := is64SizeBitToSize(i.u2 >> 32)
1055 str = fmt.Sprintf("%s %s, %s, %s", aluOp(i.u1).String(),
1056 formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), size), i.rm.format(size))
1057 case aluRRBitmaskImm:
1058 size := is64SizeBitToSize(i.u1 >> 32)
1059 rd, rn := formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), size)
1060 if size == 32 {
1061 str = fmt.Sprintf("%s %s, %s, #%#x", aluOp(i.u1).String(), rd, rn, uint32(i.u2))
1062 } else {
1063 str = fmt.Sprintf("%s %s, %s, #%#x", aluOp(i.u1).String(), rd, rn, i.u2)
1064 }
1065 case aluRRImmShift:
1066 size := is64SizeBitToSize(i.u2 >> 32)
1067 str = fmt.Sprintf("%s %s, %s, %#x",
1068 aluOp(i.u1).String(),
1069 formatVRegSized(i.rd, size),
1070 formatVRegSized(i.rn.nr(), size),
1071 i.rm.shiftImm(),
1072 )
1073 case aluRRRShift:
1074 size := is64SizeBitToSize(i.u2 >> 32)
1075 str = fmt.Sprintf("%s %s, %s, %s",
1076 aluOp(i.u1).String(),
1077 formatVRegSized(i.rd, size),
1078 formatVRegSized(i.rn.nr(), size),
1079 i.rm.format(size),
1080 )
1081 case aluRRRExtend:
1082 size := is64SizeBitToSize(i.u2 >> 32)
1083 str = fmt.Sprintf("%s %s, %s, %s", aluOp(i.u1).String(),
1084 formatVRegSized(i.rd, size),
1085 formatVRegSized(i.rn.nr(), size),
1086 // Regardless of the source size, the register is formatted in 32-bit.
1087 i.rm.format(32),
1088 )
1089 case bitRR:
1090 size := is64SizeBitToSize(i.u2)
1091 str = fmt.Sprintf("%s %s, %s",
1092 bitOp(i.u1),
1093 formatVRegSized(i.rd, size),
1094 formatVRegSized(i.rn.nr(), size),
1095 )
1096 case uLoad8:
1097 str = fmt.Sprintf("ldrb %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1098 case sLoad8:
1099 str = fmt.Sprintf("ldrsb %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1100 case uLoad16:
1101 str = fmt.Sprintf("ldrh %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1102 case sLoad16:
1103 str = fmt.Sprintf("ldrsh %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1104 case uLoad32:
1105 str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1106 case sLoad32:
1107 str = fmt.Sprintf("ldrs %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1108 case uLoad64:
1109 str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd, 64), i.getAmode().format(64))
1110 case store8:
1111 str = fmt.Sprintf("strb %s, %s", formatVRegSized(i.rn.nr(), 32), i.getAmode().format(8))
1112 case store16:
1113 str = fmt.Sprintf("strh %s, %s", formatVRegSized(i.rn.nr(), 32), i.getAmode().format(16))
1114 case store32:
1115 str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 32), i.getAmode().format(32))
1116 case store64:
1117 str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 64), i.getAmode().format(64))
1118 case storeP64:
1119 str = fmt.Sprintf("stp %s, %s, %s",
1120 formatVRegSized(i.rn.nr(), 64), formatVRegSized(i.rm.nr(), 64), i.getAmode().format(64))
1121 case loadP64:
1122 str = fmt.Sprintf("ldp %s, %s, %s",
1123 formatVRegSized(i.rn.nr(), 64), formatVRegSized(i.rm.nr(), 64), i.getAmode().format(64))
1124 case mov64:
1125 str = fmt.Sprintf("mov %s, %s",
1126 formatVRegSized(i.rd, 64),
1127 formatVRegSized(i.rn.nr(), 64))
1128 case mov32:
1129 str = fmt.Sprintf("mov %s, %s", formatVRegSized(i.rd, 32), formatVRegSized(i.rn.nr(), 32))
1130 case movZ:
1131 size := is64SizeBitToSize(i.u2 >> 32)
1132 str = fmt.Sprintf("movz %s, #%#x, lsl %d", formatVRegSized(i.rd, size), uint16(i.u1), uint32(i.u2)*16)
1133 case movN:
1134 size := is64SizeBitToSize(i.u2 >> 32)
1135 str = fmt.Sprintf("movn %s, #%#x, lsl %d", formatVRegSized(i.rd, size), uint16(i.u1), uint32(i.u2)*16)
1136 case movK:
1137 size := is64SizeBitToSize(i.u2 >> 32)
1138 str = fmt.Sprintf("movk %s, #%#x, lsl %d", formatVRegSized(i.rd, size), uint16(i.u1), uint32(i.u2)*16)
1139 case extend:
1140 fromBits, toBits := byte(i.u1), byte(i.u2)
1141
1142 var signedStr string
1143 if i.u2>>32 == 1 {
1144 signedStr = "s"
1145 } else {
1146 signedStr = "u"
1147 }
1148 var fromStr string
1149 switch fromBits {
1150 case 8:
1151 fromStr = "b"
1152 case 16:
1153 fromStr = "h"
1154 case 32:
1155 fromStr = "w"
1156 }
1157 str = fmt.Sprintf("%sxt%s %s, %s", signedStr, fromStr, formatVRegSized(i.rd, toBits), formatVRegSized(i.rn.nr(), 32))
1158 case cSel:
1159 size := is64SizeBitToSize(i.u2)
1160 str = fmt.Sprintf("csel %s, %s, %s, %s",
1161 formatVRegSized(i.rd, size),
1162 formatVRegSized(i.rn.nr(), size),
1163 formatVRegSized(i.rm.nr(), size),
1164 condFlag(i.u1),
1165 )
1166 case cSet:
1167 if i.u2 != 0 {
1168 str = fmt.Sprintf("csetm %s, %s", formatVRegSized(i.rd, 64), condFlag(i.u1))
1169 } else {
1170 str = fmt.Sprintf("cset %s, %s", formatVRegSized(i.rd, 64), condFlag(i.u1))
1171 }
1172 case cCmpImm:
1173 size := is64SizeBitToSize(i.u2 >> 32)
1174 str = fmt.Sprintf("ccmp %s, #%#x, #%#x, %s",
1175 formatVRegSized(i.rn.nr(), size), i.rm.data,
1176 i.u2&0b1111,
1177 condFlag(i.u1))
1178 case fpuMov64:
1179 str = fmt.Sprintf("mov %s, %s",
1180 formatVRegVec(i.rd, vecArrangement8B, vecIndexNone),
1181 formatVRegVec(i.rn.nr(), vecArrangement8B, vecIndexNone))
1182 case fpuMov128:
1183 str = fmt.Sprintf("mov %s, %s",
1184 formatVRegVec(i.rd, vecArrangement16B, vecIndexNone),
1185 formatVRegVec(i.rn.nr(), vecArrangement16B, vecIndexNone))
1186 case fpuMovFromVec:
1187 panic("TODO")
1188 case fpuRR:
1189 dstSz := is64SizeBitToSize(i.u2)
1190 srcSz := dstSz
1191 op := fpuUniOp(i.u1)
1192 switch op {
1193 case fpuUniOpCvt32To64:
1194 srcSz = 32
1195 case fpuUniOpCvt64To32:
1196 srcSz = 64
1197 }
1198 str = fmt.Sprintf("%s %s, %s", op.String(),
1199 formatVRegSized(i.rd, dstSz), formatVRegSized(i.rn.nr(), srcSz))
1200 case fpuRRR:
1201 size := is64SizeBitToSize(i.u2)
1202 str = fmt.Sprintf("%s %s, %s, %s", fpuBinOp(i.u1).String(),
1203 formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), size), formatVRegSized(i.rm.nr(), size))
1204 case fpuRRI:
1205 panic("TODO")
1206 case fpuRRRR:
1207 panic("TODO")
1208 case fpuCmp:
1209 size := is64SizeBitToSize(i.u1)
1210 str = fmt.Sprintf("fcmp %s, %s",
1211 formatVRegSized(i.rn.nr(), size), formatVRegSized(i.rm.nr(), size))
1212 case fpuLoad32:
1213 str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd, 32), i.getAmode().format(32))
1214 case fpuStore32:
1215 str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 32), i.getAmode().format(64))
1216 case fpuLoad64:
1217 str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd, 64), i.getAmode().format(64))
1218 case fpuStore64:
1219 str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 64), i.getAmode().format(64))
1220 case fpuLoad128:
1221 str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd, 128), i.getAmode().format(64))
1222 case fpuStore128:
1223 str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 128), i.getAmode().format(64))
1224 case loadFpuConst32:
1225 str = fmt.Sprintf("ldr %s, #8; b 8; data.f32 %f", formatVRegSized(i.rd, 32), math.Float32frombits(uint32(i.u1)))
1226 case loadFpuConst64:
1227 str = fmt.Sprintf("ldr %s, #8; b 16; data.f64 %f", formatVRegSized(i.rd, 64), math.Float64frombits(i.u1))
1228 case loadFpuConst128:
1229 str = fmt.Sprintf("ldr %s, #8; b 32; data.v128 %016x %016x",
1230 formatVRegSized(i.rd, 128), i.u1, i.u2)
1231 case fpuToInt:
1232 var op, src, dst string
1233 if signed := i.u1 == 1; signed {
1234 op = "fcvtzs"
1235 } else {
1236 op = "fcvtzu"
1237 }
1238 if src64 := i.u2&1 != 0; src64 {
1239 src = formatVRegWidthVec(i.rn.nr(), vecArrangementD)
1240 } else {
1241 src = formatVRegWidthVec(i.rn.nr(), vecArrangementS)
1242 }
1243 if dst64 := i.u2&2 != 0; dst64 {
1244 dst = formatVRegSized(i.rd, 64)
1245 } else {
1246 dst = formatVRegSized(i.rd, 32)
1247 }
1248 str = fmt.Sprintf("%s %s, %s", op, dst, src)
1249
1250 case intToFpu:
1251 var op, src, dst string
1252 if signed := i.u1 == 1; signed {
1253 op = "scvtf"
1254 } else {
1255 op = "ucvtf"
1256 }
1257 if src64 := i.u2&1 != 0; src64 {
1258 src = formatVRegSized(i.rn.nr(), 64)
1259 } else {
1260 src = formatVRegSized(i.rn.nr(), 32)
1261 }
1262 if dst64 := i.u2&2 != 0; dst64 {
1263 dst = formatVRegWidthVec(i.rd, vecArrangementD)
1264 } else {
1265 dst = formatVRegWidthVec(i.rd, vecArrangementS)
1266 }
1267 str = fmt.Sprintf("%s %s, %s", op, dst, src)
1268 case fpuCSel:
1269 size := is64SizeBitToSize(i.u2)
1270 str = fmt.Sprintf("fcsel %s, %s, %s, %s",
1271 formatVRegSized(i.rd, size),
1272 formatVRegSized(i.rn.nr(), size),
1273 formatVRegSized(i.rm.nr(), size),
1274 condFlag(i.u1),
1275 )
1276 case movToVec:
1277 var size byte
1278 arr := vecArrangement(i.u1)
1279 switch arr {
1280 case vecArrangementB, vecArrangementH, vecArrangementS:
1281 size = 32
1282 case vecArrangementD:
1283 size = 64
1284 default:
1285 panic("unsupported arrangement " + arr.String())
1286 }
1287 str = fmt.Sprintf("ins %s, %s", formatVRegVec(i.rd, arr, vecIndex(i.u2)), formatVRegSized(i.rn.nr(), size))
1288 case movFromVec, movFromVecSigned:
1289 var size byte
1290 var opcode string
1291 arr := vecArrangement(i.u1)
1292 signed := i.kind == movFromVecSigned
1293 switch arr {
1294 case vecArrangementB, vecArrangementH, vecArrangementS:
1295 size = 32
1296 if signed {
1297 opcode = "smov"
1298 } else {
1299 opcode = "umov"
1300 }
1301 case vecArrangementD:
1302 size = 64
1303 if signed {
1304 opcode = "smov"
1305 } else {
1306 opcode = "mov"
1307 }
1308 default:
1309 panic("unsupported arrangement " + arr.String())
1310 }
1311 str = fmt.Sprintf("%s %s, %s", opcode, formatVRegSized(i.rd, size), formatVRegVec(i.rn.nr(), arr, vecIndex(i.u2)))
1312 case vecDup:
1313 str = fmt.Sprintf("dup %s, %s",
1314 formatVRegVec(i.rd, vecArrangement(i.u1), vecIndexNone),
1315 formatVRegSized(i.rn.nr(), 64),
1316 )
1317 case vecDupElement:
1318 arr := vecArrangement(i.u1)
1319 str = fmt.Sprintf("dup %s, %s",
1320 formatVRegVec(i.rd, arr, vecIndexNone),
1321 formatVRegVec(i.rn.nr(), arr, vecIndex(i.u2)),
1322 )
1323 case vecDupFromFpu:
1324 panic("TODO")
1325 case vecExtract:
1326 str = fmt.Sprintf("ext %s, %s, %s, #%d",
1327 formatVRegVec(i.rd, vecArrangement(i.u1), vecIndexNone),
1328 formatVRegVec(i.rn.nr(), vecArrangement(i.u1), vecIndexNone),
1329 formatVRegVec(i.rm.nr(), vecArrangement(i.u1), vecIndexNone),
1330 uint32(i.u2),
1331 )
1332 case vecExtend:
1333 panic("TODO")
1334 case vecMovElement:
1335 str = fmt.Sprintf("mov %s, %s",
1336 formatVRegVec(i.rd, vecArrangement(i.u1), vecIndex(i.u2&0xffffffff)),
1337 formatVRegVec(i.rn.nr(), vecArrangement(i.u1), vecIndex(i.u2>>32)),
1338 )
1339 case vecMiscNarrow:
1340 panic("TODO")
1341 case vecRRR, vecRRRRewrite:
1342 str = fmt.Sprintf("%s %s, %s, %s",
1343 vecOp(i.u1),
1344 formatVRegVec(i.rd, vecArrangement(i.u2), vecIndexNone),
1345 formatVRegVec(i.rn.nr(), vecArrangement(i.u2), vecIndexNone),
1346 formatVRegVec(i.rm.nr(), vecArrangement(i.u2), vecIndexNone),
1347 )
1348 case vecMisc:
1349 vop := vecOp(i.u1)
1350 if vop == vecOpCmeq0 {
1351 str = fmt.Sprintf("cmeq %s, %s, #0",
1352 formatVRegVec(i.rd, vecArrangement(i.u2), vecIndexNone),
1353 formatVRegVec(i.rn.nr(), vecArrangement(i.u2), vecIndexNone))
1354 } else {
1355 str = fmt.Sprintf("%s %s, %s",
1356 vop,
1357 formatVRegVec(i.rd, vecArrangement(i.u2), vecIndexNone),
1358 formatVRegVec(i.rn.nr(), vecArrangement(i.u2), vecIndexNone))
1359 }
1360 case vecLanes:
1361 arr := vecArrangement(i.u2)
1362 var destArr vecArrangement
1363 switch arr {
1364 case vecArrangement8B, vecArrangement16B:
1365 destArr = vecArrangementH
1366 case vecArrangement4H, vecArrangement8H:
1367 destArr = vecArrangementS
1368 case vecArrangement4S:
1369 destArr = vecArrangementD
1370 default:
1371 panic("invalid arrangement " + arr.String())
1372 }
1373 str = fmt.Sprintf("%s %s, %s",
1374 vecOp(i.u1),
1375 formatVRegWidthVec(i.rd, destArr),
1376 formatVRegVec(i.rn.nr(), arr, vecIndexNone))
1377 case vecShiftImm:
1378 arr := vecArrangement(i.u2)
1379 str = fmt.Sprintf("%s %s, %s, #%d",
1380 vecOp(i.u1),
1381 formatVRegVec(i.rd, arr, vecIndexNone),
1382 formatVRegVec(i.rn.nr(), arr, vecIndexNone),
1383 i.rm.shiftImm())
1384 case vecTbl:
1385 arr := vecArrangement(i.u2)
1386 str = fmt.Sprintf("tbl %s, { %s }, %s",
1387 formatVRegVec(i.rd, arr, vecIndexNone),
1388 formatVRegVec(i.rn.nr(), vecArrangement16B, vecIndexNone),
1389 formatVRegVec(i.rm.nr(), arr, vecIndexNone))
1390 case vecTbl2:
1391 arr := vecArrangement(i.u2)
1392 rd, rn, rm := i.rd, i.rn.nr(), i.rm.nr()
1393 rn1 := regalloc.FromRealReg(rn.RealReg()+1, rn.RegType())
1394 str = fmt.Sprintf("tbl %s, { %s, %s }, %s",
1395 formatVRegVec(rd, arr, vecIndexNone),
1396 formatVRegVec(rn, vecArrangement16B, vecIndexNone),
1397 formatVRegVec(rn1, vecArrangement16B, vecIndexNone),
1398 formatVRegVec(rm, arr, vecIndexNone))
1399 case vecPermute:
1400 arr := vecArrangement(i.u2)
1401 str = fmt.Sprintf("%s %s, %s, %s",
1402 vecOp(i.u1),
1403 formatVRegVec(i.rd, arr, vecIndexNone),
1404 formatVRegVec(i.rn.nr(), arr, vecIndexNone),
1405 formatVRegVec(i.rm.nr(), arr, vecIndexNone))
1406 case movToFPSR:
1407 str = fmt.Sprintf("msr fpsr, %s", formatVRegSized(i.rn.nr(), 64))
1408 case movFromFPSR:
1409 str = fmt.Sprintf("mrs %s fpsr", formatVRegSized(i.rd, 64))
1410 case call:
1411 str = fmt.Sprintf("bl %s", ssa.FuncRef(i.u1))
1412 case callInd:
1413 str = fmt.Sprintf("bl %s", formatVRegSized(i.rn.nr(), 64))
1414 case ret:
1415 str = "ret"
1416 case br:
1417 target := label(i.u1)
1418 if i.rm.data != 0 {
1419 str = fmt.Sprintf("b #%#x (%s)", i.brOffset(), target.String())
1420 } else {
1421 str = fmt.Sprintf("b %s", target.String())
1422 }
1423 case condBr:
1424 size := is64SizeBitToSize(i.u2 >> 32)
1425 c := cond(i.u1)
1426 target := label(i.u2 & 0xffffffff)
1427 switch c.kind() {
1428 case condKindRegisterZero:
1429 if !i.condBrOffsetResolved() {
1430 str = fmt.Sprintf("cbz %s, (%s)", formatVRegSized(c.register(), size), target.String())
1431 } else {
1432 str = fmt.Sprintf("cbz %s, #%#x %s", formatVRegSized(c.register(), size), i.condBrOffset(), target.String())
1433 }
1434 case condKindRegisterNotZero:
1435 if offset := i.condBrOffset(); offset != 0 {
1436 str = fmt.Sprintf("cbnz %s, #%#x (%s)", formatVRegSized(c.register(), size), offset, target.String())
1437 } else {
1438 str = fmt.Sprintf("cbnz %s, %s", formatVRegSized(c.register(), size), target.String())
1439 }
1440 case condKindCondFlagSet:
1441 if offset := i.condBrOffset(); offset != 0 {
1442 if target == labelInvalid {
1443 str = fmt.Sprintf("b.%s #%#x", c.flag(), offset)
1444 } else {
1445 str = fmt.Sprintf("b.%s #%#x, (%s)", c.flag(), offset, target.String())
1446 }
1447 } else {
1448 str = fmt.Sprintf("b.%s %s", c.flag(), target.String())
1449 }
1450 }
1451 case adr:
1452 str = fmt.Sprintf("adr %s, #%#x", formatVRegSized(i.rd, 64), int64(i.u1))
1453 case brTableSequence:
1454 targetIndex := i.u1
1455 str = fmt.Sprintf("br_table_sequence %s, table_index=%d", formatVRegSized(i.rn.nr(), 64), targetIndex)
1456 case exitSequence:
1457 str = fmt.Sprintf("exit_sequence %s", formatVRegSized(i.rn.nr(), 64))
1458 case atomicRmw:
1459 m := atomicRmwOp(i.u1).String()
1460 size := byte(32)
1461 switch i.u2 {
1462 case 8:
1463 size = 64
1464 case 2:
1465 m = m + "h"
1466 case 1:
1467 m = m + "b"
1468 }
1469 str = fmt.Sprintf("%s %s, %s, %s", m, formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), 64))
1470 case atomicCas:
1471 m := "casal"
1472 size := byte(32)
1473 switch i.u2 {
1474 case 8:
1475 size = 64
1476 case 2:
1477 m = m + "h"
1478 case 1:
1479 m = m + "b"
1480 }
1481 str = fmt.Sprintf("%s %s, %s, %s", m, formatVRegSized(i.rd, size), formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rn.nr(), 64))
1482 case atomicLoad:
1483 m := "ldar"
1484 size := byte(32)
1485 switch i.u2 {
1486 case 8:
1487 size = 64
1488 case 2:
1489 m = m + "h"
1490 case 1:
1491 m = m + "b"
1492 }
1493 str = fmt.Sprintf("%s %s, %s", m, formatVRegSized(i.rd, size), formatVRegSized(i.rn.nr(), 64))
1494 case atomicStore:
1495 m := "stlr"
1496 size := byte(32)
1497 switch i.u2 {
1498 case 8:
1499 size = 64
1500 case 2:
1501 m = m + "h"
1502 case 1:
1503 m = m + "b"
1504 }
1505 str = fmt.Sprintf("%s %s, %s", m, formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rn.nr(), 64))
1506 case dmb:
1507 str = "dmb"
1508 case tailCall:
1509 str = fmt.Sprintf("b %s", ssa.FuncRef(i.u1))
1510 case tailCallInd:
1511 str = fmt.Sprintf("b %s", formatVRegSized(i.rn.nr(), 64))
1512 case udf:
1513 str = "udf"
1514 case emitSourceOffsetInfo:
1515 str = fmt.Sprintf("source_offset_info %d", ssa.SourceOffset(i.u1))
1516 case vecLoad1R:
1517 str = fmt.Sprintf("ld1r {%s}, [%s]", formatVRegVec(i.rd, vecArrangement(i.u1), vecIndexNone), formatVRegSized(i.rn.nr(), 64))
1518 case loadConstBlockArg:
1519 str = fmt.Sprintf("load_const_block_arg %s, %#x", formatVRegSized(i.rd, 64), i.u1)
1520 default:
1521 panic(i.kind)
1522 }
1523 return
1524 }
1525
1526 func (i *instruction) asAdr(rd regalloc.VReg, offset int64) {
1527 i.kind = adr
1528 i.rd = rd
1529 i.u1 = uint64(offset)
1530 }
1531
1532 func (i *instruction) asAtomicRmw(op atomicRmwOp, rn, rs, rt regalloc.VReg, size uint64) {
1533 i.kind = atomicRmw
1534 i.rd, i.rn, i.rm = rt, operandNR(rn), operandNR(rs)
1535 i.u1 = uint64(op)
1536 i.u2 = size
1537 }
1538
1539 func (i *instruction) asAtomicCas(rn, rs, rt regalloc.VReg, size uint64) {
1540 i.kind = atomicCas
1541 i.rm, i.rn, i.rd = operandNR(rt), operandNR(rn), rs
1542 i.u2 = size
1543 }
1544
1545 func (i *instruction) asAtomicLoad(rn, rt regalloc.VReg, size uint64) {
1546 i.kind = atomicLoad
1547 i.rn, i.rd = operandNR(rn), rt
1548 i.u2 = size
1549 }
1550
1551 func (i *instruction) asAtomicStore(rn, rt operand, size uint64) {
1552 i.kind = atomicStore
1553 i.rn, i.rm = rn, rt
1554 i.u2 = size
1555 }
1556
1557 func (i *instruction) asDMB() {
1558 i.kind = dmb
1559 }
1560
1561 func (i *instruction) asTailCall(ref ssa.FuncRef, abi *backend.FunctionABI) {
1562 i.kind = tailCall
1563 i.u1 = uint64(ref)
1564 if abi != nil {
1565 i.u2 = abi.ABIInfoAsUint64()
1566 }
1567 }
1568
1569 func (i *instruction) asTailCallIndirect(ptr regalloc.VReg, abi *backend.FunctionABI) {
1570 i.kind = tailCallInd
1571 i.rn = operandNR(ptr)
1572 if abi != nil {
1573 i.u2 = abi.ABIInfoAsUint64()
1574 }
1575 }
1576
1577 // TODO: delete unnecessary things.
1578 const (
1579 // nop0 represents a no-op of zero size.
1580 nop0 instructionKind = iota + 1
1581 // aluRRR represents an ALU operation with two register sources and a register destination.
1582 aluRRR
1583 // aluRRRR represents an ALU operation with three register sources and a register destination.
1584 aluRRRR
1585 // aluRRImm12 represents an ALU operation with a register source and an immediate-12 source, with a register destination.
1586 aluRRImm12
1587 // aluRRBitmaskImm represents an ALU operation with a register source and a bitmask immediate, with a register destination.
1588 aluRRBitmaskImm
1589 // aluRRImmShift represents an ALU operation with a register source and an immediate-shifted source, with a register destination.
1590 aluRRImmShift
1591 // aluRRRShift represents an ALU operation with two register sources, one of which can be shifted, with a register destination.
1592 aluRRRShift
1593 // aluRRRExtend represents an ALU operation with two register sources, one of which can be extended, with a register destination.
1594 aluRRRExtend
1595 // bitRR represents a bit op instruction with a single register source.
1596 bitRR
1597 // uLoad8 represents an unsigned 8-bit load.
1598 uLoad8
1599 // sLoad8 represents a signed 8-bit load into 64-bit register.
1600 sLoad8
1601 // uLoad16 represents an unsigned 16-bit load into 64-bit register.
1602 uLoad16
1603 // sLoad16 represents a signed 16-bit load into 64-bit register.
1604 sLoad16
1605 // uLoad32 represents an unsigned 32-bit load into 64-bit register.
1606 uLoad32
1607 // sLoad32 represents a signed 32-bit load into 64-bit register.
1608 sLoad32
1609 // uLoad64 represents a 64-bit load.
1610 uLoad64
1611 // store8 represents an 8-bit store.
1612 store8
1613 // store16 represents a 16-bit store.
1614 store16
1615 // store32 represents a 32-bit store.
1616 store32
1617 // store64 represents a 64-bit store.
1618 store64
1619 // storeP64 represents a store of a pair of registers.
1620 storeP64
1621 // loadP64 represents a load of a pair of registers.
1622 loadP64
1623 // mov64 represents a MOV instruction. These are encoded as ORR's but we keep them separate for better handling.
1624 mov64
1625 // mov32 represents a 32-bit MOV. This zeroes the top 32 bits of the destination.
1626 mov32
1627 // movZ represents a MOVZ with a 16-bit immediate.
1628 movZ
1629 // movN represents a MOVN with a 16-bit immediate.
1630 movN
1631 // movK represents a MOVK with a 16-bit immediate.
1632 movK
1633 // extend represents a sign- or zero-extend operation.
1634 extend
1635 // cSel represents a conditional-select operation.
1636 cSel
1637 // cSet represents a conditional-set operation.
1638 cSet
1639 // cCmpImm represents a conditional comparison with an immediate.
1640 cCmpImm
1641 // fpuMov64 represents a FPU move. Distinct from a vector-register move; moving just 64 bits appears to be significantly faster.
1642 fpuMov64
1643 // fpuMov128 represents a vector register move.
1644 fpuMov128
1645 // fpuMovFromVec represents a move to scalar from a vector element.
1646 fpuMovFromVec
1647 // fpuRR represents a 1-op FPU instruction.
1648 fpuRR
1649 // fpuRRR represents a 2-op FPU instruction.
1650 fpuRRR
1651 // fpuRRI represents a 2-op FPU instruction with immediate value.
1652 fpuRRI
1653 // fpuRRRR represents a 3-op FPU instruction.
1654 fpuRRRR
1655 // fpuCmp represents a FPU comparison, either 32 or 64 bit.
1656 fpuCmp
1657 // fpuLoad32 represents a floating-point load, single-precision (32 bit).
1658 fpuLoad32
1659 // fpuStore32 represents a floating-point store, single-precision (32 bit).
1660 fpuStore32
1661 // fpuLoad64 represents a floating-point load, double-precision (64 bit).
1662 fpuLoad64
1663 // fpuStore64 represents a floating-point store, double-precision (64 bit).
1664 fpuStore64
1665 // fpuLoad128 represents a floating-point/vector load, 128 bit.
1666 fpuLoad128
1667 // fpuStore128 represents a floating-point/vector store, 128 bit.
1668 fpuStore128
1669 // loadFpuConst32 represents a load of a 32-bit floating-point constant.
1670 loadFpuConst32
1671 // loadFpuConst64 represents a load of a 64-bit floating-point constant.
1672 loadFpuConst64
1673 // loadFpuConst128 represents a load of a 128-bit floating-point constant.
1674 loadFpuConst128
1675 // vecLoad1R represents a load of a one single-element structure that replicates to all lanes of a vector.
1676 vecLoad1R
1677 // fpuToInt represents a conversion from FP to integer.
1678 fpuToInt
1679 // intToFpu represents a conversion from integer to FP.
1680 intToFpu
1681 // fpuCSel represents a 32/64-bit FP conditional select.
1682 fpuCSel
1683 // movToVec represents a move to a vector element from a GPR.
1684 movToVec
1685 // movFromVec represents an unsigned move from a vector element to a GPR.
1686 movFromVec
1687 // movFromVecSigned represents a signed move from a vector element to a GPR.
1688 movFromVecSigned
1689 // vecDup represents a duplication of general-purpose register to vector.
1690 vecDup
1691 // vecDupElement represents a duplication of a vector element to vector or scalar.
1692 vecDupElement
1693 // vecDupFromFpu represents a duplication of scalar to vector.
1694 vecDupFromFpu
1695 // vecExtract represents a vector extraction operation.
1696 vecExtract
1697 // vecExtend represents a vector extension operation.
1698 vecExtend
1699 // vecMovElement represents a move vector element to another vector element operation.
1700 vecMovElement
1701 // vecMiscNarrow represents a vector narrowing operation.
1702 vecMiscNarrow
1703 // vecRRR represents a vector ALU operation.
1704 vecRRR
1705 // vecRRRRewrite is exactly the same as vecRRR except that this rewrites the destination register.
1706 // For example, BSL instruction rewrites the destination register, and the existing value influences the result.
1707 // Therefore, the "destination" register in vecRRRRewrite will be treated as "use" which makes the register outlive
1708 // the instruction while this instruction doesn't have "def" in the context of register allocation.
1709 vecRRRRewrite
1710 // vecMisc represents a vector two register miscellaneous instruction.
1711 vecMisc
1712 // vecLanes represents a vector instruction across lanes.
1713 vecLanes
1714 // vecShiftImm represents a SIMD scalar shift by immediate instruction.
1715 vecShiftImm
1716 // vecTbl represents a table vector lookup - single register table.
1717 vecTbl
1718 // vecTbl2 represents a table vector lookup - two register table.
1719 vecTbl2
1720 // vecPermute represents a vector permute instruction.
1721 vecPermute
1722 // movToNZCV represents a move to the FPSR.
1723 movToFPSR
1724 // movFromNZCV represents a move from the FPSR.
1725 movFromFPSR
1726 // call represents a machine call instruction.
1727 call
1728 // callInd represents a machine indirect-call instruction.
1729 callInd
1730 // ret represents a machine return instruction.
1731 ret
1732 // br represents an unconditional branch.
1733 br
1734 // condBr represents a conditional branch.
1735 condBr
1736 // adr represents a compute the address (using a PC-relative offset) of a memory location.
1737 adr
1738 // brTableSequence represents a jump-table sequence.
1739 brTableSequence
1740 // exitSequence consists of multiple instructions, and exits the execution immediately.
1741 // See encodeExitSequence.
1742 exitSequence
1743 // atomicRmw represents an atomic read-modify-write operation with two register sources and a register destination.
1744 atomicRmw
1745 // atomicCas represents an atomic compare-and-swap operation with three register sources. The value is loaded to
1746 // the source register containing the comparison value.
1747 atomicCas
1748 // atomicLoad represents an atomic load with one source register and a register destination.
1749 atomicLoad
1750 // atomicStore represents an atomic store with two source registers and no destination.
1751 atomicStore
1752 // dmb represents the data memory barrier instruction in inner-shareable (ish) mode.
1753 dmb
1754 // tailCall represents a tail call instruction.
1755 tailCall
1756 // tailCallInd represents a tail call indirect instruction.
1757 tailCallInd
1758 // UDF is the undefined instruction. For debugging only.
1759 udf
1760 // loadConstBlockArg represents a load of a constant block argument.
1761 loadConstBlockArg
1762
1763 // emitSourceOffsetInfo is a dummy instruction to emit source offset info.
1764 // The existence of this instruction does not affect the execution.
1765 emitSourceOffsetInfo
1766
1767 // ------------------- do not define below this line -------------------
1768 numInstructionKinds
1769 )
1770
1771 func (i *instruction) asLoadConstBlockArg(v uint64, typ ssa.Type, dst regalloc.VReg) *instruction {
1772 i.kind = loadConstBlockArg
1773 i.u1 = v
1774 i.u2 = uint64(typ)
1775 i.rd = dst
1776 return i
1777 }
1778
1779 func (i *instruction) loadConstBlockArgData() (v uint64, typ ssa.Type, dst regalloc.VReg) {
1780 return i.u1, ssa.Type(i.u2), i.rd
1781 }
1782
1783 func (i *instruction) asEmitSourceOffsetInfo(l ssa.SourceOffset) *instruction {
1784 i.kind = emitSourceOffsetInfo
1785 i.u1 = uint64(l)
1786 return i
1787 }
1788
1789 func (i *instruction) sourceOffsetInfo() ssa.SourceOffset {
1790 return ssa.SourceOffset(i.u1)
1791 }
1792
1793 func (i *instruction) asUDF() *instruction {
1794 i.kind = udf
1795 return i
1796 }
1797
1798 func (i *instruction) asFpuToInt(rd regalloc.VReg, rn operand, rdSigned, src64bit, dst64bit bool) {
1799 i.kind = fpuToInt
1800 i.rn = rn
1801 i.rd = rd
1802 if rdSigned {
1803 i.u1 = 1
1804 }
1805 if src64bit {
1806 i.u2 = 1
1807 }
1808 if dst64bit {
1809 i.u2 |= 2
1810 }
1811 }
1812
1813 func (i *instruction) asIntToFpu(rd regalloc.VReg, rn operand, rnSigned, src64bit, dst64bit bool) {
1814 i.kind = intToFpu
1815 i.rn = rn
1816 i.rd = rd
1817 if rnSigned {
1818 i.u1 = 1
1819 }
1820 if src64bit {
1821 i.u2 = 1
1822 }
1823 if dst64bit {
1824 i.u2 |= 2
1825 }
1826 }
1827
1828 func (i *instruction) asExitSequence(ctx regalloc.VReg) *instruction {
1829 i.kind = exitSequence
1830 i.rn = operandNR(ctx)
1831 return i
1832 }
1833
1834 // aluOp determines the type of ALU operation. Instructions whose kind is one of
1835 // aluRRR, aluRRRR, aluRRImm12, aluRRBitmaskImm, aluRRImmShift, aluRRRShift and aluRRRExtend
1836 // would use this type.
1837 type aluOp uint32
1838
1839 func (a aluOp) String() string {
1840 switch a {
1841 case aluOpAdd:
1842 return "add"
1843 case aluOpSub:
1844 return "sub"
1845 case aluOpOrr:
1846 return "orr"
1847 case aluOpOrn:
1848 return "orn"
1849 case aluOpAnd:
1850 return "and"
1851 case aluOpAnds:
1852 return "ands"
1853 case aluOpBic:
1854 return "bic"
1855 case aluOpEor:
1856 return "eor"
1857 case aluOpAddS:
1858 return "adds"
1859 case aluOpSubS:
1860 return "subs"
1861 case aluOpSMulH:
1862 return "sMulH"
1863 case aluOpUMulH:
1864 return "uMulH"
1865 case aluOpSDiv:
1866 return "sdiv"
1867 case aluOpUDiv:
1868 return "udiv"
1869 case aluOpRotR:
1870 return "ror"
1871 case aluOpLsr:
1872 return "lsr"
1873 case aluOpAsr:
1874 return "asr"
1875 case aluOpLsl:
1876 return "lsl"
1877 case aluOpMAdd:
1878 return "madd"
1879 case aluOpMSub:
1880 return "msub"
1881 }
1882 panic(int(a))
1883 }
1884
1885 const (
1886 // 32/64-bit Add.
1887 aluOpAdd aluOp = iota
1888 // 32/64-bit Subtract.
1889 aluOpSub
1890 // 32/64-bit Bitwise OR.
1891 aluOpOrr
1892 // 32/64-bit Bitwise OR NOT.
1893 aluOpOrn
1894 // 32/64-bit Bitwise AND.
1895 aluOpAnd
1896 // 32/64-bit Bitwise ANDS.
1897 aluOpAnds
1898 // 32/64-bit Bitwise AND NOT.
1899 aluOpBic
1900 // 32/64-bit Bitwise XOR (Exclusive OR).
1901 aluOpEor
1902 // 32/64-bit Add setting flags.
1903 aluOpAddS
1904 // 32/64-bit Subtract setting flags.
1905 aluOpSubS
1906 // Signed multiply, high-word result.
1907 aluOpSMulH
1908 // Unsigned multiply, high-word result.
1909 aluOpUMulH
1910 // 64-bit Signed divide.
1911 aluOpSDiv
1912 // 64-bit Unsigned divide.
1913 aluOpUDiv
1914 // 32/64-bit Rotate right.
1915 aluOpRotR
1916 // 32/64-bit Logical shift right.
1917 aluOpLsr
1918 // 32/64-bit Arithmetic shift right.
1919 aluOpAsr
1920 // 32/64-bit Logical shift left.
1921 aluOpLsl /// Multiply-add
1922
1923 // MAdd and MSub are only applicable for aluRRRR.
1924 aluOpMAdd
1925 aluOpMSub
1926 )
1927
1928 // vecOp determines the type of vector operation. Instructions whose kind is one of
1929 // vecOpCnt would use this type.
1930 type vecOp int
1931
1932 // String implements fmt.Stringer.
1933 func (b vecOp) String() string {
1934 switch b {
1935 case vecOpCnt:
1936 return "cnt"
1937 case vecOpCmeq:
1938 return "cmeq"
1939 case vecOpCmgt:
1940 return "cmgt"
1941 case vecOpCmhi:
1942 return "cmhi"
1943 case vecOpCmge:
1944 return "cmge"
1945 case vecOpCmhs:
1946 return "cmhs"
1947 case vecOpFcmeq:
1948 return "fcmeq"
1949 case vecOpFcmgt:
1950 return "fcmgt"
1951 case vecOpFcmge:
1952 return "fcmge"
1953 case vecOpCmeq0:
1954 return "cmeq0"
1955 case vecOpUaddlv:
1956 return "uaddlv"
1957 case vecOpBit:
1958 return "bit"
1959 case vecOpBic:
1960 return "bic"
1961 case vecOpBsl:
1962 return "bsl"
1963 case vecOpNot:
1964 return "not"
1965 case vecOpAnd:
1966 return "and"
1967 case vecOpOrr:
1968 return "orr"
1969 case vecOpEOR:
1970 return "eor"
1971 case vecOpFadd:
1972 return "fadd"
1973 case vecOpAdd:
1974 return "add"
1975 case vecOpAddp:
1976 return "addp"
1977 case vecOpAddv:
1978 return "addv"
1979 case vecOpSub:
1980 return "sub"
1981 case vecOpFsub:
1982 return "fsub"
1983 case vecOpSmin:
1984 return "smin"
1985 case vecOpUmin:
1986 return "umin"
1987 case vecOpUminv:
1988 return "uminv"
1989 case vecOpSmax:
1990 return "smax"
1991 case vecOpUmax:
1992 return "umax"
1993 case vecOpUmaxp:
1994 return "umaxp"
1995 case vecOpUrhadd:
1996 return "urhadd"
1997 case vecOpFmul:
1998 return "fmul"
1999 case vecOpSqrdmulh:
2000 return "sqrdmulh"
2001 case vecOpMul:
2002 return "mul"
2003 case vecOpUmlal:
2004 return "umlal"
2005 case vecOpFdiv:
2006 return "fdiv"
2007 case vecOpFsqrt:
2008 return "fsqrt"
2009 case vecOpAbs:
2010 return "abs"
2011 case vecOpFabs:
2012 return "fabs"
2013 case vecOpNeg:
2014 return "neg"
2015 case vecOpFneg:
2016 return "fneg"
2017 case vecOpFrintp:
2018 return "frintp"
2019 case vecOpFrintm:
2020 return "frintm"
2021 case vecOpFrintn:
2022 return "frintn"
2023 case vecOpFrintz:
2024 return "frintz"
2025 case vecOpFcvtl:
2026 return "fcvtl"
2027 case vecOpFcvtn:
2028 return "fcvtn"
2029 case vecOpFcvtzu:
2030 return "fcvtzu"
2031 case vecOpFcvtzs:
2032 return "fcvtzs"
2033 case vecOpScvtf:
2034 return "scvtf"
2035 case vecOpUcvtf:
2036 return "ucvtf"
2037 case vecOpSqxtn:
2038 return "sqxtn"
2039 case vecOpUqxtn:
2040 return "uqxtn"
2041 case vecOpSqxtun:
2042 return "sqxtun"
2043 case vecOpRev64:
2044 return "rev64"
2045 case vecOpXtn:
2046 return "xtn"
2047 case vecOpShll:
2048 return "shll"
2049 case vecOpSshl:
2050 return "sshl"
2051 case vecOpSshll:
2052 return "sshll"
2053 case vecOpUshl:
2054 return "ushl"
2055 case vecOpUshll:
2056 return "ushll"
2057 case vecOpSshr:
2058 return "sshr"
2059 case vecOpZip1:
2060 return "zip1"
2061 case vecOpFmin:
2062 return "fmin"
2063 case vecOpFmax:
2064 return "fmax"
2065 case vecOpSmull:
2066 return "smull"
2067 case vecOpSmull2:
2068 return "smull2"
2069 }
2070 panic(int(b))
2071 }
2072
2073 const (
2074 vecOpCnt vecOp = iota
2075 vecOpCmeq0
2076 vecOpCmeq
2077 vecOpCmgt
2078 vecOpCmhi
2079 vecOpCmge
2080 vecOpCmhs
2081 vecOpFcmeq
2082 vecOpFcmgt
2083 vecOpFcmge
2084 vecOpUaddlv
2085 vecOpBit
2086 vecOpBic
2087 vecOpBsl
2088 vecOpNot
2089 vecOpAnd
2090 vecOpOrr
2091 vecOpEOR
2092 vecOpAdd
2093 vecOpFadd
2094 vecOpAddv
2095 vecOpSqadd
2096 vecOpUqadd
2097 vecOpAddp
2098 vecOpSub
2099 vecOpFsub
2100 vecOpSqsub
2101 vecOpUqsub
2102 vecOpSmin
2103 vecOpUmin
2104 vecOpUminv
2105 vecOpFmin
2106 vecOpSmax
2107 vecOpUmax
2108 vecOpUmaxp
2109 vecOpFmax
2110 vecOpUrhadd
2111 vecOpMul
2112 vecOpFmul
2113 vecOpSqrdmulh
2114 vecOpUmlal
2115 vecOpFdiv
2116 vecOpFsqrt
2117 vecOpAbs
2118 vecOpFabs
2119 vecOpNeg
2120 vecOpFneg
2121 vecOpFrintm
2122 vecOpFrintn
2123 vecOpFrintp
2124 vecOpFrintz
2125 vecOpFcvtl
2126 vecOpFcvtn
2127 vecOpFcvtzs
2128 vecOpFcvtzu
2129 vecOpScvtf
2130 vecOpUcvtf
2131 vecOpSqxtn
2132 vecOpSqxtun
2133 vecOpUqxtn
2134 vecOpRev64
2135 vecOpXtn
2136 vecOpShll
2137 vecOpSshl
2138 vecOpSshll
2139 vecOpUshl
2140 vecOpUshll
2141 vecOpSshr
2142 vecOpZip1
2143 vecOpSmull
2144 vecOpSmull2
2145 )
2146
2147 // bitOp determines the type of bitwise operation. Instructions whose kind is one of
2148 // bitOpRbit and bitOpClz would use this type.
2149 type bitOp int
2150
2151 // String implements fmt.Stringer.
2152 func (b bitOp) String() string {
2153 switch b {
2154 case bitOpRbit:
2155 return "rbit"
2156 case bitOpClz:
2157 return "clz"
2158 }
2159 panic(int(b))
2160 }
2161
2162 const (
2163 // 32/64-bit Rbit.
2164 bitOpRbit bitOp = iota
2165 // 32/64-bit Clz.
2166 bitOpClz
2167 )
2168
2169 // fpuUniOp represents a unary floating-point unit (FPU) operation.
2170 type fpuUniOp byte
2171
2172 const (
2173 fpuUniOpNeg fpuUniOp = iota
2174 fpuUniOpCvt32To64
2175 fpuUniOpCvt64To32
2176 fpuUniOpSqrt
2177 fpuUniOpRoundPlus
2178 fpuUniOpRoundMinus
2179 fpuUniOpRoundZero
2180 fpuUniOpRoundNearest
2181 fpuUniOpAbs
2182 )
2183
2184 // String implements the fmt.Stringer.
2185 func (f fpuUniOp) String() string {
2186 switch f {
2187 case fpuUniOpNeg:
2188 return "fneg"
2189 case fpuUniOpCvt32To64:
2190 return "fcvt"
2191 case fpuUniOpCvt64To32:
2192 return "fcvt"
2193 case fpuUniOpSqrt:
2194 return "fsqrt"
2195 case fpuUniOpRoundPlus:
2196 return "frintp"
2197 case fpuUniOpRoundMinus:
2198 return "frintm"
2199 case fpuUniOpRoundZero:
2200 return "frintz"
2201 case fpuUniOpRoundNearest:
2202 return "frintn"
2203 case fpuUniOpAbs:
2204 return "fabs"
2205 }
2206 panic(int(f))
2207 }
2208
2209 // fpuBinOp represents a binary floating-point unit (FPU) operation.
2210 type fpuBinOp byte
2211
2212 const (
2213 fpuBinOpAdd = iota
2214 fpuBinOpSub
2215 fpuBinOpMul
2216 fpuBinOpDiv
2217 fpuBinOpMax
2218 fpuBinOpMin
2219 )
2220
2221 // String implements the fmt.Stringer.
2222 func (f fpuBinOp) String() string {
2223 switch f {
2224 case fpuBinOpAdd:
2225 return "fadd"
2226 case fpuBinOpSub:
2227 return "fsub"
2228 case fpuBinOpMul:
2229 return "fmul"
2230 case fpuBinOpDiv:
2231 return "fdiv"
2232 case fpuBinOpMax:
2233 return "fmax"
2234 case fpuBinOpMin:
2235 return "fmin"
2236 }
2237 panic(int(f))
2238 }
2239
2240 // extMode represents the mode of a register operand extension.
2241 // For example, aluRRRExtend instructions need this info to determine the extensions.
2242 type extMode byte
2243
2244 const (
2245 extModeNone extMode = iota
2246 // extModeZeroExtend64 suggests a zero-extension to 32 bits if the original bit size is less than 32.
2247 extModeZeroExtend32
2248 // extModeSignExtend64 stands for a sign-extension to 32 bits if the original bit size is less than 32.
2249 extModeSignExtend32
2250 // extModeZeroExtend64 suggests a zero-extension to 64 bits if the original bit size is less than 64.
2251 extModeZeroExtend64
2252 // extModeSignExtend64 stands for a sign-extension to 64 bits if the original bit size is less than 64.
2253 extModeSignExtend64
2254 )
2255
2256 func (e extMode) bits() byte {
2257 switch e {
2258 case extModeZeroExtend32, extModeSignExtend32:
2259 return 32
2260 case extModeZeroExtend64, extModeSignExtend64:
2261 return 64
2262 default:
2263 return 0
2264 }
2265 }
2266
2267 func (e extMode) signed() bool {
2268 switch e {
2269 case extModeSignExtend32, extModeSignExtend64:
2270 return true
2271 default:
2272 return false
2273 }
2274 }
2275
2276 func extModeOf(t ssa.Type, signed bool) extMode {
2277 switch t.Bits() {
2278 case 32:
2279 if signed {
2280 return extModeSignExtend32
2281 }
2282 return extModeZeroExtend32
2283 case 64:
2284 if signed {
2285 return extModeSignExtend64
2286 }
2287 return extModeZeroExtend64
2288 default:
2289 panic("TODO? do we need narrower than 32 bits?")
2290 }
2291 }
2292
2293 type extendOp byte
2294
2295 const (
2296 extendOpUXTB extendOp = 0b000
2297 extendOpUXTH extendOp = 0b001
2298 extendOpUXTW extendOp = 0b010
2299 // extendOpUXTX does nothing, but convenient symbol that officially exists. See:
2300 // https://stackoverflow.com/questions/72041372/what-do-the-uxtx-and-sxtx-extensions-mean-for-32-bit-aarch64-adds-instruct
2301 extendOpUXTX extendOp = 0b011
2302 extendOpSXTB extendOp = 0b100
2303 extendOpSXTH extendOp = 0b101
2304 extendOpSXTW extendOp = 0b110
2305 // extendOpSXTX does nothing, but convenient symbol that officially exists. See:
2306 // https://stackoverflow.com/questions/72041372/what-do-the-uxtx-and-sxtx-extensions-mean-for-32-bit-aarch64-adds-instruct
2307 extendOpSXTX extendOp = 0b111
2308 extendOpNone extendOp = 0xff
2309 )
2310
2311 func (e extendOp) srcBits() byte {
2312 switch e {
2313 case extendOpUXTB, extendOpSXTB:
2314 return 8
2315 case extendOpUXTH, extendOpSXTH:
2316 return 16
2317 case extendOpUXTW, extendOpSXTW:
2318 return 32
2319 case extendOpUXTX, extendOpSXTX:
2320 return 64
2321 }
2322 panic(int(e))
2323 }
2324
2325 func (e extendOp) String() string {
2326 switch e {
2327 case extendOpUXTB:
2328 return "UXTB"
2329 case extendOpUXTH:
2330 return "UXTH"
2331 case extendOpUXTW:
2332 return "UXTW"
2333 case extendOpUXTX:
2334 return "UXTX"
2335 case extendOpSXTB:
2336 return "SXTB"
2337 case extendOpSXTH:
2338 return "SXTH"
2339 case extendOpSXTW:
2340 return "SXTW"
2341 case extendOpSXTX:
2342 return "SXTX"
2343 }
2344 panic(int(e))
2345 }
2346
2347 func extendOpFrom(signed bool, from byte) extendOp {
2348 switch from {
2349 case 8:
2350 if signed {
2351 return extendOpSXTB
2352 }
2353 return extendOpUXTB
2354 case 16:
2355 if signed {
2356 return extendOpSXTH
2357 }
2358 return extendOpUXTH
2359 case 32:
2360 if signed {
2361 return extendOpSXTW
2362 }
2363 return extendOpUXTW
2364 case 64:
2365 if signed {
2366 return extendOpSXTX
2367 }
2368 return extendOpUXTX
2369 }
2370 panic("invalid extendOpFrom")
2371 }
2372
2373 type shiftOp byte
2374
2375 const (
2376 shiftOpLSL shiftOp = 0b00
2377 shiftOpLSR shiftOp = 0b01
2378 shiftOpASR shiftOp = 0b10
2379 shiftOpROR shiftOp = 0b11
2380 )
2381
2382 func (s shiftOp) String() string {
2383 switch s {
2384 case shiftOpLSL:
2385 return "lsl"
2386 case shiftOpLSR:
2387 return "lsr"
2388 case shiftOpASR:
2389 return "asr"
2390 case shiftOpROR:
2391 return "ror"
2392 }
2393 panic(int(s))
2394 }
2395
2396 const exitSequenceSize = 6 * 4 // 6 instructions as in encodeExitSequence.
2397
2398 // size returns the size of the instruction in encoded bytes.
2399 func (i *instruction) size() int64 {
2400 switch i.kind {
2401 case exitSequence:
2402 return exitSequenceSize // 5 instructions as in encodeExitSequence.
2403 case nop0, loadConstBlockArg:
2404 return 0
2405 case emitSourceOffsetInfo:
2406 return 0
2407 case loadFpuConst32:
2408 if i.u1 == 0 {
2409 return 4 // zero loading can be encoded as a single instruction.
2410 }
2411 return 4 + 4 + 4
2412 case loadFpuConst64:
2413 if i.u1 == 0 {
2414 return 4 // zero loading can be encoded as a single instruction.
2415 }
2416 return 4 + 4 + 8
2417 case loadFpuConst128:
2418 if i.u1 == 0 && i.u2 == 0 {
2419 return 4 // zero loading can be encoded as a single instruction.
2420 }
2421 return 4 + 4 + 16
2422 case brTableSequence:
2423 return 4*4 + int64(i.u2)*4
2424 default:
2425 return 4
2426 }
2427 }
2428
2429 // vecArrangement is the arrangement of data within a vector register.
2430 type vecArrangement byte
2431
2432 const (
2433 // vecArrangementNone is an arrangement indicating no data is stored.
2434 vecArrangementNone vecArrangement = iota
2435 // vecArrangement8B is an arrangement of 8 bytes (64-bit vector)
2436 vecArrangement8B
2437 // vecArrangement16B is an arrangement of 16 bytes (128-bit vector)
2438 vecArrangement16B
2439 // vecArrangement4H is an arrangement of 4 half precisions (64-bit vector)
2440 vecArrangement4H
2441 // vecArrangement8H is an arrangement of 8 half precisions (128-bit vector)
2442 vecArrangement8H
2443 // vecArrangement2S is an arrangement of 2 single precisions (64-bit vector)
2444 vecArrangement2S
2445 // vecArrangement4S is an arrangement of 4 single precisions (128-bit vector)
2446 vecArrangement4S
2447 // vecArrangement1D is an arrangement of 1 double precision (64-bit vector)
2448 vecArrangement1D
2449 // vecArrangement2D is an arrangement of 2 double precisions (128-bit vector)
2450 vecArrangement2D
2451
2452 // Assign each vector size specifier to a vector arrangement ID.
2453 // Instructions can only have an arrangement or a size specifier, but not both, so it
2454 // simplifies the internal representation of vector instructions by being able to
2455 // store either into the same field.
2456
2457 // vecArrangementB is a size specifier of byte
2458 vecArrangementB
2459 // vecArrangementH is a size specifier of word (16-bit)
2460 vecArrangementH
2461 // vecArrangementS is a size specifier of double word (32-bit)
2462 vecArrangementS
2463 // vecArrangementD is a size specifier of quad word (64-bit)
2464 vecArrangementD
2465 // vecArrangementQ is a size specifier of the entire vector (128-bit)
2466 vecArrangementQ
2467 )
2468
2469 // String implements fmt.Stringer
2470 func (v vecArrangement) String() (ret string) {
2471 switch v {
2472 case vecArrangement8B:
2473 ret = "8B"
2474 case vecArrangement16B:
2475 ret = "16B"
2476 case vecArrangement4H:
2477 ret = "4H"
2478 case vecArrangement8H:
2479 ret = "8H"
2480 case vecArrangement2S:
2481 ret = "2S"
2482 case vecArrangement4S:
2483 ret = "4S"
2484 case vecArrangement1D:
2485 ret = "1D"
2486 case vecArrangement2D:
2487 ret = "2D"
2488 case vecArrangementB:
2489 ret = "B"
2490 case vecArrangementH:
2491 ret = "H"
2492 case vecArrangementS:
2493 ret = "S"
2494 case vecArrangementD:
2495 ret = "D"
2496 case vecArrangementQ:
2497 ret = "Q"
2498 case vecArrangementNone:
2499 ret = "none"
2500 default:
2501 panic(v)
2502 }
2503 return
2504 }
2505
2506 // vecIndex is the index of an element of a vector register
2507 type vecIndex byte
2508
2509 // vecIndexNone indicates no vector index specified.
2510 const vecIndexNone = ^vecIndex(0)
2511
2512 func ssaLaneToArrangement(lane ssa.VecLane) vecArrangement {
2513 switch lane {
2514 case ssa.VecLaneI8x16:
2515 return vecArrangement16B
2516 case ssa.VecLaneI16x8:
2517 return vecArrangement8H
2518 case ssa.VecLaneI32x4:
2519 return vecArrangement4S
2520 case ssa.VecLaneI64x2:
2521 return vecArrangement2D
2522 case ssa.VecLaneF32x4:
2523 return vecArrangement4S
2524 case ssa.VecLaneF64x2:
2525 return vecArrangement2D
2526 default:
2527 panic(lane)
2528 }
2529 }
2530
2531 // atomicRmwOp is the type of atomic read-modify-write operation.
2532 type atomicRmwOp byte
2533
2534 const (
2535 // atomicRmwOpAdd is an atomic add operation.
2536 atomicRmwOpAdd atomicRmwOp = iota
2537 // atomicRmwOpClr is an atomic clear operation, i.e. AND NOT.
2538 atomicRmwOpClr
2539 // atomicRmwOpSet is an atomic set operation, i.e. OR.
2540 atomicRmwOpSet
2541 // atomicRmwOpEor is an atomic exclusive OR operation.
2542 atomicRmwOpEor
2543 // atomicRmwOpSwp is an atomic swap operation.
2544 atomicRmwOpSwp
2545 )
2546
2547 // String implements fmt.Stringer
2548 func (a atomicRmwOp) String() string {
2549 switch a {
2550 case atomicRmwOpAdd:
2551 return "ldaddal"
2552 case atomicRmwOpClr:
2553 return "ldclral"
2554 case atomicRmwOpSet:
2555 return "ldsetal"
2556 case atomicRmwOpEor:
2557 return "ldeoral"
2558 case atomicRmwOpSwp:
2559 return "swpal"
2560 }
2561 panic(fmt.Sprintf("unknown atomicRmwOp: %d", a))
2562 }