Debian Codegraph @

internal/engine/wazevo/backend/isa/amd64/instr.go

raw
1 package amd64
2
3 import (
4 "fmt"
5
6 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
7 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
8 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
9 )
10
11 type instruction struct {
12 prev, next *instruction
13 op1, op2 operand
14 u1, u2 uint64
15 b1 bool
16 addedBeforeRegAlloc bool
17 kind instructionKind
18 }
19
20 // IsCall implements regalloc.Instr.
21 func (i *instruction) IsCall() bool { return i.kind == call }
22
23 // IsIndirectCall implements regalloc.Instr.
24 func (i *instruction) IsIndirectCall() bool {
25 return i.kind == callIndirect
26 }
27
28 // IsReturn implements regalloc.Instr.
29 func (i *instruction) IsReturn() bool { return i.kind == ret }
30
31 // String implements regalloc.Instr.
32 func (i *instruction) String() string {
33 switch i.kind {
34 case nop0:
35 return "nop"
36 case sourceOffsetInfo:
37 return fmt.Sprintf("source_offset_info %d", i.u1)
38 case ret:
39 return "ret"
40 case imm:
41 if i.b1 {
42 return fmt.Sprintf("movabsq $%d, %s", int64(i.u1), i.op2.format(true))
43 } else {
44 return fmt.Sprintf("movl $%d, %s", int32(i.u1), i.op2.format(false))
45 }
46 case aluRmiR:
47 return fmt.Sprintf("%s %s, %s", aluRmiROpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1))
48 case movRR:
49 if i.b1 {
50 return fmt.Sprintf("movq %s, %s", i.op1.format(true), i.op2.format(true))
51 } else {
52 return fmt.Sprintf("movl %s, %s", i.op1.format(false), i.op2.format(false))
53 }
54 case xmmRmR:
55 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(false), i.op2.format(false))
56 case gprToXmm:
57 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1))
58 case xmmUnaryRmR:
59 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(false), i.op2.format(false))
60 case xmmUnaryRmRImm:
61 return fmt.Sprintf("%s $%d, %s, %s", sseOpcode(i.u1), roundingMode(i.u2), i.op1.format(false), i.op2.format(false))
62 case unaryRmR:
63 var suffix string
64 if i.b1 {
65 suffix = "q"
66 } else {
67 suffix = "l"
68 }
69 return fmt.Sprintf("%s%s %s, %s", unaryRmROpcode(i.u1), suffix, i.op1.format(i.b1), i.op2.format(i.b1))
70 case not:
71 var op string
72 if i.b1 {
73 op = "notq"
74 } else {
75 op = "notl"
76 }
77 return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
78 case neg:
79 var op string
80 if i.b1 {
81 op = "negq"
82 } else {
83 op = "negl"
84 }
85 return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
86 case div:
87 var prefix string
88 var op string
89 if i.b1 {
90 op = "divq"
91 } else {
92 op = "divl"
93 }
94 if i.u1 != 0 {
95 prefix = "i"
96 }
97 return fmt.Sprintf("%s%s %s", prefix, op, i.op1.format(i.b1))
98 case mulHi:
99 signed, _64 := i.u1 != 0, i.b1
100 var op string
101 switch {
102 case signed && _64:
103 op = "imulq"
104 case !signed && _64:
105 op = "mulq"
106 case signed && !_64:
107 op = "imull"
108 case !signed && !_64:
109 op = "mull"
110 }
111 return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
112 case signExtendData:
113 var op string
114 if i.b1 {
115 op = "cqo"
116 } else {
117 op = "cdq"
118 }
119 return op
120 case movzxRmR:
121 return fmt.Sprintf("movzx.%s %s, %s", extMode(i.u1), i.op1.format(true), i.op2.format(true))
122 case mov64MR:
123 return fmt.Sprintf("movq %s, %s", i.op1.format(true), i.op2.format(true))
124 case lea:
125 return fmt.Sprintf("lea %s, %s", i.op1.format(true), i.op2.format(true))
126 case movsxRmR:
127 return fmt.Sprintf("movsx.%s %s, %s", extMode(i.u1), i.op1.format(true), i.op2.format(true))
128 case movRM:
129 var suffix string
130 switch i.u1 {
131 case 1:
132 suffix = "b"
133 case 2:
134 suffix = "w"
135 case 4:
136 suffix = "l"
137 case 8:
138 suffix = "q"
139 }
140 return fmt.Sprintf("mov.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
141 case shiftR:
142 var suffix string
143 if i.b1 {
144 suffix = "q"
145 } else {
146 suffix = "l"
147 }
148 return fmt.Sprintf("%s%s %s, %s", shiftROp(i.u1), suffix, i.op1.format(false), i.op2.format(i.b1))
149 case xmmRmiReg:
150 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(true), i.op2.format(true))
151 case cmpRmiR:
152 var op, suffix string
153 if i.u1 != 0 {
154 op = "cmp"
155 } else {
156 op = "test"
157 }
158 if i.b1 {
159 suffix = "q"
160 } else {
161 suffix = "l"
162 }
163 if op == "test" && i.op1.kind == operandKindMem {
164 // Print consistently with AT&T syntax.
165 return fmt.Sprintf("%s%s %s, %s", op, suffix, i.op2.format(i.b1), i.op1.format(i.b1))
166 }
167 return fmt.Sprintf("%s%s %s, %s", op, suffix, i.op1.format(i.b1), i.op2.format(i.b1))
168 case setcc:
169 return fmt.Sprintf("set%s %s", cond(i.u1), i.op2.format(true))
170 case cmove:
171 var suffix string
172 if i.b1 {
173 suffix = "q"
174 } else {
175 suffix = "l"
176 }
177 return fmt.Sprintf("cmov%s%s %s, %s", cond(i.u1), suffix, i.op1.format(i.b1), i.op2.format(i.b1))
178 case push64:
179 return fmt.Sprintf("pushq %s", i.op1.format(true))
180 case pop64:
181 return fmt.Sprintf("popq %s", i.op1.format(true))
182 case xmmMovRM:
183 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(true), i.op2.format(true))
184 case xmmLoadConst:
185 panic("TODO")
186 case xmmToGpr:
187 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1))
188 case cvtUint64ToFloatSeq:
189 panic("TODO")
190 case cvtFloatToSintSeq:
191 panic("TODO")
192 case cvtFloatToUintSeq:
193 panic("TODO")
194 case xmmMinMaxSeq:
195 panic("TODO")
196 case xmmCmpRmR:
197 return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(false), i.op2.format(false))
198 case xmmRmRImm:
199 op := sseOpcode(i.u1)
200 r1, r2 := i.op1.format(op == sseOpcodePextrq || op == sseOpcodePinsrq),
201 i.op2.format(op == sseOpcodePextrq || op == sseOpcodePinsrq)
202 return fmt.Sprintf("%s $%d, %s, %s", op, i.u2, r1, r2)
203 case jmp:
204 return fmt.Sprintf("jmp %s", i.op1.format(true))
205 case jmpIf:
206 return fmt.Sprintf("j%s %s", cond(i.u1), i.op1.format(true))
207 case jmpTableIsland:
208 return fmt.Sprintf("jump_table_island: jmp_table_index=%d", i.u1)
209 case exitSequence:
210 return fmt.Sprintf("exit_sequence %s", i.op1.format(true))
211 case ud2:
212 return "ud2"
213 case call:
214 return fmt.Sprintf("call %s", ssa.FuncRef(i.u1))
215 case callIndirect:
216 return fmt.Sprintf("callq *%s", i.op1.format(true))
217 case xchg:
218 var suffix string
219 switch i.u1 {
220 case 1:
221 suffix = "b"
222 case 2:
223 suffix = "w"
224 case 4:
225 suffix = "l"
226 case 8:
227 suffix = "q"
228 }
229 return fmt.Sprintf("xchg.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
230 case zeros:
231 return fmt.Sprintf("xor %s, %s", i.op2.format(true), i.op2.format(true))
232 case fcvtToSintSequence:
233 execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData()
234 return fmt.Sprintf(
235 "fcvtToSintSequence execCtx=%s, src=%s, tmpGp=%s, tmpGp2=%s, tmpXmm=%s, src64=%v, dst64=%v, sat=%v",
236 formatVRegSized(execCtx, true),
237 formatVRegSized(src, true),
238 formatVRegSized(tmpGp, true),
239 formatVRegSized(tmpGp2, true),
240 formatVRegSized(tmpXmm, true), src64, dst64, sat)
241 case fcvtToUintSequence:
242 execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData()
243 return fmt.Sprintf(
244 "fcvtToUintSequence execCtx=%s, src=%s, tmpGp=%s, tmpGp2=%s, tmpXmm=%s, tmpXmm2=%s, src64=%v, dst64=%v, sat=%v",
245 formatVRegSized(execCtx, true),
246 formatVRegSized(src, true),
247 formatVRegSized(tmpGp, true),
248 formatVRegSized(tmpGp2, true),
249 formatVRegSized(tmpXmm, true),
250 formatVRegSized(tmpXmm2, true), src64, dst64, sat)
251 case idivRemSequence:
252 execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData()
253 return fmt.Sprintf("idivRemSequence execCtx=%s, divisor=%s, tmpGp=%s, isDiv=%v, signed=%v, _64=%v",
254 formatVRegSized(execCtx, true), formatVRegSized(divisor, _64), formatVRegSized(tmpGp, _64), isDiv, signed, _64)
255 case defineUninitializedReg:
256 return fmt.Sprintf("defineUninitializedReg %s", i.op2.format(true))
257 case xmmCMov:
258 return fmt.Sprintf("xmmcmov%s %s, %s", cond(i.u1), i.op1.format(true), i.op2.format(true))
259 case blendvpd:
260 return fmt.Sprintf("blendvpd %s, %s, %%xmm0", i.op1.format(false), i.op2.format(false))
261 case mfence:
262 return "mfence"
263 case lockcmpxchg:
264 var suffix string
265 switch i.u1 {
266 case 1:
267 suffix = "b"
268 case 2:
269 suffix = "w"
270 case 4:
271 suffix = "l"
272 case 8:
273 suffix = "q"
274 }
275 return fmt.Sprintf("lock cmpxchg.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
276 case lockxadd:
277 var suffix string
278 switch i.u1 {
279 case 1:
280 suffix = "b"
281 case 2:
282 suffix = "w"
283 case 4:
284 suffix = "l"
285 case 8:
286 suffix = "q"
287 }
288 return fmt.Sprintf("lock xadd.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
289
290 case nopUseReg:
291 return fmt.Sprintf("nop_use_reg %s", i.op1.format(true))
292
293 case tailCall:
294 return fmt.Sprintf("tailCall %s", ssa.FuncRef(i.u1))
295 case tailCallIndirect:
296 return fmt.Sprintf("tailCallIndirect %s", i.op1.format(true))
297
298 default:
299 panic(fmt.Sprintf("BUG: %d", int(i.kind)))
300 }
301 }
302
303 // Defs implements regalloc.Instr.
304 func (i *instruction) Defs(regs *[]regalloc.VReg) []regalloc.VReg {
305 *regs = (*regs)[:0]
306 switch dk := defKinds[i.kind]; dk {
307 case defKindNone:
308 case defKindOp2:
309 *regs = append(*regs, i.op2.reg())
310 case defKindCall:
311 _, _, retIntRealRegs, retFloatRealRegs, _ := backend.ABIInfoFromUint64(i.u2)
312 for i := byte(0); i < retIntRealRegs; i++ {
313 *regs = append(*regs, regInfo.RealRegToVReg[intArgResultRegs[i]])
314 }
315 for i := byte(0); i < retFloatRealRegs; i++ {
316 *regs = append(*regs, regInfo.RealRegToVReg[floatArgResultRegs[i]])
317 }
318 case defKindDivRem:
319 _, _, _, isDiv, _, _ := i.idivRemSequenceData()
320 if isDiv {
321 *regs = append(*regs, raxVReg)
322 } else {
323 *regs = append(*regs, rdxVReg)
324 }
325 default:
326 panic(fmt.Sprintf("BUG: invalid defKind \"%s\" for %s", dk, i))
327 }
328 return *regs
329 }
330
331 // Uses implements regalloc.Instr.
332 func (i *instruction) Uses(regs *[]regalloc.VReg) []regalloc.VReg {
333 *regs = (*regs)[:0]
334 switch uk := useKinds[i.kind]; uk {
335 case useKindNone:
336 case useKindOp1Op2Reg, useKindOp1RegOp2:
337 opAny, opReg := &i.op1, &i.op2
338 if uk == useKindOp1RegOp2 {
339 opAny, opReg = opReg, opAny
340 }
341 // The destination operand (op2) can be only reg,
342 // the source operand (op1) can be imm32, reg or mem.
343 switch opAny.kind {
344 case operandKindReg:
345 *regs = append(*regs, opAny.reg())
346 case operandKindMem:
347 opAny.addressMode().uses(regs)
348 case operandKindImm32:
349 default:
350 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
351 }
352 if opReg.kind != operandKindReg {
353 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
354 }
355 *regs = append(*regs, opReg.reg())
356 case useKindOp1:
357 op := i.op1
358 switch op.kind {
359 case operandKindReg:
360 *regs = append(*regs, op.reg())
361 case operandKindMem:
362 op.addressMode().uses(regs)
363 case operandKindImm32, operandKindLabel:
364 default:
365 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
366 }
367 case useKindCallInd, useKindTailCallInd:
368 op := i.op1
369 switch op.kind {
370 case operandKindReg:
371 *regs = append(*regs, op.reg())
372 case operandKindMem:
373 op.addressMode().uses(regs)
374 default:
375 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
376 }
377 fallthrough
378 case useKindCall:
379 argIntRealRegs, argFloatRealRegs, _, _, _ := backend.ABIInfoFromUint64(i.u2)
380 for i := byte(0); i < argIntRealRegs; i++ {
381 *regs = append(*regs, regInfo.RealRegToVReg[intArgResultRegs[i]])
382 }
383 for i := byte(0); i < argFloatRealRegs; i++ {
384 *regs = append(*regs, regInfo.RealRegToVReg[floatArgResultRegs[i]])
385 }
386 case useKindFcvtToSintSequence:
387 execCtx, src, tmpGp, tmpGp2, tmpXmm, _, _, _ := i.fcvtToSintSequenceData()
388 *regs = append(*regs, execCtx, src, tmpGp, tmpGp2, tmpXmm)
389 case useKindFcvtToUintSequence:
390 execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, _, _, _ := i.fcvtToUintSequenceData()
391 *regs = append(*regs, execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2)
392 case useKindDivRem:
393 execCtx, divisor, tmpGp, _, _, _ := i.idivRemSequenceData()
394 // idiv uses rax and rdx as implicit operands.
395 *regs = append(*regs, raxVReg, rdxVReg, execCtx, divisor, tmpGp)
396 case useKindBlendvpd:
397 *regs = append(*regs, xmm0VReg)
398
399 opAny, opReg := &i.op1, &i.op2
400 switch opAny.kind {
401 case operandKindReg:
402 *regs = append(*regs, opAny.reg())
403 case operandKindMem:
404 opAny.addressMode().uses(regs)
405 default:
406 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
407 }
408 if opReg.kind != operandKindReg {
409 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
410 }
411 *regs = append(*regs, opReg.reg())
412
413 case useKindRaxOp1RegOp2:
414 opReg, opAny := &i.op1, &i.op2
415 *regs = append(*regs, raxVReg, opReg.reg())
416 switch opAny.kind {
417 case operandKindReg:
418 *regs = append(*regs, opAny.reg())
419 case operandKindMem:
420 opAny.addressMode().uses(regs)
421 default:
422 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
423 }
424 if opReg.kind != operandKindReg {
425 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
426 }
427
428 default:
429 panic(fmt.Sprintf("BUG: invalid useKind %s for %s", uk, i))
430 }
431 return *regs
432 }
433
434 // AssignUse implements regalloc.Instr.
435 func (i *instruction) AssignUse(index int, v regalloc.VReg) {
436 switch uk := useKinds[i.kind]; uk {
437 case useKindNone:
438 case useKindCallInd, useKindTailCallInd:
439 if index != 0 {
440 panic("BUG")
441 }
442 op := &i.op1
443 switch op.kind {
444 case operandKindReg:
445 if uk == useKindTailCallInd && v != r11VReg {
446 panic("BUG")
447 }
448 op.setReg(v)
449 case operandKindMem:
450 op.addressMode().assignUses(index, v)
451 default:
452 panic("BUG")
453 }
454 case useKindOp1Op2Reg, useKindOp1RegOp2:
455 op, opMustBeReg := &i.op1, &i.op2
456 if uk == useKindOp1RegOp2 {
457 op, opMustBeReg = opMustBeReg, op
458 }
459 switch op.kind {
460 case operandKindReg:
461 if index == 0 {
462 op.setReg(v)
463 } else if index == 1 {
464 opMustBeReg.setReg(v)
465 } else {
466 panic("BUG")
467 }
468 case operandKindMem:
469 nregs := op.addressMode().nregs()
470 if index < nregs {
471 op.addressMode().assignUses(index, v)
472 } else if index == nregs {
473 opMustBeReg.setReg(v)
474 } else {
475 panic("BUG")
476 }
477 case operandKindImm32:
478 if index == 0 {
479 opMustBeReg.setReg(v)
480 } else {
481 panic("BUG")
482 }
483 default:
484 panic(fmt.Sprintf("BUG: invalid operand pair: %s", i))
485 }
486 case useKindOp1:
487 op := &i.op1
488 switch op.kind {
489 case operandKindReg:
490 if index != 0 {
491 panic("BUG")
492 }
493 op.setReg(v)
494 case operandKindMem:
495 op.addressMode().assignUses(index, v)
496 default:
497 panic(fmt.Sprintf("BUG: invalid operand: %s", i))
498 }
499 case useKindFcvtToSintSequence:
500 switch index {
501 case 0:
502 i.op1.addressMode().base = v
503 case 1:
504 i.op1.addressMode().index = v
505 case 2:
506 i.op2.addressMode().base = v
507 case 3:
508 i.op2.addressMode().index = v
509 case 4:
510 i.u1 = uint64(v)
511 default:
512 panic("BUG")
513 }
514 case useKindFcvtToUintSequence:
515 switch index {
516 case 0:
517 i.op1.addressMode().base = v
518 case 1:
519 i.op1.addressMode().index = v
520 case 2:
521 i.op2.addressMode().base = v
522 case 3:
523 i.op2.addressMode().index = v
524 case 4:
525 i.u1 = uint64(v)
526 case 5:
527 i.u2 = uint64(v)
528 default:
529 panic("BUG")
530 }
531 case useKindDivRem:
532 switch index {
533 case 0:
534 if v != raxVReg {
535 panic("BUG")
536 }
537 case 1:
538 if v != rdxVReg {
539 panic("BUG")
540 }
541 case 2:
542 i.op1.setReg(v)
543 case 3:
544 i.op2.setReg(v)
545 case 4:
546 i.u1 = uint64(v)
547 default:
548 panic("BUG")
549 }
550 case useKindBlendvpd:
551 op, opMustBeReg := &i.op1, &i.op2
552 if index == 0 {
553 if v.RealReg() != xmm0 {
554 panic("BUG")
555 }
556 } else {
557 switch op.kind {
558 case operandKindReg:
559 switch index {
560 case 1:
561 op.setReg(v)
562 case 2:
563 opMustBeReg.setReg(v)
564 default:
565 panic("BUG")
566 }
567 case operandKindMem:
568 nregs := op.addressMode().nregs()
569 index--
570 if index < nregs {
571 op.addressMode().assignUses(index, v)
572 } else if index == nregs {
573 opMustBeReg.setReg(v)
574 } else {
575 panic("BUG")
576 }
577 default:
578 panic(fmt.Sprintf("BUG: invalid operand pair: %s", i))
579 }
580 }
581
582 case useKindRaxOp1RegOp2:
583 switch index {
584 case 0:
585 if v.RealReg() != rax {
586 panic("BUG")
587 }
588 case 1:
589 i.op1.setReg(v)
590 default:
591 op := &i.op2
592 switch op.kind {
593 case operandKindReg:
594 switch index {
595 case 1:
596 op.setReg(v)
597 case 2:
598 op.setReg(v)
599 default:
600 panic("BUG")
601 }
602 case operandKindMem:
603 nregs := op.addressMode().nregs()
604 index -= 2
605 if index < nregs {
606 op.addressMode().assignUses(index, v)
607 } else if index == nregs {
608 op.setReg(v)
609 } else {
610 panic("BUG")
611 }
612 default:
613 panic(fmt.Sprintf("BUG: invalid operand pair: %s", i))
614 }
615 }
616 default:
617 panic(fmt.Sprintf("BUG: invalid useKind %s for %s", uk, i))
618 }
619 }
620
621 // AssignDef implements regalloc.Instr.
622 func (i *instruction) AssignDef(reg regalloc.VReg) {
623 switch dk := defKinds[i.kind]; dk {
624 case defKindNone:
625 case defKindOp2:
626 i.op2.setReg(reg)
627 default:
628 panic(fmt.Sprintf("BUG: invalid defKind \"%s\" for %s", dk, i))
629 }
630 }
631
632 // IsCopy implements regalloc.Instr.
633 func (i *instruction) IsCopy() bool {
634 k := i.kind
635 if k == movRR {
636 return true
637 }
638 if k == xmmUnaryRmR {
639 if i.op1.kind == operandKindReg {
640 sse := sseOpcode(i.u1)
641 return sse == sseOpcodeMovss || sse == sseOpcodeMovsd || sse == sseOpcodeMovdqu
642 }
643 }
644 return false
645 }
646
647 func resetInstruction(i *instruction) {
648 *i = instruction{}
649 }
650
651 func (i *instruction) asNop0WithLabel(label label) *instruction { //nolint
652 i.kind = nop0
653 i.u1 = uint64(label)
654 return i
655 }
656
657 func (i *instruction) nop0Label() label {
658 return label(i.u1)
659 }
660
661 type instructionKind byte
662
663 const (
664 nop0 instructionKind = iota + 1
665
666 // Integer arithmetic/bit-twiddling: (add sub and or xor mul, etc.) (32 64) (reg addr imm) reg
667 aluRmiR
668
669 // Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc.
670 unaryRmR
671
672 // Bitwise not
673 not
674
675 // Integer negation
676 neg
677
678 // Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
679 div
680
681 // The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs.
682 mulHi
683
684 // Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo)
685 // or al into ah: (cbw)
686 signExtendData
687
688 // Constant materialization: (imm32 imm64) reg.
689 // Either: movl $imm32, %reg32 or movabsq $imm64, %reg64.
690 imm
691
692 // GPR to GPR move: mov (64 32) reg reg.
693 movRR
694
695 // movzxRmR is zero-extended loads or move (R to R), except for 64 bits: movz (bl bq wl wq lq) addr reg.
696 // Note that the lq variant doesn't really exist since the default zero-extend rule makes it
697 // unnecessary. For that case we emit the equivalent "movl AM, reg32".
698 movzxRmR
699
700 // mov64MR is a plain 64-bit integer load, since movzxRmR can't represent that.
701 mov64MR
702
703 // Loads the memory address of addr into dst.
704 lea
705
706 // Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg.
707 movsxRmR
708
709 // Integer stores: mov (b w l q) reg addr.
710 movRM
711
712 // Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
713 shiftR
714
715 // Arithmetic SIMD shifts.
716 xmmRmiReg
717
718 // Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
719 cmpRmiR
720
721 // Materializes the requested condition code in the destination reg.
722 setcc
723
724 // Integer conditional move.
725 // Overwrites the destination register.
726 cmove
727
728 // pushq (reg addr imm)
729 push64
730
731 // popq reg
732 pop64
733
734 // XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg
735 xmmRmR
736
737 // XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg.
738 //
739 // This differs from xmmRmR in that the dst register of xmmUnaryRmR is not used in the
740 // computation of the instruction dst value and so does not have to be a previously valid
741 // value. This is characteristic of mov instructions.
742 xmmUnaryRmR
743
744 // XMM (scalar or vector) unary op with immediate: roundss, roundsd, etc.
745 //
746 // This differs from XMM_RM_R_IMM in that the dst register of
747 // XmmUnaryRmRImm is not used in the computation of the instruction dst
748 // value and so does not have to be a previously valid value.
749 xmmUnaryRmRImm
750
751 // XMM (scalar or vector) unary op (from xmm to mem): stores, movd, movq
752 xmmMovRM
753
754 // XMM (vector) unary op (to move a constant value into an xmm register): movups
755 xmmLoadConst
756
757 // XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si
758 xmmToGpr
759
760 // XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d}
761 gprToXmm
762
763 // Converts an unsigned int64 to a float32/float64.
764 cvtUint64ToFloatSeq
765
766 // Converts a scalar xmm to a signed int32/int64.
767 cvtFloatToSintSeq
768
769 // Converts a scalar xmm to an unsigned int32/int64.
770 cvtFloatToUintSeq
771
772 // A sequence to compute min/max with the proper NaN semantics for xmm registers.
773 xmmMinMaxSeq
774
775 // Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
776 xmmCmpRmR
777
778 // A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg
779 xmmRmRImm
780
781 // Direct call: call simm32.
782 // Note that the offset is the relative to the *current RIP*, which points to the first byte of the next instruction.
783 call
784
785 // Indirect call: callq (reg mem).
786 callIndirect
787
788 // Return.
789 ret
790
791 // Jump: jmp (reg, mem, imm32 or label)
792 jmp
793
794 // Jump conditionally: jcond cond label.
795 jmpIf
796
797 // jmpTableIsland is to emit the jump table.
798 jmpTableIsland
799
800 // exitSequence exits the execution and go back to the Go world.
801 exitSequence
802
803 // An instruction that will always trigger the illegal instruction exception.
804 ud2
805
806 // xchg is described in https://www.felixcloutier.com/x86/xchg.
807 // This instruction uses two operands, where one of them can be a memory address, and swaps their values.
808 // If the dst is a memory address, the execution is atomic.
809 xchg
810
811 // lockcmpxchg is the cmpxchg instruction https://www.felixcloutier.com/x86/cmpxchg with a lock prefix.
812 lockcmpxchg
813
814 // zeros puts zeros into the destination register. This is implemented as xor reg, reg for
815 // either integer or XMM registers. The reason why we have this instruction instead of using aluRmiR
816 // is that it requires the already-defined registers. From reg alloc's perspective, this defines
817 // the destination register and takes no inputs.
818 zeros
819
820 // sourceOffsetInfo is a dummy instruction to emit source offset info.
821 // The existence of this instruction does not affect the execution.
822 sourceOffsetInfo
823
824 // defineUninitializedReg is a no-op instruction that defines a register without a defining instruction.
825 defineUninitializedReg
826
827 // fcvtToSintSequence is a sequence of instructions to convert a float to a signed integer.
828 fcvtToSintSequence
829
830 // fcvtToUintSequence is a sequence of instructions to convert a float to an unsigned integer.
831 fcvtToUintSequence
832
833 // xmmCMov is a conditional move instruction for XMM registers. Lowered after register allocation.
834 xmmCMov
835
836 // idivRemSequence is a sequence of instructions to compute both the quotient and remainder of a division.
837 idivRemSequence
838
839 // blendvpd is https://www.felixcloutier.com/x86/blendvpd.
840 blendvpd
841
842 // mfence is https://www.felixcloutier.com/x86/mfence
843 mfence
844
845 // lockxadd is xadd https://www.felixcloutier.com/x86/xadd with a lock prefix.
846 lockxadd
847
848 // nopUseReg is a meta instruction that uses one register and does nothing.
849 nopUseReg
850
851 // tailCall is a meta instruction that emits a tail call.
852 tailCall
853
854 // tailCallIndirect is a meta instruction that emits a tail call with an indirect call.
855 tailCallIndirect
856
857 instrMax
858 )
859
860 func (i *instruction) asMFence() *instruction {
861 i.kind = mfence
862 return i
863 }
864
865 func (i *instruction) asNopUseReg(r regalloc.VReg) *instruction {
866 i.kind = nopUseReg
867 i.op1 = newOperandReg(r)
868 return i
869 }
870
871 func (i *instruction) asIdivRemSequence(execCtx, divisor, tmpGp regalloc.VReg, isDiv, signed, _64 bool) *instruction {
872 i.kind = idivRemSequence
873 i.op1 = newOperandReg(execCtx)
874 i.op2 = newOperandReg(divisor)
875 i.u1 = uint64(tmpGp)
876 if isDiv {
877 i.u2 |= 1
878 }
879 if signed {
880 i.u2 |= 2
881 }
882 if _64 {
883 i.u2 |= 4
884 }
885 return i
886 }
887
888 func (i *instruction) idivRemSequenceData() (
889 execCtx, divisor, tmpGp regalloc.VReg, isDiv, signed, _64 bool,
890 ) {
891 if i.kind != idivRemSequence {
892 panic("BUG")
893 }
894 return i.op1.reg(), i.op2.reg(), regalloc.VReg(i.u1), i.u2&1 != 0, i.u2&2 != 0, i.u2&4 != 0
895 }
896
897 func (i *instruction) asXmmCMov(cc cond, x operand, rd regalloc.VReg, size byte) *instruction {
898 i.kind = xmmCMov
899 i.op1 = x
900 i.op2 = newOperandReg(rd)
901 i.u1 = uint64(cc)
902 i.u2 = uint64(size)
903 return i
904 }
905
906 func (i *instruction) asDefineUninitializedReg(r regalloc.VReg) *instruction {
907 i.kind = defineUninitializedReg
908 i.op2 = newOperandReg(r)
909 return i
910 }
911
912 func (m *machine) allocateFcvtToUintSequence(
913 execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2 regalloc.VReg,
914 src64, dst64, sat bool,
915 ) *instruction {
916 i := m.allocateInstr()
917 i.kind = fcvtToUintSequence
918 op1a := m.amodePool.Allocate()
919 op2a := m.amodePool.Allocate()
920 i.op1 = newOperandMem(op1a)
921 i.op2 = newOperandMem(op2a)
922 if src64 {
923 op1a.imm32 = 1
924 } else {
925 op1a.imm32 = 0
926 }
927 if dst64 {
928 op1a.imm32 |= 2
929 }
930 if sat {
931 op1a.imm32 |= 4
932 }
933
934 op1a.base = execCtx
935 op1a.index = src
936 op2a.base = tmpGp
937 op2a.index = tmpGp2
938 i.u1 = uint64(tmpXmm)
939 i.u2 = uint64(tmpXmm2)
940 return i
941 }
942
943 func (i *instruction) fcvtToUintSequenceData() (
944 execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2 regalloc.VReg, src64, dst64, sat bool,
945 ) {
946 if i.kind != fcvtToUintSequence {
947 panic("BUG")
948 }
949 op1a := i.op1.addressMode()
950 op2a := i.op2.addressMode()
951 return op1a.base, op1a.index, op2a.base, op2a.index, regalloc.VReg(i.u1), regalloc.VReg(i.u2),
952 op1a.imm32&1 != 0, op1a.imm32&2 != 0, op1a.imm32&4 != 0
953 }
954
955 func (m *machine) allocateFcvtToSintSequence(
956 execCtx, src, tmpGp, tmpGp2, tmpXmm regalloc.VReg,
957 src64, dst64, sat bool,
958 ) *instruction {
959 i := m.allocateInstr()
960 i.kind = fcvtToSintSequence
961 op1a := m.amodePool.Allocate()
962 op2a := m.amodePool.Allocate()
963 i.op1 = newOperandMem(op1a)
964 i.op2 = newOperandMem(op2a)
965 op1a.base = execCtx
966 op1a.index = src
967 op2a.base = tmpGp
968 op2a.index = tmpGp2
969 i.u1 = uint64(tmpXmm)
970 if src64 {
971 i.u2 = 1
972 } else {
973 i.u2 = 0
974 }
975 if dst64 {
976 i.u2 |= 2
977 }
978 if sat {
979 i.u2 |= 4
980 }
981 return i
982 }
983
984 func (i *instruction) fcvtToSintSequenceData() (
985 execCtx, src, tmpGp, tmpGp2, tmpXmm regalloc.VReg, src64, dst64, sat bool,
986 ) {
987 if i.kind != fcvtToSintSequence {
988 panic("BUG")
989 }
990 op1a := i.op1.addressMode()
991 op2a := i.op2.addressMode()
992 return op1a.base, op1a.index, op2a.base, op2a.index, regalloc.VReg(i.u1),
993 i.u2&1 != 0, i.u2&2 != 0, i.u2&4 != 0
994 }
995
996 func (k instructionKind) String() string {
997 switch k {
998 case nop0:
999 return "nop"
1000 case ret:
1001 return "ret"
1002 case imm:
1003 return "imm"
1004 case aluRmiR:
1005 return "aluRmiR"
1006 case movRR:
1007 return "movRR"
1008 case xmmRmR:
1009 return "xmmRmR"
1010 case gprToXmm:
1011 return "gprToXmm"
1012 case xmmUnaryRmR:
1013 return "xmmUnaryRmR"
1014 case xmmUnaryRmRImm:
1015 return "xmmUnaryRmRImm"
1016 case unaryRmR:
1017 return "unaryRmR"
1018 case not:
1019 return "not"
1020 case neg:
1021 return "neg"
1022 case div:
1023 return "div"
1024 case mulHi:
1025 return "mulHi"
1026 case signExtendData:
1027 return "signExtendData"
1028 case movzxRmR:
1029 return "movzxRmR"
1030 case mov64MR:
1031 return "mov64MR"
1032 case lea:
1033 return "lea"
1034 case movsxRmR:
1035 return "movsxRmR"
1036 case movRM:
1037 return "movRM"
1038 case shiftR:
1039 return "shiftR"
1040 case xmmRmiReg:
1041 return "xmmRmiReg"
1042 case cmpRmiR:
1043 return "cmpRmiR"
1044 case setcc:
1045 return "setcc"
1046 case cmove:
1047 return "cmove"
1048 case push64:
1049 return "push64"
1050 case pop64:
1051 return "pop64"
1052 case xmmMovRM:
1053 return "xmmMovRM"
1054 case xmmLoadConst:
1055 return "xmmLoadConst"
1056 case xmmToGpr:
1057 return "xmmToGpr"
1058 case cvtUint64ToFloatSeq:
1059 return "cvtUint64ToFloatSeq"
1060 case cvtFloatToSintSeq:
1061 return "cvtFloatToSintSeq"
1062 case cvtFloatToUintSeq:
1063 return "cvtFloatToUintSeq"
1064 case xmmMinMaxSeq:
1065 return "xmmMinMaxSeq"
1066 case xmmCmpRmR:
1067 return "xmmCmpRmR"
1068 case xmmRmRImm:
1069 return "xmmRmRImm"
1070 case jmpIf:
1071 return "jmpIf"
1072 case jmp:
1073 return "jmp"
1074 case jmpTableIsland:
1075 return "jmpTableIsland"
1076 case exitSequence:
1077 return "exit_sequence"
1078 case ud2:
1079 return "ud2"
1080 case xchg:
1081 return "xchg"
1082 case zeros:
1083 return "zeros"
1084 case fcvtToSintSequence:
1085 return "fcvtToSintSequence"
1086 case fcvtToUintSequence:
1087 return "fcvtToUintSequence"
1088 case xmmCMov:
1089 return "xmmCMov"
1090 case idivRemSequence:
1091 return "idivRemSequence"
1092 case mfence:
1093 return "mfence"
1094 case lockcmpxchg:
1095 return "lockcmpxchg"
1096 case lockxadd:
1097 return "lockxadd"
1098 case tailCall:
1099 return "tailCall"
1100 case tailCallIndirect:
1101 return "tailCallIndirect"
1102 default:
1103 panic("BUG")
1104 }
1105 }
1106
1107 type aluRmiROpcode byte
1108
1109 const (
1110 aluRmiROpcodeAdd aluRmiROpcode = iota + 1
1111 aluRmiROpcodeSub
1112 aluRmiROpcodeAnd
1113 aluRmiROpcodeOr
1114 aluRmiROpcodeXor
1115 aluRmiROpcodeMul
1116 )
1117
1118 func (a aluRmiROpcode) String() string {
1119 switch a {
1120 case aluRmiROpcodeAdd:
1121 return "add"
1122 case aluRmiROpcodeSub:
1123 return "sub"
1124 case aluRmiROpcodeAnd:
1125 return "and"
1126 case aluRmiROpcodeOr:
1127 return "or"
1128 case aluRmiROpcodeXor:
1129 return "xor"
1130 case aluRmiROpcodeMul:
1131 return "imul"
1132 default:
1133 panic("BUG")
1134 }
1135 }
1136
1137 func (i *instruction) asJmpIf(cond cond, target operand) *instruction {
1138 i.kind = jmpIf
1139 i.u1 = uint64(cond)
1140 i.op1 = target
1141 return i
1142 }
1143
1144 // asJmpTableSequence is used to emit the jump table.
1145 // targetSliceIndex is the index of the target slice in machine.jmpTableTargets.
1146 func (i *instruction) asJmpTableSequence(targetSliceIndex int, targetCount int) *instruction {
1147 i.kind = jmpTableIsland
1148 i.u1 = uint64(targetSliceIndex)
1149 i.u2 = uint64(targetCount)
1150 return i
1151 }
1152
1153 func (i *instruction) asJmp(target operand) *instruction {
1154 i.kind = jmp
1155 i.op1 = target
1156 return i
1157 }
1158
1159 func (i *instruction) jmpLabel() label {
1160 switch i.kind {
1161 case jmp, jmpIf, lea, xmmUnaryRmR:
1162 return i.op1.label()
1163 default:
1164 panic("BUG")
1165 }
1166 }
1167
1168 func (i *instruction) asLEA(target operand, rd regalloc.VReg) *instruction {
1169 i.kind = lea
1170 i.op1 = target
1171 i.op2 = newOperandReg(rd)
1172 return i
1173 }
1174
1175 func (i *instruction) asCall(ref ssa.FuncRef, abi *backend.FunctionABI) *instruction {
1176 i.kind = call
1177 i.u1 = uint64(ref)
1178 if abi != nil {
1179 i.u2 = abi.ABIInfoAsUint64()
1180 }
1181 return i
1182 }
1183
1184 func (i *instruction) asCallIndirect(ptr operand, abi *backend.FunctionABI) *instruction {
1185 if ptr.kind != operandKindReg && ptr.kind != operandKindMem {
1186 panic("BUG")
1187 }
1188 i.kind = callIndirect
1189 i.op1 = ptr
1190 if abi != nil {
1191 i.u2 = abi.ABIInfoAsUint64()
1192 }
1193 return i
1194 }
1195
1196 func (i *instruction) asTailCallReturnCall(ref ssa.FuncRef, abi *backend.FunctionABI) *instruction {
1197 i.kind = tailCall
1198 i.u1 = uint64(ref)
1199 if abi != nil {
1200 i.u2 = abi.ABIInfoAsUint64()
1201 }
1202 return i
1203 }
1204
1205 func (i *instruction) asTailCallReturnCallIndirect(ptr operand, abi *backend.FunctionABI) *instruction {
1206 if ptr.kind != operandKindReg && ptr.kind != operandKindMem {
1207 panic("BUG")
1208 }
1209 i.kind = tailCallIndirect
1210 i.op1 = ptr
1211 if abi != nil {
1212 i.u2 = abi.ABIInfoAsUint64()
1213 }
1214 return i
1215 }
1216
1217 func (i *instruction) asRet() *instruction {
1218 i.kind = ret
1219 return i
1220 }
1221
1222 func (i *instruction) asImm(dst regalloc.VReg, value uint64, _64 bool) *instruction {
1223 i.kind = imm
1224 i.op2 = newOperandReg(dst)
1225 i.u1 = value
1226 i.b1 = _64
1227 return i
1228 }
1229
1230 func (i *instruction) asAluRmiR(op aluRmiROpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
1231 if rm.kind != operandKindReg && rm.kind != operandKindMem && rm.kind != operandKindImm32 {
1232 panic("BUG")
1233 }
1234 i.kind = aluRmiR
1235 i.op1 = rm
1236 i.op2 = newOperandReg(rd)
1237 i.u1 = uint64(op)
1238 i.b1 = _64
1239 return i
1240 }
1241
1242 func (i *instruction) asZeros(dst regalloc.VReg) *instruction {
1243 i.kind = zeros
1244 i.op2 = newOperandReg(dst)
1245 return i
1246 }
1247
1248 func (i *instruction) asBlendvpd(rm operand, rd regalloc.VReg) *instruction {
1249 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1250 panic("BUG")
1251 }
1252 i.kind = blendvpd
1253 i.op1 = rm
1254 i.op2 = newOperandReg(rd)
1255 return i
1256 }
1257
1258 func (i *instruction) asXmmRmR(op sseOpcode, rm operand, rd regalloc.VReg) *instruction {
1259 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1260 panic("BUG")
1261 }
1262 i.kind = xmmRmR
1263 i.op1 = rm
1264 i.op2 = newOperandReg(rd)
1265 i.u1 = uint64(op)
1266 return i
1267 }
1268
1269 func (i *instruction) asXmmRmRImm(op sseOpcode, imm uint8, rm operand, rd regalloc.VReg) *instruction {
1270 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1271 panic("BUG")
1272 }
1273 i.kind = xmmRmRImm
1274 i.op1 = rm
1275 i.op2 = newOperandReg(rd)
1276 i.u1 = uint64(op)
1277 i.u2 = uint64(imm)
1278 return i
1279 }
1280
1281 func (i *instruction) asGprToXmm(op sseOpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
1282 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1283 panic("BUG")
1284 }
1285 i.kind = gprToXmm
1286 i.op1 = rm
1287 i.op2 = newOperandReg(rd)
1288 i.u1 = uint64(op)
1289 i.b1 = _64
1290 return i
1291 }
1292
1293 func (i *instruction) asEmitSourceOffsetInfo(l ssa.SourceOffset) *instruction {
1294 i.kind = sourceOffsetInfo
1295 i.u1 = uint64(l)
1296 return i
1297 }
1298
1299 func (i *instruction) sourceOffsetInfo() ssa.SourceOffset {
1300 return ssa.SourceOffset(i.u1)
1301 }
1302
1303 func (i *instruction) asXmmToGpr(op sseOpcode, rm, rd regalloc.VReg, _64 bool) *instruction {
1304 i.kind = xmmToGpr
1305 i.op1 = newOperandReg(rm)
1306 i.op2 = newOperandReg(rd)
1307 i.u1 = uint64(op)
1308 i.b1 = _64
1309 return i
1310 }
1311
1312 func (i *instruction) asMovRM(rm regalloc.VReg, rd operand, size byte) *instruction {
1313 if rd.kind != operandKindMem {
1314 panic("BUG")
1315 }
1316 i.kind = movRM
1317 i.op1 = newOperandReg(rm)
1318 i.op2 = rd
1319 i.u1 = uint64(size)
1320 return i
1321 }
1322
1323 func (i *instruction) asMovsxRmR(ext extMode, src operand, rd regalloc.VReg) *instruction {
1324 if src.kind != operandKindReg && src.kind != operandKindMem {
1325 panic("BUG")
1326 }
1327 i.kind = movsxRmR
1328 i.op1 = src
1329 i.op2 = newOperandReg(rd)
1330 i.u1 = uint64(ext)
1331 return i
1332 }
1333
1334 func (i *instruction) asMovzxRmR(ext extMode, src operand, rd regalloc.VReg) *instruction {
1335 if src.kind != operandKindReg && src.kind != operandKindMem {
1336 panic("BUG")
1337 }
1338 i.kind = movzxRmR
1339 i.op1 = src
1340 i.op2 = newOperandReg(rd)
1341 i.u1 = uint64(ext)
1342 return i
1343 }
1344
1345 func (i *instruction) asSignExtendData(_64 bool) *instruction {
1346 i.kind = signExtendData
1347 i.b1 = _64
1348 return i
1349 }
1350
1351 func (i *instruction) asUD2() *instruction {
1352 i.kind = ud2
1353 return i
1354 }
1355
1356 func (i *instruction) asDiv(rn operand, signed bool, _64 bool) *instruction {
1357 i.kind = div
1358 i.op1 = rn
1359 i.b1 = _64
1360 if signed {
1361 i.u1 = 1
1362 }
1363 return i
1364 }
1365
1366 func (i *instruction) asMov64MR(rm operand, rd regalloc.VReg) *instruction {
1367 if rm.kind != operandKindMem {
1368 panic("BUG")
1369 }
1370 i.kind = mov64MR
1371 i.op1 = rm
1372 i.op2 = newOperandReg(rd)
1373 return i
1374 }
1375
1376 func (i *instruction) asMovRR(rm, rd regalloc.VReg, _64 bool) *instruction {
1377 i.kind = movRR
1378 i.op1 = newOperandReg(rm)
1379 i.op2 = newOperandReg(rd)
1380 i.b1 = _64
1381 return i
1382 }
1383
1384 func (i *instruction) asNot(rm operand, _64 bool) *instruction {
1385 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1386 panic("BUG")
1387 }
1388 i.kind = not
1389 i.op1 = rm
1390 i.b1 = _64
1391 return i
1392 }
1393
1394 func (i *instruction) asNeg(rm operand, _64 bool) *instruction {
1395 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1396 panic("BUG")
1397 }
1398 i.kind = neg
1399 i.op1 = rm
1400 i.b1 = _64
1401 return i
1402 }
1403
1404 func (i *instruction) asMulHi(rm operand, signed, _64 bool) *instruction {
1405 if rm.kind != operandKindReg && (rm.kind != operandKindMem) {
1406 panic("BUG")
1407 }
1408 i.kind = mulHi
1409 i.op1 = rm
1410 i.b1 = _64
1411 if signed {
1412 i.u1 = 1
1413 }
1414 return i
1415 }
1416
1417 func (i *instruction) asUnaryRmR(op unaryRmROpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
1418 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1419 panic("BUG")
1420 }
1421 i.kind = unaryRmR
1422 i.op1 = rm
1423 i.op2 = newOperandReg(rd)
1424 i.u1 = uint64(op)
1425 i.b1 = _64
1426 return i
1427 }
1428
1429 func (i *instruction) asShiftR(op shiftROp, amount operand, rd regalloc.VReg, _64 bool) *instruction {
1430 if amount.kind != operandKindReg && amount.kind != operandKindImm32 {
1431 panic("BUG")
1432 }
1433 i.kind = shiftR
1434 i.op1 = amount
1435 i.op2 = newOperandReg(rd)
1436 i.u1 = uint64(op)
1437 i.b1 = _64
1438 return i
1439 }
1440
1441 func (i *instruction) asXmmRmiReg(op sseOpcode, rm operand, rd regalloc.VReg) *instruction {
1442 if rm.kind != operandKindReg && rm.kind != operandKindImm32 && rm.kind != operandKindMem {
1443 panic("BUG")
1444 }
1445 i.kind = xmmRmiReg
1446 i.op1 = rm
1447 i.op2 = newOperandReg(rd)
1448 i.u1 = uint64(op)
1449 return i
1450 }
1451
1452 func (i *instruction) asCmpRmiR(cmp bool, rm operand, rn regalloc.VReg, _64 bool) *instruction {
1453 if rm.kind != operandKindReg && rm.kind != operandKindImm32 && rm.kind != operandKindMem {
1454 panic("BUG")
1455 }
1456 i.kind = cmpRmiR
1457 i.op1 = rm
1458 i.op2 = newOperandReg(rn)
1459 if cmp {
1460 i.u1 = 1
1461 }
1462 i.b1 = _64
1463 return i
1464 }
1465
1466 func (i *instruction) asSetcc(c cond, rd regalloc.VReg) *instruction {
1467 i.kind = setcc
1468 i.op2 = newOperandReg(rd)
1469 i.u1 = uint64(c)
1470 return i
1471 }
1472
1473 func (i *instruction) asCmove(c cond, rm operand, rd regalloc.VReg, _64 bool) *instruction {
1474 i.kind = cmove
1475 i.op1 = rm
1476 i.op2 = newOperandReg(rd)
1477 i.u1 = uint64(c)
1478 i.b1 = _64
1479 return i
1480 }
1481
1482 func (m *machine) allocateExitSeq(execCtx regalloc.VReg) *instruction {
1483 i := m.allocateInstr()
1484 i.kind = exitSequence
1485 i.op1 = newOperandReg(execCtx)
1486 // Allocate the address mode that will be used in encoding the exit sequence.
1487 i.op2 = newOperandMem(m.amodePool.Allocate())
1488 return i
1489 }
1490
1491 func (i *instruction) asXmmUnaryRmR(op sseOpcode, rm operand, rd regalloc.VReg) *instruction {
1492 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1493 panic("BUG")
1494 }
1495 i.kind = xmmUnaryRmR
1496 i.op1 = rm
1497 i.op2 = newOperandReg(rd)
1498 i.u1 = uint64(op)
1499 return i
1500 }
1501
1502 func (i *instruction) asXmmUnaryRmRImm(op sseOpcode, imm byte, rm operand, rd regalloc.VReg) *instruction {
1503 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1504 panic("BUG")
1505 }
1506 i.kind = xmmUnaryRmRImm
1507 i.op1 = rm
1508 i.op2 = newOperandReg(rd)
1509 i.u1 = uint64(op)
1510 i.u2 = uint64(imm)
1511 return i
1512 }
1513
1514 func (i *instruction) asXmmCmpRmR(op sseOpcode, rm operand, rd regalloc.VReg) *instruction {
1515 if rm.kind != operandKindReg && rm.kind != operandKindMem {
1516 panic("BUG")
1517 }
1518 i.kind = xmmCmpRmR
1519 i.op1 = rm
1520 i.op2 = newOperandReg(rd)
1521 i.u1 = uint64(op)
1522 return i
1523 }
1524
1525 func (i *instruction) asXmmMovRM(op sseOpcode, rm regalloc.VReg, rd operand) *instruction {
1526 if rd.kind != operandKindMem {
1527 panic("BUG")
1528 }
1529 i.kind = xmmMovRM
1530 i.op1 = newOperandReg(rm)
1531 i.op2 = rd
1532 i.u1 = uint64(op)
1533 return i
1534 }
1535
1536 func (i *instruction) asPop64(rm regalloc.VReg) *instruction {
1537 i.kind = pop64
1538 i.op1 = newOperandReg(rm)
1539 return i
1540 }
1541
1542 func (i *instruction) asPush64(op operand) *instruction {
1543 if op.kind != operandKindReg && op.kind != operandKindMem && op.kind != operandKindImm32 {
1544 panic("BUG")
1545 }
1546 i.kind = push64
1547 i.op1 = op
1548 return i
1549 }
1550
1551 func (i *instruction) asXCHG(rm regalloc.VReg, rd operand, size byte) *instruction {
1552 i.kind = xchg
1553 i.op1 = newOperandReg(rm)
1554 i.op2 = rd
1555 i.u1 = uint64(size)
1556 return i
1557 }
1558
1559 func (i *instruction) asLockCmpXCHG(rm regalloc.VReg, rd *amode, size byte) *instruction {
1560 i.kind = lockcmpxchg
1561 i.op1 = newOperandReg(rm)
1562 i.op2 = newOperandMem(rd)
1563 i.u1 = uint64(size)
1564 return i
1565 }
1566
1567 func (i *instruction) asLockXAdd(rm regalloc.VReg, rd *amode, size byte) *instruction {
1568 i.kind = lockxadd
1569 i.op1 = newOperandReg(rm)
1570 i.op2 = newOperandMem(rd)
1571 i.u1 = uint64(size)
1572 return i
1573 }
1574
1575 type unaryRmROpcode byte
1576
1577 const (
1578 unaryRmROpcodeBsr unaryRmROpcode = iota
1579 unaryRmROpcodeBsf
1580 unaryRmROpcodeLzcnt
1581 unaryRmROpcodeTzcnt
1582 unaryRmROpcodePopcnt
1583 )
1584
1585 func (u unaryRmROpcode) String() string {
1586 switch u {
1587 case unaryRmROpcodeBsr:
1588 return "bsr"
1589 case unaryRmROpcodeBsf:
1590 return "bsf"
1591 case unaryRmROpcodeLzcnt:
1592 return "lzcnt"
1593 case unaryRmROpcodeTzcnt:
1594 return "tzcnt"
1595 case unaryRmROpcodePopcnt:
1596 return "popcnt"
1597 default:
1598 panic("BUG")
1599 }
1600 }
1601
1602 type shiftROp byte
1603
1604 const (
1605 shiftROpRotateLeft shiftROp = 0
1606 shiftROpRotateRight shiftROp = 1
1607 shiftROpShiftLeft shiftROp = 4
1608 shiftROpShiftRightLogical shiftROp = 5
1609 shiftROpShiftRightArithmetic shiftROp = 7
1610 )
1611
1612 func (s shiftROp) String() string {
1613 switch s {
1614 case shiftROpRotateLeft:
1615 return "rol"
1616 case shiftROpRotateRight:
1617 return "ror"
1618 case shiftROpShiftLeft:
1619 return "shl"
1620 case shiftROpShiftRightLogical:
1621 return "shr"
1622 case shiftROpShiftRightArithmetic:
1623 return "sar"
1624 default:
1625 panic("BUG")
1626 }
1627 }
1628
1629 type sseOpcode byte
1630
1631 const (
1632 sseOpcodeInvalid sseOpcode = iota
1633 sseOpcodeAddps
1634 sseOpcodeAddpd
1635 sseOpcodeAddss
1636 sseOpcodeAddsd
1637 sseOpcodeAndps
1638 sseOpcodeAndpd
1639 sseOpcodeAndnps
1640 sseOpcodeAndnpd
1641 sseOpcodeBlendvps
1642 sseOpcodeBlendvpd
1643 sseOpcodeComiss
1644 sseOpcodeComisd
1645 sseOpcodeCmpps
1646 sseOpcodeCmppd
1647 sseOpcodeCmpss
1648 sseOpcodeCmpsd
1649 sseOpcodeCvtdq2ps
1650 sseOpcodeCvtdq2pd
1651 sseOpcodeCvtsd2ss
1652 sseOpcodeCvtsd2si
1653 sseOpcodeCvtsi2ss
1654 sseOpcodeCvtsi2sd
1655 sseOpcodeCvtss2si
1656 sseOpcodeCvtss2sd
1657 sseOpcodeCvttps2dq
1658 sseOpcodeCvttss2si
1659 sseOpcodeCvttsd2si
1660 sseOpcodeDivps
1661 sseOpcodeDivpd
1662 sseOpcodeDivss
1663 sseOpcodeDivsd
1664 sseOpcodeInsertps
1665 sseOpcodeMaxps
1666 sseOpcodeMaxpd
1667 sseOpcodeMaxss
1668 sseOpcodeMaxsd
1669 sseOpcodeMinps
1670 sseOpcodeMinpd
1671 sseOpcodeMinss
1672 sseOpcodeMinsd
1673 sseOpcodeMovaps
1674 sseOpcodeMovapd
1675 sseOpcodeMovd
1676 sseOpcodeMovdqa
1677 sseOpcodeMovdqu
1678 sseOpcodeMovlhps
1679 sseOpcodeMovmskps
1680 sseOpcodeMovmskpd
1681 sseOpcodeMovq
1682 sseOpcodeMovss
1683 sseOpcodeMovsd
1684 sseOpcodeMovups
1685 sseOpcodeMovupd
1686 sseOpcodeMulps
1687 sseOpcodeMulpd
1688 sseOpcodeMulss
1689 sseOpcodeMulsd
1690 sseOpcodeOrps
1691 sseOpcodeOrpd
1692 sseOpcodePabsb
1693 sseOpcodePabsw
1694 sseOpcodePabsd
1695 sseOpcodePackssdw
1696 sseOpcodePacksswb
1697 sseOpcodePackusdw
1698 sseOpcodePackuswb
1699 sseOpcodePaddb
1700 sseOpcodePaddd
1701 sseOpcodePaddq
1702 sseOpcodePaddw
1703 sseOpcodePaddsb
1704 sseOpcodePaddsw
1705 sseOpcodePaddusb
1706 sseOpcodePaddusw
1707 sseOpcodePalignr
1708 sseOpcodePand
1709 sseOpcodePandn
1710 sseOpcodePavgb
1711 sseOpcodePavgw
1712 sseOpcodePcmpeqb
1713 sseOpcodePcmpeqw
1714 sseOpcodePcmpeqd
1715 sseOpcodePcmpeqq
1716 sseOpcodePcmpgtb
1717 sseOpcodePcmpgtw
1718 sseOpcodePcmpgtd
1719 sseOpcodePcmpgtq
1720 sseOpcodePextrb
1721 sseOpcodePextrw
1722 sseOpcodePextrd
1723 sseOpcodePextrq
1724 sseOpcodePinsrb
1725 sseOpcodePinsrw
1726 sseOpcodePinsrd
1727 sseOpcodePinsrq
1728 sseOpcodePmaddwd
1729 sseOpcodePmaxsb
1730 sseOpcodePmaxsw
1731 sseOpcodePmaxsd
1732 sseOpcodePmaxub
1733 sseOpcodePmaxuw
1734 sseOpcodePmaxud
1735 sseOpcodePminsb
1736 sseOpcodePminsw
1737 sseOpcodePminsd
1738 sseOpcodePminub
1739 sseOpcodePminuw
1740 sseOpcodePminud
1741 sseOpcodePmovmskb
1742 sseOpcodePmovsxbd
1743 sseOpcodePmovsxbw
1744 sseOpcodePmovsxbq
1745 sseOpcodePmovsxwd
1746 sseOpcodePmovsxwq
1747 sseOpcodePmovsxdq
1748 sseOpcodePmovzxbd
1749 sseOpcodePmovzxbw
1750 sseOpcodePmovzxbq
1751 sseOpcodePmovzxwd
1752 sseOpcodePmovzxwq
1753 sseOpcodePmovzxdq
1754 sseOpcodePmulld
1755 sseOpcodePmullw
1756 sseOpcodePmuludq
1757 sseOpcodePor
1758 sseOpcodePshufb
1759 sseOpcodePshufd
1760 sseOpcodePsllw
1761 sseOpcodePslld
1762 sseOpcodePsllq
1763 sseOpcodePsraw
1764 sseOpcodePsrad
1765 sseOpcodePsrlw
1766 sseOpcodePsrld
1767 sseOpcodePsrlq
1768 sseOpcodePsubb
1769 sseOpcodePsubd
1770 sseOpcodePsubq
1771 sseOpcodePsubw
1772 sseOpcodePsubsb
1773 sseOpcodePsubsw
1774 sseOpcodePsubusb
1775 sseOpcodePsubusw
1776 sseOpcodePtest
1777 sseOpcodePunpckhbw
1778 sseOpcodePunpcklbw
1779 sseOpcodePxor
1780 sseOpcodeRcpss
1781 sseOpcodeRoundps
1782 sseOpcodeRoundpd
1783 sseOpcodeRoundss
1784 sseOpcodeRoundsd
1785 sseOpcodeRsqrtss
1786 sseOpcodeSqrtps
1787 sseOpcodeSqrtpd
1788 sseOpcodeSqrtss
1789 sseOpcodeSqrtsd
1790 sseOpcodeSubps
1791 sseOpcodeSubpd
1792 sseOpcodeSubss
1793 sseOpcodeSubsd
1794 sseOpcodeUcomiss
1795 sseOpcodeUcomisd
1796 sseOpcodeXorps
1797 sseOpcodeXorpd
1798 sseOpcodePmulhrsw
1799 sseOpcodeUnpcklps
1800 sseOpcodeCvtps2pd
1801 sseOpcodeCvtpd2ps
1802 sseOpcodeCvttpd2dq
1803 sseOpcodeShufps
1804 sseOpcodePmaddubsw
1805 )
1806
1807 func (s sseOpcode) String() string {
1808 switch s {
1809 case sseOpcodeInvalid:
1810 return "invalid"
1811 case sseOpcodeAddps:
1812 return "addps"
1813 case sseOpcodeAddpd:
1814 return "addpd"
1815 case sseOpcodeAddss:
1816 return "addss"
1817 case sseOpcodeAddsd:
1818 return "addsd"
1819 case sseOpcodeAndps:
1820 return "andps"
1821 case sseOpcodeAndpd:
1822 return "andpd"
1823 case sseOpcodeAndnps:
1824 return "andnps"
1825 case sseOpcodeAndnpd:
1826 return "andnpd"
1827 case sseOpcodeBlendvps:
1828 return "blendvps"
1829 case sseOpcodeBlendvpd:
1830 return "blendvpd"
1831 case sseOpcodeComiss:
1832 return "comiss"
1833 case sseOpcodeComisd:
1834 return "comisd"
1835 case sseOpcodeCmpps:
1836 return "cmpps"
1837 case sseOpcodeCmppd:
1838 return "cmppd"
1839 case sseOpcodeCmpss:
1840 return "cmpss"
1841 case sseOpcodeCmpsd:
1842 return "cmpsd"
1843 case sseOpcodeCvtdq2ps:
1844 return "cvtdq2ps"
1845 case sseOpcodeCvtdq2pd:
1846 return "cvtdq2pd"
1847 case sseOpcodeCvtsd2ss:
1848 return "cvtsd2ss"
1849 case sseOpcodeCvtsd2si:
1850 return "cvtsd2si"
1851 case sseOpcodeCvtsi2ss:
1852 return "cvtsi2ss"
1853 case sseOpcodeCvtsi2sd:
1854 return "cvtsi2sd"
1855 case sseOpcodeCvtss2si:
1856 return "cvtss2si"
1857 case sseOpcodeCvtss2sd:
1858 return "cvtss2sd"
1859 case sseOpcodeCvttps2dq:
1860 return "cvttps2dq"
1861 case sseOpcodeCvttss2si:
1862 return "cvttss2si"
1863 case sseOpcodeCvttsd2si:
1864 return "cvttsd2si"
1865 case sseOpcodeDivps:
1866 return "divps"
1867 case sseOpcodeDivpd:
1868 return "divpd"
1869 case sseOpcodeDivss:
1870 return "divss"
1871 case sseOpcodeDivsd:
1872 return "divsd"
1873 case sseOpcodeInsertps:
1874 return "insertps"
1875 case sseOpcodeMaxps:
1876 return "maxps"
1877 case sseOpcodeMaxpd:
1878 return "maxpd"
1879 case sseOpcodeMaxss:
1880 return "maxss"
1881 case sseOpcodeMaxsd:
1882 return "maxsd"
1883 case sseOpcodeMinps:
1884 return "minps"
1885 case sseOpcodeMinpd:
1886 return "minpd"
1887 case sseOpcodeMinss:
1888 return "minss"
1889 case sseOpcodeMinsd:
1890 return "minsd"
1891 case sseOpcodeMovaps:
1892 return "movaps"
1893 case sseOpcodeMovapd:
1894 return "movapd"
1895 case sseOpcodeMovd:
1896 return "movd"
1897 case sseOpcodeMovdqa:
1898 return "movdqa"
1899 case sseOpcodeMovdqu:
1900 return "movdqu"
1901 case sseOpcodeMovlhps:
1902 return "movlhps"
1903 case sseOpcodeMovmskps:
1904 return "movmskps"
1905 case sseOpcodeMovmskpd:
1906 return "movmskpd"
1907 case sseOpcodeMovq:
1908 return "movq"
1909 case sseOpcodeMovss:
1910 return "movss"
1911 case sseOpcodeMovsd:
1912 return "movsd"
1913 case sseOpcodeMovups:
1914 return "movups"
1915 case sseOpcodeMovupd:
1916 return "movupd"
1917 case sseOpcodeMulps:
1918 return "mulps"
1919 case sseOpcodeMulpd:
1920 return "mulpd"
1921 case sseOpcodeMulss:
1922 return "mulss"
1923 case sseOpcodeMulsd:
1924 return "mulsd"
1925 case sseOpcodeOrps:
1926 return "orps"
1927 case sseOpcodeOrpd:
1928 return "orpd"
1929 case sseOpcodePabsb:
1930 return "pabsb"
1931 case sseOpcodePabsw:
1932 return "pabsw"
1933 case sseOpcodePabsd:
1934 return "pabsd"
1935 case sseOpcodePackssdw:
1936 return "packssdw"
1937 case sseOpcodePacksswb:
1938 return "packsswb"
1939 case sseOpcodePackusdw:
1940 return "packusdw"
1941 case sseOpcodePackuswb:
1942 return "packuswb"
1943 case sseOpcodePaddb:
1944 return "paddb"
1945 case sseOpcodePaddd:
1946 return "paddd"
1947 case sseOpcodePaddq:
1948 return "paddq"
1949 case sseOpcodePaddw:
1950 return "paddw"
1951 case sseOpcodePaddsb:
1952 return "paddsb"
1953 case sseOpcodePaddsw:
1954 return "paddsw"
1955 case sseOpcodePaddusb:
1956 return "paddusb"
1957 case sseOpcodePaddusw:
1958 return "paddusw"
1959 case sseOpcodePalignr:
1960 return "palignr"
1961 case sseOpcodePand:
1962 return "pand"
1963 case sseOpcodePandn:
1964 return "pandn"
1965 case sseOpcodePavgb:
1966 return "pavgb"
1967 case sseOpcodePavgw:
1968 return "pavgw"
1969 case sseOpcodePcmpeqb:
1970 return "pcmpeqb"
1971 case sseOpcodePcmpeqw:
1972 return "pcmpeqw"
1973 case sseOpcodePcmpeqd:
1974 return "pcmpeqd"
1975 case sseOpcodePcmpeqq:
1976 return "pcmpeqq"
1977 case sseOpcodePcmpgtb:
1978 return "pcmpgtb"
1979 case sseOpcodePcmpgtw:
1980 return "pcmpgtw"
1981 case sseOpcodePcmpgtd:
1982 return "pcmpgtd"
1983 case sseOpcodePcmpgtq:
1984 return "pcmpgtq"
1985 case sseOpcodePextrb:
1986 return "pextrb"
1987 case sseOpcodePextrw:
1988 return "pextrw"
1989 case sseOpcodePextrd:
1990 return "pextrd"
1991 case sseOpcodePextrq:
1992 return "pextrq"
1993 case sseOpcodePinsrb:
1994 return "pinsrb"
1995 case sseOpcodePinsrw:
1996 return "pinsrw"
1997 case sseOpcodePinsrd:
1998 return "pinsrd"
1999 case sseOpcodePinsrq:
2000 return "pinsrq"
2001 case sseOpcodePmaddwd:
2002 return "pmaddwd"
2003 case sseOpcodePmaxsb:
2004 return "pmaxsb"
2005 case sseOpcodePmaxsw:
2006 return "pmaxsw"
2007 case sseOpcodePmaxsd:
2008 return "pmaxsd"
2009 case sseOpcodePmaxub:
2010 return "pmaxub"
2011 case sseOpcodePmaxuw:
2012 return "pmaxuw"
2013 case sseOpcodePmaxud:
2014 return "pmaxud"
2015 case sseOpcodePminsb:
2016 return "pminsb"
2017 case sseOpcodePminsw:
2018 return "pminsw"
2019 case sseOpcodePminsd:
2020 return "pminsd"
2021 case sseOpcodePminub:
2022 return "pminub"
2023 case sseOpcodePminuw:
2024 return "pminuw"
2025 case sseOpcodePminud:
2026 return "pminud"
2027 case sseOpcodePmovmskb:
2028 return "pmovmskb"
2029 case sseOpcodePmovsxbd:
2030 return "pmovsxbd"
2031 case sseOpcodePmovsxbw:
2032 return "pmovsxbw"
2033 case sseOpcodePmovsxbq:
2034 return "pmovsxbq"
2035 case sseOpcodePmovsxwd:
2036 return "pmovsxwd"
2037 case sseOpcodePmovsxwq:
2038 return "pmovsxwq"
2039 case sseOpcodePmovsxdq:
2040 return "pmovsxdq"
2041 case sseOpcodePmovzxbd:
2042 return "pmovzxbd"
2043 case sseOpcodePmovzxbw:
2044 return "pmovzxbw"
2045 case sseOpcodePmovzxbq:
2046 return "pmovzxbq"
2047 case sseOpcodePmovzxwd:
2048 return "pmovzxwd"
2049 case sseOpcodePmovzxwq:
2050 return "pmovzxwq"
2051 case sseOpcodePmovzxdq:
2052 return "pmovzxdq"
2053 case sseOpcodePmulld:
2054 return "pmulld"
2055 case sseOpcodePmullw:
2056 return "pmullw"
2057 case sseOpcodePmuludq:
2058 return "pmuludq"
2059 case sseOpcodePor:
2060 return "por"
2061 case sseOpcodePshufb:
2062 return "pshufb"
2063 case sseOpcodePshufd:
2064 return "pshufd"
2065 case sseOpcodePsllw:
2066 return "psllw"
2067 case sseOpcodePslld:
2068 return "pslld"
2069 case sseOpcodePsllq:
2070 return "psllq"
2071 case sseOpcodePsraw:
2072 return "psraw"
2073 case sseOpcodePsrad:
2074 return "psrad"
2075 case sseOpcodePsrlw:
2076 return "psrlw"
2077 case sseOpcodePsrld:
2078 return "psrld"
2079 case sseOpcodePsrlq:
2080 return "psrlq"
2081 case sseOpcodePsubb:
2082 return "psubb"
2083 case sseOpcodePsubd:
2084 return "psubd"
2085 case sseOpcodePsubq:
2086 return "psubq"
2087 case sseOpcodePsubw:
2088 return "psubw"
2089 case sseOpcodePsubsb:
2090 return "psubsb"
2091 case sseOpcodePsubsw:
2092 return "psubsw"
2093 case sseOpcodePsubusb:
2094 return "psubusb"
2095 case sseOpcodePsubusw:
2096 return "psubusw"
2097 case sseOpcodePtest:
2098 return "ptest"
2099 case sseOpcodePunpckhbw:
2100 return "punpckhbw"
2101 case sseOpcodePunpcklbw:
2102 return "punpcklbw"
2103 case sseOpcodePxor:
2104 return "pxor"
2105 case sseOpcodeRcpss:
2106 return "rcpss"
2107 case sseOpcodeRoundps:
2108 return "roundps"
2109 case sseOpcodeRoundpd:
2110 return "roundpd"
2111 case sseOpcodeRoundss:
2112 return "roundss"
2113 case sseOpcodeRoundsd:
2114 return "roundsd"
2115 case sseOpcodeRsqrtss:
2116 return "rsqrtss"
2117 case sseOpcodeSqrtps:
2118 return "sqrtps"
2119 case sseOpcodeSqrtpd:
2120 return "sqrtpd"
2121 case sseOpcodeSqrtss:
2122 return "sqrtss"
2123 case sseOpcodeSqrtsd:
2124 return "sqrtsd"
2125 case sseOpcodeSubps:
2126 return "subps"
2127 case sseOpcodeSubpd:
2128 return "subpd"
2129 case sseOpcodeSubss:
2130 return "subss"
2131 case sseOpcodeSubsd:
2132 return "subsd"
2133 case sseOpcodeUcomiss:
2134 return "ucomiss"
2135 case sseOpcodeUcomisd:
2136 return "ucomisd"
2137 case sseOpcodeXorps:
2138 return "xorps"
2139 case sseOpcodeXorpd:
2140 return "xorpd"
2141 case sseOpcodePmulhrsw:
2142 return "pmulhrsw"
2143 case sseOpcodeUnpcklps:
2144 return "unpcklps"
2145 case sseOpcodeCvtps2pd:
2146 return "cvtps2pd"
2147 case sseOpcodeCvtpd2ps:
2148 return "cvtpd2ps"
2149 case sseOpcodeCvttpd2dq:
2150 return "cvttpd2dq"
2151 case sseOpcodeShufps:
2152 return "shufps"
2153 case sseOpcodePmaddubsw:
2154 return "pmaddubsw"
2155 default:
2156 panic("BUG")
2157 }
2158 }
2159
2160 type roundingMode uint8
2161
2162 const (
2163 roundingModeNearest roundingMode = iota
2164 roundingModeDown
2165 roundingModeUp
2166 roundingModeZero
2167 )
2168
2169 func (r roundingMode) String() string {
2170 switch r {
2171 case roundingModeNearest:
2172 return "nearest"
2173 case roundingModeDown:
2174 return "down"
2175 case roundingModeUp:
2176 return "up"
2177 case roundingModeZero:
2178 return "zero"
2179 default:
2180 panic("BUG")
2181 }
2182 }
2183
2184 // cmpPred is the immediate value for a comparison operation in xmmRmRImm.
2185 type cmpPred uint8
2186
2187 const (
2188 // cmpPredEQ_OQ is Equal (ordered, non-signaling)
2189 cmpPredEQ_OQ cmpPred = iota
2190 // cmpPredLT_OS is Less-than (ordered, signaling)
2191 cmpPredLT_OS
2192 // cmpPredLE_OS is Less-than-or-equal (ordered, signaling)
2193 cmpPredLE_OS
2194 // cmpPredUNORD_Q is Unordered (non-signaling)
2195 cmpPredUNORD_Q
2196 // cmpPredNEQ_UQ is Not-equal (unordered, non-signaling)
2197 cmpPredNEQ_UQ
2198 // cmpPredNLT_US is Not-less-than (unordered, signaling)
2199 cmpPredNLT_US
2200 // cmpPredNLE_US is Not-less-than-or-equal (unordered, signaling)
2201 cmpPredNLE_US
2202 // cmpPredORD_Q is Ordered (non-signaling)
2203 cmpPredORD_Q
2204 // cmpPredEQ_UQ is Equal (unordered, non-signaling)
2205 cmpPredEQ_UQ
2206 // cmpPredNGE_US is Not-greater-than-or-equal (unordered, signaling)
2207 cmpPredNGE_US
2208 // cmpPredNGT_US is Not-greater-than (unordered, signaling)
2209 cmpPredNGT_US
2210 // cmpPredFALSE_OQ is False (ordered, non-signaling)
2211 cmpPredFALSE_OQ
2212 // cmpPredNEQ_OQ is Not-equal (ordered, non-signaling)
2213 cmpPredNEQ_OQ
2214 // cmpPredGE_OS is Greater-than-or-equal (ordered, signaling)
2215 cmpPredGE_OS
2216 // cmpPredGT_OS is Greater-than (ordered, signaling)
2217 cmpPredGT_OS
2218 // cmpPredTRUE_UQ is True (unordered, non-signaling)
2219 cmpPredTRUE_UQ
2220 // Equal (ordered, signaling)
2221 cmpPredEQ_OS
2222 // Less-than (ordered, nonsignaling)
2223 cmpPredLT_OQ
2224 // Less-than-or-equal (ordered, nonsignaling)
2225 cmpPredLE_OQ
2226 // Unordered (signaling)
2227 cmpPredUNORD_S
2228 // Not-equal (unordered, signaling)
2229 cmpPredNEQ_US
2230 // Not-less-than (unordered, nonsignaling)
2231 cmpPredNLT_UQ
2232 // Not-less-than-or-equal (unordered, nonsignaling)
2233 cmpPredNLE_UQ
2234 // Ordered (signaling)
2235 cmpPredORD_S
2236 // Equal (unordered, signaling)
2237 cmpPredEQ_US
2238 // Not-greater-than-or-equal (unordered, non-signaling)
2239 cmpPredNGE_UQ
2240 // Not-greater-than (unordered, nonsignaling)
2241 cmpPredNGT_UQ
2242 // False (ordered, signaling)
2243 cmpPredFALSE_OS
2244 // Not-equal (ordered, signaling)
2245 cmpPredNEQ_OS
2246 // Greater-than-or-equal (ordered, nonsignaling)
2247 cmpPredGE_OQ
2248 // Greater-than (ordered, nonsignaling)
2249 cmpPredGT_OQ
2250 // True (unordered, signaling)
2251 cmpPredTRUE_US
2252 )
2253
2254 func (r cmpPred) String() string {
2255 switch r {
2256 case cmpPredEQ_OQ:
2257 return "eq_oq"
2258 case cmpPredLT_OS:
2259 return "lt_os"
2260 case cmpPredLE_OS:
2261 return "le_os"
2262 case cmpPredUNORD_Q:
2263 return "unord_q"
2264 case cmpPredNEQ_UQ:
2265 return "neq_uq"
2266 case cmpPredNLT_US:
2267 return "nlt_us"
2268 case cmpPredNLE_US:
2269 return "nle_us"
2270 case cmpPredORD_Q:
2271 return "ord_q"
2272 case cmpPredEQ_UQ:
2273 return "eq_uq"
2274 case cmpPredNGE_US:
2275 return "nge_us"
2276 case cmpPredNGT_US:
2277 return "ngt_us"
2278 case cmpPredFALSE_OQ:
2279 return "false_oq"
2280 case cmpPredNEQ_OQ:
2281 return "neq_oq"
2282 case cmpPredGE_OS:
2283 return "ge_os"
2284 case cmpPredGT_OS:
2285 return "gt_os"
2286 case cmpPredTRUE_UQ:
2287 return "true_uq"
2288 case cmpPredEQ_OS:
2289 return "eq_os"
2290 case cmpPredLT_OQ:
2291 return "lt_oq"
2292 case cmpPredLE_OQ:
2293 return "le_oq"
2294 case cmpPredUNORD_S:
2295 return "unord_s"
2296 case cmpPredNEQ_US:
2297 return "neq_us"
2298 case cmpPredNLT_UQ:
2299 return "nlt_uq"
2300 case cmpPredNLE_UQ:
2301 return "nle_uq"
2302 case cmpPredORD_S:
2303 return "ord_s"
2304 case cmpPredEQ_US:
2305 return "eq_us"
2306 case cmpPredNGE_UQ:
2307 return "nge_uq"
2308 case cmpPredNGT_UQ:
2309 return "ngt_uq"
2310 case cmpPredFALSE_OS:
2311 return "false_os"
2312 case cmpPredNEQ_OS:
2313 return "neq_os"
2314 case cmpPredGE_OQ:
2315 return "ge_oq"
2316 case cmpPredGT_OQ:
2317 return "gt_oq"
2318 case cmpPredTRUE_US:
2319 return "true_us"
2320 default:
2321 panic("BUG")
2322 }
2323 }
2324
2325 func linkInstr(prev, next *instruction) *instruction {
2326 prev.next = next
2327 next.prev = prev
2328 return next
2329 }
2330
2331 type defKind byte
2332
2333 const (
2334 defKindNone defKind = iota + 1
2335 defKindOp2
2336 defKindCall
2337 defKindDivRem
2338 )
2339
2340 var defKinds = [instrMax]defKind{
2341 nop0: defKindNone,
2342 ret: defKindNone,
2343 movRR: defKindOp2,
2344 movRM: defKindNone,
2345 xmmMovRM: defKindNone,
2346 aluRmiR: defKindNone,
2347 shiftR: defKindNone,
2348 imm: defKindOp2,
2349 unaryRmR: defKindOp2,
2350 xmmRmiReg: defKindNone,
2351 xmmUnaryRmR: defKindOp2,
2352 xmmUnaryRmRImm: defKindOp2,
2353 xmmCmpRmR: defKindNone,
2354 xmmRmR: defKindNone,
2355 xmmRmRImm: defKindNone,
2356 mov64MR: defKindOp2,
2357 movsxRmR: defKindOp2,
2358 movzxRmR: defKindOp2,
2359 gprToXmm: defKindOp2,
2360 xmmToGpr: defKindOp2,
2361 cmove: defKindNone,
2362 call: defKindCall,
2363 callIndirect: defKindCall,
2364 ud2: defKindNone,
2365 jmp: defKindNone,
2366 jmpIf: defKindNone,
2367 jmpTableIsland: defKindNone,
2368 cmpRmiR: defKindNone,
2369 exitSequence: defKindNone,
2370 lea: defKindOp2,
2371 setcc: defKindOp2,
2372 zeros: defKindOp2,
2373 sourceOffsetInfo: defKindNone,
2374 fcvtToSintSequence: defKindNone,
2375 defineUninitializedReg: defKindOp2,
2376 fcvtToUintSequence: defKindNone,
2377 xmmCMov: defKindOp2,
2378 idivRemSequence: defKindDivRem,
2379 blendvpd: defKindNone,
2380 mfence: defKindNone,
2381 xchg: defKindNone,
2382 lockcmpxchg: defKindNone,
2383 lockxadd: defKindNone,
2384 neg: defKindNone,
2385 nopUseReg: defKindNone,
2386 tailCall: defKindCall,
2387 tailCallIndirect: defKindCall,
2388 }
2389
2390 // String implements fmt.Stringer.
2391 func (d defKind) String() string {
2392 switch d {
2393 case defKindNone:
2394 return "none"
2395 case defKindOp2:
2396 return "op2"
2397 case defKindCall:
2398 return "call"
2399 case defKindDivRem:
2400 return "divrem"
2401 default:
2402 return "invalid"
2403 }
2404 }
2405
2406 type useKind byte
2407
2408 const (
2409 useKindNone useKind = iota + 1
2410 useKindOp1
2411 // useKindOp1Op2Reg is Op1 can be any operand, Op2 must be a register.
2412 useKindOp1Op2Reg
2413 // useKindOp1RegOp2 is Op1 must be a register, Op2 can be any operand.
2414 useKindOp1RegOp2
2415 // useKindRaxOp1RegOp2 is Op1 must be a register, Op2 can be any operand, and RAX is used.
2416 useKindRaxOp1RegOp2
2417 useKindDivRem
2418 useKindBlendvpd
2419 useKindCall
2420 useKindCallInd
2421 useKindTailCallInd
2422 useKindFcvtToSintSequence
2423 useKindFcvtToUintSequence
2424 )
2425
2426 var useKinds = [instrMax]useKind{
2427 nop0: useKindNone,
2428 ret: useKindNone,
2429 movRR: useKindOp1,
2430 movRM: useKindOp1RegOp2,
2431 xmmMovRM: useKindOp1RegOp2,
2432 cmove: useKindOp1Op2Reg,
2433 aluRmiR: useKindOp1Op2Reg,
2434 shiftR: useKindOp1Op2Reg,
2435 imm: useKindNone,
2436 unaryRmR: useKindOp1,
2437 xmmRmiReg: useKindOp1Op2Reg,
2438 xmmUnaryRmR: useKindOp1,
2439 xmmUnaryRmRImm: useKindOp1,
2440 xmmCmpRmR: useKindOp1Op2Reg,
2441 xmmRmR: useKindOp1Op2Reg,
2442 xmmRmRImm: useKindOp1Op2Reg,
2443 mov64MR: useKindOp1,
2444 movzxRmR: useKindOp1,
2445 movsxRmR: useKindOp1,
2446 gprToXmm: useKindOp1,
2447 xmmToGpr: useKindOp1,
2448 call: useKindCall,
2449 callIndirect: useKindCallInd,
2450 ud2: useKindNone,
2451 jmpIf: useKindOp1,
2452 jmp: useKindOp1,
2453 cmpRmiR: useKindOp1Op2Reg,
2454 exitSequence: useKindOp1,
2455 lea: useKindOp1,
2456 jmpTableIsland: useKindNone,
2457 setcc: useKindNone,
2458 zeros: useKindNone,
2459 sourceOffsetInfo: useKindNone,
2460 fcvtToSintSequence: useKindFcvtToSintSequence,
2461 defineUninitializedReg: useKindNone,
2462 fcvtToUintSequence: useKindFcvtToUintSequence,
2463 xmmCMov: useKindOp1,
2464 idivRemSequence: useKindDivRem,
2465 blendvpd: useKindBlendvpd,
2466 mfence: useKindNone,
2467 xchg: useKindOp1RegOp2,
2468 lockcmpxchg: useKindRaxOp1RegOp2,
2469 lockxadd: useKindOp1RegOp2,
2470 neg: useKindOp1,
2471 nopUseReg: useKindOp1,
2472 tailCall: useKindCall,
2473 tailCallIndirect: useKindTailCallInd,
2474 }
2475
2476 func (u useKind) String() string {
2477 switch u {
2478 case useKindNone:
2479 return "none"
2480 case useKindOp1:
2481 return "op1"
2482 case useKindOp1Op2Reg:
2483 return "op1op2Reg"
2484 case useKindOp1RegOp2:
2485 return "op1RegOp2"
2486 case useKindCall:
2487 return "call"
2488 case useKindCallInd:
2489 return "callInd"
2490 case useKindTailCallInd:
2491 return "tailCallInd"
2492 default:
2493 return "invalid"
2494 }
2495 }