Debian Codegraph @

internal/engine/wazevo/backend/isa/arm64/lower_instr_test.go

raw
1 package arm64
2
3 import (
4 "encoding/hex"
5 "fmt"
6 "strings"
7 "testing"
8
9 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
10 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
11 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
12 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
13 "github.com/tetratelabs/wazero/internal/testing/require"
14 )
15
16 func TestMachine_LowerConditionalBranch(t *testing.T) {
17 cmpInSameGroupFromParams := func(
18 brz bool, intCond ssa.IntegerCmpCond, floatCond ssa.FloatCmpCond,
19 ctx *mockCompiler, builder ssa.Builder, m *machine,
20 ) (instr *ssa.Instruction, verify func(t *testing.T)) {
21 entry := builder.CurrentBlock()
22 isInt := intCond != ssa.IntegerCmpCondInvalid
23
24 var val1, val2 ssa.Value
25 if isInt {
26 val1 = entry.AddParam(builder, ssa.TypeI64)
27 val2 = entry.AddParam(builder, ssa.TypeI64)
28 ctx.vRegMap[val1], ctx.vRegMap[val2] = regToVReg(x1).SetRegType(regalloc.RegTypeInt), regToVReg(x2).SetRegType(regalloc.RegTypeInt)
29 } else {
30 val1 = entry.AddParam(builder, ssa.TypeF64)
31 val2 = entry.AddParam(builder, ssa.TypeF64)
32 ctx.vRegMap[val1], ctx.vRegMap[val2] = regToVReg(v1).SetRegType(regalloc.RegTypeFloat), regToVReg(v2).SetRegType(regalloc.RegTypeFloat)
33 }
34
35 var cmpInstr *ssa.Instruction
36 if isInt {
37 cmpInstr = builder.AllocateInstruction()
38 cmpInstr.AsIcmp(val1, val2, intCond)
39 builder.InsertInstruction(cmpInstr)
40 } else {
41 cmpInstr = builder.AllocateInstruction()
42 cmpInstr.AsFcmp(val1, val2, floatCond)
43 builder.InsertInstruction(cmpInstr)
44 }
45
46 cmpVal := cmpInstr.Return()
47 ctx.vRegMap[cmpVal] = 3
48
49 ctx.definitions[val1] = backend.SSAValueDefinition{V: val1}
50 ctx.definitions[val2] = backend.SSAValueDefinition{V: val2}
51 ctx.definitions[cmpVal] = backend.SSAValueDefinition{Instr: cmpInstr}
52 b := builder.AllocateInstruction()
53 if brz {
54 b.AsBrz(cmpVal, ssa.ValuesNil, builder.AllocateBasicBlock())
55 } else {
56 b.AsBrnz(cmpVal, ssa.ValuesNil, builder.AllocateBasicBlock())
57 }
58 builder.InsertInstruction(b)
59 return b, func(t *testing.T) {
60 require.True(t, cmpInstr.Lowered())
61 }
62 }
63
64 icmpInSameGroupFromParamAndImm12 := func(brz bool, ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
65 entry := builder.CurrentBlock()
66 v1 := entry.AddParam(builder, ssa.TypeI32)
67
68 iconst := builder.AllocateInstruction()
69 iconst.AsIconst32(0x4d2)
70 builder.InsertInstruction(iconst)
71 v2 := iconst.Return()
72
73 icmp := builder.AllocateInstruction()
74 icmp.AsIcmp(v1, v2, ssa.IntegerCmpCondEqual)
75 builder.InsertInstruction(icmp)
76 icmpVal := icmp.Return()
77 ctx.definitions[v1] = backend.SSAValueDefinition{V: v1}
78 ctx.definitions[v2] = backend.SSAValueDefinition{Instr: iconst}
79 ctx.definitions[icmpVal] = backend.SSAValueDefinition{Instr: icmp}
80 ctx.vRegMap[v1], ctx.vRegMap[v2], ctx.vRegMap[icmpVal] = intToVReg(1), intToVReg(2), intToVReg(3)
81 b := builder.AllocateInstruction()
82 if brz {
83 b.AsBrz(icmpVal, ssa.ValuesNil, builder.AllocateBasicBlock())
84 } else {
85 b.AsBrnz(icmpVal, ssa.ValuesNil, builder.AllocateBasicBlock())
86 }
87 builder.InsertInstruction(b)
88 return b, func(t *testing.T) {
89 require.True(t, icmp.Lowered())
90 }
91 }
92
93 for _, tc := range []struct {
94 name string
95 setup func(*mockCompiler, ssa.Builder, *machine) (instr *ssa.Instruction, verify func(t *testing.T))
96 instructions []string
97 }{
98 {
99 name: "icmp in different group",
100 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
101 entry := builder.CurrentBlock()
102 v1, v2 := entry.AddParam(builder, ssa.TypeI64), entry.AddParam(builder, ssa.TypeI64)
103
104 icmp := builder.AllocateInstruction()
105 icmp.AsIcmp(v1, v2, ssa.IntegerCmpCondEqual)
106 builder.InsertInstruction(icmp)
107 icmpVal := icmp.Return()
108 ctx.definitions[icmpVal] = backend.SSAValueDefinition{Instr: icmp, V: icmpVal}
109 ctx.vRegMap[v1], ctx.vRegMap[v2], ctx.vRegMap[icmpVal] = intToVReg(1), intToVReg(2), intToVReg(3)
110
111 brz := builder.AllocateInstruction()
112 brz.AsBrz(icmpVal, ssa.ValuesNil, builder.AllocateBasicBlock())
113 builder.InsertInstruction(brz)
114
115 // Indicate that currently compiling in the different group.
116 ctx.currentGID = 1000
117 return brz, func(t *testing.T) {
118 require.False(t, icmp.Lowered())
119 }
120 },
121 instructions: []string{"cbz w3?, (L1)"},
122 },
123 {
124 name: "brz / icmp in the same group / params",
125 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
126 return cmpInSameGroupFromParams(true, ssa.IntegerCmpCondUnsignedGreaterThan, ssa.FloatCmpCondInvalid, ctx, builder, m)
127 },
128 instructions: []string{
129 "subs xzr, x1, x2",
130 "b.ls L1",
131 },
132 },
133 {
134 name: "brnz / icmp in the same group / params",
135 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
136 return cmpInSameGroupFromParams(false, ssa.IntegerCmpCondEqual, ssa.FloatCmpCondInvalid, ctx, builder, m)
137 },
138 instructions: []string{
139 "subs xzr, x1, x2",
140 "b.eq L1",
141 },
142 },
143 {
144 name: "brz / fcmp in the same group / params",
145 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
146 return cmpInSameGroupFromParams(true, ssa.IntegerCmpCondInvalid, ssa.FloatCmpCondEqual, ctx, builder, m)
147 },
148 instructions: []string{
149 "fcmp d1, d2",
150 "b.ne L1",
151 },
152 },
153 {
154 name: "brnz / fcmp in the same group / params",
155 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
156 return cmpInSameGroupFromParams(false, ssa.IntegerCmpCondInvalid, ssa.FloatCmpCondGreaterThan, ctx, builder, m)
157 },
158 instructions: []string{
159 "fcmp d1, d2",
160 "b.gt L1",
161 },
162 },
163 {
164 name: "brz / icmp in the same group / params",
165 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
166 return icmpInSameGroupFromParamAndImm12(true, ctx, builder, m)
167 },
168 instructions: []string{
169 "subs wzr, w1?, #0x4d2",
170 "b.ne L1",
171 },
172 },
173 {
174 name: "brz / icmp in the same group / params",
175 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction, verify func(t *testing.T)) {
176 return icmpInSameGroupFromParamAndImm12(false, ctx, builder, m)
177 },
178 instructions: []string{
179 "subs wzr, w1?, #0x4d2",
180 "b.eq L1",
181 },
182 },
183 } {
184 t.Run(tc.name, func(t *testing.T) {
185 ctx, b, m := newSetupWithMockContext()
186 instr, verify := tc.setup(ctx, b, m)
187 m.LowerConditionalBranch(instr)
188 verify(t)
189 require.Equal(t, strings.Join(tc.instructions, "\n"),
190 formatEmittedInstructionsInCurrentBlock(m))
191 })
192 }
193 }
194
195 func TestMachine_LowerSingleBranch(t *testing.T) {
196 for _, tc := range []struct {
197 name string
198 setup func(*mockCompiler, ssa.Builder, *machine) (instr *ssa.Instruction)
199 instructions []string
200 }{
201 {
202 name: "jump-fallthrough",
203 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction) {
204 jump := builder.AllocateInstruction()
205 jump.AsJump(ssa.ValuesNil, builder.AllocateBasicBlock())
206 builder.InsertInstruction(jump)
207 jump.AsFallthroughJump()
208 return jump
209 },
210 instructions: []string{}, // Fallthrough jump should be optimized out.
211 },
212 {
213 name: "b",
214 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction) {
215 jump := builder.AllocateInstruction()
216 jump.AsJump(ssa.ValuesNil, builder.AllocateBasicBlock())
217 builder.InsertInstruction(jump)
218 return jump
219 },
220 instructions: []string{"b L1"},
221 },
222 {
223 name: "ret",
224 setup: func(ctx *mockCompiler, builder ssa.Builder, m *machine) (instr *ssa.Instruction) {
225 jump := builder.AllocateInstruction()
226 jump.AsJump(ssa.ValuesNil, builder.ReturnBlock())
227 builder.InsertInstruction(jump)
228 return jump
229 },
230 // Jump which targets the return block should be translated as "ret".
231 instructions: []string{"ret"},
232 },
233 } {
234 tc := tc
235 t.Run(tc.name, func(t *testing.T) {
236 ctx, b, m := newSetupWithMockContext()
237 instr := tc.setup(ctx, b, m)
238 m.LowerSingleBranch(instr)
239 require.Equal(t, strings.Join(tc.instructions, "\n"), formatEmittedInstructionsInCurrentBlock(m))
240 })
241 }
242 }
243
244 func TestMachine_InsertMove(t *testing.T) {
245 for _, tc := range []struct {
246 name string
247 src, dst regalloc.VReg
248 typ ssa.Type
249 instruction string
250 }{
251 {
252 name: "int",
253 src: regalloc.VReg(1).SetRegType(regalloc.RegTypeInt),
254 dst: regalloc.VReg(2).SetRegType(regalloc.RegTypeInt),
255 instruction: "mov x1?, x2?",
256 typ: ssa.TypeI64,
257 },
258 {
259 name: "float",
260 src: regalloc.VReg(1).SetRegType(regalloc.RegTypeFloat),
261 dst: regalloc.VReg(2).SetRegType(regalloc.RegTypeFloat),
262 instruction: "mov v1?.8b, v2?.8b",
263 typ: ssa.TypeF64,
264 },
265 {
266 name: "vector",
267 src: regalloc.VReg(1).SetRegType(regalloc.RegTypeFloat),
268 dst: regalloc.VReg(2).SetRegType(regalloc.RegTypeFloat),
269 instruction: "mov v1?.16b, v2?.16b",
270 typ: ssa.TypeV128,
271 },
272 } {
273 t.Run(tc.name, func(t *testing.T) {
274 _, _, m := newSetupWithMockContext()
275 m.InsertMove(tc.src, tc.dst, tc.typ)
276 require.Equal(t, tc.instruction, formatEmittedInstructionsInCurrentBlock(m))
277 })
278 }
279 }
280
281 func TestMachine_lowerIDiv(t *testing.T) {
282 for _, tc := range []struct {
283 name string
284 _64bit bool
285 signed bool
286 exp string
287 }{
288 {
289 name: "32bit unsigned", _64bit: false, signed: false,
290 exp: `
291 udiv w1?, w2?, w3?
292 mov x1?, x65535?
293 cbnz w3?, L1
294 movz x2?, #0xa, lsl 0
295 str w2?, [x1?]
296 mov x3?, sp
297 str x3?, [x1?, #0x38]
298 adr x4?, #0x0
299 str x4?, [x1?, #0x30]
300 exit_sequence x1?
301 L1:
302 `,
303 },
304 {name: "32bit signed", _64bit: false, signed: true, exp: `
305 sdiv w1?, w2?, w3?
306 mov x1?, x65535?
307 cbnz w3?, L1
308 movz x2?, #0xa, lsl 0
309 str w2?, [x1?]
310 mov x3?, sp
311 str x3?, [x1?, #0x38]
312 adr x4?, #0x0
313 str x4?, [x1?, #0x30]
314 exit_sequence x1?
315 L1:
316 adds wzr, w3?, #0x1
317 ccmp w2?, #0x1, #0x0, eq
318 mov x5?, x65535?
319 b.vc L2
320 movz x6?, #0xb, lsl 0
321 str w6?, [x5?]
322 mov x7?, sp
323 str x7?, [x5?, #0x38]
324 adr x8?, #0x0
325 str x8?, [x5?, #0x30]
326 exit_sequence x5?
327 L2:
328 `},
329 {name: "64bit unsigned", _64bit: true, signed: false, exp: `
330 udiv x1?, x2?, x3?
331 mov x1?, x65535?
332 cbnz x3?, L1
333 movz x2?, #0xa, lsl 0
334 str w2?, [x1?]
335 mov x3?, sp
336 str x3?, [x1?, #0x38]
337 adr x4?, #0x0
338 str x4?, [x1?, #0x30]
339 exit_sequence x1?
340 L1:
341 `},
342 {name: "64bit signed", _64bit: true, signed: true, exp: `
343 sdiv x1?, x2?, x3?
344 mov x1?, x65535?
345 cbnz x3?, L1
346 movz x2?, #0xa, lsl 0
347 str w2?, [x1?]
348 mov x3?, sp
349 str x3?, [x1?, #0x38]
350 adr x4?, #0x0
351 str x4?, [x1?, #0x30]
352 exit_sequence x1?
353 L1:
354 adds xzr, x3?, #0x1
355 ccmp x2?, #0x1, #0x0, eq
356 mov x5?, x65535?
357 b.vc L2
358 movz x6?, #0xb, lsl 0
359 str w6?, [x5?]
360 mov x7?, sp
361 str x7?, [x5?, #0x38]
362 adr x8?, #0x0
363 str x8?, [x5?, #0x30]
364 exit_sequence x5?
365 L2:
366 `},
367 } {
368 t.Run(tc.name, func(t *testing.T) {
369 execCtx := regalloc.VReg(0xffff).SetRegType(regalloc.RegTypeInt)
370 rd, rn, rm := regalloc.VReg(1).SetRegType(regalloc.RegTypeInt),
371 regalloc.VReg(2).SetRegType(regalloc.RegTypeInt),
372 regalloc.VReg(3).SetRegType(regalloc.RegTypeInt)
373 mc, _, m := newSetupWithMockContext()
374 m.maxSSABlockID, m.nextLabel = 1, 1
375 mc.typeOf = map[regalloc.VRegID]ssa.Type{execCtx.ID(): ssa.TypeI64, 2: ssa.TypeI64, 3: ssa.TypeI64}
376 m.lowerIDiv(execCtx, rd, operandNR(rn), operandNR(rm), tc._64bit, tc.signed)
377 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
378 })
379 }
380 }
381
382 func TestMachine_exitWithCode(t *testing.T) {
383 _, _, m := newSetupWithMockContext()
384 m.lowerExitWithCode(x1VReg, wazevoapi.ExitCodeGrowStack)
385 m.FlushPendingInstructions()
386 m.encode(m.perBlockHead)
387 require.Equal(t, `
388 movz x1?, #0x1, lsl 0
389 str w1?, [x1]
390 mov x2?, sp
391 str x2?, [x1, #0x38]
392 adr x3?, #0x0
393 str x3?, [x1, #0x30]
394 exit_sequence x1
395 `, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
396 }
397
398 func TestMachine_lowerFpuToInt(t *testing.T) {
399 for _, tc := range []struct {
400 name string
401 nontrapping bool
402 expectedAsm string
403 }{
404 {
405 name: "trapping",
406 nontrapping: false,
407 expectedAsm: `
408 msr fpsr, xzr
409 fcvtzu w1, s2
410 mrs x1? fpsr
411 mov x2?, x15
412 mov x3?, d2
413 subs xzr, x1?, #0x1
414 b.ne L2
415 fcmp w3?, w3?
416 mov x4?, x2?
417 b.vc L1
418 movz x5?, #0xc, lsl 0
419 str w5?, [x4?]
420 mov x6?, sp
421 str x6?, [x4?, #0x38]
422 adr x7?, #0x0
423 str x7?, [x4?, #0x30]
424 exit_sequence x4?
425 L1:
426 movz x8?, #0xb, lsl 0
427 str w8?, [x2?]
428 mov x9?, sp
429 str x9?, [x2?, #0x38]
430 adr x10?, #0x0
431 str x10?, [x2?, #0x30]
432 exit_sequence x2?
433 L2:
434 `,
435 },
436 {
437 name: "nontrapping",
438 nontrapping: true,
439 expectedAsm: `
440 fcvtzu w1, s2
441 `,
442 },
443 } {
444 t.Run(tc.name, func(t *testing.T) {
445 mc, _, m := newSetupWithMockContext()
446 m.maxSSABlockID, m.nextLabel = 1, 1
447 mc.typeOf = map[regalloc.VRegID]ssa.Type{v2VReg.ID(): ssa.TypeI64, x15VReg.ID(): ssa.TypeI64}
448 m.lowerFpuToInt(x1VReg, operandNR(v2VReg), x15VReg, false, false, false, tc.nontrapping)
449 require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
450
451 m.FlushPendingInstructions()
452 m.encode(m.perBlockHead)
453 })
454 }
455 }
456
457 func TestMachine_lowerVIMul(t *testing.T) {
458 for _, tc := range []struct {
459 name string
460 expectedAsm string
461 arrangement vecArrangement
462 expectedBytes string
463 }{
464 {
465 name: "2D",
466 arrangement: vecArrangement2D,
467 expectedAsm: `
468 rev64 v2?.4s, x15.4s
469 mul v2?.4s, v2?.4s, x2.4s
470 xtn v1?.2s, x2.2s
471 addp v2?.4s, v2?.4s, v2?.4s
472 xtn v3?.2s, x15.2s
473 shll v4?.2s, v2?.2s
474 umlal v4?.2s, v3?.2s, v1?.2s
475 mov x1.16b, v4?.16b
476 `,
477 expectedBytes: "e009a04e009ca24e4028a10e00bca04ee029a10e0038a12e0080a02e011ca04e",
478 },
479 {
480 name: "8B",
481 arrangement: vecArrangement8B,
482 expectedAsm: `
483 mul x1.8b, x2.8b, x15.8b
484 `,
485 expectedBytes: "419c2f0e",
486 },
487 {
488 name: "16B",
489 arrangement: vecArrangement16B,
490 expectedAsm: `
491 mul x1.16b, x2.16b, x15.16b
492 `,
493 expectedBytes: "419c2f4e",
494 },
495 {
496 name: "4H",
497 arrangement: vecArrangement4H,
498 expectedAsm: `
499 mul x1.4h, x2.4h, x15.4h
500 `,
501 expectedBytes: "419c6f0e",
502 },
503 {
504 name: "8H",
505 arrangement: vecArrangement8H,
506 expectedAsm: `
507 mul x1.8h, x2.8h, x15.8h
508 `,
509 expectedBytes: "419c6f4e",
510 },
511 {
512 name: "2S",
513 arrangement: vecArrangement2S,
514 expectedAsm: `
515 mul x1.2s, x2.2s, x15.2s
516 `,
517 expectedBytes: "419caf0e",
518 },
519 {
520 name: "4S",
521 arrangement: vecArrangement4S,
522 expectedAsm: `
523 mul x1.4s, x2.4s, x15.4s
524 `,
525 expectedBytes: "419caf4e",
526 },
527 } {
528 t.Run(tc.name, func(t *testing.T) {
529 _, _, m := newSetupWithMockContext()
530 m.lowerVIMul(x1VReg, operandNR(x2VReg), operandNR(x15VReg), tc.arrangement)
531 require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
532
533 m.FlushPendingInstructions()
534 m.encode(m.perBlockHead)
535 buf := m.compiler.Buf()
536 require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
537 })
538 }
539 }
540
541 func TestMachine_lowerVcheckTrue(t *testing.T) {
542 for _, tc := range []struct {
543 name string
544 op ssa.Opcode
545 expectedAsm string
546 arrangement vecArrangement
547 expectedBytes string
548 }{
549 {
550 name: "anyTrue",
551 op: ssa.OpcodeVanyTrue,
552 expectedAsm: `
553 umaxp v1?.16b, x1.16b, x1.16b
554 mov x15, v1?.d[0]
555 ccmp x15, #0x0, #0x0, al
556 cset x15, ne
557 `,
558 expectedBytes: "20a4216e0f3c084ee0e940faef079f9a",
559 },
560 {
561 name: "allTrue 2D",
562 op: ssa.OpcodeVallTrue,
563 arrangement: vecArrangement2D,
564 expectedAsm: `
565 cmeq v1?.2d, x1.2d, #0
566 addp v1?.2d, v1?.2d, v1?.2d
567 fcmp d1?, d1?
568 cset x15, eq
569 `,
570 expectedBytes: "2098e04e00bce04e0020601eef179f9a",
571 },
572 {
573 name: "allTrue 8B",
574 arrangement: vecArrangement8B,
575 op: ssa.OpcodeVallTrue,
576 expectedAsm: `
577 uminv h1?, x1.8b
578 mov x15, v1?.d[0]
579 ccmp x15, #0x0, #0x0, al
580 cset x15, ne
581 `,
582 expectedBytes: "20a8312e0f3c084ee0e940faef079f9a",
583 },
584 {
585 name: "allTrue 16B",
586 arrangement: vecArrangement16B,
587 op: ssa.OpcodeVallTrue,
588 expectedAsm: `
589 uminv h1?, x1.16b
590 mov x15, v1?.d[0]
591 ccmp x15, #0x0, #0x0, al
592 cset x15, ne
593 `,
594 expectedBytes: "20a8316e0f3c084ee0e940faef079f9a",
595 },
596 {
597 name: "allTrue 4H",
598 arrangement: vecArrangement4H,
599 op: ssa.OpcodeVallTrue,
600 expectedAsm: `
601 uminv s1?, x1.4h
602 mov x15, v1?.d[0]
603 ccmp x15, #0x0, #0x0, al
604 cset x15, ne
605 `,
606 expectedBytes: "20a8712e0f3c084ee0e940faef079f9a",
607 },
608 {
609 name: "allTrue 8H",
610 arrangement: vecArrangement8H,
611 op: ssa.OpcodeVallTrue,
612 expectedAsm: `
613 uminv s1?, x1.8h
614 mov x15, v1?.d[0]
615 ccmp x15, #0x0, #0x0, al
616 cset x15, ne
617 `,
618 expectedBytes: "20a8716e0f3c084ee0e940faef079f9a",
619 },
620 {
621 name: "allTrue 4S",
622 arrangement: vecArrangement4S,
623 op: ssa.OpcodeVallTrue,
624 expectedAsm: `
625 uminv d1?, x1.4s
626 mov x15, v1?.d[0]
627 ccmp x15, #0x0, #0x0, al
628 cset x15, ne
629 `,
630 expectedBytes: "20a8b16e0f3c084ee0e940faef079f9a",
631 },
632 } {
633 t.Run(tc.name, func(t *testing.T) {
634 _, _, m := newSetupWithMockContext()
635 m.lowerVcheckTrue(tc.op, operandNR(x1VReg), x15VReg, tc.arrangement)
636 require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
637
638 m.FlushPendingInstructions()
639 m.encode(m.perBlockHead)
640 buf := m.compiler.Buf()
641 require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
642 })
643 }
644 }
645
646 func TestMachine_lowerVhighBits(t *testing.T) {
647 for _, tc := range []struct {
648 name string
649 expectedAsm string
650 arrangement vecArrangement
651 expectedBytes string
652 }{
653 {
654 name: "16B",
655 arrangement: vecArrangement16B,
656 expectedAsm: `
657 sshr v3?.16b, x1.16b, #7
658 movz x1?, #0x201, lsl 0
659 movk x1?, #0x804, lsl 16
660 movk x1?, #0x2010, lsl 32
661 movk x1?, #0x8040, lsl 48
662 dup v2?.2d, x1?
663 and v3?.16b, v3?.16b, v2?.16b
664 ext v2?.16b, v3?.16b, v3?.16b, #8
665 zip1 v2?.16b, v3?.16b, v2?.16b
666 addv s2?, v2?.8h
667 umov w15, v2?.h[0]
668 `,
669 expectedBytes: "2004094f204080d28000a1f20002c4f20008f0f2000c084e001c204e0040006e0038004e00b8714e0f3c020e",
670 },
671 {
672 name: "8H",
673 arrangement: vecArrangement8H,
674 expectedAsm: `
675 sshr v3?.8h, x1.8h, #15
676 movz x1?, #0x1, lsl 0
677 movk x1?, #0x2, lsl 16
678 movk x1?, #0x4, lsl 32
679 movk x1?, #0x8, lsl 48
680 dup v2?.2d, x1?
681 lsl x1?, x1?, 0x4
682 ins v2?.d[1], x1?
683 and v2?.16b, v3?.16b, v2?.16b
684 addv s2?, v2?.8h
685 umov w15, v2?.h[0]
686 `,
687 expectedBytes: "2004114f200080d24000a0f28000c0f20001e0f2000c084e00ec7cd3001c184e001c204e00b8714e0f3c020e",
688 },
689 {
690 name: "4S",
691 arrangement: vecArrangement4S,
692 expectedAsm: `
693 sshr v3?.4s, x1.4s, #31
694 movz x1?, #0x1, lsl 0
695 movk x1?, #0x2, lsl 32
696 dup v2?.2d, x1?
697 lsl x1?, x1?, 0x2
698 ins v2?.d[1], x1?
699 and v2?.16b, v3?.16b, v2?.16b
700 addv d2?, v2?.4s
701 umov w15, v2?.s[0]
702 `,
703 expectedBytes: "2004214f200080d24000c0f2000c084e00f47ed3001c184e001c204e00b8b14e0f3c040e",
704 },
705 {
706 name: "2D",
707 arrangement: vecArrangement2D,
708 expectedAsm: `
709 mov x15, x1.d[0]
710 mov x1?, x1.d[1]
711 lsr x1?, x1?, 0x3f
712 lsr x15, x15, 0x3f
713 add w15, w15, w1?, lsl #1
714 `,
715 expectedBytes: "2f3c084e203c184e00fc7fd3effd7fd3ef05000b",
716 },
717 } {
718 t.Run(tc.name, func(t *testing.T) {
719 _, _, m := newSetupWithMockContext()
720 m.lowerVhighBits(operandNR(x1VReg), x15VReg, tc.arrangement)
721 require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
722
723 m.FlushPendingInstructions()
724 m.encode(m.perBlockHead)
725 buf := m.compiler.Buf()
726 require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
727 })
728 }
729 }
730
731 func TestMachine_lowerShuffle(t *testing.T) {
732 for _, tc := range []struct {
733 name string
734 lanes []uint64
735 expectedAsm string
736 expectedBytes string
737 }{
738 {
739 name: "lanes 0..15",
740 lanes: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
741 expectedAsm: `
742 mov v29.16b, x2.16b
743 mov v30.16b, x15.16b
744 ldr q1?, #8; b 32; data.v128 0706050403020100 0f0e0d0c0b0a0908
745 tbl x1.16b, { v29.16b, v30.16b }, v1?.16b
746 `,
747 expectedBytes: "5d1ca24efe1daf4e4000009c05000014000102030405060708090a0b0c0d0e0fa123004e",
748 },
749 {
750 name: "lanes 0101...",
751 lanes: []uint64{0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1},
752 expectedAsm: `
753 mov v29.16b, x2.16b
754 mov v30.16b, x15.16b
755 ldr q1?, #8; b 32; data.v128 0100010001000100 0100010001000100
756 tbl x1.16b, { v29.16b, v30.16b }, v1?.16b
757 `,
758 expectedBytes: "5d1ca24efe1daf4e4000009c0500001400010001000100010001000100010001a123004e",
759 },
760 } {
761 t.Run(tc.name, func(t *testing.T) {
762 _, _, m := newSetupWithMockContext()
763 lanes := tc.lanes
764
765 // Encode the 16 bytes as 8 bytes in u1, and 8 bytes in u2.
766 lane1 := lanes[7]<<56 | lanes[6]<<48 | lanes[5]<<40 | lanes[4]<<32 | lanes[3]<<24 | lanes[2]<<16 | lanes[1]<<8 | lanes[0]
767 lane2 := lanes[15]<<56 | lanes[14]<<48 | lanes[13]<<40 | lanes[12]<<32 | lanes[11]<<24 | lanes[10]<<16 | lanes[9]<<8 | lanes[8]
768
769 m.lowerShuffle(x1VReg, operandNR(x2VReg), operandNR(x15VReg), lane1, lane2)
770 require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
771
772 m.FlushPendingInstructions()
773 m.encode(m.perBlockHead)
774 buf := m.compiler.Buf()
775 require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
776 })
777 }
778 }
779
780 func TestMachine_lowerVShift(t *testing.T) {
781 for _, tc := range []struct {
782 name string
783 expectedAsm string
784 op ssa.Opcode
785 arrangement vecArrangement
786 expectedBytes string
787 }{
788 {
789 name: "VIshl",
790 op: ssa.OpcodeVIshl,
791 arrangement: vecArrangement16B,
792 expectedAsm: `
793 and x1?, x15, #0x7
794 dup v2?.16b, x1?
795 sshl x1.16b, x2.16b, v2?.16b
796 `,
797 expectedBytes: "e0094092000c014e4144204e",
798 },
799 {
800 name: "VSshr",
801 op: ssa.OpcodeVSshr,
802 arrangement: vecArrangement16B,
803 expectedAsm: `
804 and x1?, x15, #0x7
805 sub x1?, xzr, x1?
806 dup v2?.16b, x1?
807 sshl x1.16b, x2.16b, v2?.16b
808 `,
809 expectedBytes: "e0094092e00300cb000c014e4144204e",
810 },
811 {
812 name: "VUshr",
813 op: ssa.OpcodeVUshr,
814 arrangement: vecArrangement16B,
815 expectedAsm: `
816 and x1?, x15, #0x7
817 sub x1?, xzr, x1?
818 dup v2?.16b, x1?
819 ushl x1.16b, x2.16b, v2?.16b
820 `,
821 expectedBytes: "e0094092e00300cb000c014e4144206e",
822 },
823 } {
824 t.Run(tc.name, func(t *testing.T) {
825 _, _, m := newSetupWithMockContext()
826 m.lowerVShift(tc.op, x1VReg, operandNR(x2VReg), operandNR(x15VReg), tc.arrangement)
827 require.Equal(t, tc.expectedAsm, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
828
829 m.FlushPendingInstructions()
830 m.encode(m.perBlockHead)
831 buf := m.compiler.Buf()
832 require.Equal(t, tc.expectedBytes, hex.EncodeToString(buf))
833 })
834 }
835 }
836
837 func TestMachine_lowerSelectVec(t *testing.T) {
838 _, _, m := newSetupWithMockContext()
839 c := operandNR(m.compiler.AllocateVReg(ssa.TypeI32))
840 rn := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
841 rm := operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
842 rd := m.compiler.AllocateVReg(ssa.TypeV128)
843
844 require.Equal(t, 1, int(c.reg().ID()))
845 require.Equal(t, 2, int(rn.reg().ID()))
846 require.Equal(t, 3, int(rm.reg().ID()))
847 require.Equal(t, 4, int(rd.ID()))
848
849 m.lowerSelectVec(c, rn, rm, rd)
850 require.Equal(t, `
851 subs wzr, w1?, wzr
852 csetm x5?, ne
853 dup v6?.2d, x5?
854 bsl v6?.16b, v2?.16b, v3?.16b
855 mov v4?.16b, v6?.16b
856 `, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
857 }
858
859 func TestMachine_lowerFcopysign(t *testing.T) {
860 for _, tc := range []struct {
861 _64bit bool
862 exp string
863 }{
864 {
865 _64bit: false,
866 exp: `
867 movz w1?, #0x8000, lsl 16
868 ins v2?.s[0], w1?
869 mov v6?.8b, v3?.8b
870 bit v6?.8b, v4?.8b, v2?.8b
871 mov v5?.8b, v6?.8b
872 `,
873 },
874 {
875 _64bit: true,
876 exp: `
877 movz x1?, #0x8000, lsl 48
878 ins v2?.d[0], x1?
879 mov v6?.8b, v3?.8b
880 bit v6?.8b, v4?.8b, v2?.8b
881 mov v5?.8b, v6?.8b
882 `,
883 },
884 } {
885 t.Run(fmt.Sprintf("64bit=%v", tc._64bit), func(t *testing.T) {
886 _, _, m := newSetupWithMockContext()
887 var typ, ftyp ssa.Type
888 if tc._64bit {
889 typ = ssa.TypeI64
890 ftyp = ssa.TypeF64
891 } else {
892 typ = ssa.TypeI32
893 ftyp = ssa.TypeF32
894 }
895 tmpI := m.compiler.AllocateVReg(typ)
896 tmpF := m.compiler.AllocateVReg(ftyp)
897 rn := operandNR(m.compiler.AllocateVReg(ftyp))
898 rm := operandNR(m.compiler.AllocateVReg(ftyp))
899 rd := m.compiler.AllocateVReg(ftyp)
900
901 require.Equal(t, 1, int(tmpI.ID()))
902 require.Equal(t, 2, int(tmpF.ID()))
903 require.Equal(t, 3, int(rn.reg().ID()))
904 require.Equal(t, 4, int(rm.reg().ID()))
905 require.Equal(t, 5, int(rd.ID()))
906
907 m.lowerFcopysignImpl(rd, rn, rm, tmpI, tmpF, tc._64bit)
908 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
909 })
910 }
911 }
912
913 func TestMachine_lowerRotl(t *testing.T) {
914 for _, tc := range []struct {
915 _64bit bool
916 exp string
917 }{
918 {
919 _64bit: false,
920 exp: `
921 sub w1?, wzr, w3?
922 ror w4?, w2?, w1?
923 `,
924 },
925 {
926 _64bit: true,
927 exp: `
928 sub x1?, xzr, x3?
929 ror x4?, x2?, x1?
930 `,
931 },
932 } {
933 t.Run(fmt.Sprintf("64bit=%v", tc._64bit), func(t *testing.T) {
934 _, _, m := newSetupWithMockContext()
935 var typ ssa.Type
936 if tc._64bit {
937 typ = ssa.TypeI64
938 } else {
939 typ = ssa.TypeI32
940 }
941 tmpI := m.compiler.AllocateVReg(typ)
942 rn := operandNR(m.compiler.AllocateVReg(typ))
943 rm := operandNR(m.compiler.AllocateVReg(typ))
944 rd := m.compiler.AllocateVReg(typ)
945
946 require.Equal(t, 1, int(tmpI.ID()))
947 require.Equal(t, 2, int(rn.reg().ID()))
948 require.Equal(t, 3, int(rm.reg().ID()))
949 require.Equal(t, 4, int(rd.ID()))
950
951 m.lowerRotlImpl(rd, rn, rm, tmpI, tc._64bit)
952 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
953 })
954 }
955 }
956
957 func TestMachine_lowerAtomicRmw(t *testing.T) {
958 tests := []struct {
959 name string
960 op atomicRmwOp
961 negateArg bool
962 flipArg bool
963 _64bit bool
964 size uint64
965 exp string
966 }{
967 {
968 name: "add 32",
969 op: atomicRmwOpAdd,
970 size: 4,
971 exp: `
972 ldaddal w3?, w4?, x2?
973 `,
974 },
975 {
976 name: "add 32_16u",
977 op: atomicRmwOpAdd,
978 size: 2,
979 exp: `
980 ldaddalh w3?, w4?, x2?
981 `,
982 },
983 {
984 name: "add 32_8u",
985 op: atomicRmwOpAdd,
986 size: 1,
987 exp: `
988 ldaddalb w3?, w4?, x2?
989 `,
990 },
991 {
992 name: "add 64",
993 op: atomicRmwOpAdd,
994 size: 8,
995 _64bit: true,
996 exp: `
997 ldaddal x3?, x4?, x2?
998 `,
999 },
1000 {
1001 name: "add 64_32u",
1002 op: atomicRmwOpAdd,
1003 size: 4,
1004 _64bit: true,
1005 exp: `
1006 ldaddal w3?, w4?, x2?
1007 `,
1008 },
1009 {
1010 name: "add 64_16u",
1011 op: atomicRmwOpAdd,
1012 size: 2,
1013 _64bit: true,
1014 exp: `
1015 ldaddalh w3?, w4?, x2?
1016 `,
1017 },
1018 {
1019 name: "add 64_8u",
1020 op: atomicRmwOpAdd,
1021 size: 1,
1022 _64bit: true,
1023 exp: `
1024 ldaddalb w3?, w4?, x2?
1025 `,
1026 },
1027 {
1028 name: "sub 32",
1029 op: atomicRmwOpAdd,
1030 negateArg: true,
1031 size: 4,
1032 exp: `
1033 sub w1?, wzr, w3?
1034 ldaddal w1?, w4?, x2?
1035 `,
1036 },
1037 {
1038 name: "sub 32_16u",
1039 op: atomicRmwOpAdd,
1040 negateArg: true,
1041 size: 2,
1042 exp: `
1043 sub w1?, wzr, w3?
1044 ldaddalh w1?, w4?, x2?
1045 `,
1046 },
1047 {
1048 name: "sub 32_8u",
1049 op: atomicRmwOpAdd,
1050 negateArg: true,
1051 size: 1,
1052 exp: `
1053 sub w1?, wzr, w3?
1054 ldaddalb w1?, w4?, x2?
1055 `,
1056 },
1057 {
1058 name: "sub 64",
1059 op: atomicRmwOpAdd,
1060 negateArg: true,
1061 size: 8,
1062 _64bit: true,
1063 exp: `
1064 sub x1?, xzr, x3?
1065 ldaddal x1?, x4?, x2?
1066 `,
1067 },
1068 {
1069 name: "sub 64_32u",
1070 op: atomicRmwOpAdd,
1071 negateArg: true,
1072 size: 4,
1073 _64bit: true,
1074 exp: `
1075 sub x1?, xzr, x3?
1076 ldaddal w1?, w4?, x2?
1077 `,
1078 },
1079 {
1080 name: "sub 64_16u",
1081 op: atomicRmwOpAdd,
1082 negateArg: true,
1083 size: 2,
1084 _64bit: true,
1085 exp: `
1086 sub x1?, xzr, x3?
1087 ldaddalh w1?, w4?, x2?
1088 `,
1089 },
1090 {
1091 name: "sub 64_8u",
1092 op: atomicRmwOpAdd,
1093 negateArg: true,
1094 size: 1,
1095 _64bit: true,
1096 exp: `
1097 sub x1?, xzr, x3?
1098 ldaddalb w1?, w4?, x2?
1099 `,
1100 },
1101 {
1102 name: "and 32",
1103 op: atomicRmwOpClr,
1104 flipArg: true,
1105 size: 4,
1106 exp: `
1107 orn w1?, wzr, w3?
1108 ldclral w1?, w4?, x2?
1109 `,
1110 },
1111 {
1112 name: "and 32_16u",
1113 op: atomicRmwOpClr,
1114 flipArg: true,
1115 size: 2,
1116 exp: `
1117 orn w1?, wzr, w3?
1118 ldclralh w1?, w4?, x2?
1119 `,
1120 },
1121 {
1122 name: "and 32_8u",
1123 op: atomicRmwOpClr,
1124 flipArg: true,
1125 size: 1,
1126 exp: `
1127 orn w1?, wzr, w3?
1128 ldclralb w1?, w4?, x2?
1129 `,
1130 },
1131 {
1132 name: "and 64",
1133 op: atomicRmwOpClr,
1134 flipArg: true,
1135 size: 8,
1136 _64bit: true,
1137 exp: `
1138 orn x1?, xzr, x3?
1139 ldclral x1?, x4?, x2?
1140 `,
1141 },
1142 {
1143 name: "and 64_32u",
1144 op: atomicRmwOpClr,
1145 flipArg: true,
1146 size: 4,
1147 _64bit: true,
1148 exp: `
1149 orn x1?, xzr, x3?
1150 ldclral w1?, w4?, x2?
1151 `,
1152 },
1153 {
1154 name: "and 64_16u",
1155 op: atomicRmwOpClr,
1156 flipArg: true,
1157 size: 2,
1158 _64bit: true,
1159 exp: `
1160 orn x1?, xzr, x3?
1161 ldclralh w1?, w4?, x2?
1162 `,
1163 },
1164 {
1165 name: "and 64_8u",
1166 op: atomicRmwOpClr,
1167 flipArg: true,
1168 size: 1,
1169 _64bit: true,
1170 exp: `
1171 orn x1?, xzr, x3?
1172 ldclralb w1?, w4?, x2?
1173 `,
1174 },
1175 {
1176 name: "or 32",
1177 op: atomicRmwOpSet,
1178 size: 4,
1179 exp: `
1180 ldsetal w3?, w4?, x2?
1181 `,
1182 },
1183 {
1184 name: "or 32_16u",
1185 op: atomicRmwOpSet,
1186 size: 2,
1187 exp: `
1188 ldsetalh w3?, w4?, x2?
1189 `,
1190 },
1191 {
1192 name: "or 32_8u",
1193 op: atomicRmwOpSet,
1194 size: 1,
1195 exp: `
1196 ldsetalb w3?, w4?, x2?
1197 `,
1198 },
1199 {
1200 name: "or 64",
1201 op: atomicRmwOpSet,
1202 size: 8,
1203 _64bit: true,
1204 exp: `
1205 ldsetal x3?, x4?, x2?
1206 `,
1207 },
1208 {
1209 name: "or 64_32u",
1210 op: atomicRmwOpSet,
1211 size: 4,
1212 _64bit: true,
1213 exp: `
1214 ldsetal w3?, w4?, x2?
1215 `,
1216 },
1217 {
1218 name: "or 64_16u",
1219 op: atomicRmwOpSet,
1220 size: 2,
1221 _64bit: true,
1222 exp: `
1223 ldsetalh w3?, w4?, x2?
1224 `,
1225 },
1226 {
1227 name: "or 64_8u",
1228 op: atomicRmwOpSet,
1229 size: 1,
1230 _64bit: true,
1231 exp: `
1232 ldsetalb w3?, w4?, x2?
1233 `,
1234 },
1235 {
1236 name: "xor 32",
1237 op: atomicRmwOpEor,
1238 size: 4,
1239 exp: `
1240 ldeoral w3?, w4?, x2?
1241 `,
1242 },
1243 {
1244 name: "xor 32_16u",
1245 op: atomicRmwOpEor,
1246 size: 2,
1247 exp: `
1248 ldeoralh w3?, w4?, x2?
1249 `,
1250 },
1251 {
1252 name: "xor 32_8u",
1253 op: atomicRmwOpEor,
1254 size: 1,
1255 exp: `
1256 ldeoralb w3?, w4?, x2?
1257 `,
1258 },
1259 {
1260 name: "xor 64",
1261 op: atomicRmwOpEor,
1262 size: 8,
1263 _64bit: true,
1264 exp: `
1265 ldeoral x3?, x4?, x2?
1266 `,
1267 },
1268 {
1269 name: "xor 64_32u",
1270 op: atomicRmwOpEor,
1271 size: 4,
1272 _64bit: true,
1273 exp: `
1274 ldeoral w3?, w4?, x2?
1275 `,
1276 },
1277 {
1278 name: "xor 64_16u",
1279 op: atomicRmwOpEor,
1280 size: 2,
1281 _64bit: true,
1282 exp: `
1283 ldeoralh w3?, w4?, x2?
1284 `,
1285 },
1286 {
1287 name: "xor 64_8u",
1288 op: atomicRmwOpEor,
1289 size: 1,
1290 _64bit: true,
1291 exp: `
1292 ldeoralb w3?, w4?, x2?
1293 `,
1294 },
1295 {
1296 name: "xchg 32",
1297 op: atomicRmwOpSwp,
1298 size: 4,
1299 exp: `
1300 swpal w3?, w4?, x2?
1301 `,
1302 },
1303 {
1304 name: "xchg 32_16u",
1305 op: atomicRmwOpSwp,
1306 size: 2,
1307 exp: `
1308 swpalh w3?, w4?, x2?
1309 `,
1310 },
1311 {
1312 name: "xchg 32_8u",
1313 op: atomicRmwOpSwp,
1314 size: 1,
1315 exp: `
1316 swpalb w3?, w4?, x2?
1317 `,
1318 },
1319 {
1320 name: "xchg 64",
1321 op: atomicRmwOpSwp,
1322 size: 8,
1323 _64bit: true,
1324 exp: `
1325 swpal x3?, x4?, x2?
1326 `,
1327 },
1328 {
1329 name: "xchg 64_32u",
1330 op: atomicRmwOpSwp,
1331 size: 4,
1332 _64bit: true,
1333 exp: `
1334 swpal w3?, w4?, x2?
1335 `,
1336 },
1337 {
1338 name: "xchg 64_16u",
1339 op: atomicRmwOpSwp,
1340 size: 2,
1341 _64bit: true,
1342 exp: `
1343 swpalh w3?, w4?, x2?
1344 `,
1345 },
1346 {
1347 name: "xchg 64_8u",
1348 op: atomicRmwOpSwp,
1349 size: 1,
1350 _64bit: true,
1351 exp: `
1352 swpalb w3?, w4?, x2?
1353 `,
1354 },
1355 }
1356
1357 for _, tc := range tests {
1358 tc := tc
1359 t.Run(tc.name, func(t *testing.T) {
1360 _, _, m := newSetupWithMockContext()
1361 var typ ssa.Type
1362 if tc._64bit {
1363 typ = ssa.TypeI64
1364 } else {
1365 typ = ssa.TypeI32
1366 }
1367 tmp := m.compiler.AllocateVReg(typ)
1368 rn := m.compiler.AllocateVReg(ssa.TypeI64)
1369 rs := m.compiler.AllocateVReg(typ)
1370 rt := m.compiler.AllocateVReg(typ)
1371
1372 require.Equal(t, 1, int(tmp.ID()))
1373 require.Equal(t, 2, int(rn.ID()))
1374 require.Equal(t, 3, int(rs.ID()))
1375 require.Equal(t, 4, int(rt.ID()))
1376
1377 m.lowerAtomicRmwImpl(tc.op, rn, rs, rt, tmp, tc.size, tc.negateArg, tc.flipArg, tc._64bit)
1378 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
1379 })
1380 }
1381 }
1382
1383 func TestMachine_lowerAtomicCas(t *testing.T) {
1384 tests := []struct {
1385 name string
1386 _64bit bool
1387 size uint64
1388 exp string
1389 }{
1390 {
1391 name: "cas 32",
1392 size: 4,
1393 exp: `
1394 casal w2?, w3?, x1?
1395 `,
1396 },
1397 {
1398 name: "cas 32_16u",
1399 size: 2,
1400 exp: `
1401 casalh w2?, w3?, x1?
1402 `,
1403 },
1404 {
1405 name: "cas 32_8u",
1406 size: 1,
1407 exp: `
1408 casalb w2?, w3?, x1?
1409 `,
1410 },
1411 {
1412 name: "cas 64",
1413 size: 8,
1414 _64bit: true,
1415 exp: `
1416 casal x2?, x3?, x1?
1417 `,
1418 },
1419 {
1420 name: "cas 64_32u",
1421 size: 4,
1422 _64bit: true,
1423 exp: `
1424 casal w2?, w3?, x1?
1425 `,
1426 },
1427 {
1428 name: "cas 64_16u",
1429 size: 2,
1430 _64bit: true,
1431 exp: `
1432 casalh w2?, w3?, x1?
1433 `,
1434 },
1435 {
1436 name: "cas 64_8u",
1437 size: 1,
1438 _64bit: true,
1439 exp: `
1440 casalb w2?, w3?, x1?
1441 `,
1442 },
1443 }
1444
1445 for _, tc := range tests {
1446 tc := tc
1447 t.Run(tc.name, func(t *testing.T) {
1448 _, _, m := newSetupWithMockContext()
1449 var typ ssa.Type
1450 if tc._64bit {
1451 typ = ssa.TypeI64
1452 } else {
1453 typ = ssa.TypeI32
1454 }
1455 rn := m.compiler.AllocateVReg(ssa.TypeI64)
1456 rs := m.compiler.AllocateVReg(typ)
1457 rt := m.compiler.AllocateVReg(typ)
1458
1459 require.Equal(t, 1, int(rn.ID()))
1460 require.Equal(t, 2, int(rs.ID()))
1461 require.Equal(t, 3, int(rt.ID()))
1462
1463 m.lowerAtomicCasImpl(rn, rs, rt, tc.size)
1464 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
1465 })
1466 }
1467 }
1468
1469 func TestMachine_lowerAtomicLoad(t *testing.T) {
1470 tests := []struct {
1471 name string
1472 _64bit bool
1473 size uint64
1474 exp string
1475 }{
1476 {
1477 name: "load 32",
1478 size: 4,
1479 exp: `
1480 ldar w2?, x1?
1481 `,
1482 },
1483 {
1484 name: "load 32_16u",
1485 size: 2,
1486 exp: `
1487 ldarh w2?, x1?
1488 `,
1489 },
1490 {
1491 name: "load 32_8u",
1492 size: 1,
1493 exp: `
1494 ldarb w2?, x1?
1495 `,
1496 },
1497 {
1498 name: "load 64",
1499 size: 8,
1500 _64bit: true,
1501 exp: `
1502 ldar x2?, x1?
1503 `,
1504 },
1505 {
1506 name: "load 64_32u",
1507 size: 4,
1508 _64bit: true,
1509 exp: `
1510 ldar w2?, x1?
1511 `,
1512 },
1513 {
1514 name: "load 64_16u",
1515 size: 2,
1516 _64bit: true,
1517 exp: `
1518 ldarh w2?, x1?
1519 `,
1520 },
1521 {
1522 name: "load 64_8u",
1523 size: 1,
1524 _64bit: true,
1525 exp: `
1526 ldarb w2?, x1?
1527 `,
1528 },
1529 }
1530
1531 for _, tc := range tests {
1532 tc := tc
1533 t.Run(tc.name, func(t *testing.T) {
1534 _, _, m := newSetupWithMockContext()
1535 var typ ssa.Type
1536 if tc._64bit {
1537 typ = ssa.TypeI64
1538 } else {
1539 typ = ssa.TypeI32
1540 }
1541 rn := m.compiler.AllocateVReg(ssa.TypeI64)
1542 rt := m.compiler.AllocateVReg(typ)
1543
1544 require.Equal(t, 1, int(rn.ID()))
1545 require.Equal(t, 2, int(rt.ID()))
1546
1547 m.lowerAtomicLoadImpl(rn, rt, tc.size)
1548 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
1549 })
1550 }
1551 }
1552
1553 func TestMachine_lowerAtomicStore(t *testing.T) {
1554 tests := []struct {
1555 name string
1556 _64bit bool
1557 size uint64
1558 exp string
1559 }{
1560 {
1561 name: "store 32",
1562 size: 4,
1563 exp: `
1564 stlr w2?, x1?
1565 `,
1566 },
1567 {
1568 name: "store 32_16u",
1569 size: 2,
1570 exp: `
1571 stlrh w2?, x1?
1572 `,
1573 },
1574 {
1575 name: "store 32_8u",
1576 size: 1,
1577 exp: `
1578 stlrb w2?, x1?
1579 `,
1580 },
1581 {
1582 name: "store 64",
1583 size: 8,
1584 _64bit: true,
1585 exp: `
1586 stlr x2?, x1?
1587 `,
1588 },
1589 {
1590 name: "store 64_32u",
1591 size: 4,
1592 _64bit: true,
1593 exp: `
1594 stlr w2?, x1?
1595 `,
1596 },
1597 {
1598 name: "store 64_16u",
1599 size: 2,
1600 _64bit: true,
1601 exp: `
1602 stlrh w2?, x1?
1603 `,
1604 },
1605 {
1606 name: "store 64_8u",
1607 size: 1,
1608 _64bit: true,
1609 exp: `
1610 stlrb w2?, x1?
1611 `,
1612 },
1613 }
1614
1615 for _, tc := range tests {
1616 tc := tc
1617 t.Run(tc.name, func(t *testing.T) {
1618 _, _, m := newSetupWithMockContext()
1619 var typ ssa.Type
1620 if tc._64bit {
1621 typ = ssa.TypeI64
1622 } else {
1623 typ = ssa.TypeI32
1624 }
1625 rn := operandNR(m.compiler.AllocateVReg(ssa.TypeI64))
1626 rt := operandNR(m.compiler.AllocateVReg(typ))
1627
1628 require.Equal(t, 1, int(rn.reg().ID()))
1629 require.Equal(t, 2, int(rt.reg().ID()))
1630
1631 m.lowerAtomicStoreImpl(rn, rt, tc.size)
1632 require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
1633 })
1634 }
1635 }