; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s define void @foo_st2_nxv16i8( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st2_nxv8i16( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv8i16: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: st2h { z0.h, z1.h }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv16i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv16i16( %val1, %val2) call void @llvm.masked.store.nxv16i16.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st2_nxv4i32( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv4i32: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv8i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv8i32( %val1, %val2) call void @llvm.masked.store.nxv8i32.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st2_nxv2i64( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv2i64: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: st2d { z0.d, z1.d }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv4i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv4i64( %val1, %val2) call void @llvm.masked.store.nxv4i64.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st4_nxv16i8( %mask, %val1, %val2, %val3, %val4, ptr %p) { ; CHECK-LABEL: foo_st4_nxv16i8: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv64i1( %mask, %mask, %mask, %mask) %interleaved.value = call @llvm.vector.interleave4.nxv64i8( %val1, %val2, %val3, %val4) call void @llvm.masked.store.nxv64i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st4_nxv8i16( %mask, %val1, %val2, %val3, %val4, ptr %p) { ; CHECK-LABEL: foo_st4_nxv8i16: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: st4h { z0.h - z3.h }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv32i1( %mask, %mask, %mask, %mask) %interleaved.value = call @llvm.vector.interleave4.nxv32i16( %val1, %val2, %val3, %val4) call void @llvm.masked.store.nxv32i16.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st4_nxv4i32( %mask, %val1, %val2, %val3, %val4, ptr %p) { ; CHECK-LABEL: foo_st4_nxv4i32: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: st4w { z0.s - z3.s }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv16i1( %mask, %mask, %mask, %mask) %interleaved.value = call @llvm.vector.interleave4.nxv16i32( %val1, %val2, %val3, %val4) call void @llvm.masked.store.nxv16i32.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st4_nxv2i64( %mask, %val1, %val2, %val3, %val4, ptr %p) { ; CHECK-LABEL: foo_st4_nxv2i64: ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; CHECK-NEXT: st4d { z0.d - z3.d }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv4i1( %mask, %mask, %mask, %mask) %interleaved.value = call @llvm.vector.interleave4.nxv8i64( %val1, %val2, %val3, %val4) call void @llvm.masked.store.nxv8i64.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st2_nxv16i8_mul_use_mask( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_mul_use_mask: ; CHECK: // %bb.0: ; CHECK-NEXT: zip1 p1.b, p0.b, p0.b ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: zip2 p2.b, p0.b, p0.b ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0] ; CHECK-NEXT: // fake_use: $p1 ; CHECK-NEXT: // fake_use: $p2 ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) call void (...) @llvm.fake.use( %interleaved.mask) ret void } define void @foo_st2_nxv16i8_mask_of_interleaved_ones( %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_mask_of_interleaved_ones: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1 ; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( splat(i1 1), splat(i1 1)) %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st2_nxv16i8_all_false_mask( %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_all_false_mask: ; CHECK: // %bb.0: ; CHECK-NEXT: ret %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, splat(i1 0)) ret void } ; DAGCombiner optimises the masked store to a normal store before we even ; reach performMSTORECombine so we never get a chance to convert this to st2b. define void @foo_st2_nxv16i8_all_true_mask( %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_all_true_mask: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z2.b, z0.b, z1.b ; CHECK-NEXT: zip1 z0.b, z0.b, z1.b ; CHECK-NEXT: str z2, [x0, #1, mul vl] ; CHECK-NEXT: str z0, [x0] ; CHECK-NEXT: ret %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, splat(i1 1)) ret void } ; Negative tests define void @foo_st2_nxv16i8_mul_use_value( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_mul_use_value: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z2.b, z0.b, z1.b ; CHECK-NEXT: zip1 z0.b, z0.b, z1.b ; CHECK-NEXT: zip2 p1.b, p0.b, p0.b ; CHECK-NEXT: zip1 p0.b, p0.b, p0.b ; CHECK-NEXT: st1b { z2.b }, p1, [x0, #1, mul vl] ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: // fake_use: $z0 ; CHECK-NEXT: // fake_use: $z2 ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) call void (...) @llvm.fake.use( %interleaved.value) ret void } ; Mask must be an interleave of identical masks. define void @foo_st2_nxv16i8_bad_mask( %mask, %mask2, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_bad_mask: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z2.b, z0.b, z1.b ; CHECK-NEXT: zip1 z0.b, z0.b, z1.b ; CHECK-NEXT: zip2 p2.b, p0.b, p1.b ; CHECK-NEXT: zip1 p0.b, p0.b, p1.b ; CHECK-NEXT: st1b { z2.b }, p2, [x0, #1, mul vl] ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask2) %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } ; Number of parts in mask interleave must match interleave. define void @foo_st4_nxv4i32_bad_mask2( %mask, %val1, %val2, %val3, %val4, ptr %p) { ; CHECK-LABEL: foo_st4_nxv4i32_bad_mask2: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z4.s, z1.s, z3.s ; CHECK-NEXT: zip2 z5.s, z0.s, z2.s ; CHECK-NEXT: zip2 p1.h, p0.h, p0.h ; CHECK-NEXT: zip1 z1.s, z1.s, z3.s ; CHECK-NEXT: zip1 z0.s, z0.s, z2.s ; CHECK-NEXT: zip1 p0.h, p0.h, p0.h ; CHECK-NEXT: punpkhi p2.h, p1.b ; CHECK-NEXT: zip2 z2.s, z5.s, z4.s ; CHECK-NEXT: zip1 z3.s, z5.s, z4.s ; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: zip2 z4.s, z0.s, z1.s ; CHECK-NEXT: zip1 z0.s, z0.s, z1.s ; CHECK-NEXT: st1w { z2.s }, p2, [x0, #3, mul vl] ; CHECK-NEXT: punpkhi p2.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1w { z3.s }, p1, [x0, #2, mul vl] ; CHECK-NEXT: st1w { z4.s }, p2, [x0, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv16i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave4.nxv16i32( %val1, %val2, %val3, %val4) call void @llvm.masked.store.nxv16i32.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } ; Mask must come from an interleave or a splat. define void @foo_st2_nxv16i8_bad_mask3( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv16i8_bad_mask3: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z2.b, z0.b, z1.b ; CHECK-NEXT: zip1 z0.b, z0.b, z1.b ; CHECK-NEXT: st1b { z2.b }, p1, [x0, #1, mul vl] ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %interleaved.value = call @llvm.vector.interleave2.nxv32i8( %val1, %val2) call void @llvm.masked.store.nxv32i8.p0( %interleaved.value, ptr %p, i32 1, %mask) ret void } ; Each interleaved vector must be exactly 128 bits. define void @foo_st2_nxv8i8( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv8i8: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z2.h, z0.h, z1.h ; CHECK-NEXT: zip1 z0.h, z0.h, z1.h ; CHECK-NEXT: zip2 p1.h, p0.h, p0.h ; CHECK-NEXT: zip1 p0.h, p0.h, p0.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z2.b ; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv16i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv16i8( %val1, %val2) call void @llvm.masked.store.nxv16i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) ret void } define void @foo_st2_nxv8i8_trunc( %mask, %val1, %val2, ptr %p) { ; CHECK-LABEL: foo_st2_nxv8i8_trunc: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 z2.s, z0.s, z1.s ; CHECK-NEXT: zip1 z0.s, z0.s, z1.s ; CHECK-NEXT: zip2 p1.s, p0.s, p0.s ; CHECK-NEXT: zip1 p0.s, p0.s, p0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h ; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h ; CHECK-NEXT: st1b { z0.h }, p0, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv8i1( %mask, %mask) %interleaved.value = call @llvm.vector.interleave2.nxv8i16( %val1, %val2) %trunc.value = trunc %interleaved.value to call void @llvm.masked.store.nxv8i8.p0( %trunc.value, ptr %p, i32 1, %interleaved.mask) ret void }