; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s define { , } @foo_ld2_nxv16i8( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv16i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv32i8(ptr %p, i32 1, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv32i8( %wide.masked.vec) ret { , } %deinterleaved.vec } define { , } @foo_ld2_nxv8i16( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv8i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv16i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv16i16.p0(ptr %p, i32 2, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv16i16( %wide.masked.vec) ret { , } %deinterleaved.vec } define { , } @foo_ld2_nxv4f32( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv4f32: ; CHECK: // %bb.0: ; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv8i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv8f32(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv8f32( %wide.masked.vec) ret { , } %deinterleaved.vec } define { , } @foo_ld2_nxv2f64( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv2f64: ; CHECK: // %bb.0: ; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv4i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv4f64(ptr %p, i32 8, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv4f64( %wide.masked.vec) ret { , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv16i8( %mask, ptr %p) { ; CHECK-LABEL: foo_ld4_nxv16i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv64i1( %mask, %mask, %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv64i8(ptr %p, i32 1, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv64i8( %wide.masked.vec) ret { , , , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv8i16( %mask, ptr %p) { ; CHECK-LABEL: foo_ld4_nxv8i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv32i1( %mask, %mask, %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv32i16(ptr %p, i32 2, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv32i16( %wide.masked.vec) ret { , , , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv4f32( %mask, ptr %p) { ; CHECK-LABEL: foo_ld4_nxv4f32: ; CHECK: // %bb.0: ; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv16i1( %mask, %mask, %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv16f32(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv16f32( %wide.masked.vec) ret { , , , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv2f64( %mask, ptr %p) { ; CHECK-LABEL: foo_ld4_nxv2f64: ; CHECK: // %bb.0: ; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv8i1( %mask, %mask, %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv8f64(ptr %p, i32 8, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv8f64( %wide.masked.vec) ret { , , , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv16i8_mul_use_of_mask( %mask, ptr %p, ptr %p2) { ; CHECK-LABEL: foo_ld4_nxv16i8_mul_use_of_mask: ; CHECK: // %bb.0: ; CHECK-NEXT: zip1 p2.b, p0.b, p0.b ; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0] ; CHECK-NEXT: zip2 p1.b, p0.b, p0.b ; CHECK-NEXT: zip1 p3.b, p2.b, p2.b ; CHECK-NEXT: zip2 p0.b, p1.b, p1.b ; CHECK-NEXT: zip1 p1.b, p1.b, p1.b ; CHECK-NEXT: zip2 p2.b, p2.b, p2.b ; CHECK-NEXT: // fake_use: $p3 ; CHECK-NEXT: // fake_use: $p2 ; CHECK-NEXT: // fake_use: $p1 ; CHECK-NEXT: // fake_use: $p0 ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv64i1( %mask, %mask, %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv64i8(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv64i8( %wide.masked.vec) call void (...) @llvm.fake.use( %interleaved.mask) ret { , , , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv16i8_mask_of_interleaved_ones(ptr %p) { ; CHECK-LABEL: foo_ld4_nxv16i8_mask_of_interleaved_ones: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0] ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave4.nxv64i1( splat(i1 1), splat(i1 1), splat(i1 1), splat(i1 1)) %wide.masked.vec = call @llvm.masked.load.nxv64i8(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv64i8( %wide.masked.vec) ret { , , , } %deinterleaved.vec } define { , , , } @foo_ld4_nxv16i8_mask_of_ones(ptr %p) { ; CHECK-LABEL: foo_ld4_nxv16i8_mask_of_ones: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0] ; CHECK-NEXT: ret %wide.masked.vec = call @llvm.masked.load.nxv64i8(ptr %p, i32 4, splat(i1 1), poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv64i8( %wide.masked.vec) ret { , , , } %deinterleaved.vec } ; Negative tests define { , } @foo_ld2_nxv16i8_mul_use_of_load( %mask, ptr %p, ptr %p2) { ; CHECK-LABEL: foo_ld2_nxv16i8_mul_use_of_load: ; CHECK: // %bb.0: ; CHECK-NEXT: zip1 p1.b, p0.b, p0.b ; CHECK-NEXT: zip2 p0.b, p0.b, p0.b ; CHECK-NEXT: ld1b { z3.b }, p1/z, [x0] ; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, #1, mul vl] ; CHECK-NEXT: uzp1 z0.b, z3.b, z2.b ; CHECK-NEXT: uzp2 z1.b, z3.b, z2.b ; CHECK-NEXT: // fake_use: $z3 ; CHECK-NEXT: // fake_use: $z2 ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv32i8(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv32i8( %wide.masked.vec) call void (...) @llvm.fake.use( %wide.masked.vec) ret { , } %deinterleaved.vec } ; Mask must be an interleave of identical masks. define { , } @foo_ld2_nxv16i8_bad_mask( %mask, %mask2, ptr %p, ptr %p2) { ; CHECK-LABEL: foo_ld2_nxv16i8_bad_mask: ; CHECK: // %bb.0: ; CHECK-NEXT: zip1 p2.b, p0.b, p1.b ; CHECK-NEXT: zip2 p0.b, p0.b, p1.b ; CHECK-NEXT: ld1b { z2.b }, p2/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, #1, mul vl] ; CHECK-NEXT: uzp1 z0.b, z2.b, z1.b ; CHECK-NEXT: uzp2 z1.b, z2.b, z1.b ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask2) %wide.masked.vec = call @llvm.masked.load.nxv32i8(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv32i8( %wide.masked.vec) ret { , } %deinterleaved.vec } ; Number of parts in mask interleave must match deinterleave. define { , , , } @foo_ld4_nxv16i8_bad_mask2( %mask, ptr %p, ptr %p2) { ; CHECK-LABEL: foo_ld4_nxv16i8_bad_mask2: ; CHECK: // %bb.0: ; CHECK-NEXT: zip1 p2.b, p1.b, p1.b ; CHECK-NEXT: zip2 p1.b, p1.b, p1.b ; CHECK-NEXT: zip2 p3.b, p0.b, p0.b ; CHECK-NEXT: ld1b { z3.b }, p2/z, [x0, #2, mul vl] ; CHECK-NEXT: zip1 p0.b, p0.b, p0.b ; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, #3, mul vl] ; CHECK-NEXT: ld1b { z0.b }, p3/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] ; CHECK-NEXT: uzp1 z4.b, z3.b, z2.b ; CHECK-NEXT: uzp2 z3.b, z3.b, z2.b ; CHECK-NEXT: uzp1 z5.b, z1.b, z0.b ; CHECK-NEXT: uzp2 z6.b, z1.b, z0.b ; CHECK-NEXT: uzp1 z0.b, z5.b, z4.b ; CHECK-NEXT: uzp1 z1.b, z6.b, z3.b ; CHECK-NEXT: uzp2 z2.b, z5.b, z4.b ; CHECK-NEXT: uzp2 z3.b, z6.b, z3.b ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv64i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv64i8(ptr %p, i32 4, %interleaved.mask, poison) %deinterleaved.vec = call { , , , } @llvm.vector.deinterleave4.nxv64i8( %wide.masked.vec) ret { , , , } %deinterleaved.vec } ; Mask must come from an interleave or a splat. define { , } @foo_ld2_nxv16i8_bad_mask3( %mask, ptr %p, ptr %p2) { ; CHECK-LABEL: foo_ld2_nxv16i8_bad_mask3: ; CHECK: // %bb.0: ; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0] ; CHECK-NEXT: uzp1 z0.b, z2.b, z1.b ; CHECK-NEXT: uzp2 z1.b, z2.b, z1.b ; CHECK-NEXT: ret %wide.masked.vec = call @llvm.masked.load.nxv32i8(ptr %p, i32 4, %mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv32i8( %wide.masked.vec) ret { , } %deinterleaved.vec } ; Each deinterleaved vector must be exactly 128 bits. define { , } @foo_ld2_nxv8i8( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv8i8: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 p1.h, p0.h, p0.h ; CHECK-NEXT: zip1 p0.h, p0.h, p0.h ; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: uunpkhi z1.h, z0.b ; CHECK-NEXT: uunpklo z2.h, z0.b ; CHECK-NEXT: uzp1 z0.h, z2.h, z1.h ; CHECK-NEXT: uzp2 z1.h, z2.h, z1.h ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv16i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv16i8(ptr %p, i32 1, %interleaved.mask, poison) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv16i8( %wide.masked.vec) ret { , } %deinterleaved.vec } ; Passthru must be poison or zero. define { , } @foo_ld2_nxv16i8_bad_passthru( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv16i8_bad_passthru: ; CHECK: // %bb.0: ; CHECK-NEXT: zip1 p1.b, p0.b, p0.b ; CHECK-NEXT: mov z0.b, #3 // =0x3 ; CHECK-NEXT: zip2 p0.b, p0.b, p0.b ; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, #1, mul vl] ; CHECK-NEXT: sel z2.b, p1, z2.b, z0.b ; CHECK-NEXT: sel z1.b, p0, z1.b, z0.b ; CHECK-NEXT: uzp1 z0.b, z2.b, z1.b ; CHECK-NEXT: uzp2 z1.b, z2.b, z1.b ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv32i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv32i8(ptr %p, i32 1, %interleaved.mask, splat(i8 3)) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv32i8( %wide.masked.vec) ret { , } %deinterleaved.vec } define { , } @foo_deinterleave2_not_load( %vec1, %vec2) { ; CHECK-LABEL: foo_deinterleave2_not_load: ; CHECK: // %bb.0: ; CHECK-NEXT: uzp1 z2.h, z0.h, z1.h ; CHECK-NEXT: uzp2 z1.h, z0.h, z1.h ; CHECK-NEXT: mov z0.d, z2.d ; CHECK-NEXT: ret %bad.vec.init = call @llvm.vector.insert.nxv16i16( poison, %vec1, i64 0) %bad.vec = call @llvm.vector.insert.nxv16i16( %bad.vec.init, %vec2, i64 8) %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv16i16( %bad.vec) ret { , } %deinterleaved.vec } define { , } @foo_ld2_nxv8i8_exti16( %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_nxv8i8_exti16: ; CHECK: // %bb.0: ; CHECK-NEXT: zip2 p1.s, p0.s, p0.s ; CHECK-NEXT: zip1 p0.s, p0.s, p0.s ; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h ; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] ; CHECK-NEXT: uunpkhi z1.s, z0.h ; CHECK-NEXT: uunpklo z2.s, z0.h ; CHECK-NEXT: uzp1 z0.s, z2.s, z1.s ; CHECK-NEXT: uzp2 z1.s, z2.s, z1.s ; CHECK-NEXT: ret %interleaved.mask = call @llvm.vector.interleave2.nxv8i1( %mask, %mask) %wide.masked.vec = call @llvm.masked.load.nxv8i8(ptr %p, i32 1, %interleaved.mask, poison) %wide.masked.vec.ext = zext %wide.masked.vec to %deinterleaved.vec = call { , } @llvm.vector.deinterleave2.nxv8i16( %wide.masked.vec.ext) ret { , } %deinterleaved.vec }