/* * Copyright (C) 2023 Linux Studio Plugins Project * (C) 2023 Vladimir Sadovnikov * * This file is part of lsp-dsp-lib * Created on: 3 февр. 2023 г. * * lsp-dsp-lib is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * lsp-dsp-lib is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with lsp-dsp-lib. If not, see . */ #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LRAMP_H_ #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LRAMP_H_ #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL #error "This header should not be included directly" #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */ namespace lsp { namespace asimd { IF_ARCH_AARCH64( static const float lramp_const[] __lsp_aligned16 = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, /* Initial values 0..7 */ 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, /* Initial values 8..15 */ 16.0f, 16.0f, 16.0f, 16.0f, 16.0f, 16.0f, 16.0f, 16.0f /* Step */ }; ) void lramp_set1(float *dst, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fill(dst, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( __ASM_EMIT("ld1r {v14.4s}, [%[v1]]") __ASM_EMIT("ld1r {v15.4s}, [%[delta]]") __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") __ASM_EMIT("ldp q10, q11, [%[CC], #0x20]") __ASM_EMIT("ldp q12, q13, [%[CC], #0x40]") /* v8.4s = i[0], v9.4s = i[1], v10.4s = i[2], v11.4s = i[3] */ /* v12.4s = step, v13.4s = step */ /* v14.4s = v1, v15.4s = delta */ /* 16x blocks */ __ASM_EMIT("subs %[count], %[count], #16") __ASM_EMIT("b.lo 2f") __ASM_EMIT("1:") __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ __ASM_EMIT("fmul v2.4s, v10.4s, v15.4s") /* v2.4s = i[2] * delta */ __ASM_EMIT("fmul v3.4s, v11.4s, v15.4s") /* v3.4s = i[3] * delta */ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ __ASM_EMIT("fadd v2.4s, v2.4s, v14.4s") /* v2.4s = v1 + i[0] * delta */ __ASM_EMIT("fadd v3.4s, v3.4s, v14.4s") /* v3.4s = v1 + i[0] * delta */ __ASM_EMIT("fadd v8.4s, v8.4s, v12.4s") /* v8.4s = i[0]' = i[0] + step */ __ASM_EMIT("fadd v9.4s, v9.4s, v13.4s") /* v9.4s = i[1]' = i[1] + step */ __ASM_EMIT("fadd v10.4s, v10.4s, v12.4s") /* v10.4s = i[2]' = i[2] + step */ __ASM_EMIT("fadd v11.4s, v11.4s, v13.4s") /* v11.4s = i[3]' = i[3] + step */ __ASM_EMIT("stp q0, q1, [%[dst], 0x00]") __ASM_EMIT("stp q2, q3, [%[dst], 0x20]") __ASM_EMIT("subs %[count], %[count], #16") __ASM_EMIT("add %[dst], %[dst], #0x40") __ASM_EMIT("b.hs 1b") __ASM_EMIT("2:") /* 8x blocks */ __ASM_EMIT("adds %[count], %[count], #8") __ASM_EMIT("b.lt 4f") __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ __ASM_EMIT("mov v8.16b, v10.16b") /* v8.4s = i[0]' = i[0] + step */ __ASM_EMIT("mov v9.16b, v11.16b") /* v9.4s = i[1]' = i[1] + step */ __ASM_EMIT("stp q0, q1, [%[dst], 0x00]") __ASM_EMIT("sub %[count], %[count], #8") __ASM_EMIT("add %[dst], %[dst], #0x20") __ASM_EMIT("4:") /* 4x block */ __ASM_EMIT("adds %[count], %[count], #4") __ASM_EMIT("b.lt 6f") __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ __ASM_EMIT("str q0, [%[dst], 0x00]") __ASM_EMIT("sub %[count], %[count], #4") __ASM_EMIT("add %[dst], %[dst], #0x10") __ASM_EMIT("6:") /* 1x blocks */ __ASM_EMIT("adds %[count], %[count], #3") __ASM_EMIT("b.lt 8f") __ASM_EMIT("7:") __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ __ASM_EMIT("st1 {v0.s}[0], [%[dst]]") __ASM_EMIT("subs %[count], %[count], #1") __ASM_EMIT("add %[dst], %[dst], #0x04") __ASM_EMIT("b.ge 7b") __ASM_EMIT("8:") : [count] "+r" (count), [dst] "+r" (dst) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } #define OP_SINC(a) a #define OP_SNOINC(a) #define LRAMP_CORE(DST, SRC, SINC) \ __ASM_EMIT("ld1r {v14.4s}, [%[v1]]") \ __ASM_EMIT("ld1r {v15.4s}, [%[delta]]") \ __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") \ __ASM_EMIT("ldp q10, q11, [%[CC], #0x20]") \ __ASM_EMIT("ldp q12, q13, [%[CC], #0x40]") \ /* v8.4s = i[0], v9.4s = i[1], v10.4s = i[2], v11.4s = i[3] */ \ /* v12.4s = step, v13.4s = step */ \ /* v14.4s = v1, v15.4s = delta */ \ /* 16x blocks */ \ __ASM_EMIT("subs %[count], %[count], #16") \ __ASM_EMIT("b.lo 2f") \ __ASM_EMIT("1:") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \ __ASM_EMIT("fmul v2.4s, v10.4s, v15.4s") /* v2.4s = i[2] * delta */ \ __ASM_EMIT("fmul v3.4s, v11.4s, v15.4s") /* v3.4s = i[3] * delta */ \ __ASM_EMIT("ldp q4, q5, [%[" SRC "], 0x00]") \ __ASM_EMIT("ldp q6, q7, [%[" SRC "], 0x20]") \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v2.4s, v2.4s, v14.4s") /* v2.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v3.4s, v3.4s, v14.4s") /* v3.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v8.4s, v8.4s, v12.4s") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("fadd v9.4s, v9.4s, v13.4s") /* v9.4s = i[1]' = i[1] + step */ \ __ASM_EMIT("fadd v10.4s, v10.4s, v12.4s") /* v10.4s = i[2]' = i[2] + step */ \ __ASM_EMIT("fadd v11.4s, v11.4s, v13.4s") /* v11.4s = i[3]' = i[3] + step */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = src[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") /* v2.4s = src[2] * (v1 + i[2] * delta) */ \ __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") /* v3.4s = src[3] * (v1 + i[3] * delta) */ \ __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \ __ASM_EMIT("stp q2, q3, [%[" DST "], 0x20]") \ __ASM_EMIT("subs %[count], %[count], #16") \ __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x40")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \ __ASM_EMIT("b.hs 1b") \ __ASM_EMIT("2:") \ /* 8x blocks */ \ __ASM_EMIT("adds %[count], %[count], #8") \ __ASM_EMIT("b.lt 4f") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \ __ASM_EMIT("ldp q4, q5, [%[" SRC "], 0x00]") \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \ __ASM_EMIT("mov v8.16b, v10.16b") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("mov v9.16b, v11.16b") /* v9.4s = i[1]' = i[1] + step */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = src[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \ __ASM_EMIT("sub %[count], %[count], #8") \ __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x20")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \ __ASM_EMIT("4:") \ /* 4x block */ \ __ASM_EMIT("adds %[count], %[count], #4") \ __ASM_EMIT("b.lt 6f") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("ldr q4, [%[" SRC "], 0x00]") \ __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("str q0, [%[" DST "], 0x00]") \ __ASM_EMIT("sub %[count], %[count], #4") \ __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x10")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \ __ASM_EMIT("6:") \ /* 1x blocks */ \ __ASM_EMIT("adds %[count], %[count], #3") \ __ASM_EMIT("b.lt 8f") \ __ASM_EMIT("7:") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("ld1r {v4.4s}, [%[" SRC "]]") \ __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \ __ASM_EMIT("subs %[count], %[count], #1") \ __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x04")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \ __ASM_EMIT("b.ge 7b") \ __ASM_EMIT("8:") void lramp1(float *dst, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::mul_k2(dst, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_CORE("dst", "dst", OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::mul_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_CORE("dst", "src", OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } #undef LRAMP_CORE #define OP_DSEL(a, b) a #define OP_RSEL(a, b) b #define LRAMP_OP3_CORE(DST, A, B, OP, SEL, AINC) \ __ASM_EMIT("ld1r {v14.4s}, [%[v1]]") \ __ASM_EMIT("ld1r {v15.4s}, [%[delta]]") \ __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") \ __ASM_EMIT("ldp q10, q11, [%[CC], #0x20]") \ __ASM_EMIT("ldp q12, q13, [%[CC], #0x40]") \ /* v8.4s = i[0], v9.4s = i[1], v10.4s = i[2], v11.4s = i[3] */ \ /* v12.4s = step, v13.4s = step */ \ /* v14.4s = v1, v15.4s = delta */ \ /* 16x blocks */ \ __ASM_EMIT("subs %[count], %[count], #16") \ __ASM_EMIT("b.lo 2f") \ __ASM_EMIT("1:") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \ __ASM_EMIT("fmul v2.4s, v10.4s, v15.4s") /* v2.4s = i[2] * delta */ \ __ASM_EMIT("fmul v3.4s, v11.4s, v15.4s") /* v3.4s = i[3] * delta */ \ __ASM_EMIT("ldp q4, q5, [%[" B "], 0x00]") \ __ASM_EMIT("ldp q6, q7, [%[" B "], 0x20]") \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v2.4s, v2.4s, v14.4s") /* v2.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v3.4s, v3.4s, v14.4s") /* v3.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = b[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") /* v2.4s = b[2] * (v1 + i[2] * delta) */ \ __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") /* v3.4s = b[3] * (v1 + i[3] * delta) */ \ __ASM_EMIT("ldp q4, q5, [%[" A "], 0x00]") \ __ASM_EMIT("ldp q6, q7, [%[" A "], 0x20]") \ __ASM_EMIT("fadd v8.4s, v8.4s, v12.4s") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("fadd v9.4s, v9.4s, v13.4s") /* v9.4s = i[1]' = i[1] + step */ \ __ASM_EMIT("fadd v10.4s, v10.4s, v12.4s") /* v10.4s = i[2]' = i[2] + step */ \ __ASM_EMIT("fadd v11.4s, v11.4s, v13.4s") /* v11.4s = i[3]' = i[3] + step */ \ __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT(OP " v1.4s, " SEL("v5.4s, v1.4s", "v1.4s, v5.4s")) /* v1.4s = a[1] OP b[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT(OP " v2.4s, " SEL("v6.4s, v2.4s", "v2.4s, v6.4s")) /* v2.4s = a[2] OP b[2] * (v1 + i[2] * delta) */ \ __ASM_EMIT(OP " v3.4s, " SEL("v7.4s, v3.4s", "v3.4s, v7.4s")) /* v3.4s = a[3] OP b[3] * (v1 + i[3] * delta) */ \ __ASM_EMIT("subs %[count], %[count], #16") \ __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \ __ASM_EMIT("stp q2, q3, [%[" DST "], 0x20]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x40") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x40")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \ __ASM_EMIT("b.hs 1b") \ __ASM_EMIT("2:") \ /* 8x blocks */ \ __ASM_EMIT("adds %[count], %[count], #8") \ __ASM_EMIT("b.lt 4f") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \ __ASM_EMIT("ldp q4, q5, [%[" B "], 0x00]") \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = b[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT("ldp q4, q5, [%[" A "], 0x00]") \ __ASM_EMIT("mov v8.16b, v10.16b") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("mov v9.16b, v11.16b") /* v9.4s = i[1]' = i[1] + step */ \ __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT(OP " v1.4s, " SEL("v5.4s, v1.4s", "v1.4s, v5.4s")) /* v1.4s = a[1] OP b[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT("sub %[count], %[count], #8") \ __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x20") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x20")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \ __ASM_EMIT("4:") \ /* 4x block */ \ __ASM_EMIT("adds %[count], %[count], #4") \ __ASM_EMIT("b.lt 6f") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("ldr q4, [%[" B "], 0x00]") \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("ldr q4, [%[" A "], 0x00]") \ __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("sub %[count], %[count], #4") \ __ASM_EMIT("str q0, [%[" DST "], 0x00]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x10") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x10")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \ __ASM_EMIT("6:") \ /* 1x blocks */ \ __ASM_EMIT("adds %[count], %[count], #3") \ __ASM_EMIT("b.lt 8f") \ __ASM_EMIT("7:") \ __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \ __ASM_EMIT("ld1r {v4.4s}, [%[" B "]]") \ __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("ld1r {v4.4s}, [%[" A "]]") \ __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("subs %[count], %[count], #1") \ __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x04") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x04")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \ __ASM_EMIT("b.ge 7b") \ __ASM_EMIT("8:") void lramp_add2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmadd_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "dst", "src", "fadd", OP_DSEL, OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_sub2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmsub_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "dst", "src", "fsub", OP_DSEL, OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_rsub2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmrsub_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "dst", "src", "fsub", OP_RSEL, OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_mul2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmmul_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "dst", "src", "fmul", OP_DSEL, OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_add3(float *dst, const float *a, const float *b, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmadd_k4(dst, a, b, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "a", "b", "fadd", OP_DSEL, OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_sub3(float *dst, const float *a, const float *b, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmsub_k4(dst, a, b, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "a", "b", "fsub", OP_DSEL, OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_rsub3(float *dst, const float *a, const float *b, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmrsub_k4(dst, a, b, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "a", "b", "fsub", OP_RSEL, OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_mul3(float *dst, const float *a, const float *b, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmmul_k4(dst, a, b, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_OP3_CORE("dst", "a", "b", "fmul", OP_DSEL, OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b) : [CC] "r" (&lramp_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } #undef LRAMP_OP3_CORE IF_ARCH_AARCH64( static const float lramp_div_const[] __lsp_aligned16 = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, /* Initial values 0..7 */ 8.0f, 8.0f, 8.0f, 8.0f /* Step */ }; ) #define LRAMP_DIV3_CORE(DST, A, B, SEL, AINC) \ __ASM_EMIT("ld1r {v12.4s}, [%[v1]]") \ __ASM_EMIT("ld1r {v13.4s}, [%[delta]]") \ __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") \ __ASM_EMIT("ldr q10, [%[CC], #0x20]") \ /* v8.4s = i[0], v9.4s = i[1] */ \ /* v10.4s = step */ \ /* v12.4s = v1, v13.4s = delta */ \ /* 8x blocks */ \ __ASM_EMIT("subs %[count], %[count], #8") \ __ASM_EMIT("b.lo 2f") \ __ASM_EMIT("1:") \ __ASM_EMIT("fmul v4.4s, v8.4s, v13.4s") /* v4.4s = i[0] * delta */ \ __ASM_EMIT("fmul v5.4s, v9.4s, v13.4s") /* v5.4s = i[1] * delta */ \ __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0.4s = a[0], v1.4s = a[1] */ \ __ASM_EMIT("ldp q2, q3, [%[" B "], #0x00]") /* v2.4s = b[0], v3.4s = b[1] */ \ __ASM_EMIT("fadd v4.4s, v4.4s, v12.4s") /* v4.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fadd v5.4s, v5.4s, v12.4s") /* v5.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v2.4s, v2.4s, v4.4s") /* v2.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fmul v3.4s, v3.4s, v5.4s") /* v3.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fadd v8.4s, v8.4s, v10.4s") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("fadd v9.4s, v9.4s, v10.4s") /* v9.4s = i[1]' = i[1] + step */ \ __ASM_EMIT("frecpe v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("frecpe v5.4s, " SEL("v3.4s", "v1.4s")) \ __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("frecps v7.4s, v5.4s, " SEL("v3.4s", "v1.4s")) \ __ASM_EMIT("fmul v4.4s, v6.4s, v4.4s") \ __ASM_EMIT("fmul v5.4s, v7.4s, v5.4s") \ __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("frecps v7.4s, v5.4s, " SEL("v3.4s", "v1.4s")) \ __ASM_EMIT("fmul " SEL("v2.4s", "v0.4s") ", v6.4s, v4.4s") /* v2.4s = 1/(b[0] * (v1 + i[0] * delta)) */ \ __ASM_EMIT("fmul " SEL("v3.4s", "v1.4s") ", v7.4s, v5.4s") /* v3.4s = 1/(b[1] * (v1 + i[1] * delta)) */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0.4s = a[0] / b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s") /* v1.4s = a[1] / b[1] * (v1 + i[1] * delta) */ \ __ASM_EMIT("subs %[count], %[count], #8") \ __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x20") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x20")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \ __ASM_EMIT("b.hs 1b") \ __ASM_EMIT("2:") \ /* 4x blocks */ \ __ASM_EMIT("adds %[count], %[count], #4") \ __ASM_EMIT("b.lt 4f") \ __ASM_EMIT("fmul v4.4s, v8.4s, v13.4s") /* v4.4s = i[0] * delta */ \ __ASM_EMIT("ldr q0, [%[" A "], #0x00]") /* v0.4s = a[0] */ \ __ASM_EMIT("ldr q2, [%[" B "], #0x00]") /* v2.4s = b[0] */ \ __ASM_EMIT("fadd v4.4s, v4.4s, v12.4s") /* v4.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v2.4s, v2.4s, v4.4s") /* v2.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("frecpe v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("fmul v4.4s, v6.4s, v4.4s") \ __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("fmul " SEL("v2.4s", "v0.4s") ", v6.4s, v4.4s") /* v2.4s = 1/(b[0] * (v1 + i[0] * delta)) */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0.4s = a[0] / b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("sub %[count], %[count], #4") \ __ASM_EMIT("str q0, [%[" DST "], #0x00]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x10") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x10")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \ __ASM_EMIT("4:") \ /* 1x blocks */ \ __ASM_EMIT("adds %[count], %[count], #3") \ __ASM_EMIT("b.lt 6f") \ __ASM_EMIT("5:") \ __ASM_EMIT("fmul v4.4s, v8.4s, v13.4s") /* v4.4s = i[0] * delta */ \ __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \ __ASM_EMIT("ld1r {v2.4s}, [%[" B "]]") \ __ASM_EMIT("fadd v4.4s, v4.4s, v12.4s") /* v4.4s = v1 + i[0] * delta */ \ __ASM_EMIT("fmul v2.4s, v2.4s, v4.4s") /* v2.4s = b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ \ __ASM_EMIT("frecpe v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("fmul v4.4s, v6.4s, v4.4s") \ __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \ __ASM_EMIT("fmul " SEL("v2.4s", "v0.4s") ", v6.4s, v4.4s") /* v2.4s = 1/(b[0] * (v1 + i[0] * delta)) */ \ __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0.4s = a[0] / b[0] * (v1 + i[0] * delta) */ \ __ASM_EMIT("subs %[count], %[count], #1") \ __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \ __ASM_EMIT("add %[" B "], %[" B "], #0x04") \ __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x04")) \ __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \ __ASM_EMIT("b.ge 5b") \ __ASM_EMIT("6:") void lramp_div2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmdiv_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_DIV3_CORE("dst", "dst", "src", OP_DSEL, OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_div_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_rdiv2(float *dst, const float *src, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmrdiv_k3(dst, src, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_DIV3_CORE("dst", "dst", "src", OP_RSEL, OP_SNOINC) : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src) : [CC] "r" (&lramp_div_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_div3(float *dst, const float *a, const float *b, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmdiv_k4(dst, a, b, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_DIV3_CORE("dst", "a", "b", OP_DSEL, OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b) : [CC] "r" (&lramp_div_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } void lramp_rdiv3(float *dst, const float *a, const float *b, float v1, float v2, size_t count) { float delta = v2 - v1; if (delta == 0.0f) { dsp::fmrdiv_k4(dst, a, b, v1, count); return; } if (count == 0) return; delta /= count; ARCH_AARCH64_ASM( LRAMP_DIV3_CORE("dst", "a", "b", OP_RSEL, OP_SINC) : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b) : [CC] "r" (&lramp_div_const[0]), [delta] "r" (&delta), [v1] "r" (&v1) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" ); } #undef LRAMP_DIV3_CORE #undef OP_DSEL #undef OP_RSEL #undef OP_SINC #undef OP_SNOINC } /* namespace asimd */ } /* namespace lsp */ #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LRAMP_H_ */