/*
* Copyright (C) 2020 Linux Studio Plugins Project
* (C) 2020 Vladimir Sadovnikov
*
* This file is part of lsp-dsp-lib
* Created on: 31 мар. 2020 г.
*
* lsp-dsp-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-dsp-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-dsp-lib. If not, see .
*/
#ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_FLOAT_H_
#define PRIVATE_DSP_ARCH_AARCH64_ASIMD_FLOAT_H_
#ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
#error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
namespace lsp
{
namespace asimd
{
#define SEL_DST(a, b) a
#define SEL_NODST(a, b) b
IF_ARCH_AARCH64(
static uint32_t SAT_IARGS[] __lsp_aligned16 =
{
LSP_DSP_VEC4(0x7f800000), // X_P_INF
LSP_DSP_VEC4(0xff800000), // X_N_INF
LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_P_NAN_I), // SX_P_NAN
LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_P_INF_I), // SX_P_INF
LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_N_NAN_I), // SX_N_NAN
LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_N_INF_I) // SX_N_INF
};
)
#define MULTIPLE_SAT_BODY(DST, SRC, SEL) \
__ASM_EMIT("ldp q16, q17, [%[IARGS], #0x00]") /* v16 = +inf, v17 = -inf */ \
__ASM_EMIT("ldp q18, q19, [%[IARGS], #0x20]") /* v18 = X_P_NAN, v19 = X_P_INF */ \
__ASM_EMIT("ldp q20, q21, [%[IARGS], #0x40]") /* v20 = X_N_NAN, v21 = X_N_INF */ \
/* x8 blocks */ \
__ASM_EMIT("subs %[count], %[count], #8") \
__ASM_EMIT("b.lo 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT("ldp q0, q1, [%[" SRC "]]") /* v0 = s0..s3, v1 = s4..s7 */ \
__ASM_EMIT("cmgt v2.4s, v0.4s, v16.4s") /* v2 = [s > +inf] */ \
__ASM_EMIT("cmgt v3.4s, v1.4s, v16.4s") \
__ASM_EMIT("cmeq v4.4s, v0.4s, v16.4s") /* v4 = [s == +inf] */ \
__ASM_EMIT("cmeq v5.4s, v1.4s, v16.4s") \
__ASM_EMIT("bit v0.16b, v18.16b, v2.16b") /* v0 = s' = X_P_NAN * [s > +inf] | s * [s <= +inf] */ \
__ASM_EMIT("bit v1.16b, v18.16b, v3.16b") \
__ASM_EMIT("bit v0.16b, v19.16b, v4.16b") /* v0 = X_P_INF * [s == +inf] | s' * [s != +inf] */ \
__ASM_EMIT("bit v1.16b, v19.16b, v5.16b") \
__ASM_EMIT("cmhi v2.4s, v0.4s, v17.4s") /* v2 = [s > -inf] */ \
__ASM_EMIT("cmhi v3.4s, v1.4s, v17.4s") \
__ASM_EMIT("cmeq v4.4s, v0.4s, v17.4s") /* v4 = [s == -inf] */ \
__ASM_EMIT("cmeq v5.4s, v1.4s, v17.4s") \
__ASM_EMIT("bit v0.16b, v20.16b, v2.16b") /* v0 = s' = X_N_NAN * [s > -inf] | s * [s <= -inf] */ \
__ASM_EMIT("bit v1.16b, v20.16b, v3.16b") \
__ASM_EMIT("bit v0.16b, v21.16b, v4.16b") /* v0 = X_N_INF * [s == +inf] | s' * [s != +inf] */ \
__ASM_EMIT("bit v1.16b, v21.16b, v5.16b") \
__ASM_EMIT("stp q0, q1, [%[" DST "]]") \
__ASM_EMIT("subs %[count], %[count], #8") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
__ASM_EMIT("b.hs 1b") \
/* x4 block */ \
__ASM_EMIT("2:") \
__ASM_EMIT("adds %[count], %[count], #4") \
__ASM_EMIT("b.lt 4f") \
__ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s0..s3 */ \
__ASM_EMIT("cmgt v2.4s, v0.4s, v16.4s") /* v2 = [s > +inf] */ \
__ASM_EMIT("cmeq v4.4s, v0.4s, v16.4s") /* v4 = [s == +inf] */ \
__ASM_EMIT("bit v0.16b, v18.16b, v2.16b") /* v0 = s' = X_P_NAN * [s > +inf] | s * [s <= +inf] */ \
__ASM_EMIT("bit v0.16b, v19.16b, v4.16b") /* v0 = X_P_INF * [s == +inf] | s' * [s != +inf] */ \
__ASM_EMIT("cmhi v2.4s, v0.4s, v17.4s") /* v2 = [s > -inf] */ \
__ASM_EMIT("cmeq v4.4s, v0.4s, v17.4s") /* v4 = [s == -inf] */ \
__ASM_EMIT("bit v0.16b, v20.16b, v2.16b") /* v0 = s' = X_N_NAN * [s > -inf] | s * [s <= -inf] */ \
__ASM_EMIT("bit v0.16b, v21.16b, v4.16b") /* v0 = X_N_INF * [s == +inf] | s' * [s != +inf] */ \
__ASM_EMIT("str q0, [%[" DST "]]") \
__ASM_EMIT("sub %[count], %[count], #4") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
__ASM_EMIT("4:") \
/* x1 blocks */ \
__ASM_EMIT("adds %[count], %[count], #3") \
__ASM_EMIT("b.lt 6f") \
__ASM_EMIT("5:") \
__ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s */ \
__ASM_EMIT("cmgt v2.4s, v0.4s, v16.4s") /* v2 = [s > +inf] */ \
__ASM_EMIT("cmeq v4.4s, v0.4s, v16.4s") /* v4 = [s == +inf] */ \
__ASM_EMIT("bit v0.16b, v18.16b, v2.16b") /* v0 = s' = X_P_NAN * [s > +inf] | s * [s <= +inf] */ \
__ASM_EMIT("bit v0.16b, v19.16b, v4.16b") /* v0 = X_P_INF * [s == +inf] | s' * [s != +inf] */ \
__ASM_EMIT("cmhi v2.4s, v0.4s, v17.4s") /* v2 = [s > -inf] */ \
__ASM_EMIT("cmeq v4.4s, v0.4s, v17.4s") /* v4 = [s == -inf] */ \
__ASM_EMIT("bit v0.16b, v20.16b, v2.16b") /* v0 = s' = X_N_NAN * [s > -inf] | s * [s <= -inf] */ \
__ASM_EMIT("bit v0.16b, v21.16b, v4.16b") /* v0 = X_N_INF * [s == +inf] | s' * [s != +inf] */ \
__ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
__ASM_EMIT("subs %[count], %[count], #1") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
__ASM_EMIT("b.ge 5b") \
__ASM_EMIT("6:")
void copy_saturated(float *dst, const float *src, size_t count)
{
ARCH_AARCH64_ASM (
MULTIPLE_SAT_BODY("dst", "src", SEL_DST)
: [dst] "+r" (dst), [src] "+r" (src),
[count] "+r" (count)
: [IARGS] "r" (&SAT_IARGS[0])
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5",
"v16", "v17", "v18", "v19",
"v20", "v21"
);
}
void saturate(float *dst, size_t count)
{
ARCH_AARCH64_ASM(
MULTIPLE_SAT_BODY("dst", "dst", SEL_NODST)
: [dst] "+r" (dst), [count] "+r" (count)
: [IARGS] "r" (&SAT_IARGS[0])
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5",
"v16", "v17", "v18", "v19",
"v20", "v21"
);
}
#undef MULTIPLE_SAT_BODY
#define LIMIT_SAT_BODY(DST, SRC, SEL) \
__ASM_EMIT("ldp q16, q17, [%[XC], #0x00]") /* v16 = abs, v17 = +1 */ \
__ASM_EMIT("ldp q18, q19, [%[XC], #0x20]") /* v18 = sign, v19 = +Inf */ \
/* x8 blocks */ \
__ASM_EMIT("subs %[count], %[count], #8") \
__ASM_EMIT("b.lo 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT("ldp q0, q1, [%[" SRC "]]") /* v0 = s0, v1 = s1 */ \
__ASM_EMIT("and v6.16b, v0.16b, v16.16b") /* v6 = abs(s) */ \
__ASM_EMIT("and v7.16b, v1.16b, v16.16b") \
__ASM_EMIT("fcmge v2.4s, v17.4s, v6.4s") /* v2 = [ abs(s) <= +1 ] */ \
__ASM_EMIT("fcmge v3.4s, v17.4s, v7.4s") \
__ASM_EMIT("and v4.16b, v0.16b, v18.16b") /* v4 = sign(s) */ \
__ASM_EMIT("and v5.16b, v1.16b, v18.16b") \
__ASM_EMIT("cmge v6.4s, v19.4s, v6.4s") /* v6 = [ abs(s) <= +Inf ] */ \
__ASM_EMIT("cmge v7.4s, v19.4s, v7.4s") \
__ASM_EMIT("orr v4.16b, v4.16b, v17.16b") /* v4 = +1 * sign(s) */ \
__ASM_EMIT("orr v5.16b, v5.16b, v17.16b") \
__ASM_EMIT("and v4.16b, v4.16b, v6.16b") /* v4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
__ASM_EMIT("and v5.16b, v5.16b, v7.16b") \
__ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ( s & [ abs(s) <= +1 ]) | ( r & [ abs(s) > +1 ]) */ \
__ASM_EMIT("bif v1.16b, v5.16b, v3.16b") \
__ASM_EMIT("stp q0, q1, [%[" DST "]]") \
__ASM_EMIT("subs %[count], %[count], #8") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
__ASM_EMIT("b.hs 1b") \
__ASM_EMIT("2:") \
/* x4 block */ \
__ASM_EMIT("adds %[count], %[count], #4") \
__ASM_EMIT("b.lt 4f") \
__ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s0, v1 = s1 */ \
__ASM_EMIT("and v6.16b, v0.16b, v16.16b") /* v6 = abs(s) */ \
__ASM_EMIT("fcmge v2.4s, v17.4s, v6.4s") /* v2 = [ abs(s) <= +1 ] */ \
__ASM_EMIT("and v4.16b, v0.16b, v18.16b") /* v4 = sign(s) */ \
__ASM_EMIT("cmge v6.4s, v19.4s, v6.4s") /* v6 = [ abs(s) <= +Inf ] */ \
__ASM_EMIT("orr v4.16b, v4.16b, v17.16b") /* v4 = +1 * sign(s) */ \
__ASM_EMIT("and v4.16b, v4.16b, v6.16b") /* v4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
__ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ( s & [ abs(s) <= +1 ]) | ( r & [ abs(s) > +1 ]) */ \
__ASM_EMIT("str q0, [%[" DST "]]") \
__ASM_EMIT("sub %[count], %[count], #4") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
__ASM_EMIT("4:") \
/* 1x blocks */ \
__ASM_EMIT("adds %[count], %[count], #3") \
__ASM_EMIT("b.lt 6f") \
__ASM_EMIT("5:") \
__ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s0, v1 = s1 */ \
__ASM_EMIT("and v6.16b, v0.16b, v16.16b") /* v6 = abs(s) */ \
__ASM_EMIT("fcmge v2.4s, v17.4s, v6.4s") /* v2 = [ abs(s) <= +1 ] */ \
__ASM_EMIT("and v4.16b, v0.16b, v18.16b") /* v4 = sign(s) */ \
__ASM_EMIT("cmge v6.4s, v19.4s, v6.4s") /* v6 = [ abs(s) <= +Inf ] */ \
__ASM_EMIT("orr v4.16b, v4.16b, v17.16b") /* v4 = +1 * sign(s) */ \
__ASM_EMIT("and v4.16b, v4.16b, v6.16b") /* v4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
__ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ( s & [ abs(s) <= +1 ]) | ( r & [ abs(s) > +1 ]) */ \
__ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
__ASM_EMIT("subs %[count], %[count], #1") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
__ASM_EMIT("b.ge 5b") \
__ASM_EMIT("6:")
IF_ARCH_AARCH64(
static uint32_t XLIM_SAT[] __lsp_aligned16 =
{
LSP_DSP_VEC4(0x7fffffff), // abs
LSP_DSP_VEC4(0x3f800000), // +1
LSP_DSP_VEC4(0x80000000), // sign
LSP_DSP_VEC4(0x7f800000) // +Inf
};
)
void limit_saturate1(float *dst, size_t count)
{
ARCH_AARCH64_ASM(
LIMIT_SAT_BODY("dst", "dst", SEL_NODST)
: [dst] "+r" (dst), [count] "+r" (count)
: [XC] "r" (&XLIM_SAT[0])
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19"
);
}
void limit_saturate2(float *dst, const float *src, size_t count)
{
ARCH_AARCH64_ASM(
LIMIT_SAT_BODY("dst", "src", SEL_DST)
: [dst] "+r" (dst), [src] "+r" (src),
[count] "+r" (count)
: [XC] "r" (&XLIM_SAT[0])
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19"
);
}
#undef LIMIT_SAT_BODY
#define LIMIT_BODY(DST, SRC, SEL) \
__ASM_EMIT("dup v16.4s, %[min].s[0]") /* v16 = min */ \
__ASM_EMIT("dup v17.4s, %[max].s[0]") /* v17 = max */ \
/* x16 blocks */ \
__ASM_EMIT("subs %[count], %[count], #16") \
__ASM_EMIT("b.lo 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") /* v0 = s0, v1 = s1 */ \
__ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") /* v0 = s0, v1 = s1 */ \
__ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
__ASM_EMIT("fcmge v5.4s, v1.4s, v16.4s") \
__ASM_EMIT("fcmge v6.4s, v2.4s, v16.4s") \
__ASM_EMIT("fcmge v7.4s, v3.4s, v16.4s") \
__ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("bif v1.16b, v16.16b, v5.16b") \
__ASM_EMIT("bif v2.16b, v16.16b, v6.16b") \
__ASM_EMIT("bif v3.16b, v16.16b, v7.16b") \
__ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
__ASM_EMIT("fcmge v5.4s, v17.4s, v1.4s") \
__ASM_EMIT("fcmge v6.4s, v17.4s, v2.4s") \
__ASM_EMIT("fcmge v7.4s, v17.4s, v3.4s") \
__ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("bif v1.16b, v17.16b, v5.16b") \
__ASM_EMIT("bif v2.16b, v17.16b, v6.16b") \
__ASM_EMIT("bif v3.16b, v17.16b, v7.16b") \
__ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
__ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
__ASM_EMIT("subs %[count], %[count], #16") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x40", "")) \
__ASM_EMIT("b.hs 1b") \
__ASM_EMIT("2:") \
/* x8 block */ \
__ASM_EMIT("adds %[count], %[count], #8") \
__ASM_EMIT("b.lt 4f") \
__ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") /* v0 = s0, v1 = s1 */ \
__ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
__ASM_EMIT("fcmge v5.4s, v1.4s, v16.4s") \
__ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("bif v1.16b, v16.16b, v5.16b") \
__ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
__ASM_EMIT("fcmge v5.4s, v17.4s, v1.4s") \
__ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("bif v1.16b, v17.16b, v5.16b") \
__ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
__ASM_EMIT("sub %[count], %[count], #8") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
__ASM_EMIT("4:") \
/* x4 block */ \
__ASM_EMIT("adds %[count], %[count], #4") \
__ASM_EMIT("b.lt 6f") \
__ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s0 */ \
__ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
__ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
__ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("str q0, [%[" DST "]]") \
__ASM_EMIT("sub %[count], %[count], #4") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
__ASM_EMIT("6:") \
/* 1x blocks */ \
__ASM_EMIT("adds %[count], %[count], #3") \
__ASM_EMIT("b.lt 8f") \
__ASM_EMIT("7:") \
__ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s0 */ \
__ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
__ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
__ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
__ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
__ASM_EMIT("subs %[count], %[count], #1") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
__ASM_EMIT("b.ge 7b") \
__ASM_EMIT("8:")
void limit1(float *dst, float min, float max, size_t count)
{
ARCH_AARCH64_ASM(
LIMIT_BODY("dst", "dst", SEL_NODST)
: [dst] "+r" (dst), [count] "+r" (count),
[min] "+w" (min), [max] "+w" (max)
:
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5", "v6", "v7",
"v16", "v17"
);
}
void limit2(float *dst, const float *src, float min, float max, size_t count)
{
ARCH_AARCH64_ASM(
LIMIT_BODY("dst", "src", SEL_DST)
: [dst] "+r" (dst), [src] "+r" (src), [count] "+r" (count),
[min] "+w" (min), [max] "+w" (max)
:
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5", "v6", "v7",
"v16", "v17"
);
}
#undef LIMIT_BODY
#define SANITIZE_BODY(DST, SRC, SEL) \
__ASM_EMIT("ldp q16, q17, [%[CVAL], #0x00]") \
__ASM_EMIT("ldp q18, q19, [%[CVAL], #0x20]") \
__ASM_EMIT("subs %[count], %[count], #8") \
__ASM_EMIT("b.lo 2f") \
/* 8x blocks */ \
__ASM_EMIT("1:") \
__ASM_EMIT("ldp q0, q1, [%[" SRC "]]") /* v0 = s */ \
__ASM_EMIT("and v2.16b, v0.16b, v16.16b") /* v2 = abs(s) */ \
__ASM_EMIT("and v3.16b, v1.16b, v16.16b") \
__ASM_EMIT("and v4.16b, v0.16b, v17.16b") /* v4 = sign(s) */ \
__ASM_EMIT("and v5.16b, v1.16b, v17.16b") \
__ASM_EMIT("cmge v6.4s, v18.4s, v2.4s") /* v6 = abs(s) <= X_MAX */ \
__ASM_EMIT("cmge v7.4s, v18.4s, v3.4s") \
__ASM_EMIT("cmgt v2.4s, v2.4s, v19.4s") /* v2 = abs(s) > X_MIN */ \
__ASM_EMIT("cmgt v3.4s, v3.4s, v19.4s") \
__ASM_EMIT("and v2.16b, v2.16b, v6.16b") /* v2 = (abs(s) > X_MIN) & (abs(s) <= X_MIN) */ \
__ASM_EMIT("and v3.16b, v3.16b, v7.16b") \
__ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((abs(s) > X_MIN) & (abs(s) <= X_MIN)) ? s : sign(s) */ \
__ASM_EMIT("bif v1.16b, v5.16b, v3.16b") \
__ASM_EMIT("stp q0, q1, [%[" DST "]]") /* q0 = s */ \
__ASM_EMIT("subs %[count], %[count], #8") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
__ASM_EMIT("b.hs 1b") \
/* 4x block */ \
__ASM_EMIT("2:") \
__ASM_EMIT("adds %[count], %[count], #4") \
__ASM_EMIT("b.lt 4f") \
__ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s */ \
__ASM_EMIT("and v2.16b, v0.16b, v16.16b") /* v2 = abs(s) */ \
__ASM_EMIT("and v4.16b, v0.16b, v17.16b") /* v4 = sign(s) */ \
__ASM_EMIT("cmge v6.4s, v18.4s, v2.4s") /* v6 = abs(s) <= X_MAX */ \
__ASM_EMIT("cmgt v2.4s, v2.4s, v19.4s") /* v2 = abs(s) > X_MIN */ \
__ASM_EMIT("and v2.16b, v2.16b, v6.16b") /* v2 = (abs(s) > X_MIN) & (abs(s) <= X_MIN) */ \
__ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((abs(s) > X_MIN) & (abs(s) <= X_MIN)) ? s : sign(s) */ \
__ASM_EMIT("str q0, [%[" DST "]]") /* q0 = s */ \
__ASM_EMIT("sub %[count], %[count], #4") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
/* 1x blocks */ \
__ASM_EMIT("4:") \
__ASM_EMIT("adds %[count], %[count], #3") \
__ASM_EMIT("b.lt 6f") \
__ASM_EMIT("5:") \
__ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s */ \
__ASM_EMIT("and v2.16b, v0.16b, v16.16b") /* v2 = abs(s) */ \
__ASM_EMIT("and v4.16b, v0.16b, v17.16b") /* v4 = sign(s) */ \
__ASM_EMIT("cmge v6.4s, v18.4s, v2.4s") /* v6 = abs(s) <= X_MAX */ \
__ASM_EMIT("cmgt v2.4s, v2.4s, v19.4s") /* v2 = abs(s) > X_MIN */ \
__ASM_EMIT("and v2.16b, v2.16b, v6.16b") /* v2 = (abs(s) > X_MIN) & (abs(s) <= X_MIN) */ \
__ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((abs(s) > X_MIN) & (abs(s) <= X_MIN)) ? s : sign(s) */ \
__ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") /* q0 = s */ \
__ASM_EMIT("subs %[count], %[count], #1") \
__ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
__ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
__ASM_EMIT("b.ge 5b") \
/* end */ \
__ASM_EMIT("6:")
IF_ARCH_AARCH64(
static uint32_t SANITIZE_CVAL[] __lsp_aligned16 =
{
LSP_DSP_VEC4(0x7fffffff), // X_ABS
LSP_DSP_VEC4(0x80000000), // X_SIGN
LSP_DSP_VEC4(0x7f7fffff), // X_MAX
LSP_DSP_VEC4(0x007fffff) // X_MIN
};
)
void sanitize1(float *dst, size_t count)
{
ARCH_AARCH64_ASM
(
SANITIZE_BODY("dst", "dst", SEL_NODST)
: [dst] "+r" (dst),
[count] "+r" (count)
: [CVAL] "r" (&SANITIZE_CVAL[0])
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19"
);
}
void sanitize2(float *dst, const float *src, size_t count)
{
ARCH_AARCH64_ASM
(
SANITIZE_BODY("dst", "src", SEL_DST)
: [dst] "+r" (dst), [src] "+r" (src),
[count] "+r" (count)
: [CVAL] "r" (&SANITIZE_CVAL[0])
: "cc", "memory",
"v0", "v1", "v2", "v3",
"v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19"
);
}
#undef SANITIZE_BODY
#undef SEL_DST
#undef SEL_NODST
}
}
#endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_FLOAT_H_ */