/*
* Copyright (C) 2020 Linux Studio Plugins Project
* (C) 2020 Vladimir Sadovnikov
*
* This file is part of lsp-dsp-lib
* Created on: 31 мар. 2020 г.
*
* lsp-dsp-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-dsp-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-dsp-lib. If not, see .
*/
#ifndef PRIVATE_DSP_ARCH_X86_AVX2_SEARCH_IMINMAX_H_
#define PRIVATE_DSP_ARCH_X86_AVX2_SEARCH_IMINMAX_H_
#ifndef PRIVATE_DSP_ARCH_X86_AVX2_IMPL
#error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_X86_AVX2_IMPL */
namespace lsp
{
namespace avx2
{
IF_ARCH_X86(
static const uint32_t minmax_incr[] __lsp_aligned64 =
{
0, 1, 2, 3, // Index
4, 5, 6, 7,
8, 8, 8, 8, // Increment
8, 8, 8, 8,
4, 4, 4, 4,
1, 1, 1, 1,
0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, // Abs mask
0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff
}
);
#define SEL_NONE(x, y) x
#define SEL_ABS(x, y) y
/*
* Register allocation:
* ymm0 = imin
* ymm1 = vmin
* ymm2 = cind
* ymm3 = cval
* ymm4 = cmask
* ymm5 = temp
* ymm6 = amask
* ymm7 = incr
*/
#define IMINMAX_CORE(COND, SEL) \
__ASM_EMIT("vpxor %%ymm0, %%ymm0, %%ymm0") \
__ASM_EMIT("test %[count], %[count]") \
__ASM_EMIT("jz 6f") \
/* 8x block */ \
__ASM_EMIT("vbroadcastss 0x00(%[src]), %%ymm1") \
__ASM_EMIT(SEL("", "vmovaps 0x60 + %[INCR], %%ymm6")) \
__ASM_EMIT("vmovdqu 0x00 + %[INCR], %%ymm2") \
__ASM_EMIT(SEL("", "vandps %%ymm6, %%ymm1, %%ymm1")) \
__ASM_EMIT("vmovdqu 0x20 + %[INCR], %%ymm7") \
__ASM_EMIT("sub $8, %[count]") \
__ASM_EMIT("jb 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT(SEL("vmovups 0x00(%[src]), %%ymm3", "vandps 0x00(%[src]), %%ymm6, %%ymm3")) \
__ASM_EMIT("vcmpps " COND ", %%ymm1, %%ymm3, %%ymm4") \
__ASM_EMIT("vpblendvb %%ymm4, %%ymm2, %%ymm0, %%ymm0") \
__ASM_EMIT("vblendvps %%ymm4, %%ymm3, %%ymm1, %%ymm1") \
__ASM_EMIT("vpaddd %%ymm7, %%ymm2, %%ymm2") \
__ASM_EMIT("add $0x20, %[src]") \
__ASM_EMIT("sub $8, %[count]") \
__ASM_EMIT("jae 1b") \
__ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm5") \
__ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm3") \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("2:") \
/* 4x block */ \
__ASM_EMIT("add $4, %[count]") \
__ASM_EMIT("jl 4f") \
__ASM_EMIT(SEL("vmovups 0x00(%[src]), %%xmm3", "vandps 0x00(%[src]), %%xmm6, %%xmm3")) \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("vpaddd 0x40 + %[INCR], %%xmm2, %%xmm2") \
__ASM_EMIT("sub $4, %[count]") \
__ASM_EMIT("add $0x10, %[src]") \
__ASM_EMIT("4:") \
__ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm5") \
__ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm3") \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm5") \
__ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm3") \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
/* 1x block */ \
__ASM_EMIT("add $3, %[count]") \
__ASM_EMIT("jl 6f") \
__ASM_EMIT("5:") \
__ASM_EMIT("movss 0x00(%[src]), %%xmm3") \
__ASM_EMIT(SEL("", "vandps %%xmm6, %%xmm3, %%xmm3")) \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("vpaddd 0x50 + %[INCR], %%xmm2, %%xmm2") \
__ASM_EMIT("add $0x04, %[src]") \
__ASM_EMIT("dec %[count]") \
__ASM_EMIT("jge 5b") \
/* end */ \
__ASM_EMIT("6:")
size_t min_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$1", SEL_NONE)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
return index;
}
size_t max_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$6", SEL_NONE)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
return index;
}
size_t abs_min_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$1", SEL_ABS)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
return index;
}
size_t abs_max_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$6", SEL_ABS)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
return index;
}
#undef IMINMAX_CORE
/*
* Register allocation:
* ymm0 = imin
* ymm1 = vmin
* ymm2 = imax
* ymm3 = vmax
* ymm4 = cind
* ymm5 = cval
* ymm6 = cmask
* ymm7 = temp
*/
#define IMINMAX2_CORE(SEL) \
__ASM_EMIT("vpxor %%ymm0, %%ymm0, %%ymm0") /* imin */ \
__ASM_EMIT("vpxor %%ymm2, %%ymm2, %%ymm2") /* imax */ \
__ASM_EMIT("test %[count], %[count]") \
__ASM_EMIT("jz 6f") \
/* 8x block */ \
__ASM_EMIT("vbroadcastss 0x00(%[src]), %%ymm1") /* vmin */ \
__ASM_EMIT("vmovdqu 0x00 + %[INCR], %%ymm4") /* cind */ \
__ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%ymm1, %%ymm1")) \
__ASM_EMIT("vmovdqa %%ymm1, %%ymm3") /* vmax */ \
__ASM_EMIT("sub $8, %[count]") \
__ASM_EMIT("jb 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT("vmovups 0x00(%[src]), %%ymm5") \
__ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%ymm5, %%ymm5")) \
__ASM_EMIT("vcmpps $1, %%ymm1, %%ymm5, %%ymm6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%ymm3, %%ymm5, %%ymm7") /* cmpgtps */ \
__ASM_EMIT("vpblendvb %%ymm6, %%ymm4, %%ymm0, %%ymm0") \
__ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm1, %%ymm1") \
__ASM_EMIT("vpblendvb %%ymm7, %%ymm4, %%ymm2, %%ymm2") \
__ASM_EMIT("vblendvps %%ymm7, %%ymm5, %%ymm3, %%ymm3") \
__ASM_EMIT("vpaddd 0x20 + %[INCR], %%ymm4, %%ymm4") \
__ASM_EMIT("add $0x20, %[src]") \
__ASM_EMIT("sub $8, %[count]") \
__ASM_EMIT("jae 1b") \
__ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm7") \
__ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm5") \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vextracti128 $1, %%ymm2, %%xmm7") \
__ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("2:") \
/* 4x block */ \
__ASM_EMIT("add $4, %[count]") \
__ASM_EMIT("jl 4f") \
__ASM_EMIT("vmovups 0x00(%[src]), %%xmm5") \
__ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%xmm5, %%xmm5")) \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("vpaddd 0x40 + %[INCR], %%xmm4, %%xmm4") \
__ASM_EMIT("sub $4, %[count]") \
__ASM_EMIT("add $0x10, %[src]") \
__ASM_EMIT("4:") \
__ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm7") \
__ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm5") \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm7") \
__ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm5") \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpshufd $0x39, %%xmm2, %%xmm7") \
__ASM_EMIT("vshufps $0x39, %%xmm3, %%xmm3, %%xmm5") \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("vpunpckhdq %%xmm2, %%xmm2, %%xmm7") \
__ASM_EMIT("vunpckhps %%xmm3, %%xmm3, %%xmm5") \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
/* 1x block */ \
__ASM_EMIT("add $3, %[count]") \
__ASM_EMIT("jl 6f") \
__ASM_EMIT("5:") \
__ASM_EMIT("movss 0x00(%[src]), %%xmm5") \
__ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%xmm5, %%xmm5")) \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("vpaddd 0x50 + %[INCR], %%xmm4, %%xmm4") \
__ASM_EMIT("add $0x04, %[src]") \
__ASM_EMIT("dec %[count]") \
__ASM_EMIT("jge 5b") \
/* end */ \
__ASM_EMIT("6:") \
__ASM_EMIT("movss %%xmm0, (%[min])") \
__ASM_EMIT("movss %%xmm2, (%[max])")
void minmax_index(const float *src, size_t count, size_t *min, size_t *max)
{
*min = 0;
*max = 0;
ARCH_X86_ASM(
IMINMAX2_CORE(SEL_NONE)
: [src] "+r" (src), [count] "+r" (count)
: [min] "r" (min), [max] "r" (max),
[INCR] "o" (minmax_incr)
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
}
void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max)
{
*min = 0;
*max = 0;
ARCH_X86_ASM(
IMINMAX2_CORE(SEL_ABS)
: [src] "+r" (src), [count] "+r" (count)
: [min] "r" (min), [max] "r" (max),
[INCR] "o" (minmax_incr)
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
}
#undef IMINMAX2_CORE
#undef SEL_NONE
#undef SEL_ABS
} /* namespace avx2 */
} /* namespace lsp */
#endif /* PRIVATE_DSP_ARCH_X86_AVX2_SEARCH_IMINMAX_H_ */