/*
* Copyright (C) 2024 Linux Studio Plugins Project
* (C) 2024 Vladimir Sadovnikov
*
* This file is part of lsp-dsp-lib
* Created on: 29 июл. 2024 г.
*
* lsp-dsp-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-dsp-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-dsp-lib. If not, see .
*/
#ifndef PRIVATE_DSP_ARCH_X86_AVX512_SEARCH_IMINMAX_H_
#define PRIVATE_DSP_ARCH_X86_AVX512_SEARCH_IMINMAX_H_
#ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
#error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
namespace lsp
{
namespace avx512
{
IF_ARCH_X86(
static const uint32_t minmax_incr[] __lsp_aligned64 =
{
// +0x00 Index
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
// Increments
LSP_DSP_VEC16(16), // +0x40
LSP_DSP_VEC8(8), // +0x80
LSP_DSP_VEC4(4), // +0xa0
LSP_DSP_VEC4(1), // +0xb0
// Absolute mask
LSP_DSP_VEC16(0x7fffffff) // +0xc0
}
);
#define SEL_NONE(x, y) x
#define SEL_ABS(x, y) y
/*
* Register allocation:
* zmm0 = imin
* zmm1 = vmin
* zmm2 = cind
* zmm3 = cval
* zmm4/k4 = cmask
* zmm5 = temp
* zmm6 = amask
* zmm7 = incr
*/
#define IMINMAX_CORE(COND, SEL) \
__ASM_EMIT("vpxord %%zmm0, %%zmm0, %%zmm0") \
__ASM_EMIT("test %[count], %[count]") \
__ASM_EMIT("jz 8f") \
/* 16x block */ \
__ASM_EMIT("vbroadcastss 0x00(%[src]), %%zmm1") \
__ASM_EMIT(SEL("", "vmovaps 0xc0 + %[INCR], %%zmm6")) \
__ASM_EMIT("vmovdqa32 0x00 + %[INCR], %%zmm2") \
__ASM_EMIT(SEL("", "vandps %%zmm6, %%zmm1, %%zmm1")) \
__ASM_EMIT("vmovdqa32 0x40 + %[INCR], %%zmm7") \
__ASM_EMIT("sub $16, %[count]") \
__ASM_EMIT("jb 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT(SEL("vmovups 0x00(%[src]), %%zmm3", "vandps 0x00(%[src]), %%zmm6, %%zmm3")) \
__ASM_EMIT("vcmpps " COND ", %%zmm1, %%zmm3, %%k4") \
__ASM_EMIT("vmovdqa32 %%zmm2, %%zmm0 %{%%k4%}") \
__ASM_EMIT("vmovaps %%zmm3, %%zmm1 %{%%k4%}") \
__ASM_EMIT("vpaddd %%zmm7, %%zmm2, %%zmm2") \
__ASM_EMIT("add $0x40, %[src]") \
__ASM_EMIT("sub $16, %[count]") \
__ASM_EMIT("jae 1b") \
__ASM_EMIT("vextracti64x4 $1, %%zmm0, %%ymm5") \
__ASM_EMIT("vextractf64x4 $1, %%zmm1, %%ymm3") \
__ASM_EMIT("vcmpps " COND ", %%ymm1, %%ymm3, %%ymm4") \
__ASM_EMIT("vpblendvb %%ymm4, %%ymm5, %%ymm0, %%ymm0") \
__ASM_EMIT("vblendvps %%ymm4, %%ymm3, %%ymm1, %%ymm1") \
__ASM_EMIT("2:") \
/* 8x block */ \
__ASM_EMIT("add $8, %[count]") \
__ASM_EMIT("jl 4f") \
__ASM_EMIT("1:") \
__ASM_EMIT(SEL("vmovups 0x00(%[src]), %%ymm3", "vandps 0x00(%[src]), %%ymm6, %%ymm3")) \
__ASM_EMIT("vcmpps " COND ", %%ymm1, %%ymm3, %%ymm4") \
__ASM_EMIT("vpblendvb %%ymm4, %%ymm2, %%ymm0, %%ymm0") \
__ASM_EMIT("vblendvps %%ymm4, %%ymm3, %%ymm1, %%ymm1") \
__ASM_EMIT("vpaddd 0x80 + %[INCR], %%ymm2, %%ymm2") \
__ASM_EMIT("add $0x20, %[src]") \
__ASM_EMIT("sub $8, %[count]") \
__ASM_EMIT("4:") \
__ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm5") \
__ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm3") \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
/* 4x block */ \
__ASM_EMIT("add $4, %[count]") \
__ASM_EMIT("jl 6f") \
__ASM_EMIT(SEL("vmovups 0x00(%[src]), %%xmm3", "vandps 0x00(%[src]), %%xmm6, %%xmm3")) \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("vpaddd 0xa0 + %[INCR], %%xmm2, %%xmm2") \
__ASM_EMIT("sub $4, %[count]") \
__ASM_EMIT("add $0x10, %[src]") \
__ASM_EMIT("6:") \
__ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm5") \
__ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm3") \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm5") \
__ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm3") \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
/* 1x block */ \
__ASM_EMIT("add $3, %[count]") \
__ASM_EMIT("jl 8f") \
__ASM_EMIT("7:") \
__ASM_EMIT("movss 0x00(%[src]), %%xmm3") \
__ASM_EMIT(SEL("", "vandps %%xmm6, %%xmm3, %%xmm3")) \
__ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
__ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
__ASM_EMIT("vpaddd 0xb0 + %[INCR], %%xmm2, %%xmm2") \
__ASM_EMIT("add $0x04, %[src]") \
__ASM_EMIT("dec %[count]") \
__ASM_EMIT("jge 7b") \
__ASM_EMIT("8:") \
/* end */
size_t min_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$1", SEL_NONE)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%k4"
);
return index;
}
size_t max_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$6", SEL_NONE)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%k4"
);
return index;
}
size_t abs_min_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$1", SEL_ABS)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%k4"
);
return index;
}
size_t abs_max_index(const float *src, size_t count)
{
uint32_t index;
ARCH_X86_ASM(
IMINMAX_CORE("$6", SEL_ABS)
: [src] "+r" (src), [count] "+r" (count),
[index] "=Yz" (index)
: [INCR] "o" (minmax_incr)
: "cc",
"%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%k4"
);
return index;
}
#undef IMINMAX_CORE
/*
* Register allocation:
* ymm0 = imin
* ymm1 = vmin
* ymm2 = imax
* ymm3 = vmax
* ymm4 = cind
* ymm5 = cval
* ymm6/k6 = cmask
* ymm7/k7 = temp
*/
#define IMINMAX2_CORE(SEL) \
__ASM_EMIT("vpxord %%zmm0, %%zmm0, %%zmm0") /* imin */ \
__ASM_EMIT("vpxord %%zmm2, %%zmm2, %%zmm2") /* imax */ \
__ASM_EMIT("test %[count], %[count]") \
__ASM_EMIT("jz 8f") \
/* 16x block */ \
__ASM_EMIT("vbroadcastss 0x00(%[src]), %%zmm1") /* vmin */ \
__ASM_EMIT("vmovdqa32 0x00 + %[INCR], %%zmm4") /* cind */ \
__ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%zmm1, %%zmm1")) \
__ASM_EMIT("vmovdqa32 %%zmm1, %%zmm3") /* vmax */ \
__ASM_EMIT("sub $16, %[count]") \
__ASM_EMIT("jb 2f") \
__ASM_EMIT("1:") \
__ASM_EMIT("vmovups 0x00(%[src]), %%zmm5") \
__ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%zmm5, %%zmm5")) \
__ASM_EMIT("vcmpps $1, %%zmm1, %%zmm5, %%k6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%zmm3, %%zmm5, %%k7") /* cmpgtps */ \
__ASM_EMIT("vmovdqa32 %%zmm4, %%zmm0 %{%%k6%}") \
__ASM_EMIT("vmovaps %%zmm5, %%zmm1 %{%%k6%}") \
__ASM_EMIT("vmovdqa32 %%zmm4, %%zmm2 %{%%k7%}") \
__ASM_EMIT("vmovaps %%zmm5, %%zmm3 %{%%k7%}") \
__ASM_EMIT("vpaddd 0x40 + %[INCR], %%zmm4, %%zmm4") \
__ASM_EMIT("add $0x40, %[src]") \
__ASM_EMIT("sub $16, %[count]") \
__ASM_EMIT("jae 1b") \
__ASM_EMIT("vextracti64x4 $1, %%zmm0, %%ymm7") \
__ASM_EMIT("vextractf64x4 $1, %%zmm1, %%ymm5") \
__ASM_EMIT("vcmpps $1, %%ymm1, %%ymm5, %%ymm6") \
__ASM_EMIT("vpblendvb %%ymm6, %%ymm7, %%ymm0, %%ymm0") \
__ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm1, %%ymm1") \
__ASM_EMIT("vextracti64x4 $1, %%zmm2, %%ymm7") \
__ASM_EMIT("vextractf64x4 $1, %%zmm3, %%ymm5") \
__ASM_EMIT("vcmpps $6, %%ymm3, %%ymm5, %%ymm6") \
__ASM_EMIT("vpblendvb %%ymm6, %%ymm7, %%ymm2, %%ymm2") \
__ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm3, %%ymm3") \
__ASM_EMIT("2:") \
/* 8x block */ \
__ASM_EMIT("add $8, %[count]") \
__ASM_EMIT("jl 4f") \
__ASM_EMIT("vmovups 0x00(%[src]), %%ymm5") \
__ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%ymm5, %%ymm5")) \
__ASM_EMIT("vcmpps $1, %%ymm1, %%ymm5, %%ymm6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%ymm3, %%ymm5, %%ymm7") /* cmpgtps */ \
__ASM_EMIT("vpblendvb %%ymm6, %%ymm4, %%ymm0, %%ymm0") \
__ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm1, %%ymm1") \
__ASM_EMIT("vpblendvb %%ymm7, %%ymm4, %%ymm2, %%ymm2") \
__ASM_EMIT("vblendvps %%ymm7, %%ymm5, %%ymm3, %%ymm3") \
__ASM_EMIT("vpaddd 0x80 + %[INCR], %%ymm4, %%ymm4") \
__ASM_EMIT("add $0x20, %[src]") \
__ASM_EMIT("sub $8, %[count]") \
__ASM_EMIT("4:") \
__ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm7") \
__ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm5") \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vextracti128 $1, %%ymm2, %%xmm7") \
__ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
/* 4x block */ \
__ASM_EMIT("add $4, %[count]") \
__ASM_EMIT("jl 6f") \
__ASM_EMIT("vmovups 0x00(%[src]), %%xmm5") \
__ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%xmm5, %%xmm5")) \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("vpaddd 0xa0 + %[INCR], %%xmm4, %%xmm4") \
__ASM_EMIT("sub $4, %[count]") \
__ASM_EMIT("add $0x10, %[src]") \
__ASM_EMIT("6:") \
__ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm7") \
__ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm5") \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm7") \
__ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm5") \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpshufd $0x39, %%xmm2, %%xmm7") \
__ASM_EMIT("vshufps $0x39, %%xmm3, %%xmm3, %%xmm5") \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("vpunpckhdq %%xmm2, %%xmm2, %%xmm7") \
__ASM_EMIT("vunpckhps %%xmm3, %%xmm3, %%xmm5") \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
/* 1x block */ \
__ASM_EMIT("add $3, %[count]") \
__ASM_EMIT("jl 8f") \
__ASM_EMIT("7:") \
__ASM_EMIT("movss 0x00(%[src]), %%xmm5") \
__ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%xmm5, %%xmm5")) \
__ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
__ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
__ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
__ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
__ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
__ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
__ASM_EMIT("vpaddd 0xb0 + %[INCR], %%xmm4, %%xmm4") \
__ASM_EMIT("add $0x04, %[src]") \
__ASM_EMIT("dec %[count]") \
__ASM_EMIT("jge 7b") \
__ASM_EMIT("8:") \
/* end */ \
__ASM_EMIT("movss %%xmm0, (%[min])") \
__ASM_EMIT("movss %%xmm2, (%[max])")
void minmax_index(const float *src, size_t count, size_t *min, size_t *max)
{
ARCH_X86_ASM(
IMINMAX2_CORE(SEL_NONE)
: [src] "+r" (src), [count] "+r" (count)
: [min] "r" (min), [max] "r" (max),
[INCR] "o" (minmax_incr)
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%k6", "%k7"
);
}
void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max)
{
ARCH_X86_ASM(
IMINMAX2_CORE(SEL_ABS)
: [src] "+r" (src), [count] "+r" (count)
: [min] "r" (min), [max] "r" (max),
[INCR] "o" (minmax_incr)
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7",
"%k6", "%k7"
);
}
#undef IMINMAX2_CORE
#undef SEL_NONE
#undef SEL_ABS
} /* namespace avx512 */
} /* namespace lsp */
#endif /* PRIVATE_DSP_ARCH_X86_AVX512_SEARCH_IMINMAX_H_ */