/* * Copyright (C) 2020 Linux Studio Plugins Project * (C) 2020 Vladimir Sadovnikov * * This file is part of lsp-dsp-lib * Created on: 31 мар. 2020 г. * * lsp-dsp-lib is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * lsp-dsp-lib is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with lsp-dsp-lib. If not, see . */ #ifndef PRIVATE_DSP_ARCH_X86_AVX_FASTCONV_H_ #define PRIVATE_DSP_ARCH_X86_AVX_FASTCONV_H_ #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL #error "This header should not be included directly" #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */ #include #include #include namespace lsp { namespace avx { void fastconv_parse(float *dst, const float *src, size_t rank) { const float *ak = &FFT_A[(rank - 3) << 4]; const float *wk = &FFT_DW[(rank - 3) << 4]; size_t np = 1 << (rank - 1); size_t nb = 1; if (np > 4) { fastconv_direct_prepare(dst, src, ak, wk, np); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } else fastconv_direct_unpack(dst, src); while (np > 4) { fastconv_direct_butterfly(dst, ak, wk, np, nb); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } fastconv_direct_butterfly_last(dst, nb); } void fastconv_parse_fma3(float *dst, const float *src, size_t rank) { const float *ak = &FFT_A[(rank - 3) << 4]; const float *wk = &FFT_DW[(rank - 3) << 4]; size_t np = 1 << (rank - 1); size_t nb = 1; if (np > 4) { fastconv_direct_prepare_fma3(dst, src, ak, wk, np); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } else fastconv_direct_unpack(dst, src); while (np > 4) { fastconv_direct_butterfly_fma3(dst, ak, wk, np, nb); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } fastconv_direct_butterfly_last_fma3(dst, nb); } void fastconv_restore(float *dst, float *tmp, size_t rank) { size_t nb = 1 << (rank - 3), np = 4; const float *ak = FFT_A; const float *wk = FFT_DW; fastconv_reverse_prepare(tmp, nb); if ((nb >>= 1) <= 0) { fastconv_reverse_unpack(dst, tmp, rank); return; } ak += 16; wk += 16; np <<= 1; while (nb > 1) { fastconv_reverse_butterfly(tmp, ak, wk, np, nb); ak += 16; wk += 16; np <<= 1; nb >>= 1; } fastconv_reverse_butterfly_last(dst, tmp, ak, wk, np); } void fastconv_restore_fma3(float *dst, float *tmp, size_t rank) { size_t nb = 1 << (rank - 3), np = 4; const float *ak = FFT_A; const float *wk = FFT_DW; fastconv_reverse_prepare_fma3(tmp, nb); if ((nb >>= 1) <= 0) { fastconv_reverse_unpack(dst, tmp, rank); return; } ak += 16; wk += 16; np <<= 1; while (nb > 1) { fastconv_reverse_butterfly_fma3(tmp, ak, wk, np, nb); ak += 16; wk += 16; np <<= 1; nb >>= 1; } fastconv_reverse_butterfly_last_fma3(dst, tmp, ak, wk, np); } void fastconv_apply(float *dst, float *tmp, const float *c1, const float *c2, size_t rank) { size_t nb = 1 << (rank - 3), np = 4; const float *ak = FFT_A; const float *wk = FFT_DW; fastconv_apply_prepare(tmp, c1, c2, nb); if ((nb >>= 1) <= 0) { fastconv_reverse_unpack_adding(dst, tmp, rank); return; } ak += 16; wk += 16; np <<= 1; while (nb > 1) { fastconv_reverse_butterfly(tmp, ak, wk, np, nb); ak += 16; wk += 16; np <<= 1; nb >>= 1; } fastconv_reverse_butterfly_last_adding(dst, tmp, ak, wk, np); } void fastconv_apply_fma3(float *dst, float *tmp, const float *c1, const float *c2, size_t rank) { size_t nb = 1 << (rank - 3), np = 4; const float *ak = FFT_A; const float *wk = FFT_DW; fastconv_apply_prepare_fma3(tmp, c1, c2, nb); if ((nb >>= 1) <= 0) { fastconv_reverse_unpack_adding(dst, tmp, rank); return; } ak += 16; wk += 16; np <<= 1; while (nb > 1) { fastconv_reverse_butterfly_fma3(tmp, ak, wk, np, nb); ak += 16; wk += 16; np <<= 1; nb >>= 1; } fastconv_reverse_butterfly_last_adding_fma3(dst, tmp, ak, wk, np); } void fastconv_parse_apply(float *dst, float *tmp, const float *c, const float *src, size_t rank) { const float *ak = &FFT_A[(rank - 3) << 4]; const float *wk = &FFT_DW[(rank - 3) << 4]; size_t np = 1 << (rank - 1); size_t nb = 1; if (np > 4) { fastconv_direct_prepare(tmp, src, ak, wk, np); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } else fastconv_direct_unpack(tmp, src); while (np > 4) { fastconv_direct_butterfly(tmp, ak, wk, np, nb); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } fastconv_apply_internal(tmp, c, nb); if ((nb >>= 1) <= 0) { fastconv_reverse_unpack_adding(dst, tmp, rank); return; } ak += 16; wk += 16; np <<= 1; while (nb > 1) { fastconv_reverse_butterfly(tmp, ak, wk, np, nb); ak += 16; wk += 16; np <<= 1; nb >>= 1; } fastconv_reverse_butterfly_last_adding(dst, tmp, ak, wk, np); } void fastconv_parse_apply_fma3(float *dst, float *tmp, const float *c, const float *src, size_t rank) { const float *ak = &FFT_A[(rank - 3) << 4]; const float *wk = &FFT_DW[(rank - 3) << 4]; size_t np = 1 << (rank - 1); size_t nb = 1; if (np > 4) { fastconv_direct_prepare_fma3(tmp, src, ak, wk, np); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } else fastconv_direct_unpack(tmp, src); while (np > 4) { fastconv_direct_butterfly_fma3(tmp, ak, wk, np, nb); ak -= 16; wk -= 16; np >>= 1; nb <<= 1; } fastconv_apply_internal_fma3(tmp, c, nb); if ((nb >>= 1) <= 0) { fastconv_reverse_unpack_adding(dst, tmp, rank); return; } ak += 16; wk += 16; np <<= 1; while (nb > 1) { fastconv_reverse_butterfly_fma3(tmp, ak, wk, np, nb); ak += 16; wk += 16; np <<= 1; nb >>= 1; } fastconv_reverse_butterfly_last_adding_fma3(dst, tmp, ak, wk, np); } } } #endif /* PRIVATE_DSP_ARCH_X86_AVX_FASTCONV_H_ */