/*
* Copyright (C) 2020 Linux Studio Plugins Project
* (C) 2020 Vladimir Sadovnikov
*
* This file is part of lsp-dsp-lib
* Created on: 31 мар. 2020 г.
*
* lsp-dsp-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-dsp-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-dsp-lib. If not, see .
*/
#ifndef PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_
#define PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_
#ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
#error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
namespace lsp
{
namespace sse
{
void fastconv_parse(float *dst, const float *src, size_t rank)
{
// Prepare for butterflies
const float *wk = &XFFT_W[(rank-3) << 3];
const float *ak = &XFFT_A[(rank-3) << 3];
size_t items = size_t(1) << (rank + 1);
size_t bs = items;
size_t n = bs >> 1;
// Iterate first cycle
if (n > 4)
{
// ONE LARGE CYCLE
// Set initial values of pointers
float *a = dst;
float *b = &a[n];
size_t k = n;
ARCH_X86_ASM
(
__ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */
__ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */
__ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 */
:
: [ak] "r"(ak)
:
"%xmm5", "%xmm6", "%xmm7"
);
ARCH_X86_ASM
(
// __ASM_EMIT(".align 16")
__ASM_EMIT("1:")
__ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
__ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 */
__ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = s[i] */
__ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = s[i] */
__ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = s[i]*iA[i] */
__ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = s[i]*rA[i] */
__ASM_EMIT("subps %%xmm4, %%xmm3") /* xmm3 = -s[i]*iA[i] */
__ASM_EMIT("movups %%xmm0, 0x00(%[a])")
__ASM_EMIT("movups %%xmm5, 0x10(%[a])")
__ASM_EMIT("movups %%xmm2, 0x00(%[b])")
__ASM_EMIT("movups %%xmm3, 0x10(%[b])")
__ASM_EMIT("add $0x10, %[src]")
__ASM_EMIT("add $0x20, %[a]")
__ASM_EMIT("add $0x20, %[b]")
/* Repeat loop */
__ASM_EMIT("sub $8, %[k]")
__ASM_EMIT("jz 2f")
/* Rotate angle */
__ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw[i] */
__ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw[i] */
__ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw[i] */
__ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw[i] */
__ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i] * iw[i] */
__ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = iA[i] * iw[i] */
__ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = rA[i] * rw[i] */
__ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = iA[i] * rw[i] */
__ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = rA[i] * rw[i] - iA[i] * iw[i] */
__ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = rA[i] * iw[i] + iA[i] * rw[i] */
__ASM_EMIT("jmp 1b")
__ASM_EMIT("2:")
: [a] "+r" (a), [b] "+r" (b), [src] "+r" (src), [k] "+r" (k)
: [wk] "r"(wk)
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
wk -= 8;
ak -= 8;
n >>= 1;
bs >>= 1;
}
else
{
// Unpack 4x real to 4x split complex
ARCH_X86_ASM
(
__ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
__ASM_EMIT("xorps %%xmm1, %%xmm1") /* xmm1 = 0 */
__ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
__ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
:
: [items] "r" (items), [src] "r" (src), [dst] "r" (dst)
: "cc", "memory",
"%xmm0", "%xmm1"
);
}
// Iterate butterflies
for (; n > 4; n >>= 1, bs >>= 1)
{
for (size_t p=0; p> 1;
// Iterate first cycle
if (n > 4)
{
// ONE LARGE CYCLE
// Set initial values of pointers
float *a = dst;
float *b = &a[n];
size_t k = n;
ARCH_X86_ASM
(
__ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */
__ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */
__ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 */
:
: [ak] "r"(ak)
:
"%xmm5", "%xmm6", "%xmm7"
);
ARCH_X86_ASM
(
// __ASM_EMIT(".align 16")
__ASM_EMIT("1:")
__ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
__ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 */
__ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = s[i] */
__ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = s[i] */
__ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = s[i]*iA[i] */
__ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = s[i]*rA[i] */
__ASM_EMIT("subps %%xmm4, %%xmm3") /* xmm3 = -s[i]*iA[i] */
__ASM_EMIT("movups %%xmm0, 0x00(%[a])")
__ASM_EMIT("movups %%xmm5, 0x10(%[a])")
__ASM_EMIT("movups %%xmm2, 0x00(%[b])")
__ASM_EMIT("movups %%xmm3, 0x10(%[b])")
__ASM_EMIT("add $0x10, %[src]")
__ASM_EMIT("add $0x20, %[a]")
__ASM_EMIT("add $0x20, %[b]")
/* Repeat loop */
__ASM_EMIT32("subl $8, %[k]")
__ASM_EMIT64("sub $8, %[k]")
__ASM_EMIT("jz 2f")
/* Rotate angle */
__ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw[i] */
__ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw[i] */
__ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw[i] */
__ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw[i] */
__ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i] * iw[i] */
__ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = iA[i] * iw[i] */
__ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = rA[i] * rw[i] */
__ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = iA[i] * rw[i] */
__ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = rA[i] * rw[i] - iA[i] * iw[i] */
__ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = rA[i] * iw[i] + iA[i] * rw[i] */
__ASM_EMIT("jmp 1b")
__ASM_EMIT("2:")
: [a] "+r" (a), [b] "+r" (b), [src] "+r" (src), [k] "+r" (k)
: [wk] "r" (wk)
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
wk -= 8;
ak -= 8;
n >>= 1;
bs >>= 1;
}
else
{
// Unpack 4x real to 4x split complex
ARCH_X86_ASM
(
__ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
__ASM_EMIT("xorps %%xmm1, %%xmm1") /* xmm1 = 0 */
__ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
__ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
:
: [src] "r" (src), [dst] "r" (dst)
: "cc", "memory",
"%xmm0", "%xmm1"
);
}
// Iterate butterflies
for (; n > 4; n >>= 1, bs >>= 1)
{
for (size_t p=0; p