/* * Copyright (C) 2020 Linux Studio Plugins Project * (C) 2020 Vladimir Sadovnikov * * This file is part of lsp-dsp-lib * Created on: 31 мар. 2020 г. * * lsp-dsp-lib is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * lsp-dsp-lib is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with lsp-dsp-lib. If not, see . */ #ifndef PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_ #define PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_ #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL #error "This header should not be included directly" #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */ namespace lsp { namespace sse { void fastconv_parse(float *dst, const float *src, size_t rank) { // Prepare for butterflies const float *wk = &XFFT_W[(rank-3) << 3]; const float *ak = &XFFT_A[(rank-3) << 3]; size_t items = size_t(1) << (rank + 1); size_t bs = items; size_t n = bs >> 1; // Iterate first cycle if (n > 4) { // ONE LARGE CYCLE // Set initial values of pointers float *a = dst; float *b = &a[n]; size_t k = n; ARCH_X86_ASM ( __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */ __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */ __ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 */ : : [ak] "r"(ak) : "%xmm5", "%xmm6", "%xmm7" ); ARCH_X86_ASM ( // __ASM_EMIT(".align 16") __ASM_EMIT("1:") __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */ __ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 */ __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = s[i] */ __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = s[i] */ __ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = s[i]*iA[i] */ __ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = s[i]*rA[i] */ __ASM_EMIT("subps %%xmm4, %%xmm3") /* xmm3 = -s[i]*iA[i] */ __ASM_EMIT("movups %%xmm0, 0x00(%[a])") __ASM_EMIT("movups %%xmm5, 0x10(%[a])") __ASM_EMIT("movups %%xmm2, 0x00(%[b])") __ASM_EMIT("movups %%xmm3, 0x10(%[b])") __ASM_EMIT("add $0x10, %[src]") __ASM_EMIT("add $0x20, %[a]") __ASM_EMIT("add $0x20, %[b]") /* Repeat loop */ __ASM_EMIT("sub $8, %[k]") __ASM_EMIT("jz 2f") /* Rotate angle */ __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw[i] */ __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw[i] */ __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw[i] */ __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw[i] */ __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i] * iw[i] */ __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = iA[i] * iw[i] */ __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = rA[i] * rw[i] */ __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = iA[i] * rw[i] */ __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = rA[i] * rw[i] - iA[i] * iw[i] */ __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = rA[i] * iw[i] + iA[i] * rw[i] */ __ASM_EMIT("jmp 1b") __ASM_EMIT("2:") : [a] "+r" (a), [b] "+r" (b), [src] "+r" (src), [k] "+r" (k) : [wk] "r"(wk) : "cc", "memory", "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7" ); wk -= 8; ak -= 8; n >>= 1; bs >>= 1; } else { // Unpack 4x real to 4x split complex ARCH_X86_ASM ( __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */ __ASM_EMIT("xorps %%xmm1, %%xmm1") /* xmm1 = 0 */ __ASM_EMIT("movups %%xmm0, 0x00(%[dst])") __ASM_EMIT("movups %%xmm1, 0x10(%[dst])") : : [items] "r" (items), [src] "r" (src), [dst] "r" (dst) : "cc", "memory", "%xmm0", "%xmm1" ); } // Iterate butterflies for (; n > 4; n >>= 1, bs >>= 1) { for (size_t p=0; p> 1; // Iterate first cycle if (n > 4) { // ONE LARGE CYCLE // Set initial values of pointers float *a = dst; float *b = &a[n]; size_t k = n; ARCH_X86_ASM ( __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */ __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */ __ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 */ : : [ak] "r"(ak) : "%xmm5", "%xmm6", "%xmm7" ); ARCH_X86_ASM ( // __ASM_EMIT(".align 16") __ASM_EMIT("1:") __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */ __ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 */ __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = s[i] */ __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = s[i] */ __ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = s[i]*iA[i] */ __ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = s[i]*rA[i] */ __ASM_EMIT("subps %%xmm4, %%xmm3") /* xmm3 = -s[i]*iA[i] */ __ASM_EMIT("movups %%xmm0, 0x00(%[a])") __ASM_EMIT("movups %%xmm5, 0x10(%[a])") __ASM_EMIT("movups %%xmm2, 0x00(%[b])") __ASM_EMIT("movups %%xmm3, 0x10(%[b])") __ASM_EMIT("add $0x10, %[src]") __ASM_EMIT("add $0x20, %[a]") __ASM_EMIT("add $0x20, %[b]") /* Repeat loop */ __ASM_EMIT32("subl $8, %[k]") __ASM_EMIT64("sub $8, %[k]") __ASM_EMIT("jz 2f") /* Rotate angle */ __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw[i] */ __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw[i] */ __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw[i] */ __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw[i] */ __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i] * iw[i] */ __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = iA[i] * iw[i] */ __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = rA[i] * rw[i] */ __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = iA[i] * rw[i] */ __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = rA[i] * rw[i] - iA[i] * iw[i] */ __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = rA[i] * iw[i] + iA[i] * rw[i] */ __ASM_EMIT("jmp 1b") __ASM_EMIT("2:") : [a] "+r" (a), [b] "+r" (b), [src] "+r" (src), [k] "+r" (k) : [wk] "r" (wk) : "cc", "memory", "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7" ); wk -= 8; ak -= 8; n >>= 1; bs >>= 1; } else { // Unpack 4x real to 4x split complex ARCH_X86_ASM ( __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */ __ASM_EMIT("xorps %%xmm1, %%xmm1") /* xmm1 = 0 */ __ASM_EMIT("movups %%xmm0, 0x00(%[dst])") __ASM_EMIT("movups %%xmm1, 0x10(%[dst])") : : [src] "r" (src), [dst] "r" (dst) : "cc", "memory", "%xmm0", "%xmm1" ); } // Iterate butterflies for (; n > 4; n >>= 1, bs >>= 1) { for (size_t p=0; p