/* * Copyright (C) 2020 Linux Studio Plugins Project * (C) 2020 Vladimir Sadovnikov * * This file is part of lsp-dsp-lib * Created on: 31 мар. 2020 г. * * lsp-dsp-lib is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * lsp-dsp-lib is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with lsp-dsp-lib. If not, see . */ #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL #error "This header should not be included directly" #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */ namespace lsp { namespace sse { static inline void FFT_SCRAMBLE_SELF_DIRECT_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank) { // Calculate number of items size_t items = (1 << rank) - 1; for (size_t i = 1; i < items; ++i) { size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */ if (i >= j) continue; /* Copy the values from the reversed position */ ARCH_X86_ASM ( __ASM_EMIT("movss (%[dst_re], %[i], 4), %%xmm0") __ASM_EMIT("movss (%[dst_im], %[i], 4), %%xmm1") __ASM_EMIT("movss (%[dst_re], %[j], 4), %%xmm2") __ASM_EMIT("movss (%[dst_im], %[j], 4), %%xmm3") __ASM_EMIT("movss %%xmm2, (%[dst_re], %[i], 4)") __ASM_EMIT("movss %%xmm3, (%[dst_im], %[i], 4)") __ASM_EMIT("movss %%xmm0, (%[dst_re], %[j], 4)") __ASM_EMIT("movss %%xmm1, (%[dst_im], %[j], 4)") : : [dst_re] "r"(dst_re), [dst_im] "r"(dst_im), [i] "r"(i), [j] "r"(j) : "memory", "%xmm0", "%xmm1", "%xmm2", "%xmm3" ); } // Perform butterfly 8x items = 1 << (rank - 3); // Perform 4-element butterflies ARCH_X86_ASM ( /* Loop */ __ASM_EMIT(".align 16") __ASM_EMIT("1:") /* Load data to registers */ __ASM_EMIT("movups 0x00(%[dst_re]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */ __ASM_EMIT("movups 0x10(%[dst_re]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */ __ASM_EMIT("movups 0x00(%[dst_im]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */ __ASM_EMIT("movups 0x10(%[dst_im]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */ /* 1st-order 4x butterfly */ __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 r1 r2 r3 */ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0 i1 i2 i3 */ __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0 r2 r4 r6 */ __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0 i2 i4 i6 */ __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r1 r3 r5 r7 */ __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i1 i3 i5 i7 */ __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0 r2 r4 r6 */ __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0 i2 i4 i6 */ __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */ __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */ __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */ __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */ /* 2nd-order 4x butterfly */ __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' r2' r4' r6' */ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0' i2' i4' i6' */ __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0' r4' r1' r5' */ __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm1") /* xmm1 = r2' r6' i3' i7' */ __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm3") /* xmm3 = i2' i6' r3' r7' */ __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0' i4' i1' i5' */ __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0' r4' r1' r5' */ __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0' i4' i1' i5' */ __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */ __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */ __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */ __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */ /* Reorder and store */ __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" r4" r1" r5" */ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0" i4" i3" i7" */ __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0" r1" r2" r3" */ __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0" i3" i2" i1" */ __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r4" r5" r6" r7" */ __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i4" i7" i6" i5" */ __ASM_EMIT("shufps $0x6c, %%xmm2, %%xmm2") /* xmm2 = i0" i1" i2" i3" */ __ASM_EMIT("shufps $0x6c, %%xmm3, %%xmm3") /* xmm3 = i4" i5" i6" i7" */ __ASM_EMIT("movups %%xmm0, 0x00(%[dst_re])") __ASM_EMIT("movups %%xmm1, 0x10(%[dst_re])") __ASM_EMIT("movups %%xmm2, 0x00(%[dst_im])") __ASM_EMIT("movups %%xmm3, 0x10(%[dst_im])") /* Move pointers */ __ASM_EMIT("add $0x20, %[dst_re]") __ASM_EMIT("add $0x20, %[dst_im]") /* Repeat cycle */ __ASM_EMIT("dec %[items]") __ASM_EMIT("jnz 1b") __ASM_EMIT("2:") : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im), [items] "+r"(items) : : "cc", "memory", "%xmm0", "%xmm1", "%xmm2" ); } static inline void FFT_SCRAMBLE_COPY_DIRECT_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank) { size_t regs = 1 << rank; for (size_t i=0; i= j) continue; /* Copy the values from the reversed position */ ARCH_X86_ASM ( __ASM_EMIT("movss (%[dst_re], %[i], 4), %%xmm0") __ASM_EMIT("movss (%[dst_im], %[i], 4), %%xmm1") __ASM_EMIT("movss (%[dst_re], %[j], 4), %%xmm2") __ASM_EMIT("movss (%[dst_im], %[j], 4), %%xmm3") __ASM_EMIT("movss %%xmm2, (%[dst_re], %[i], 4)") __ASM_EMIT("movss %%xmm3, (%[dst_im], %[i], 4)") __ASM_EMIT("movss %%xmm0, (%[dst_re], %[j], 4)") __ASM_EMIT("movss %%xmm1, (%[dst_im], %[j], 4)") : : [dst_re] "r"(dst_re), [dst_im] "r"(dst_im), [i] "r"(i), [j] "r"(j) : "memory", "%xmm0", "%xmm1", "%xmm2", "%xmm3" ); } // Perform butterfly 4x items = 1 << (rank - 3); // Perform 4-element butterflies ARCH_X86_ASM ( /* Loop */ __ASM_EMIT(".align 16") __ASM_EMIT("1:") /* Load data to registers */ __ASM_EMIT("movups 0x00(%[dst_re]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */ __ASM_EMIT("movups 0x10(%[dst_re]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */ __ASM_EMIT("movups 0x00(%[dst_im]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */ __ASM_EMIT("movups 0x10(%[dst_im]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */ /* 1st-order 4x butterfly */ __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 r1 r2 r3 */ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0 i1 i2 i3 */ __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0 r2 r4 r6 */ __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0 i2 i4 i6 */ __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r1 r3 r5 r7 */ __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i1 i3 i5 i7 */ __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0 r2 r4 r6 */ __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0 i2 i4 i6 */ __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */ __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */ __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */ __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */ /* 2nd-order 4x butterfly */ __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' r2' r4' r6' */ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0' i2' i4' i6' */ __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0' r4' r1' r5' */ __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm1") /* xmm1 = r2' r6' i3' i7' */ __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm3") /* xmm3 = i2' i6' r3' r7' */ __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0' i4' i1' i5' */ __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0' r4' r1' r5' */ __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0' i4' i1' i5' */ __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */ __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+i3' i5'+i7' = i0" i4" i1" i5" */ __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */ __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0'-i2' i4'-i6' i1'-i3' i5'-i7' = i2" i6" i3" i7" */ /* Reorder and store */ __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" r4" r3" r7" */ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0" i4" i1" i5" */ __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0" r3" r2" r1" */ __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0" i1" i2" i3" */ __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r4" r7" r6" r5" */ __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i4" i5" i6" i7" */ __ASM_EMIT("shufps $0x6c, %%xmm0, %%xmm0") /* xmm0 = r0" r1" r2" r3" */ __ASM_EMIT("shufps $0x6c, %%xmm1, %%xmm1") /* xmm1 = r4" r5" r6" r7" */ __ASM_EMIT("movups %%xmm0, 0x00(%[dst_re])") __ASM_EMIT("movups %%xmm1, 0x10(%[dst_re])") __ASM_EMIT("movups %%xmm2, 0x00(%[dst_im])") __ASM_EMIT("movups %%xmm3, 0x10(%[dst_im])") /* Move pointers */ __ASM_EMIT("add $0x20, %[dst_re]") __ASM_EMIT("add $0x20, %[dst_im]") /* Repeat cycle */ __ASM_EMIT("dec %[items]") __ASM_EMIT("jnz 1b") __ASM_EMIT("2:") : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im), [items] "+r"(items) : : "cc", "memory", "%xmm0", "%xmm1", "%xmm2" ); } static inline void FFT_SCRAMBLE_COPY_REVERSE_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank) { size_t regs = 1 << rank; for (size_t i=0; i