Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/fft/normalize.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_FFT_NORMALIZE_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_FFT_NORMALIZE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 void normalize_fft3(float *dre, float *dim, const float *re, const float *im, size_t rank)
34 {
35 IF_ARCH_X86(
36 float k = 1.0f/(1 << rank);
37 size_t count = 1 << rank, off = 0;
38 );
39 ARCH_X86_ASM(
40 // x16 blocks
41 __ASM_EMIT ("vbroadcastss %[k], %%ymm0") // ymm0 = k
42 __ASM_EMIT32("subl $16, %[count]")
43 __ASM_EMIT64("sub $16, %[count]")
44 __ASM_EMIT ("vmovaps %%ymm0, %%ymm1")
45 __ASM_EMIT ("jb 2f")
46 __ASM_EMIT ("1:")
47 __ASM_EMIT ("vmulps 0x00(%[s_re], %[off]), %%ymm0, %%ymm4")
48 __ASM_EMIT ("vmulps 0x20(%[s_re], %[off]), %%ymm1, %%ymm5")
49 __ASM_EMIT ("vmulps 0x00(%[s_im], %[off]), %%ymm0, %%ymm6")
50 __ASM_EMIT ("vmulps 0x20(%[s_im], %[off]), %%ymm1, %%ymm7")
51 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
52 __ASM_EMIT ("vmovups %%ymm5, 0x20(%[d_re], %[off])")
53 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
54 __ASM_EMIT ("vmovups %%ymm7, 0x20(%[d_im], %[off])")
55 __ASM_EMIT ("add $0x40, %[off]")
56 __ASM_EMIT32("subl $16, %[count]")
57 __ASM_EMIT64("sub $16, %[count]")
58 __ASM_EMIT ("jae 1b")
59 __ASM_EMIT ("2:")
60 // x8 block
61 __ASM_EMIT32("addl $8, %[count]")
62 __ASM_EMIT64("add $8, %[count]")
63 __ASM_EMIT ("jl 4f")
64 __ASM_EMIT ("vmulps 0x00(%[s_re], %[off]), %%ymm0, %%ymm4")
65 __ASM_EMIT ("vmulps 0x00(%[s_im], %[off]), %%ymm1, %%ymm6")
66 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
67 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
68 __ASM_EMIT ("4:")
69 : [off] "+r" (off),
70 [count] X86_PGREG (count)
71 : [s_re] "r" (re), [s_im] "r" (im),
72 [d_re] "r" (dre), [d_im] "r" (dim),
73 [k] "m" (k)
74 : "cc", "memory",
75 "%xmm0", "%xmm1",
76 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
77 );
78 }
79
80 void normalize_fft2(float *re, float *im, size_t rank)
81 {
82 IF_ARCH_X86(
83 float k = 1.0f/(1 << rank);
84 size_t count = 1 << rank, off = 0;
85 );
86 ARCH_X86_ASM(
87 // x16 blocks
88 __ASM_EMIT ("vbroadcastss %[k], %%ymm0") // ymm0 = k
89 __ASM_EMIT32("subl $16, %[count]")
90 __ASM_EMIT64("sub $16, %[count]")
91 __ASM_EMIT ("vmovaps %%ymm0, %%ymm1")
92 __ASM_EMIT ("jb 2f")
93 __ASM_EMIT ("1:")
94 __ASM_EMIT ("vmulps 0x00(%[d_re], %[off]), %%ymm0, %%ymm4")
95 __ASM_EMIT ("vmulps 0x20(%[d_re], %[off]), %%ymm1, %%ymm5")
96 __ASM_EMIT ("vmulps 0x00(%[d_im], %[off]), %%ymm0, %%ymm6")
97 __ASM_EMIT ("vmulps 0x20(%[d_im], %[off]), %%ymm1, %%ymm7")
98 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
99 __ASM_EMIT ("vmovups %%ymm5, 0x20(%[d_re], %[off])")
100 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
101 __ASM_EMIT ("vmovups %%ymm7, 0x20(%[d_im], %[off])")
102 __ASM_EMIT ("add $0x40, %[off]")
103 __ASM_EMIT32("subl $16, %[count]")
104 __ASM_EMIT64("sub $16, %[count]")
105 __ASM_EMIT ("jae 1b")
106 __ASM_EMIT ("2:")
107 // x8 block
108 __ASM_EMIT32("addl $8, %[count]")
109 __ASM_EMIT64("add $8, %[count]")
110 __ASM_EMIT ("jl 4f")
111 __ASM_EMIT ("vmulps 0x00(%[d_re], %[off]), %%ymm0, %%ymm4")
112 __ASM_EMIT ("vmulps 0x00(%[d_im], %[off]), %%ymm1, %%ymm6")
113 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
114 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
115 __ASM_EMIT ("4:")
116 : [off] "+r" (off), [count] "+r" (count)
117 : [d_re] "r" (re), [d_im] "r" (im),
118 [k] "m" (k)
119 : "cc", "memory",
120 "%xmm0", "%xmm1",
121 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
122 );
123 }
124 } /* namespace avx */
125 } /* namespace lsp */
126
127 #endif /* PRIVATE_DSP_ARCH_X86_AVX_FFT_NORMALIZE_H_ */