Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/fft/normalize.h

raw
1 /*
2 * Copyright (C) 2025 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2025 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 23 февр. 2025 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_FFT_NORMALIZE_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_FFT_NORMALIZE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 void normalize_fft3(float *dre, float *dim, const float *re, const float *im, size_t rank)
34 {
35 IF_ARCH_X86(
36 float k = 1.0f/(1 << rank);
37 size_t count = 1 << rank, off = 0;
38 );
39 ARCH_X86_ASM(
40 __ASM_EMIT ("vbroadcastss %[k], %%zmm0") // zmm0 = k
41 // x32 blocks
42 __ASM_EMIT32("subl $32, %[count]")
43 __ASM_EMIT64("sub $32, %[count]")
44 __ASM_EMIT ("vmovaps %%zmm0, %%zmm1")
45 __ASM_EMIT ("jb 2f")
46 __ASM_EMIT ("1:")
47 __ASM_EMIT ("vmulps 0x00(%[s_re], %[off]), %%zmm0, %%zmm4")
48 __ASM_EMIT ("vmulps 0x40(%[s_re], %[off]), %%zmm1, %%zmm5")
49 __ASM_EMIT ("vmulps 0x00(%[s_im], %[off]), %%zmm0, %%zmm6")
50 __ASM_EMIT ("vmulps 0x40(%[s_im], %[off]), %%zmm1, %%zmm7")
51 __ASM_EMIT ("vmovups %%zmm4, 0x00(%[d_re], %[off])")
52 __ASM_EMIT ("vmovups %%zmm5, 0x40(%[d_re], %[off])")
53 __ASM_EMIT ("vmovups %%zmm6, 0x00(%[d_im], %[off])")
54 __ASM_EMIT ("vmovups %%zmm7, 0x40(%[d_im], %[off])")
55 __ASM_EMIT ("add $0x80, %[off]")
56 __ASM_EMIT32("subl $32, %[count]")
57 __ASM_EMIT64("sub $32, %[count]")
58 __ASM_EMIT ("jae 1b")
59 // x16 block
60 __ASM_EMIT ("2:")
61 __ASM_EMIT32("addl $16, %[count]")
62 __ASM_EMIT64("add $16, %[count]")
63 __ASM_EMIT ("jl 4f")
64 __ASM_EMIT ("vmulps 0x00(%[s_re], %[off]), %%ymm0, %%ymm4")
65 __ASM_EMIT ("vmulps 0x20(%[s_re], %[off]), %%ymm1, %%ymm5")
66 __ASM_EMIT ("vmulps 0x00(%[s_im], %[off]), %%ymm0, %%ymm6")
67 __ASM_EMIT ("vmulps 0x20(%[s_im], %[off]), %%ymm1, %%ymm7")
68 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
69 __ASM_EMIT ("vmovups %%ymm5, 0x20(%[d_re], %[off])")
70 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
71 __ASM_EMIT ("vmovups %%ymm7, 0x20(%[d_im], %[off])")
72 __ASM_EMIT ("add $0x40, %[off]")
73 __ASM_EMIT32("subl $16, %[count]")
74 __ASM_EMIT64("sub $16, %[count]")
75 // x8 block
76 __ASM_EMIT ("4:")
77 __ASM_EMIT32("addl $8, %[count]")
78 __ASM_EMIT64("add $8, %[count]")
79 __ASM_EMIT ("jl 6f")
80 __ASM_EMIT ("vmulps 0x00(%[s_re], %[off]), %%ymm0, %%ymm4")
81 __ASM_EMIT ("vmulps 0x00(%[s_im], %[off]), %%ymm1, %%ymm6")
82 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
83 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
84 __ASM_EMIT ("6:")
85 : [off] "+r" (off),
86 [count] X86_PGREG (count),
87 [k] "+Yz" (k)
88 : [s_re] "r" (re), [s_im] "r" (im),
89 [d_re] "r" (dre), [d_im] "r" (dim)
90 : "cc", "memory",
91 "%xmm1",
92 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
93 );
94 }
95
96 void normalize_fft2(float *re, float *im, size_t rank)
97 {
98 IF_ARCH_X86(
99 float k = 1.0f/(1 << rank);
100 size_t count = 1 << rank, off = 0;
101 );
102 ARCH_X86_ASM(
103 __ASM_EMIT ("vbroadcastss %[k], %%zmm0") // zmm0 = k
104 // x32 blocks
105 __ASM_EMIT32("subl $32, %[count]")
106 __ASM_EMIT64("sub $32, %[count]")
107 __ASM_EMIT ("vmovaps %%zmm0, %%zmm1")
108 __ASM_EMIT ("jb 2f")
109 __ASM_EMIT ("1:")
110 __ASM_EMIT ("vmulps 0x00(%[d_re], %[off]), %%zmm0, %%zmm4")
111 __ASM_EMIT ("vmulps 0x40(%[d_re], %[off]), %%zmm1, %%zmm5")
112 __ASM_EMIT ("vmulps 0x00(%[d_im], %[off]), %%zmm0, %%zmm6")
113 __ASM_EMIT ("vmulps 0x40(%[d_im], %[off]), %%zmm1, %%zmm7")
114 __ASM_EMIT ("vmovups %%zmm4, 0x00(%[d_re], %[off])")
115 __ASM_EMIT ("vmovups %%zmm5, 0x40(%[d_re], %[off])")
116 __ASM_EMIT ("vmovups %%zmm6, 0x00(%[d_im], %[off])")
117 __ASM_EMIT ("vmovups %%zmm7, 0x40(%[d_im], %[off])")
118 __ASM_EMIT ("add $0x80, %[off]")
119 __ASM_EMIT32("subl $32, %[count]")
120 __ASM_EMIT64("sub $32, %[count]")
121 __ASM_EMIT ("jae 1b")
122 // x16 block
123 __ASM_EMIT ("2:")
124 __ASM_EMIT32("addl $16, %[count]")
125 __ASM_EMIT64("add $16, %[count]")
126 __ASM_EMIT ("jl 4f")
127 __ASM_EMIT ("vmulps 0x00(%[d_re], %[off]), %%ymm0, %%ymm4")
128 __ASM_EMIT ("vmulps 0x20(%[d_re], %[off]), %%ymm1, %%ymm5")
129 __ASM_EMIT ("vmulps 0x00(%[d_im], %[off]), %%ymm0, %%ymm6")
130 __ASM_EMIT ("vmulps 0x20(%[d_im], %[off]), %%ymm1, %%ymm7")
131 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
132 __ASM_EMIT ("vmovups %%ymm5, 0x20(%[d_re], %[off])")
133 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
134 __ASM_EMIT ("vmovups %%ymm7, 0x20(%[d_im], %[off])")
135 __ASM_EMIT ("add $0x40, %[off]")
136 __ASM_EMIT32("subl $16, %[count]")
137 __ASM_EMIT64("sub $16, %[count]")
138 // x8 block
139 __ASM_EMIT ("4:")
140 __ASM_EMIT32("addl $8, %[count]")
141 __ASM_EMIT64("add $8, %[count]")
142 __ASM_EMIT ("jl 6f")
143 __ASM_EMIT ("vmulps 0x00(%[d_re], %[off]), %%ymm0, %%ymm4")
144 __ASM_EMIT ("vmulps 0x00(%[d_im], %[off]), %%ymm1, %%ymm6")
145 __ASM_EMIT ("vmovups %%ymm4, 0x00(%[d_re], %[off])")
146 __ASM_EMIT ("vmovups %%ymm6, 0x00(%[d_im], %[off])")
147 __ASM_EMIT ("6:")
148 : [off] "+r" (off), [count] "+r" (count),
149 [k] "+Yz" (k)
150 : [d_re] "r" (re), [d_im] "r" (im)
151 : "cc", "memory",
152 "%xmm1",
153 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
154 );
155 }
156 } /* namespace avx512 */
157 } /* namespace lsp */
158
159
160
161 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_FFT_NORMALIZE_H_ */