Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/fft.h

raw
1 /*
2 * Copyright (C) 2025 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2025 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
25
26 #include <private/dsp/arch/x86/avx/fft/const.h>
27 #include <private/dsp/arch/x86/avx/fft/butterfly.h>
28 #include <private/dsp/arch/x86/avx/fft/normalize.h>
29
30 #define FFT_SCRAMBLE_SELF_DIRECT_NAME scramble_self_direct8
31 #define FFT_SCRAMBLE_SELF_REVERSE_NAME scramble_self_reverse8
32 #define FFT_SCRAMBLE_COPY_DIRECT_NAME scramble_copy_direct8
33 #define FFT_SCRAMBLE_COPY_REVERSE_NAME scramble_copy_reverse8
34 #define FFT_TYPE uint8_t
35 #define FFT_FMA(a, b) a
36 #include <private/dsp/arch/x86/avx/fft/scramble.h>
37
38 #define FFT_SCRAMBLE_SELF_DIRECT_NAME scramble_self_direct16
39 #define FFT_SCRAMBLE_SELF_REVERSE_NAME scramble_self_reverse16
40 #define FFT_SCRAMBLE_COPY_DIRECT_NAME scramble_copy_direct16
41 #define FFT_SCRAMBLE_COPY_REVERSE_NAME scramble_copy_reverse16
42 #define FFT_TYPE uint16_t
43 #define FFT_FMA(a, b) a
44 #include <private/dsp/arch/x86/avx/fft/scramble.h>
45
46 #define FFT_SCRAMBLE_SELF_DIRECT_NAME scramble_self_direct8_fma3
47 #define FFT_SCRAMBLE_SELF_REVERSE_NAME scramble_self_reverse8_fma3
48 #define FFT_SCRAMBLE_COPY_DIRECT_NAME scramble_copy_direct8_fma3
49 #define FFT_SCRAMBLE_COPY_REVERSE_NAME scramble_copy_reverse8_fma3
50 #define FFT_TYPE uint8_t
51 #define FFT_FMA(a, b) b
52 #include <private/dsp/arch/x86/avx/fft/scramble.h>
53
54 #define FFT_SCRAMBLE_SELF_DIRECT_NAME scramble_self_direct16_fma3
55 #define FFT_SCRAMBLE_SELF_REVERSE_NAME scramble_self_reverse16_fma3
56 #define FFT_SCRAMBLE_COPY_DIRECT_NAME scramble_copy_direct16_fma3
57 #define FFT_SCRAMBLE_COPY_REVERSE_NAME scramble_copy_reverse16_fma3
58 #define FFT_TYPE uint16_t
59 #define FFT_FMA(a, b) b
60 #include <private/dsp/arch/x86/avx/fft/scramble.h>
61
62 namespace lsp
63 {
64 namespace avx
65 {
66 static inline void small_direct_fft(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
67 {
68 if (rank == 2)
69 {
70 float s0_re = src_re[0] + src_re[2];
71 float s1_re = src_re[0] - src_re[2];
72 float s2_re = src_re[1] + src_re[3];
73 float s3_re = src_re[1] - src_re[3];
74
75 float s0_im = src_im[0] + src_im[2];
76 float s1_im = src_im[0] - src_im[2];
77 float s2_im = src_im[1] + src_im[3];
78 float s3_im = src_im[1] - src_im[3];
79
80 dst_re[0] = s0_re + s2_re;
81 dst_re[1] = s1_re + s3_im;
82 dst_re[2] = s0_re - s2_re;
83 dst_re[3] = s1_re - s3_im;
84
85 dst_im[0] = s0_im + s2_im;
86 dst_im[1] = s1_im - s3_re;
87 dst_im[2] = s0_im - s2_im;
88 dst_im[3] = s1_im + s3_re;
89 }
90 else if (rank == 1)
91 {
92 // s0' = s0 + s1
93 // s1' = s0 - s1
94 float s1_re = src_re[1];
95 float s1_im = src_im[1];
96 dst_re[1] = src_re[0] - s1_re;
97 dst_im[1] = src_im[0] - s1_im;
98 dst_re[0] = src_re[0] + s1_re;
99 dst_im[0] = src_im[0] + s1_im;
100 }
101 else
102 {
103 dst_re[0] = src_re[0];
104 dst_im[0] = src_im[0];
105 }
106 }
107
108 static inline void small_reverse_fft(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
109 {
110 if (rank == 2)
111 {
112 float s0_re = src_re[0] + src_re[2];
113 float s1_re = src_re[0] - src_re[2];
114 float s2_re = src_re[1] + src_re[3];
115 float s3_re = src_re[1] - src_re[3];
116
117 float s0_im = src_im[0] + src_im[2];
118 float s1_im = src_im[0] - src_im[2];
119 float s2_im = src_im[1] + src_im[3];
120 float s3_im = src_im[1] - src_im[3];
121
122 dst_re[0] = (s0_re + s2_re) * 0.25f;
123 dst_re[1] = (s1_re - s3_im) * 0.25f;
124 dst_re[2] = (s0_re - s2_re) * 0.25f;
125 dst_re[3] = (s1_re + s3_im) * 0.25f;
126
127 dst_im[0] = (s0_im + s2_im) * 0.25f;
128 dst_im[1] = (s1_im + s3_re) * 0.25f;
129 dst_im[2] = (s0_im - s2_im) * 0.25f;
130 dst_im[3] = (s1_im - s3_re) * 0.25f;
131 }
132 else if (rank == 1)
133 {
134 // s0' = s0 + s1
135 // s1' = s0 - s1
136 float s1_re = src_re[1];
137 float s1_im = src_im[1];
138 dst_re[1] = src_re[0] - s1_re;
139 dst_im[1] = src_im[0] - s1_im;
140 dst_re[0] = src_re[0] + s1_re;
141 dst_im[0] = src_im[0] + s1_im;
142 }
143 else
144 {
145 dst_re[0] = src_re[0];
146 dst_im[0] = src_im[0];
147 }
148 }
149
150 void direct_fft(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
151 {
152 // Check bounds
153 if (rank <= 2)
154 {
155 small_direct_fft(dst_re, dst_im, src_re, src_im, rank);
156 return;
157 }
158
159 if ((dst_re == src_re) || (dst_im == src_im) || (rank < 4))
160 {
161 dsp::move(dst_re, src_re, 1 << rank);
162 dsp::move(dst_im, src_im, 1 << rank);
163 if (rank <= 8)
164 scramble_self_direct8(dst_re, dst_im, rank);
165 else
166 scramble_self_direct16(dst_re, dst_im, rank);
167 }
168 else
169 {
170 if (rank <= 12)
171 scramble_copy_direct8(dst_re, dst_im, src_re, src_im, rank-4);
172 else
173 scramble_copy_direct16(dst_re, dst_im, src_re, src_im, rank-4);
174 }
175
176 for (size_t i=3; i < rank; ++i)
177 butterfly_direct8p(dst_re, dst_im, i, 1 << (rank - i - 1));
178 }
179
180 void direct_fft_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
181 {
182 // Check bounds
183 if (rank <= 2)
184 {
185 small_direct_fft(dst_re, dst_im, src_re, src_im, rank);
186 return;
187 }
188
189 if ((dst_re == src_re) || (dst_im == src_im) || (rank < 4))
190 {
191 dsp::move(dst_re, src_re, 1 << rank);
192 dsp::move(dst_im, src_im, 1 << rank);
193 if (rank <= 8)
194 scramble_self_direct8_fma3(dst_re, dst_im, rank);
195 else
196 scramble_self_direct16_fma3(dst_re, dst_im, rank);
197 }
198 else
199 {
200 if (rank <= 12)
201 scramble_copy_direct8_fma3(dst_re, dst_im, src_re, src_im, rank-4);
202 else
203 scramble_copy_direct16_fma3(dst_re, dst_im, src_re, src_im, rank-4);
204 }
205
206 for (size_t i=3; i < rank; ++i)
207 butterfly_direct8p_fma3(dst_re, dst_im, i, 1 << (rank - i - 1));
208 }
209
210 void reverse_fft(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
211 {
212 // Check bounds
213 if (rank <= 2)
214 {
215 small_reverse_fft(dst_re, dst_im, src_re, src_im, rank);
216 return;
217 }
218
219 if ((dst_re == src_re) || (dst_im == src_im) || (rank < 4))
220 {
221 dsp::move(dst_re, src_re, 1 << rank);
222 dsp::move(dst_im, src_im, 1 << rank);
223 if (rank <= 8)
224 scramble_self_reverse8(dst_re, dst_im, rank);
225 else
226 scramble_self_reverse16(dst_re, dst_im, rank);
227 }
228 else
229 {
230 if (rank <= 12)
231 scramble_copy_reverse8(dst_re, dst_im, src_re, src_im, rank-4);
232 else
233 scramble_copy_reverse16(dst_re, dst_im, src_re, src_im, rank-4);
234 }
235
236 for (size_t i=3; i < rank; ++i)
237 butterfly_reverse8p(dst_re, dst_im, i, 1 << (rank - i - 1));
238
239 dsp::normalize_fft2(dst_re, dst_im, rank);
240 }
241
242 void reverse_fft_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
243 {
244 // Check bounds
245 if (rank <= 2)
246 {
247 small_reverse_fft(dst_re, dst_im, src_re, src_im, rank);
248 return;
249 }
250
251 if ((dst_re == src_re) || (dst_im == src_im) || (rank < 4))
252 {
253 dsp::move(dst_re, src_re, 1 << rank);
254 dsp::move(dst_im, src_im, 1 << rank);
255 if (rank <= 8)
256 scramble_self_reverse8_fma3(dst_re, dst_im, rank);
257 else
258 scramble_self_reverse16_fma3(dst_re, dst_im, rank);
259 }
260 else
261 {
262 if (rank <= 12)
263 scramble_copy_reverse8_fma3(dst_re, dst_im, src_re, src_im, rank-4);
264 else
265 scramble_copy_reverse16_fma3(dst_re, dst_im, src_re, src_im, rank-4);
266 }
267
268 for (size_t i=3; i < rank; ++i)
269 butterfly_reverse8p_fma3(dst_re, dst_im, i, 1 << (rank - i - 1));
270
271 dsp::normalize_fft2(dst_re, dst_im, rank);
272 }
273
274 } /* namespace avx */
275 } /* namespace lsp */