Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/fastconv.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_FASTCONV_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_FASTCONV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 #include <private/dsp/arch/x86/avx/fastconv/prepare.h>
30 #include <private/dsp/arch/x86/avx/fastconv/butterfly.h>
31 #include <private/dsp/arch/x86/avx/fastconv/apply.h>
32
33 namespace lsp
34 {
35 namespace avx
36 {
37 void fastconv_parse(float *dst, const float *src, size_t rank)
38 {
39 const float *ak = &FFT_A[(rank - 3) << 4];
40 const float *wk = &FFT_DW[(rank - 3) << 4];
41 size_t np = 1 << (rank - 1);
42 size_t nb = 1;
43
44 if (np > 4)
45 {
46 fastconv_direct_prepare(dst, src, ak, wk, np);
47 ak -= 16;
48 wk -= 16;
49 np >>= 1;
50 nb <<= 1;
51 }
52 else
53 fastconv_direct_unpack(dst, src);
54
55 while (np > 4)
56 {
57 fastconv_direct_butterfly(dst, ak, wk, np, nb);
58 ak -= 16;
59 wk -= 16;
60 np >>= 1;
61 nb <<= 1;
62 }
63
64 fastconv_direct_butterfly_last(dst, nb);
65 }
66
67 void fastconv_parse_fma3(float *dst, const float *src, size_t rank)
68 {
69 const float *ak = &FFT_A[(rank - 3) << 4];
70 const float *wk = &FFT_DW[(rank - 3) << 4];
71 size_t np = 1 << (rank - 1);
72 size_t nb = 1;
73
74 if (np > 4)
75 {
76 fastconv_direct_prepare_fma3(dst, src, ak, wk, np);
77 ak -= 16;
78 wk -= 16;
79 np >>= 1;
80 nb <<= 1;
81 }
82 else
83 fastconv_direct_unpack(dst, src);
84
85 while (np > 4)
86 {
87 fastconv_direct_butterfly_fma3(dst, ak, wk, np, nb);
88 ak -= 16;
89 wk -= 16;
90 np >>= 1;
91 nb <<= 1;
92 }
93
94 fastconv_direct_butterfly_last_fma3(dst, nb);
95 }
96
97 void fastconv_restore(float *dst, float *tmp, size_t rank)
98 {
99 size_t nb = 1 << (rank - 3), np = 4;
100 const float *ak = FFT_A;
101 const float *wk = FFT_DW;
102
103 fastconv_reverse_prepare(tmp, nb);
104 if ((nb >>= 1) <= 0)
105 {
106 fastconv_reverse_unpack(dst, tmp, rank);
107 return;
108 }
109 ak += 16;
110 wk += 16;
111 np <<= 1;
112
113 while (nb > 1)
114 {
115 fastconv_reverse_butterfly(tmp, ak, wk, np, nb);
116 ak += 16;
117 wk += 16;
118 np <<= 1;
119 nb >>= 1;
120 }
121
122 fastconv_reverse_butterfly_last(dst, tmp, ak, wk, np);
123 }
124
125 void fastconv_restore_fma3(float *dst, float *tmp, size_t rank)
126 {
127 size_t nb = 1 << (rank - 3), np = 4;
128 const float *ak = FFT_A;
129 const float *wk = FFT_DW;
130
131 fastconv_reverse_prepare_fma3(tmp, nb);
132 if ((nb >>= 1) <= 0)
133 {
134 fastconv_reverse_unpack(dst, tmp, rank);
135 return;
136 }
137 ak += 16;
138 wk += 16;
139 np <<= 1;
140
141 while (nb > 1)
142 {
143 fastconv_reverse_butterfly_fma3(tmp, ak, wk, np, nb);
144 ak += 16;
145 wk += 16;
146 np <<= 1;
147 nb >>= 1;
148 }
149
150 fastconv_reverse_butterfly_last_fma3(dst, tmp, ak, wk, np);
151 }
152
153 void fastconv_apply(float *dst, float *tmp, const float *c1, const float *c2, size_t rank)
154 {
155 size_t nb = 1 << (rank - 3), np = 4;
156 const float *ak = FFT_A;
157 const float *wk = FFT_DW;
158
159 fastconv_apply_prepare(tmp, c1, c2, nb);
160 if ((nb >>= 1) <= 0)
161 {
162 fastconv_reverse_unpack_adding(dst, tmp, rank);
163 return;
164 }
165 ak += 16;
166 wk += 16;
167 np <<= 1;
168
169 while (nb > 1)
170 {
171 fastconv_reverse_butterfly(tmp, ak, wk, np, nb);
172 ak += 16;
173 wk += 16;
174 np <<= 1;
175 nb >>= 1;
176 }
177
178 fastconv_reverse_butterfly_last_adding(dst, tmp, ak, wk, np);
179 }
180
181 void fastconv_apply_fma3(float *dst, float *tmp, const float *c1, const float *c2, size_t rank)
182 {
183 size_t nb = 1 << (rank - 3), np = 4;
184 const float *ak = FFT_A;
185 const float *wk = FFT_DW;
186
187 fastconv_apply_prepare_fma3(tmp, c1, c2, nb);
188 if ((nb >>= 1) <= 0)
189 {
190 fastconv_reverse_unpack_adding(dst, tmp, rank);
191 return;
192 }
193 ak += 16;
194 wk += 16;
195 np <<= 1;
196
197 while (nb > 1)
198 {
199 fastconv_reverse_butterfly_fma3(tmp, ak, wk, np, nb);
200 ak += 16;
201 wk += 16;
202 np <<= 1;
203 nb >>= 1;
204 }
205
206 fastconv_reverse_butterfly_last_adding_fma3(dst, tmp, ak, wk, np);
207 }
208
209 void fastconv_parse_apply(float *dst, float *tmp, const float *c, const float *src, size_t rank)
210 {
211 const float *ak = &FFT_A[(rank - 3) << 4];
212 const float *wk = &FFT_DW[(rank - 3) << 4];
213 size_t np = 1 << (rank - 1);
214 size_t nb = 1;
215
216 if (np > 4)
217 {
218 fastconv_direct_prepare(tmp, src, ak, wk, np);
219 ak -= 16;
220 wk -= 16;
221 np >>= 1;
222 nb <<= 1;
223 }
224 else
225 fastconv_direct_unpack(tmp, src);
226
227 while (np > 4)
228 {
229 fastconv_direct_butterfly(tmp, ak, wk, np, nb);
230 ak -= 16;
231 wk -= 16;
232 np >>= 1;
233 nb <<= 1;
234 }
235
236 fastconv_apply_internal(tmp, c, nb);
237
238 if ((nb >>= 1) <= 0)
239 {
240 fastconv_reverse_unpack_adding(dst, tmp, rank);
241 return;
242 }
243 ak += 16;
244 wk += 16;
245 np <<= 1;
246
247 while (nb > 1)
248 {
249 fastconv_reverse_butterfly(tmp, ak, wk, np, nb);
250 ak += 16;
251 wk += 16;
252 np <<= 1;
253 nb >>= 1;
254 }
255
256 fastconv_reverse_butterfly_last_adding(dst, tmp, ak, wk, np);
257 }
258
259 void fastconv_parse_apply_fma3(float *dst, float *tmp, const float *c, const float *src, size_t rank)
260 {
261 const float *ak = &FFT_A[(rank - 3) << 4];
262 const float *wk = &FFT_DW[(rank - 3) << 4];
263 size_t np = 1 << (rank - 1);
264 size_t nb = 1;
265
266 if (np > 4)
267 {
268 fastconv_direct_prepare_fma3(tmp, src, ak, wk, np);
269 ak -= 16;
270 wk -= 16;
271 np >>= 1;
272 nb <<= 1;
273 }
274 else
275 fastconv_direct_unpack(tmp, src);
276
277 while (np > 4)
278 {
279 fastconv_direct_butterfly_fma3(tmp, ak, wk, np, nb);
280 ak -= 16;
281 wk -= 16;
282 np >>= 1;
283 nb <<= 1;
284 }
285
286 fastconv_apply_internal_fma3(tmp, c, nb);
287
288 if ((nb >>= 1) <= 0)
289 {
290 fastconv_reverse_unpack_adding(dst, tmp, rank);
291 return;
292 }
293 ak += 16;
294 wk += 16;
295 np <<= 1;
296
297 while (nb > 1)
298 {
299 fastconv_reverse_butterfly_fma3(tmp, ak, wk, np, nb);
300 ak += 16;
301 wk += 16;
302 np <<= 1;
303 nb >>= 1;
304 }
305
306 fastconv_reverse_butterfly_last_adding_fma3(dst, tmp, ak, wk, np);
307 }
308 }
309 }
310
311 #endif /* PRIVATE_DSP_ARCH_X86_AVX_FASTCONV_H_ */