Debian Codegraph @

.cargo/vendor/rustfft/src/wasm_simd/wasm_simd_prime_butterflies.rs

raw
1 /// Auto-generated prime length butterflies
2 /// The code here is mostly autogenerated by the python script tools/gen_sse_butterflies.py, and then translated from SSE to WASM SIMD.
3 ///
4 /// The algorithm is derived directly from the definition of the DFT, by eliminating any repeated calculations.
5 /// See the comments in src/algorithm/butterflies.rs for a detailed description.
6 ///
7 /// The script generates the code for performing a single f64 fft, as well as dual f32 fft.
8 /// It also generates the code for reading and writing the input and output.
9 /// The single 32-bit ffts reuse the dual ffts.
10 use core::arch::wasm32::*;
11 use num_complex::Complex;
12
13 use crate::{common::FftNum, FftDirection};
14
15 use crate::array_utils;
16 use crate::array_utils::workaround_transmute_mut;
17 use crate::array_utils::DoubleBuf;
18 use crate::common::{fft_error_inplace, fft_error_outofplace};
19 use crate::twiddles;
20 use crate::{Direction, Fft, Length};
21
22 use super::wasm_simd_butterflies::{parallel_fft2_interleaved_f32, solo_fft2_f64};
23 use super::wasm_simd_common::{assert_f32, assert_f64};
24 use super::wasm_simd_utils::*;
25 use super::wasm_simd_vector::WasmSimdArrayMut;
26
27 // _____ _________ _ _ _
28 // |___ | |___ /___ \| |__ (_) |_
29 // / / _____ |_ \ __) | '_ \| | __|
30 // / / |_____| ___) / __/| |_) | | |_
31 // /_/ |____/_____|_.__/|_|\__|
32 //
33
34 pub struct WasmSimdF32Butterfly7<T> {
35 direction: FftDirection,
36 _phantom: std::marker::PhantomData<T>,
37 rotate: Rotate90F32,
38 twiddle1re: v128,
39 twiddle1im: v128,
40 twiddle2re: v128,
41 twiddle2im: v128,
42 twiddle3re: v128,
43 twiddle3im: v128,
44 }
45
46 boilerplate_fft_wasm_simd_f32_butterfly!(
47 WasmSimdF32Butterfly7,
48 7,
49 |this: &WasmSimdF32Butterfly7<_>| this.direction
50 );
51 boilerplate_fft_wasm_simd_common_butterfly!(
52 WasmSimdF32Butterfly7,
53 7,
54 |this: &WasmSimdF32Butterfly7<_>| { this.direction }
55 );
56 impl<T: FftNum> WasmSimdF32Butterfly7<T> {
57 #[inline(always)]
58 pub fn new(direction: FftDirection) -> Self {
59 assert_f32::<T>();
60 let rotate = Rotate90F32::new(true);
61 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 7, direction);
62 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 7, direction);
63 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 7, direction);
64 let twiddle1re = f32x4_splat(tw1.re);
65 let twiddle1im = f32x4_splat(tw1.im);
66 let twiddle2re = f32x4_splat(tw2.re);
67 let twiddle2im = f32x4_splat(tw2.im);
68 let twiddle3re = f32x4_splat(tw3.re);
69 let twiddle3im = f32x4_splat(tw3.im);
70
71 Self {
72 direction,
73 _phantom: std::marker::PhantomData,
74 rotate,
75 twiddle1re,
76 twiddle1im,
77 twiddle2re,
78 twiddle2im,
79 twiddle3re,
80 twiddle3im,
81 }
82 }
83 #[inline(always)]
84 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
85 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6});
86
87 let out = self.perform_parallel_fft_direct(values);
88
89 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6});
90 }
91
92 #[inline(always)]
93 pub(crate) unsafe fn perform_parallel_fft_contiguous(
94 &self,
95 mut buffer: impl WasmSimdArrayMut<f32>,
96 ) {
97 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12});
98
99 let values = [
100 extract_lo_hi_f32(input_packed[0], input_packed[3]),
101 extract_hi_lo_f32(input_packed[0], input_packed[4]),
102 extract_lo_hi_f32(input_packed[1], input_packed[4]),
103 extract_hi_lo_f32(input_packed[1], input_packed[5]),
104 extract_lo_hi_f32(input_packed[2], input_packed[5]),
105 extract_hi_lo_f32(input_packed[2], input_packed[6]),
106 extract_lo_hi_f32(input_packed[3], input_packed[6]),
107 ];
108
109 let out = self.perform_parallel_fft_direct(values);
110
111 let out_packed = [
112 extract_lo_lo_f32(out[0], out[1]),
113 extract_lo_lo_f32(out[2], out[3]),
114 extract_lo_lo_f32(out[4], out[5]),
115 extract_lo_hi_f32(out[6], out[0]),
116 extract_hi_hi_f32(out[1], out[2]),
117 extract_hi_hi_f32(out[3], out[4]),
118 extract_hi_hi_f32(out[5], out[6]),
119 ];
120
121 write_complex_to_array_strided!(out_packed, buffer, 2, {0,1,2,3,4,5,6});
122 }
123
124 #[inline(always)]
125 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 7]) -> [v128; 7] {
126 let [x1p6, x1m6] = parallel_fft2_interleaved_f32(values[1], values[6]);
127 let [x2p5, x2m5] = parallel_fft2_interleaved_f32(values[2], values[5]);
128 let [x3p4, x3m4] = parallel_fft2_interleaved_f32(values[3], values[4]);
129
130 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p6);
131 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p5);
132 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p4);
133 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p6);
134 let t_a2_2 = f32x4_mul(self.twiddle3re, x2p5);
135 let t_a2_3 = f32x4_mul(self.twiddle1re, x3p4);
136 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p6);
137 let t_a3_2 = f32x4_mul(self.twiddle1re, x2p5);
138 let t_a3_3 = f32x4_mul(self.twiddle2re, x3p4);
139
140 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m6);
141 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m5);
142 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m4);
143 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m6);
144 let t_b2_2 = f32x4_mul(self.twiddle3im, x2m5);
145 let t_b2_3 = f32x4_mul(self.twiddle1im, x3m4);
146 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m6);
147 let t_b3_2 = f32x4_mul(self.twiddle1im, x2m5);
148 let t_b3_3 = f32x4_mul(self.twiddle2im, x3m4);
149
150 let x0 = values[0];
151 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3);
152 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3);
153 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3);
154
155 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3);
156 let t_b2 = calc_f32!(t_b2_1 - t_b2_2 - t_b2_3);
157 let t_b3 = calc_f32!(t_b3_1 - t_b3_2 + t_b3_3);
158
159 let t_b1_rot = self.rotate.rotate_both(t_b1);
160 let t_b2_rot = self.rotate.rotate_both(t_b2);
161 let t_b3_rot = self.rotate.rotate_both(t_b3);
162
163 let y0 = calc_f32!(x0 + x1p6 + x2p5 + x3p4);
164 let [y1, y6] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
165 let [y2, y5] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
166 let [y3, y4] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
167 [y0, y1, y2, y3, y4, y5, y6]
168 }
169 }
170
171 // _____ __ _ _ _ _ _
172 // |___ | / /_ | || | | |__ (_) |_
173 // / / _____ | '_ \| || |_| '_ \| | __|
174 // / / |_____| | (_) |__ _| |_) | | |_
175 // /_/ \___/ |_| |_.__/|_|\__|
176 //
177
178 pub struct WasmSimdF64Butterfly7<T> {
179 direction: FftDirection,
180 _phantom: std::marker::PhantomData<T>,
181 rotate: Rotate90F64,
182 twiddle1re: v128,
183 twiddle1im: v128,
184 twiddle2re: v128,
185 twiddle2im: v128,
186 twiddle3re: v128,
187 twiddle3im: v128,
188 }
189
190 boilerplate_fft_wasm_simd_f64_butterfly!(
191 WasmSimdF64Butterfly7,
192 7,
193 |this: &WasmSimdF64Butterfly7<_>| this.direction
194 );
195 boilerplate_fft_wasm_simd_common_butterfly!(
196 WasmSimdF64Butterfly7,
197 7,
198 |this: &WasmSimdF64Butterfly7<_>| { this.direction }
199 );
200 impl<T: FftNum> WasmSimdF64Butterfly7<T> {
201 #[inline(always)]
202 pub fn new(direction: FftDirection) -> Self {
203 assert_f64::<T>();
204 let rotate = Rotate90F64::new(true);
205 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 7, direction);
206 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 7, direction);
207 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 7, direction);
208 let twiddle1re = f64x2_splat(tw1.re);
209 let twiddle1im = f64x2_splat(tw1.im);
210 let twiddle2re = f64x2_splat(tw2.re);
211 let twiddle2im = f64x2_splat(tw2.im);
212 let twiddle3re = f64x2_splat(tw3.re);
213 let twiddle3im = f64x2_splat(tw3.im);
214
215 Self {
216 direction,
217 _phantom: std::marker::PhantomData,
218 rotate,
219 twiddle1re,
220 twiddle1im,
221 twiddle2re,
222 twiddle2im,
223 twiddle3re,
224 twiddle3im,
225 }
226 }
227
228 #[inline(always)]
229 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
230 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6});
231
232 let out = self.perform_fft_direct(values);
233
234 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6});
235 }
236
237 #[inline(always)]
238 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 7]) -> [v128; 7] {
239 let [x1p6, x1m6] = solo_fft2_f64(values[1], values[6]);
240 let [x2p5, x2m5] = solo_fft2_f64(values[2], values[5]);
241 let [x3p4, x3m4] = solo_fft2_f64(values[3], values[4]);
242
243 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p6);
244 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p5);
245 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p4);
246 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p6);
247 let t_a2_2 = f64x2_mul(self.twiddle3re, x2p5);
248 let t_a2_3 = f64x2_mul(self.twiddle1re, x3p4);
249 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p6);
250 let t_a3_2 = f64x2_mul(self.twiddle1re, x2p5);
251 let t_a3_3 = f64x2_mul(self.twiddle2re, x3p4);
252
253 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m6);
254 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m5);
255 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m4);
256 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m6);
257 let t_b2_2 = f64x2_mul(self.twiddle3im, x2m5);
258 let t_b2_3 = f64x2_mul(self.twiddle1im, x3m4);
259 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m6);
260 let t_b3_2 = f64x2_mul(self.twiddle1im, x2m5);
261 let t_b3_3 = f64x2_mul(self.twiddle2im, x3m4);
262
263 let x0 = values[0];
264 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3);
265 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3);
266 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3);
267
268 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3);
269 let t_b2 = calc_f64!(t_b2_1 - t_b2_2 - t_b2_3);
270 let t_b3 = calc_f64!(t_b3_1 - t_b3_2 + t_b3_3);
271
272 let t_b1_rot = self.rotate.rotate(t_b1);
273 let t_b2_rot = self.rotate.rotate(t_b2);
274 let t_b3_rot = self.rotate.rotate(t_b3);
275
276 let y0 = calc_f64!(x0 + x1p6 + x2p5 + x3p4);
277 let [y1, y6] = solo_fft2_f64(t_a1, t_b1_rot);
278 let [y2, y5] = solo_fft2_f64(t_a2, t_b2_rot);
279 let [y3, y4] = solo_fft2_f64(t_a3, t_b3_rot);
280 [y0, y1, y2, y3, y4, y5, y6]
281 }
282 }
283
284 // _ _ _________ _ _ _
285 // / / | |___ /___ \| |__ (_) |_
286 // | | | _____ |_ \ __) | '_ \| | __|
287 // | | | |_____| ___) / __/| |_) | | |_
288 // |_|_| |____/_____|_.__/|_|\__|
289 //
290
291 pub struct WasmSimdF32Butterfly11<T> {
292 direction: FftDirection,
293 _phantom: std::marker::PhantomData<T>,
294 rotate: Rotate90F32,
295 twiddle1re: v128,
296 twiddle1im: v128,
297 twiddle2re: v128,
298 twiddle2im: v128,
299 twiddle3re: v128,
300 twiddle3im: v128,
301 twiddle4re: v128,
302 twiddle4im: v128,
303 twiddle5re: v128,
304 twiddle5im: v128,
305 }
306
307 boilerplate_fft_wasm_simd_f32_butterfly!(
308 WasmSimdF32Butterfly11,
309 11,
310 |this: &WasmSimdF32Butterfly11<_>| this.direction
311 );
312 boilerplate_fft_wasm_simd_common_butterfly!(
313 WasmSimdF32Butterfly11,
314 11,
315 |this: &WasmSimdF32Butterfly11<_>| this.direction
316 );
317 impl<T: FftNum> WasmSimdF32Butterfly11<T> {
318 #[inline(always)]
319 pub fn new(direction: FftDirection) -> Self {
320 assert_f32::<T>();
321 let rotate = Rotate90F32::new(true);
322 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 11, direction);
323 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 11, direction);
324 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 11, direction);
325 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 11, direction);
326 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 11, direction);
327 let twiddle1re = f32x4_splat(tw1.re);
328 let twiddle1im = f32x4_splat(tw1.im);
329 let twiddle2re = f32x4_splat(tw2.re);
330 let twiddle2im = f32x4_splat(tw2.im);
331 let twiddle3re = f32x4_splat(tw3.re);
332 let twiddle3im = f32x4_splat(tw3.im);
333 let twiddle4re = f32x4_splat(tw4.re);
334 let twiddle4im = f32x4_splat(tw4.im);
335 let twiddle5re = f32x4_splat(tw5.re);
336 let twiddle5im = f32x4_splat(tw5.im);
337
338 Self {
339 direction,
340 _phantom: std::marker::PhantomData,
341 rotate,
342 twiddle1re,
343 twiddle1im,
344 twiddle2re,
345 twiddle2im,
346 twiddle3re,
347 twiddle3im,
348 twiddle4re,
349 twiddle4im,
350 twiddle5re,
351 twiddle5im,
352 }
353 }
354
355 #[inline(always)]
356 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
357 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
358
359 let out = self.perform_parallel_fft_direct(values);
360
361 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
362 }
363
364 #[inline(always)]
365 pub(crate) unsafe fn perform_parallel_fft_contiguous(
366 &self,
367 mut buffer: impl WasmSimdArrayMut<f32>,
368 ) {
369 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20});
370
371 let values = [
372 extract_lo_hi_f32(input_packed[0], input_packed[5]),
373 extract_hi_lo_f32(input_packed[0], input_packed[6]),
374 extract_lo_hi_f32(input_packed[1], input_packed[6]),
375 extract_hi_lo_f32(input_packed[1], input_packed[7]),
376 extract_lo_hi_f32(input_packed[2], input_packed[7]),
377 extract_hi_lo_f32(input_packed[2], input_packed[8]),
378 extract_lo_hi_f32(input_packed[3], input_packed[8]),
379 extract_hi_lo_f32(input_packed[3], input_packed[9]),
380 extract_lo_hi_f32(input_packed[4], input_packed[9]),
381 extract_hi_lo_f32(input_packed[4], input_packed[10]),
382 extract_lo_hi_f32(input_packed[5], input_packed[10]),
383 ];
384
385 let out = self.perform_parallel_fft_direct(values);
386
387 let out_packed = [
388 extract_lo_lo_f32(out[0], out[1]),
389 extract_lo_lo_f32(out[2], out[3]),
390 extract_lo_lo_f32(out[4], out[5]),
391 extract_lo_lo_f32(out[6], out[7]),
392 extract_lo_lo_f32(out[8], out[9]),
393 extract_lo_hi_f32(out[10], out[0]),
394 extract_hi_hi_f32(out[1], out[2]),
395 extract_hi_hi_f32(out[3], out[4]),
396 extract_hi_hi_f32(out[5], out[6]),
397 extract_hi_hi_f32(out[7], out[8]),
398 extract_hi_hi_f32(out[9], out[10]),
399 ];
400
401 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
402 }
403
404 #[inline(always)]
405 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 11]) -> [v128; 11] {
406 let [x1p10, x1m10] = parallel_fft2_interleaved_f32(values[1], values[10]);
407 let [x2p9, x2m9] = parallel_fft2_interleaved_f32(values[2], values[9]);
408 let [x3p8, x3m8] = parallel_fft2_interleaved_f32(values[3], values[8]);
409 let [x4p7, x4m7] = parallel_fft2_interleaved_f32(values[4], values[7]);
410 let [x5p6, x5m6] = parallel_fft2_interleaved_f32(values[5], values[6]);
411
412 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p10);
413 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p9);
414 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p8);
415 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p7);
416 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p6);
417 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p10);
418 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p9);
419 let t_a2_3 = f32x4_mul(self.twiddle5re, x3p8);
420 let t_a2_4 = f32x4_mul(self.twiddle3re, x4p7);
421 let t_a2_5 = f32x4_mul(self.twiddle1re, x5p6);
422 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p10);
423 let t_a3_2 = f32x4_mul(self.twiddle5re, x2p9);
424 let t_a3_3 = f32x4_mul(self.twiddle2re, x3p8);
425 let t_a3_4 = f32x4_mul(self.twiddle1re, x4p7);
426 let t_a3_5 = f32x4_mul(self.twiddle4re, x5p6);
427 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p10);
428 let t_a4_2 = f32x4_mul(self.twiddle3re, x2p9);
429 let t_a4_3 = f32x4_mul(self.twiddle1re, x3p8);
430 let t_a4_4 = f32x4_mul(self.twiddle5re, x4p7);
431 let t_a4_5 = f32x4_mul(self.twiddle2re, x5p6);
432 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p10);
433 let t_a5_2 = f32x4_mul(self.twiddle1re, x2p9);
434 let t_a5_3 = f32x4_mul(self.twiddle4re, x3p8);
435 let t_a5_4 = f32x4_mul(self.twiddle2re, x4p7);
436 let t_a5_5 = f32x4_mul(self.twiddle3re, x5p6);
437
438 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m10);
439 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m9);
440 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m8);
441 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m7);
442 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m6);
443 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m10);
444 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m9);
445 let t_b2_3 = f32x4_mul(self.twiddle5im, x3m8);
446 let t_b2_4 = f32x4_mul(self.twiddle3im, x4m7);
447 let t_b2_5 = f32x4_mul(self.twiddle1im, x5m6);
448 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m10);
449 let t_b3_2 = f32x4_mul(self.twiddle5im, x2m9);
450 let t_b3_3 = f32x4_mul(self.twiddle2im, x3m8);
451 let t_b3_4 = f32x4_mul(self.twiddle1im, x4m7);
452 let t_b3_5 = f32x4_mul(self.twiddle4im, x5m6);
453 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m10);
454 let t_b4_2 = f32x4_mul(self.twiddle3im, x2m9);
455 let t_b4_3 = f32x4_mul(self.twiddle1im, x3m8);
456 let t_b4_4 = f32x4_mul(self.twiddle5im, x4m7);
457 let t_b4_5 = f32x4_mul(self.twiddle2im, x5m6);
458 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m10);
459 let t_b5_2 = f32x4_mul(self.twiddle1im, x2m9);
460 let t_b5_3 = f32x4_mul(self.twiddle4im, x3m8);
461 let t_b5_4 = f32x4_mul(self.twiddle2im, x4m7);
462 let t_b5_5 = f32x4_mul(self.twiddle3im, x5m6);
463
464 let x0 = values[0];
465 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5);
466 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5);
467 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5);
468 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5);
469 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5);
470
471 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5);
472 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 - t_b2_3 - t_b2_4 - t_b2_5);
473 let t_b3 = calc_f32!(t_b3_1 - t_b3_2 - t_b3_3 + t_b3_4 + t_b3_5);
474 let t_b4 = calc_f32!(t_b4_1 - t_b4_2 + t_b4_3 + t_b4_4 - t_b4_5);
475 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 + t_b5_5);
476
477 let t_b1_rot = self.rotate.rotate_both(t_b1);
478 let t_b2_rot = self.rotate.rotate_both(t_b2);
479 let t_b3_rot = self.rotate.rotate_both(t_b3);
480 let t_b4_rot = self.rotate.rotate_both(t_b4);
481 let t_b5_rot = self.rotate.rotate_both(t_b5);
482
483 let y0 = calc_f32!(x0 + x1p10 + x2p9 + x3p8 + x4p7 + x5p6);
484 let [y1, y10] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
485 let [y2, y9] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
486 let [y3, y8] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
487 let [y4, y7] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
488 let [y5, y6] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
489 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10]
490 }
491 }
492
493 // _ _ __ _ _ _ _ _
494 // / / | / /_ | || | | |__ (_) |_
495 // | | | _____ | '_ \| || |_| '_ \| | __|
496 // | | | |_____| | (_) |__ _| |_) | | |_
497 // |_|_| \___/ |_| |_.__/|_|\__|
498 //
499
500 pub struct WasmSimdF64Butterfly11<T> {
501 direction: FftDirection,
502 _phantom: std::marker::PhantomData<T>,
503 rotate: Rotate90F64,
504 twiddle1re: v128,
505 twiddle1im: v128,
506 twiddle2re: v128,
507 twiddle2im: v128,
508 twiddle3re: v128,
509 twiddle3im: v128,
510 twiddle4re: v128,
511 twiddle4im: v128,
512 twiddle5re: v128,
513 twiddle5im: v128,
514 }
515
516 boilerplate_fft_wasm_simd_f64_butterfly!(
517 WasmSimdF64Butterfly11,
518 11,
519 |this: &WasmSimdF64Butterfly11<_>| this.direction
520 );
521 boilerplate_fft_wasm_simd_common_butterfly!(
522 WasmSimdF64Butterfly11,
523 11,
524 |this: &WasmSimdF64Butterfly11<_>| this.direction
525 );
526 impl<T: FftNum> WasmSimdF64Butterfly11<T> {
527 #[inline(always)]
528 pub fn new(direction: FftDirection) -> Self {
529 assert_f64::<T>();
530 let rotate = Rotate90F64::new(true);
531 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 11, direction);
532 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 11, direction);
533 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 11, direction);
534 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 11, direction);
535 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 11, direction);
536 let twiddle1re = f64x2_splat(tw1.re);
537 let twiddle1im = f64x2_splat(tw1.im);
538 let twiddle2re = f64x2_splat(tw2.re);
539 let twiddle2im = f64x2_splat(tw2.im);
540 let twiddle3re = f64x2_splat(tw3.re);
541 let twiddle3im = f64x2_splat(tw3.im);
542 let twiddle4re = f64x2_splat(tw4.re);
543 let twiddle4im = f64x2_splat(tw4.im);
544 let twiddle5re = f64x2_splat(tw5.re);
545 let twiddle5im = f64x2_splat(tw5.im);
546
547 Self {
548 direction,
549 _phantom: std::marker::PhantomData,
550 rotate,
551 twiddle1re,
552 twiddle1im,
553 twiddle2re,
554 twiddle2im,
555 twiddle3re,
556 twiddle3im,
557 twiddle4re,
558 twiddle4im,
559 twiddle5re,
560 twiddle5im,
561 }
562 }
563
564 #[inline(always)]
565 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
566 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
567
568 let out = self.perform_fft_direct(values);
569
570 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
571 }
572
573 #[inline(always)]
574 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 11]) -> [v128; 11] {
575 let [x1p10, x1m10] = solo_fft2_f64(values[1], values[10]);
576 let [x2p9, x2m9] = solo_fft2_f64(values[2], values[9]);
577 let [x3p8, x3m8] = solo_fft2_f64(values[3], values[8]);
578 let [x4p7, x4m7] = solo_fft2_f64(values[4], values[7]);
579 let [x5p6, x5m6] = solo_fft2_f64(values[5], values[6]);
580
581 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p10);
582 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p9);
583 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p8);
584 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p7);
585 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p6);
586 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p10);
587 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p9);
588 let t_a2_3 = f64x2_mul(self.twiddle5re, x3p8);
589 let t_a2_4 = f64x2_mul(self.twiddle3re, x4p7);
590 let t_a2_5 = f64x2_mul(self.twiddle1re, x5p6);
591 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p10);
592 let t_a3_2 = f64x2_mul(self.twiddle5re, x2p9);
593 let t_a3_3 = f64x2_mul(self.twiddle2re, x3p8);
594 let t_a3_4 = f64x2_mul(self.twiddle1re, x4p7);
595 let t_a3_5 = f64x2_mul(self.twiddle4re, x5p6);
596 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p10);
597 let t_a4_2 = f64x2_mul(self.twiddle3re, x2p9);
598 let t_a4_3 = f64x2_mul(self.twiddle1re, x3p8);
599 let t_a4_4 = f64x2_mul(self.twiddle5re, x4p7);
600 let t_a4_5 = f64x2_mul(self.twiddle2re, x5p6);
601 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p10);
602 let t_a5_2 = f64x2_mul(self.twiddle1re, x2p9);
603 let t_a5_3 = f64x2_mul(self.twiddle4re, x3p8);
604 let t_a5_4 = f64x2_mul(self.twiddle2re, x4p7);
605 let t_a5_5 = f64x2_mul(self.twiddle3re, x5p6);
606
607 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m10);
608 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m9);
609 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m8);
610 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m7);
611 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m6);
612 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m10);
613 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m9);
614 let t_b2_3 = f64x2_mul(self.twiddle5im, x3m8);
615 let t_b2_4 = f64x2_mul(self.twiddle3im, x4m7);
616 let t_b2_5 = f64x2_mul(self.twiddle1im, x5m6);
617 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m10);
618 let t_b3_2 = f64x2_mul(self.twiddle5im, x2m9);
619 let t_b3_3 = f64x2_mul(self.twiddle2im, x3m8);
620 let t_b3_4 = f64x2_mul(self.twiddle1im, x4m7);
621 let t_b3_5 = f64x2_mul(self.twiddle4im, x5m6);
622 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m10);
623 let t_b4_2 = f64x2_mul(self.twiddle3im, x2m9);
624 let t_b4_3 = f64x2_mul(self.twiddle1im, x3m8);
625 let t_b4_4 = f64x2_mul(self.twiddle5im, x4m7);
626 let t_b4_5 = f64x2_mul(self.twiddle2im, x5m6);
627 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m10);
628 let t_b5_2 = f64x2_mul(self.twiddle1im, x2m9);
629 let t_b5_3 = f64x2_mul(self.twiddle4im, x3m8);
630 let t_b5_4 = f64x2_mul(self.twiddle2im, x4m7);
631 let t_b5_5 = f64x2_mul(self.twiddle3im, x5m6);
632
633 let x0 = values[0];
634 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5);
635 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5);
636 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5);
637 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5);
638 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5);
639
640 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5);
641 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 - t_b2_3 - t_b2_4 - t_b2_5);
642 let t_b3 = calc_f64!(t_b3_1 - t_b3_2 - t_b3_3 + t_b3_4 + t_b3_5);
643 let t_b4 = calc_f64!(t_b4_1 - t_b4_2 + t_b4_3 + t_b4_4 - t_b4_5);
644 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 + t_b5_5);
645
646 let t_b1_rot = self.rotate.rotate(t_b1);
647 let t_b2_rot = self.rotate.rotate(t_b2);
648 let t_b3_rot = self.rotate.rotate(t_b3);
649 let t_b4_rot = self.rotate.rotate(t_b4);
650 let t_b5_rot = self.rotate.rotate(t_b5);
651
652 let y0 = calc_f64!(x0 + x1p10 + x2p9 + x3p8 + x4p7 + x5p6);
653 let [y1, y10] = solo_fft2_f64(t_a1, t_b1_rot);
654 let [y2, y9] = solo_fft2_f64(t_a2, t_b2_rot);
655 let [y3, y8] = solo_fft2_f64(t_a3, t_b3_rot);
656 let [y4, y7] = solo_fft2_f64(t_a4, t_b4_rot);
657 let [y5, y6] = solo_fft2_f64(t_a5, t_b5_rot);
658 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10]
659 }
660 }
661
662 // _ _____ _________ _ _ _
663 // / |___ / |___ /___ \| |__ (_) |_
664 // | | |_ \ _____ |_ \ __) | '_ \| | __|
665 // | |___) | |_____| ___) / __/| |_) | | |_
666 // |_|____/ |____/_____|_.__/|_|\__|
667 //
668
669 pub struct WasmSimdF32Butterfly13<T> {
670 direction: FftDirection,
671 _phantom: std::marker::PhantomData<T>,
672 rotate: Rotate90F32,
673 twiddle1re: v128,
674 twiddle1im: v128,
675 twiddle2re: v128,
676 twiddle2im: v128,
677 twiddle3re: v128,
678 twiddle3im: v128,
679 twiddle4re: v128,
680 twiddle4im: v128,
681 twiddle5re: v128,
682 twiddle5im: v128,
683 twiddle6re: v128,
684 twiddle6im: v128,
685 }
686
687 boilerplate_fft_wasm_simd_f32_butterfly!(
688 WasmSimdF32Butterfly13,
689 13,
690 |this: &WasmSimdF32Butterfly13<_>| this.direction
691 );
692 boilerplate_fft_wasm_simd_common_butterfly!(
693 WasmSimdF32Butterfly13,
694 13,
695 |this: &WasmSimdF32Butterfly13<_>| this.direction
696 );
697 impl<T: FftNum> WasmSimdF32Butterfly13<T> {
698 #[inline(always)]
699 pub fn new(direction: FftDirection) -> Self {
700 assert_f32::<T>();
701 let rotate = Rotate90F32::new(true);
702 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 13, direction);
703 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 13, direction);
704 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 13, direction);
705 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 13, direction);
706 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 13, direction);
707 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 13, direction);
708 let twiddle1re = f32x4_splat(tw1.re);
709 let twiddle1im = f32x4_splat(tw1.im);
710 let twiddle2re = f32x4_splat(tw2.re);
711 let twiddle2im = f32x4_splat(tw2.im);
712 let twiddle3re = f32x4_splat(tw3.re);
713 let twiddle3im = f32x4_splat(tw3.im);
714 let twiddle4re = f32x4_splat(tw4.re);
715 let twiddle4im = f32x4_splat(tw4.im);
716 let twiddle5re = f32x4_splat(tw5.re);
717 let twiddle5im = f32x4_splat(tw5.im);
718 let twiddle6re = f32x4_splat(tw6.re);
719 let twiddle6im = f32x4_splat(tw6.im);
720
721 Self {
722 direction,
723 _phantom: std::marker::PhantomData,
724 rotate,
725 twiddle1re,
726 twiddle1im,
727 twiddle2re,
728 twiddle2im,
729 twiddle3re,
730 twiddle3im,
731 twiddle4re,
732 twiddle4im,
733 twiddle5re,
734 twiddle5im,
735 twiddle6re,
736 twiddle6im,
737 }
738 }
739
740 #[inline(always)]
741 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
742 let values =
743 read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
744
745 let out = self.perform_parallel_fft_direct(values);
746
747 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
748 }
749
750 #[inline(always)]
751 pub(crate) unsafe fn perform_parallel_fft_contiguous(
752 &self,
753 mut buffer: impl WasmSimdArrayMut<f32>,
754 ) {
755 let input_packed =
756 read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24});
757
758 let values = [
759 extract_lo_hi_f32(input_packed[0], input_packed[6]),
760 extract_hi_lo_f32(input_packed[0], input_packed[7]),
761 extract_lo_hi_f32(input_packed[1], input_packed[7]),
762 extract_hi_lo_f32(input_packed[1], input_packed[8]),
763 extract_lo_hi_f32(input_packed[2], input_packed[8]),
764 extract_hi_lo_f32(input_packed[2], input_packed[9]),
765 extract_lo_hi_f32(input_packed[3], input_packed[9]),
766 extract_hi_lo_f32(input_packed[3], input_packed[10]),
767 extract_lo_hi_f32(input_packed[4], input_packed[10]),
768 extract_hi_lo_f32(input_packed[4], input_packed[11]),
769 extract_lo_hi_f32(input_packed[5], input_packed[11]),
770 extract_hi_lo_f32(input_packed[5], input_packed[12]),
771 extract_lo_hi_f32(input_packed[6], input_packed[12]),
772 ];
773
774 let out = self.perform_parallel_fft_direct(values);
775
776 let out_packed = [
777 extract_lo_lo_f32(out[0], out[1]),
778 extract_lo_lo_f32(out[2], out[3]),
779 extract_lo_lo_f32(out[4], out[5]),
780 extract_lo_lo_f32(out[6], out[7]),
781 extract_lo_lo_f32(out[8], out[9]),
782 extract_lo_lo_f32(out[10], out[11]),
783 extract_lo_hi_f32(out[12], out[0]),
784 extract_hi_hi_f32(out[1], out[2]),
785 extract_hi_hi_f32(out[3], out[4]),
786 extract_hi_hi_f32(out[5], out[6]),
787 extract_hi_hi_f32(out[7], out[8]),
788 extract_hi_hi_f32(out[9], out[10]),
789 extract_hi_hi_f32(out[11], out[12]),
790 ];
791
792 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
793 }
794
795 #[inline(always)]
796 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 13]) -> [v128; 13] {
797 let [x1p12, x1m12] = parallel_fft2_interleaved_f32(values[1], values[12]);
798 let [x2p11, x2m11] = parallel_fft2_interleaved_f32(values[2], values[11]);
799 let [x3p10, x3m10] = parallel_fft2_interleaved_f32(values[3], values[10]);
800 let [x4p9, x4m9] = parallel_fft2_interleaved_f32(values[4], values[9]);
801 let [x5p8, x5m8] = parallel_fft2_interleaved_f32(values[5], values[8]);
802 let [x6p7, x6m7] = parallel_fft2_interleaved_f32(values[6], values[7]);
803
804 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p12);
805 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p11);
806 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p10);
807 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p9);
808 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p8);
809 let t_a1_6 = f32x4_mul(self.twiddle6re, x6p7);
810 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p12);
811 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p11);
812 let t_a2_3 = f32x4_mul(self.twiddle6re, x3p10);
813 let t_a2_4 = f32x4_mul(self.twiddle5re, x4p9);
814 let t_a2_5 = f32x4_mul(self.twiddle3re, x5p8);
815 let t_a2_6 = f32x4_mul(self.twiddle1re, x6p7);
816 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p12);
817 let t_a3_2 = f32x4_mul(self.twiddle6re, x2p11);
818 let t_a3_3 = f32x4_mul(self.twiddle4re, x3p10);
819 let t_a3_4 = f32x4_mul(self.twiddle1re, x4p9);
820 let t_a3_5 = f32x4_mul(self.twiddle2re, x5p8);
821 let t_a3_6 = f32x4_mul(self.twiddle5re, x6p7);
822 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p12);
823 let t_a4_2 = f32x4_mul(self.twiddle5re, x2p11);
824 let t_a4_3 = f32x4_mul(self.twiddle1re, x3p10);
825 let t_a4_4 = f32x4_mul(self.twiddle3re, x4p9);
826 let t_a4_5 = f32x4_mul(self.twiddle6re, x5p8);
827 let t_a4_6 = f32x4_mul(self.twiddle2re, x6p7);
828 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p12);
829 let t_a5_2 = f32x4_mul(self.twiddle3re, x2p11);
830 let t_a5_3 = f32x4_mul(self.twiddle2re, x3p10);
831 let t_a5_4 = f32x4_mul(self.twiddle6re, x4p9);
832 let t_a5_5 = f32x4_mul(self.twiddle1re, x5p8);
833 let t_a5_6 = f32x4_mul(self.twiddle4re, x6p7);
834 let t_a6_1 = f32x4_mul(self.twiddle6re, x1p12);
835 let t_a6_2 = f32x4_mul(self.twiddle1re, x2p11);
836 let t_a6_3 = f32x4_mul(self.twiddle5re, x3p10);
837 let t_a6_4 = f32x4_mul(self.twiddle2re, x4p9);
838 let t_a6_5 = f32x4_mul(self.twiddle4re, x5p8);
839 let t_a6_6 = f32x4_mul(self.twiddle3re, x6p7);
840
841 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m12);
842 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m11);
843 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m10);
844 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m9);
845 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m8);
846 let t_b1_6 = f32x4_mul(self.twiddle6im, x6m7);
847 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m12);
848 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m11);
849 let t_b2_3 = f32x4_mul(self.twiddle6im, x3m10);
850 let t_b2_4 = f32x4_mul(self.twiddle5im, x4m9);
851 let t_b2_5 = f32x4_mul(self.twiddle3im, x5m8);
852 let t_b2_6 = f32x4_mul(self.twiddle1im, x6m7);
853 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m12);
854 let t_b3_2 = f32x4_mul(self.twiddle6im, x2m11);
855 let t_b3_3 = f32x4_mul(self.twiddle4im, x3m10);
856 let t_b3_4 = f32x4_mul(self.twiddle1im, x4m9);
857 let t_b3_5 = f32x4_mul(self.twiddle2im, x5m8);
858 let t_b3_6 = f32x4_mul(self.twiddle5im, x6m7);
859 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m12);
860 let t_b4_2 = f32x4_mul(self.twiddle5im, x2m11);
861 let t_b4_3 = f32x4_mul(self.twiddle1im, x3m10);
862 let t_b4_4 = f32x4_mul(self.twiddle3im, x4m9);
863 let t_b4_5 = f32x4_mul(self.twiddle6im, x5m8);
864 let t_b4_6 = f32x4_mul(self.twiddle2im, x6m7);
865 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m12);
866 let t_b5_2 = f32x4_mul(self.twiddle3im, x2m11);
867 let t_b5_3 = f32x4_mul(self.twiddle2im, x3m10);
868 let t_b5_4 = f32x4_mul(self.twiddle6im, x4m9);
869 let t_b5_5 = f32x4_mul(self.twiddle1im, x5m8);
870 let t_b5_6 = f32x4_mul(self.twiddle4im, x6m7);
871 let t_b6_1 = f32x4_mul(self.twiddle6im, x1m12);
872 let t_b6_2 = f32x4_mul(self.twiddle1im, x2m11);
873 let t_b6_3 = f32x4_mul(self.twiddle5im, x3m10);
874 let t_b6_4 = f32x4_mul(self.twiddle2im, x4m9);
875 let t_b6_5 = f32x4_mul(self.twiddle4im, x5m8);
876 let t_b6_6 = f32x4_mul(self.twiddle3im, x6m7);
877
878 let x0 = values[0];
879 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6);
880 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6);
881 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6);
882 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6);
883 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6);
884 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6);
885
886 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6);
887 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 - t_b2_4 - t_b2_5 - t_b2_6);
888 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 + t_b3_5 + t_b3_6);
889 let t_b4 = calc_f32!(t_b4_1 - t_b4_2 - t_b4_3 + t_b4_4 - t_b4_5 - t_b4_6);
890 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6);
891 let t_b6 = calc_f32!(t_b6_1 - t_b6_2 + t_b6_3 - t_b6_4 + t_b6_5 - t_b6_6);
892
893 let t_b1_rot = self.rotate.rotate_both(t_b1);
894 let t_b2_rot = self.rotate.rotate_both(t_b2);
895 let t_b3_rot = self.rotate.rotate_both(t_b3);
896 let t_b4_rot = self.rotate.rotate_both(t_b4);
897 let t_b5_rot = self.rotate.rotate_both(t_b5);
898 let t_b6_rot = self.rotate.rotate_both(t_b6);
899
900 let y0 = calc_f32!(x0 + x1p12 + x2p11 + x3p10 + x4p9 + x5p8 + x6p7);
901 let [y1, y12] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
902 let [y2, y11] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
903 let [y3, y10] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
904 let [y4, y9] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
905 let [y5, y8] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
906 let [y6, y7] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
907 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12]
908 }
909 }
910
911 // _ _____ __ _ _ _ _ _
912 // / |___ / / /_ | || | | |__ (_) |_
913 // | | |_ \ _____ | '_ \| || |_| '_ \| | __|
914 // | |___) | |_____| | (_) |__ _| |_) | | |_
915 // |_|____/ \___/ |_| |_.__/|_|\__|
916 //
917
918 pub struct WasmSimdF64Butterfly13<T> {
919 direction: FftDirection,
920 _phantom: std::marker::PhantomData<T>,
921 rotate: Rotate90F64,
922 twiddle1re: v128,
923 twiddle1im: v128,
924 twiddle2re: v128,
925 twiddle2im: v128,
926 twiddle3re: v128,
927 twiddle3im: v128,
928 twiddle4re: v128,
929 twiddle4im: v128,
930 twiddle5re: v128,
931 twiddle5im: v128,
932 twiddle6re: v128,
933 twiddle6im: v128,
934 }
935
936 boilerplate_fft_wasm_simd_f64_butterfly!(
937 WasmSimdF64Butterfly13,
938 13,
939 |this: &WasmSimdF64Butterfly13<_>| this.direction
940 );
941 boilerplate_fft_wasm_simd_common_butterfly!(
942 WasmSimdF64Butterfly13,
943 13,
944 |this: &WasmSimdF64Butterfly13<_>| this.direction
945 );
946 impl<T: FftNum> WasmSimdF64Butterfly13<T> {
947 #[inline(always)]
948 pub fn new(direction: FftDirection) -> Self {
949 assert_f64::<T>();
950 let rotate = Rotate90F64::new(true);
951 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 13, direction);
952 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 13, direction);
953 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 13, direction);
954 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 13, direction);
955 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 13, direction);
956 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 13, direction);
957 let twiddle1re = f64x2_splat(tw1.re);
958 let twiddle1im = f64x2_splat(tw1.im);
959 let twiddle2re = f64x2_splat(tw2.re);
960 let twiddle2im = f64x2_splat(tw2.im);
961 let twiddle3re = f64x2_splat(tw3.re);
962 let twiddle3im = f64x2_splat(tw3.im);
963 let twiddle4re = f64x2_splat(tw4.re);
964 let twiddle4im = f64x2_splat(tw4.im);
965 let twiddle5re = f64x2_splat(tw5.re);
966 let twiddle5im = f64x2_splat(tw5.im);
967 let twiddle6re = f64x2_splat(tw6.re);
968 let twiddle6im = f64x2_splat(tw6.im);
969
970 Self {
971 direction,
972 _phantom: std::marker::PhantomData,
973 rotate,
974 twiddle1re,
975 twiddle1im,
976 twiddle2re,
977 twiddle2im,
978 twiddle3re,
979 twiddle3im,
980 twiddle4re,
981 twiddle4im,
982 twiddle5re,
983 twiddle5im,
984 twiddle6re,
985 twiddle6im,
986 }
987 }
988
989 #[inline(always)]
990 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
991 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
992
993 let out = self.perform_fft_direct(values);
994
995 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
996 }
997
998 #[inline(always)]
999 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 13]) -> [v128; 13] {
1000 let [x1p12, x1m12] = solo_fft2_f64(values[1], values[12]);
1001 let [x2p11, x2m11] = solo_fft2_f64(values[2], values[11]);
1002 let [x3p10, x3m10] = solo_fft2_f64(values[3], values[10]);
1003 let [x4p9, x4m9] = solo_fft2_f64(values[4], values[9]);
1004 let [x5p8, x5m8] = solo_fft2_f64(values[5], values[8]);
1005 let [x6p7, x6m7] = solo_fft2_f64(values[6], values[7]);
1006
1007 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p12);
1008 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p11);
1009 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p10);
1010 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p9);
1011 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p8);
1012 let t_a1_6 = f64x2_mul(self.twiddle6re, x6p7);
1013 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p12);
1014 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p11);
1015 let t_a2_3 = f64x2_mul(self.twiddle6re, x3p10);
1016 let t_a2_4 = f64x2_mul(self.twiddle5re, x4p9);
1017 let t_a2_5 = f64x2_mul(self.twiddle3re, x5p8);
1018 let t_a2_6 = f64x2_mul(self.twiddle1re, x6p7);
1019 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p12);
1020 let t_a3_2 = f64x2_mul(self.twiddle6re, x2p11);
1021 let t_a3_3 = f64x2_mul(self.twiddle4re, x3p10);
1022 let t_a3_4 = f64x2_mul(self.twiddle1re, x4p9);
1023 let t_a3_5 = f64x2_mul(self.twiddle2re, x5p8);
1024 let t_a3_6 = f64x2_mul(self.twiddle5re, x6p7);
1025 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p12);
1026 let t_a4_2 = f64x2_mul(self.twiddle5re, x2p11);
1027 let t_a4_3 = f64x2_mul(self.twiddle1re, x3p10);
1028 let t_a4_4 = f64x2_mul(self.twiddle3re, x4p9);
1029 let t_a4_5 = f64x2_mul(self.twiddle6re, x5p8);
1030 let t_a4_6 = f64x2_mul(self.twiddle2re, x6p7);
1031 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p12);
1032 let t_a5_2 = f64x2_mul(self.twiddle3re, x2p11);
1033 let t_a5_3 = f64x2_mul(self.twiddle2re, x3p10);
1034 let t_a5_4 = f64x2_mul(self.twiddle6re, x4p9);
1035 let t_a5_5 = f64x2_mul(self.twiddle1re, x5p8);
1036 let t_a5_6 = f64x2_mul(self.twiddle4re, x6p7);
1037 let t_a6_1 = f64x2_mul(self.twiddle6re, x1p12);
1038 let t_a6_2 = f64x2_mul(self.twiddle1re, x2p11);
1039 let t_a6_3 = f64x2_mul(self.twiddle5re, x3p10);
1040 let t_a6_4 = f64x2_mul(self.twiddle2re, x4p9);
1041 let t_a6_5 = f64x2_mul(self.twiddle4re, x5p8);
1042 let t_a6_6 = f64x2_mul(self.twiddle3re, x6p7);
1043
1044 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m12);
1045 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m11);
1046 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m10);
1047 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m9);
1048 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m8);
1049 let t_b1_6 = f64x2_mul(self.twiddle6im, x6m7);
1050 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m12);
1051 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m11);
1052 let t_b2_3 = f64x2_mul(self.twiddle6im, x3m10);
1053 let t_b2_4 = f64x2_mul(self.twiddle5im, x4m9);
1054 let t_b2_5 = f64x2_mul(self.twiddle3im, x5m8);
1055 let t_b2_6 = f64x2_mul(self.twiddle1im, x6m7);
1056 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m12);
1057 let t_b3_2 = f64x2_mul(self.twiddle6im, x2m11);
1058 let t_b3_3 = f64x2_mul(self.twiddle4im, x3m10);
1059 let t_b3_4 = f64x2_mul(self.twiddle1im, x4m9);
1060 let t_b3_5 = f64x2_mul(self.twiddle2im, x5m8);
1061 let t_b3_6 = f64x2_mul(self.twiddle5im, x6m7);
1062 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m12);
1063 let t_b4_2 = f64x2_mul(self.twiddle5im, x2m11);
1064 let t_b4_3 = f64x2_mul(self.twiddle1im, x3m10);
1065 let t_b4_4 = f64x2_mul(self.twiddle3im, x4m9);
1066 let t_b4_5 = f64x2_mul(self.twiddle6im, x5m8);
1067 let t_b4_6 = f64x2_mul(self.twiddle2im, x6m7);
1068 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m12);
1069 let t_b5_2 = f64x2_mul(self.twiddle3im, x2m11);
1070 let t_b5_3 = f64x2_mul(self.twiddle2im, x3m10);
1071 let t_b5_4 = f64x2_mul(self.twiddle6im, x4m9);
1072 let t_b5_5 = f64x2_mul(self.twiddle1im, x5m8);
1073 let t_b5_6 = f64x2_mul(self.twiddle4im, x6m7);
1074 let t_b6_1 = f64x2_mul(self.twiddle6im, x1m12);
1075 let t_b6_2 = f64x2_mul(self.twiddle1im, x2m11);
1076 let t_b6_3 = f64x2_mul(self.twiddle5im, x3m10);
1077 let t_b6_4 = f64x2_mul(self.twiddle2im, x4m9);
1078 let t_b6_5 = f64x2_mul(self.twiddle4im, x5m8);
1079 let t_b6_6 = f64x2_mul(self.twiddle3im, x6m7);
1080
1081 let x0 = values[0];
1082 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6);
1083 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6);
1084 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6);
1085 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6);
1086 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6);
1087 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6);
1088
1089 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6);
1090 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 - t_b2_4 - t_b2_5 - t_b2_6);
1091 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 + t_b3_5 + t_b3_6);
1092 let t_b4 = calc_f64!(t_b4_1 - t_b4_2 - t_b4_3 + t_b4_4 - t_b4_5 - t_b4_6);
1093 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6);
1094 let t_b6 = calc_f64!(t_b6_1 - t_b6_2 + t_b6_3 - t_b6_4 + t_b6_5 - t_b6_6);
1095
1096 let t_b1_rot = self.rotate.rotate(t_b1);
1097 let t_b2_rot = self.rotate.rotate(t_b2);
1098 let t_b3_rot = self.rotate.rotate(t_b3);
1099 let t_b4_rot = self.rotate.rotate(t_b4);
1100 let t_b5_rot = self.rotate.rotate(t_b5);
1101 let t_b6_rot = self.rotate.rotate(t_b6);
1102
1103 let y0 = calc_f64!(x0 + x1p12 + x2p11 + x3p10 + x4p9 + x5p8 + x6p7);
1104 let [y1, y12] = solo_fft2_f64(t_a1, t_b1_rot);
1105 let [y2, y11] = solo_fft2_f64(t_a2, t_b2_rot);
1106 let [y3, y10] = solo_fft2_f64(t_a3, t_b3_rot);
1107 let [y4, y9] = solo_fft2_f64(t_a4, t_b4_rot);
1108 let [y5, y8] = solo_fft2_f64(t_a5, t_b5_rot);
1109 let [y6, y7] = solo_fft2_f64(t_a6, t_b6_rot);
1110 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12]
1111 }
1112 }
1113
1114 // _ _____ _________ _ _ _
1115 // / |___ | |___ /___ \| |__ (_) |_
1116 // | | / / _____ |_ \ __) | '_ \| | __|
1117 // | | / / |_____| ___) / __/| |_) | | |_
1118 // |_|/_/ |____/_____|_.__/|_|\__|
1119 //
1120
1121 pub struct WasmSimdF32Butterfly17<T> {
1122 direction: FftDirection,
1123 _phantom: std::marker::PhantomData<T>,
1124 rotate: Rotate90F32,
1125 twiddle1re: v128,
1126 twiddle1im: v128,
1127 twiddle2re: v128,
1128 twiddle2im: v128,
1129 twiddle3re: v128,
1130 twiddle3im: v128,
1131 twiddle4re: v128,
1132 twiddle4im: v128,
1133 twiddle5re: v128,
1134 twiddle5im: v128,
1135 twiddle6re: v128,
1136 twiddle6im: v128,
1137 twiddle7re: v128,
1138 twiddle7im: v128,
1139 twiddle8re: v128,
1140 twiddle8im: v128,
1141 }
1142
1143 boilerplate_fft_wasm_simd_f32_butterfly!(
1144 WasmSimdF32Butterfly17,
1145 17,
1146 |this: &WasmSimdF32Butterfly17<_>| this.direction
1147 );
1148 boilerplate_fft_wasm_simd_common_butterfly!(
1149 WasmSimdF32Butterfly17,
1150 17,
1151 |this: &WasmSimdF32Butterfly17<_>| this.direction
1152 );
1153 impl<T: FftNum> WasmSimdF32Butterfly17<T> {
1154 #[inline(always)]
1155 pub fn new(direction: FftDirection) -> Self {
1156 assert_f32::<T>();
1157 let rotate = Rotate90F32::new(true);
1158 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 17, direction);
1159 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 17, direction);
1160 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 17, direction);
1161 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 17, direction);
1162 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 17, direction);
1163 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 17, direction);
1164 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 17, direction);
1165 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 17, direction);
1166 let twiddle1re = f32x4_splat(tw1.re);
1167 let twiddle1im = f32x4_splat(tw1.im);
1168 let twiddle2re = f32x4_splat(tw2.re);
1169 let twiddle2im = f32x4_splat(tw2.im);
1170 let twiddle3re = f32x4_splat(tw3.re);
1171 let twiddle3im = f32x4_splat(tw3.im);
1172 let twiddle4re = f32x4_splat(tw4.re);
1173 let twiddle4im = f32x4_splat(tw4.im);
1174 let twiddle5re = f32x4_splat(tw5.re);
1175 let twiddle5im = f32x4_splat(tw5.im);
1176 let twiddle6re = f32x4_splat(tw6.re);
1177 let twiddle6im = f32x4_splat(tw6.im);
1178 let twiddle7re = f32x4_splat(tw7.re);
1179 let twiddle7im = f32x4_splat(tw7.im);
1180 let twiddle8re = f32x4_splat(tw8.re);
1181 let twiddle8im = f32x4_splat(tw8.im);
1182
1183 Self {
1184 direction,
1185 _phantom: std::marker::PhantomData,
1186 rotate,
1187 twiddle1re,
1188 twiddle1im,
1189 twiddle2re,
1190 twiddle2im,
1191 twiddle3re,
1192 twiddle3im,
1193 twiddle4re,
1194 twiddle4im,
1195 twiddle5re,
1196 twiddle5im,
1197 twiddle6re,
1198 twiddle6im,
1199 twiddle7re,
1200 twiddle7im,
1201 twiddle8re,
1202 twiddle8im,
1203 }
1204 }
1205
1206 #[inline(always)]
1207 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
1208 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1209
1210 let out = self.perform_parallel_fft_direct(values);
1211
1212 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1213 }
1214
1215 #[inline(always)]
1216 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1217 &self,
1218 mut buffer: impl WasmSimdArrayMut<f32>,
1219 ) {
1220 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32});
1221
1222 let values = [
1223 extract_lo_hi_f32(input_packed[0], input_packed[8]),
1224 extract_hi_lo_f32(input_packed[0], input_packed[9]),
1225 extract_lo_hi_f32(input_packed[1], input_packed[9]),
1226 extract_hi_lo_f32(input_packed[1], input_packed[10]),
1227 extract_lo_hi_f32(input_packed[2], input_packed[10]),
1228 extract_hi_lo_f32(input_packed[2], input_packed[11]),
1229 extract_lo_hi_f32(input_packed[3], input_packed[11]),
1230 extract_hi_lo_f32(input_packed[3], input_packed[12]),
1231 extract_lo_hi_f32(input_packed[4], input_packed[12]),
1232 extract_hi_lo_f32(input_packed[4], input_packed[13]),
1233 extract_lo_hi_f32(input_packed[5], input_packed[13]),
1234 extract_hi_lo_f32(input_packed[5], input_packed[14]),
1235 extract_lo_hi_f32(input_packed[6], input_packed[14]),
1236 extract_hi_lo_f32(input_packed[6], input_packed[15]),
1237 extract_lo_hi_f32(input_packed[7], input_packed[15]),
1238 extract_hi_lo_f32(input_packed[7], input_packed[16]),
1239 extract_lo_hi_f32(input_packed[8], input_packed[16]),
1240 ];
1241
1242 let out = self.perform_parallel_fft_direct(values);
1243
1244 let out_packed = [
1245 extract_lo_lo_f32(out[0], out[1]),
1246 extract_lo_lo_f32(out[2], out[3]),
1247 extract_lo_lo_f32(out[4], out[5]),
1248 extract_lo_lo_f32(out[6], out[7]),
1249 extract_lo_lo_f32(out[8], out[9]),
1250 extract_lo_lo_f32(out[10], out[11]),
1251 extract_lo_lo_f32(out[12], out[13]),
1252 extract_lo_lo_f32(out[14], out[15]),
1253 extract_lo_hi_f32(out[16], out[0]),
1254 extract_hi_hi_f32(out[1], out[2]),
1255 extract_hi_hi_f32(out[3], out[4]),
1256 extract_hi_hi_f32(out[5], out[6]),
1257 extract_hi_hi_f32(out[7], out[8]),
1258 extract_hi_hi_f32(out[9], out[10]),
1259 extract_hi_hi_f32(out[11], out[12]),
1260 extract_hi_hi_f32(out[13], out[14]),
1261 extract_hi_hi_f32(out[15], out[16]),
1262 ];
1263
1264 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1265 }
1266
1267 #[inline(always)]
1268 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 17]) -> [v128; 17] {
1269 let [x1p16, x1m16] = parallel_fft2_interleaved_f32(values[1], values[16]);
1270 let [x2p15, x2m15] = parallel_fft2_interleaved_f32(values[2], values[15]);
1271 let [x3p14, x3m14] = parallel_fft2_interleaved_f32(values[3], values[14]);
1272 let [x4p13, x4m13] = parallel_fft2_interleaved_f32(values[4], values[13]);
1273 let [x5p12, x5m12] = parallel_fft2_interleaved_f32(values[5], values[12]);
1274 let [x6p11, x6m11] = parallel_fft2_interleaved_f32(values[6], values[11]);
1275 let [x7p10, x7m10] = parallel_fft2_interleaved_f32(values[7], values[10]);
1276 let [x8p9, x8m9] = parallel_fft2_interleaved_f32(values[8], values[9]);
1277
1278 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p16);
1279 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p15);
1280 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p14);
1281 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p13);
1282 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p12);
1283 let t_a1_6 = f32x4_mul(self.twiddle6re, x6p11);
1284 let t_a1_7 = f32x4_mul(self.twiddle7re, x7p10);
1285 let t_a1_8 = f32x4_mul(self.twiddle8re, x8p9);
1286 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p16);
1287 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p15);
1288 let t_a2_3 = f32x4_mul(self.twiddle6re, x3p14);
1289 let t_a2_4 = f32x4_mul(self.twiddle8re, x4p13);
1290 let t_a2_5 = f32x4_mul(self.twiddle7re, x5p12);
1291 let t_a2_6 = f32x4_mul(self.twiddle5re, x6p11);
1292 let t_a2_7 = f32x4_mul(self.twiddle3re, x7p10);
1293 let t_a2_8 = f32x4_mul(self.twiddle1re, x8p9);
1294 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p16);
1295 let t_a3_2 = f32x4_mul(self.twiddle6re, x2p15);
1296 let t_a3_3 = f32x4_mul(self.twiddle8re, x3p14);
1297 let t_a3_4 = f32x4_mul(self.twiddle5re, x4p13);
1298 let t_a3_5 = f32x4_mul(self.twiddle2re, x5p12);
1299 let t_a3_6 = f32x4_mul(self.twiddle1re, x6p11);
1300 let t_a3_7 = f32x4_mul(self.twiddle4re, x7p10);
1301 let t_a3_8 = f32x4_mul(self.twiddle7re, x8p9);
1302 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p16);
1303 let t_a4_2 = f32x4_mul(self.twiddle8re, x2p15);
1304 let t_a4_3 = f32x4_mul(self.twiddle5re, x3p14);
1305 let t_a4_4 = f32x4_mul(self.twiddle1re, x4p13);
1306 let t_a4_5 = f32x4_mul(self.twiddle3re, x5p12);
1307 let t_a4_6 = f32x4_mul(self.twiddle7re, x6p11);
1308 let t_a4_7 = f32x4_mul(self.twiddle6re, x7p10);
1309 let t_a4_8 = f32x4_mul(self.twiddle2re, x8p9);
1310 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p16);
1311 let t_a5_2 = f32x4_mul(self.twiddle7re, x2p15);
1312 let t_a5_3 = f32x4_mul(self.twiddle2re, x3p14);
1313 let t_a5_4 = f32x4_mul(self.twiddle3re, x4p13);
1314 let t_a5_5 = f32x4_mul(self.twiddle8re, x5p12);
1315 let t_a5_6 = f32x4_mul(self.twiddle4re, x6p11);
1316 let t_a5_7 = f32x4_mul(self.twiddle1re, x7p10);
1317 let t_a5_8 = f32x4_mul(self.twiddle6re, x8p9);
1318 let t_a6_1 = f32x4_mul(self.twiddle6re, x1p16);
1319 let t_a6_2 = f32x4_mul(self.twiddle5re, x2p15);
1320 let t_a6_3 = f32x4_mul(self.twiddle1re, x3p14);
1321 let t_a6_4 = f32x4_mul(self.twiddle7re, x4p13);
1322 let t_a6_5 = f32x4_mul(self.twiddle4re, x5p12);
1323 let t_a6_6 = f32x4_mul(self.twiddle2re, x6p11);
1324 let t_a6_7 = f32x4_mul(self.twiddle8re, x7p10);
1325 let t_a6_8 = f32x4_mul(self.twiddle3re, x8p9);
1326 let t_a7_1 = f32x4_mul(self.twiddle7re, x1p16);
1327 let t_a7_2 = f32x4_mul(self.twiddle3re, x2p15);
1328 let t_a7_3 = f32x4_mul(self.twiddle4re, x3p14);
1329 let t_a7_4 = f32x4_mul(self.twiddle6re, x4p13);
1330 let t_a7_5 = f32x4_mul(self.twiddle1re, x5p12);
1331 let t_a7_6 = f32x4_mul(self.twiddle8re, x6p11);
1332 let t_a7_7 = f32x4_mul(self.twiddle2re, x7p10);
1333 let t_a7_8 = f32x4_mul(self.twiddle5re, x8p9);
1334 let t_a8_1 = f32x4_mul(self.twiddle8re, x1p16);
1335 let t_a8_2 = f32x4_mul(self.twiddle1re, x2p15);
1336 let t_a8_3 = f32x4_mul(self.twiddle7re, x3p14);
1337 let t_a8_4 = f32x4_mul(self.twiddle2re, x4p13);
1338 let t_a8_5 = f32x4_mul(self.twiddle6re, x5p12);
1339 let t_a8_6 = f32x4_mul(self.twiddle3re, x6p11);
1340 let t_a8_7 = f32x4_mul(self.twiddle5re, x7p10);
1341 let t_a8_8 = f32x4_mul(self.twiddle4re, x8p9);
1342
1343 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m16);
1344 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m15);
1345 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m14);
1346 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m13);
1347 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m12);
1348 let t_b1_6 = f32x4_mul(self.twiddle6im, x6m11);
1349 let t_b1_7 = f32x4_mul(self.twiddle7im, x7m10);
1350 let t_b1_8 = f32x4_mul(self.twiddle8im, x8m9);
1351 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m16);
1352 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m15);
1353 let t_b2_3 = f32x4_mul(self.twiddle6im, x3m14);
1354 let t_b2_4 = f32x4_mul(self.twiddle8im, x4m13);
1355 let t_b2_5 = f32x4_mul(self.twiddle7im, x5m12);
1356 let t_b2_6 = f32x4_mul(self.twiddle5im, x6m11);
1357 let t_b2_7 = f32x4_mul(self.twiddle3im, x7m10);
1358 let t_b2_8 = f32x4_mul(self.twiddle1im, x8m9);
1359 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m16);
1360 let t_b3_2 = f32x4_mul(self.twiddle6im, x2m15);
1361 let t_b3_3 = f32x4_mul(self.twiddle8im, x3m14);
1362 let t_b3_4 = f32x4_mul(self.twiddle5im, x4m13);
1363 let t_b3_5 = f32x4_mul(self.twiddle2im, x5m12);
1364 let t_b3_6 = f32x4_mul(self.twiddle1im, x6m11);
1365 let t_b3_7 = f32x4_mul(self.twiddle4im, x7m10);
1366 let t_b3_8 = f32x4_mul(self.twiddle7im, x8m9);
1367 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m16);
1368 let t_b4_2 = f32x4_mul(self.twiddle8im, x2m15);
1369 let t_b4_3 = f32x4_mul(self.twiddle5im, x3m14);
1370 let t_b4_4 = f32x4_mul(self.twiddle1im, x4m13);
1371 let t_b4_5 = f32x4_mul(self.twiddle3im, x5m12);
1372 let t_b4_6 = f32x4_mul(self.twiddle7im, x6m11);
1373 let t_b4_7 = f32x4_mul(self.twiddle6im, x7m10);
1374 let t_b4_8 = f32x4_mul(self.twiddle2im, x8m9);
1375 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m16);
1376 let t_b5_2 = f32x4_mul(self.twiddle7im, x2m15);
1377 let t_b5_3 = f32x4_mul(self.twiddle2im, x3m14);
1378 let t_b5_4 = f32x4_mul(self.twiddle3im, x4m13);
1379 let t_b5_5 = f32x4_mul(self.twiddle8im, x5m12);
1380 let t_b5_6 = f32x4_mul(self.twiddle4im, x6m11);
1381 let t_b5_7 = f32x4_mul(self.twiddle1im, x7m10);
1382 let t_b5_8 = f32x4_mul(self.twiddle6im, x8m9);
1383 let t_b6_1 = f32x4_mul(self.twiddle6im, x1m16);
1384 let t_b6_2 = f32x4_mul(self.twiddle5im, x2m15);
1385 let t_b6_3 = f32x4_mul(self.twiddle1im, x3m14);
1386 let t_b6_4 = f32x4_mul(self.twiddle7im, x4m13);
1387 let t_b6_5 = f32x4_mul(self.twiddle4im, x5m12);
1388 let t_b6_6 = f32x4_mul(self.twiddle2im, x6m11);
1389 let t_b6_7 = f32x4_mul(self.twiddle8im, x7m10);
1390 let t_b6_8 = f32x4_mul(self.twiddle3im, x8m9);
1391 let t_b7_1 = f32x4_mul(self.twiddle7im, x1m16);
1392 let t_b7_2 = f32x4_mul(self.twiddle3im, x2m15);
1393 let t_b7_3 = f32x4_mul(self.twiddle4im, x3m14);
1394 let t_b7_4 = f32x4_mul(self.twiddle6im, x4m13);
1395 let t_b7_5 = f32x4_mul(self.twiddle1im, x5m12);
1396 let t_b7_6 = f32x4_mul(self.twiddle8im, x6m11);
1397 let t_b7_7 = f32x4_mul(self.twiddle2im, x7m10);
1398 let t_b7_8 = f32x4_mul(self.twiddle5im, x8m9);
1399 let t_b8_1 = f32x4_mul(self.twiddle8im, x1m16);
1400 let t_b8_2 = f32x4_mul(self.twiddle1im, x2m15);
1401 let t_b8_3 = f32x4_mul(self.twiddle7im, x3m14);
1402 let t_b8_4 = f32x4_mul(self.twiddle2im, x4m13);
1403 let t_b8_5 = f32x4_mul(self.twiddle6im, x5m12);
1404 let t_b8_6 = f32x4_mul(self.twiddle3im, x6m11);
1405 let t_b8_7 = f32x4_mul(self.twiddle5im, x7m10);
1406 let t_b8_8 = f32x4_mul(self.twiddle4im, x8m9);
1407
1408 let x0 = values[0];
1409 let t_a1 =
1410 calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8);
1411 let t_a2 =
1412 calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8);
1413 let t_a3 =
1414 calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8);
1415 let t_a4 =
1416 calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8);
1417 let t_a5 =
1418 calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8);
1419 let t_a6 =
1420 calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8);
1421 let t_a7 =
1422 calc_f32!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8);
1423 let t_a8 =
1424 calc_f32!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8);
1425
1426 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8);
1427 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8);
1428 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 - t_b3_5 + t_b3_6 + t_b3_7 + t_b3_8);
1429 let t_b4 = calc_f32!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 - t_b4_7 - t_b4_8);
1430 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8);
1431 let t_b6 = calc_f32!(t_b6_1 - t_b6_2 + t_b6_3 + t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8);
1432 let t_b7 = calc_f32!(t_b7_1 - t_b7_2 + t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 + t_b7_8);
1433 let t_b8 = calc_f32!(t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 + t_b8_7 - t_b8_8);
1434
1435 let t_b1_rot = self.rotate.rotate_both(t_b1);
1436 let t_b2_rot = self.rotate.rotate_both(t_b2);
1437 let t_b3_rot = self.rotate.rotate_both(t_b3);
1438 let t_b4_rot = self.rotate.rotate_both(t_b4);
1439 let t_b5_rot = self.rotate.rotate_both(t_b5);
1440 let t_b6_rot = self.rotate.rotate_both(t_b6);
1441 let t_b7_rot = self.rotate.rotate_both(t_b7);
1442 let t_b8_rot = self.rotate.rotate_both(t_b8);
1443
1444 let y0 = calc_f32!(x0 + x1p16 + x2p15 + x3p14 + x4p13 + x5p12 + x6p11 + x7p10 + x8p9);
1445 let [y1, y16] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
1446 let [y2, y15] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
1447 let [y3, y14] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
1448 let [y4, y13] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
1449 let [y5, y12] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
1450 let [y6, y11] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
1451 let [y7, y10] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
1452 let [y8, y9] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
1453 [
1454 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16,
1455 ]
1456 }
1457 }
1458
1459 // _ _____ __ _ _ _ _ _
1460 // / |___ | / /_ | || | | |__ (_) |_
1461 // | | / / _____ | '_ \| || |_| '_ \| | __|
1462 // | | / / |_____| | (_) |__ _| |_) | | |_
1463 // |_|/_/ \___/ |_| |_.__/|_|\__|
1464 //
1465
1466 pub struct WasmSimdF64Butterfly17<T> {
1467 direction: FftDirection,
1468 _phantom: std::marker::PhantomData<T>,
1469 rotate: Rotate90F64,
1470 twiddle1re: v128,
1471 twiddle1im: v128,
1472 twiddle2re: v128,
1473 twiddle2im: v128,
1474 twiddle3re: v128,
1475 twiddle3im: v128,
1476 twiddle4re: v128,
1477 twiddle4im: v128,
1478 twiddle5re: v128,
1479 twiddle5im: v128,
1480 twiddle6re: v128,
1481 twiddle6im: v128,
1482 twiddle7re: v128,
1483 twiddle7im: v128,
1484 twiddle8re: v128,
1485 twiddle8im: v128,
1486 }
1487
1488 boilerplate_fft_wasm_simd_f64_butterfly!(
1489 WasmSimdF64Butterfly17,
1490 17,
1491 |this: &WasmSimdF64Butterfly17<_>| this.direction
1492 );
1493 boilerplate_fft_wasm_simd_common_butterfly!(
1494 WasmSimdF64Butterfly17,
1495 17,
1496 |this: &WasmSimdF64Butterfly17<_>| this.direction
1497 );
1498 impl<T: FftNum> WasmSimdF64Butterfly17<T> {
1499 #[inline(always)]
1500 pub fn new(direction: FftDirection) -> Self {
1501 assert_f64::<T>();
1502 let rotate = Rotate90F64::new(true);
1503 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 17, direction);
1504 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 17, direction);
1505 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 17, direction);
1506 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 17, direction);
1507 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 17, direction);
1508 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 17, direction);
1509 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 17, direction);
1510 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 17, direction);
1511 let twiddle1re = f64x2_splat(tw1.re);
1512 let twiddle1im = f64x2_splat(tw1.im);
1513 let twiddle2re = f64x2_splat(tw2.re);
1514 let twiddle2im = f64x2_splat(tw2.im);
1515 let twiddle3re = f64x2_splat(tw3.re);
1516 let twiddle3im = f64x2_splat(tw3.im);
1517 let twiddle4re = f64x2_splat(tw4.re);
1518 let twiddle4im = f64x2_splat(tw4.im);
1519 let twiddle5re = f64x2_splat(tw5.re);
1520 let twiddle5im = f64x2_splat(tw5.im);
1521 let twiddle6re = f64x2_splat(tw6.re);
1522 let twiddle6im = f64x2_splat(tw6.im);
1523 let twiddle7re = f64x2_splat(tw7.re);
1524 let twiddle7im = f64x2_splat(tw7.im);
1525 let twiddle8re = f64x2_splat(tw8.re);
1526 let twiddle8im = f64x2_splat(tw8.im);
1527
1528 Self {
1529 direction,
1530 _phantom: std::marker::PhantomData,
1531 rotate,
1532 twiddle1re,
1533 twiddle1im,
1534 twiddle2re,
1535 twiddle2im,
1536 twiddle3re,
1537 twiddle3im,
1538 twiddle4re,
1539 twiddle4im,
1540 twiddle5re,
1541 twiddle5im,
1542 twiddle6re,
1543 twiddle6im,
1544 twiddle7re,
1545 twiddle7im,
1546 twiddle8re,
1547 twiddle8im,
1548 }
1549 }
1550
1551 #[inline(always)]
1552 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
1553 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1554
1555 let out = self.perform_fft_direct(values);
1556
1557 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1558 }
1559
1560 #[inline(always)]
1561 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 17]) -> [v128; 17] {
1562 let [x1p16, x1m16] = solo_fft2_f64(values[1], values[16]);
1563 let [x2p15, x2m15] = solo_fft2_f64(values[2], values[15]);
1564 let [x3p14, x3m14] = solo_fft2_f64(values[3], values[14]);
1565 let [x4p13, x4m13] = solo_fft2_f64(values[4], values[13]);
1566 let [x5p12, x5m12] = solo_fft2_f64(values[5], values[12]);
1567 let [x6p11, x6m11] = solo_fft2_f64(values[6], values[11]);
1568 let [x7p10, x7m10] = solo_fft2_f64(values[7], values[10]);
1569 let [x8p9, x8m9] = solo_fft2_f64(values[8], values[9]);
1570
1571 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p16);
1572 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p15);
1573 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p14);
1574 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p13);
1575 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p12);
1576 let t_a1_6 = f64x2_mul(self.twiddle6re, x6p11);
1577 let t_a1_7 = f64x2_mul(self.twiddle7re, x7p10);
1578 let t_a1_8 = f64x2_mul(self.twiddle8re, x8p9);
1579 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p16);
1580 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p15);
1581 let t_a2_3 = f64x2_mul(self.twiddle6re, x3p14);
1582 let t_a2_4 = f64x2_mul(self.twiddle8re, x4p13);
1583 let t_a2_5 = f64x2_mul(self.twiddle7re, x5p12);
1584 let t_a2_6 = f64x2_mul(self.twiddle5re, x6p11);
1585 let t_a2_7 = f64x2_mul(self.twiddle3re, x7p10);
1586 let t_a2_8 = f64x2_mul(self.twiddle1re, x8p9);
1587 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p16);
1588 let t_a3_2 = f64x2_mul(self.twiddle6re, x2p15);
1589 let t_a3_3 = f64x2_mul(self.twiddle8re, x3p14);
1590 let t_a3_4 = f64x2_mul(self.twiddle5re, x4p13);
1591 let t_a3_5 = f64x2_mul(self.twiddle2re, x5p12);
1592 let t_a3_6 = f64x2_mul(self.twiddle1re, x6p11);
1593 let t_a3_7 = f64x2_mul(self.twiddle4re, x7p10);
1594 let t_a3_8 = f64x2_mul(self.twiddle7re, x8p9);
1595 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p16);
1596 let t_a4_2 = f64x2_mul(self.twiddle8re, x2p15);
1597 let t_a4_3 = f64x2_mul(self.twiddle5re, x3p14);
1598 let t_a4_4 = f64x2_mul(self.twiddle1re, x4p13);
1599 let t_a4_5 = f64x2_mul(self.twiddle3re, x5p12);
1600 let t_a4_6 = f64x2_mul(self.twiddle7re, x6p11);
1601 let t_a4_7 = f64x2_mul(self.twiddle6re, x7p10);
1602 let t_a4_8 = f64x2_mul(self.twiddle2re, x8p9);
1603 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p16);
1604 let t_a5_2 = f64x2_mul(self.twiddle7re, x2p15);
1605 let t_a5_3 = f64x2_mul(self.twiddle2re, x3p14);
1606 let t_a5_4 = f64x2_mul(self.twiddle3re, x4p13);
1607 let t_a5_5 = f64x2_mul(self.twiddle8re, x5p12);
1608 let t_a5_6 = f64x2_mul(self.twiddle4re, x6p11);
1609 let t_a5_7 = f64x2_mul(self.twiddle1re, x7p10);
1610 let t_a5_8 = f64x2_mul(self.twiddle6re, x8p9);
1611 let t_a6_1 = f64x2_mul(self.twiddle6re, x1p16);
1612 let t_a6_2 = f64x2_mul(self.twiddle5re, x2p15);
1613 let t_a6_3 = f64x2_mul(self.twiddle1re, x3p14);
1614 let t_a6_4 = f64x2_mul(self.twiddle7re, x4p13);
1615 let t_a6_5 = f64x2_mul(self.twiddle4re, x5p12);
1616 let t_a6_6 = f64x2_mul(self.twiddle2re, x6p11);
1617 let t_a6_7 = f64x2_mul(self.twiddle8re, x7p10);
1618 let t_a6_8 = f64x2_mul(self.twiddle3re, x8p9);
1619 let t_a7_1 = f64x2_mul(self.twiddle7re, x1p16);
1620 let t_a7_2 = f64x2_mul(self.twiddle3re, x2p15);
1621 let t_a7_3 = f64x2_mul(self.twiddle4re, x3p14);
1622 let t_a7_4 = f64x2_mul(self.twiddle6re, x4p13);
1623 let t_a7_5 = f64x2_mul(self.twiddle1re, x5p12);
1624 let t_a7_6 = f64x2_mul(self.twiddle8re, x6p11);
1625 let t_a7_7 = f64x2_mul(self.twiddle2re, x7p10);
1626 let t_a7_8 = f64x2_mul(self.twiddle5re, x8p9);
1627 let t_a8_1 = f64x2_mul(self.twiddle8re, x1p16);
1628 let t_a8_2 = f64x2_mul(self.twiddle1re, x2p15);
1629 let t_a8_3 = f64x2_mul(self.twiddle7re, x3p14);
1630 let t_a8_4 = f64x2_mul(self.twiddle2re, x4p13);
1631 let t_a8_5 = f64x2_mul(self.twiddle6re, x5p12);
1632 let t_a8_6 = f64x2_mul(self.twiddle3re, x6p11);
1633 let t_a8_7 = f64x2_mul(self.twiddle5re, x7p10);
1634 let t_a8_8 = f64x2_mul(self.twiddle4re, x8p9);
1635
1636 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m16);
1637 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m15);
1638 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m14);
1639 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m13);
1640 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m12);
1641 let t_b1_6 = f64x2_mul(self.twiddle6im, x6m11);
1642 let t_b1_7 = f64x2_mul(self.twiddle7im, x7m10);
1643 let t_b1_8 = f64x2_mul(self.twiddle8im, x8m9);
1644 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m16);
1645 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m15);
1646 let t_b2_3 = f64x2_mul(self.twiddle6im, x3m14);
1647 let t_b2_4 = f64x2_mul(self.twiddle8im, x4m13);
1648 let t_b2_5 = f64x2_mul(self.twiddle7im, x5m12);
1649 let t_b2_6 = f64x2_mul(self.twiddle5im, x6m11);
1650 let t_b2_7 = f64x2_mul(self.twiddle3im, x7m10);
1651 let t_b2_8 = f64x2_mul(self.twiddle1im, x8m9);
1652 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m16);
1653 let t_b3_2 = f64x2_mul(self.twiddle6im, x2m15);
1654 let t_b3_3 = f64x2_mul(self.twiddle8im, x3m14);
1655 let t_b3_4 = f64x2_mul(self.twiddle5im, x4m13);
1656 let t_b3_5 = f64x2_mul(self.twiddle2im, x5m12);
1657 let t_b3_6 = f64x2_mul(self.twiddle1im, x6m11);
1658 let t_b3_7 = f64x2_mul(self.twiddle4im, x7m10);
1659 let t_b3_8 = f64x2_mul(self.twiddle7im, x8m9);
1660 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m16);
1661 let t_b4_2 = f64x2_mul(self.twiddle8im, x2m15);
1662 let t_b4_3 = f64x2_mul(self.twiddle5im, x3m14);
1663 let t_b4_4 = f64x2_mul(self.twiddle1im, x4m13);
1664 let t_b4_5 = f64x2_mul(self.twiddle3im, x5m12);
1665 let t_b4_6 = f64x2_mul(self.twiddle7im, x6m11);
1666 let t_b4_7 = f64x2_mul(self.twiddle6im, x7m10);
1667 let t_b4_8 = f64x2_mul(self.twiddle2im, x8m9);
1668 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m16);
1669 let t_b5_2 = f64x2_mul(self.twiddle7im, x2m15);
1670 let t_b5_3 = f64x2_mul(self.twiddle2im, x3m14);
1671 let t_b5_4 = f64x2_mul(self.twiddle3im, x4m13);
1672 let t_b5_5 = f64x2_mul(self.twiddle8im, x5m12);
1673 let t_b5_6 = f64x2_mul(self.twiddle4im, x6m11);
1674 let t_b5_7 = f64x2_mul(self.twiddle1im, x7m10);
1675 let t_b5_8 = f64x2_mul(self.twiddle6im, x8m9);
1676 let t_b6_1 = f64x2_mul(self.twiddle6im, x1m16);
1677 let t_b6_2 = f64x2_mul(self.twiddle5im, x2m15);
1678 let t_b6_3 = f64x2_mul(self.twiddle1im, x3m14);
1679 let t_b6_4 = f64x2_mul(self.twiddle7im, x4m13);
1680 let t_b6_5 = f64x2_mul(self.twiddle4im, x5m12);
1681 let t_b6_6 = f64x2_mul(self.twiddle2im, x6m11);
1682 let t_b6_7 = f64x2_mul(self.twiddle8im, x7m10);
1683 let t_b6_8 = f64x2_mul(self.twiddle3im, x8m9);
1684 let t_b7_1 = f64x2_mul(self.twiddle7im, x1m16);
1685 let t_b7_2 = f64x2_mul(self.twiddle3im, x2m15);
1686 let t_b7_3 = f64x2_mul(self.twiddle4im, x3m14);
1687 let t_b7_4 = f64x2_mul(self.twiddle6im, x4m13);
1688 let t_b7_5 = f64x2_mul(self.twiddle1im, x5m12);
1689 let t_b7_6 = f64x2_mul(self.twiddle8im, x6m11);
1690 let t_b7_7 = f64x2_mul(self.twiddle2im, x7m10);
1691 let t_b7_8 = f64x2_mul(self.twiddle5im, x8m9);
1692 let t_b8_1 = f64x2_mul(self.twiddle8im, x1m16);
1693 let t_b8_2 = f64x2_mul(self.twiddle1im, x2m15);
1694 let t_b8_3 = f64x2_mul(self.twiddle7im, x3m14);
1695 let t_b8_4 = f64x2_mul(self.twiddle2im, x4m13);
1696 let t_b8_5 = f64x2_mul(self.twiddle6im, x5m12);
1697 let t_b8_6 = f64x2_mul(self.twiddle3im, x6m11);
1698 let t_b8_7 = f64x2_mul(self.twiddle5im, x7m10);
1699 let t_b8_8 = f64x2_mul(self.twiddle4im, x8m9);
1700
1701 let x0 = values[0];
1702 let t_a1 =
1703 calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8);
1704 let t_a2 =
1705 calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8);
1706 let t_a3 =
1707 calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8);
1708 let t_a4 =
1709 calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8);
1710 let t_a5 =
1711 calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8);
1712 let t_a6 =
1713 calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8);
1714 let t_a7 =
1715 calc_f64!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8);
1716 let t_a8 =
1717 calc_f64!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8);
1718
1719 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8);
1720 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8);
1721 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 - t_b3_5 + t_b3_6 + t_b3_7 + t_b3_8);
1722 let t_b4 = calc_f64!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 - t_b4_7 - t_b4_8);
1723 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8);
1724 let t_b6 = calc_f64!(t_b6_1 - t_b6_2 + t_b6_3 + t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8);
1725 let t_b7 = calc_f64!(t_b7_1 - t_b7_2 + t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 + t_b7_8);
1726 let t_b8 = calc_f64!(t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 + t_b8_7 - t_b8_8);
1727
1728 let t_b1_rot = self.rotate.rotate(t_b1);
1729 let t_b2_rot = self.rotate.rotate(t_b2);
1730 let t_b3_rot = self.rotate.rotate(t_b3);
1731 let t_b4_rot = self.rotate.rotate(t_b4);
1732 let t_b5_rot = self.rotate.rotate(t_b5);
1733 let t_b6_rot = self.rotate.rotate(t_b6);
1734 let t_b7_rot = self.rotate.rotate(t_b7);
1735 let t_b8_rot = self.rotate.rotate(t_b8);
1736
1737 let y0 = calc_f64!(x0 + x1p16 + x2p15 + x3p14 + x4p13 + x5p12 + x6p11 + x7p10 + x8p9);
1738 let [y1, y16] = solo_fft2_f64(t_a1, t_b1_rot);
1739 let [y2, y15] = solo_fft2_f64(t_a2, t_b2_rot);
1740 let [y3, y14] = solo_fft2_f64(t_a3, t_b3_rot);
1741 let [y4, y13] = solo_fft2_f64(t_a4, t_b4_rot);
1742 let [y5, y12] = solo_fft2_f64(t_a5, t_b5_rot);
1743 let [y6, y11] = solo_fft2_f64(t_a6, t_b6_rot);
1744 let [y7, y10] = solo_fft2_f64(t_a7, t_b7_rot);
1745 let [y8, y9] = solo_fft2_f64(t_a8, t_b8_rot);
1746 [
1747 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16,
1748 ]
1749 }
1750 }
1751
1752 // _ ___ _________ _ _ _
1753 // / |/ _ \ |___ /___ \| |__ (_) |_
1754 // | | (_) | _____ |_ \ __) | '_ \| | __|
1755 // | |\__, | |_____| ___) / __/| |_) | | |_
1756 // |_| /_/ |____/_____|_.__/|_|\__|
1757 //
1758 pub struct WasmSimdF32Butterfly19<T> {
1759 direction: FftDirection,
1760 _phantom: std::marker::PhantomData<T>,
1761 rotate: Rotate90F32,
1762 twiddle1re: v128,
1763 twiddle1im: v128,
1764 twiddle2re: v128,
1765 twiddle2im: v128,
1766 twiddle3re: v128,
1767 twiddle3im: v128,
1768 twiddle4re: v128,
1769 twiddle4im: v128,
1770 twiddle5re: v128,
1771 twiddle5im: v128,
1772 twiddle6re: v128,
1773 twiddle6im: v128,
1774 twiddle7re: v128,
1775 twiddle7im: v128,
1776 twiddle8re: v128,
1777 twiddle8im: v128,
1778 twiddle9re: v128,
1779 twiddle9im: v128,
1780 }
1781
1782 boilerplate_fft_wasm_simd_f32_butterfly!(
1783 WasmSimdF32Butterfly19,
1784 19,
1785 |this: &WasmSimdF32Butterfly19<_>| this.direction
1786 );
1787 boilerplate_fft_wasm_simd_common_butterfly!(
1788 WasmSimdF32Butterfly19,
1789 19,
1790 |this: &WasmSimdF32Butterfly19<_>| this.direction
1791 );
1792 impl<T: FftNum> WasmSimdF32Butterfly19<T> {
1793 #[inline(always)]
1794 pub fn new(direction: FftDirection) -> Self {
1795 assert_f32::<T>();
1796 let rotate = Rotate90F32::new(true);
1797 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 19, direction);
1798 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 19, direction);
1799 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 19, direction);
1800 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 19, direction);
1801 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 19, direction);
1802 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 19, direction);
1803 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 19, direction);
1804 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 19, direction);
1805 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 19, direction);
1806 let twiddle1re = f32x4_splat(tw1.re);
1807 let twiddle1im = f32x4_splat(tw1.im);
1808 let twiddle2re = f32x4_splat(tw2.re);
1809 let twiddle2im = f32x4_splat(tw2.im);
1810 let twiddle3re = f32x4_splat(tw3.re);
1811 let twiddle3im = f32x4_splat(tw3.im);
1812 let twiddle4re = f32x4_splat(tw4.re);
1813 let twiddle4im = f32x4_splat(tw4.im);
1814 let twiddle5re = f32x4_splat(tw5.re);
1815 let twiddle5im = f32x4_splat(tw5.im);
1816 let twiddle6re = f32x4_splat(tw6.re);
1817 let twiddle6im = f32x4_splat(tw6.im);
1818 let twiddle7re = f32x4_splat(tw7.re);
1819 let twiddle7im = f32x4_splat(tw7.im);
1820 let twiddle8re = f32x4_splat(tw8.re);
1821 let twiddle8im = f32x4_splat(tw8.im);
1822 let twiddle9re = f32x4_splat(tw9.re);
1823 let twiddle9im = f32x4_splat(tw9.im);
1824
1825 Self {
1826 direction,
1827 _phantom: std::marker::PhantomData,
1828 rotate,
1829 twiddle1re,
1830 twiddle1im,
1831 twiddle2re,
1832 twiddle2im,
1833 twiddle3re,
1834 twiddle3im,
1835 twiddle4re,
1836 twiddle4im,
1837 twiddle5re,
1838 twiddle5im,
1839 twiddle6re,
1840 twiddle6im,
1841 twiddle7re,
1842 twiddle7im,
1843 twiddle8re,
1844 twiddle8im,
1845 twiddle9re,
1846 twiddle9im,
1847 }
1848 }
1849
1850 #[inline(always)]
1851 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
1852 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
1853
1854 let out = self.perform_parallel_fft_direct(values);
1855
1856 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
1857 }
1858
1859 #[inline(always)]
1860 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1861 &self,
1862 mut buffer: impl WasmSimdArrayMut<f32>,
1863 ) {
1864 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36});
1865
1866 let values = [
1867 extract_lo_hi_f32(input_packed[0], input_packed[9]),
1868 extract_hi_lo_f32(input_packed[0], input_packed[10]),
1869 extract_lo_hi_f32(input_packed[1], input_packed[10]),
1870 extract_hi_lo_f32(input_packed[1], input_packed[11]),
1871 extract_lo_hi_f32(input_packed[2], input_packed[11]),
1872 extract_hi_lo_f32(input_packed[2], input_packed[12]),
1873 extract_lo_hi_f32(input_packed[3], input_packed[12]),
1874 extract_hi_lo_f32(input_packed[3], input_packed[13]),
1875 extract_lo_hi_f32(input_packed[4], input_packed[13]),
1876 extract_hi_lo_f32(input_packed[4], input_packed[14]),
1877 extract_lo_hi_f32(input_packed[5], input_packed[14]),
1878 extract_hi_lo_f32(input_packed[5], input_packed[15]),
1879 extract_lo_hi_f32(input_packed[6], input_packed[15]),
1880 extract_hi_lo_f32(input_packed[6], input_packed[16]),
1881 extract_lo_hi_f32(input_packed[7], input_packed[16]),
1882 extract_hi_lo_f32(input_packed[7], input_packed[17]),
1883 extract_lo_hi_f32(input_packed[8], input_packed[17]),
1884 extract_hi_lo_f32(input_packed[8], input_packed[18]),
1885 extract_lo_hi_f32(input_packed[9], input_packed[18]),
1886 ];
1887
1888 let out = self.perform_parallel_fft_direct(values);
1889
1890 let out_packed = [
1891 extract_lo_lo_f32(out[0], out[1]),
1892 extract_lo_lo_f32(out[2], out[3]),
1893 extract_lo_lo_f32(out[4], out[5]),
1894 extract_lo_lo_f32(out[6], out[7]),
1895 extract_lo_lo_f32(out[8], out[9]),
1896 extract_lo_lo_f32(out[10], out[11]),
1897 extract_lo_lo_f32(out[12], out[13]),
1898 extract_lo_lo_f32(out[14], out[15]),
1899 extract_lo_lo_f32(out[16], out[17]),
1900 extract_lo_hi_f32(out[18], out[0]),
1901 extract_hi_hi_f32(out[1], out[2]),
1902 extract_hi_hi_f32(out[3], out[4]),
1903 extract_hi_hi_f32(out[5], out[6]),
1904 extract_hi_hi_f32(out[7], out[8]),
1905 extract_hi_hi_f32(out[9], out[10]),
1906 extract_hi_hi_f32(out[11], out[12]),
1907 extract_hi_hi_f32(out[13], out[14]),
1908 extract_hi_hi_f32(out[15], out[16]),
1909 extract_hi_hi_f32(out[17], out[18]),
1910 ];
1911
1912 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
1913 }
1914
1915 #[inline(always)]
1916 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 19]) -> [v128; 19] {
1917 let [x1p18, x1m18] = parallel_fft2_interleaved_f32(values[1], values[18]);
1918 let [x2p17, x2m17] = parallel_fft2_interleaved_f32(values[2], values[17]);
1919 let [x3p16, x3m16] = parallel_fft2_interleaved_f32(values[3], values[16]);
1920 let [x4p15, x4m15] = parallel_fft2_interleaved_f32(values[4], values[15]);
1921 let [x5p14, x5m14] = parallel_fft2_interleaved_f32(values[5], values[14]);
1922 let [x6p13, x6m13] = parallel_fft2_interleaved_f32(values[6], values[13]);
1923 let [x7p12, x7m12] = parallel_fft2_interleaved_f32(values[7], values[12]);
1924 let [x8p11, x8m11] = parallel_fft2_interleaved_f32(values[8], values[11]);
1925 let [x9p10, x9m10] = parallel_fft2_interleaved_f32(values[9], values[10]);
1926
1927 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p18);
1928 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p17);
1929 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p16);
1930 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p15);
1931 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p14);
1932 let t_a1_6 = f32x4_mul(self.twiddle6re, x6p13);
1933 let t_a1_7 = f32x4_mul(self.twiddle7re, x7p12);
1934 let t_a1_8 = f32x4_mul(self.twiddle8re, x8p11);
1935 let t_a1_9 = f32x4_mul(self.twiddle9re, x9p10);
1936 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p18);
1937 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p17);
1938 let t_a2_3 = f32x4_mul(self.twiddle6re, x3p16);
1939 let t_a2_4 = f32x4_mul(self.twiddle8re, x4p15);
1940 let t_a2_5 = f32x4_mul(self.twiddle9re, x5p14);
1941 let t_a2_6 = f32x4_mul(self.twiddle7re, x6p13);
1942 let t_a2_7 = f32x4_mul(self.twiddle5re, x7p12);
1943 let t_a2_8 = f32x4_mul(self.twiddle3re, x8p11);
1944 let t_a2_9 = f32x4_mul(self.twiddle1re, x9p10);
1945 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p18);
1946 let t_a3_2 = f32x4_mul(self.twiddle6re, x2p17);
1947 let t_a3_3 = f32x4_mul(self.twiddle9re, x3p16);
1948 let t_a3_4 = f32x4_mul(self.twiddle7re, x4p15);
1949 let t_a3_5 = f32x4_mul(self.twiddle4re, x5p14);
1950 let t_a3_6 = f32x4_mul(self.twiddle1re, x6p13);
1951 let t_a3_7 = f32x4_mul(self.twiddle2re, x7p12);
1952 let t_a3_8 = f32x4_mul(self.twiddle5re, x8p11);
1953 let t_a3_9 = f32x4_mul(self.twiddle8re, x9p10);
1954 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p18);
1955 let t_a4_2 = f32x4_mul(self.twiddle8re, x2p17);
1956 let t_a4_3 = f32x4_mul(self.twiddle7re, x3p16);
1957 let t_a4_4 = f32x4_mul(self.twiddle3re, x4p15);
1958 let t_a4_5 = f32x4_mul(self.twiddle1re, x5p14);
1959 let t_a4_6 = f32x4_mul(self.twiddle5re, x6p13);
1960 let t_a4_7 = f32x4_mul(self.twiddle9re, x7p12);
1961 let t_a4_8 = f32x4_mul(self.twiddle6re, x8p11);
1962 let t_a4_9 = f32x4_mul(self.twiddle2re, x9p10);
1963 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p18);
1964 let t_a5_2 = f32x4_mul(self.twiddle9re, x2p17);
1965 let t_a5_3 = f32x4_mul(self.twiddle4re, x3p16);
1966 let t_a5_4 = f32x4_mul(self.twiddle1re, x4p15);
1967 let t_a5_5 = f32x4_mul(self.twiddle6re, x5p14);
1968 let t_a5_6 = f32x4_mul(self.twiddle8re, x6p13);
1969 let t_a5_7 = f32x4_mul(self.twiddle3re, x7p12);
1970 let t_a5_8 = f32x4_mul(self.twiddle2re, x8p11);
1971 let t_a5_9 = f32x4_mul(self.twiddle7re, x9p10);
1972 let t_a6_1 = f32x4_mul(self.twiddle6re, x1p18);
1973 let t_a6_2 = f32x4_mul(self.twiddle7re, x2p17);
1974 let t_a6_3 = f32x4_mul(self.twiddle1re, x3p16);
1975 let t_a6_4 = f32x4_mul(self.twiddle5re, x4p15);
1976 let t_a6_5 = f32x4_mul(self.twiddle8re, x5p14);
1977 let t_a6_6 = f32x4_mul(self.twiddle2re, x6p13);
1978 let t_a6_7 = f32x4_mul(self.twiddle4re, x7p12);
1979 let t_a6_8 = f32x4_mul(self.twiddle9re, x8p11);
1980 let t_a6_9 = f32x4_mul(self.twiddle3re, x9p10);
1981 let t_a7_1 = f32x4_mul(self.twiddle7re, x1p18);
1982 let t_a7_2 = f32x4_mul(self.twiddle5re, x2p17);
1983 let t_a7_3 = f32x4_mul(self.twiddle2re, x3p16);
1984 let t_a7_4 = f32x4_mul(self.twiddle9re, x4p15);
1985 let t_a7_5 = f32x4_mul(self.twiddle3re, x5p14);
1986 let t_a7_6 = f32x4_mul(self.twiddle4re, x6p13);
1987 let t_a7_7 = f32x4_mul(self.twiddle8re, x7p12);
1988 let t_a7_8 = f32x4_mul(self.twiddle1re, x8p11);
1989 let t_a7_9 = f32x4_mul(self.twiddle6re, x9p10);
1990 let t_a8_1 = f32x4_mul(self.twiddle8re, x1p18);
1991 let t_a8_2 = f32x4_mul(self.twiddle3re, x2p17);
1992 let t_a8_3 = f32x4_mul(self.twiddle5re, x3p16);
1993 let t_a8_4 = f32x4_mul(self.twiddle6re, x4p15);
1994 let t_a8_5 = f32x4_mul(self.twiddle2re, x5p14);
1995 let t_a8_6 = f32x4_mul(self.twiddle9re, x6p13);
1996 let t_a8_7 = f32x4_mul(self.twiddle1re, x7p12);
1997 let t_a8_8 = f32x4_mul(self.twiddle7re, x8p11);
1998 let t_a8_9 = f32x4_mul(self.twiddle4re, x9p10);
1999 let t_a9_1 = f32x4_mul(self.twiddle9re, x1p18);
2000 let t_a9_2 = f32x4_mul(self.twiddle1re, x2p17);
2001 let t_a9_3 = f32x4_mul(self.twiddle8re, x3p16);
2002 let t_a9_4 = f32x4_mul(self.twiddle2re, x4p15);
2003 let t_a9_5 = f32x4_mul(self.twiddle7re, x5p14);
2004 let t_a9_6 = f32x4_mul(self.twiddle3re, x6p13);
2005 let t_a9_7 = f32x4_mul(self.twiddle6re, x7p12);
2006 let t_a9_8 = f32x4_mul(self.twiddle4re, x8p11);
2007 let t_a9_9 = f32x4_mul(self.twiddle5re, x9p10);
2008
2009 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m18);
2010 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m17);
2011 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m16);
2012 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m15);
2013 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m14);
2014 let t_b1_6 = f32x4_mul(self.twiddle6im, x6m13);
2015 let t_b1_7 = f32x4_mul(self.twiddle7im, x7m12);
2016 let t_b1_8 = f32x4_mul(self.twiddle8im, x8m11);
2017 let t_b1_9 = f32x4_mul(self.twiddle9im, x9m10);
2018 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m18);
2019 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m17);
2020 let t_b2_3 = f32x4_mul(self.twiddle6im, x3m16);
2021 let t_b2_4 = f32x4_mul(self.twiddle8im, x4m15);
2022 let t_b2_5 = f32x4_mul(self.twiddle9im, x5m14);
2023 let t_b2_6 = f32x4_mul(self.twiddle7im, x6m13);
2024 let t_b2_7 = f32x4_mul(self.twiddle5im, x7m12);
2025 let t_b2_8 = f32x4_mul(self.twiddle3im, x8m11);
2026 let t_b2_9 = f32x4_mul(self.twiddle1im, x9m10);
2027 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m18);
2028 let t_b3_2 = f32x4_mul(self.twiddle6im, x2m17);
2029 let t_b3_3 = f32x4_mul(self.twiddle9im, x3m16);
2030 let t_b3_4 = f32x4_mul(self.twiddle7im, x4m15);
2031 let t_b3_5 = f32x4_mul(self.twiddle4im, x5m14);
2032 let t_b3_6 = f32x4_mul(self.twiddle1im, x6m13);
2033 let t_b3_7 = f32x4_mul(self.twiddle2im, x7m12);
2034 let t_b3_8 = f32x4_mul(self.twiddle5im, x8m11);
2035 let t_b3_9 = f32x4_mul(self.twiddle8im, x9m10);
2036 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m18);
2037 let t_b4_2 = f32x4_mul(self.twiddle8im, x2m17);
2038 let t_b4_3 = f32x4_mul(self.twiddle7im, x3m16);
2039 let t_b4_4 = f32x4_mul(self.twiddle3im, x4m15);
2040 let t_b4_5 = f32x4_mul(self.twiddle1im, x5m14);
2041 let t_b4_6 = f32x4_mul(self.twiddle5im, x6m13);
2042 let t_b4_7 = f32x4_mul(self.twiddle9im, x7m12);
2043 let t_b4_8 = f32x4_mul(self.twiddle6im, x8m11);
2044 let t_b4_9 = f32x4_mul(self.twiddle2im, x9m10);
2045 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m18);
2046 let t_b5_2 = f32x4_mul(self.twiddle9im, x2m17);
2047 let t_b5_3 = f32x4_mul(self.twiddle4im, x3m16);
2048 let t_b5_4 = f32x4_mul(self.twiddle1im, x4m15);
2049 let t_b5_5 = f32x4_mul(self.twiddle6im, x5m14);
2050 let t_b5_6 = f32x4_mul(self.twiddle8im, x6m13);
2051 let t_b5_7 = f32x4_mul(self.twiddle3im, x7m12);
2052 let t_b5_8 = f32x4_mul(self.twiddle2im, x8m11);
2053 let t_b5_9 = f32x4_mul(self.twiddle7im, x9m10);
2054 let t_b6_1 = f32x4_mul(self.twiddle6im, x1m18);
2055 let t_b6_2 = f32x4_mul(self.twiddle7im, x2m17);
2056 let t_b6_3 = f32x4_mul(self.twiddle1im, x3m16);
2057 let t_b6_4 = f32x4_mul(self.twiddle5im, x4m15);
2058 let t_b6_5 = f32x4_mul(self.twiddle8im, x5m14);
2059 let t_b6_6 = f32x4_mul(self.twiddle2im, x6m13);
2060 let t_b6_7 = f32x4_mul(self.twiddle4im, x7m12);
2061 let t_b6_8 = f32x4_mul(self.twiddle9im, x8m11);
2062 let t_b6_9 = f32x4_mul(self.twiddle3im, x9m10);
2063 let t_b7_1 = f32x4_mul(self.twiddle7im, x1m18);
2064 let t_b7_2 = f32x4_mul(self.twiddle5im, x2m17);
2065 let t_b7_3 = f32x4_mul(self.twiddle2im, x3m16);
2066 let t_b7_4 = f32x4_mul(self.twiddle9im, x4m15);
2067 let t_b7_5 = f32x4_mul(self.twiddle3im, x5m14);
2068 let t_b7_6 = f32x4_mul(self.twiddle4im, x6m13);
2069 let t_b7_7 = f32x4_mul(self.twiddle8im, x7m12);
2070 let t_b7_8 = f32x4_mul(self.twiddle1im, x8m11);
2071 let t_b7_9 = f32x4_mul(self.twiddle6im, x9m10);
2072 let t_b8_1 = f32x4_mul(self.twiddle8im, x1m18);
2073 let t_b8_2 = f32x4_mul(self.twiddle3im, x2m17);
2074 let t_b8_3 = f32x4_mul(self.twiddle5im, x3m16);
2075 let t_b8_4 = f32x4_mul(self.twiddle6im, x4m15);
2076 let t_b8_5 = f32x4_mul(self.twiddle2im, x5m14);
2077 let t_b8_6 = f32x4_mul(self.twiddle9im, x6m13);
2078 let t_b8_7 = f32x4_mul(self.twiddle1im, x7m12);
2079 let t_b8_8 = f32x4_mul(self.twiddle7im, x8m11);
2080 let t_b8_9 = f32x4_mul(self.twiddle4im, x9m10);
2081 let t_b9_1 = f32x4_mul(self.twiddle9im, x1m18);
2082 let t_b9_2 = f32x4_mul(self.twiddle1im, x2m17);
2083 let t_b9_3 = f32x4_mul(self.twiddle8im, x3m16);
2084 let t_b9_4 = f32x4_mul(self.twiddle2im, x4m15);
2085 let t_b9_5 = f32x4_mul(self.twiddle7im, x5m14);
2086 let t_b9_6 = f32x4_mul(self.twiddle3im, x6m13);
2087 let t_b9_7 = f32x4_mul(self.twiddle6im, x7m12);
2088 let t_b9_8 = f32x4_mul(self.twiddle4im, x8m11);
2089 let t_b9_9 = f32x4_mul(self.twiddle5im, x9m10);
2090
2091 let x0 = values[0];
2092 let t_a1 = calc_f32!(
2093 x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9
2094 );
2095 let t_a2 = calc_f32!(
2096 x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9
2097 );
2098 let t_a3 = calc_f32!(
2099 x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9
2100 );
2101 let t_a4 = calc_f32!(
2102 x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9
2103 );
2104 let t_a5 = calc_f32!(
2105 x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9
2106 );
2107 let t_a6 = calc_f32!(
2108 x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9
2109 );
2110 let t_a7 = calc_f32!(
2111 x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9
2112 );
2113 let t_a8 = calc_f32!(
2114 x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9
2115 );
2116 let t_a9 = calc_f32!(
2117 x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9
2118 );
2119
2120 let t_b1 = calc_f32!(
2121 t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9
2122 );
2123 let t_b2 = calc_f32!(
2124 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8 - t_b2_9
2125 );
2126 let t_b3 = calc_f32!(
2127 t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 + t_b3_7 + t_b3_8 + t_b3_9
2128 );
2129 let t_b4 = calc_f32!(
2130 t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 + t_b4_7 - t_b4_8 - t_b4_9
2131 );
2132 let t_b5 = calc_f32!(
2133 t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 - t_b5_7 + t_b5_8 + t_b5_9
2134 );
2135 let t_b6 = calc_f32!(
2136 t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 - t_b6_5 - t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9
2137 );
2138 let t_b7 = calc_f32!(
2139 t_b7_1 - t_b7_2 + t_b7_3 + t_b7_4 - t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9
2140 );
2141 let t_b8 = calc_f32!(
2142 t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 - t_b8_9
2143 );
2144 let t_b9 = calc_f32!(
2145 t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 - t_b9_8 + t_b9_9
2146 );
2147
2148 let t_b1_rot = self.rotate.rotate_both(t_b1);
2149 let t_b2_rot = self.rotate.rotate_both(t_b2);
2150 let t_b3_rot = self.rotate.rotate_both(t_b3);
2151 let t_b4_rot = self.rotate.rotate_both(t_b4);
2152 let t_b5_rot = self.rotate.rotate_both(t_b5);
2153 let t_b6_rot = self.rotate.rotate_both(t_b6);
2154 let t_b7_rot = self.rotate.rotate_both(t_b7);
2155 let t_b8_rot = self.rotate.rotate_both(t_b8);
2156 let t_b9_rot = self.rotate.rotate_both(t_b9);
2157
2158 let y0 =
2159 calc_f32!(x0 + x1p18 + x2p17 + x3p16 + x4p15 + x5p14 + x6p13 + x7p12 + x8p11 + x9p10);
2160 let [y1, y18] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
2161 let [y2, y17] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
2162 let [y3, y16] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
2163 let [y4, y15] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
2164 let [y5, y14] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
2165 let [y6, y13] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
2166 let [y7, y12] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
2167 let [y8, y11] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
2168 let [y9, y10] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
2169 [
2170 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
2171 ]
2172 }
2173 }
2174
2175 // _ ___ __ _ _ _ _ _
2176 // / |/ _ \ / /_ | || | | |__ (_) |_
2177 // | | (_) | _____ | '_ \| || |_| '_ \| | __|
2178 // | |\__, | |_____| | (_) |__ _| |_) | | |_
2179 // |_| /_/ \___/ |_| |_.__/|_|\__|
2180 //
2181
2182 pub struct WasmSimdF64Butterfly19<T> {
2183 direction: FftDirection,
2184 _phantom: std::marker::PhantomData<T>,
2185 rotate: Rotate90F64,
2186 twiddle1re: v128,
2187 twiddle1im: v128,
2188 twiddle2re: v128,
2189 twiddle2im: v128,
2190 twiddle3re: v128,
2191 twiddle3im: v128,
2192 twiddle4re: v128,
2193 twiddle4im: v128,
2194 twiddle5re: v128,
2195 twiddle5im: v128,
2196 twiddle6re: v128,
2197 twiddle6im: v128,
2198 twiddle7re: v128,
2199 twiddle7im: v128,
2200 twiddle8re: v128,
2201 twiddle8im: v128,
2202 twiddle9re: v128,
2203 twiddle9im: v128,
2204 }
2205
2206 boilerplate_fft_wasm_simd_f64_butterfly!(
2207 WasmSimdF64Butterfly19,
2208 19,
2209 |this: &WasmSimdF64Butterfly19<_>| this.direction
2210 );
2211 boilerplate_fft_wasm_simd_common_butterfly!(
2212 WasmSimdF64Butterfly19,
2213 19,
2214 |this: &WasmSimdF64Butterfly19<_>| this.direction
2215 );
2216 impl<T: FftNum> WasmSimdF64Butterfly19<T> {
2217 #[inline(always)]
2218 pub fn new(direction: FftDirection) -> Self {
2219 assert_f64::<T>();
2220 let rotate = Rotate90F64::new(true);
2221 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 19, direction);
2222 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 19, direction);
2223 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 19, direction);
2224 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 19, direction);
2225 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 19, direction);
2226 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 19, direction);
2227 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 19, direction);
2228 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 19, direction);
2229 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 19, direction);
2230 let twiddle1re = f64x2_splat(tw1.re);
2231 let twiddle1im = f64x2_splat(tw1.im);
2232 let twiddle2re = f64x2_splat(tw2.re);
2233 let twiddle2im = f64x2_splat(tw2.im);
2234 let twiddle3re = f64x2_splat(tw3.re);
2235 let twiddle3im = f64x2_splat(tw3.im);
2236 let twiddle4re = f64x2_splat(tw4.re);
2237 let twiddle4im = f64x2_splat(tw4.im);
2238 let twiddle5re = f64x2_splat(tw5.re);
2239 let twiddle5im = f64x2_splat(tw5.im);
2240 let twiddle6re = f64x2_splat(tw6.re);
2241 let twiddle6im = f64x2_splat(tw6.im);
2242 let twiddle7re = f64x2_splat(tw7.re);
2243 let twiddle7im = f64x2_splat(tw7.im);
2244 let twiddle8re = f64x2_splat(tw8.re);
2245 let twiddle8im = f64x2_splat(tw8.im);
2246 let twiddle9re = f64x2_splat(tw9.re);
2247 let twiddle9im = f64x2_splat(tw9.im);
2248
2249 Self {
2250 direction,
2251 _phantom: std::marker::PhantomData,
2252 rotate,
2253 twiddle1re,
2254 twiddle1im,
2255 twiddle2re,
2256 twiddle2im,
2257 twiddle3re,
2258 twiddle3im,
2259 twiddle4re,
2260 twiddle4im,
2261 twiddle5re,
2262 twiddle5im,
2263 twiddle6re,
2264 twiddle6im,
2265 twiddle7re,
2266 twiddle7im,
2267 twiddle8re,
2268 twiddle8im,
2269 twiddle9re,
2270 twiddle9im,
2271 }
2272 }
2273
2274 #[inline(always)]
2275 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
2276 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
2277
2278 let out = self.perform_fft_direct(values);
2279
2280 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
2281 }
2282
2283 #[inline(always)]
2284 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 19]) -> [v128; 19] {
2285 let [x1p18, x1m18] = solo_fft2_f64(values[1], values[18]);
2286 let [x2p17, x2m17] = solo_fft2_f64(values[2], values[17]);
2287 let [x3p16, x3m16] = solo_fft2_f64(values[3], values[16]);
2288 let [x4p15, x4m15] = solo_fft2_f64(values[4], values[15]);
2289 let [x5p14, x5m14] = solo_fft2_f64(values[5], values[14]);
2290 let [x6p13, x6m13] = solo_fft2_f64(values[6], values[13]);
2291 let [x7p12, x7m12] = solo_fft2_f64(values[7], values[12]);
2292 let [x8p11, x8m11] = solo_fft2_f64(values[8], values[11]);
2293 let [x9p10, x9m10] = solo_fft2_f64(values[9], values[10]);
2294
2295 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p18);
2296 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p17);
2297 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p16);
2298 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p15);
2299 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p14);
2300 let t_a1_6 = f64x2_mul(self.twiddle6re, x6p13);
2301 let t_a1_7 = f64x2_mul(self.twiddle7re, x7p12);
2302 let t_a1_8 = f64x2_mul(self.twiddle8re, x8p11);
2303 let t_a1_9 = f64x2_mul(self.twiddle9re, x9p10);
2304 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p18);
2305 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p17);
2306 let t_a2_3 = f64x2_mul(self.twiddle6re, x3p16);
2307 let t_a2_4 = f64x2_mul(self.twiddle8re, x4p15);
2308 let t_a2_5 = f64x2_mul(self.twiddle9re, x5p14);
2309 let t_a2_6 = f64x2_mul(self.twiddle7re, x6p13);
2310 let t_a2_7 = f64x2_mul(self.twiddle5re, x7p12);
2311 let t_a2_8 = f64x2_mul(self.twiddle3re, x8p11);
2312 let t_a2_9 = f64x2_mul(self.twiddle1re, x9p10);
2313 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p18);
2314 let t_a3_2 = f64x2_mul(self.twiddle6re, x2p17);
2315 let t_a3_3 = f64x2_mul(self.twiddle9re, x3p16);
2316 let t_a3_4 = f64x2_mul(self.twiddle7re, x4p15);
2317 let t_a3_5 = f64x2_mul(self.twiddle4re, x5p14);
2318 let t_a3_6 = f64x2_mul(self.twiddle1re, x6p13);
2319 let t_a3_7 = f64x2_mul(self.twiddle2re, x7p12);
2320 let t_a3_8 = f64x2_mul(self.twiddle5re, x8p11);
2321 let t_a3_9 = f64x2_mul(self.twiddle8re, x9p10);
2322 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p18);
2323 let t_a4_2 = f64x2_mul(self.twiddle8re, x2p17);
2324 let t_a4_3 = f64x2_mul(self.twiddle7re, x3p16);
2325 let t_a4_4 = f64x2_mul(self.twiddle3re, x4p15);
2326 let t_a4_5 = f64x2_mul(self.twiddle1re, x5p14);
2327 let t_a4_6 = f64x2_mul(self.twiddle5re, x6p13);
2328 let t_a4_7 = f64x2_mul(self.twiddle9re, x7p12);
2329 let t_a4_8 = f64x2_mul(self.twiddle6re, x8p11);
2330 let t_a4_9 = f64x2_mul(self.twiddle2re, x9p10);
2331 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p18);
2332 let t_a5_2 = f64x2_mul(self.twiddle9re, x2p17);
2333 let t_a5_3 = f64x2_mul(self.twiddle4re, x3p16);
2334 let t_a5_4 = f64x2_mul(self.twiddle1re, x4p15);
2335 let t_a5_5 = f64x2_mul(self.twiddle6re, x5p14);
2336 let t_a5_6 = f64x2_mul(self.twiddle8re, x6p13);
2337 let t_a5_7 = f64x2_mul(self.twiddle3re, x7p12);
2338 let t_a5_8 = f64x2_mul(self.twiddle2re, x8p11);
2339 let t_a5_9 = f64x2_mul(self.twiddle7re, x9p10);
2340 let t_a6_1 = f64x2_mul(self.twiddle6re, x1p18);
2341 let t_a6_2 = f64x2_mul(self.twiddle7re, x2p17);
2342 let t_a6_3 = f64x2_mul(self.twiddle1re, x3p16);
2343 let t_a6_4 = f64x2_mul(self.twiddle5re, x4p15);
2344 let t_a6_5 = f64x2_mul(self.twiddle8re, x5p14);
2345 let t_a6_6 = f64x2_mul(self.twiddle2re, x6p13);
2346 let t_a6_7 = f64x2_mul(self.twiddle4re, x7p12);
2347 let t_a6_8 = f64x2_mul(self.twiddle9re, x8p11);
2348 let t_a6_9 = f64x2_mul(self.twiddle3re, x9p10);
2349 let t_a7_1 = f64x2_mul(self.twiddle7re, x1p18);
2350 let t_a7_2 = f64x2_mul(self.twiddle5re, x2p17);
2351 let t_a7_3 = f64x2_mul(self.twiddle2re, x3p16);
2352 let t_a7_4 = f64x2_mul(self.twiddle9re, x4p15);
2353 let t_a7_5 = f64x2_mul(self.twiddle3re, x5p14);
2354 let t_a7_6 = f64x2_mul(self.twiddle4re, x6p13);
2355 let t_a7_7 = f64x2_mul(self.twiddle8re, x7p12);
2356 let t_a7_8 = f64x2_mul(self.twiddle1re, x8p11);
2357 let t_a7_9 = f64x2_mul(self.twiddle6re, x9p10);
2358 let t_a8_1 = f64x2_mul(self.twiddle8re, x1p18);
2359 let t_a8_2 = f64x2_mul(self.twiddle3re, x2p17);
2360 let t_a8_3 = f64x2_mul(self.twiddle5re, x3p16);
2361 let t_a8_4 = f64x2_mul(self.twiddle6re, x4p15);
2362 let t_a8_5 = f64x2_mul(self.twiddle2re, x5p14);
2363 let t_a8_6 = f64x2_mul(self.twiddle9re, x6p13);
2364 let t_a8_7 = f64x2_mul(self.twiddle1re, x7p12);
2365 let t_a8_8 = f64x2_mul(self.twiddle7re, x8p11);
2366 let t_a8_9 = f64x2_mul(self.twiddle4re, x9p10);
2367 let t_a9_1 = f64x2_mul(self.twiddle9re, x1p18);
2368 let t_a9_2 = f64x2_mul(self.twiddle1re, x2p17);
2369 let t_a9_3 = f64x2_mul(self.twiddle8re, x3p16);
2370 let t_a9_4 = f64x2_mul(self.twiddle2re, x4p15);
2371 let t_a9_5 = f64x2_mul(self.twiddle7re, x5p14);
2372 let t_a9_6 = f64x2_mul(self.twiddle3re, x6p13);
2373 let t_a9_7 = f64x2_mul(self.twiddle6re, x7p12);
2374 let t_a9_8 = f64x2_mul(self.twiddle4re, x8p11);
2375 let t_a9_9 = f64x2_mul(self.twiddle5re, x9p10);
2376
2377 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m18);
2378 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m17);
2379 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m16);
2380 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m15);
2381 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m14);
2382 let t_b1_6 = f64x2_mul(self.twiddle6im, x6m13);
2383 let t_b1_7 = f64x2_mul(self.twiddle7im, x7m12);
2384 let t_b1_8 = f64x2_mul(self.twiddle8im, x8m11);
2385 let t_b1_9 = f64x2_mul(self.twiddle9im, x9m10);
2386 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m18);
2387 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m17);
2388 let t_b2_3 = f64x2_mul(self.twiddle6im, x3m16);
2389 let t_b2_4 = f64x2_mul(self.twiddle8im, x4m15);
2390 let t_b2_5 = f64x2_mul(self.twiddle9im, x5m14);
2391 let t_b2_6 = f64x2_mul(self.twiddle7im, x6m13);
2392 let t_b2_7 = f64x2_mul(self.twiddle5im, x7m12);
2393 let t_b2_8 = f64x2_mul(self.twiddle3im, x8m11);
2394 let t_b2_9 = f64x2_mul(self.twiddle1im, x9m10);
2395 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m18);
2396 let t_b3_2 = f64x2_mul(self.twiddle6im, x2m17);
2397 let t_b3_3 = f64x2_mul(self.twiddle9im, x3m16);
2398 let t_b3_4 = f64x2_mul(self.twiddle7im, x4m15);
2399 let t_b3_5 = f64x2_mul(self.twiddle4im, x5m14);
2400 let t_b3_6 = f64x2_mul(self.twiddle1im, x6m13);
2401 let t_b3_7 = f64x2_mul(self.twiddle2im, x7m12);
2402 let t_b3_8 = f64x2_mul(self.twiddle5im, x8m11);
2403 let t_b3_9 = f64x2_mul(self.twiddle8im, x9m10);
2404 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m18);
2405 let t_b4_2 = f64x2_mul(self.twiddle8im, x2m17);
2406 let t_b4_3 = f64x2_mul(self.twiddle7im, x3m16);
2407 let t_b4_4 = f64x2_mul(self.twiddle3im, x4m15);
2408 let t_b4_5 = f64x2_mul(self.twiddle1im, x5m14);
2409 let t_b4_6 = f64x2_mul(self.twiddle5im, x6m13);
2410 let t_b4_7 = f64x2_mul(self.twiddle9im, x7m12);
2411 let t_b4_8 = f64x2_mul(self.twiddle6im, x8m11);
2412 let t_b4_9 = f64x2_mul(self.twiddle2im, x9m10);
2413 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m18);
2414 let t_b5_2 = f64x2_mul(self.twiddle9im, x2m17);
2415 let t_b5_3 = f64x2_mul(self.twiddle4im, x3m16);
2416 let t_b5_4 = f64x2_mul(self.twiddle1im, x4m15);
2417 let t_b5_5 = f64x2_mul(self.twiddle6im, x5m14);
2418 let t_b5_6 = f64x2_mul(self.twiddle8im, x6m13);
2419 let t_b5_7 = f64x2_mul(self.twiddle3im, x7m12);
2420 let t_b5_8 = f64x2_mul(self.twiddle2im, x8m11);
2421 let t_b5_9 = f64x2_mul(self.twiddle7im, x9m10);
2422 let t_b6_1 = f64x2_mul(self.twiddle6im, x1m18);
2423 let t_b6_2 = f64x2_mul(self.twiddle7im, x2m17);
2424 let t_b6_3 = f64x2_mul(self.twiddle1im, x3m16);
2425 let t_b6_4 = f64x2_mul(self.twiddle5im, x4m15);
2426 let t_b6_5 = f64x2_mul(self.twiddle8im, x5m14);
2427 let t_b6_6 = f64x2_mul(self.twiddle2im, x6m13);
2428 let t_b6_7 = f64x2_mul(self.twiddle4im, x7m12);
2429 let t_b6_8 = f64x2_mul(self.twiddle9im, x8m11);
2430 let t_b6_9 = f64x2_mul(self.twiddle3im, x9m10);
2431 let t_b7_1 = f64x2_mul(self.twiddle7im, x1m18);
2432 let t_b7_2 = f64x2_mul(self.twiddle5im, x2m17);
2433 let t_b7_3 = f64x2_mul(self.twiddle2im, x3m16);
2434 let t_b7_4 = f64x2_mul(self.twiddle9im, x4m15);
2435 let t_b7_5 = f64x2_mul(self.twiddle3im, x5m14);
2436 let t_b7_6 = f64x2_mul(self.twiddle4im, x6m13);
2437 let t_b7_7 = f64x2_mul(self.twiddle8im, x7m12);
2438 let t_b7_8 = f64x2_mul(self.twiddle1im, x8m11);
2439 let t_b7_9 = f64x2_mul(self.twiddle6im, x9m10);
2440 let t_b8_1 = f64x2_mul(self.twiddle8im, x1m18);
2441 let t_b8_2 = f64x2_mul(self.twiddle3im, x2m17);
2442 let t_b8_3 = f64x2_mul(self.twiddle5im, x3m16);
2443 let t_b8_4 = f64x2_mul(self.twiddle6im, x4m15);
2444 let t_b8_5 = f64x2_mul(self.twiddle2im, x5m14);
2445 let t_b8_6 = f64x2_mul(self.twiddle9im, x6m13);
2446 let t_b8_7 = f64x2_mul(self.twiddle1im, x7m12);
2447 let t_b8_8 = f64x2_mul(self.twiddle7im, x8m11);
2448 let t_b8_9 = f64x2_mul(self.twiddle4im, x9m10);
2449 let t_b9_1 = f64x2_mul(self.twiddle9im, x1m18);
2450 let t_b9_2 = f64x2_mul(self.twiddle1im, x2m17);
2451 let t_b9_3 = f64x2_mul(self.twiddle8im, x3m16);
2452 let t_b9_4 = f64x2_mul(self.twiddle2im, x4m15);
2453 let t_b9_5 = f64x2_mul(self.twiddle7im, x5m14);
2454 let t_b9_6 = f64x2_mul(self.twiddle3im, x6m13);
2455 let t_b9_7 = f64x2_mul(self.twiddle6im, x7m12);
2456 let t_b9_8 = f64x2_mul(self.twiddle4im, x8m11);
2457 let t_b9_9 = f64x2_mul(self.twiddle5im, x9m10);
2458
2459 let x0 = values[0];
2460 let t_a1 = calc_f64!(
2461 x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9
2462 );
2463 let t_a2 = calc_f64!(
2464 x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9
2465 );
2466 let t_a3 = calc_f64!(
2467 x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9
2468 );
2469 let t_a4 = calc_f64!(
2470 x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9
2471 );
2472 let t_a5 = calc_f64!(
2473 x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9
2474 );
2475 let t_a6 = calc_f64!(
2476 x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9
2477 );
2478 let t_a7 = calc_f64!(
2479 x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9
2480 );
2481 let t_a8 = calc_f64!(
2482 x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9
2483 );
2484 let t_a9 = calc_f64!(
2485 x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9
2486 );
2487
2488 let t_b1 = calc_f64!(
2489 t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9
2490 );
2491 let t_b2 = calc_f64!(
2492 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8 - t_b2_9
2493 );
2494 let t_b3 = calc_f64!(
2495 t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 + t_b3_7 + t_b3_8 + t_b3_9
2496 );
2497 let t_b4 = calc_f64!(
2498 t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 + t_b4_7 - t_b4_8 - t_b4_9
2499 );
2500 let t_b5 = calc_f64!(
2501 t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 - t_b5_7 + t_b5_8 + t_b5_9
2502 );
2503 let t_b6 = calc_f64!(
2504 t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 - t_b6_5 - t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9
2505 );
2506 let t_b7 = calc_f64!(
2507 t_b7_1 - t_b7_2 + t_b7_3 + t_b7_4 - t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9
2508 );
2509 let t_b8 = calc_f64!(
2510 t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 - t_b8_9
2511 );
2512 let t_b9 = calc_f64!(
2513 t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 - t_b9_8 + t_b9_9
2514 );
2515
2516 let t_b1_rot = self.rotate.rotate(t_b1);
2517 let t_b2_rot = self.rotate.rotate(t_b2);
2518 let t_b3_rot = self.rotate.rotate(t_b3);
2519 let t_b4_rot = self.rotate.rotate(t_b4);
2520 let t_b5_rot = self.rotate.rotate(t_b5);
2521 let t_b6_rot = self.rotate.rotate(t_b6);
2522 let t_b7_rot = self.rotate.rotate(t_b7);
2523 let t_b8_rot = self.rotate.rotate(t_b8);
2524 let t_b9_rot = self.rotate.rotate(t_b9);
2525
2526 let y0 =
2527 calc_f64!(x0 + x1p18 + x2p17 + x3p16 + x4p15 + x5p14 + x6p13 + x7p12 + x8p11 + x9p10);
2528 let [y1, y18] = solo_fft2_f64(t_a1, t_b1_rot);
2529 let [y2, y17] = solo_fft2_f64(t_a2, t_b2_rot);
2530 let [y3, y16] = solo_fft2_f64(t_a3, t_b3_rot);
2531 let [y4, y15] = solo_fft2_f64(t_a4, t_b4_rot);
2532 let [y5, y14] = solo_fft2_f64(t_a5, t_b5_rot);
2533 let [y6, y13] = solo_fft2_f64(t_a6, t_b6_rot);
2534 let [y7, y12] = solo_fft2_f64(t_a7, t_b7_rot);
2535 let [y8, y11] = solo_fft2_f64(t_a8, t_b8_rot);
2536 let [y9, y10] = solo_fft2_f64(t_a9, t_b9_rot);
2537 [
2538 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
2539 ]
2540 }
2541 }
2542
2543 // ____ _____ _________ _ _ _
2544 // |___ \|___ / |___ /___ \| |__ (_) |_
2545 // __) | |_ \ _____ |_ \ __) | '_ \| | __|
2546 // / __/ ___) | |_____| ___) / __/| |_) | | |_
2547 // |_____|____/ |____/_____|_.__/|_|\__|
2548 //
2549
2550 pub struct WasmSimdF32Butterfly23<T> {
2551 direction: FftDirection,
2552 _phantom: std::marker::PhantomData<T>,
2553 rotate: Rotate90F32,
2554 twiddle1re: v128,
2555 twiddle1im: v128,
2556 twiddle2re: v128,
2557 twiddle2im: v128,
2558 twiddle3re: v128,
2559 twiddle3im: v128,
2560 twiddle4re: v128,
2561 twiddle4im: v128,
2562 twiddle5re: v128,
2563 twiddle5im: v128,
2564 twiddle6re: v128,
2565 twiddle6im: v128,
2566 twiddle7re: v128,
2567 twiddle7im: v128,
2568 twiddle8re: v128,
2569 twiddle8im: v128,
2570 twiddle9re: v128,
2571 twiddle9im: v128,
2572 twiddle10re: v128,
2573 twiddle10im: v128,
2574 twiddle11re: v128,
2575 twiddle11im: v128,
2576 }
2577
2578 boilerplate_fft_wasm_simd_f32_butterfly!(
2579 WasmSimdF32Butterfly23,
2580 23,
2581 |this: &WasmSimdF32Butterfly23<_>| this.direction
2582 );
2583 boilerplate_fft_wasm_simd_common_butterfly!(
2584 WasmSimdF32Butterfly23,
2585 23,
2586 |this: &WasmSimdF32Butterfly23<_>| this.direction
2587 );
2588 impl<T: FftNum> WasmSimdF32Butterfly23<T> {
2589 #[inline(always)]
2590 pub fn new(direction: FftDirection) -> Self {
2591 assert_f32::<T>();
2592 let rotate = Rotate90F32::new(true);
2593 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 23, direction);
2594 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 23, direction);
2595 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 23, direction);
2596 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 23, direction);
2597 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 23, direction);
2598 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 23, direction);
2599 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 23, direction);
2600 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 23, direction);
2601 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 23, direction);
2602 let tw10: Complex<f32> = twiddles::compute_twiddle(10, 23, direction);
2603 let tw11: Complex<f32> = twiddles::compute_twiddle(11, 23, direction);
2604 let twiddle1re = f32x4_splat(tw1.re);
2605 let twiddle1im = f32x4_splat(tw1.im);
2606 let twiddle2re = f32x4_splat(tw2.re);
2607 let twiddle2im = f32x4_splat(tw2.im);
2608 let twiddle3re = f32x4_splat(tw3.re);
2609 let twiddle3im = f32x4_splat(tw3.im);
2610 let twiddle4re = f32x4_splat(tw4.re);
2611 let twiddle4im = f32x4_splat(tw4.im);
2612 let twiddle5re = f32x4_splat(tw5.re);
2613 let twiddle5im = f32x4_splat(tw5.im);
2614 let twiddle6re = f32x4_splat(tw6.re);
2615 let twiddle6im = f32x4_splat(tw6.im);
2616 let twiddle7re = f32x4_splat(tw7.re);
2617 let twiddle7im = f32x4_splat(tw7.im);
2618 let twiddle8re = f32x4_splat(tw8.re);
2619 let twiddle8im = f32x4_splat(tw8.im);
2620 let twiddle9re = f32x4_splat(tw9.re);
2621 let twiddle9im = f32x4_splat(tw9.im);
2622 let twiddle10re = f32x4_splat(tw10.re);
2623 let twiddle10im = f32x4_splat(tw10.im);
2624 let twiddle11re = f32x4_splat(tw11.re);
2625 let twiddle11im = f32x4_splat(tw11.im);
2626
2627 Self {
2628 direction,
2629 _phantom: std::marker::PhantomData,
2630 rotate,
2631 twiddle1re,
2632 twiddle1im,
2633 twiddle2re,
2634 twiddle2im,
2635 twiddle3re,
2636 twiddle3im,
2637 twiddle4re,
2638 twiddle4im,
2639 twiddle5re,
2640 twiddle5im,
2641 twiddle6re,
2642 twiddle6im,
2643 twiddle7re,
2644 twiddle7im,
2645 twiddle8re,
2646 twiddle8im,
2647 twiddle9re,
2648 twiddle9im,
2649 twiddle10re,
2650 twiddle10im,
2651 twiddle11re,
2652 twiddle11im,
2653 }
2654 }
2655
2656 #[inline(always)]
2657 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
2658 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2659
2660 let out = self.perform_parallel_fft_direct(values);
2661
2662 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2663 }
2664
2665 #[inline(always)]
2666 pub(crate) unsafe fn perform_parallel_fft_contiguous(
2667 &self,
2668 mut buffer: impl WasmSimdArrayMut<f32>,
2669 ) {
2670 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44});
2671
2672 let values = [
2673 extract_lo_hi_f32(input_packed[0], input_packed[11]),
2674 extract_hi_lo_f32(input_packed[0], input_packed[12]),
2675 extract_lo_hi_f32(input_packed[1], input_packed[12]),
2676 extract_hi_lo_f32(input_packed[1], input_packed[13]),
2677 extract_lo_hi_f32(input_packed[2], input_packed[13]),
2678 extract_hi_lo_f32(input_packed[2], input_packed[14]),
2679 extract_lo_hi_f32(input_packed[3], input_packed[14]),
2680 extract_hi_lo_f32(input_packed[3], input_packed[15]),
2681 extract_lo_hi_f32(input_packed[4], input_packed[15]),
2682 extract_hi_lo_f32(input_packed[4], input_packed[16]),
2683 extract_lo_hi_f32(input_packed[5], input_packed[16]),
2684 extract_hi_lo_f32(input_packed[5], input_packed[17]),
2685 extract_lo_hi_f32(input_packed[6], input_packed[17]),
2686 extract_hi_lo_f32(input_packed[6], input_packed[18]),
2687 extract_lo_hi_f32(input_packed[7], input_packed[18]),
2688 extract_hi_lo_f32(input_packed[7], input_packed[19]),
2689 extract_lo_hi_f32(input_packed[8], input_packed[19]),
2690 extract_hi_lo_f32(input_packed[8], input_packed[20]),
2691 extract_lo_hi_f32(input_packed[9], input_packed[20]),
2692 extract_hi_lo_f32(input_packed[9], input_packed[21]),
2693 extract_lo_hi_f32(input_packed[10], input_packed[21]),
2694 extract_hi_lo_f32(input_packed[10], input_packed[22]),
2695 extract_lo_hi_f32(input_packed[11], input_packed[22]),
2696 ];
2697
2698 let out = self.perform_parallel_fft_direct(values);
2699
2700 let out_packed = [
2701 extract_lo_lo_f32(out[0], out[1]),
2702 extract_lo_lo_f32(out[2], out[3]),
2703 extract_lo_lo_f32(out[4], out[5]),
2704 extract_lo_lo_f32(out[6], out[7]),
2705 extract_lo_lo_f32(out[8], out[9]),
2706 extract_lo_lo_f32(out[10], out[11]),
2707 extract_lo_lo_f32(out[12], out[13]),
2708 extract_lo_lo_f32(out[14], out[15]),
2709 extract_lo_lo_f32(out[16], out[17]),
2710 extract_lo_lo_f32(out[18], out[19]),
2711 extract_lo_lo_f32(out[20], out[21]),
2712 extract_lo_hi_f32(out[22], out[0]),
2713 extract_hi_hi_f32(out[1], out[2]),
2714 extract_hi_hi_f32(out[3], out[4]),
2715 extract_hi_hi_f32(out[5], out[6]),
2716 extract_hi_hi_f32(out[7], out[8]),
2717 extract_hi_hi_f32(out[9], out[10]),
2718 extract_hi_hi_f32(out[11], out[12]),
2719 extract_hi_hi_f32(out[13], out[14]),
2720 extract_hi_hi_f32(out[15], out[16]),
2721 extract_hi_hi_f32(out[17], out[18]),
2722 extract_hi_hi_f32(out[19], out[20]),
2723 extract_hi_hi_f32(out[21], out[22]),
2724 ];
2725
2726 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2727 }
2728
2729 #[inline(always)]
2730 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 23]) -> [v128; 23] {
2731 let [x1p22, x1m22] = parallel_fft2_interleaved_f32(values[1], values[22]);
2732 let [x2p21, x2m21] = parallel_fft2_interleaved_f32(values[2], values[21]);
2733 let [x3p20, x3m20] = parallel_fft2_interleaved_f32(values[3], values[20]);
2734 let [x4p19, x4m19] = parallel_fft2_interleaved_f32(values[4], values[19]);
2735 let [x5p18, x5m18] = parallel_fft2_interleaved_f32(values[5], values[18]);
2736 let [x6p17, x6m17] = parallel_fft2_interleaved_f32(values[6], values[17]);
2737 let [x7p16, x7m16] = parallel_fft2_interleaved_f32(values[7], values[16]);
2738 let [x8p15, x8m15] = parallel_fft2_interleaved_f32(values[8], values[15]);
2739 let [x9p14, x9m14] = parallel_fft2_interleaved_f32(values[9], values[14]);
2740 let [x10p13, x10m13] = parallel_fft2_interleaved_f32(values[10], values[13]);
2741 let [x11p12, x11m12] = parallel_fft2_interleaved_f32(values[11], values[12]);
2742
2743 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p22);
2744 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p21);
2745 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p20);
2746 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p19);
2747 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p18);
2748 let t_a1_6 = f32x4_mul(self.twiddle6re, x6p17);
2749 let t_a1_7 = f32x4_mul(self.twiddle7re, x7p16);
2750 let t_a1_8 = f32x4_mul(self.twiddle8re, x8p15);
2751 let t_a1_9 = f32x4_mul(self.twiddle9re, x9p14);
2752 let t_a1_10 = f32x4_mul(self.twiddle10re, x10p13);
2753 let t_a1_11 = f32x4_mul(self.twiddle11re, x11p12);
2754 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p22);
2755 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p21);
2756 let t_a2_3 = f32x4_mul(self.twiddle6re, x3p20);
2757 let t_a2_4 = f32x4_mul(self.twiddle8re, x4p19);
2758 let t_a2_5 = f32x4_mul(self.twiddle10re, x5p18);
2759 let t_a2_6 = f32x4_mul(self.twiddle11re, x6p17);
2760 let t_a2_7 = f32x4_mul(self.twiddle9re, x7p16);
2761 let t_a2_8 = f32x4_mul(self.twiddle7re, x8p15);
2762 let t_a2_9 = f32x4_mul(self.twiddle5re, x9p14);
2763 let t_a2_10 = f32x4_mul(self.twiddle3re, x10p13);
2764 let t_a2_11 = f32x4_mul(self.twiddle1re, x11p12);
2765 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p22);
2766 let t_a3_2 = f32x4_mul(self.twiddle6re, x2p21);
2767 let t_a3_3 = f32x4_mul(self.twiddle9re, x3p20);
2768 let t_a3_4 = f32x4_mul(self.twiddle11re, x4p19);
2769 let t_a3_5 = f32x4_mul(self.twiddle8re, x5p18);
2770 let t_a3_6 = f32x4_mul(self.twiddle5re, x6p17);
2771 let t_a3_7 = f32x4_mul(self.twiddle2re, x7p16);
2772 let t_a3_8 = f32x4_mul(self.twiddle1re, x8p15);
2773 let t_a3_9 = f32x4_mul(self.twiddle4re, x9p14);
2774 let t_a3_10 = f32x4_mul(self.twiddle7re, x10p13);
2775 let t_a3_11 = f32x4_mul(self.twiddle10re, x11p12);
2776 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p22);
2777 let t_a4_2 = f32x4_mul(self.twiddle8re, x2p21);
2778 let t_a4_3 = f32x4_mul(self.twiddle11re, x3p20);
2779 let t_a4_4 = f32x4_mul(self.twiddle7re, x4p19);
2780 let t_a4_5 = f32x4_mul(self.twiddle3re, x5p18);
2781 let t_a4_6 = f32x4_mul(self.twiddle1re, x6p17);
2782 let t_a4_7 = f32x4_mul(self.twiddle5re, x7p16);
2783 let t_a4_8 = f32x4_mul(self.twiddle9re, x8p15);
2784 let t_a4_9 = f32x4_mul(self.twiddle10re, x9p14);
2785 let t_a4_10 = f32x4_mul(self.twiddle6re, x10p13);
2786 let t_a4_11 = f32x4_mul(self.twiddle2re, x11p12);
2787 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p22);
2788 let t_a5_2 = f32x4_mul(self.twiddle10re, x2p21);
2789 let t_a5_3 = f32x4_mul(self.twiddle8re, x3p20);
2790 let t_a5_4 = f32x4_mul(self.twiddle3re, x4p19);
2791 let t_a5_5 = f32x4_mul(self.twiddle2re, x5p18);
2792 let t_a5_6 = f32x4_mul(self.twiddle7re, x6p17);
2793 let t_a5_7 = f32x4_mul(self.twiddle11re, x7p16);
2794 let t_a5_8 = f32x4_mul(self.twiddle6re, x8p15);
2795 let t_a5_9 = f32x4_mul(self.twiddle1re, x9p14);
2796 let t_a5_10 = f32x4_mul(self.twiddle4re, x10p13);
2797 let t_a5_11 = f32x4_mul(self.twiddle9re, x11p12);
2798 let t_a6_1 = f32x4_mul(self.twiddle6re, x1p22);
2799 let t_a6_2 = f32x4_mul(self.twiddle11re, x2p21);
2800 let t_a6_3 = f32x4_mul(self.twiddle5re, x3p20);
2801 let t_a6_4 = f32x4_mul(self.twiddle1re, x4p19);
2802 let t_a6_5 = f32x4_mul(self.twiddle7re, x5p18);
2803 let t_a6_6 = f32x4_mul(self.twiddle10re, x6p17);
2804 let t_a6_7 = f32x4_mul(self.twiddle4re, x7p16);
2805 let t_a6_8 = f32x4_mul(self.twiddle2re, x8p15);
2806 let t_a6_9 = f32x4_mul(self.twiddle8re, x9p14);
2807 let t_a6_10 = f32x4_mul(self.twiddle9re, x10p13);
2808 let t_a6_11 = f32x4_mul(self.twiddle3re, x11p12);
2809 let t_a7_1 = f32x4_mul(self.twiddle7re, x1p22);
2810 let t_a7_2 = f32x4_mul(self.twiddle9re, x2p21);
2811 let t_a7_3 = f32x4_mul(self.twiddle2re, x3p20);
2812 let t_a7_4 = f32x4_mul(self.twiddle5re, x4p19);
2813 let t_a7_5 = f32x4_mul(self.twiddle11re, x5p18);
2814 let t_a7_6 = f32x4_mul(self.twiddle4re, x6p17);
2815 let t_a7_7 = f32x4_mul(self.twiddle3re, x7p16);
2816 let t_a7_8 = f32x4_mul(self.twiddle10re, x8p15);
2817 let t_a7_9 = f32x4_mul(self.twiddle6re, x9p14);
2818 let t_a7_10 = f32x4_mul(self.twiddle1re, x10p13);
2819 let t_a7_11 = f32x4_mul(self.twiddle8re, x11p12);
2820 let t_a8_1 = f32x4_mul(self.twiddle8re, x1p22);
2821 let t_a8_2 = f32x4_mul(self.twiddle7re, x2p21);
2822 let t_a8_3 = f32x4_mul(self.twiddle1re, x3p20);
2823 let t_a8_4 = f32x4_mul(self.twiddle9re, x4p19);
2824 let t_a8_5 = f32x4_mul(self.twiddle6re, x5p18);
2825 let t_a8_6 = f32x4_mul(self.twiddle2re, x6p17);
2826 let t_a8_7 = f32x4_mul(self.twiddle10re, x7p16);
2827 let t_a8_8 = f32x4_mul(self.twiddle5re, x8p15);
2828 let t_a8_9 = f32x4_mul(self.twiddle3re, x9p14);
2829 let t_a8_10 = f32x4_mul(self.twiddle11re, x10p13);
2830 let t_a8_11 = f32x4_mul(self.twiddle4re, x11p12);
2831 let t_a9_1 = f32x4_mul(self.twiddle9re, x1p22);
2832 let t_a9_2 = f32x4_mul(self.twiddle5re, x2p21);
2833 let t_a9_3 = f32x4_mul(self.twiddle4re, x3p20);
2834 let t_a9_4 = f32x4_mul(self.twiddle10re, x4p19);
2835 let t_a9_5 = f32x4_mul(self.twiddle1re, x5p18);
2836 let t_a9_6 = f32x4_mul(self.twiddle8re, x6p17);
2837 let t_a9_7 = f32x4_mul(self.twiddle6re, x7p16);
2838 let t_a9_8 = f32x4_mul(self.twiddle3re, x8p15);
2839 let t_a9_9 = f32x4_mul(self.twiddle11re, x9p14);
2840 let t_a9_10 = f32x4_mul(self.twiddle2re, x10p13);
2841 let t_a9_11 = f32x4_mul(self.twiddle7re, x11p12);
2842 let t_a10_1 = f32x4_mul(self.twiddle10re, x1p22);
2843 let t_a10_2 = f32x4_mul(self.twiddle3re, x2p21);
2844 let t_a10_3 = f32x4_mul(self.twiddle7re, x3p20);
2845 let t_a10_4 = f32x4_mul(self.twiddle6re, x4p19);
2846 let t_a10_5 = f32x4_mul(self.twiddle4re, x5p18);
2847 let t_a10_6 = f32x4_mul(self.twiddle9re, x6p17);
2848 let t_a10_7 = f32x4_mul(self.twiddle1re, x7p16);
2849 let t_a10_8 = f32x4_mul(self.twiddle11re, x8p15);
2850 let t_a10_9 = f32x4_mul(self.twiddle2re, x9p14);
2851 let t_a10_10 = f32x4_mul(self.twiddle8re, x10p13);
2852 let t_a10_11 = f32x4_mul(self.twiddle5re, x11p12);
2853 let t_a11_1 = f32x4_mul(self.twiddle11re, x1p22);
2854 let t_a11_2 = f32x4_mul(self.twiddle1re, x2p21);
2855 let t_a11_3 = f32x4_mul(self.twiddle10re, x3p20);
2856 let t_a11_4 = f32x4_mul(self.twiddle2re, x4p19);
2857 let t_a11_5 = f32x4_mul(self.twiddle9re, x5p18);
2858 let t_a11_6 = f32x4_mul(self.twiddle3re, x6p17);
2859 let t_a11_7 = f32x4_mul(self.twiddle8re, x7p16);
2860 let t_a11_8 = f32x4_mul(self.twiddle4re, x8p15);
2861 let t_a11_9 = f32x4_mul(self.twiddle7re, x9p14);
2862 let t_a11_10 = f32x4_mul(self.twiddle5re, x10p13);
2863 let t_a11_11 = f32x4_mul(self.twiddle6re, x11p12);
2864
2865 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m22);
2866 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m21);
2867 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m20);
2868 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m19);
2869 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m18);
2870 let t_b1_6 = f32x4_mul(self.twiddle6im, x6m17);
2871 let t_b1_7 = f32x4_mul(self.twiddle7im, x7m16);
2872 let t_b1_8 = f32x4_mul(self.twiddle8im, x8m15);
2873 let t_b1_9 = f32x4_mul(self.twiddle9im, x9m14);
2874 let t_b1_10 = f32x4_mul(self.twiddle10im, x10m13);
2875 let t_b1_11 = f32x4_mul(self.twiddle11im, x11m12);
2876 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m22);
2877 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m21);
2878 let t_b2_3 = f32x4_mul(self.twiddle6im, x3m20);
2879 let t_b2_4 = f32x4_mul(self.twiddle8im, x4m19);
2880 let t_b2_5 = f32x4_mul(self.twiddle10im, x5m18);
2881 let t_b2_6 = f32x4_mul(self.twiddle11im, x6m17);
2882 let t_b2_7 = f32x4_mul(self.twiddle9im, x7m16);
2883 let t_b2_8 = f32x4_mul(self.twiddle7im, x8m15);
2884 let t_b2_9 = f32x4_mul(self.twiddle5im, x9m14);
2885 let t_b2_10 = f32x4_mul(self.twiddle3im, x10m13);
2886 let t_b2_11 = f32x4_mul(self.twiddle1im, x11m12);
2887 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m22);
2888 let t_b3_2 = f32x4_mul(self.twiddle6im, x2m21);
2889 let t_b3_3 = f32x4_mul(self.twiddle9im, x3m20);
2890 let t_b3_4 = f32x4_mul(self.twiddle11im, x4m19);
2891 let t_b3_5 = f32x4_mul(self.twiddle8im, x5m18);
2892 let t_b3_6 = f32x4_mul(self.twiddle5im, x6m17);
2893 let t_b3_7 = f32x4_mul(self.twiddle2im, x7m16);
2894 let t_b3_8 = f32x4_mul(self.twiddle1im, x8m15);
2895 let t_b3_9 = f32x4_mul(self.twiddle4im, x9m14);
2896 let t_b3_10 = f32x4_mul(self.twiddle7im, x10m13);
2897 let t_b3_11 = f32x4_mul(self.twiddle10im, x11m12);
2898 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m22);
2899 let t_b4_2 = f32x4_mul(self.twiddle8im, x2m21);
2900 let t_b4_3 = f32x4_mul(self.twiddle11im, x3m20);
2901 let t_b4_4 = f32x4_mul(self.twiddle7im, x4m19);
2902 let t_b4_5 = f32x4_mul(self.twiddle3im, x5m18);
2903 let t_b4_6 = f32x4_mul(self.twiddle1im, x6m17);
2904 let t_b4_7 = f32x4_mul(self.twiddle5im, x7m16);
2905 let t_b4_8 = f32x4_mul(self.twiddle9im, x8m15);
2906 let t_b4_9 = f32x4_mul(self.twiddle10im, x9m14);
2907 let t_b4_10 = f32x4_mul(self.twiddle6im, x10m13);
2908 let t_b4_11 = f32x4_mul(self.twiddle2im, x11m12);
2909 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m22);
2910 let t_b5_2 = f32x4_mul(self.twiddle10im, x2m21);
2911 let t_b5_3 = f32x4_mul(self.twiddle8im, x3m20);
2912 let t_b5_4 = f32x4_mul(self.twiddle3im, x4m19);
2913 let t_b5_5 = f32x4_mul(self.twiddle2im, x5m18);
2914 let t_b5_6 = f32x4_mul(self.twiddle7im, x6m17);
2915 let t_b5_7 = f32x4_mul(self.twiddle11im, x7m16);
2916 let t_b5_8 = f32x4_mul(self.twiddle6im, x8m15);
2917 let t_b5_9 = f32x4_mul(self.twiddle1im, x9m14);
2918 let t_b5_10 = f32x4_mul(self.twiddle4im, x10m13);
2919 let t_b5_11 = f32x4_mul(self.twiddle9im, x11m12);
2920 let t_b6_1 = f32x4_mul(self.twiddle6im, x1m22);
2921 let t_b6_2 = f32x4_mul(self.twiddle11im, x2m21);
2922 let t_b6_3 = f32x4_mul(self.twiddle5im, x3m20);
2923 let t_b6_4 = f32x4_mul(self.twiddle1im, x4m19);
2924 let t_b6_5 = f32x4_mul(self.twiddle7im, x5m18);
2925 let t_b6_6 = f32x4_mul(self.twiddle10im, x6m17);
2926 let t_b6_7 = f32x4_mul(self.twiddle4im, x7m16);
2927 let t_b6_8 = f32x4_mul(self.twiddle2im, x8m15);
2928 let t_b6_9 = f32x4_mul(self.twiddle8im, x9m14);
2929 let t_b6_10 = f32x4_mul(self.twiddle9im, x10m13);
2930 let t_b6_11 = f32x4_mul(self.twiddle3im, x11m12);
2931 let t_b7_1 = f32x4_mul(self.twiddle7im, x1m22);
2932 let t_b7_2 = f32x4_mul(self.twiddle9im, x2m21);
2933 let t_b7_3 = f32x4_mul(self.twiddle2im, x3m20);
2934 let t_b7_4 = f32x4_mul(self.twiddle5im, x4m19);
2935 let t_b7_5 = f32x4_mul(self.twiddle11im, x5m18);
2936 let t_b7_6 = f32x4_mul(self.twiddle4im, x6m17);
2937 let t_b7_7 = f32x4_mul(self.twiddle3im, x7m16);
2938 let t_b7_8 = f32x4_mul(self.twiddle10im, x8m15);
2939 let t_b7_9 = f32x4_mul(self.twiddle6im, x9m14);
2940 let t_b7_10 = f32x4_mul(self.twiddle1im, x10m13);
2941 let t_b7_11 = f32x4_mul(self.twiddle8im, x11m12);
2942 let t_b8_1 = f32x4_mul(self.twiddle8im, x1m22);
2943 let t_b8_2 = f32x4_mul(self.twiddle7im, x2m21);
2944 let t_b8_3 = f32x4_mul(self.twiddle1im, x3m20);
2945 let t_b8_4 = f32x4_mul(self.twiddle9im, x4m19);
2946 let t_b8_5 = f32x4_mul(self.twiddle6im, x5m18);
2947 let t_b8_6 = f32x4_mul(self.twiddle2im, x6m17);
2948 let t_b8_7 = f32x4_mul(self.twiddle10im, x7m16);
2949 let t_b8_8 = f32x4_mul(self.twiddle5im, x8m15);
2950 let t_b8_9 = f32x4_mul(self.twiddle3im, x9m14);
2951 let t_b8_10 = f32x4_mul(self.twiddle11im, x10m13);
2952 let t_b8_11 = f32x4_mul(self.twiddle4im, x11m12);
2953 let t_b9_1 = f32x4_mul(self.twiddle9im, x1m22);
2954 let t_b9_2 = f32x4_mul(self.twiddle5im, x2m21);
2955 let t_b9_3 = f32x4_mul(self.twiddle4im, x3m20);
2956 let t_b9_4 = f32x4_mul(self.twiddle10im, x4m19);
2957 let t_b9_5 = f32x4_mul(self.twiddle1im, x5m18);
2958 let t_b9_6 = f32x4_mul(self.twiddle8im, x6m17);
2959 let t_b9_7 = f32x4_mul(self.twiddle6im, x7m16);
2960 let t_b9_8 = f32x4_mul(self.twiddle3im, x8m15);
2961 let t_b9_9 = f32x4_mul(self.twiddle11im, x9m14);
2962 let t_b9_10 = f32x4_mul(self.twiddle2im, x10m13);
2963 let t_b9_11 = f32x4_mul(self.twiddle7im, x11m12);
2964 let t_b10_1 = f32x4_mul(self.twiddle10im, x1m22);
2965 let t_b10_2 = f32x4_mul(self.twiddle3im, x2m21);
2966 let t_b10_3 = f32x4_mul(self.twiddle7im, x3m20);
2967 let t_b10_4 = f32x4_mul(self.twiddle6im, x4m19);
2968 let t_b10_5 = f32x4_mul(self.twiddle4im, x5m18);
2969 let t_b10_6 = f32x4_mul(self.twiddle9im, x6m17);
2970 let t_b10_7 = f32x4_mul(self.twiddle1im, x7m16);
2971 let t_b10_8 = f32x4_mul(self.twiddle11im, x8m15);
2972 let t_b10_9 = f32x4_mul(self.twiddle2im, x9m14);
2973 let t_b10_10 = f32x4_mul(self.twiddle8im, x10m13);
2974 let t_b10_11 = f32x4_mul(self.twiddle5im, x11m12);
2975 let t_b11_1 = f32x4_mul(self.twiddle11im, x1m22);
2976 let t_b11_2 = f32x4_mul(self.twiddle1im, x2m21);
2977 let t_b11_3 = f32x4_mul(self.twiddle10im, x3m20);
2978 let t_b11_4 = f32x4_mul(self.twiddle2im, x4m19);
2979 let t_b11_5 = f32x4_mul(self.twiddle9im, x5m18);
2980 let t_b11_6 = f32x4_mul(self.twiddle3im, x6m17);
2981 let t_b11_7 = f32x4_mul(self.twiddle8im, x7m16);
2982 let t_b11_8 = f32x4_mul(self.twiddle4im, x8m15);
2983 let t_b11_9 = f32x4_mul(self.twiddle7im, x9m14);
2984 let t_b11_10 = f32x4_mul(self.twiddle5im, x10m13);
2985 let t_b11_11 = f32x4_mul(self.twiddle6im, x11m12);
2986
2987 let x0 = values[0];
2988 let t_a1 = calc_f32!(
2989 x0 + t_a1_1
2990 + t_a1_2
2991 + t_a1_3
2992 + t_a1_4
2993 + t_a1_5
2994 + t_a1_6
2995 + t_a1_7
2996 + t_a1_8
2997 + t_a1_9
2998 + t_a1_10
2999 + t_a1_11
3000 );
3001 let t_a2 = calc_f32!(
3002 x0 + t_a2_1
3003 + t_a2_2
3004 + t_a2_3
3005 + t_a2_4
3006 + t_a2_5
3007 + t_a2_6
3008 + t_a2_7
3009 + t_a2_8
3010 + t_a2_9
3011 + t_a2_10
3012 + t_a2_11
3013 );
3014 let t_a3 = calc_f32!(
3015 x0 + t_a3_1
3016 + t_a3_2
3017 + t_a3_3
3018 + t_a3_4
3019 + t_a3_5
3020 + t_a3_6
3021 + t_a3_7
3022 + t_a3_8
3023 + t_a3_9
3024 + t_a3_10
3025 + t_a3_11
3026 );
3027 let t_a4 = calc_f32!(
3028 x0 + t_a4_1
3029 + t_a4_2
3030 + t_a4_3
3031 + t_a4_4
3032 + t_a4_5
3033 + t_a4_6
3034 + t_a4_7
3035 + t_a4_8
3036 + t_a4_9
3037 + t_a4_10
3038 + t_a4_11
3039 );
3040 let t_a5 = calc_f32!(
3041 x0 + t_a5_1
3042 + t_a5_2
3043 + t_a5_3
3044 + t_a5_4
3045 + t_a5_5
3046 + t_a5_6
3047 + t_a5_7
3048 + t_a5_8
3049 + t_a5_9
3050 + t_a5_10
3051 + t_a5_11
3052 );
3053 let t_a6 = calc_f32!(
3054 x0 + t_a6_1
3055 + t_a6_2
3056 + t_a6_3
3057 + t_a6_4
3058 + t_a6_5
3059 + t_a6_6
3060 + t_a6_7
3061 + t_a6_8
3062 + t_a6_9
3063 + t_a6_10
3064 + t_a6_11
3065 );
3066 let t_a7 = calc_f32!(
3067 x0 + t_a7_1
3068 + t_a7_2
3069 + t_a7_3
3070 + t_a7_4
3071 + t_a7_5
3072 + t_a7_6
3073 + t_a7_7
3074 + t_a7_8
3075 + t_a7_9
3076 + t_a7_10
3077 + t_a7_11
3078 );
3079 let t_a8 = calc_f32!(
3080 x0 + t_a8_1
3081 + t_a8_2
3082 + t_a8_3
3083 + t_a8_4
3084 + t_a8_5
3085 + t_a8_6
3086 + t_a8_7
3087 + t_a8_8
3088 + t_a8_9
3089 + t_a8_10
3090 + t_a8_11
3091 );
3092 let t_a9 = calc_f32!(
3093 x0 + t_a9_1
3094 + t_a9_2
3095 + t_a9_3
3096 + t_a9_4
3097 + t_a9_5
3098 + t_a9_6
3099 + t_a9_7
3100 + t_a9_8
3101 + t_a9_9
3102 + t_a9_10
3103 + t_a9_11
3104 );
3105 let t_a10 = calc_f32!(
3106 x0 + t_a10_1
3107 + t_a10_2
3108 + t_a10_3
3109 + t_a10_4
3110 + t_a10_5
3111 + t_a10_6
3112 + t_a10_7
3113 + t_a10_8
3114 + t_a10_9
3115 + t_a10_10
3116 + t_a10_11
3117 );
3118 let t_a11 = calc_f32!(
3119 x0 + t_a11_1
3120 + t_a11_2
3121 + t_a11_3
3122 + t_a11_4
3123 + t_a11_5
3124 + t_a11_6
3125 + t_a11_7
3126 + t_a11_8
3127 + t_a11_9
3128 + t_a11_10
3129 + t_a11_11
3130 );
3131
3132 let t_b1 = calc_f32!(
3133 t_b1_1
3134 + t_b1_2
3135 + t_b1_3
3136 + t_b1_4
3137 + t_b1_5
3138 + t_b1_6
3139 + t_b1_7
3140 + t_b1_8
3141 + t_b1_9
3142 + t_b1_10
3143 + t_b1_11
3144 );
3145 let t_b2 = calc_f32!(
3146 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5
3147 - t_b2_6
3148 - t_b2_7
3149 - t_b2_8
3150 - t_b2_9
3151 - t_b2_10
3152 - t_b2_11
3153 );
3154 let t_b3 = calc_f32!(
3155 t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7
3156 + t_b3_8
3157 + t_b3_9
3158 + t_b3_10
3159 + t_b3_11
3160 );
3161 let t_b4 = calc_f32!(
3162 t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 - t_b4_5 + t_b4_6 + t_b4_7 + t_b4_8
3163 - t_b4_9
3164 - t_b4_10
3165 - t_b4_11
3166 );
3167 let t_b5 = calc_f32!(
3168 t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 + t_b5_5 + t_b5_6 - t_b5_7 - t_b5_8 - t_b5_9
3169 + t_b5_10
3170 + t_b5_11
3171 );
3172 let t_b6 = calc_f32!(
3173 t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 + t_b6_5 - t_b6_6 - t_b6_7 + t_b6_8 + t_b6_9
3174 - t_b6_10
3175 - t_b6_11
3176 );
3177 let t_b7 = calc_f32!(
3178 t_b7_1 - t_b7_2 - t_b7_3 + t_b7_4 - t_b7_5 - t_b7_6 + t_b7_7 + t_b7_8 - t_b7_9
3179 + t_b7_10
3180 + t_b7_11
3181 );
3182 let t_b8 = calc_f32!(
3183 t_b8_1 - t_b8_2 + t_b8_3 + t_b8_4 - t_b8_5 + t_b8_6 + t_b8_7 - t_b8_8
3184 + t_b8_9
3185 + t_b8_10
3186 - t_b8_11
3187 );
3188 let t_b9 = calc_f32!(
3189 t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 - t_b9_5 + t_b9_6 - t_b9_7 + t_b9_8
3190 - t_b9_9
3191 - t_b9_10
3192 + t_b9_11
3193 );
3194 let t_b10 = calc_f32!(
3195 t_b10_1 - t_b10_2 + t_b10_3 - t_b10_4 + t_b10_5 - t_b10_6 + t_b10_7 + t_b10_8 - t_b10_9
3196 + t_b10_10
3197 - t_b10_11
3198 );
3199 let t_b11 = calc_f32!(
3200 t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 + t_b11_5 - t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9
3201 - t_b11_10
3202 + t_b11_11
3203 );
3204
3205 let t_b1_rot = self.rotate.rotate_both(t_b1);
3206 let t_b2_rot = self.rotate.rotate_both(t_b2);
3207 let t_b3_rot = self.rotate.rotate_both(t_b3);
3208 let t_b4_rot = self.rotate.rotate_both(t_b4);
3209 let t_b5_rot = self.rotate.rotate_both(t_b5);
3210 let t_b6_rot = self.rotate.rotate_both(t_b6);
3211 let t_b7_rot = self.rotate.rotate_both(t_b7);
3212 let t_b8_rot = self.rotate.rotate_both(t_b8);
3213 let t_b9_rot = self.rotate.rotate_both(t_b9);
3214 let t_b10_rot = self.rotate.rotate_both(t_b10);
3215 let t_b11_rot = self.rotate.rotate_both(t_b11);
3216
3217 let y0 = calc_f32!(
3218 x0 + x1p22
3219 + x2p21
3220 + x3p20
3221 + x4p19
3222 + x5p18
3223 + x6p17
3224 + x7p16
3225 + x8p15
3226 + x9p14
3227 + x10p13
3228 + x11p12
3229 );
3230 let [y1, y22] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
3231 let [y2, y21] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
3232 let [y3, y20] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
3233 let [y4, y19] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
3234 let [y5, y18] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
3235 let [y6, y17] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
3236 let [y7, y16] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
3237 let [y8, y15] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
3238 let [y9, y14] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
3239 let [y10, y13] = parallel_fft2_interleaved_f32(t_a10, t_b10_rot);
3240 let [y11, y12] = parallel_fft2_interleaved_f32(t_a11, t_b11_rot);
3241 [
3242 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
3243 y19, y20, y21, y22,
3244 ]
3245 }
3246 }
3247
3248 // ____ _____ __ _ _ _ _ _
3249 // |___ \|___ / / /_ | || | | |__ (_) |_
3250 // __) | |_ \ _____ | '_ \| || |_| '_ \| | __|
3251 // / __/ ___) | |_____| | (_) |__ _| |_) | | |_
3252 // |_____|____/ \___/ |_| |_.__/|_|\__|
3253 //
3254
3255 pub struct WasmSimdF64Butterfly23<T> {
3256 direction: FftDirection,
3257 _phantom: std::marker::PhantomData<T>,
3258 rotate: Rotate90F64,
3259 twiddle1re: v128,
3260 twiddle1im: v128,
3261 twiddle2re: v128,
3262 twiddle2im: v128,
3263 twiddle3re: v128,
3264 twiddle3im: v128,
3265 twiddle4re: v128,
3266 twiddle4im: v128,
3267 twiddle5re: v128,
3268 twiddle5im: v128,
3269 twiddle6re: v128,
3270 twiddle6im: v128,
3271 twiddle7re: v128,
3272 twiddle7im: v128,
3273 twiddle8re: v128,
3274 twiddle8im: v128,
3275 twiddle9re: v128,
3276 twiddle9im: v128,
3277 twiddle10re: v128,
3278 twiddle10im: v128,
3279 twiddle11re: v128,
3280 twiddle11im: v128,
3281 }
3282
3283 boilerplate_fft_wasm_simd_f64_butterfly!(
3284 WasmSimdF64Butterfly23,
3285 23,
3286 |this: &WasmSimdF64Butterfly23<_>| this.direction
3287 );
3288 boilerplate_fft_wasm_simd_common_butterfly!(
3289 WasmSimdF64Butterfly23,
3290 23,
3291 |this: &WasmSimdF64Butterfly23<_>| this.direction
3292 );
3293 impl<T: FftNum> WasmSimdF64Butterfly23<T> {
3294 #[inline(always)]
3295 pub fn new(direction: FftDirection) -> Self {
3296 assert_f64::<T>();
3297 let rotate = Rotate90F64::new(true);
3298 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 23, direction);
3299 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 23, direction);
3300 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 23, direction);
3301 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 23, direction);
3302 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 23, direction);
3303 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 23, direction);
3304 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 23, direction);
3305 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 23, direction);
3306 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 23, direction);
3307 let tw10: Complex<f64> = twiddles::compute_twiddle(10, 23, direction);
3308 let tw11: Complex<f64> = twiddles::compute_twiddle(11, 23, direction);
3309 let twiddle1re = f64x2_splat(tw1.re);
3310 let twiddle1im = f64x2_splat(tw1.im);
3311 let twiddle2re = f64x2_splat(tw2.re);
3312 let twiddle2im = f64x2_splat(tw2.im);
3313 let twiddle3re = f64x2_splat(tw3.re);
3314 let twiddle3im = f64x2_splat(tw3.im);
3315 let twiddle4re = f64x2_splat(tw4.re);
3316 let twiddle4im = f64x2_splat(tw4.im);
3317 let twiddle5re = f64x2_splat(tw5.re);
3318 let twiddle5im = f64x2_splat(tw5.im);
3319 let twiddle6re = f64x2_splat(tw6.re);
3320 let twiddle6im = f64x2_splat(tw6.im);
3321 let twiddle7re = f64x2_splat(tw7.re);
3322 let twiddle7im = f64x2_splat(tw7.im);
3323 let twiddle8re = f64x2_splat(tw8.re);
3324 let twiddle8im = f64x2_splat(tw8.im);
3325 let twiddle9re = f64x2_splat(tw9.re);
3326 let twiddle9im = f64x2_splat(tw9.im);
3327 let twiddle10re = f64x2_splat(tw10.re);
3328 let twiddle10im = f64x2_splat(tw10.im);
3329 let twiddle11re = f64x2_splat(tw11.re);
3330 let twiddle11im = f64x2_splat(tw11.im);
3331
3332 Self {
3333 direction,
3334 _phantom: std::marker::PhantomData,
3335 rotate,
3336 twiddle1re,
3337 twiddle1im,
3338 twiddle2re,
3339 twiddle2im,
3340 twiddle3re,
3341 twiddle3im,
3342 twiddle4re,
3343 twiddle4im,
3344 twiddle5re,
3345 twiddle5im,
3346 twiddle6re,
3347 twiddle6im,
3348 twiddle7re,
3349 twiddle7im,
3350 twiddle8re,
3351 twiddle8im,
3352 twiddle9re,
3353 twiddle9im,
3354 twiddle10re,
3355 twiddle10im,
3356 twiddle11re,
3357 twiddle11im,
3358 }
3359 }
3360
3361 #[inline(always)]
3362 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
3363 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
3364
3365 let out = self.perform_fft_direct(values);
3366
3367 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
3368 }
3369
3370 #[inline(always)]
3371 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 23]) -> [v128; 23] {
3372 let [x1p22, x1m22] = solo_fft2_f64(values[1], values[22]);
3373 let [x2p21, x2m21] = solo_fft2_f64(values[2], values[21]);
3374 let [x3p20, x3m20] = solo_fft2_f64(values[3], values[20]);
3375 let [x4p19, x4m19] = solo_fft2_f64(values[4], values[19]);
3376 let [x5p18, x5m18] = solo_fft2_f64(values[5], values[18]);
3377 let [x6p17, x6m17] = solo_fft2_f64(values[6], values[17]);
3378 let [x7p16, x7m16] = solo_fft2_f64(values[7], values[16]);
3379 let [x8p15, x8m15] = solo_fft2_f64(values[8], values[15]);
3380 let [x9p14, x9m14] = solo_fft2_f64(values[9], values[14]);
3381 let [x10p13, x10m13] = solo_fft2_f64(values[10], values[13]);
3382 let [x11p12, x11m12] = solo_fft2_f64(values[11], values[12]);
3383
3384 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p22);
3385 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p21);
3386 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p20);
3387 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p19);
3388 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p18);
3389 let t_a1_6 = f64x2_mul(self.twiddle6re, x6p17);
3390 let t_a1_7 = f64x2_mul(self.twiddle7re, x7p16);
3391 let t_a1_8 = f64x2_mul(self.twiddle8re, x8p15);
3392 let t_a1_9 = f64x2_mul(self.twiddle9re, x9p14);
3393 let t_a1_10 = f64x2_mul(self.twiddle10re, x10p13);
3394 let t_a1_11 = f64x2_mul(self.twiddle11re, x11p12);
3395 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p22);
3396 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p21);
3397 let t_a2_3 = f64x2_mul(self.twiddle6re, x3p20);
3398 let t_a2_4 = f64x2_mul(self.twiddle8re, x4p19);
3399 let t_a2_5 = f64x2_mul(self.twiddle10re, x5p18);
3400 let t_a2_6 = f64x2_mul(self.twiddle11re, x6p17);
3401 let t_a2_7 = f64x2_mul(self.twiddle9re, x7p16);
3402 let t_a2_8 = f64x2_mul(self.twiddle7re, x8p15);
3403 let t_a2_9 = f64x2_mul(self.twiddle5re, x9p14);
3404 let t_a2_10 = f64x2_mul(self.twiddle3re, x10p13);
3405 let t_a2_11 = f64x2_mul(self.twiddle1re, x11p12);
3406 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p22);
3407 let t_a3_2 = f64x2_mul(self.twiddle6re, x2p21);
3408 let t_a3_3 = f64x2_mul(self.twiddle9re, x3p20);
3409 let t_a3_4 = f64x2_mul(self.twiddle11re, x4p19);
3410 let t_a3_5 = f64x2_mul(self.twiddle8re, x5p18);
3411 let t_a3_6 = f64x2_mul(self.twiddle5re, x6p17);
3412 let t_a3_7 = f64x2_mul(self.twiddle2re, x7p16);
3413 let t_a3_8 = f64x2_mul(self.twiddle1re, x8p15);
3414 let t_a3_9 = f64x2_mul(self.twiddle4re, x9p14);
3415 let t_a3_10 = f64x2_mul(self.twiddle7re, x10p13);
3416 let t_a3_11 = f64x2_mul(self.twiddle10re, x11p12);
3417 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p22);
3418 let t_a4_2 = f64x2_mul(self.twiddle8re, x2p21);
3419 let t_a4_3 = f64x2_mul(self.twiddle11re, x3p20);
3420 let t_a4_4 = f64x2_mul(self.twiddle7re, x4p19);
3421 let t_a4_5 = f64x2_mul(self.twiddle3re, x5p18);
3422 let t_a4_6 = f64x2_mul(self.twiddle1re, x6p17);
3423 let t_a4_7 = f64x2_mul(self.twiddle5re, x7p16);
3424 let t_a4_8 = f64x2_mul(self.twiddle9re, x8p15);
3425 let t_a4_9 = f64x2_mul(self.twiddle10re, x9p14);
3426 let t_a4_10 = f64x2_mul(self.twiddle6re, x10p13);
3427 let t_a4_11 = f64x2_mul(self.twiddle2re, x11p12);
3428 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p22);
3429 let t_a5_2 = f64x2_mul(self.twiddle10re, x2p21);
3430 let t_a5_3 = f64x2_mul(self.twiddle8re, x3p20);
3431 let t_a5_4 = f64x2_mul(self.twiddle3re, x4p19);
3432 let t_a5_5 = f64x2_mul(self.twiddle2re, x5p18);
3433 let t_a5_6 = f64x2_mul(self.twiddle7re, x6p17);
3434 let t_a5_7 = f64x2_mul(self.twiddle11re, x7p16);
3435 let t_a5_8 = f64x2_mul(self.twiddle6re, x8p15);
3436 let t_a5_9 = f64x2_mul(self.twiddle1re, x9p14);
3437 let t_a5_10 = f64x2_mul(self.twiddle4re, x10p13);
3438 let t_a5_11 = f64x2_mul(self.twiddle9re, x11p12);
3439 let t_a6_1 = f64x2_mul(self.twiddle6re, x1p22);
3440 let t_a6_2 = f64x2_mul(self.twiddle11re, x2p21);
3441 let t_a6_3 = f64x2_mul(self.twiddle5re, x3p20);
3442 let t_a6_4 = f64x2_mul(self.twiddle1re, x4p19);
3443 let t_a6_5 = f64x2_mul(self.twiddle7re, x5p18);
3444 let t_a6_6 = f64x2_mul(self.twiddle10re, x6p17);
3445 let t_a6_7 = f64x2_mul(self.twiddle4re, x7p16);
3446 let t_a6_8 = f64x2_mul(self.twiddle2re, x8p15);
3447 let t_a6_9 = f64x2_mul(self.twiddle8re, x9p14);
3448 let t_a6_10 = f64x2_mul(self.twiddle9re, x10p13);
3449 let t_a6_11 = f64x2_mul(self.twiddle3re, x11p12);
3450 let t_a7_1 = f64x2_mul(self.twiddle7re, x1p22);
3451 let t_a7_2 = f64x2_mul(self.twiddle9re, x2p21);
3452 let t_a7_3 = f64x2_mul(self.twiddle2re, x3p20);
3453 let t_a7_4 = f64x2_mul(self.twiddle5re, x4p19);
3454 let t_a7_5 = f64x2_mul(self.twiddle11re, x5p18);
3455 let t_a7_6 = f64x2_mul(self.twiddle4re, x6p17);
3456 let t_a7_7 = f64x2_mul(self.twiddle3re, x7p16);
3457 let t_a7_8 = f64x2_mul(self.twiddle10re, x8p15);
3458 let t_a7_9 = f64x2_mul(self.twiddle6re, x9p14);
3459 let t_a7_10 = f64x2_mul(self.twiddle1re, x10p13);
3460 let t_a7_11 = f64x2_mul(self.twiddle8re, x11p12);
3461 let t_a8_1 = f64x2_mul(self.twiddle8re, x1p22);
3462 let t_a8_2 = f64x2_mul(self.twiddle7re, x2p21);
3463 let t_a8_3 = f64x2_mul(self.twiddle1re, x3p20);
3464 let t_a8_4 = f64x2_mul(self.twiddle9re, x4p19);
3465 let t_a8_5 = f64x2_mul(self.twiddle6re, x5p18);
3466 let t_a8_6 = f64x2_mul(self.twiddle2re, x6p17);
3467 let t_a8_7 = f64x2_mul(self.twiddle10re, x7p16);
3468 let t_a8_8 = f64x2_mul(self.twiddle5re, x8p15);
3469 let t_a8_9 = f64x2_mul(self.twiddle3re, x9p14);
3470 let t_a8_10 = f64x2_mul(self.twiddle11re, x10p13);
3471 let t_a8_11 = f64x2_mul(self.twiddle4re, x11p12);
3472 let t_a9_1 = f64x2_mul(self.twiddle9re, x1p22);
3473 let t_a9_2 = f64x2_mul(self.twiddle5re, x2p21);
3474 let t_a9_3 = f64x2_mul(self.twiddle4re, x3p20);
3475 let t_a9_4 = f64x2_mul(self.twiddle10re, x4p19);
3476 let t_a9_5 = f64x2_mul(self.twiddle1re, x5p18);
3477 let t_a9_6 = f64x2_mul(self.twiddle8re, x6p17);
3478 let t_a9_7 = f64x2_mul(self.twiddle6re, x7p16);
3479 let t_a9_8 = f64x2_mul(self.twiddle3re, x8p15);
3480 let t_a9_9 = f64x2_mul(self.twiddle11re, x9p14);
3481 let t_a9_10 = f64x2_mul(self.twiddle2re, x10p13);
3482 let t_a9_11 = f64x2_mul(self.twiddle7re, x11p12);
3483 let t_a10_1 = f64x2_mul(self.twiddle10re, x1p22);
3484 let t_a10_2 = f64x2_mul(self.twiddle3re, x2p21);
3485 let t_a10_3 = f64x2_mul(self.twiddle7re, x3p20);
3486 let t_a10_4 = f64x2_mul(self.twiddle6re, x4p19);
3487 let t_a10_5 = f64x2_mul(self.twiddle4re, x5p18);
3488 let t_a10_6 = f64x2_mul(self.twiddle9re, x6p17);
3489 let t_a10_7 = f64x2_mul(self.twiddle1re, x7p16);
3490 let t_a10_8 = f64x2_mul(self.twiddle11re, x8p15);
3491 let t_a10_9 = f64x2_mul(self.twiddle2re, x9p14);
3492 let t_a10_10 = f64x2_mul(self.twiddle8re, x10p13);
3493 let t_a10_11 = f64x2_mul(self.twiddle5re, x11p12);
3494 let t_a11_1 = f64x2_mul(self.twiddle11re, x1p22);
3495 let t_a11_2 = f64x2_mul(self.twiddle1re, x2p21);
3496 let t_a11_3 = f64x2_mul(self.twiddle10re, x3p20);
3497 let t_a11_4 = f64x2_mul(self.twiddle2re, x4p19);
3498 let t_a11_5 = f64x2_mul(self.twiddle9re, x5p18);
3499 let t_a11_6 = f64x2_mul(self.twiddle3re, x6p17);
3500 let t_a11_7 = f64x2_mul(self.twiddle8re, x7p16);
3501 let t_a11_8 = f64x2_mul(self.twiddle4re, x8p15);
3502 let t_a11_9 = f64x2_mul(self.twiddle7re, x9p14);
3503 let t_a11_10 = f64x2_mul(self.twiddle5re, x10p13);
3504 let t_a11_11 = f64x2_mul(self.twiddle6re, x11p12);
3505
3506 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m22);
3507 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m21);
3508 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m20);
3509 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m19);
3510 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m18);
3511 let t_b1_6 = f64x2_mul(self.twiddle6im, x6m17);
3512 let t_b1_7 = f64x2_mul(self.twiddle7im, x7m16);
3513 let t_b1_8 = f64x2_mul(self.twiddle8im, x8m15);
3514 let t_b1_9 = f64x2_mul(self.twiddle9im, x9m14);
3515 let t_b1_10 = f64x2_mul(self.twiddle10im, x10m13);
3516 let t_b1_11 = f64x2_mul(self.twiddle11im, x11m12);
3517 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m22);
3518 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m21);
3519 let t_b2_3 = f64x2_mul(self.twiddle6im, x3m20);
3520 let t_b2_4 = f64x2_mul(self.twiddle8im, x4m19);
3521 let t_b2_5 = f64x2_mul(self.twiddle10im, x5m18);
3522 let t_b2_6 = f64x2_mul(self.twiddle11im, x6m17);
3523 let t_b2_7 = f64x2_mul(self.twiddle9im, x7m16);
3524 let t_b2_8 = f64x2_mul(self.twiddle7im, x8m15);
3525 let t_b2_9 = f64x2_mul(self.twiddle5im, x9m14);
3526 let t_b2_10 = f64x2_mul(self.twiddle3im, x10m13);
3527 let t_b2_11 = f64x2_mul(self.twiddle1im, x11m12);
3528 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m22);
3529 let t_b3_2 = f64x2_mul(self.twiddle6im, x2m21);
3530 let t_b3_3 = f64x2_mul(self.twiddle9im, x3m20);
3531 let t_b3_4 = f64x2_mul(self.twiddle11im, x4m19);
3532 let t_b3_5 = f64x2_mul(self.twiddle8im, x5m18);
3533 let t_b3_6 = f64x2_mul(self.twiddle5im, x6m17);
3534 let t_b3_7 = f64x2_mul(self.twiddle2im, x7m16);
3535 let t_b3_8 = f64x2_mul(self.twiddle1im, x8m15);
3536 let t_b3_9 = f64x2_mul(self.twiddle4im, x9m14);
3537 let t_b3_10 = f64x2_mul(self.twiddle7im, x10m13);
3538 let t_b3_11 = f64x2_mul(self.twiddle10im, x11m12);
3539 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m22);
3540 let t_b4_2 = f64x2_mul(self.twiddle8im, x2m21);
3541 let t_b4_3 = f64x2_mul(self.twiddle11im, x3m20);
3542 let t_b4_4 = f64x2_mul(self.twiddle7im, x4m19);
3543 let t_b4_5 = f64x2_mul(self.twiddle3im, x5m18);
3544 let t_b4_6 = f64x2_mul(self.twiddle1im, x6m17);
3545 let t_b4_7 = f64x2_mul(self.twiddle5im, x7m16);
3546 let t_b4_8 = f64x2_mul(self.twiddle9im, x8m15);
3547 let t_b4_9 = f64x2_mul(self.twiddle10im, x9m14);
3548 let t_b4_10 = f64x2_mul(self.twiddle6im, x10m13);
3549 let t_b4_11 = f64x2_mul(self.twiddle2im, x11m12);
3550 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m22);
3551 let t_b5_2 = f64x2_mul(self.twiddle10im, x2m21);
3552 let t_b5_3 = f64x2_mul(self.twiddle8im, x3m20);
3553 let t_b5_4 = f64x2_mul(self.twiddle3im, x4m19);
3554 let t_b5_5 = f64x2_mul(self.twiddle2im, x5m18);
3555 let t_b5_6 = f64x2_mul(self.twiddle7im, x6m17);
3556 let t_b5_7 = f64x2_mul(self.twiddle11im, x7m16);
3557 let t_b5_8 = f64x2_mul(self.twiddle6im, x8m15);
3558 let t_b5_9 = f64x2_mul(self.twiddle1im, x9m14);
3559 let t_b5_10 = f64x2_mul(self.twiddle4im, x10m13);
3560 let t_b5_11 = f64x2_mul(self.twiddle9im, x11m12);
3561 let t_b6_1 = f64x2_mul(self.twiddle6im, x1m22);
3562 let t_b6_2 = f64x2_mul(self.twiddle11im, x2m21);
3563 let t_b6_3 = f64x2_mul(self.twiddle5im, x3m20);
3564 let t_b6_4 = f64x2_mul(self.twiddle1im, x4m19);
3565 let t_b6_5 = f64x2_mul(self.twiddle7im, x5m18);
3566 let t_b6_6 = f64x2_mul(self.twiddle10im, x6m17);
3567 let t_b6_7 = f64x2_mul(self.twiddle4im, x7m16);
3568 let t_b6_8 = f64x2_mul(self.twiddle2im, x8m15);
3569 let t_b6_9 = f64x2_mul(self.twiddle8im, x9m14);
3570 let t_b6_10 = f64x2_mul(self.twiddle9im, x10m13);
3571 let t_b6_11 = f64x2_mul(self.twiddle3im, x11m12);
3572 let t_b7_1 = f64x2_mul(self.twiddle7im, x1m22);
3573 let t_b7_2 = f64x2_mul(self.twiddle9im, x2m21);
3574 let t_b7_3 = f64x2_mul(self.twiddle2im, x3m20);
3575 let t_b7_4 = f64x2_mul(self.twiddle5im, x4m19);
3576 let t_b7_5 = f64x2_mul(self.twiddle11im, x5m18);
3577 let t_b7_6 = f64x2_mul(self.twiddle4im, x6m17);
3578 let t_b7_7 = f64x2_mul(self.twiddle3im, x7m16);
3579 let t_b7_8 = f64x2_mul(self.twiddle10im, x8m15);
3580 let t_b7_9 = f64x2_mul(self.twiddle6im, x9m14);
3581 let t_b7_10 = f64x2_mul(self.twiddle1im, x10m13);
3582 let t_b7_11 = f64x2_mul(self.twiddle8im, x11m12);
3583 let t_b8_1 = f64x2_mul(self.twiddle8im, x1m22);
3584 let t_b8_2 = f64x2_mul(self.twiddle7im, x2m21);
3585 let t_b8_3 = f64x2_mul(self.twiddle1im, x3m20);
3586 let t_b8_4 = f64x2_mul(self.twiddle9im, x4m19);
3587 let t_b8_5 = f64x2_mul(self.twiddle6im, x5m18);
3588 let t_b8_6 = f64x2_mul(self.twiddle2im, x6m17);
3589 let t_b8_7 = f64x2_mul(self.twiddle10im, x7m16);
3590 let t_b8_8 = f64x2_mul(self.twiddle5im, x8m15);
3591 let t_b8_9 = f64x2_mul(self.twiddle3im, x9m14);
3592 let t_b8_10 = f64x2_mul(self.twiddle11im, x10m13);
3593 let t_b8_11 = f64x2_mul(self.twiddle4im, x11m12);
3594 let t_b9_1 = f64x2_mul(self.twiddle9im, x1m22);
3595 let t_b9_2 = f64x2_mul(self.twiddle5im, x2m21);
3596 let t_b9_3 = f64x2_mul(self.twiddle4im, x3m20);
3597 let t_b9_4 = f64x2_mul(self.twiddle10im, x4m19);
3598 let t_b9_5 = f64x2_mul(self.twiddle1im, x5m18);
3599 let t_b9_6 = f64x2_mul(self.twiddle8im, x6m17);
3600 let t_b9_7 = f64x2_mul(self.twiddle6im, x7m16);
3601 let t_b9_8 = f64x2_mul(self.twiddle3im, x8m15);
3602 let t_b9_9 = f64x2_mul(self.twiddle11im, x9m14);
3603 let t_b9_10 = f64x2_mul(self.twiddle2im, x10m13);
3604 let t_b9_11 = f64x2_mul(self.twiddle7im, x11m12);
3605 let t_b10_1 = f64x2_mul(self.twiddle10im, x1m22);
3606 let t_b10_2 = f64x2_mul(self.twiddle3im, x2m21);
3607 let t_b10_3 = f64x2_mul(self.twiddle7im, x3m20);
3608 let t_b10_4 = f64x2_mul(self.twiddle6im, x4m19);
3609 let t_b10_5 = f64x2_mul(self.twiddle4im, x5m18);
3610 let t_b10_6 = f64x2_mul(self.twiddle9im, x6m17);
3611 let t_b10_7 = f64x2_mul(self.twiddle1im, x7m16);
3612 let t_b10_8 = f64x2_mul(self.twiddle11im, x8m15);
3613 let t_b10_9 = f64x2_mul(self.twiddle2im, x9m14);
3614 let t_b10_10 = f64x2_mul(self.twiddle8im, x10m13);
3615 let t_b10_11 = f64x2_mul(self.twiddle5im, x11m12);
3616 let t_b11_1 = f64x2_mul(self.twiddle11im, x1m22);
3617 let t_b11_2 = f64x2_mul(self.twiddle1im, x2m21);
3618 let t_b11_3 = f64x2_mul(self.twiddle10im, x3m20);
3619 let t_b11_4 = f64x2_mul(self.twiddle2im, x4m19);
3620 let t_b11_5 = f64x2_mul(self.twiddle9im, x5m18);
3621 let t_b11_6 = f64x2_mul(self.twiddle3im, x6m17);
3622 let t_b11_7 = f64x2_mul(self.twiddle8im, x7m16);
3623 let t_b11_8 = f64x2_mul(self.twiddle4im, x8m15);
3624 let t_b11_9 = f64x2_mul(self.twiddle7im, x9m14);
3625 let t_b11_10 = f64x2_mul(self.twiddle5im, x10m13);
3626 let t_b11_11 = f64x2_mul(self.twiddle6im, x11m12);
3627
3628 let x0 = values[0];
3629 let t_a1 = calc_f64!(
3630 x0 + t_a1_1
3631 + t_a1_2
3632 + t_a1_3
3633 + t_a1_4
3634 + t_a1_5
3635 + t_a1_6
3636 + t_a1_7
3637 + t_a1_8
3638 + t_a1_9
3639 + t_a1_10
3640 + t_a1_11
3641 );
3642 let t_a2 = calc_f64!(
3643 x0 + t_a2_1
3644 + t_a2_2
3645 + t_a2_3
3646 + t_a2_4
3647 + t_a2_5
3648 + t_a2_6
3649 + t_a2_7
3650 + t_a2_8
3651 + t_a2_9
3652 + t_a2_10
3653 + t_a2_11
3654 );
3655 let t_a3 = calc_f64!(
3656 x0 + t_a3_1
3657 + t_a3_2
3658 + t_a3_3
3659 + t_a3_4
3660 + t_a3_5
3661 + t_a3_6
3662 + t_a3_7
3663 + t_a3_8
3664 + t_a3_9
3665 + t_a3_10
3666 + t_a3_11
3667 );
3668 let t_a4 = calc_f64!(
3669 x0 + t_a4_1
3670 + t_a4_2
3671 + t_a4_3
3672 + t_a4_4
3673 + t_a4_5
3674 + t_a4_6
3675 + t_a4_7
3676 + t_a4_8
3677 + t_a4_9
3678 + t_a4_10
3679 + t_a4_11
3680 );
3681 let t_a5 = calc_f64!(
3682 x0 + t_a5_1
3683 + t_a5_2
3684 + t_a5_3
3685 + t_a5_4
3686 + t_a5_5
3687 + t_a5_6
3688 + t_a5_7
3689 + t_a5_8
3690 + t_a5_9
3691 + t_a5_10
3692 + t_a5_11
3693 );
3694 let t_a6 = calc_f64!(
3695 x0 + t_a6_1
3696 + t_a6_2
3697 + t_a6_3
3698 + t_a6_4
3699 + t_a6_5
3700 + t_a6_6
3701 + t_a6_7
3702 + t_a6_8
3703 + t_a6_9
3704 + t_a6_10
3705 + t_a6_11
3706 );
3707 let t_a7 = calc_f64!(
3708 x0 + t_a7_1
3709 + t_a7_2
3710 + t_a7_3
3711 + t_a7_4
3712 + t_a7_5
3713 + t_a7_6
3714 + t_a7_7
3715 + t_a7_8
3716 + t_a7_9
3717 + t_a7_10
3718 + t_a7_11
3719 );
3720 let t_a8 = calc_f64!(
3721 x0 + t_a8_1
3722 + t_a8_2
3723 + t_a8_3
3724 + t_a8_4
3725 + t_a8_5
3726 + t_a8_6
3727 + t_a8_7
3728 + t_a8_8
3729 + t_a8_9
3730 + t_a8_10
3731 + t_a8_11
3732 );
3733 let t_a9 = calc_f64!(
3734 x0 + t_a9_1
3735 + t_a9_2
3736 + t_a9_3
3737 + t_a9_4
3738 + t_a9_5
3739 + t_a9_6
3740 + t_a9_7
3741 + t_a9_8
3742 + t_a9_9
3743 + t_a9_10
3744 + t_a9_11
3745 );
3746 let t_a10 = calc_f64!(
3747 x0 + t_a10_1
3748 + t_a10_2
3749 + t_a10_3
3750 + t_a10_4
3751 + t_a10_5
3752 + t_a10_6
3753 + t_a10_7
3754 + t_a10_8
3755 + t_a10_9
3756 + t_a10_10
3757 + t_a10_11
3758 );
3759 let t_a11 = calc_f64!(
3760 x0 + t_a11_1
3761 + t_a11_2
3762 + t_a11_3
3763 + t_a11_4
3764 + t_a11_5
3765 + t_a11_6
3766 + t_a11_7
3767 + t_a11_8
3768 + t_a11_9
3769 + t_a11_10
3770 + t_a11_11
3771 );
3772
3773 let t_b1 = calc_f64!(
3774 t_b1_1
3775 + t_b1_2
3776 + t_b1_3
3777 + t_b1_4
3778 + t_b1_5
3779 + t_b1_6
3780 + t_b1_7
3781 + t_b1_8
3782 + t_b1_9
3783 + t_b1_10
3784 + t_b1_11
3785 );
3786 let t_b2 = calc_f64!(
3787 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5
3788 - t_b2_6
3789 - t_b2_7
3790 - t_b2_8
3791 - t_b2_9
3792 - t_b2_10
3793 - t_b2_11
3794 );
3795 let t_b3 = calc_f64!(
3796 t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7
3797 + t_b3_8
3798 + t_b3_9
3799 + t_b3_10
3800 + t_b3_11
3801 );
3802 let t_b4 = calc_f64!(
3803 t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 - t_b4_5 + t_b4_6 + t_b4_7 + t_b4_8
3804 - t_b4_9
3805 - t_b4_10
3806 - t_b4_11
3807 );
3808 let t_b5 = calc_f64!(
3809 t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 + t_b5_5 + t_b5_6 - t_b5_7 - t_b5_8 - t_b5_9
3810 + t_b5_10
3811 + t_b5_11
3812 );
3813 let t_b6 = calc_f64!(
3814 t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 + t_b6_5 - t_b6_6 - t_b6_7 + t_b6_8 + t_b6_9
3815 - t_b6_10
3816 - t_b6_11
3817 );
3818 let t_b7 = calc_f64!(
3819 t_b7_1 - t_b7_2 - t_b7_3 + t_b7_4 - t_b7_5 - t_b7_6 + t_b7_7 + t_b7_8 - t_b7_9
3820 + t_b7_10
3821 + t_b7_11
3822 );
3823 let t_b8 = calc_f64!(
3824 t_b8_1 - t_b8_2 + t_b8_3 + t_b8_4 - t_b8_5 + t_b8_6 + t_b8_7 - t_b8_8
3825 + t_b8_9
3826 + t_b8_10
3827 - t_b8_11
3828 );
3829 let t_b9 = calc_f64!(
3830 t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 - t_b9_5 + t_b9_6 - t_b9_7 + t_b9_8
3831 - t_b9_9
3832 - t_b9_10
3833 + t_b9_11
3834 );
3835 let t_b10 = calc_f64!(
3836 t_b10_1 - t_b10_2 + t_b10_3 - t_b10_4 + t_b10_5 - t_b10_6 + t_b10_7 + t_b10_8 - t_b10_9
3837 + t_b10_10
3838 - t_b10_11
3839 );
3840 let t_b11 = calc_f64!(
3841 t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 + t_b11_5 - t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9
3842 - t_b11_10
3843 + t_b11_11
3844 );
3845
3846 let t_b1_rot = self.rotate.rotate(t_b1);
3847 let t_b2_rot = self.rotate.rotate(t_b2);
3848 let t_b3_rot = self.rotate.rotate(t_b3);
3849 let t_b4_rot = self.rotate.rotate(t_b4);
3850 let t_b5_rot = self.rotate.rotate(t_b5);
3851 let t_b6_rot = self.rotate.rotate(t_b6);
3852 let t_b7_rot = self.rotate.rotate(t_b7);
3853 let t_b8_rot = self.rotate.rotate(t_b8);
3854 let t_b9_rot = self.rotate.rotate(t_b9);
3855 let t_b10_rot = self.rotate.rotate(t_b10);
3856 let t_b11_rot = self.rotate.rotate(t_b11);
3857
3858 let y0 = calc_f64!(
3859 x0 + x1p22
3860 + x2p21
3861 + x3p20
3862 + x4p19
3863 + x5p18
3864 + x6p17
3865 + x7p16
3866 + x8p15
3867 + x9p14
3868 + x10p13
3869 + x11p12
3870 );
3871 let [y1, y22] = solo_fft2_f64(t_a1, t_b1_rot);
3872 let [y2, y21] = solo_fft2_f64(t_a2, t_b2_rot);
3873 let [y3, y20] = solo_fft2_f64(t_a3, t_b3_rot);
3874 let [y4, y19] = solo_fft2_f64(t_a4, t_b4_rot);
3875 let [y5, y18] = solo_fft2_f64(t_a5, t_b5_rot);
3876 let [y6, y17] = solo_fft2_f64(t_a6, t_b6_rot);
3877 let [y7, y16] = solo_fft2_f64(t_a7, t_b7_rot);
3878 let [y8, y15] = solo_fft2_f64(t_a8, t_b8_rot);
3879 let [y9, y14] = solo_fft2_f64(t_a9, t_b9_rot);
3880 let [y10, y13] = solo_fft2_f64(t_a10, t_b10_rot);
3881 let [y11, y12] = solo_fft2_f64(t_a11, t_b11_rot);
3882 [
3883 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
3884 y19, y20, y21, y22,
3885 ]
3886 }
3887 }
3888
3889 // ____ ___ _________ _ _ _
3890 // |___ \ / _ \ |___ /___ \| |__ (_) |_
3891 // __) | (_) | _____ |_ \ __) | '_ \| | __|
3892 // / __/ \__, | |_____| ___) / __/| |_) | | |_
3893 // |_____| /_/ |____/_____|_.__/|_|\__|
3894 //
3895
3896 pub struct WasmSimdF32Butterfly29<T> {
3897 direction: FftDirection,
3898 _phantom: std::marker::PhantomData<T>,
3899 rotate: Rotate90F32,
3900 twiddle1re: v128,
3901 twiddle1im: v128,
3902 twiddle2re: v128,
3903 twiddle2im: v128,
3904 twiddle3re: v128,
3905 twiddle3im: v128,
3906 twiddle4re: v128,
3907 twiddle4im: v128,
3908 twiddle5re: v128,
3909 twiddle5im: v128,
3910 twiddle6re: v128,
3911 twiddle6im: v128,
3912 twiddle7re: v128,
3913 twiddle7im: v128,
3914 twiddle8re: v128,
3915 twiddle8im: v128,
3916 twiddle9re: v128,
3917 twiddle9im: v128,
3918 twiddle10re: v128,
3919 twiddle10im: v128,
3920 twiddle11re: v128,
3921 twiddle11im: v128,
3922 twiddle12re: v128,
3923 twiddle12im: v128,
3924 twiddle13re: v128,
3925 twiddle13im: v128,
3926 twiddle14re: v128,
3927 twiddle14im: v128,
3928 }
3929
3930 boilerplate_fft_wasm_simd_f32_butterfly!(
3931 WasmSimdF32Butterfly29,
3932 29,
3933 |this: &WasmSimdF32Butterfly29<_>| this.direction
3934 );
3935 boilerplate_fft_wasm_simd_common_butterfly!(
3936 WasmSimdF32Butterfly29,
3937 29,
3938 |this: &WasmSimdF32Butterfly29<_>| this.direction
3939 );
3940 impl<T: FftNum> WasmSimdF32Butterfly29<T> {
3941 #[inline(always)]
3942 pub fn new(direction: FftDirection) -> Self {
3943 assert_f32::<T>();
3944 let rotate = Rotate90F32::new(true);
3945 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 29, direction);
3946 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 29, direction);
3947 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 29, direction);
3948 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 29, direction);
3949 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 29, direction);
3950 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 29, direction);
3951 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 29, direction);
3952 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 29, direction);
3953 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 29, direction);
3954 let tw10: Complex<f32> = twiddles::compute_twiddle(10, 29, direction);
3955 let tw11: Complex<f32> = twiddles::compute_twiddle(11, 29, direction);
3956 let tw12: Complex<f32> = twiddles::compute_twiddle(12, 29, direction);
3957 let tw13: Complex<f32> = twiddles::compute_twiddle(13, 29, direction);
3958 let tw14: Complex<f32> = twiddles::compute_twiddle(14, 29, direction);
3959 let twiddle1re = f32x4_splat(tw1.re);
3960 let twiddle1im = f32x4_splat(tw1.im);
3961 let twiddle2re = f32x4_splat(tw2.re);
3962 let twiddle2im = f32x4_splat(tw2.im);
3963 let twiddle3re = f32x4_splat(tw3.re);
3964 let twiddle3im = f32x4_splat(tw3.im);
3965 let twiddle4re = f32x4_splat(tw4.re);
3966 let twiddle4im = f32x4_splat(tw4.im);
3967 let twiddle5re = f32x4_splat(tw5.re);
3968 let twiddle5im = f32x4_splat(tw5.im);
3969 let twiddle6re = f32x4_splat(tw6.re);
3970 let twiddle6im = f32x4_splat(tw6.im);
3971 let twiddle7re = f32x4_splat(tw7.re);
3972 let twiddle7im = f32x4_splat(tw7.im);
3973 let twiddle8re = f32x4_splat(tw8.re);
3974 let twiddle8im = f32x4_splat(tw8.im);
3975 let twiddle9re = f32x4_splat(tw9.re);
3976 let twiddle9im = f32x4_splat(tw9.im);
3977 let twiddle10re = f32x4_splat(tw10.re);
3978 let twiddle10im = f32x4_splat(tw10.im);
3979 let twiddle11re = f32x4_splat(tw11.re);
3980 let twiddle11im = f32x4_splat(tw11.im);
3981 let twiddle12re = f32x4_splat(tw12.re);
3982 let twiddle12im = f32x4_splat(tw12.im);
3983 let twiddle13re = f32x4_splat(tw13.re);
3984 let twiddle13im = f32x4_splat(tw13.im);
3985 let twiddle14re = f32x4_splat(tw14.re);
3986 let twiddle14im = f32x4_splat(tw14.im);
3987
3988 Self {
3989 direction,
3990 _phantom: std::marker::PhantomData,
3991 rotate,
3992 twiddle1re,
3993 twiddle1im,
3994 twiddle2re,
3995 twiddle2im,
3996 twiddle3re,
3997 twiddle3im,
3998 twiddle4re,
3999 twiddle4im,
4000 twiddle5re,
4001 twiddle5im,
4002 twiddle6re,
4003 twiddle6im,
4004 twiddle7re,
4005 twiddle7im,
4006 twiddle8re,
4007 twiddle8im,
4008 twiddle9re,
4009 twiddle9im,
4010 twiddle10re,
4011 twiddle10im,
4012 twiddle11re,
4013 twiddle11im,
4014 twiddle12re,
4015 twiddle12im,
4016 twiddle13re,
4017 twiddle13im,
4018 twiddle14re,
4019 twiddle14im,
4020 }
4021 }
4022
4023 #[inline(always)]
4024 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
4025 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
4026
4027 let out = self.perform_parallel_fft_direct(values);
4028
4029 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
4030 }
4031
4032 #[inline(always)]
4033 pub(crate) unsafe fn perform_parallel_fft_contiguous(
4034 &self,
4035 mut buffer: impl WasmSimdArrayMut<f32>,
4036 ) {
4037 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56});
4038
4039 let values = [
4040 extract_lo_hi_f32(input_packed[0], input_packed[14]),
4041 extract_hi_lo_f32(input_packed[0], input_packed[15]),
4042 extract_lo_hi_f32(input_packed[1], input_packed[15]),
4043 extract_hi_lo_f32(input_packed[1], input_packed[16]),
4044 extract_lo_hi_f32(input_packed[2], input_packed[16]),
4045 extract_hi_lo_f32(input_packed[2], input_packed[17]),
4046 extract_lo_hi_f32(input_packed[3], input_packed[17]),
4047 extract_hi_lo_f32(input_packed[3], input_packed[18]),
4048 extract_lo_hi_f32(input_packed[4], input_packed[18]),
4049 extract_hi_lo_f32(input_packed[4], input_packed[19]),
4050 extract_lo_hi_f32(input_packed[5], input_packed[19]),
4051 extract_hi_lo_f32(input_packed[5], input_packed[20]),
4052 extract_lo_hi_f32(input_packed[6], input_packed[20]),
4053 extract_hi_lo_f32(input_packed[6], input_packed[21]),
4054 extract_lo_hi_f32(input_packed[7], input_packed[21]),
4055 extract_hi_lo_f32(input_packed[7], input_packed[22]),
4056 extract_lo_hi_f32(input_packed[8], input_packed[22]),
4057 extract_hi_lo_f32(input_packed[8], input_packed[23]),
4058 extract_lo_hi_f32(input_packed[9], input_packed[23]),
4059 extract_hi_lo_f32(input_packed[9], input_packed[24]),
4060 extract_lo_hi_f32(input_packed[10], input_packed[24]),
4061 extract_hi_lo_f32(input_packed[10], input_packed[25]),
4062 extract_lo_hi_f32(input_packed[11], input_packed[25]),
4063 extract_hi_lo_f32(input_packed[11], input_packed[26]),
4064 extract_lo_hi_f32(input_packed[12], input_packed[26]),
4065 extract_hi_lo_f32(input_packed[12], input_packed[27]),
4066 extract_lo_hi_f32(input_packed[13], input_packed[27]),
4067 extract_hi_lo_f32(input_packed[13], input_packed[28]),
4068 extract_lo_hi_f32(input_packed[14], input_packed[28]),
4069 ];
4070
4071 let out = self.perform_parallel_fft_direct(values);
4072
4073 let out_packed = [
4074 extract_lo_lo_f32(out[0], out[1]),
4075 extract_lo_lo_f32(out[2], out[3]),
4076 extract_lo_lo_f32(out[4], out[5]),
4077 extract_lo_lo_f32(out[6], out[7]),
4078 extract_lo_lo_f32(out[8], out[9]),
4079 extract_lo_lo_f32(out[10], out[11]),
4080 extract_lo_lo_f32(out[12], out[13]),
4081 extract_lo_lo_f32(out[14], out[15]),
4082 extract_lo_lo_f32(out[16], out[17]),
4083 extract_lo_lo_f32(out[18], out[19]),
4084 extract_lo_lo_f32(out[20], out[21]),
4085 extract_lo_lo_f32(out[22], out[23]),
4086 extract_lo_lo_f32(out[24], out[25]),
4087 extract_lo_lo_f32(out[26], out[27]),
4088 extract_lo_hi_f32(out[28], out[0]),
4089 extract_hi_hi_f32(out[1], out[2]),
4090 extract_hi_hi_f32(out[3], out[4]),
4091 extract_hi_hi_f32(out[5], out[6]),
4092 extract_hi_hi_f32(out[7], out[8]),
4093 extract_hi_hi_f32(out[9], out[10]),
4094 extract_hi_hi_f32(out[11], out[12]),
4095 extract_hi_hi_f32(out[13], out[14]),
4096 extract_hi_hi_f32(out[15], out[16]),
4097 extract_hi_hi_f32(out[17], out[18]),
4098 extract_hi_hi_f32(out[19], out[20]),
4099 extract_hi_hi_f32(out[21], out[22]),
4100 extract_hi_hi_f32(out[23], out[24]),
4101 extract_hi_hi_f32(out[25], out[26]),
4102 extract_hi_hi_f32(out[27], out[28]),
4103 ];
4104
4105 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
4106 }
4107
4108 #[inline(always)]
4109 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 29]) -> [v128; 29] {
4110 let [x1p28, x1m28] = parallel_fft2_interleaved_f32(values[1], values[28]);
4111 let [x2p27, x2m27] = parallel_fft2_interleaved_f32(values[2], values[27]);
4112 let [x3p26, x3m26] = parallel_fft2_interleaved_f32(values[3], values[26]);
4113 let [x4p25, x4m25] = parallel_fft2_interleaved_f32(values[4], values[25]);
4114 let [x5p24, x5m24] = parallel_fft2_interleaved_f32(values[5], values[24]);
4115 let [x6p23, x6m23] = parallel_fft2_interleaved_f32(values[6], values[23]);
4116 let [x7p22, x7m22] = parallel_fft2_interleaved_f32(values[7], values[22]);
4117 let [x8p21, x8m21] = parallel_fft2_interleaved_f32(values[8], values[21]);
4118 let [x9p20, x9m20] = parallel_fft2_interleaved_f32(values[9], values[20]);
4119 let [x10p19, x10m19] = parallel_fft2_interleaved_f32(values[10], values[19]);
4120 let [x11p18, x11m18] = parallel_fft2_interleaved_f32(values[11], values[18]);
4121 let [x12p17, x12m17] = parallel_fft2_interleaved_f32(values[12], values[17]);
4122 let [x13p16, x13m16] = parallel_fft2_interleaved_f32(values[13], values[16]);
4123 let [x14p15, x14m15] = parallel_fft2_interleaved_f32(values[14], values[15]);
4124
4125 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p28);
4126 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p27);
4127 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p26);
4128 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p25);
4129 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p24);
4130 let t_a1_6 = f32x4_mul(self.twiddle6re, x6p23);
4131 let t_a1_7 = f32x4_mul(self.twiddle7re, x7p22);
4132 let t_a1_8 = f32x4_mul(self.twiddle8re, x8p21);
4133 let t_a1_9 = f32x4_mul(self.twiddle9re, x9p20);
4134 let t_a1_10 = f32x4_mul(self.twiddle10re, x10p19);
4135 let t_a1_11 = f32x4_mul(self.twiddle11re, x11p18);
4136 let t_a1_12 = f32x4_mul(self.twiddle12re, x12p17);
4137 let t_a1_13 = f32x4_mul(self.twiddle13re, x13p16);
4138 let t_a1_14 = f32x4_mul(self.twiddle14re, x14p15);
4139 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p28);
4140 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p27);
4141 let t_a2_3 = f32x4_mul(self.twiddle6re, x3p26);
4142 let t_a2_4 = f32x4_mul(self.twiddle8re, x4p25);
4143 let t_a2_5 = f32x4_mul(self.twiddle10re, x5p24);
4144 let t_a2_6 = f32x4_mul(self.twiddle12re, x6p23);
4145 let t_a2_7 = f32x4_mul(self.twiddle14re, x7p22);
4146 let t_a2_8 = f32x4_mul(self.twiddle13re, x8p21);
4147 let t_a2_9 = f32x4_mul(self.twiddle11re, x9p20);
4148 let t_a2_10 = f32x4_mul(self.twiddle9re, x10p19);
4149 let t_a2_11 = f32x4_mul(self.twiddle7re, x11p18);
4150 let t_a2_12 = f32x4_mul(self.twiddle5re, x12p17);
4151 let t_a2_13 = f32x4_mul(self.twiddle3re, x13p16);
4152 let t_a2_14 = f32x4_mul(self.twiddle1re, x14p15);
4153 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p28);
4154 let t_a3_2 = f32x4_mul(self.twiddle6re, x2p27);
4155 let t_a3_3 = f32x4_mul(self.twiddle9re, x3p26);
4156 let t_a3_4 = f32x4_mul(self.twiddle12re, x4p25);
4157 let t_a3_5 = f32x4_mul(self.twiddle14re, x5p24);
4158 let t_a3_6 = f32x4_mul(self.twiddle11re, x6p23);
4159 let t_a3_7 = f32x4_mul(self.twiddle8re, x7p22);
4160 let t_a3_8 = f32x4_mul(self.twiddle5re, x8p21);
4161 let t_a3_9 = f32x4_mul(self.twiddle2re, x9p20);
4162 let t_a3_10 = f32x4_mul(self.twiddle1re, x10p19);
4163 let t_a3_11 = f32x4_mul(self.twiddle4re, x11p18);
4164 let t_a3_12 = f32x4_mul(self.twiddle7re, x12p17);
4165 let t_a3_13 = f32x4_mul(self.twiddle10re, x13p16);
4166 let t_a3_14 = f32x4_mul(self.twiddle13re, x14p15);
4167 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p28);
4168 let t_a4_2 = f32x4_mul(self.twiddle8re, x2p27);
4169 let t_a4_3 = f32x4_mul(self.twiddle12re, x3p26);
4170 let t_a4_4 = f32x4_mul(self.twiddle13re, x4p25);
4171 let t_a4_5 = f32x4_mul(self.twiddle9re, x5p24);
4172 let t_a4_6 = f32x4_mul(self.twiddle5re, x6p23);
4173 let t_a4_7 = f32x4_mul(self.twiddle1re, x7p22);
4174 let t_a4_8 = f32x4_mul(self.twiddle3re, x8p21);
4175 let t_a4_9 = f32x4_mul(self.twiddle7re, x9p20);
4176 let t_a4_10 = f32x4_mul(self.twiddle11re, x10p19);
4177 let t_a4_11 = f32x4_mul(self.twiddle14re, x11p18);
4178 let t_a4_12 = f32x4_mul(self.twiddle10re, x12p17);
4179 let t_a4_13 = f32x4_mul(self.twiddle6re, x13p16);
4180 let t_a4_14 = f32x4_mul(self.twiddle2re, x14p15);
4181 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p28);
4182 let t_a5_2 = f32x4_mul(self.twiddle10re, x2p27);
4183 let t_a5_3 = f32x4_mul(self.twiddle14re, x3p26);
4184 let t_a5_4 = f32x4_mul(self.twiddle9re, x4p25);
4185 let t_a5_5 = f32x4_mul(self.twiddle4re, x5p24);
4186 let t_a5_6 = f32x4_mul(self.twiddle1re, x6p23);
4187 let t_a5_7 = f32x4_mul(self.twiddle6re, x7p22);
4188 let t_a5_8 = f32x4_mul(self.twiddle11re, x8p21);
4189 let t_a5_9 = f32x4_mul(self.twiddle13re, x9p20);
4190 let t_a5_10 = f32x4_mul(self.twiddle8re, x10p19);
4191 let t_a5_11 = f32x4_mul(self.twiddle3re, x11p18);
4192 let t_a5_12 = f32x4_mul(self.twiddle2re, x12p17);
4193 let t_a5_13 = f32x4_mul(self.twiddle7re, x13p16);
4194 let t_a5_14 = f32x4_mul(self.twiddle12re, x14p15);
4195 let t_a6_1 = f32x4_mul(self.twiddle6re, x1p28);
4196 let t_a6_2 = f32x4_mul(self.twiddle12re, x2p27);
4197 let t_a6_3 = f32x4_mul(self.twiddle11re, x3p26);
4198 let t_a6_4 = f32x4_mul(self.twiddle5re, x4p25);
4199 let t_a6_5 = f32x4_mul(self.twiddle1re, x5p24);
4200 let t_a6_6 = f32x4_mul(self.twiddle7re, x6p23);
4201 let t_a6_7 = f32x4_mul(self.twiddle13re, x7p22);
4202 let t_a6_8 = f32x4_mul(self.twiddle10re, x8p21);
4203 let t_a6_9 = f32x4_mul(self.twiddle4re, x9p20);
4204 let t_a6_10 = f32x4_mul(self.twiddle2re, x10p19);
4205 let t_a6_11 = f32x4_mul(self.twiddle8re, x11p18);
4206 let t_a6_12 = f32x4_mul(self.twiddle14re, x12p17);
4207 let t_a6_13 = f32x4_mul(self.twiddle9re, x13p16);
4208 let t_a6_14 = f32x4_mul(self.twiddle3re, x14p15);
4209 let t_a7_1 = f32x4_mul(self.twiddle7re, x1p28);
4210 let t_a7_2 = f32x4_mul(self.twiddle14re, x2p27);
4211 let t_a7_3 = f32x4_mul(self.twiddle8re, x3p26);
4212 let t_a7_4 = f32x4_mul(self.twiddle1re, x4p25);
4213 let t_a7_5 = f32x4_mul(self.twiddle6re, x5p24);
4214 let t_a7_6 = f32x4_mul(self.twiddle13re, x6p23);
4215 let t_a7_7 = f32x4_mul(self.twiddle9re, x7p22);
4216 let t_a7_8 = f32x4_mul(self.twiddle2re, x8p21);
4217 let t_a7_9 = f32x4_mul(self.twiddle5re, x9p20);
4218 let t_a7_10 = f32x4_mul(self.twiddle12re, x10p19);
4219 let t_a7_11 = f32x4_mul(self.twiddle10re, x11p18);
4220 let t_a7_12 = f32x4_mul(self.twiddle3re, x12p17);
4221 let t_a7_13 = f32x4_mul(self.twiddle4re, x13p16);
4222 let t_a7_14 = f32x4_mul(self.twiddle11re, x14p15);
4223 let t_a8_1 = f32x4_mul(self.twiddle8re, x1p28);
4224 let t_a8_2 = f32x4_mul(self.twiddle13re, x2p27);
4225 let t_a8_3 = f32x4_mul(self.twiddle5re, x3p26);
4226 let t_a8_4 = f32x4_mul(self.twiddle3re, x4p25);
4227 let t_a8_5 = f32x4_mul(self.twiddle11re, x5p24);
4228 let t_a8_6 = f32x4_mul(self.twiddle10re, x6p23);
4229 let t_a8_7 = f32x4_mul(self.twiddle2re, x7p22);
4230 let t_a8_8 = f32x4_mul(self.twiddle6re, x8p21);
4231 let t_a8_9 = f32x4_mul(self.twiddle14re, x9p20);
4232 let t_a8_10 = f32x4_mul(self.twiddle7re, x10p19);
4233 let t_a8_11 = f32x4_mul(self.twiddle1re, x11p18);
4234 let t_a8_12 = f32x4_mul(self.twiddle9re, x12p17);
4235 let t_a8_13 = f32x4_mul(self.twiddle12re, x13p16);
4236 let t_a8_14 = f32x4_mul(self.twiddle4re, x14p15);
4237 let t_a9_1 = f32x4_mul(self.twiddle9re, x1p28);
4238 let t_a9_2 = f32x4_mul(self.twiddle11re, x2p27);
4239 let t_a9_3 = f32x4_mul(self.twiddle2re, x3p26);
4240 let t_a9_4 = f32x4_mul(self.twiddle7re, x4p25);
4241 let t_a9_5 = f32x4_mul(self.twiddle13re, x5p24);
4242 let t_a9_6 = f32x4_mul(self.twiddle4re, x6p23);
4243 let t_a9_7 = f32x4_mul(self.twiddle5re, x7p22);
4244 let t_a9_8 = f32x4_mul(self.twiddle14re, x8p21);
4245 let t_a9_9 = f32x4_mul(self.twiddle6re, x9p20);
4246 let t_a9_10 = f32x4_mul(self.twiddle3re, x10p19);
4247 let t_a9_11 = f32x4_mul(self.twiddle12re, x11p18);
4248 let t_a9_12 = f32x4_mul(self.twiddle8re, x12p17);
4249 let t_a9_13 = f32x4_mul(self.twiddle1re, x13p16);
4250 let t_a9_14 = f32x4_mul(self.twiddle10re, x14p15);
4251 let t_a10_1 = f32x4_mul(self.twiddle10re, x1p28);
4252 let t_a10_2 = f32x4_mul(self.twiddle9re, x2p27);
4253 let t_a10_3 = f32x4_mul(self.twiddle1re, x3p26);
4254 let t_a10_4 = f32x4_mul(self.twiddle11re, x4p25);
4255 let t_a10_5 = f32x4_mul(self.twiddle8re, x5p24);
4256 let t_a10_6 = f32x4_mul(self.twiddle2re, x6p23);
4257 let t_a10_7 = f32x4_mul(self.twiddle12re, x7p22);
4258 let t_a10_8 = f32x4_mul(self.twiddle7re, x8p21);
4259 let t_a10_9 = f32x4_mul(self.twiddle3re, x9p20);
4260 let t_a10_10 = f32x4_mul(self.twiddle13re, x10p19);
4261 let t_a10_11 = f32x4_mul(self.twiddle6re, x11p18);
4262 let t_a10_12 = f32x4_mul(self.twiddle4re, x12p17);
4263 let t_a10_13 = f32x4_mul(self.twiddle14re, x13p16);
4264 let t_a10_14 = f32x4_mul(self.twiddle5re, x14p15);
4265 let t_a11_1 = f32x4_mul(self.twiddle11re, x1p28);
4266 let t_a11_2 = f32x4_mul(self.twiddle7re, x2p27);
4267 let t_a11_3 = f32x4_mul(self.twiddle4re, x3p26);
4268 let t_a11_4 = f32x4_mul(self.twiddle14re, x4p25);
4269 let t_a11_5 = f32x4_mul(self.twiddle3re, x5p24);
4270 let t_a11_6 = f32x4_mul(self.twiddle8re, x6p23);
4271 let t_a11_7 = f32x4_mul(self.twiddle10re, x7p22);
4272 let t_a11_8 = f32x4_mul(self.twiddle1re, x8p21);
4273 let t_a11_9 = f32x4_mul(self.twiddle12re, x9p20);
4274 let t_a11_10 = f32x4_mul(self.twiddle6re, x10p19);
4275 let t_a11_11 = f32x4_mul(self.twiddle5re, x11p18);
4276 let t_a11_12 = f32x4_mul(self.twiddle13re, x12p17);
4277 let t_a11_13 = f32x4_mul(self.twiddle2re, x13p16);
4278 let t_a11_14 = f32x4_mul(self.twiddle9re, x14p15);
4279 let t_a12_1 = f32x4_mul(self.twiddle12re, x1p28);
4280 let t_a12_2 = f32x4_mul(self.twiddle5re, x2p27);
4281 let t_a12_3 = f32x4_mul(self.twiddle7re, x3p26);
4282 let t_a12_4 = f32x4_mul(self.twiddle10re, x4p25);
4283 let t_a12_5 = f32x4_mul(self.twiddle2re, x5p24);
4284 let t_a12_6 = f32x4_mul(self.twiddle14re, x6p23);
4285 let t_a12_7 = f32x4_mul(self.twiddle3re, x7p22);
4286 let t_a12_8 = f32x4_mul(self.twiddle9re, x8p21);
4287 let t_a12_9 = f32x4_mul(self.twiddle8re, x9p20);
4288 let t_a12_10 = f32x4_mul(self.twiddle4re, x10p19);
4289 let t_a12_11 = f32x4_mul(self.twiddle13re, x11p18);
4290 let t_a12_12 = f32x4_mul(self.twiddle1re, x12p17);
4291 let t_a12_13 = f32x4_mul(self.twiddle11re, x13p16);
4292 let t_a12_14 = f32x4_mul(self.twiddle6re, x14p15);
4293 let t_a13_1 = f32x4_mul(self.twiddle13re, x1p28);
4294 let t_a13_2 = f32x4_mul(self.twiddle3re, x2p27);
4295 let t_a13_3 = f32x4_mul(self.twiddle10re, x3p26);
4296 let t_a13_4 = f32x4_mul(self.twiddle6re, x4p25);
4297 let t_a13_5 = f32x4_mul(self.twiddle7re, x5p24);
4298 let t_a13_6 = f32x4_mul(self.twiddle9re, x6p23);
4299 let t_a13_7 = f32x4_mul(self.twiddle4re, x7p22);
4300 let t_a13_8 = f32x4_mul(self.twiddle12re, x8p21);
4301 let t_a13_9 = f32x4_mul(self.twiddle1re, x9p20);
4302 let t_a13_10 = f32x4_mul(self.twiddle14re, x10p19);
4303 let t_a13_11 = f32x4_mul(self.twiddle2re, x11p18);
4304 let t_a13_12 = f32x4_mul(self.twiddle11re, x12p17);
4305 let t_a13_13 = f32x4_mul(self.twiddle5re, x13p16);
4306 let t_a13_14 = f32x4_mul(self.twiddle8re, x14p15);
4307 let t_a14_1 = f32x4_mul(self.twiddle14re, x1p28);
4308 let t_a14_2 = f32x4_mul(self.twiddle1re, x2p27);
4309 let t_a14_3 = f32x4_mul(self.twiddle13re, x3p26);
4310 let t_a14_4 = f32x4_mul(self.twiddle2re, x4p25);
4311 let t_a14_5 = f32x4_mul(self.twiddle12re, x5p24);
4312 let t_a14_6 = f32x4_mul(self.twiddle3re, x6p23);
4313 let t_a14_7 = f32x4_mul(self.twiddle11re, x7p22);
4314 let t_a14_8 = f32x4_mul(self.twiddle4re, x8p21);
4315 let t_a14_9 = f32x4_mul(self.twiddle10re, x9p20);
4316 let t_a14_10 = f32x4_mul(self.twiddle5re, x10p19);
4317 let t_a14_11 = f32x4_mul(self.twiddle9re, x11p18);
4318 let t_a14_12 = f32x4_mul(self.twiddle6re, x12p17);
4319 let t_a14_13 = f32x4_mul(self.twiddle8re, x13p16);
4320 let t_a14_14 = f32x4_mul(self.twiddle7re, x14p15);
4321
4322 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m28);
4323 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m27);
4324 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m26);
4325 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m25);
4326 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m24);
4327 let t_b1_6 = f32x4_mul(self.twiddle6im, x6m23);
4328 let t_b1_7 = f32x4_mul(self.twiddle7im, x7m22);
4329 let t_b1_8 = f32x4_mul(self.twiddle8im, x8m21);
4330 let t_b1_9 = f32x4_mul(self.twiddle9im, x9m20);
4331 let t_b1_10 = f32x4_mul(self.twiddle10im, x10m19);
4332 let t_b1_11 = f32x4_mul(self.twiddle11im, x11m18);
4333 let t_b1_12 = f32x4_mul(self.twiddle12im, x12m17);
4334 let t_b1_13 = f32x4_mul(self.twiddle13im, x13m16);
4335 let t_b1_14 = f32x4_mul(self.twiddle14im, x14m15);
4336 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m28);
4337 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m27);
4338 let t_b2_3 = f32x4_mul(self.twiddle6im, x3m26);
4339 let t_b2_4 = f32x4_mul(self.twiddle8im, x4m25);
4340 let t_b2_5 = f32x4_mul(self.twiddle10im, x5m24);
4341 let t_b2_6 = f32x4_mul(self.twiddle12im, x6m23);
4342 let t_b2_7 = f32x4_mul(self.twiddle14im, x7m22);
4343 let t_b2_8 = f32x4_mul(self.twiddle13im, x8m21);
4344 let t_b2_9 = f32x4_mul(self.twiddle11im, x9m20);
4345 let t_b2_10 = f32x4_mul(self.twiddle9im, x10m19);
4346 let t_b2_11 = f32x4_mul(self.twiddle7im, x11m18);
4347 let t_b2_12 = f32x4_mul(self.twiddle5im, x12m17);
4348 let t_b2_13 = f32x4_mul(self.twiddle3im, x13m16);
4349 let t_b2_14 = f32x4_mul(self.twiddle1im, x14m15);
4350 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m28);
4351 let t_b3_2 = f32x4_mul(self.twiddle6im, x2m27);
4352 let t_b3_3 = f32x4_mul(self.twiddle9im, x3m26);
4353 let t_b3_4 = f32x4_mul(self.twiddle12im, x4m25);
4354 let t_b3_5 = f32x4_mul(self.twiddle14im, x5m24);
4355 let t_b3_6 = f32x4_mul(self.twiddle11im, x6m23);
4356 let t_b3_7 = f32x4_mul(self.twiddle8im, x7m22);
4357 let t_b3_8 = f32x4_mul(self.twiddle5im, x8m21);
4358 let t_b3_9 = f32x4_mul(self.twiddle2im, x9m20);
4359 let t_b3_10 = f32x4_mul(self.twiddle1im, x10m19);
4360 let t_b3_11 = f32x4_mul(self.twiddle4im, x11m18);
4361 let t_b3_12 = f32x4_mul(self.twiddle7im, x12m17);
4362 let t_b3_13 = f32x4_mul(self.twiddle10im, x13m16);
4363 let t_b3_14 = f32x4_mul(self.twiddle13im, x14m15);
4364 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m28);
4365 let t_b4_2 = f32x4_mul(self.twiddle8im, x2m27);
4366 let t_b4_3 = f32x4_mul(self.twiddle12im, x3m26);
4367 let t_b4_4 = f32x4_mul(self.twiddle13im, x4m25);
4368 let t_b4_5 = f32x4_mul(self.twiddle9im, x5m24);
4369 let t_b4_6 = f32x4_mul(self.twiddle5im, x6m23);
4370 let t_b4_7 = f32x4_mul(self.twiddle1im, x7m22);
4371 let t_b4_8 = f32x4_mul(self.twiddle3im, x8m21);
4372 let t_b4_9 = f32x4_mul(self.twiddle7im, x9m20);
4373 let t_b4_10 = f32x4_mul(self.twiddle11im, x10m19);
4374 let t_b4_11 = f32x4_mul(self.twiddle14im, x11m18);
4375 let t_b4_12 = f32x4_mul(self.twiddle10im, x12m17);
4376 let t_b4_13 = f32x4_mul(self.twiddle6im, x13m16);
4377 let t_b4_14 = f32x4_mul(self.twiddle2im, x14m15);
4378 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m28);
4379 let t_b5_2 = f32x4_mul(self.twiddle10im, x2m27);
4380 let t_b5_3 = f32x4_mul(self.twiddle14im, x3m26);
4381 let t_b5_4 = f32x4_mul(self.twiddle9im, x4m25);
4382 let t_b5_5 = f32x4_mul(self.twiddle4im, x5m24);
4383 let t_b5_6 = f32x4_mul(self.twiddle1im, x6m23);
4384 let t_b5_7 = f32x4_mul(self.twiddle6im, x7m22);
4385 let t_b5_8 = f32x4_mul(self.twiddle11im, x8m21);
4386 let t_b5_9 = f32x4_mul(self.twiddle13im, x9m20);
4387 let t_b5_10 = f32x4_mul(self.twiddle8im, x10m19);
4388 let t_b5_11 = f32x4_mul(self.twiddle3im, x11m18);
4389 let t_b5_12 = f32x4_mul(self.twiddle2im, x12m17);
4390 let t_b5_13 = f32x4_mul(self.twiddle7im, x13m16);
4391 let t_b5_14 = f32x4_mul(self.twiddle12im, x14m15);
4392 let t_b6_1 = f32x4_mul(self.twiddle6im, x1m28);
4393 let t_b6_2 = f32x4_mul(self.twiddle12im, x2m27);
4394 let t_b6_3 = f32x4_mul(self.twiddle11im, x3m26);
4395 let t_b6_4 = f32x4_mul(self.twiddle5im, x4m25);
4396 let t_b6_5 = f32x4_mul(self.twiddle1im, x5m24);
4397 let t_b6_6 = f32x4_mul(self.twiddle7im, x6m23);
4398 let t_b6_7 = f32x4_mul(self.twiddle13im, x7m22);
4399 let t_b6_8 = f32x4_mul(self.twiddle10im, x8m21);
4400 let t_b6_9 = f32x4_mul(self.twiddle4im, x9m20);
4401 let t_b6_10 = f32x4_mul(self.twiddle2im, x10m19);
4402 let t_b6_11 = f32x4_mul(self.twiddle8im, x11m18);
4403 let t_b6_12 = f32x4_mul(self.twiddle14im, x12m17);
4404 let t_b6_13 = f32x4_mul(self.twiddle9im, x13m16);
4405 let t_b6_14 = f32x4_mul(self.twiddle3im, x14m15);
4406 let t_b7_1 = f32x4_mul(self.twiddle7im, x1m28);
4407 let t_b7_2 = f32x4_mul(self.twiddle14im, x2m27);
4408 let t_b7_3 = f32x4_mul(self.twiddle8im, x3m26);
4409 let t_b7_4 = f32x4_mul(self.twiddle1im, x4m25);
4410 let t_b7_5 = f32x4_mul(self.twiddle6im, x5m24);
4411 let t_b7_6 = f32x4_mul(self.twiddle13im, x6m23);
4412 let t_b7_7 = f32x4_mul(self.twiddle9im, x7m22);
4413 let t_b7_8 = f32x4_mul(self.twiddle2im, x8m21);
4414 let t_b7_9 = f32x4_mul(self.twiddle5im, x9m20);
4415 let t_b7_10 = f32x4_mul(self.twiddle12im, x10m19);
4416 let t_b7_11 = f32x4_mul(self.twiddle10im, x11m18);
4417 let t_b7_12 = f32x4_mul(self.twiddle3im, x12m17);
4418 let t_b7_13 = f32x4_mul(self.twiddle4im, x13m16);
4419 let t_b7_14 = f32x4_mul(self.twiddle11im, x14m15);
4420 let t_b8_1 = f32x4_mul(self.twiddle8im, x1m28);
4421 let t_b8_2 = f32x4_mul(self.twiddle13im, x2m27);
4422 let t_b8_3 = f32x4_mul(self.twiddle5im, x3m26);
4423 let t_b8_4 = f32x4_mul(self.twiddle3im, x4m25);
4424 let t_b8_5 = f32x4_mul(self.twiddle11im, x5m24);
4425 let t_b8_6 = f32x4_mul(self.twiddle10im, x6m23);
4426 let t_b8_7 = f32x4_mul(self.twiddle2im, x7m22);
4427 let t_b8_8 = f32x4_mul(self.twiddle6im, x8m21);
4428 let t_b8_9 = f32x4_mul(self.twiddle14im, x9m20);
4429 let t_b8_10 = f32x4_mul(self.twiddle7im, x10m19);
4430 let t_b8_11 = f32x4_mul(self.twiddle1im, x11m18);
4431 let t_b8_12 = f32x4_mul(self.twiddle9im, x12m17);
4432 let t_b8_13 = f32x4_mul(self.twiddle12im, x13m16);
4433 let t_b8_14 = f32x4_mul(self.twiddle4im, x14m15);
4434 let t_b9_1 = f32x4_mul(self.twiddle9im, x1m28);
4435 let t_b9_2 = f32x4_mul(self.twiddle11im, x2m27);
4436 let t_b9_3 = f32x4_mul(self.twiddle2im, x3m26);
4437 let t_b9_4 = f32x4_mul(self.twiddle7im, x4m25);
4438 let t_b9_5 = f32x4_mul(self.twiddle13im, x5m24);
4439 let t_b9_6 = f32x4_mul(self.twiddle4im, x6m23);
4440 let t_b9_7 = f32x4_mul(self.twiddle5im, x7m22);
4441 let t_b9_8 = f32x4_mul(self.twiddle14im, x8m21);
4442 let t_b9_9 = f32x4_mul(self.twiddle6im, x9m20);
4443 let t_b9_10 = f32x4_mul(self.twiddle3im, x10m19);
4444 let t_b9_11 = f32x4_mul(self.twiddle12im, x11m18);
4445 let t_b9_12 = f32x4_mul(self.twiddle8im, x12m17);
4446 let t_b9_13 = f32x4_mul(self.twiddle1im, x13m16);
4447 let t_b9_14 = f32x4_mul(self.twiddle10im, x14m15);
4448 let t_b10_1 = f32x4_mul(self.twiddle10im, x1m28);
4449 let t_b10_2 = f32x4_mul(self.twiddle9im, x2m27);
4450 let t_b10_3 = f32x4_mul(self.twiddle1im, x3m26);
4451 let t_b10_4 = f32x4_mul(self.twiddle11im, x4m25);
4452 let t_b10_5 = f32x4_mul(self.twiddle8im, x5m24);
4453 let t_b10_6 = f32x4_mul(self.twiddle2im, x6m23);
4454 let t_b10_7 = f32x4_mul(self.twiddle12im, x7m22);
4455 let t_b10_8 = f32x4_mul(self.twiddle7im, x8m21);
4456 let t_b10_9 = f32x4_mul(self.twiddle3im, x9m20);
4457 let t_b10_10 = f32x4_mul(self.twiddle13im, x10m19);
4458 let t_b10_11 = f32x4_mul(self.twiddle6im, x11m18);
4459 let t_b10_12 = f32x4_mul(self.twiddle4im, x12m17);
4460 let t_b10_13 = f32x4_mul(self.twiddle14im, x13m16);
4461 let t_b10_14 = f32x4_mul(self.twiddle5im, x14m15);
4462 let t_b11_1 = f32x4_mul(self.twiddle11im, x1m28);
4463 let t_b11_2 = f32x4_mul(self.twiddle7im, x2m27);
4464 let t_b11_3 = f32x4_mul(self.twiddle4im, x3m26);
4465 let t_b11_4 = f32x4_mul(self.twiddle14im, x4m25);
4466 let t_b11_5 = f32x4_mul(self.twiddle3im, x5m24);
4467 let t_b11_6 = f32x4_mul(self.twiddle8im, x6m23);
4468 let t_b11_7 = f32x4_mul(self.twiddle10im, x7m22);
4469 let t_b11_8 = f32x4_mul(self.twiddle1im, x8m21);
4470 let t_b11_9 = f32x4_mul(self.twiddle12im, x9m20);
4471 let t_b11_10 = f32x4_mul(self.twiddle6im, x10m19);
4472 let t_b11_11 = f32x4_mul(self.twiddle5im, x11m18);
4473 let t_b11_12 = f32x4_mul(self.twiddle13im, x12m17);
4474 let t_b11_13 = f32x4_mul(self.twiddle2im, x13m16);
4475 let t_b11_14 = f32x4_mul(self.twiddle9im, x14m15);
4476 let t_b12_1 = f32x4_mul(self.twiddle12im, x1m28);
4477 let t_b12_2 = f32x4_mul(self.twiddle5im, x2m27);
4478 let t_b12_3 = f32x4_mul(self.twiddle7im, x3m26);
4479 let t_b12_4 = f32x4_mul(self.twiddle10im, x4m25);
4480 let t_b12_5 = f32x4_mul(self.twiddle2im, x5m24);
4481 let t_b12_6 = f32x4_mul(self.twiddle14im, x6m23);
4482 let t_b12_7 = f32x4_mul(self.twiddle3im, x7m22);
4483 let t_b12_8 = f32x4_mul(self.twiddle9im, x8m21);
4484 let t_b12_9 = f32x4_mul(self.twiddle8im, x9m20);
4485 let t_b12_10 = f32x4_mul(self.twiddle4im, x10m19);
4486 let t_b12_11 = f32x4_mul(self.twiddle13im, x11m18);
4487 let t_b12_12 = f32x4_mul(self.twiddle1im, x12m17);
4488 let t_b12_13 = f32x4_mul(self.twiddle11im, x13m16);
4489 let t_b12_14 = f32x4_mul(self.twiddle6im, x14m15);
4490 let t_b13_1 = f32x4_mul(self.twiddle13im, x1m28);
4491 let t_b13_2 = f32x4_mul(self.twiddle3im, x2m27);
4492 let t_b13_3 = f32x4_mul(self.twiddle10im, x3m26);
4493 let t_b13_4 = f32x4_mul(self.twiddle6im, x4m25);
4494 let t_b13_5 = f32x4_mul(self.twiddle7im, x5m24);
4495 let t_b13_6 = f32x4_mul(self.twiddle9im, x6m23);
4496 let t_b13_7 = f32x4_mul(self.twiddle4im, x7m22);
4497 let t_b13_8 = f32x4_mul(self.twiddle12im, x8m21);
4498 let t_b13_9 = f32x4_mul(self.twiddle1im, x9m20);
4499 let t_b13_10 = f32x4_mul(self.twiddle14im, x10m19);
4500 let t_b13_11 = f32x4_mul(self.twiddle2im, x11m18);
4501 let t_b13_12 = f32x4_mul(self.twiddle11im, x12m17);
4502 let t_b13_13 = f32x4_mul(self.twiddle5im, x13m16);
4503 let t_b13_14 = f32x4_mul(self.twiddle8im, x14m15);
4504 let t_b14_1 = f32x4_mul(self.twiddle14im, x1m28);
4505 let t_b14_2 = f32x4_mul(self.twiddle1im, x2m27);
4506 let t_b14_3 = f32x4_mul(self.twiddle13im, x3m26);
4507 let t_b14_4 = f32x4_mul(self.twiddle2im, x4m25);
4508 let t_b14_5 = f32x4_mul(self.twiddle12im, x5m24);
4509 let t_b14_6 = f32x4_mul(self.twiddle3im, x6m23);
4510 let t_b14_7 = f32x4_mul(self.twiddle11im, x7m22);
4511 let t_b14_8 = f32x4_mul(self.twiddle4im, x8m21);
4512 let t_b14_9 = f32x4_mul(self.twiddle10im, x9m20);
4513 let t_b14_10 = f32x4_mul(self.twiddle5im, x10m19);
4514 let t_b14_11 = f32x4_mul(self.twiddle9im, x11m18);
4515 let t_b14_12 = f32x4_mul(self.twiddle6im, x12m17);
4516 let t_b14_13 = f32x4_mul(self.twiddle8im, x13m16);
4517 let t_b14_14 = f32x4_mul(self.twiddle7im, x14m15);
4518
4519 let x0 = values[0];
4520 let t_a1 = calc_f32!(
4521 x0 + t_a1_1
4522 + t_a1_2
4523 + t_a1_3
4524 + t_a1_4
4525 + t_a1_5
4526 + t_a1_6
4527 + t_a1_7
4528 + t_a1_8
4529 + t_a1_9
4530 + t_a1_10
4531 + t_a1_11
4532 + t_a1_12
4533 + t_a1_13
4534 + t_a1_14
4535 );
4536 let t_a2 = calc_f32!(
4537 x0 + t_a2_1
4538 + t_a2_2
4539 + t_a2_3
4540 + t_a2_4
4541 + t_a2_5
4542 + t_a2_6
4543 + t_a2_7
4544 + t_a2_8
4545 + t_a2_9
4546 + t_a2_10
4547 + t_a2_11
4548 + t_a2_12
4549 + t_a2_13
4550 + t_a2_14
4551 );
4552 let t_a3 = calc_f32!(
4553 x0 + t_a3_1
4554 + t_a3_2
4555 + t_a3_3
4556 + t_a3_4
4557 + t_a3_5
4558 + t_a3_6
4559 + t_a3_7
4560 + t_a3_8
4561 + t_a3_9
4562 + t_a3_10
4563 + t_a3_11
4564 + t_a3_12
4565 + t_a3_13
4566 + t_a3_14
4567 );
4568 let t_a4 = calc_f32!(
4569 x0 + t_a4_1
4570 + t_a4_2
4571 + t_a4_3
4572 + t_a4_4
4573 + t_a4_5
4574 + t_a4_6
4575 + t_a4_7
4576 + t_a4_8
4577 + t_a4_9
4578 + t_a4_10
4579 + t_a4_11
4580 + t_a4_12
4581 + t_a4_13
4582 + t_a4_14
4583 );
4584 let t_a5 = calc_f32!(
4585 x0 + t_a5_1
4586 + t_a5_2
4587 + t_a5_3
4588 + t_a5_4
4589 + t_a5_5
4590 + t_a5_6
4591 + t_a5_7
4592 + t_a5_8
4593 + t_a5_9
4594 + t_a5_10
4595 + t_a5_11
4596 + t_a5_12
4597 + t_a5_13
4598 + t_a5_14
4599 );
4600 let t_a6 = calc_f32!(
4601 x0 + t_a6_1
4602 + t_a6_2
4603 + t_a6_3
4604 + t_a6_4
4605 + t_a6_5
4606 + t_a6_6
4607 + t_a6_7
4608 + t_a6_8
4609 + t_a6_9
4610 + t_a6_10
4611 + t_a6_11
4612 + t_a6_12
4613 + t_a6_13
4614 + t_a6_14
4615 );
4616 let t_a7 = calc_f32!(
4617 x0 + t_a7_1
4618 + t_a7_2
4619 + t_a7_3
4620 + t_a7_4
4621 + t_a7_5
4622 + t_a7_6
4623 + t_a7_7
4624 + t_a7_8
4625 + t_a7_9
4626 + t_a7_10
4627 + t_a7_11
4628 + t_a7_12
4629 + t_a7_13
4630 + t_a7_14
4631 );
4632 let t_a8 = calc_f32!(
4633 x0 + t_a8_1
4634 + t_a8_2
4635 + t_a8_3
4636 + t_a8_4
4637 + t_a8_5
4638 + t_a8_6
4639 + t_a8_7
4640 + t_a8_8
4641 + t_a8_9
4642 + t_a8_10
4643 + t_a8_11
4644 + t_a8_12
4645 + t_a8_13
4646 + t_a8_14
4647 );
4648 let t_a9 = calc_f32!(
4649 x0 + t_a9_1
4650 + t_a9_2
4651 + t_a9_3
4652 + t_a9_4
4653 + t_a9_5
4654 + t_a9_6
4655 + t_a9_7
4656 + t_a9_8
4657 + t_a9_9
4658 + t_a9_10
4659 + t_a9_11
4660 + t_a9_12
4661 + t_a9_13
4662 + t_a9_14
4663 );
4664 let t_a10 = calc_f32!(
4665 x0 + t_a10_1
4666 + t_a10_2
4667 + t_a10_3
4668 + t_a10_4
4669 + t_a10_5
4670 + t_a10_6
4671 + t_a10_7
4672 + t_a10_8
4673 + t_a10_9
4674 + t_a10_10
4675 + t_a10_11
4676 + t_a10_12
4677 + t_a10_13
4678 + t_a10_14
4679 );
4680 let t_a11 = calc_f32!(
4681 x0 + t_a11_1
4682 + t_a11_2
4683 + t_a11_3
4684 + t_a11_4
4685 + t_a11_5
4686 + t_a11_6
4687 + t_a11_7
4688 + t_a11_8
4689 + t_a11_9
4690 + t_a11_10
4691 + t_a11_11
4692 + t_a11_12
4693 + t_a11_13
4694 + t_a11_14
4695 );
4696 let t_a12 = calc_f32!(
4697 x0 + t_a12_1
4698 + t_a12_2
4699 + t_a12_3
4700 + t_a12_4
4701 + t_a12_5
4702 + t_a12_6
4703 + t_a12_7
4704 + t_a12_8
4705 + t_a12_9
4706 + t_a12_10
4707 + t_a12_11
4708 + t_a12_12
4709 + t_a12_13
4710 + t_a12_14
4711 );
4712 let t_a13 = calc_f32!(
4713 x0 + t_a13_1
4714 + t_a13_2
4715 + t_a13_3
4716 + t_a13_4
4717 + t_a13_5
4718 + t_a13_6
4719 + t_a13_7
4720 + t_a13_8
4721 + t_a13_9
4722 + t_a13_10
4723 + t_a13_11
4724 + t_a13_12
4725 + t_a13_13
4726 + t_a13_14
4727 );
4728 let t_a14 = calc_f32!(
4729 x0 + t_a14_1
4730 + t_a14_2
4731 + t_a14_3
4732 + t_a14_4
4733 + t_a14_5
4734 + t_a14_6
4735 + t_a14_7
4736 + t_a14_8
4737 + t_a14_9
4738 + t_a14_10
4739 + t_a14_11
4740 + t_a14_12
4741 + t_a14_13
4742 + t_a14_14
4743 );
4744
4745 let t_b1 = calc_f32!(
4746 t_b1_1
4747 + t_b1_2
4748 + t_b1_3
4749 + t_b1_4
4750 + t_b1_5
4751 + t_b1_6
4752 + t_b1_7
4753 + t_b1_8
4754 + t_b1_9
4755 + t_b1_10
4756 + t_b1_11
4757 + t_b1_12
4758 + t_b1_13
4759 + t_b1_14
4760 );
4761 let t_b2 = calc_f32!(
4762 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7
4763 - t_b2_8
4764 - t_b2_9
4765 - t_b2_10
4766 - t_b2_11
4767 - t_b2_12
4768 - t_b2_13
4769 - t_b2_14
4770 );
4771 let t_b3 = calc_f32!(
4772 t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7 - t_b3_8 - t_b3_9
4773 + t_b3_10
4774 + t_b3_11
4775 + t_b3_12
4776 + t_b3_13
4777 + t_b3_14
4778 );
4779 let t_b4 = calc_f32!(
4780 t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7
4781 + t_b4_8
4782 + t_b4_9
4783 + t_b4_10
4784 - t_b4_11
4785 - t_b4_12
4786 - t_b4_13
4787 - t_b4_14
4788 );
4789 let t_b5 = calc_f32!(
4790 t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6 + t_b5_7 + t_b5_8
4791 - t_b5_9
4792 - t_b5_10
4793 - t_b5_11
4794 + t_b5_12
4795 + t_b5_13
4796 + t_b5_14
4797 );
4798 let t_b6 = calc_f32!(
4799 t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 + t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9
4800 + t_b6_10
4801 + t_b6_11
4802 + t_b6_12
4803 - t_b6_13
4804 - t_b6_14
4805 );
4806 let t_b7 = calc_f32!(
4807 t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8
4808 + t_b7_9
4809 + t_b7_10
4810 - t_b7_11
4811 - t_b7_12
4812 + t_b7_13
4813 + t_b7_14
4814 );
4815 let t_b8 = calc_f32!(
4816 t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9
4817 - t_b8_10
4818 + t_b8_11
4819 + t_b8_12
4820 - t_b8_13
4821 - t_b8_14
4822 );
4823 let t_b9 = calc_f32!(
4824 t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 - t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8 - t_b9_9
4825 + t_b9_10
4826 + t_b9_11
4827 - t_b9_12
4828 + t_b9_13
4829 + t_b9_14
4830 );
4831 let t_b10 = calc_f32!(
4832 t_b10_1 - t_b10_2 + t_b10_3 + t_b10_4 - t_b10_5 + t_b10_6 + t_b10_7 - t_b10_8
4833 + t_b10_9
4834 + t_b10_10
4835 - t_b10_11
4836 + t_b10_12
4837 + t_b10_13
4838 - t_b10_14
4839 );
4840 let t_b11 = calc_f32!(
4841 t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 - t_b11_5 + t_b11_6 - t_b11_7 + t_b11_8 + t_b11_9
4842 - t_b11_10
4843 + t_b11_11
4844 - t_b11_12
4845 - t_b11_13
4846 + t_b11_14
4847 );
4848 let t_b12 = calc_f32!(
4849 t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 + t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 - t_b12_9
4850 + t_b12_10
4851 - t_b12_11
4852 - t_b12_12
4853 + t_b12_13
4854 - t_b12_14
4855 );
4856 let t_b13 = calc_f32!(
4857 t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 + t_b13_7 - t_b13_8
4858 + t_b13_9
4859 + t_b13_10
4860 - t_b13_11
4861 + t_b13_12
4862 - t_b13_13
4863 + t_b13_14
4864 );
4865 let t_b14 = calc_f32!(
4866 t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9
4867 - t_b14_10
4868 + t_b14_11
4869 - t_b14_12
4870 + t_b14_13
4871 - t_b14_14
4872 );
4873
4874 let t_b1_rot = self.rotate.rotate_both(t_b1);
4875 let t_b2_rot = self.rotate.rotate_both(t_b2);
4876 let t_b3_rot = self.rotate.rotate_both(t_b3);
4877 let t_b4_rot = self.rotate.rotate_both(t_b4);
4878 let t_b5_rot = self.rotate.rotate_both(t_b5);
4879 let t_b6_rot = self.rotate.rotate_both(t_b6);
4880 let t_b7_rot = self.rotate.rotate_both(t_b7);
4881 let t_b8_rot = self.rotate.rotate_both(t_b8);
4882 let t_b9_rot = self.rotate.rotate_both(t_b9);
4883 let t_b10_rot = self.rotate.rotate_both(t_b10);
4884 let t_b11_rot = self.rotate.rotate_both(t_b11);
4885 let t_b12_rot = self.rotate.rotate_both(t_b12);
4886 let t_b13_rot = self.rotate.rotate_both(t_b13);
4887 let t_b14_rot = self.rotate.rotate_both(t_b14);
4888
4889 let y0 = calc_f32!(
4890 x0 + x1p28
4891 + x2p27
4892 + x3p26
4893 + x4p25
4894 + x5p24
4895 + x6p23
4896 + x7p22
4897 + x8p21
4898 + x9p20
4899 + x10p19
4900 + x11p18
4901 + x12p17
4902 + x13p16
4903 + x14p15
4904 );
4905 let [y1, y28] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
4906 let [y2, y27] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
4907 let [y3, y26] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
4908 let [y4, y25] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
4909 let [y5, y24] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
4910 let [y6, y23] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
4911 let [y7, y22] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
4912 let [y8, y21] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
4913 let [y9, y20] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
4914 let [y10, y19] = parallel_fft2_interleaved_f32(t_a10, t_b10_rot);
4915 let [y11, y18] = parallel_fft2_interleaved_f32(t_a11, t_b11_rot);
4916 let [y12, y17] = parallel_fft2_interleaved_f32(t_a12, t_b12_rot);
4917 let [y13, y16] = parallel_fft2_interleaved_f32(t_a13, t_b13_rot);
4918 let [y14, y15] = parallel_fft2_interleaved_f32(t_a14, t_b14_rot);
4919 [
4920 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
4921 y19, y20, y21, y22, y23, y24, y25, y26, y27, y28,
4922 ]
4923 }
4924 }
4925
4926 // ____ ___ __ _ _ _ _ _
4927 // |___ \ / _ \ / /_ | || | | |__ (_) |_
4928 // __) | (_) | _____ | '_ \| || |_| '_ \| | __|
4929 // / __/ \__, | |_____| | (_) |__ _| |_) | | |_
4930 // |_____| /_/ \___/ |_| |_.__/|_|\__|
4931 //
4932
4933 pub struct WasmSimdF64Butterfly29<T> {
4934 direction: FftDirection,
4935 _phantom: std::marker::PhantomData<T>,
4936 rotate: Rotate90F64,
4937 twiddle1re: v128,
4938 twiddle1im: v128,
4939 twiddle2re: v128,
4940 twiddle2im: v128,
4941 twiddle3re: v128,
4942 twiddle3im: v128,
4943 twiddle4re: v128,
4944 twiddle4im: v128,
4945 twiddle5re: v128,
4946 twiddle5im: v128,
4947 twiddle6re: v128,
4948 twiddle6im: v128,
4949 twiddle7re: v128,
4950 twiddle7im: v128,
4951 twiddle8re: v128,
4952 twiddle8im: v128,
4953 twiddle9re: v128,
4954 twiddle9im: v128,
4955 twiddle10re: v128,
4956 twiddle10im: v128,
4957 twiddle11re: v128,
4958 twiddle11im: v128,
4959 twiddle12re: v128,
4960 twiddle12im: v128,
4961 twiddle13re: v128,
4962 twiddle13im: v128,
4963 twiddle14re: v128,
4964 twiddle14im: v128,
4965 }
4966
4967 boilerplate_fft_wasm_simd_f64_butterfly!(
4968 WasmSimdF64Butterfly29,
4969 29,
4970 |this: &WasmSimdF64Butterfly29<_>| this.direction
4971 );
4972 boilerplate_fft_wasm_simd_common_butterfly!(
4973 WasmSimdF64Butterfly29,
4974 29,
4975 |this: &WasmSimdF64Butterfly29<_>| this.direction
4976 );
4977 impl<T: FftNum> WasmSimdF64Butterfly29<T> {
4978 #[inline(always)]
4979 pub fn new(direction: FftDirection) -> Self {
4980 assert_f64::<T>();
4981 let rotate = Rotate90F64::new(true);
4982 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 29, direction);
4983 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 29, direction);
4984 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 29, direction);
4985 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 29, direction);
4986 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 29, direction);
4987 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 29, direction);
4988 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 29, direction);
4989 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 29, direction);
4990 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 29, direction);
4991 let tw10: Complex<f64> = twiddles::compute_twiddle(10, 29, direction);
4992 let tw11: Complex<f64> = twiddles::compute_twiddle(11, 29, direction);
4993 let tw12: Complex<f64> = twiddles::compute_twiddle(12, 29, direction);
4994 let tw13: Complex<f64> = twiddles::compute_twiddle(13, 29, direction);
4995 let tw14: Complex<f64> = twiddles::compute_twiddle(14, 29, direction);
4996 let twiddle1re = f64x2_splat(tw1.re);
4997 let twiddle1im = f64x2_splat(tw1.im);
4998 let twiddle2re = f64x2_splat(tw2.re);
4999 let twiddle2im = f64x2_splat(tw2.im);
5000 let twiddle3re = f64x2_splat(tw3.re);
5001 let twiddle3im = f64x2_splat(tw3.im);
5002 let twiddle4re = f64x2_splat(tw4.re);
5003 let twiddle4im = f64x2_splat(tw4.im);
5004 let twiddle5re = f64x2_splat(tw5.re);
5005 let twiddle5im = f64x2_splat(tw5.im);
5006 let twiddle6re = f64x2_splat(tw6.re);
5007 let twiddle6im = f64x2_splat(tw6.im);
5008 let twiddle7re = f64x2_splat(tw7.re);
5009 let twiddle7im = f64x2_splat(tw7.im);
5010 let twiddle8re = f64x2_splat(tw8.re);
5011 let twiddle8im = f64x2_splat(tw8.im);
5012 let twiddle9re = f64x2_splat(tw9.re);
5013 let twiddle9im = f64x2_splat(tw9.im);
5014 let twiddle10re = f64x2_splat(tw10.re);
5015 let twiddle10im = f64x2_splat(tw10.im);
5016 let twiddle11re = f64x2_splat(tw11.re);
5017 let twiddle11im = f64x2_splat(tw11.im);
5018 let twiddle12re = f64x2_splat(tw12.re);
5019 let twiddle12im = f64x2_splat(tw12.im);
5020 let twiddle13re = f64x2_splat(tw13.re);
5021 let twiddle13im = f64x2_splat(tw13.im);
5022 let twiddle14re = f64x2_splat(tw14.re);
5023 let twiddle14im = f64x2_splat(tw14.im);
5024
5025 Self {
5026 direction,
5027 _phantom: std::marker::PhantomData,
5028 rotate,
5029 twiddle1re,
5030 twiddle1im,
5031 twiddle2re,
5032 twiddle2im,
5033 twiddle3re,
5034 twiddle3im,
5035 twiddle4re,
5036 twiddle4im,
5037 twiddle5re,
5038 twiddle5im,
5039 twiddle6re,
5040 twiddle6im,
5041 twiddle7re,
5042 twiddle7im,
5043 twiddle8re,
5044 twiddle8im,
5045 twiddle9re,
5046 twiddle9im,
5047 twiddle10re,
5048 twiddle10im,
5049 twiddle11re,
5050 twiddle11im,
5051 twiddle12re,
5052 twiddle12im,
5053 twiddle13re,
5054 twiddle13im,
5055 twiddle14re,
5056 twiddle14im,
5057 }
5058 }
5059
5060 #[inline(always)]
5061 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
5062 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
5063
5064 let out = self.perform_fft_direct(values);
5065
5066 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
5067 }
5068
5069 #[inline(always)]
5070 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 29]) -> [v128; 29] {
5071 let [x1p28, x1m28] = solo_fft2_f64(values[1], values[28]);
5072 let [x2p27, x2m27] = solo_fft2_f64(values[2], values[27]);
5073 let [x3p26, x3m26] = solo_fft2_f64(values[3], values[26]);
5074 let [x4p25, x4m25] = solo_fft2_f64(values[4], values[25]);
5075 let [x5p24, x5m24] = solo_fft2_f64(values[5], values[24]);
5076 let [x6p23, x6m23] = solo_fft2_f64(values[6], values[23]);
5077 let [x7p22, x7m22] = solo_fft2_f64(values[7], values[22]);
5078 let [x8p21, x8m21] = solo_fft2_f64(values[8], values[21]);
5079 let [x9p20, x9m20] = solo_fft2_f64(values[9], values[20]);
5080 let [x10p19, x10m19] = solo_fft2_f64(values[10], values[19]);
5081 let [x11p18, x11m18] = solo_fft2_f64(values[11], values[18]);
5082 let [x12p17, x12m17] = solo_fft2_f64(values[12], values[17]);
5083 let [x13p16, x13m16] = solo_fft2_f64(values[13], values[16]);
5084 let [x14p15, x14m15] = solo_fft2_f64(values[14], values[15]);
5085
5086 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p28);
5087 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p27);
5088 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p26);
5089 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p25);
5090 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p24);
5091 let t_a1_6 = f64x2_mul(self.twiddle6re, x6p23);
5092 let t_a1_7 = f64x2_mul(self.twiddle7re, x7p22);
5093 let t_a1_8 = f64x2_mul(self.twiddle8re, x8p21);
5094 let t_a1_9 = f64x2_mul(self.twiddle9re, x9p20);
5095 let t_a1_10 = f64x2_mul(self.twiddle10re, x10p19);
5096 let t_a1_11 = f64x2_mul(self.twiddle11re, x11p18);
5097 let t_a1_12 = f64x2_mul(self.twiddle12re, x12p17);
5098 let t_a1_13 = f64x2_mul(self.twiddle13re, x13p16);
5099 let t_a1_14 = f64x2_mul(self.twiddle14re, x14p15);
5100 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p28);
5101 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p27);
5102 let t_a2_3 = f64x2_mul(self.twiddle6re, x3p26);
5103 let t_a2_4 = f64x2_mul(self.twiddle8re, x4p25);
5104 let t_a2_5 = f64x2_mul(self.twiddle10re, x5p24);
5105 let t_a2_6 = f64x2_mul(self.twiddle12re, x6p23);
5106 let t_a2_7 = f64x2_mul(self.twiddle14re, x7p22);
5107 let t_a2_8 = f64x2_mul(self.twiddle13re, x8p21);
5108 let t_a2_9 = f64x2_mul(self.twiddle11re, x9p20);
5109 let t_a2_10 = f64x2_mul(self.twiddle9re, x10p19);
5110 let t_a2_11 = f64x2_mul(self.twiddle7re, x11p18);
5111 let t_a2_12 = f64x2_mul(self.twiddle5re, x12p17);
5112 let t_a2_13 = f64x2_mul(self.twiddle3re, x13p16);
5113 let t_a2_14 = f64x2_mul(self.twiddle1re, x14p15);
5114 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p28);
5115 let t_a3_2 = f64x2_mul(self.twiddle6re, x2p27);
5116 let t_a3_3 = f64x2_mul(self.twiddle9re, x3p26);
5117 let t_a3_4 = f64x2_mul(self.twiddle12re, x4p25);
5118 let t_a3_5 = f64x2_mul(self.twiddle14re, x5p24);
5119 let t_a3_6 = f64x2_mul(self.twiddle11re, x6p23);
5120 let t_a3_7 = f64x2_mul(self.twiddle8re, x7p22);
5121 let t_a3_8 = f64x2_mul(self.twiddle5re, x8p21);
5122 let t_a3_9 = f64x2_mul(self.twiddle2re, x9p20);
5123 let t_a3_10 = f64x2_mul(self.twiddle1re, x10p19);
5124 let t_a3_11 = f64x2_mul(self.twiddle4re, x11p18);
5125 let t_a3_12 = f64x2_mul(self.twiddle7re, x12p17);
5126 let t_a3_13 = f64x2_mul(self.twiddle10re, x13p16);
5127 let t_a3_14 = f64x2_mul(self.twiddle13re, x14p15);
5128 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p28);
5129 let t_a4_2 = f64x2_mul(self.twiddle8re, x2p27);
5130 let t_a4_3 = f64x2_mul(self.twiddle12re, x3p26);
5131 let t_a4_4 = f64x2_mul(self.twiddle13re, x4p25);
5132 let t_a4_5 = f64x2_mul(self.twiddle9re, x5p24);
5133 let t_a4_6 = f64x2_mul(self.twiddle5re, x6p23);
5134 let t_a4_7 = f64x2_mul(self.twiddle1re, x7p22);
5135 let t_a4_8 = f64x2_mul(self.twiddle3re, x8p21);
5136 let t_a4_9 = f64x2_mul(self.twiddle7re, x9p20);
5137 let t_a4_10 = f64x2_mul(self.twiddle11re, x10p19);
5138 let t_a4_11 = f64x2_mul(self.twiddle14re, x11p18);
5139 let t_a4_12 = f64x2_mul(self.twiddle10re, x12p17);
5140 let t_a4_13 = f64x2_mul(self.twiddle6re, x13p16);
5141 let t_a4_14 = f64x2_mul(self.twiddle2re, x14p15);
5142 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p28);
5143 let t_a5_2 = f64x2_mul(self.twiddle10re, x2p27);
5144 let t_a5_3 = f64x2_mul(self.twiddle14re, x3p26);
5145 let t_a5_4 = f64x2_mul(self.twiddle9re, x4p25);
5146 let t_a5_5 = f64x2_mul(self.twiddle4re, x5p24);
5147 let t_a5_6 = f64x2_mul(self.twiddle1re, x6p23);
5148 let t_a5_7 = f64x2_mul(self.twiddle6re, x7p22);
5149 let t_a5_8 = f64x2_mul(self.twiddle11re, x8p21);
5150 let t_a5_9 = f64x2_mul(self.twiddle13re, x9p20);
5151 let t_a5_10 = f64x2_mul(self.twiddle8re, x10p19);
5152 let t_a5_11 = f64x2_mul(self.twiddle3re, x11p18);
5153 let t_a5_12 = f64x2_mul(self.twiddle2re, x12p17);
5154 let t_a5_13 = f64x2_mul(self.twiddle7re, x13p16);
5155 let t_a5_14 = f64x2_mul(self.twiddle12re, x14p15);
5156 let t_a6_1 = f64x2_mul(self.twiddle6re, x1p28);
5157 let t_a6_2 = f64x2_mul(self.twiddle12re, x2p27);
5158 let t_a6_3 = f64x2_mul(self.twiddle11re, x3p26);
5159 let t_a6_4 = f64x2_mul(self.twiddle5re, x4p25);
5160 let t_a6_5 = f64x2_mul(self.twiddle1re, x5p24);
5161 let t_a6_6 = f64x2_mul(self.twiddle7re, x6p23);
5162 let t_a6_7 = f64x2_mul(self.twiddle13re, x7p22);
5163 let t_a6_8 = f64x2_mul(self.twiddle10re, x8p21);
5164 let t_a6_9 = f64x2_mul(self.twiddle4re, x9p20);
5165 let t_a6_10 = f64x2_mul(self.twiddle2re, x10p19);
5166 let t_a6_11 = f64x2_mul(self.twiddle8re, x11p18);
5167 let t_a6_12 = f64x2_mul(self.twiddle14re, x12p17);
5168 let t_a6_13 = f64x2_mul(self.twiddle9re, x13p16);
5169 let t_a6_14 = f64x2_mul(self.twiddle3re, x14p15);
5170 let t_a7_1 = f64x2_mul(self.twiddle7re, x1p28);
5171 let t_a7_2 = f64x2_mul(self.twiddle14re, x2p27);
5172 let t_a7_3 = f64x2_mul(self.twiddle8re, x3p26);
5173 let t_a7_4 = f64x2_mul(self.twiddle1re, x4p25);
5174 let t_a7_5 = f64x2_mul(self.twiddle6re, x5p24);
5175 let t_a7_6 = f64x2_mul(self.twiddle13re, x6p23);
5176 let t_a7_7 = f64x2_mul(self.twiddle9re, x7p22);
5177 let t_a7_8 = f64x2_mul(self.twiddle2re, x8p21);
5178 let t_a7_9 = f64x2_mul(self.twiddle5re, x9p20);
5179 let t_a7_10 = f64x2_mul(self.twiddle12re, x10p19);
5180 let t_a7_11 = f64x2_mul(self.twiddle10re, x11p18);
5181 let t_a7_12 = f64x2_mul(self.twiddle3re, x12p17);
5182 let t_a7_13 = f64x2_mul(self.twiddle4re, x13p16);
5183 let t_a7_14 = f64x2_mul(self.twiddle11re, x14p15);
5184 let t_a8_1 = f64x2_mul(self.twiddle8re, x1p28);
5185 let t_a8_2 = f64x2_mul(self.twiddle13re, x2p27);
5186 let t_a8_3 = f64x2_mul(self.twiddle5re, x3p26);
5187 let t_a8_4 = f64x2_mul(self.twiddle3re, x4p25);
5188 let t_a8_5 = f64x2_mul(self.twiddle11re, x5p24);
5189 let t_a8_6 = f64x2_mul(self.twiddle10re, x6p23);
5190 let t_a8_7 = f64x2_mul(self.twiddle2re, x7p22);
5191 let t_a8_8 = f64x2_mul(self.twiddle6re, x8p21);
5192 let t_a8_9 = f64x2_mul(self.twiddle14re, x9p20);
5193 let t_a8_10 = f64x2_mul(self.twiddle7re, x10p19);
5194 let t_a8_11 = f64x2_mul(self.twiddle1re, x11p18);
5195 let t_a8_12 = f64x2_mul(self.twiddle9re, x12p17);
5196 let t_a8_13 = f64x2_mul(self.twiddle12re, x13p16);
5197 let t_a8_14 = f64x2_mul(self.twiddle4re, x14p15);
5198 let t_a9_1 = f64x2_mul(self.twiddle9re, x1p28);
5199 let t_a9_2 = f64x2_mul(self.twiddle11re, x2p27);
5200 let t_a9_3 = f64x2_mul(self.twiddle2re, x3p26);
5201 let t_a9_4 = f64x2_mul(self.twiddle7re, x4p25);
5202 let t_a9_5 = f64x2_mul(self.twiddle13re, x5p24);
5203 let t_a9_6 = f64x2_mul(self.twiddle4re, x6p23);
5204 let t_a9_7 = f64x2_mul(self.twiddle5re, x7p22);
5205 let t_a9_8 = f64x2_mul(self.twiddle14re, x8p21);
5206 let t_a9_9 = f64x2_mul(self.twiddle6re, x9p20);
5207 let t_a9_10 = f64x2_mul(self.twiddle3re, x10p19);
5208 let t_a9_11 = f64x2_mul(self.twiddle12re, x11p18);
5209 let t_a9_12 = f64x2_mul(self.twiddle8re, x12p17);
5210 let t_a9_13 = f64x2_mul(self.twiddle1re, x13p16);
5211 let t_a9_14 = f64x2_mul(self.twiddle10re, x14p15);
5212 let t_a10_1 = f64x2_mul(self.twiddle10re, x1p28);
5213 let t_a10_2 = f64x2_mul(self.twiddle9re, x2p27);
5214 let t_a10_3 = f64x2_mul(self.twiddle1re, x3p26);
5215 let t_a10_4 = f64x2_mul(self.twiddle11re, x4p25);
5216 let t_a10_5 = f64x2_mul(self.twiddle8re, x5p24);
5217 let t_a10_6 = f64x2_mul(self.twiddle2re, x6p23);
5218 let t_a10_7 = f64x2_mul(self.twiddle12re, x7p22);
5219 let t_a10_8 = f64x2_mul(self.twiddle7re, x8p21);
5220 let t_a10_9 = f64x2_mul(self.twiddle3re, x9p20);
5221 let t_a10_10 = f64x2_mul(self.twiddle13re, x10p19);
5222 let t_a10_11 = f64x2_mul(self.twiddle6re, x11p18);
5223 let t_a10_12 = f64x2_mul(self.twiddle4re, x12p17);
5224 let t_a10_13 = f64x2_mul(self.twiddle14re, x13p16);
5225 let t_a10_14 = f64x2_mul(self.twiddle5re, x14p15);
5226 let t_a11_1 = f64x2_mul(self.twiddle11re, x1p28);
5227 let t_a11_2 = f64x2_mul(self.twiddle7re, x2p27);
5228 let t_a11_3 = f64x2_mul(self.twiddle4re, x3p26);
5229 let t_a11_4 = f64x2_mul(self.twiddle14re, x4p25);
5230 let t_a11_5 = f64x2_mul(self.twiddle3re, x5p24);
5231 let t_a11_6 = f64x2_mul(self.twiddle8re, x6p23);
5232 let t_a11_7 = f64x2_mul(self.twiddle10re, x7p22);
5233 let t_a11_8 = f64x2_mul(self.twiddle1re, x8p21);
5234 let t_a11_9 = f64x2_mul(self.twiddle12re, x9p20);
5235 let t_a11_10 = f64x2_mul(self.twiddle6re, x10p19);
5236 let t_a11_11 = f64x2_mul(self.twiddle5re, x11p18);
5237 let t_a11_12 = f64x2_mul(self.twiddle13re, x12p17);
5238 let t_a11_13 = f64x2_mul(self.twiddle2re, x13p16);
5239 let t_a11_14 = f64x2_mul(self.twiddle9re, x14p15);
5240 let t_a12_1 = f64x2_mul(self.twiddle12re, x1p28);
5241 let t_a12_2 = f64x2_mul(self.twiddle5re, x2p27);
5242 let t_a12_3 = f64x2_mul(self.twiddle7re, x3p26);
5243 let t_a12_4 = f64x2_mul(self.twiddle10re, x4p25);
5244 let t_a12_5 = f64x2_mul(self.twiddle2re, x5p24);
5245 let t_a12_6 = f64x2_mul(self.twiddle14re, x6p23);
5246 let t_a12_7 = f64x2_mul(self.twiddle3re, x7p22);
5247 let t_a12_8 = f64x2_mul(self.twiddle9re, x8p21);
5248 let t_a12_9 = f64x2_mul(self.twiddle8re, x9p20);
5249 let t_a12_10 = f64x2_mul(self.twiddle4re, x10p19);
5250 let t_a12_11 = f64x2_mul(self.twiddle13re, x11p18);
5251 let t_a12_12 = f64x2_mul(self.twiddle1re, x12p17);
5252 let t_a12_13 = f64x2_mul(self.twiddle11re, x13p16);
5253 let t_a12_14 = f64x2_mul(self.twiddle6re, x14p15);
5254 let t_a13_1 = f64x2_mul(self.twiddle13re, x1p28);
5255 let t_a13_2 = f64x2_mul(self.twiddle3re, x2p27);
5256 let t_a13_3 = f64x2_mul(self.twiddle10re, x3p26);
5257 let t_a13_4 = f64x2_mul(self.twiddle6re, x4p25);
5258 let t_a13_5 = f64x2_mul(self.twiddle7re, x5p24);
5259 let t_a13_6 = f64x2_mul(self.twiddle9re, x6p23);
5260 let t_a13_7 = f64x2_mul(self.twiddle4re, x7p22);
5261 let t_a13_8 = f64x2_mul(self.twiddle12re, x8p21);
5262 let t_a13_9 = f64x2_mul(self.twiddle1re, x9p20);
5263 let t_a13_10 = f64x2_mul(self.twiddle14re, x10p19);
5264 let t_a13_11 = f64x2_mul(self.twiddle2re, x11p18);
5265 let t_a13_12 = f64x2_mul(self.twiddle11re, x12p17);
5266 let t_a13_13 = f64x2_mul(self.twiddle5re, x13p16);
5267 let t_a13_14 = f64x2_mul(self.twiddle8re, x14p15);
5268 let t_a14_1 = f64x2_mul(self.twiddle14re, x1p28);
5269 let t_a14_2 = f64x2_mul(self.twiddle1re, x2p27);
5270 let t_a14_3 = f64x2_mul(self.twiddle13re, x3p26);
5271 let t_a14_4 = f64x2_mul(self.twiddle2re, x4p25);
5272 let t_a14_5 = f64x2_mul(self.twiddle12re, x5p24);
5273 let t_a14_6 = f64x2_mul(self.twiddle3re, x6p23);
5274 let t_a14_7 = f64x2_mul(self.twiddle11re, x7p22);
5275 let t_a14_8 = f64x2_mul(self.twiddle4re, x8p21);
5276 let t_a14_9 = f64x2_mul(self.twiddle10re, x9p20);
5277 let t_a14_10 = f64x2_mul(self.twiddle5re, x10p19);
5278 let t_a14_11 = f64x2_mul(self.twiddle9re, x11p18);
5279 let t_a14_12 = f64x2_mul(self.twiddle6re, x12p17);
5280 let t_a14_13 = f64x2_mul(self.twiddle8re, x13p16);
5281 let t_a14_14 = f64x2_mul(self.twiddle7re, x14p15);
5282
5283 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m28);
5284 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m27);
5285 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m26);
5286 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m25);
5287 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m24);
5288 let t_b1_6 = f64x2_mul(self.twiddle6im, x6m23);
5289 let t_b1_7 = f64x2_mul(self.twiddle7im, x7m22);
5290 let t_b1_8 = f64x2_mul(self.twiddle8im, x8m21);
5291 let t_b1_9 = f64x2_mul(self.twiddle9im, x9m20);
5292 let t_b1_10 = f64x2_mul(self.twiddle10im, x10m19);
5293 let t_b1_11 = f64x2_mul(self.twiddle11im, x11m18);
5294 let t_b1_12 = f64x2_mul(self.twiddle12im, x12m17);
5295 let t_b1_13 = f64x2_mul(self.twiddle13im, x13m16);
5296 let t_b1_14 = f64x2_mul(self.twiddle14im, x14m15);
5297 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m28);
5298 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m27);
5299 let t_b2_3 = f64x2_mul(self.twiddle6im, x3m26);
5300 let t_b2_4 = f64x2_mul(self.twiddle8im, x4m25);
5301 let t_b2_5 = f64x2_mul(self.twiddle10im, x5m24);
5302 let t_b2_6 = f64x2_mul(self.twiddle12im, x6m23);
5303 let t_b2_7 = f64x2_mul(self.twiddle14im, x7m22);
5304 let t_b2_8 = f64x2_mul(self.twiddle13im, x8m21);
5305 let t_b2_9 = f64x2_mul(self.twiddle11im, x9m20);
5306 let t_b2_10 = f64x2_mul(self.twiddle9im, x10m19);
5307 let t_b2_11 = f64x2_mul(self.twiddle7im, x11m18);
5308 let t_b2_12 = f64x2_mul(self.twiddle5im, x12m17);
5309 let t_b2_13 = f64x2_mul(self.twiddle3im, x13m16);
5310 let t_b2_14 = f64x2_mul(self.twiddle1im, x14m15);
5311 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m28);
5312 let t_b3_2 = f64x2_mul(self.twiddle6im, x2m27);
5313 let t_b3_3 = f64x2_mul(self.twiddle9im, x3m26);
5314 let t_b3_4 = f64x2_mul(self.twiddle12im, x4m25);
5315 let t_b3_5 = f64x2_mul(self.twiddle14im, x5m24);
5316 let t_b3_6 = f64x2_mul(self.twiddle11im, x6m23);
5317 let t_b3_7 = f64x2_mul(self.twiddle8im, x7m22);
5318 let t_b3_8 = f64x2_mul(self.twiddle5im, x8m21);
5319 let t_b3_9 = f64x2_mul(self.twiddle2im, x9m20);
5320 let t_b3_10 = f64x2_mul(self.twiddle1im, x10m19);
5321 let t_b3_11 = f64x2_mul(self.twiddle4im, x11m18);
5322 let t_b3_12 = f64x2_mul(self.twiddle7im, x12m17);
5323 let t_b3_13 = f64x2_mul(self.twiddle10im, x13m16);
5324 let t_b3_14 = f64x2_mul(self.twiddle13im, x14m15);
5325 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m28);
5326 let t_b4_2 = f64x2_mul(self.twiddle8im, x2m27);
5327 let t_b4_3 = f64x2_mul(self.twiddle12im, x3m26);
5328 let t_b4_4 = f64x2_mul(self.twiddle13im, x4m25);
5329 let t_b4_5 = f64x2_mul(self.twiddle9im, x5m24);
5330 let t_b4_6 = f64x2_mul(self.twiddle5im, x6m23);
5331 let t_b4_7 = f64x2_mul(self.twiddle1im, x7m22);
5332 let t_b4_8 = f64x2_mul(self.twiddle3im, x8m21);
5333 let t_b4_9 = f64x2_mul(self.twiddle7im, x9m20);
5334 let t_b4_10 = f64x2_mul(self.twiddle11im, x10m19);
5335 let t_b4_11 = f64x2_mul(self.twiddle14im, x11m18);
5336 let t_b4_12 = f64x2_mul(self.twiddle10im, x12m17);
5337 let t_b4_13 = f64x2_mul(self.twiddle6im, x13m16);
5338 let t_b4_14 = f64x2_mul(self.twiddle2im, x14m15);
5339 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m28);
5340 let t_b5_2 = f64x2_mul(self.twiddle10im, x2m27);
5341 let t_b5_3 = f64x2_mul(self.twiddle14im, x3m26);
5342 let t_b5_4 = f64x2_mul(self.twiddle9im, x4m25);
5343 let t_b5_5 = f64x2_mul(self.twiddle4im, x5m24);
5344 let t_b5_6 = f64x2_mul(self.twiddle1im, x6m23);
5345 let t_b5_7 = f64x2_mul(self.twiddle6im, x7m22);
5346 let t_b5_8 = f64x2_mul(self.twiddle11im, x8m21);
5347 let t_b5_9 = f64x2_mul(self.twiddle13im, x9m20);
5348 let t_b5_10 = f64x2_mul(self.twiddle8im, x10m19);
5349 let t_b5_11 = f64x2_mul(self.twiddle3im, x11m18);
5350 let t_b5_12 = f64x2_mul(self.twiddle2im, x12m17);
5351 let t_b5_13 = f64x2_mul(self.twiddle7im, x13m16);
5352 let t_b5_14 = f64x2_mul(self.twiddle12im, x14m15);
5353 let t_b6_1 = f64x2_mul(self.twiddle6im, x1m28);
5354 let t_b6_2 = f64x2_mul(self.twiddle12im, x2m27);
5355 let t_b6_3 = f64x2_mul(self.twiddle11im, x3m26);
5356 let t_b6_4 = f64x2_mul(self.twiddle5im, x4m25);
5357 let t_b6_5 = f64x2_mul(self.twiddle1im, x5m24);
5358 let t_b6_6 = f64x2_mul(self.twiddle7im, x6m23);
5359 let t_b6_7 = f64x2_mul(self.twiddle13im, x7m22);
5360 let t_b6_8 = f64x2_mul(self.twiddle10im, x8m21);
5361 let t_b6_9 = f64x2_mul(self.twiddle4im, x9m20);
5362 let t_b6_10 = f64x2_mul(self.twiddle2im, x10m19);
5363 let t_b6_11 = f64x2_mul(self.twiddle8im, x11m18);
5364 let t_b6_12 = f64x2_mul(self.twiddle14im, x12m17);
5365 let t_b6_13 = f64x2_mul(self.twiddle9im, x13m16);
5366 let t_b6_14 = f64x2_mul(self.twiddle3im, x14m15);
5367 let t_b7_1 = f64x2_mul(self.twiddle7im, x1m28);
5368 let t_b7_2 = f64x2_mul(self.twiddle14im, x2m27);
5369 let t_b7_3 = f64x2_mul(self.twiddle8im, x3m26);
5370 let t_b7_4 = f64x2_mul(self.twiddle1im, x4m25);
5371 let t_b7_5 = f64x2_mul(self.twiddle6im, x5m24);
5372 let t_b7_6 = f64x2_mul(self.twiddle13im, x6m23);
5373 let t_b7_7 = f64x2_mul(self.twiddle9im, x7m22);
5374 let t_b7_8 = f64x2_mul(self.twiddle2im, x8m21);
5375 let t_b7_9 = f64x2_mul(self.twiddle5im, x9m20);
5376 let t_b7_10 = f64x2_mul(self.twiddle12im, x10m19);
5377 let t_b7_11 = f64x2_mul(self.twiddle10im, x11m18);
5378 let t_b7_12 = f64x2_mul(self.twiddle3im, x12m17);
5379 let t_b7_13 = f64x2_mul(self.twiddle4im, x13m16);
5380 let t_b7_14 = f64x2_mul(self.twiddle11im, x14m15);
5381 let t_b8_1 = f64x2_mul(self.twiddle8im, x1m28);
5382 let t_b8_2 = f64x2_mul(self.twiddle13im, x2m27);
5383 let t_b8_3 = f64x2_mul(self.twiddle5im, x3m26);
5384 let t_b8_4 = f64x2_mul(self.twiddle3im, x4m25);
5385 let t_b8_5 = f64x2_mul(self.twiddle11im, x5m24);
5386 let t_b8_6 = f64x2_mul(self.twiddle10im, x6m23);
5387 let t_b8_7 = f64x2_mul(self.twiddle2im, x7m22);
5388 let t_b8_8 = f64x2_mul(self.twiddle6im, x8m21);
5389 let t_b8_9 = f64x2_mul(self.twiddle14im, x9m20);
5390 let t_b8_10 = f64x2_mul(self.twiddle7im, x10m19);
5391 let t_b8_11 = f64x2_mul(self.twiddle1im, x11m18);
5392 let t_b8_12 = f64x2_mul(self.twiddle9im, x12m17);
5393 let t_b8_13 = f64x2_mul(self.twiddle12im, x13m16);
5394 let t_b8_14 = f64x2_mul(self.twiddle4im, x14m15);
5395 let t_b9_1 = f64x2_mul(self.twiddle9im, x1m28);
5396 let t_b9_2 = f64x2_mul(self.twiddle11im, x2m27);
5397 let t_b9_3 = f64x2_mul(self.twiddle2im, x3m26);
5398 let t_b9_4 = f64x2_mul(self.twiddle7im, x4m25);
5399 let t_b9_5 = f64x2_mul(self.twiddle13im, x5m24);
5400 let t_b9_6 = f64x2_mul(self.twiddle4im, x6m23);
5401 let t_b9_7 = f64x2_mul(self.twiddle5im, x7m22);
5402 let t_b9_8 = f64x2_mul(self.twiddle14im, x8m21);
5403 let t_b9_9 = f64x2_mul(self.twiddle6im, x9m20);
5404 let t_b9_10 = f64x2_mul(self.twiddle3im, x10m19);
5405 let t_b9_11 = f64x2_mul(self.twiddle12im, x11m18);
5406 let t_b9_12 = f64x2_mul(self.twiddle8im, x12m17);
5407 let t_b9_13 = f64x2_mul(self.twiddle1im, x13m16);
5408 let t_b9_14 = f64x2_mul(self.twiddle10im, x14m15);
5409 let t_b10_1 = f64x2_mul(self.twiddle10im, x1m28);
5410 let t_b10_2 = f64x2_mul(self.twiddle9im, x2m27);
5411 let t_b10_3 = f64x2_mul(self.twiddle1im, x3m26);
5412 let t_b10_4 = f64x2_mul(self.twiddle11im, x4m25);
5413 let t_b10_5 = f64x2_mul(self.twiddle8im, x5m24);
5414 let t_b10_6 = f64x2_mul(self.twiddle2im, x6m23);
5415 let t_b10_7 = f64x2_mul(self.twiddle12im, x7m22);
5416 let t_b10_8 = f64x2_mul(self.twiddle7im, x8m21);
5417 let t_b10_9 = f64x2_mul(self.twiddle3im, x9m20);
5418 let t_b10_10 = f64x2_mul(self.twiddle13im, x10m19);
5419 let t_b10_11 = f64x2_mul(self.twiddle6im, x11m18);
5420 let t_b10_12 = f64x2_mul(self.twiddle4im, x12m17);
5421 let t_b10_13 = f64x2_mul(self.twiddle14im, x13m16);
5422 let t_b10_14 = f64x2_mul(self.twiddle5im, x14m15);
5423 let t_b11_1 = f64x2_mul(self.twiddle11im, x1m28);
5424 let t_b11_2 = f64x2_mul(self.twiddle7im, x2m27);
5425 let t_b11_3 = f64x2_mul(self.twiddle4im, x3m26);
5426 let t_b11_4 = f64x2_mul(self.twiddle14im, x4m25);
5427 let t_b11_5 = f64x2_mul(self.twiddle3im, x5m24);
5428 let t_b11_6 = f64x2_mul(self.twiddle8im, x6m23);
5429 let t_b11_7 = f64x2_mul(self.twiddle10im, x7m22);
5430 let t_b11_8 = f64x2_mul(self.twiddle1im, x8m21);
5431 let t_b11_9 = f64x2_mul(self.twiddle12im, x9m20);
5432 let t_b11_10 = f64x2_mul(self.twiddle6im, x10m19);
5433 let t_b11_11 = f64x2_mul(self.twiddle5im, x11m18);
5434 let t_b11_12 = f64x2_mul(self.twiddle13im, x12m17);
5435 let t_b11_13 = f64x2_mul(self.twiddle2im, x13m16);
5436 let t_b11_14 = f64x2_mul(self.twiddle9im, x14m15);
5437 let t_b12_1 = f64x2_mul(self.twiddle12im, x1m28);
5438 let t_b12_2 = f64x2_mul(self.twiddle5im, x2m27);
5439 let t_b12_3 = f64x2_mul(self.twiddle7im, x3m26);
5440 let t_b12_4 = f64x2_mul(self.twiddle10im, x4m25);
5441 let t_b12_5 = f64x2_mul(self.twiddle2im, x5m24);
5442 let t_b12_6 = f64x2_mul(self.twiddle14im, x6m23);
5443 let t_b12_7 = f64x2_mul(self.twiddle3im, x7m22);
5444 let t_b12_8 = f64x2_mul(self.twiddle9im, x8m21);
5445 let t_b12_9 = f64x2_mul(self.twiddle8im, x9m20);
5446 let t_b12_10 = f64x2_mul(self.twiddle4im, x10m19);
5447 let t_b12_11 = f64x2_mul(self.twiddle13im, x11m18);
5448 let t_b12_12 = f64x2_mul(self.twiddle1im, x12m17);
5449 let t_b12_13 = f64x2_mul(self.twiddle11im, x13m16);
5450 let t_b12_14 = f64x2_mul(self.twiddle6im, x14m15);
5451 let t_b13_1 = f64x2_mul(self.twiddle13im, x1m28);
5452 let t_b13_2 = f64x2_mul(self.twiddle3im, x2m27);
5453 let t_b13_3 = f64x2_mul(self.twiddle10im, x3m26);
5454 let t_b13_4 = f64x2_mul(self.twiddle6im, x4m25);
5455 let t_b13_5 = f64x2_mul(self.twiddle7im, x5m24);
5456 let t_b13_6 = f64x2_mul(self.twiddle9im, x6m23);
5457 let t_b13_7 = f64x2_mul(self.twiddle4im, x7m22);
5458 let t_b13_8 = f64x2_mul(self.twiddle12im, x8m21);
5459 let t_b13_9 = f64x2_mul(self.twiddle1im, x9m20);
5460 let t_b13_10 = f64x2_mul(self.twiddle14im, x10m19);
5461 let t_b13_11 = f64x2_mul(self.twiddle2im, x11m18);
5462 let t_b13_12 = f64x2_mul(self.twiddle11im, x12m17);
5463 let t_b13_13 = f64x2_mul(self.twiddle5im, x13m16);
5464 let t_b13_14 = f64x2_mul(self.twiddle8im, x14m15);
5465 let t_b14_1 = f64x2_mul(self.twiddle14im, x1m28);
5466 let t_b14_2 = f64x2_mul(self.twiddle1im, x2m27);
5467 let t_b14_3 = f64x2_mul(self.twiddle13im, x3m26);
5468 let t_b14_4 = f64x2_mul(self.twiddle2im, x4m25);
5469 let t_b14_5 = f64x2_mul(self.twiddle12im, x5m24);
5470 let t_b14_6 = f64x2_mul(self.twiddle3im, x6m23);
5471 let t_b14_7 = f64x2_mul(self.twiddle11im, x7m22);
5472 let t_b14_8 = f64x2_mul(self.twiddle4im, x8m21);
5473 let t_b14_9 = f64x2_mul(self.twiddle10im, x9m20);
5474 let t_b14_10 = f64x2_mul(self.twiddle5im, x10m19);
5475 let t_b14_11 = f64x2_mul(self.twiddle9im, x11m18);
5476 let t_b14_12 = f64x2_mul(self.twiddle6im, x12m17);
5477 let t_b14_13 = f64x2_mul(self.twiddle8im, x13m16);
5478 let t_b14_14 = f64x2_mul(self.twiddle7im, x14m15);
5479
5480 let x0 = values[0];
5481 let t_a1 = calc_f64!(
5482 x0 + t_a1_1
5483 + t_a1_2
5484 + t_a1_3
5485 + t_a1_4
5486 + t_a1_5
5487 + t_a1_6
5488 + t_a1_7
5489 + t_a1_8
5490 + t_a1_9
5491 + t_a1_10
5492 + t_a1_11
5493 + t_a1_12
5494 + t_a1_13
5495 + t_a1_14
5496 );
5497 let t_a2 = calc_f64!(
5498 x0 + t_a2_1
5499 + t_a2_2
5500 + t_a2_3
5501 + t_a2_4
5502 + t_a2_5
5503 + t_a2_6
5504 + t_a2_7
5505 + t_a2_8
5506 + t_a2_9
5507 + t_a2_10
5508 + t_a2_11
5509 + t_a2_12
5510 + t_a2_13
5511 + t_a2_14
5512 );
5513 let t_a3 = calc_f64!(
5514 x0 + t_a3_1
5515 + t_a3_2
5516 + t_a3_3
5517 + t_a3_4
5518 + t_a3_5
5519 + t_a3_6
5520 + t_a3_7
5521 + t_a3_8
5522 + t_a3_9
5523 + t_a3_10
5524 + t_a3_11
5525 + t_a3_12
5526 + t_a3_13
5527 + t_a3_14
5528 );
5529 let t_a4 = calc_f64!(
5530 x0 + t_a4_1
5531 + t_a4_2
5532 + t_a4_3
5533 + t_a4_4
5534 + t_a4_5
5535 + t_a4_6
5536 + t_a4_7
5537 + t_a4_8
5538 + t_a4_9
5539 + t_a4_10
5540 + t_a4_11
5541 + t_a4_12
5542 + t_a4_13
5543 + t_a4_14
5544 );
5545 let t_a5 = calc_f64!(
5546 x0 + t_a5_1
5547 + t_a5_2
5548 + t_a5_3
5549 + t_a5_4
5550 + t_a5_5
5551 + t_a5_6
5552 + t_a5_7
5553 + t_a5_8
5554 + t_a5_9
5555 + t_a5_10
5556 + t_a5_11
5557 + t_a5_12
5558 + t_a5_13
5559 + t_a5_14
5560 );
5561 let t_a6 = calc_f64!(
5562 x0 + t_a6_1
5563 + t_a6_2
5564 + t_a6_3
5565 + t_a6_4
5566 + t_a6_5
5567 + t_a6_6
5568 + t_a6_7
5569 + t_a6_8
5570 + t_a6_9
5571 + t_a6_10
5572 + t_a6_11
5573 + t_a6_12
5574 + t_a6_13
5575 + t_a6_14
5576 );
5577 let t_a7 = calc_f64!(
5578 x0 + t_a7_1
5579 + t_a7_2
5580 + t_a7_3
5581 + t_a7_4
5582 + t_a7_5
5583 + t_a7_6
5584 + t_a7_7
5585 + t_a7_8
5586 + t_a7_9
5587 + t_a7_10
5588 + t_a7_11
5589 + t_a7_12
5590 + t_a7_13
5591 + t_a7_14
5592 );
5593 let t_a8 = calc_f64!(
5594 x0 + t_a8_1
5595 + t_a8_2
5596 + t_a8_3
5597 + t_a8_4
5598 + t_a8_5
5599 + t_a8_6
5600 + t_a8_7
5601 + t_a8_8
5602 + t_a8_9
5603 + t_a8_10
5604 + t_a8_11
5605 + t_a8_12
5606 + t_a8_13
5607 + t_a8_14
5608 );
5609 let t_a9 = calc_f64!(
5610 x0 + t_a9_1
5611 + t_a9_2
5612 + t_a9_3
5613 + t_a9_4
5614 + t_a9_5
5615 + t_a9_6
5616 + t_a9_7
5617 + t_a9_8
5618 + t_a9_9
5619 + t_a9_10
5620 + t_a9_11
5621 + t_a9_12
5622 + t_a9_13
5623 + t_a9_14
5624 );
5625 let t_a10 = calc_f64!(
5626 x0 + t_a10_1
5627 + t_a10_2
5628 + t_a10_3
5629 + t_a10_4
5630 + t_a10_5
5631 + t_a10_6
5632 + t_a10_7
5633 + t_a10_8
5634 + t_a10_9
5635 + t_a10_10
5636 + t_a10_11
5637 + t_a10_12
5638 + t_a10_13
5639 + t_a10_14
5640 );
5641 let t_a11 = calc_f64!(
5642 x0 + t_a11_1
5643 + t_a11_2
5644 + t_a11_3
5645 + t_a11_4
5646 + t_a11_5
5647 + t_a11_6
5648 + t_a11_7
5649 + t_a11_8
5650 + t_a11_9
5651 + t_a11_10
5652 + t_a11_11
5653 + t_a11_12
5654 + t_a11_13
5655 + t_a11_14
5656 );
5657 let t_a12 = calc_f64!(
5658 x0 + t_a12_1
5659 + t_a12_2
5660 + t_a12_3
5661 + t_a12_4
5662 + t_a12_5
5663 + t_a12_6
5664 + t_a12_7
5665 + t_a12_8
5666 + t_a12_9
5667 + t_a12_10
5668 + t_a12_11
5669 + t_a12_12
5670 + t_a12_13
5671 + t_a12_14
5672 );
5673 let t_a13 = calc_f64!(
5674 x0 + t_a13_1
5675 + t_a13_2
5676 + t_a13_3
5677 + t_a13_4
5678 + t_a13_5
5679 + t_a13_6
5680 + t_a13_7
5681 + t_a13_8
5682 + t_a13_9
5683 + t_a13_10
5684 + t_a13_11
5685 + t_a13_12
5686 + t_a13_13
5687 + t_a13_14
5688 );
5689 let t_a14 = calc_f64!(
5690 x0 + t_a14_1
5691 + t_a14_2
5692 + t_a14_3
5693 + t_a14_4
5694 + t_a14_5
5695 + t_a14_6
5696 + t_a14_7
5697 + t_a14_8
5698 + t_a14_9
5699 + t_a14_10
5700 + t_a14_11
5701 + t_a14_12
5702 + t_a14_13
5703 + t_a14_14
5704 );
5705
5706 let t_b1 = calc_f64!(
5707 t_b1_1
5708 + t_b1_2
5709 + t_b1_3
5710 + t_b1_4
5711 + t_b1_5
5712 + t_b1_6
5713 + t_b1_7
5714 + t_b1_8
5715 + t_b1_9
5716 + t_b1_10
5717 + t_b1_11
5718 + t_b1_12
5719 + t_b1_13
5720 + t_b1_14
5721 );
5722 let t_b2 = calc_f64!(
5723 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7
5724 - t_b2_8
5725 - t_b2_9
5726 - t_b2_10
5727 - t_b2_11
5728 - t_b2_12
5729 - t_b2_13
5730 - t_b2_14
5731 );
5732 let t_b3 = calc_f64!(
5733 t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7 - t_b3_8 - t_b3_9
5734 + t_b3_10
5735 + t_b3_11
5736 + t_b3_12
5737 + t_b3_13
5738 + t_b3_14
5739 );
5740 let t_b4 = calc_f64!(
5741 t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7
5742 + t_b4_8
5743 + t_b4_9
5744 + t_b4_10
5745 - t_b4_11
5746 - t_b4_12
5747 - t_b4_13
5748 - t_b4_14
5749 );
5750 let t_b5 = calc_f64!(
5751 t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6 + t_b5_7 + t_b5_8
5752 - t_b5_9
5753 - t_b5_10
5754 - t_b5_11
5755 + t_b5_12
5756 + t_b5_13
5757 + t_b5_14
5758 );
5759 let t_b6 = calc_f64!(
5760 t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 + t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9
5761 + t_b6_10
5762 + t_b6_11
5763 + t_b6_12
5764 - t_b6_13
5765 - t_b6_14
5766 );
5767 let t_b7 = calc_f64!(
5768 t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8
5769 + t_b7_9
5770 + t_b7_10
5771 - t_b7_11
5772 - t_b7_12
5773 + t_b7_13
5774 + t_b7_14
5775 );
5776 let t_b8 = calc_f64!(
5777 t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9
5778 - t_b8_10
5779 + t_b8_11
5780 + t_b8_12
5781 - t_b8_13
5782 - t_b8_14
5783 );
5784 let t_b9 = calc_f64!(
5785 t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 - t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8 - t_b9_9
5786 + t_b9_10
5787 + t_b9_11
5788 - t_b9_12
5789 + t_b9_13
5790 + t_b9_14
5791 );
5792 let t_b10 = calc_f64!(
5793 t_b10_1 - t_b10_2 + t_b10_3 + t_b10_4 - t_b10_5 + t_b10_6 + t_b10_7 - t_b10_8
5794 + t_b10_9
5795 + t_b10_10
5796 - t_b10_11
5797 + t_b10_12
5798 + t_b10_13
5799 - t_b10_14
5800 );
5801 let t_b11 = calc_f64!(
5802 t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 - t_b11_5 + t_b11_6 - t_b11_7 + t_b11_8 + t_b11_9
5803 - t_b11_10
5804 + t_b11_11
5805 - t_b11_12
5806 - t_b11_13
5807 + t_b11_14
5808 );
5809 let t_b12 = calc_f64!(
5810 t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 + t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 - t_b12_9
5811 + t_b12_10
5812 - t_b12_11
5813 - t_b12_12
5814 + t_b12_13
5815 - t_b12_14
5816 );
5817 let t_b13 = calc_f64!(
5818 t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 + t_b13_7 - t_b13_8
5819 + t_b13_9
5820 + t_b13_10
5821 - t_b13_11
5822 + t_b13_12
5823 - t_b13_13
5824 + t_b13_14
5825 );
5826 let t_b14 = calc_f64!(
5827 t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9
5828 - t_b14_10
5829 + t_b14_11
5830 - t_b14_12
5831 + t_b14_13
5832 - t_b14_14
5833 );
5834
5835 let t_b1_rot = self.rotate.rotate(t_b1);
5836 let t_b2_rot = self.rotate.rotate(t_b2);
5837 let t_b3_rot = self.rotate.rotate(t_b3);
5838 let t_b4_rot = self.rotate.rotate(t_b4);
5839 let t_b5_rot = self.rotate.rotate(t_b5);
5840 let t_b6_rot = self.rotate.rotate(t_b6);
5841 let t_b7_rot = self.rotate.rotate(t_b7);
5842 let t_b8_rot = self.rotate.rotate(t_b8);
5843 let t_b9_rot = self.rotate.rotate(t_b9);
5844 let t_b10_rot = self.rotate.rotate(t_b10);
5845 let t_b11_rot = self.rotate.rotate(t_b11);
5846 let t_b12_rot = self.rotate.rotate(t_b12);
5847 let t_b13_rot = self.rotate.rotate(t_b13);
5848 let t_b14_rot = self.rotate.rotate(t_b14);
5849
5850 let y0 = calc_f64!(
5851 x0 + x1p28
5852 + x2p27
5853 + x3p26
5854 + x4p25
5855 + x5p24
5856 + x6p23
5857 + x7p22
5858 + x8p21
5859 + x9p20
5860 + x10p19
5861 + x11p18
5862 + x12p17
5863 + x13p16
5864 + x14p15
5865 );
5866 let [y1, y28] = solo_fft2_f64(t_a1, t_b1_rot);
5867 let [y2, y27] = solo_fft2_f64(t_a2, t_b2_rot);
5868 let [y3, y26] = solo_fft2_f64(t_a3, t_b3_rot);
5869 let [y4, y25] = solo_fft2_f64(t_a4, t_b4_rot);
5870 let [y5, y24] = solo_fft2_f64(t_a5, t_b5_rot);
5871 let [y6, y23] = solo_fft2_f64(t_a6, t_b6_rot);
5872 let [y7, y22] = solo_fft2_f64(t_a7, t_b7_rot);
5873 let [y8, y21] = solo_fft2_f64(t_a8, t_b8_rot);
5874 let [y9, y20] = solo_fft2_f64(t_a9, t_b9_rot);
5875 let [y10, y19] = solo_fft2_f64(t_a10, t_b10_rot);
5876 let [y11, y18] = solo_fft2_f64(t_a11, t_b11_rot);
5877 let [y12, y17] = solo_fft2_f64(t_a12, t_b12_rot);
5878 let [y13, y16] = solo_fft2_f64(t_a13, t_b13_rot);
5879 let [y14, y15] = solo_fft2_f64(t_a14, t_b14_rot);
5880 [
5881 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
5882 y19, y20, y21, y22, y23, y24, y25, y26, y27, y28,
5883 ]
5884 }
5885 }
5886
5887 // _____ _ _________ _ _ _
5888 // |___ // | |___ /___ \| |__ (_) |_
5889 // |_ \| | _____ |_ \ __) | '_ \| | __|
5890 // ___) | | |_____| ___) / __/| |_) | | |_
5891 // |____/|_| |____/_____|_.__/|_|\__|
5892 //
5893 pub struct WasmSimdF32Butterfly31<T> {
5894 direction: FftDirection,
5895 _phantom: std::marker::PhantomData<T>,
5896 rotate: Rotate90F32,
5897 twiddle1re: v128,
5898 twiddle1im: v128,
5899 twiddle2re: v128,
5900 twiddle2im: v128,
5901 twiddle3re: v128,
5902 twiddle3im: v128,
5903 twiddle4re: v128,
5904 twiddle4im: v128,
5905 twiddle5re: v128,
5906 twiddle5im: v128,
5907 twiddle6re: v128,
5908 twiddle6im: v128,
5909 twiddle7re: v128,
5910 twiddle7im: v128,
5911 twiddle8re: v128,
5912 twiddle8im: v128,
5913 twiddle9re: v128,
5914 twiddle9im: v128,
5915 twiddle10re: v128,
5916 twiddle10im: v128,
5917 twiddle11re: v128,
5918 twiddle11im: v128,
5919 twiddle12re: v128,
5920 twiddle12im: v128,
5921 twiddle13re: v128,
5922 twiddle13im: v128,
5923 twiddle14re: v128,
5924 twiddle14im: v128,
5925 twiddle15re: v128,
5926 twiddle15im: v128,
5927 }
5928
5929 boilerplate_fft_wasm_simd_f32_butterfly!(
5930 WasmSimdF32Butterfly31,
5931 31,
5932 |this: &WasmSimdF32Butterfly31<_>| this.direction
5933 );
5934 boilerplate_fft_wasm_simd_common_butterfly!(
5935 WasmSimdF32Butterfly31,
5936 31,
5937 |this: &WasmSimdF32Butterfly31<_>| this.direction
5938 );
5939 impl<T: FftNum> WasmSimdF32Butterfly31<T> {
5940 #[inline(always)]
5941 pub fn new(direction: FftDirection) -> Self {
5942 assert_f32::<T>();
5943 let rotate = Rotate90F32::new(true);
5944 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 31, direction);
5945 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 31, direction);
5946 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 31, direction);
5947 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 31, direction);
5948 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 31, direction);
5949 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 31, direction);
5950 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 31, direction);
5951 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 31, direction);
5952 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 31, direction);
5953 let tw10: Complex<f32> = twiddles::compute_twiddle(10, 31, direction);
5954 let tw11: Complex<f32> = twiddles::compute_twiddle(11, 31, direction);
5955 let tw12: Complex<f32> = twiddles::compute_twiddle(12, 31, direction);
5956 let tw13: Complex<f32> = twiddles::compute_twiddle(13, 31, direction);
5957 let tw14: Complex<f32> = twiddles::compute_twiddle(14, 31, direction);
5958 let tw15: Complex<f32> = twiddles::compute_twiddle(15, 31, direction);
5959 let twiddle1re = f32x4_splat(tw1.re);
5960 let twiddle1im = f32x4_splat(tw1.im);
5961 let twiddle2re = f32x4_splat(tw2.re);
5962 let twiddle2im = f32x4_splat(tw2.im);
5963 let twiddle3re = f32x4_splat(tw3.re);
5964 let twiddle3im = f32x4_splat(tw3.im);
5965 let twiddle4re = f32x4_splat(tw4.re);
5966 let twiddle4im = f32x4_splat(tw4.im);
5967 let twiddle5re = f32x4_splat(tw5.re);
5968 let twiddle5im = f32x4_splat(tw5.im);
5969 let twiddle6re = f32x4_splat(tw6.re);
5970 let twiddle6im = f32x4_splat(tw6.im);
5971 let twiddle7re = f32x4_splat(tw7.re);
5972 let twiddle7im = f32x4_splat(tw7.im);
5973 let twiddle8re = f32x4_splat(tw8.re);
5974 let twiddle8im = f32x4_splat(tw8.im);
5975 let twiddle9re = f32x4_splat(tw9.re);
5976 let twiddle9im = f32x4_splat(tw9.im);
5977 let twiddle10re = f32x4_splat(tw10.re);
5978 let twiddle10im = f32x4_splat(tw10.im);
5979 let twiddle11re = f32x4_splat(tw11.re);
5980 let twiddle11im = f32x4_splat(tw11.im);
5981 let twiddle12re = f32x4_splat(tw12.re);
5982 let twiddle12im = f32x4_splat(tw12.im);
5983 let twiddle13re = f32x4_splat(tw13.re);
5984 let twiddle13im = f32x4_splat(tw13.im);
5985 let twiddle14re = f32x4_splat(tw14.re);
5986 let twiddle14im = f32x4_splat(tw14.im);
5987 let twiddle15re = f32x4_splat(tw15.re);
5988 let twiddle15im = f32x4_splat(tw15.im);
5989
5990 Self {
5991 direction,
5992 _phantom: std::marker::PhantomData,
5993 rotate,
5994 twiddle1re,
5995 twiddle1im,
5996 twiddle2re,
5997 twiddle2im,
5998 twiddle3re,
5999 twiddle3im,
6000 twiddle4re,
6001 twiddle4im,
6002 twiddle5re,
6003 twiddle5im,
6004 twiddle6re,
6005 twiddle6im,
6006 twiddle7re,
6007 twiddle7im,
6008 twiddle8re,
6009 twiddle8im,
6010 twiddle9re,
6011 twiddle9im,
6012 twiddle10re,
6013 twiddle10im,
6014 twiddle11re,
6015 twiddle11im,
6016 twiddle12re,
6017 twiddle12im,
6018 twiddle13re,
6019 twiddle13im,
6020 twiddle14re,
6021 twiddle14im,
6022 twiddle15re,
6023 twiddle15im,
6024 }
6025 }
6026
6027 #[inline(always)]
6028 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f32>) {
6029 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
6030
6031 let out = self.perform_parallel_fft_direct(values);
6032
6033 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
6034 }
6035
6036 #[inline(always)]
6037 pub(crate) unsafe fn perform_parallel_fft_contiguous(
6038 &self,
6039 mut buffer: impl WasmSimdArrayMut<f32>,
6040 ) {
6041 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60});
6042
6043 let values = [
6044 extract_lo_hi_f32(input_packed[0], input_packed[15]),
6045 extract_hi_lo_f32(input_packed[0], input_packed[16]),
6046 extract_lo_hi_f32(input_packed[1], input_packed[16]),
6047 extract_hi_lo_f32(input_packed[1], input_packed[17]),
6048 extract_lo_hi_f32(input_packed[2], input_packed[17]),
6049 extract_hi_lo_f32(input_packed[2], input_packed[18]),
6050 extract_lo_hi_f32(input_packed[3], input_packed[18]),
6051 extract_hi_lo_f32(input_packed[3], input_packed[19]),
6052 extract_lo_hi_f32(input_packed[4], input_packed[19]),
6053 extract_hi_lo_f32(input_packed[4], input_packed[20]),
6054 extract_lo_hi_f32(input_packed[5], input_packed[20]),
6055 extract_hi_lo_f32(input_packed[5], input_packed[21]),
6056 extract_lo_hi_f32(input_packed[6], input_packed[21]),
6057 extract_hi_lo_f32(input_packed[6], input_packed[22]),
6058 extract_lo_hi_f32(input_packed[7], input_packed[22]),
6059 extract_hi_lo_f32(input_packed[7], input_packed[23]),
6060 extract_lo_hi_f32(input_packed[8], input_packed[23]),
6061 extract_hi_lo_f32(input_packed[8], input_packed[24]),
6062 extract_lo_hi_f32(input_packed[9], input_packed[24]),
6063 extract_hi_lo_f32(input_packed[9], input_packed[25]),
6064 extract_lo_hi_f32(input_packed[10], input_packed[25]),
6065 extract_hi_lo_f32(input_packed[10], input_packed[26]),
6066 extract_lo_hi_f32(input_packed[11], input_packed[26]),
6067 extract_hi_lo_f32(input_packed[11], input_packed[27]),
6068 extract_lo_hi_f32(input_packed[12], input_packed[27]),
6069 extract_hi_lo_f32(input_packed[12], input_packed[28]),
6070 extract_lo_hi_f32(input_packed[13], input_packed[28]),
6071 extract_hi_lo_f32(input_packed[13], input_packed[29]),
6072 extract_lo_hi_f32(input_packed[14], input_packed[29]),
6073 extract_hi_lo_f32(input_packed[14], input_packed[30]),
6074 extract_lo_hi_f32(input_packed[15], input_packed[30]),
6075 ];
6076
6077 let out = self.perform_parallel_fft_direct(values);
6078
6079 let out_packed = [
6080 extract_lo_lo_f32(out[0], out[1]),
6081 extract_lo_lo_f32(out[2], out[3]),
6082 extract_lo_lo_f32(out[4], out[5]),
6083 extract_lo_lo_f32(out[6], out[7]),
6084 extract_lo_lo_f32(out[8], out[9]),
6085 extract_lo_lo_f32(out[10], out[11]),
6086 extract_lo_lo_f32(out[12], out[13]),
6087 extract_lo_lo_f32(out[14], out[15]),
6088 extract_lo_lo_f32(out[16], out[17]),
6089 extract_lo_lo_f32(out[18], out[19]),
6090 extract_lo_lo_f32(out[20], out[21]),
6091 extract_lo_lo_f32(out[22], out[23]),
6092 extract_lo_lo_f32(out[24], out[25]),
6093 extract_lo_lo_f32(out[26], out[27]),
6094 extract_lo_lo_f32(out[28], out[29]),
6095 extract_lo_hi_f32(out[30], out[0]),
6096 extract_hi_hi_f32(out[1], out[2]),
6097 extract_hi_hi_f32(out[3], out[4]),
6098 extract_hi_hi_f32(out[5], out[6]),
6099 extract_hi_hi_f32(out[7], out[8]),
6100 extract_hi_hi_f32(out[9], out[10]),
6101 extract_hi_hi_f32(out[11], out[12]),
6102 extract_hi_hi_f32(out[13], out[14]),
6103 extract_hi_hi_f32(out[15], out[16]),
6104 extract_hi_hi_f32(out[17], out[18]),
6105 extract_hi_hi_f32(out[19], out[20]),
6106 extract_hi_hi_f32(out[21], out[22]),
6107 extract_hi_hi_f32(out[23], out[24]),
6108 extract_hi_hi_f32(out[25], out[26]),
6109 extract_hi_hi_f32(out[27], out[28]),
6110 extract_hi_hi_f32(out[29], out[30]),
6111 ];
6112
6113 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
6114 }
6115
6116 #[inline(always)]
6117 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [v128; 31]) -> [v128; 31] {
6118 let [x1p30, x1m30] = parallel_fft2_interleaved_f32(values[1], values[30]);
6119 let [x2p29, x2m29] = parallel_fft2_interleaved_f32(values[2], values[29]);
6120 let [x3p28, x3m28] = parallel_fft2_interleaved_f32(values[3], values[28]);
6121 let [x4p27, x4m27] = parallel_fft2_interleaved_f32(values[4], values[27]);
6122 let [x5p26, x5m26] = parallel_fft2_interleaved_f32(values[5], values[26]);
6123 let [x6p25, x6m25] = parallel_fft2_interleaved_f32(values[6], values[25]);
6124 let [x7p24, x7m24] = parallel_fft2_interleaved_f32(values[7], values[24]);
6125 let [x8p23, x8m23] = parallel_fft2_interleaved_f32(values[8], values[23]);
6126 let [x9p22, x9m22] = parallel_fft2_interleaved_f32(values[9], values[22]);
6127 let [x10p21, x10m21] = parallel_fft2_interleaved_f32(values[10], values[21]);
6128 let [x11p20, x11m20] = parallel_fft2_interleaved_f32(values[11], values[20]);
6129 let [x12p19, x12m19] = parallel_fft2_interleaved_f32(values[12], values[19]);
6130 let [x13p18, x13m18] = parallel_fft2_interleaved_f32(values[13], values[18]);
6131 let [x14p17, x14m17] = parallel_fft2_interleaved_f32(values[14], values[17]);
6132 let [x15p16, x15m16] = parallel_fft2_interleaved_f32(values[15], values[16]);
6133
6134 let t_a1_1 = f32x4_mul(self.twiddle1re, x1p30);
6135 let t_a1_2 = f32x4_mul(self.twiddle2re, x2p29);
6136 let t_a1_3 = f32x4_mul(self.twiddle3re, x3p28);
6137 let t_a1_4 = f32x4_mul(self.twiddle4re, x4p27);
6138 let t_a1_5 = f32x4_mul(self.twiddle5re, x5p26);
6139 let t_a1_6 = f32x4_mul(self.twiddle6re, x6p25);
6140 let t_a1_7 = f32x4_mul(self.twiddle7re, x7p24);
6141 let t_a1_8 = f32x4_mul(self.twiddle8re, x8p23);
6142 let t_a1_9 = f32x4_mul(self.twiddle9re, x9p22);
6143 let t_a1_10 = f32x4_mul(self.twiddle10re, x10p21);
6144 let t_a1_11 = f32x4_mul(self.twiddle11re, x11p20);
6145 let t_a1_12 = f32x4_mul(self.twiddle12re, x12p19);
6146 let t_a1_13 = f32x4_mul(self.twiddle13re, x13p18);
6147 let t_a1_14 = f32x4_mul(self.twiddle14re, x14p17);
6148 let t_a1_15 = f32x4_mul(self.twiddle15re, x15p16);
6149 let t_a2_1 = f32x4_mul(self.twiddle2re, x1p30);
6150 let t_a2_2 = f32x4_mul(self.twiddle4re, x2p29);
6151 let t_a2_3 = f32x4_mul(self.twiddle6re, x3p28);
6152 let t_a2_4 = f32x4_mul(self.twiddle8re, x4p27);
6153 let t_a2_5 = f32x4_mul(self.twiddle10re, x5p26);
6154 let t_a2_6 = f32x4_mul(self.twiddle12re, x6p25);
6155 let t_a2_7 = f32x4_mul(self.twiddle14re, x7p24);
6156 let t_a2_8 = f32x4_mul(self.twiddle15re, x8p23);
6157 let t_a2_9 = f32x4_mul(self.twiddle13re, x9p22);
6158 let t_a2_10 = f32x4_mul(self.twiddle11re, x10p21);
6159 let t_a2_11 = f32x4_mul(self.twiddle9re, x11p20);
6160 let t_a2_12 = f32x4_mul(self.twiddle7re, x12p19);
6161 let t_a2_13 = f32x4_mul(self.twiddle5re, x13p18);
6162 let t_a2_14 = f32x4_mul(self.twiddle3re, x14p17);
6163 let t_a2_15 = f32x4_mul(self.twiddle1re, x15p16);
6164 let t_a3_1 = f32x4_mul(self.twiddle3re, x1p30);
6165 let t_a3_2 = f32x4_mul(self.twiddle6re, x2p29);
6166 let t_a3_3 = f32x4_mul(self.twiddle9re, x3p28);
6167 let t_a3_4 = f32x4_mul(self.twiddle12re, x4p27);
6168 let t_a3_5 = f32x4_mul(self.twiddle15re, x5p26);
6169 let t_a3_6 = f32x4_mul(self.twiddle13re, x6p25);
6170 let t_a3_7 = f32x4_mul(self.twiddle10re, x7p24);
6171 let t_a3_8 = f32x4_mul(self.twiddle7re, x8p23);
6172 let t_a3_9 = f32x4_mul(self.twiddle4re, x9p22);
6173 let t_a3_10 = f32x4_mul(self.twiddle1re, x10p21);
6174 let t_a3_11 = f32x4_mul(self.twiddle2re, x11p20);
6175 let t_a3_12 = f32x4_mul(self.twiddle5re, x12p19);
6176 let t_a3_13 = f32x4_mul(self.twiddle8re, x13p18);
6177 let t_a3_14 = f32x4_mul(self.twiddle11re, x14p17);
6178 let t_a3_15 = f32x4_mul(self.twiddle14re, x15p16);
6179 let t_a4_1 = f32x4_mul(self.twiddle4re, x1p30);
6180 let t_a4_2 = f32x4_mul(self.twiddle8re, x2p29);
6181 let t_a4_3 = f32x4_mul(self.twiddle12re, x3p28);
6182 let t_a4_4 = f32x4_mul(self.twiddle15re, x4p27);
6183 let t_a4_5 = f32x4_mul(self.twiddle11re, x5p26);
6184 let t_a4_6 = f32x4_mul(self.twiddle7re, x6p25);
6185 let t_a4_7 = f32x4_mul(self.twiddle3re, x7p24);
6186 let t_a4_8 = f32x4_mul(self.twiddle1re, x8p23);
6187 let t_a4_9 = f32x4_mul(self.twiddle5re, x9p22);
6188 let t_a4_10 = f32x4_mul(self.twiddle9re, x10p21);
6189 let t_a4_11 = f32x4_mul(self.twiddle13re, x11p20);
6190 let t_a4_12 = f32x4_mul(self.twiddle14re, x12p19);
6191 let t_a4_13 = f32x4_mul(self.twiddle10re, x13p18);
6192 let t_a4_14 = f32x4_mul(self.twiddle6re, x14p17);
6193 let t_a4_15 = f32x4_mul(self.twiddle2re, x15p16);
6194 let t_a5_1 = f32x4_mul(self.twiddle5re, x1p30);
6195 let t_a5_2 = f32x4_mul(self.twiddle10re, x2p29);
6196 let t_a5_3 = f32x4_mul(self.twiddle15re, x3p28);
6197 let t_a5_4 = f32x4_mul(self.twiddle11re, x4p27);
6198 let t_a5_5 = f32x4_mul(self.twiddle6re, x5p26);
6199 let t_a5_6 = f32x4_mul(self.twiddle1re, x6p25);
6200 let t_a5_7 = f32x4_mul(self.twiddle4re, x7p24);
6201 let t_a5_8 = f32x4_mul(self.twiddle9re, x8p23);
6202 let t_a5_9 = f32x4_mul(self.twiddle14re, x9p22);
6203 let t_a5_10 = f32x4_mul(self.twiddle12re, x10p21);
6204 let t_a5_11 = f32x4_mul(self.twiddle7re, x11p20);
6205 let t_a5_12 = f32x4_mul(self.twiddle2re, x12p19);
6206 let t_a5_13 = f32x4_mul(self.twiddle3re, x13p18);
6207 let t_a5_14 = f32x4_mul(self.twiddle8re, x14p17);
6208 let t_a5_15 = f32x4_mul(self.twiddle13re, x15p16);
6209 let t_a6_1 = f32x4_mul(self.twiddle6re, x1p30);
6210 let t_a6_2 = f32x4_mul(self.twiddle12re, x2p29);
6211 let t_a6_3 = f32x4_mul(self.twiddle13re, x3p28);
6212 let t_a6_4 = f32x4_mul(self.twiddle7re, x4p27);
6213 let t_a6_5 = f32x4_mul(self.twiddle1re, x5p26);
6214 let t_a6_6 = f32x4_mul(self.twiddle5re, x6p25);
6215 let t_a6_7 = f32x4_mul(self.twiddle11re, x7p24);
6216 let t_a6_8 = f32x4_mul(self.twiddle14re, x8p23);
6217 let t_a6_9 = f32x4_mul(self.twiddle8re, x9p22);
6218 let t_a6_10 = f32x4_mul(self.twiddle2re, x10p21);
6219 let t_a6_11 = f32x4_mul(self.twiddle4re, x11p20);
6220 let t_a6_12 = f32x4_mul(self.twiddle10re, x12p19);
6221 let t_a6_13 = f32x4_mul(self.twiddle15re, x13p18);
6222 let t_a6_14 = f32x4_mul(self.twiddle9re, x14p17);
6223 let t_a6_15 = f32x4_mul(self.twiddle3re, x15p16);
6224 let t_a7_1 = f32x4_mul(self.twiddle7re, x1p30);
6225 let t_a7_2 = f32x4_mul(self.twiddle14re, x2p29);
6226 let t_a7_3 = f32x4_mul(self.twiddle10re, x3p28);
6227 let t_a7_4 = f32x4_mul(self.twiddle3re, x4p27);
6228 let t_a7_5 = f32x4_mul(self.twiddle4re, x5p26);
6229 let t_a7_6 = f32x4_mul(self.twiddle11re, x6p25);
6230 let t_a7_7 = f32x4_mul(self.twiddle13re, x7p24);
6231 let t_a7_8 = f32x4_mul(self.twiddle6re, x8p23);
6232 let t_a7_9 = f32x4_mul(self.twiddle1re, x9p22);
6233 let t_a7_10 = f32x4_mul(self.twiddle8re, x10p21);
6234 let t_a7_11 = f32x4_mul(self.twiddle15re, x11p20);
6235 let t_a7_12 = f32x4_mul(self.twiddle9re, x12p19);
6236 let t_a7_13 = f32x4_mul(self.twiddle2re, x13p18);
6237 let t_a7_14 = f32x4_mul(self.twiddle5re, x14p17);
6238 let t_a7_15 = f32x4_mul(self.twiddle12re, x15p16);
6239 let t_a8_1 = f32x4_mul(self.twiddle8re, x1p30);
6240 let t_a8_2 = f32x4_mul(self.twiddle15re, x2p29);
6241 let t_a8_3 = f32x4_mul(self.twiddle7re, x3p28);
6242 let t_a8_4 = f32x4_mul(self.twiddle1re, x4p27);
6243 let t_a8_5 = f32x4_mul(self.twiddle9re, x5p26);
6244 let t_a8_6 = f32x4_mul(self.twiddle14re, x6p25);
6245 let t_a8_7 = f32x4_mul(self.twiddle6re, x7p24);
6246 let t_a8_8 = f32x4_mul(self.twiddle2re, x8p23);
6247 let t_a8_9 = f32x4_mul(self.twiddle10re, x9p22);
6248 let t_a8_10 = f32x4_mul(self.twiddle13re, x10p21);
6249 let t_a8_11 = f32x4_mul(self.twiddle5re, x11p20);
6250 let t_a8_12 = f32x4_mul(self.twiddle3re, x12p19);
6251 let t_a8_13 = f32x4_mul(self.twiddle11re, x13p18);
6252 let t_a8_14 = f32x4_mul(self.twiddle12re, x14p17);
6253 let t_a8_15 = f32x4_mul(self.twiddle4re, x15p16);
6254 let t_a9_1 = f32x4_mul(self.twiddle9re, x1p30);
6255 let t_a9_2 = f32x4_mul(self.twiddle13re, x2p29);
6256 let t_a9_3 = f32x4_mul(self.twiddle4re, x3p28);
6257 let t_a9_4 = f32x4_mul(self.twiddle5re, x4p27);
6258 let t_a9_5 = f32x4_mul(self.twiddle14re, x5p26);
6259 let t_a9_6 = f32x4_mul(self.twiddle8re, x6p25);
6260 let t_a9_7 = f32x4_mul(self.twiddle1re, x7p24);
6261 let t_a9_8 = f32x4_mul(self.twiddle10re, x8p23);
6262 let t_a9_9 = f32x4_mul(self.twiddle12re, x9p22);
6263 let t_a9_10 = f32x4_mul(self.twiddle3re, x10p21);
6264 let t_a9_11 = f32x4_mul(self.twiddle6re, x11p20);
6265 let t_a9_12 = f32x4_mul(self.twiddle15re, x12p19);
6266 let t_a9_13 = f32x4_mul(self.twiddle7re, x13p18);
6267 let t_a9_14 = f32x4_mul(self.twiddle2re, x14p17);
6268 let t_a9_15 = f32x4_mul(self.twiddle11re, x15p16);
6269 let t_a10_1 = f32x4_mul(self.twiddle10re, x1p30);
6270 let t_a10_2 = f32x4_mul(self.twiddle11re, x2p29);
6271 let t_a10_3 = f32x4_mul(self.twiddle1re, x3p28);
6272 let t_a10_4 = f32x4_mul(self.twiddle9re, x4p27);
6273 let t_a10_5 = f32x4_mul(self.twiddle12re, x5p26);
6274 let t_a10_6 = f32x4_mul(self.twiddle2re, x6p25);
6275 let t_a10_7 = f32x4_mul(self.twiddle8re, x7p24);
6276 let t_a10_8 = f32x4_mul(self.twiddle13re, x8p23);
6277 let t_a10_9 = f32x4_mul(self.twiddle3re, x9p22);
6278 let t_a10_10 = f32x4_mul(self.twiddle7re, x10p21);
6279 let t_a10_11 = f32x4_mul(self.twiddle14re, x11p20);
6280 let t_a10_12 = f32x4_mul(self.twiddle4re, x12p19);
6281 let t_a10_13 = f32x4_mul(self.twiddle6re, x13p18);
6282 let t_a10_14 = f32x4_mul(self.twiddle15re, x14p17);
6283 let t_a10_15 = f32x4_mul(self.twiddle5re, x15p16);
6284 let t_a11_1 = f32x4_mul(self.twiddle11re, x1p30);
6285 let t_a11_2 = f32x4_mul(self.twiddle9re, x2p29);
6286 let t_a11_3 = f32x4_mul(self.twiddle2re, x3p28);
6287 let t_a11_4 = f32x4_mul(self.twiddle13re, x4p27);
6288 let t_a11_5 = f32x4_mul(self.twiddle7re, x5p26);
6289 let t_a11_6 = f32x4_mul(self.twiddle4re, x6p25);
6290 let t_a11_7 = f32x4_mul(self.twiddle15re, x7p24);
6291 let t_a11_8 = f32x4_mul(self.twiddle5re, x8p23);
6292 let t_a11_9 = f32x4_mul(self.twiddle6re, x9p22);
6293 let t_a11_10 = f32x4_mul(self.twiddle14re, x10p21);
6294 let t_a11_11 = f32x4_mul(self.twiddle3re, x11p20);
6295 let t_a11_12 = f32x4_mul(self.twiddle8re, x12p19);
6296 let t_a11_13 = f32x4_mul(self.twiddle12re, x13p18);
6297 let t_a11_14 = f32x4_mul(self.twiddle1re, x14p17);
6298 let t_a11_15 = f32x4_mul(self.twiddle10re, x15p16);
6299 let t_a12_1 = f32x4_mul(self.twiddle12re, x1p30);
6300 let t_a12_2 = f32x4_mul(self.twiddle7re, x2p29);
6301 let t_a12_3 = f32x4_mul(self.twiddle5re, x3p28);
6302 let t_a12_4 = f32x4_mul(self.twiddle14re, x4p27);
6303 let t_a12_5 = f32x4_mul(self.twiddle2re, x5p26);
6304 let t_a12_6 = f32x4_mul(self.twiddle10re, x6p25);
6305 let t_a12_7 = f32x4_mul(self.twiddle9re, x7p24);
6306 let t_a12_8 = f32x4_mul(self.twiddle3re, x8p23);
6307 let t_a12_9 = f32x4_mul(self.twiddle15re, x9p22);
6308 let t_a12_10 = f32x4_mul(self.twiddle4re, x10p21);
6309 let t_a12_11 = f32x4_mul(self.twiddle8re, x11p20);
6310 let t_a12_12 = f32x4_mul(self.twiddle11re, x12p19);
6311 let t_a12_13 = f32x4_mul(self.twiddle1re, x13p18);
6312 let t_a12_14 = f32x4_mul(self.twiddle13re, x14p17);
6313 let t_a12_15 = f32x4_mul(self.twiddle6re, x15p16);
6314 let t_a13_1 = f32x4_mul(self.twiddle13re, x1p30);
6315 let t_a13_2 = f32x4_mul(self.twiddle5re, x2p29);
6316 let t_a13_3 = f32x4_mul(self.twiddle8re, x3p28);
6317 let t_a13_4 = f32x4_mul(self.twiddle10re, x4p27);
6318 let t_a13_5 = f32x4_mul(self.twiddle3re, x5p26);
6319 let t_a13_6 = f32x4_mul(self.twiddle15re, x6p25);
6320 let t_a13_7 = f32x4_mul(self.twiddle2re, x7p24);
6321 let t_a13_8 = f32x4_mul(self.twiddle11re, x8p23);
6322 let t_a13_9 = f32x4_mul(self.twiddle7re, x9p22);
6323 let t_a13_10 = f32x4_mul(self.twiddle6re, x10p21);
6324 let t_a13_11 = f32x4_mul(self.twiddle12re, x11p20);
6325 let t_a13_12 = f32x4_mul(self.twiddle1re, x12p19);
6326 let t_a13_13 = f32x4_mul(self.twiddle14re, x13p18);
6327 let t_a13_14 = f32x4_mul(self.twiddle4re, x14p17);
6328 let t_a13_15 = f32x4_mul(self.twiddle9re, x15p16);
6329 let t_a14_1 = f32x4_mul(self.twiddle14re, x1p30);
6330 let t_a14_2 = f32x4_mul(self.twiddle3re, x2p29);
6331 let t_a14_3 = f32x4_mul(self.twiddle11re, x3p28);
6332 let t_a14_4 = f32x4_mul(self.twiddle6re, x4p27);
6333 let t_a14_5 = f32x4_mul(self.twiddle8re, x5p26);
6334 let t_a14_6 = f32x4_mul(self.twiddle9re, x6p25);
6335 let t_a14_7 = f32x4_mul(self.twiddle5re, x7p24);
6336 let t_a14_8 = f32x4_mul(self.twiddle12re, x8p23);
6337 let t_a14_9 = f32x4_mul(self.twiddle2re, x9p22);
6338 let t_a14_10 = f32x4_mul(self.twiddle15re, x10p21);
6339 let t_a14_11 = f32x4_mul(self.twiddle1re, x11p20);
6340 let t_a14_12 = f32x4_mul(self.twiddle13re, x12p19);
6341 let t_a14_13 = f32x4_mul(self.twiddle4re, x13p18);
6342 let t_a14_14 = f32x4_mul(self.twiddle10re, x14p17);
6343 let t_a14_15 = f32x4_mul(self.twiddle7re, x15p16);
6344 let t_a15_1 = f32x4_mul(self.twiddle15re, x1p30);
6345 let t_a15_2 = f32x4_mul(self.twiddle1re, x2p29);
6346 let t_a15_3 = f32x4_mul(self.twiddle14re, x3p28);
6347 let t_a15_4 = f32x4_mul(self.twiddle2re, x4p27);
6348 let t_a15_5 = f32x4_mul(self.twiddle13re, x5p26);
6349 let t_a15_6 = f32x4_mul(self.twiddle3re, x6p25);
6350 let t_a15_7 = f32x4_mul(self.twiddle12re, x7p24);
6351 let t_a15_8 = f32x4_mul(self.twiddle4re, x8p23);
6352 let t_a15_9 = f32x4_mul(self.twiddle11re, x9p22);
6353 let t_a15_10 = f32x4_mul(self.twiddle5re, x10p21);
6354 let t_a15_11 = f32x4_mul(self.twiddle10re, x11p20);
6355 let t_a15_12 = f32x4_mul(self.twiddle6re, x12p19);
6356 let t_a15_13 = f32x4_mul(self.twiddle9re, x13p18);
6357 let t_a15_14 = f32x4_mul(self.twiddle7re, x14p17);
6358 let t_a15_15 = f32x4_mul(self.twiddle8re, x15p16);
6359
6360 let t_b1_1 = f32x4_mul(self.twiddle1im, x1m30);
6361 let t_b1_2 = f32x4_mul(self.twiddle2im, x2m29);
6362 let t_b1_3 = f32x4_mul(self.twiddle3im, x3m28);
6363 let t_b1_4 = f32x4_mul(self.twiddle4im, x4m27);
6364 let t_b1_5 = f32x4_mul(self.twiddle5im, x5m26);
6365 let t_b1_6 = f32x4_mul(self.twiddle6im, x6m25);
6366 let t_b1_7 = f32x4_mul(self.twiddle7im, x7m24);
6367 let t_b1_8 = f32x4_mul(self.twiddle8im, x8m23);
6368 let t_b1_9 = f32x4_mul(self.twiddle9im, x9m22);
6369 let t_b1_10 = f32x4_mul(self.twiddle10im, x10m21);
6370 let t_b1_11 = f32x4_mul(self.twiddle11im, x11m20);
6371 let t_b1_12 = f32x4_mul(self.twiddle12im, x12m19);
6372 let t_b1_13 = f32x4_mul(self.twiddle13im, x13m18);
6373 let t_b1_14 = f32x4_mul(self.twiddle14im, x14m17);
6374 let t_b1_15 = f32x4_mul(self.twiddle15im, x15m16);
6375 let t_b2_1 = f32x4_mul(self.twiddle2im, x1m30);
6376 let t_b2_2 = f32x4_mul(self.twiddle4im, x2m29);
6377 let t_b2_3 = f32x4_mul(self.twiddle6im, x3m28);
6378 let t_b2_4 = f32x4_mul(self.twiddle8im, x4m27);
6379 let t_b2_5 = f32x4_mul(self.twiddle10im, x5m26);
6380 let t_b2_6 = f32x4_mul(self.twiddle12im, x6m25);
6381 let t_b2_7 = f32x4_mul(self.twiddle14im, x7m24);
6382 let t_b2_8 = f32x4_mul(self.twiddle15im, x8m23);
6383 let t_b2_9 = f32x4_mul(self.twiddle13im, x9m22);
6384 let t_b2_10 = f32x4_mul(self.twiddle11im, x10m21);
6385 let t_b2_11 = f32x4_mul(self.twiddle9im, x11m20);
6386 let t_b2_12 = f32x4_mul(self.twiddle7im, x12m19);
6387 let t_b2_13 = f32x4_mul(self.twiddle5im, x13m18);
6388 let t_b2_14 = f32x4_mul(self.twiddle3im, x14m17);
6389 let t_b2_15 = f32x4_mul(self.twiddle1im, x15m16);
6390 let t_b3_1 = f32x4_mul(self.twiddle3im, x1m30);
6391 let t_b3_2 = f32x4_mul(self.twiddle6im, x2m29);
6392 let t_b3_3 = f32x4_mul(self.twiddle9im, x3m28);
6393 let t_b3_4 = f32x4_mul(self.twiddle12im, x4m27);
6394 let t_b3_5 = f32x4_mul(self.twiddle15im, x5m26);
6395 let t_b3_6 = f32x4_mul(self.twiddle13im, x6m25);
6396 let t_b3_7 = f32x4_mul(self.twiddle10im, x7m24);
6397 let t_b3_8 = f32x4_mul(self.twiddle7im, x8m23);
6398 let t_b3_9 = f32x4_mul(self.twiddle4im, x9m22);
6399 let t_b3_10 = f32x4_mul(self.twiddle1im, x10m21);
6400 let t_b3_11 = f32x4_mul(self.twiddle2im, x11m20);
6401 let t_b3_12 = f32x4_mul(self.twiddle5im, x12m19);
6402 let t_b3_13 = f32x4_mul(self.twiddle8im, x13m18);
6403 let t_b3_14 = f32x4_mul(self.twiddle11im, x14m17);
6404 let t_b3_15 = f32x4_mul(self.twiddle14im, x15m16);
6405 let t_b4_1 = f32x4_mul(self.twiddle4im, x1m30);
6406 let t_b4_2 = f32x4_mul(self.twiddle8im, x2m29);
6407 let t_b4_3 = f32x4_mul(self.twiddle12im, x3m28);
6408 let t_b4_4 = f32x4_mul(self.twiddle15im, x4m27);
6409 let t_b4_5 = f32x4_mul(self.twiddle11im, x5m26);
6410 let t_b4_6 = f32x4_mul(self.twiddle7im, x6m25);
6411 let t_b4_7 = f32x4_mul(self.twiddle3im, x7m24);
6412 let t_b4_8 = f32x4_mul(self.twiddle1im, x8m23);
6413 let t_b4_9 = f32x4_mul(self.twiddle5im, x9m22);
6414 let t_b4_10 = f32x4_mul(self.twiddle9im, x10m21);
6415 let t_b4_11 = f32x4_mul(self.twiddle13im, x11m20);
6416 let t_b4_12 = f32x4_mul(self.twiddle14im, x12m19);
6417 let t_b4_13 = f32x4_mul(self.twiddle10im, x13m18);
6418 let t_b4_14 = f32x4_mul(self.twiddle6im, x14m17);
6419 let t_b4_15 = f32x4_mul(self.twiddle2im, x15m16);
6420 let t_b5_1 = f32x4_mul(self.twiddle5im, x1m30);
6421 let t_b5_2 = f32x4_mul(self.twiddle10im, x2m29);
6422 let t_b5_3 = f32x4_mul(self.twiddle15im, x3m28);
6423 let t_b5_4 = f32x4_mul(self.twiddle11im, x4m27);
6424 let t_b5_5 = f32x4_mul(self.twiddle6im, x5m26);
6425 let t_b5_6 = f32x4_mul(self.twiddle1im, x6m25);
6426 let t_b5_7 = f32x4_mul(self.twiddle4im, x7m24);
6427 let t_b5_8 = f32x4_mul(self.twiddle9im, x8m23);
6428 let t_b5_9 = f32x4_mul(self.twiddle14im, x9m22);
6429 let t_b5_10 = f32x4_mul(self.twiddle12im, x10m21);
6430 let t_b5_11 = f32x4_mul(self.twiddle7im, x11m20);
6431 let t_b5_12 = f32x4_mul(self.twiddle2im, x12m19);
6432 let t_b5_13 = f32x4_mul(self.twiddle3im, x13m18);
6433 let t_b5_14 = f32x4_mul(self.twiddle8im, x14m17);
6434 let t_b5_15 = f32x4_mul(self.twiddle13im, x15m16);
6435 let t_b6_1 = f32x4_mul(self.twiddle6im, x1m30);
6436 let t_b6_2 = f32x4_mul(self.twiddle12im, x2m29);
6437 let t_b6_3 = f32x4_mul(self.twiddle13im, x3m28);
6438 let t_b6_4 = f32x4_mul(self.twiddle7im, x4m27);
6439 let t_b6_5 = f32x4_mul(self.twiddle1im, x5m26);
6440 let t_b6_6 = f32x4_mul(self.twiddle5im, x6m25);
6441 let t_b6_7 = f32x4_mul(self.twiddle11im, x7m24);
6442 let t_b6_8 = f32x4_mul(self.twiddle14im, x8m23);
6443 let t_b6_9 = f32x4_mul(self.twiddle8im, x9m22);
6444 let t_b6_10 = f32x4_mul(self.twiddle2im, x10m21);
6445 let t_b6_11 = f32x4_mul(self.twiddle4im, x11m20);
6446 let t_b6_12 = f32x4_mul(self.twiddle10im, x12m19);
6447 let t_b6_13 = f32x4_mul(self.twiddle15im, x13m18);
6448 let t_b6_14 = f32x4_mul(self.twiddle9im, x14m17);
6449 let t_b6_15 = f32x4_mul(self.twiddle3im, x15m16);
6450 let t_b7_1 = f32x4_mul(self.twiddle7im, x1m30);
6451 let t_b7_2 = f32x4_mul(self.twiddle14im, x2m29);
6452 let t_b7_3 = f32x4_mul(self.twiddle10im, x3m28);
6453 let t_b7_4 = f32x4_mul(self.twiddle3im, x4m27);
6454 let t_b7_5 = f32x4_mul(self.twiddle4im, x5m26);
6455 let t_b7_6 = f32x4_mul(self.twiddle11im, x6m25);
6456 let t_b7_7 = f32x4_mul(self.twiddle13im, x7m24);
6457 let t_b7_8 = f32x4_mul(self.twiddle6im, x8m23);
6458 let t_b7_9 = f32x4_mul(self.twiddle1im, x9m22);
6459 let t_b7_10 = f32x4_mul(self.twiddle8im, x10m21);
6460 let t_b7_11 = f32x4_mul(self.twiddle15im, x11m20);
6461 let t_b7_12 = f32x4_mul(self.twiddle9im, x12m19);
6462 let t_b7_13 = f32x4_mul(self.twiddle2im, x13m18);
6463 let t_b7_14 = f32x4_mul(self.twiddle5im, x14m17);
6464 let t_b7_15 = f32x4_mul(self.twiddle12im, x15m16);
6465 let t_b8_1 = f32x4_mul(self.twiddle8im, x1m30);
6466 let t_b8_2 = f32x4_mul(self.twiddle15im, x2m29);
6467 let t_b8_3 = f32x4_mul(self.twiddle7im, x3m28);
6468 let t_b8_4 = f32x4_mul(self.twiddle1im, x4m27);
6469 let t_b8_5 = f32x4_mul(self.twiddle9im, x5m26);
6470 let t_b8_6 = f32x4_mul(self.twiddle14im, x6m25);
6471 let t_b8_7 = f32x4_mul(self.twiddle6im, x7m24);
6472 let t_b8_8 = f32x4_mul(self.twiddle2im, x8m23);
6473 let t_b8_9 = f32x4_mul(self.twiddle10im, x9m22);
6474 let t_b8_10 = f32x4_mul(self.twiddle13im, x10m21);
6475 let t_b8_11 = f32x4_mul(self.twiddle5im, x11m20);
6476 let t_b8_12 = f32x4_mul(self.twiddle3im, x12m19);
6477 let t_b8_13 = f32x4_mul(self.twiddle11im, x13m18);
6478 let t_b8_14 = f32x4_mul(self.twiddle12im, x14m17);
6479 let t_b8_15 = f32x4_mul(self.twiddle4im, x15m16);
6480 let t_b9_1 = f32x4_mul(self.twiddle9im, x1m30);
6481 let t_b9_2 = f32x4_mul(self.twiddle13im, x2m29);
6482 let t_b9_3 = f32x4_mul(self.twiddle4im, x3m28);
6483 let t_b9_4 = f32x4_mul(self.twiddle5im, x4m27);
6484 let t_b9_5 = f32x4_mul(self.twiddle14im, x5m26);
6485 let t_b9_6 = f32x4_mul(self.twiddle8im, x6m25);
6486 let t_b9_7 = f32x4_mul(self.twiddle1im, x7m24);
6487 let t_b9_8 = f32x4_mul(self.twiddle10im, x8m23);
6488 let t_b9_9 = f32x4_mul(self.twiddle12im, x9m22);
6489 let t_b9_10 = f32x4_mul(self.twiddle3im, x10m21);
6490 let t_b9_11 = f32x4_mul(self.twiddle6im, x11m20);
6491 let t_b9_12 = f32x4_mul(self.twiddle15im, x12m19);
6492 let t_b9_13 = f32x4_mul(self.twiddle7im, x13m18);
6493 let t_b9_14 = f32x4_mul(self.twiddle2im, x14m17);
6494 let t_b9_15 = f32x4_mul(self.twiddle11im, x15m16);
6495 let t_b10_1 = f32x4_mul(self.twiddle10im, x1m30);
6496 let t_b10_2 = f32x4_mul(self.twiddle11im, x2m29);
6497 let t_b10_3 = f32x4_mul(self.twiddle1im, x3m28);
6498 let t_b10_4 = f32x4_mul(self.twiddle9im, x4m27);
6499 let t_b10_5 = f32x4_mul(self.twiddle12im, x5m26);
6500 let t_b10_6 = f32x4_mul(self.twiddle2im, x6m25);
6501 let t_b10_7 = f32x4_mul(self.twiddle8im, x7m24);
6502 let t_b10_8 = f32x4_mul(self.twiddle13im, x8m23);
6503 let t_b10_9 = f32x4_mul(self.twiddle3im, x9m22);
6504 let t_b10_10 = f32x4_mul(self.twiddle7im, x10m21);
6505 let t_b10_11 = f32x4_mul(self.twiddle14im, x11m20);
6506 let t_b10_12 = f32x4_mul(self.twiddle4im, x12m19);
6507 let t_b10_13 = f32x4_mul(self.twiddle6im, x13m18);
6508 let t_b10_14 = f32x4_mul(self.twiddle15im, x14m17);
6509 let t_b10_15 = f32x4_mul(self.twiddle5im, x15m16);
6510 let t_b11_1 = f32x4_mul(self.twiddle11im, x1m30);
6511 let t_b11_2 = f32x4_mul(self.twiddle9im, x2m29);
6512 let t_b11_3 = f32x4_mul(self.twiddle2im, x3m28);
6513 let t_b11_4 = f32x4_mul(self.twiddle13im, x4m27);
6514 let t_b11_5 = f32x4_mul(self.twiddle7im, x5m26);
6515 let t_b11_6 = f32x4_mul(self.twiddle4im, x6m25);
6516 let t_b11_7 = f32x4_mul(self.twiddle15im, x7m24);
6517 let t_b11_8 = f32x4_mul(self.twiddle5im, x8m23);
6518 let t_b11_9 = f32x4_mul(self.twiddle6im, x9m22);
6519 let t_b11_10 = f32x4_mul(self.twiddle14im, x10m21);
6520 let t_b11_11 = f32x4_mul(self.twiddle3im, x11m20);
6521 let t_b11_12 = f32x4_mul(self.twiddle8im, x12m19);
6522 let t_b11_13 = f32x4_mul(self.twiddle12im, x13m18);
6523 let t_b11_14 = f32x4_mul(self.twiddle1im, x14m17);
6524 let t_b11_15 = f32x4_mul(self.twiddle10im, x15m16);
6525 let t_b12_1 = f32x4_mul(self.twiddle12im, x1m30);
6526 let t_b12_2 = f32x4_mul(self.twiddle7im, x2m29);
6527 let t_b12_3 = f32x4_mul(self.twiddle5im, x3m28);
6528 let t_b12_4 = f32x4_mul(self.twiddle14im, x4m27);
6529 let t_b12_5 = f32x4_mul(self.twiddle2im, x5m26);
6530 let t_b12_6 = f32x4_mul(self.twiddle10im, x6m25);
6531 let t_b12_7 = f32x4_mul(self.twiddle9im, x7m24);
6532 let t_b12_8 = f32x4_mul(self.twiddle3im, x8m23);
6533 let t_b12_9 = f32x4_mul(self.twiddle15im, x9m22);
6534 let t_b12_10 = f32x4_mul(self.twiddle4im, x10m21);
6535 let t_b12_11 = f32x4_mul(self.twiddle8im, x11m20);
6536 let t_b12_12 = f32x4_mul(self.twiddle11im, x12m19);
6537 let t_b12_13 = f32x4_mul(self.twiddle1im, x13m18);
6538 let t_b12_14 = f32x4_mul(self.twiddle13im, x14m17);
6539 let t_b12_15 = f32x4_mul(self.twiddle6im, x15m16);
6540 let t_b13_1 = f32x4_mul(self.twiddle13im, x1m30);
6541 let t_b13_2 = f32x4_mul(self.twiddle5im, x2m29);
6542 let t_b13_3 = f32x4_mul(self.twiddle8im, x3m28);
6543 let t_b13_4 = f32x4_mul(self.twiddle10im, x4m27);
6544 let t_b13_5 = f32x4_mul(self.twiddle3im, x5m26);
6545 let t_b13_6 = f32x4_mul(self.twiddle15im, x6m25);
6546 let t_b13_7 = f32x4_mul(self.twiddle2im, x7m24);
6547 let t_b13_8 = f32x4_mul(self.twiddle11im, x8m23);
6548 let t_b13_9 = f32x4_mul(self.twiddle7im, x9m22);
6549 let t_b13_10 = f32x4_mul(self.twiddle6im, x10m21);
6550 let t_b13_11 = f32x4_mul(self.twiddle12im, x11m20);
6551 let t_b13_12 = f32x4_mul(self.twiddle1im, x12m19);
6552 let t_b13_13 = f32x4_mul(self.twiddle14im, x13m18);
6553 let t_b13_14 = f32x4_mul(self.twiddle4im, x14m17);
6554 let t_b13_15 = f32x4_mul(self.twiddle9im, x15m16);
6555 let t_b14_1 = f32x4_mul(self.twiddle14im, x1m30);
6556 let t_b14_2 = f32x4_mul(self.twiddle3im, x2m29);
6557 let t_b14_3 = f32x4_mul(self.twiddle11im, x3m28);
6558 let t_b14_4 = f32x4_mul(self.twiddle6im, x4m27);
6559 let t_b14_5 = f32x4_mul(self.twiddle8im, x5m26);
6560 let t_b14_6 = f32x4_mul(self.twiddle9im, x6m25);
6561 let t_b14_7 = f32x4_mul(self.twiddle5im, x7m24);
6562 let t_b14_8 = f32x4_mul(self.twiddle12im, x8m23);
6563 let t_b14_9 = f32x4_mul(self.twiddle2im, x9m22);
6564 let t_b14_10 = f32x4_mul(self.twiddle15im, x10m21);
6565 let t_b14_11 = f32x4_mul(self.twiddle1im, x11m20);
6566 let t_b14_12 = f32x4_mul(self.twiddle13im, x12m19);
6567 let t_b14_13 = f32x4_mul(self.twiddle4im, x13m18);
6568 let t_b14_14 = f32x4_mul(self.twiddle10im, x14m17);
6569 let t_b14_15 = f32x4_mul(self.twiddle7im, x15m16);
6570 let t_b15_1 = f32x4_mul(self.twiddle15im, x1m30);
6571 let t_b15_2 = f32x4_mul(self.twiddle1im, x2m29);
6572 let t_b15_3 = f32x4_mul(self.twiddle14im, x3m28);
6573 let t_b15_4 = f32x4_mul(self.twiddle2im, x4m27);
6574 let t_b15_5 = f32x4_mul(self.twiddle13im, x5m26);
6575 let t_b15_6 = f32x4_mul(self.twiddle3im, x6m25);
6576 let t_b15_7 = f32x4_mul(self.twiddle12im, x7m24);
6577 let t_b15_8 = f32x4_mul(self.twiddle4im, x8m23);
6578 let t_b15_9 = f32x4_mul(self.twiddle11im, x9m22);
6579 let t_b15_10 = f32x4_mul(self.twiddle5im, x10m21);
6580 let t_b15_11 = f32x4_mul(self.twiddle10im, x11m20);
6581 let t_b15_12 = f32x4_mul(self.twiddle6im, x12m19);
6582 let t_b15_13 = f32x4_mul(self.twiddle9im, x13m18);
6583 let t_b15_14 = f32x4_mul(self.twiddle7im, x14m17);
6584 let t_b15_15 = f32x4_mul(self.twiddle8im, x15m16);
6585
6586 let x0 = values[0];
6587 let t_a1 = calc_f32!(
6588 x0 + t_a1_1
6589 + t_a1_2
6590 + t_a1_3
6591 + t_a1_4
6592 + t_a1_5
6593 + t_a1_6
6594 + t_a1_7
6595 + t_a1_8
6596 + t_a1_9
6597 + t_a1_10
6598 + t_a1_11
6599 + t_a1_12
6600 + t_a1_13
6601 + t_a1_14
6602 + t_a1_15
6603 );
6604 let t_a2 = calc_f32!(
6605 x0 + t_a2_1
6606 + t_a2_2
6607 + t_a2_3
6608 + t_a2_4
6609 + t_a2_5
6610 + t_a2_6
6611 + t_a2_7
6612 + t_a2_8
6613 + t_a2_9
6614 + t_a2_10
6615 + t_a2_11
6616 + t_a2_12
6617 + t_a2_13
6618 + t_a2_14
6619 + t_a2_15
6620 );
6621 let t_a3 = calc_f32!(
6622 x0 + t_a3_1
6623 + t_a3_2
6624 + t_a3_3
6625 + t_a3_4
6626 + t_a3_5
6627 + t_a3_6
6628 + t_a3_7
6629 + t_a3_8
6630 + t_a3_9
6631 + t_a3_10
6632 + t_a3_11
6633 + t_a3_12
6634 + t_a3_13
6635 + t_a3_14
6636 + t_a3_15
6637 );
6638 let t_a4 = calc_f32!(
6639 x0 + t_a4_1
6640 + t_a4_2
6641 + t_a4_3
6642 + t_a4_4
6643 + t_a4_5
6644 + t_a4_6
6645 + t_a4_7
6646 + t_a4_8
6647 + t_a4_9
6648 + t_a4_10
6649 + t_a4_11
6650 + t_a4_12
6651 + t_a4_13
6652 + t_a4_14
6653 + t_a4_15
6654 );
6655 let t_a5 = calc_f32!(
6656 x0 + t_a5_1
6657 + t_a5_2
6658 + t_a5_3
6659 + t_a5_4
6660 + t_a5_5
6661 + t_a5_6
6662 + t_a5_7
6663 + t_a5_8
6664 + t_a5_9
6665 + t_a5_10
6666 + t_a5_11
6667 + t_a5_12
6668 + t_a5_13
6669 + t_a5_14
6670 + t_a5_15
6671 );
6672 let t_a6 = calc_f32!(
6673 x0 + t_a6_1
6674 + t_a6_2
6675 + t_a6_3
6676 + t_a6_4
6677 + t_a6_5
6678 + t_a6_6
6679 + t_a6_7
6680 + t_a6_8
6681 + t_a6_9
6682 + t_a6_10
6683 + t_a6_11
6684 + t_a6_12
6685 + t_a6_13
6686 + t_a6_14
6687 + t_a6_15
6688 );
6689 let t_a7 = calc_f32!(
6690 x0 + t_a7_1
6691 + t_a7_2
6692 + t_a7_3
6693 + t_a7_4
6694 + t_a7_5
6695 + t_a7_6
6696 + t_a7_7
6697 + t_a7_8
6698 + t_a7_9
6699 + t_a7_10
6700 + t_a7_11
6701 + t_a7_12
6702 + t_a7_13
6703 + t_a7_14
6704 + t_a7_15
6705 );
6706 let t_a8 = calc_f32!(
6707 x0 + t_a8_1
6708 + t_a8_2
6709 + t_a8_3
6710 + t_a8_4
6711 + t_a8_5
6712 + t_a8_6
6713 + t_a8_7
6714 + t_a8_8
6715 + t_a8_9
6716 + t_a8_10
6717 + t_a8_11
6718 + t_a8_12
6719 + t_a8_13
6720 + t_a8_14
6721 + t_a8_15
6722 );
6723 let t_a9 = calc_f32!(
6724 x0 + t_a9_1
6725 + t_a9_2
6726 + t_a9_3
6727 + t_a9_4
6728 + t_a9_5
6729 + t_a9_6
6730 + t_a9_7
6731 + t_a9_8
6732 + t_a9_9
6733 + t_a9_10
6734 + t_a9_11
6735 + t_a9_12
6736 + t_a9_13
6737 + t_a9_14
6738 + t_a9_15
6739 );
6740 let t_a10 = calc_f32!(
6741 x0 + t_a10_1
6742 + t_a10_2
6743 + t_a10_3
6744 + t_a10_4
6745 + t_a10_5
6746 + t_a10_6
6747 + t_a10_7
6748 + t_a10_8
6749 + t_a10_9
6750 + t_a10_10
6751 + t_a10_11
6752 + t_a10_12
6753 + t_a10_13
6754 + t_a10_14
6755 + t_a10_15
6756 );
6757 let t_a11 = calc_f32!(
6758 x0 + t_a11_1
6759 + t_a11_2
6760 + t_a11_3
6761 + t_a11_4
6762 + t_a11_5
6763 + t_a11_6
6764 + t_a11_7
6765 + t_a11_8
6766 + t_a11_9
6767 + t_a11_10
6768 + t_a11_11
6769 + t_a11_12
6770 + t_a11_13
6771 + t_a11_14
6772 + t_a11_15
6773 );
6774 let t_a12 = calc_f32!(
6775 x0 + t_a12_1
6776 + t_a12_2
6777 + t_a12_3
6778 + t_a12_4
6779 + t_a12_5
6780 + t_a12_6
6781 + t_a12_7
6782 + t_a12_8
6783 + t_a12_9
6784 + t_a12_10
6785 + t_a12_11
6786 + t_a12_12
6787 + t_a12_13
6788 + t_a12_14
6789 + t_a12_15
6790 );
6791 let t_a13 = calc_f32!(
6792 x0 + t_a13_1
6793 + t_a13_2
6794 + t_a13_3
6795 + t_a13_4
6796 + t_a13_5
6797 + t_a13_6
6798 + t_a13_7
6799 + t_a13_8
6800 + t_a13_9
6801 + t_a13_10
6802 + t_a13_11
6803 + t_a13_12
6804 + t_a13_13
6805 + t_a13_14
6806 + t_a13_15
6807 );
6808 let t_a14 = calc_f32!(
6809 x0 + t_a14_1
6810 + t_a14_2
6811 + t_a14_3
6812 + t_a14_4
6813 + t_a14_5
6814 + t_a14_6
6815 + t_a14_7
6816 + t_a14_8
6817 + t_a14_9
6818 + t_a14_10
6819 + t_a14_11
6820 + t_a14_12
6821 + t_a14_13
6822 + t_a14_14
6823 + t_a14_15
6824 );
6825 let t_a15 = calc_f32!(
6826 x0 + t_a15_1
6827 + t_a15_2
6828 + t_a15_3
6829 + t_a15_4
6830 + t_a15_5
6831 + t_a15_6
6832 + t_a15_7
6833 + t_a15_8
6834 + t_a15_9
6835 + t_a15_10
6836 + t_a15_11
6837 + t_a15_12
6838 + t_a15_13
6839 + t_a15_14
6840 + t_a15_15
6841 );
6842
6843 let t_b1 = calc_f32!(
6844 t_b1_1
6845 + t_b1_2
6846 + t_b1_3
6847 + t_b1_4
6848 + t_b1_5
6849 + t_b1_6
6850 + t_b1_7
6851 + t_b1_8
6852 + t_b1_9
6853 + t_b1_10
6854 + t_b1_11
6855 + t_b1_12
6856 + t_b1_13
6857 + t_b1_14
6858 + t_b1_15
6859 );
6860 let t_b2 = calc_f32!(
6861 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7
6862 - t_b2_8
6863 - t_b2_9
6864 - t_b2_10
6865 - t_b2_11
6866 - t_b2_12
6867 - t_b2_13
6868 - t_b2_14
6869 - t_b2_15
6870 );
6871 let t_b3 = calc_f32!(
6872 t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 + t_b3_5
6873 - t_b3_6
6874 - t_b3_7
6875 - t_b3_8
6876 - t_b3_9
6877 - t_b3_10
6878 + t_b3_11
6879 + t_b3_12
6880 + t_b3_13
6881 + t_b3_14
6882 + t_b3_15
6883 );
6884 let t_b4 = calc_f32!(
6885 t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7
6886 + t_b4_8
6887 + t_b4_9
6888 + t_b4_10
6889 + t_b4_11
6890 - t_b4_12
6891 - t_b4_13
6892 - t_b4_14
6893 - t_b4_15
6894 );
6895 let t_b5 = calc_f32!(
6896 t_b5_1 + t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8 + t_b5_9
6897 - t_b5_10
6898 - t_b5_11
6899 - t_b5_12
6900 + t_b5_13
6901 + t_b5_14
6902 + t_b5_15
6903 );
6904 let t_b6 = calc_f32!(
6905 t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7
6906 - t_b6_8
6907 - t_b6_9
6908 - t_b6_10
6909 + t_b6_11
6910 + t_b6_12
6911 - t_b6_13
6912 - t_b6_14
6913 - t_b6_15
6914 );
6915 let t_b7 = calc_f32!(
6916 t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8
6917 + t_b7_9
6918 + t_b7_10
6919 + t_b7_11
6920 - t_b7_12
6921 - t_b7_13
6922 + t_b7_14
6923 + t_b7_15
6924 );
6925 let t_b8 = calc_f32!(
6926 t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9
6927 - t_b8_10
6928 - t_b8_11
6929 + t_b8_12
6930 + t_b8_13
6931 - t_b8_14
6932 - t_b8_15
6933 );
6934 let t_b9 = calc_f32!(
6935 t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8
6936 - t_b9_9
6937 - t_b9_10
6938 + t_b9_11
6939 + t_b9_12
6940 - t_b9_13
6941 + t_b9_14
6942 + t_b9_15
6943 );
6944 let t_b10 = calc_f32!(
6945 t_b10_1 - t_b10_2 - t_b10_3 + t_b10_4 - t_b10_5 - t_b10_6 + t_b10_7 - t_b10_8 - t_b10_9
6946 + t_b10_10
6947 - t_b10_11
6948 - t_b10_12
6949 + t_b10_13
6950 - t_b10_14
6951 - t_b10_15
6952 );
6953 let t_b11 = calc_f32!(
6954 t_b11_1 - t_b11_2 + t_b11_3 + t_b11_4 - t_b11_5 + t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9
6955 - t_b11_10
6956 - t_b11_11
6957 + t_b11_12
6958 - t_b11_13
6959 - t_b11_14
6960 + t_b11_15
6961 );
6962 let t_b12 = calc_f32!(
6963 t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 - t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 + t_b12_9
6964 - t_b12_10
6965 + t_b12_11
6966 - t_b12_12
6967 + t_b12_13
6968 + t_b12_14
6969 - t_b12_15
6970 );
6971 let t_b13 = calc_f32!(
6972 t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 - t_b13_7 + t_b13_8 - t_b13_9
6973 + t_b13_10
6974 - t_b13_11
6975 + t_b13_12
6976 + t_b13_13
6977 - t_b13_14
6978 + t_b13_15
6979 );
6980 let t_b14 = calc_f32!(
6981 t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9
6982 - t_b14_10
6983 - t_b14_11
6984 + t_b14_12
6985 - t_b14_13
6986 + t_b14_14
6987 - t_b14_15
6988 );
6989 let t_b15 = calc_f32!(
6990 t_b15_1 - t_b15_2 + t_b15_3 - t_b15_4 + t_b15_5 - t_b15_6 + t_b15_7 - t_b15_8 + t_b15_9
6991 - t_b15_10
6992 + t_b15_11
6993 - t_b15_12
6994 + t_b15_13
6995 - t_b15_14
6996 + t_b15_15
6997 );
6998
6999 let t_b1_rot = self.rotate.rotate_both(t_b1);
7000 let t_b2_rot = self.rotate.rotate_both(t_b2);
7001 let t_b3_rot = self.rotate.rotate_both(t_b3);
7002 let t_b4_rot = self.rotate.rotate_both(t_b4);
7003 let t_b5_rot = self.rotate.rotate_both(t_b5);
7004 let t_b6_rot = self.rotate.rotate_both(t_b6);
7005 let t_b7_rot = self.rotate.rotate_both(t_b7);
7006 let t_b8_rot = self.rotate.rotate_both(t_b8);
7007 let t_b9_rot = self.rotate.rotate_both(t_b9);
7008 let t_b10_rot = self.rotate.rotate_both(t_b10);
7009 let t_b11_rot = self.rotate.rotate_both(t_b11);
7010 let t_b12_rot = self.rotate.rotate_both(t_b12);
7011 let t_b13_rot = self.rotate.rotate_both(t_b13);
7012 let t_b14_rot = self.rotate.rotate_both(t_b14);
7013 let t_b15_rot = self.rotate.rotate_both(t_b15);
7014
7015 let y0 = calc_f32!(
7016 x0 + x1p30
7017 + x2p29
7018 + x3p28
7019 + x4p27
7020 + x5p26
7021 + x6p25
7022 + x7p24
7023 + x8p23
7024 + x9p22
7025 + x10p21
7026 + x11p20
7027 + x12p19
7028 + x13p18
7029 + x14p17
7030 + x15p16
7031 );
7032 let [y1, y30] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
7033 let [y2, y29] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
7034 let [y3, y28] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
7035 let [y4, y27] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
7036 let [y5, y26] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
7037 let [y6, y25] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
7038 let [y7, y24] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
7039 let [y8, y23] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
7040 let [y9, y22] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
7041 let [y10, y21] = parallel_fft2_interleaved_f32(t_a10, t_b10_rot);
7042 let [y11, y20] = parallel_fft2_interleaved_f32(t_a11, t_b11_rot);
7043 let [y12, y19] = parallel_fft2_interleaved_f32(t_a12, t_b12_rot);
7044 let [y13, y18] = parallel_fft2_interleaved_f32(t_a13, t_b13_rot);
7045 let [y14, y17] = parallel_fft2_interleaved_f32(t_a14, t_b14_rot);
7046 let [y15, y16] = parallel_fft2_interleaved_f32(t_a15, t_b15_rot);
7047 [
7048 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
7049 y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30,
7050 ]
7051 }
7052 }
7053
7054 // _____ _ __ _ _ _ _ _
7055 // |___ // | / /_ | || | | |__ (_) |_
7056 // |_ \| | _____ | '_ \| || |_| '_ \| | __|
7057 // ___) | | |_____| | (_) |__ _| |_) | | |_
7058 // |____/|_| \___/ |_| |_.__/|_|\__|
7059 //
7060
7061 pub struct WasmSimdF64Butterfly31<T> {
7062 direction: FftDirection,
7063 _phantom: std::marker::PhantomData<T>,
7064 rotate: Rotate90F64,
7065 twiddle1re: v128,
7066 twiddle1im: v128,
7067 twiddle2re: v128,
7068 twiddle2im: v128,
7069 twiddle3re: v128,
7070 twiddle3im: v128,
7071 twiddle4re: v128,
7072 twiddle4im: v128,
7073 twiddle5re: v128,
7074 twiddle5im: v128,
7075 twiddle6re: v128,
7076 twiddle6im: v128,
7077 twiddle7re: v128,
7078 twiddle7im: v128,
7079 twiddle8re: v128,
7080 twiddle8im: v128,
7081 twiddle9re: v128,
7082 twiddle9im: v128,
7083 twiddle10re: v128,
7084 twiddle10im: v128,
7085 twiddle11re: v128,
7086 twiddle11im: v128,
7087 twiddle12re: v128,
7088 twiddle12im: v128,
7089 twiddle13re: v128,
7090 twiddle13im: v128,
7091 twiddle14re: v128,
7092 twiddle14im: v128,
7093 twiddle15re: v128,
7094 twiddle15im: v128,
7095 }
7096
7097 boilerplate_fft_wasm_simd_f64_butterfly!(
7098 WasmSimdF64Butterfly31,
7099 31,
7100 |this: &WasmSimdF64Butterfly31<_>| this.direction
7101 );
7102 boilerplate_fft_wasm_simd_common_butterfly!(
7103 WasmSimdF64Butterfly31,
7104 31,
7105 |this: &WasmSimdF64Butterfly31<_>| this.direction
7106 );
7107 impl<T: FftNum> WasmSimdF64Butterfly31<T> {
7108 #[inline(always)]
7109 pub fn new(direction: FftDirection) -> Self {
7110 assert_f64::<T>();
7111 let rotate = Rotate90F64::new(true);
7112 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 31, direction);
7113 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 31, direction);
7114 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 31, direction);
7115 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 31, direction);
7116 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 31, direction);
7117 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 31, direction);
7118 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 31, direction);
7119 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 31, direction);
7120 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 31, direction);
7121 let tw10: Complex<f64> = twiddles::compute_twiddle(10, 31, direction);
7122 let tw11: Complex<f64> = twiddles::compute_twiddle(11, 31, direction);
7123 let tw12: Complex<f64> = twiddles::compute_twiddle(12, 31, direction);
7124 let tw13: Complex<f64> = twiddles::compute_twiddle(13, 31, direction);
7125 let tw14: Complex<f64> = twiddles::compute_twiddle(14, 31, direction);
7126 let tw15: Complex<f64> = twiddles::compute_twiddle(15, 31, direction);
7127 let twiddle1re = f64x2_splat(tw1.re);
7128 let twiddle1im = f64x2_splat(tw1.im);
7129 let twiddle2re = f64x2_splat(tw2.re);
7130 let twiddle2im = f64x2_splat(tw2.im);
7131 let twiddle3re = f64x2_splat(tw3.re);
7132 let twiddle3im = f64x2_splat(tw3.im);
7133 let twiddle4re = f64x2_splat(tw4.re);
7134 let twiddle4im = f64x2_splat(tw4.im);
7135 let twiddle5re = f64x2_splat(tw5.re);
7136 let twiddle5im = f64x2_splat(tw5.im);
7137 let twiddle6re = f64x2_splat(tw6.re);
7138 let twiddle6im = f64x2_splat(tw6.im);
7139 let twiddle7re = f64x2_splat(tw7.re);
7140 let twiddle7im = f64x2_splat(tw7.im);
7141 let twiddle8re = f64x2_splat(tw8.re);
7142 let twiddle8im = f64x2_splat(tw8.im);
7143 let twiddle9re = f64x2_splat(tw9.re);
7144 let twiddle9im = f64x2_splat(tw9.im);
7145 let twiddle10re = f64x2_splat(tw10.re);
7146 let twiddle10im = f64x2_splat(tw10.im);
7147 let twiddle11re = f64x2_splat(tw11.re);
7148 let twiddle11im = f64x2_splat(tw11.im);
7149 let twiddle12re = f64x2_splat(tw12.re);
7150 let twiddle12im = f64x2_splat(tw12.im);
7151 let twiddle13re = f64x2_splat(tw13.re);
7152 let twiddle13im = f64x2_splat(tw13.im);
7153 let twiddle14re = f64x2_splat(tw14.re);
7154 let twiddle14im = f64x2_splat(tw14.im);
7155 let twiddle15re = f64x2_splat(tw15.re);
7156 let twiddle15im = f64x2_splat(tw15.im);
7157
7158 Self {
7159 direction,
7160 _phantom: std::marker::PhantomData,
7161 rotate,
7162 twiddle1re,
7163 twiddle1im,
7164 twiddle2re,
7165 twiddle2im,
7166 twiddle3re,
7167 twiddle3im,
7168 twiddle4re,
7169 twiddle4im,
7170 twiddle5re,
7171 twiddle5im,
7172 twiddle6re,
7173 twiddle6im,
7174 twiddle7re,
7175 twiddle7im,
7176 twiddle8re,
7177 twiddle8im,
7178 twiddle9re,
7179 twiddle9im,
7180 twiddle10re,
7181 twiddle10im,
7182 twiddle11re,
7183 twiddle11im,
7184 twiddle12re,
7185 twiddle12im,
7186 twiddle13re,
7187 twiddle13im,
7188 twiddle14re,
7189 twiddle14im,
7190 twiddle15re,
7191 twiddle15im,
7192 }
7193 }
7194
7195 #[inline(always)]
7196 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl WasmSimdArrayMut<f64>) {
7197 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
7198
7199 let out = self.perform_fft_direct(values);
7200
7201 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
7202 }
7203
7204 #[inline(always)]
7205 pub(crate) unsafe fn perform_fft_direct(&self, values: [v128; 31]) -> [v128; 31] {
7206 let [x1p30, x1m30] = solo_fft2_f64(values[1], values[30]);
7207 let [x2p29, x2m29] = solo_fft2_f64(values[2], values[29]);
7208 let [x3p28, x3m28] = solo_fft2_f64(values[3], values[28]);
7209 let [x4p27, x4m27] = solo_fft2_f64(values[4], values[27]);
7210 let [x5p26, x5m26] = solo_fft2_f64(values[5], values[26]);
7211 let [x6p25, x6m25] = solo_fft2_f64(values[6], values[25]);
7212 let [x7p24, x7m24] = solo_fft2_f64(values[7], values[24]);
7213 let [x8p23, x8m23] = solo_fft2_f64(values[8], values[23]);
7214 let [x9p22, x9m22] = solo_fft2_f64(values[9], values[22]);
7215 let [x10p21, x10m21] = solo_fft2_f64(values[10], values[21]);
7216 let [x11p20, x11m20] = solo_fft2_f64(values[11], values[20]);
7217 let [x12p19, x12m19] = solo_fft2_f64(values[12], values[19]);
7218 let [x13p18, x13m18] = solo_fft2_f64(values[13], values[18]);
7219 let [x14p17, x14m17] = solo_fft2_f64(values[14], values[17]);
7220 let [x15p16, x15m16] = solo_fft2_f64(values[15], values[16]);
7221
7222 let t_a1_1 = f64x2_mul(self.twiddle1re, x1p30);
7223 let t_a1_2 = f64x2_mul(self.twiddle2re, x2p29);
7224 let t_a1_3 = f64x2_mul(self.twiddle3re, x3p28);
7225 let t_a1_4 = f64x2_mul(self.twiddle4re, x4p27);
7226 let t_a1_5 = f64x2_mul(self.twiddle5re, x5p26);
7227 let t_a1_6 = f64x2_mul(self.twiddle6re, x6p25);
7228 let t_a1_7 = f64x2_mul(self.twiddle7re, x7p24);
7229 let t_a1_8 = f64x2_mul(self.twiddle8re, x8p23);
7230 let t_a1_9 = f64x2_mul(self.twiddle9re, x9p22);
7231 let t_a1_10 = f64x2_mul(self.twiddle10re, x10p21);
7232 let t_a1_11 = f64x2_mul(self.twiddle11re, x11p20);
7233 let t_a1_12 = f64x2_mul(self.twiddle12re, x12p19);
7234 let t_a1_13 = f64x2_mul(self.twiddle13re, x13p18);
7235 let t_a1_14 = f64x2_mul(self.twiddle14re, x14p17);
7236 let t_a1_15 = f64x2_mul(self.twiddle15re, x15p16);
7237 let t_a2_1 = f64x2_mul(self.twiddle2re, x1p30);
7238 let t_a2_2 = f64x2_mul(self.twiddle4re, x2p29);
7239 let t_a2_3 = f64x2_mul(self.twiddle6re, x3p28);
7240 let t_a2_4 = f64x2_mul(self.twiddle8re, x4p27);
7241 let t_a2_5 = f64x2_mul(self.twiddle10re, x5p26);
7242 let t_a2_6 = f64x2_mul(self.twiddle12re, x6p25);
7243 let t_a2_7 = f64x2_mul(self.twiddle14re, x7p24);
7244 let t_a2_8 = f64x2_mul(self.twiddle15re, x8p23);
7245 let t_a2_9 = f64x2_mul(self.twiddle13re, x9p22);
7246 let t_a2_10 = f64x2_mul(self.twiddle11re, x10p21);
7247 let t_a2_11 = f64x2_mul(self.twiddle9re, x11p20);
7248 let t_a2_12 = f64x2_mul(self.twiddle7re, x12p19);
7249 let t_a2_13 = f64x2_mul(self.twiddle5re, x13p18);
7250 let t_a2_14 = f64x2_mul(self.twiddle3re, x14p17);
7251 let t_a2_15 = f64x2_mul(self.twiddle1re, x15p16);
7252 let t_a3_1 = f64x2_mul(self.twiddle3re, x1p30);
7253 let t_a3_2 = f64x2_mul(self.twiddle6re, x2p29);
7254 let t_a3_3 = f64x2_mul(self.twiddle9re, x3p28);
7255 let t_a3_4 = f64x2_mul(self.twiddle12re, x4p27);
7256 let t_a3_5 = f64x2_mul(self.twiddle15re, x5p26);
7257 let t_a3_6 = f64x2_mul(self.twiddle13re, x6p25);
7258 let t_a3_7 = f64x2_mul(self.twiddle10re, x7p24);
7259 let t_a3_8 = f64x2_mul(self.twiddle7re, x8p23);
7260 let t_a3_9 = f64x2_mul(self.twiddle4re, x9p22);
7261 let t_a3_10 = f64x2_mul(self.twiddle1re, x10p21);
7262 let t_a3_11 = f64x2_mul(self.twiddle2re, x11p20);
7263 let t_a3_12 = f64x2_mul(self.twiddle5re, x12p19);
7264 let t_a3_13 = f64x2_mul(self.twiddle8re, x13p18);
7265 let t_a3_14 = f64x2_mul(self.twiddle11re, x14p17);
7266 let t_a3_15 = f64x2_mul(self.twiddle14re, x15p16);
7267 let t_a4_1 = f64x2_mul(self.twiddle4re, x1p30);
7268 let t_a4_2 = f64x2_mul(self.twiddle8re, x2p29);
7269 let t_a4_3 = f64x2_mul(self.twiddle12re, x3p28);
7270 let t_a4_4 = f64x2_mul(self.twiddle15re, x4p27);
7271 let t_a4_5 = f64x2_mul(self.twiddle11re, x5p26);
7272 let t_a4_6 = f64x2_mul(self.twiddle7re, x6p25);
7273 let t_a4_7 = f64x2_mul(self.twiddle3re, x7p24);
7274 let t_a4_8 = f64x2_mul(self.twiddle1re, x8p23);
7275 let t_a4_9 = f64x2_mul(self.twiddle5re, x9p22);
7276 let t_a4_10 = f64x2_mul(self.twiddle9re, x10p21);
7277 let t_a4_11 = f64x2_mul(self.twiddle13re, x11p20);
7278 let t_a4_12 = f64x2_mul(self.twiddle14re, x12p19);
7279 let t_a4_13 = f64x2_mul(self.twiddle10re, x13p18);
7280 let t_a4_14 = f64x2_mul(self.twiddle6re, x14p17);
7281 let t_a4_15 = f64x2_mul(self.twiddle2re, x15p16);
7282 let t_a5_1 = f64x2_mul(self.twiddle5re, x1p30);
7283 let t_a5_2 = f64x2_mul(self.twiddle10re, x2p29);
7284 let t_a5_3 = f64x2_mul(self.twiddle15re, x3p28);
7285 let t_a5_4 = f64x2_mul(self.twiddle11re, x4p27);
7286 let t_a5_5 = f64x2_mul(self.twiddle6re, x5p26);
7287 let t_a5_6 = f64x2_mul(self.twiddle1re, x6p25);
7288 let t_a5_7 = f64x2_mul(self.twiddle4re, x7p24);
7289 let t_a5_8 = f64x2_mul(self.twiddle9re, x8p23);
7290 let t_a5_9 = f64x2_mul(self.twiddle14re, x9p22);
7291 let t_a5_10 = f64x2_mul(self.twiddle12re, x10p21);
7292 let t_a5_11 = f64x2_mul(self.twiddle7re, x11p20);
7293 let t_a5_12 = f64x2_mul(self.twiddle2re, x12p19);
7294 let t_a5_13 = f64x2_mul(self.twiddle3re, x13p18);
7295 let t_a5_14 = f64x2_mul(self.twiddle8re, x14p17);
7296 let t_a5_15 = f64x2_mul(self.twiddle13re, x15p16);
7297 let t_a6_1 = f64x2_mul(self.twiddle6re, x1p30);
7298 let t_a6_2 = f64x2_mul(self.twiddle12re, x2p29);
7299 let t_a6_3 = f64x2_mul(self.twiddle13re, x3p28);
7300 let t_a6_4 = f64x2_mul(self.twiddle7re, x4p27);
7301 let t_a6_5 = f64x2_mul(self.twiddle1re, x5p26);
7302 let t_a6_6 = f64x2_mul(self.twiddle5re, x6p25);
7303 let t_a6_7 = f64x2_mul(self.twiddle11re, x7p24);
7304 let t_a6_8 = f64x2_mul(self.twiddle14re, x8p23);
7305 let t_a6_9 = f64x2_mul(self.twiddle8re, x9p22);
7306 let t_a6_10 = f64x2_mul(self.twiddle2re, x10p21);
7307 let t_a6_11 = f64x2_mul(self.twiddle4re, x11p20);
7308 let t_a6_12 = f64x2_mul(self.twiddle10re, x12p19);
7309 let t_a6_13 = f64x2_mul(self.twiddle15re, x13p18);
7310 let t_a6_14 = f64x2_mul(self.twiddle9re, x14p17);
7311 let t_a6_15 = f64x2_mul(self.twiddle3re, x15p16);
7312 let t_a7_1 = f64x2_mul(self.twiddle7re, x1p30);
7313 let t_a7_2 = f64x2_mul(self.twiddle14re, x2p29);
7314 let t_a7_3 = f64x2_mul(self.twiddle10re, x3p28);
7315 let t_a7_4 = f64x2_mul(self.twiddle3re, x4p27);
7316 let t_a7_5 = f64x2_mul(self.twiddle4re, x5p26);
7317 let t_a7_6 = f64x2_mul(self.twiddle11re, x6p25);
7318 let t_a7_7 = f64x2_mul(self.twiddle13re, x7p24);
7319 let t_a7_8 = f64x2_mul(self.twiddle6re, x8p23);
7320 let t_a7_9 = f64x2_mul(self.twiddle1re, x9p22);
7321 let t_a7_10 = f64x2_mul(self.twiddle8re, x10p21);
7322 let t_a7_11 = f64x2_mul(self.twiddle15re, x11p20);
7323 let t_a7_12 = f64x2_mul(self.twiddle9re, x12p19);
7324 let t_a7_13 = f64x2_mul(self.twiddle2re, x13p18);
7325 let t_a7_14 = f64x2_mul(self.twiddle5re, x14p17);
7326 let t_a7_15 = f64x2_mul(self.twiddle12re, x15p16);
7327 let t_a8_1 = f64x2_mul(self.twiddle8re, x1p30);
7328 let t_a8_2 = f64x2_mul(self.twiddle15re, x2p29);
7329 let t_a8_3 = f64x2_mul(self.twiddle7re, x3p28);
7330 let t_a8_4 = f64x2_mul(self.twiddle1re, x4p27);
7331 let t_a8_5 = f64x2_mul(self.twiddle9re, x5p26);
7332 let t_a8_6 = f64x2_mul(self.twiddle14re, x6p25);
7333 let t_a8_7 = f64x2_mul(self.twiddle6re, x7p24);
7334 let t_a8_8 = f64x2_mul(self.twiddle2re, x8p23);
7335 let t_a8_9 = f64x2_mul(self.twiddle10re, x9p22);
7336 let t_a8_10 = f64x2_mul(self.twiddle13re, x10p21);
7337 let t_a8_11 = f64x2_mul(self.twiddle5re, x11p20);
7338 let t_a8_12 = f64x2_mul(self.twiddle3re, x12p19);
7339 let t_a8_13 = f64x2_mul(self.twiddle11re, x13p18);
7340 let t_a8_14 = f64x2_mul(self.twiddle12re, x14p17);
7341 let t_a8_15 = f64x2_mul(self.twiddle4re, x15p16);
7342 let t_a9_1 = f64x2_mul(self.twiddle9re, x1p30);
7343 let t_a9_2 = f64x2_mul(self.twiddle13re, x2p29);
7344 let t_a9_3 = f64x2_mul(self.twiddle4re, x3p28);
7345 let t_a9_4 = f64x2_mul(self.twiddle5re, x4p27);
7346 let t_a9_5 = f64x2_mul(self.twiddle14re, x5p26);
7347 let t_a9_6 = f64x2_mul(self.twiddle8re, x6p25);
7348 let t_a9_7 = f64x2_mul(self.twiddle1re, x7p24);
7349 let t_a9_8 = f64x2_mul(self.twiddle10re, x8p23);
7350 let t_a9_9 = f64x2_mul(self.twiddle12re, x9p22);
7351 let t_a9_10 = f64x2_mul(self.twiddle3re, x10p21);
7352 let t_a9_11 = f64x2_mul(self.twiddle6re, x11p20);
7353 let t_a9_12 = f64x2_mul(self.twiddle15re, x12p19);
7354 let t_a9_13 = f64x2_mul(self.twiddle7re, x13p18);
7355 let t_a9_14 = f64x2_mul(self.twiddle2re, x14p17);
7356 let t_a9_15 = f64x2_mul(self.twiddle11re, x15p16);
7357 let t_a10_1 = f64x2_mul(self.twiddle10re, x1p30);
7358 let t_a10_2 = f64x2_mul(self.twiddle11re, x2p29);
7359 let t_a10_3 = f64x2_mul(self.twiddle1re, x3p28);
7360 let t_a10_4 = f64x2_mul(self.twiddle9re, x4p27);
7361 let t_a10_5 = f64x2_mul(self.twiddle12re, x5p26);
7362 let t_a10_6 = f64x2_mul(self.twiddle2re, x6p25);
7363 let t_a10_7 = f64x2_mul(self.twiddle8re, x7p24);
7364 let t_a10_8 = f64x2_mul(self.twiddle13re, x8p23);
7365 let t_a10_9 = f64x2_mul(self.twiddle3re, x9p22);
7366 let t_a10_10 = f64x2_mul(self.twiddle7re, x10p21);
7367 let t_a10_11 = f64x2_mul(self.twiddle14re, x11p20);
7368 let t_a10_12 = f64x2_mul(self.twiddle4re, x12p19);
7369 let t_a10_13 = f64x2_mul(self.twiddle6re, x13p18);
7370 let t_a10_14 = f64x2_mul(self.twiddle15re, x14p17);
7371 let t_a10_15 = f64x2_mul(self.twiddle5re, x15p16);
7372 let t_a11_1 = f64x2_mul(self.twiddle11re, x1p30);
7373 let t_a11_2 = f64x2_mul(self.twiddle9re, x2p29);
7374 let t_a11_3 = f64x2_mul(self.twiddle2re, x3p28);
7375 let t_a11_4 = f64x2_mul(self.twiddle13re, x4p27);
7376 let t_a11_5 = f64x2_mul(self.twiddle7re, x5p26);
7377 let t_a11_6 = f64x2_mul(self.twiddle4re, x6p25);
7378 let t_a11_7 = f64x2_mul(self.twiddle15re, x7p24);
7379 let t_a11_8 = f64x2_mul(self.twiddle5re, x8p23);
7380 let t_a11_9 = f64x2_mul(self.twiddle6re, x9p22);
7381 let t_a11_10 = f64x2_mul(self.twiddle14re, x10p21);
7382 let t_a11_11 = f64x2_mul(self.twiddle3re, x11p20);
7383 let t_a11_12 = f64x2_mul(self.twiddle8re, x12p19);
7384 let t_a11_13 = f64x2_mul(self.twiddle12re, x13p18);
7385 let t_a11_14 = f64x2_mul(self.twiddle1re, x14p17);
7386 let t_a11_15 = f64x2_mul(self.twiddle10re, x15p16);
7387 let t_a12_1 = f64x2_mul(self.twiddle12re, x1p30);
7388 let t_a12_2 = f64x2_mul(self.twiddle7re, x2p29);
7389 let t_a12_3 = f64x2_mul(self.twiddle5re, x3p28);
7390 let t_a12_4 = f64x2_mul(self.twiddle14re, x4p27);
7391 let t_a12_5 = f64x2_mul(self.twiddle2re, x5p26);
7392 let t_a12_6 = f64x2_mul(self.twiddle10re, x6p25);
7393 let t_a12_7 = f64x2_mul(self.twiddle9re, x7p24);
7394 let t_a12_8 = f64x2_mul(self.twiddle3re, x8p23);
7395 let t_a12_9 = f64x2_mul(self.twiddle15re, x9p22);
7396 let t_a12_10 = f64x2_mul(self.twiddle4re, x10p21);
7397 let t_a12_11 = f64x2_mul(self.twiddle8re, x11p20);
7398 let t_a12_12 = f64x2_mul(self.twiddle11re, x12p19);
7399 let t_a12_13 = f64x2_mul(self.twiddle1re, x13p18);
7400 let t_a12_14 = f64x2_mul(self.twiddle13re, x14p17);
7401 let t_a12_15 = f64x2_mul(self.twiddle6re, x15p16);
7402 let t_a13_1 = f64x2_mul(self.twiddle13re, x1p30);
7403 let t_a13_2 = f64x2_mul(self.twiddle5re, x2p29);
7404 let t_a13_3 = f64x2_mul(self.twiddle8re, x3p28);
7405 let t_a13_4 = f64x2_mul(self.twiddle10re, x4p27);
7406 let t_a13_5 = f64x2_mul(self.twiddle3re, x5p26);
7407 let t_a13_6 = f64x2_mul(self.twiddle15re, x6p25);
7408 let t_a13_7 = f64x2_mul(self.twiddle2re, x7p24);
7409 let t_a13_8 = f64x2_mul(self.twiddle11re, x8p23);
7410 let t_a13_9 = f64x2_mul(self.twiddle7re, x9p22);
7411 let t_a13_10 = f64x2_mul(self.twiddle6re, x10p21);
7412 let t_a13_11 = f64x2_mul(self.twiddle12re, x11p20);
7413 let t_a13_12 = f64x2_mul(self.twiddle1re, x12p19);
7414 let t_a13_13 = f64x2_mul(self.twiddle14re, x13p18);
7415 let t_a13_14 = f64x2_mul(self.twiddle4re, x14p17);
7416 let t_a13_15 = f64x2_mul(self.twiddle9re, x15p16);
7417 let t_a14_1 = f64x2_mul(self.twiddle14re, x1p30);
7418 let t_a14_2 = f64x2_mul(self.twiddle3re, x2p29);
7419 let t_a14_3 = f64x2_mul(self.twiddle11re, x3p28);
7420 let t_a14_4 = f64x2_mul(self.twiddle6re, x4p27);
7421 let t_a14_5 = f64x2_mul(self.twiddle8re, x5p26);
7422 let t_a14_6 = f64x2_mul(self.twiddle9re, x6p25);
7423 let t_a14_7 = f64x2_mul(self.twiddle5re, x7p24);
7424 let t_a14_8 = f64x2_mul(self.twiddle12re, x8p23);
7425 let t_a14_9 = f64x2_mul(self.twiddle2re, x9p22);
7426 let t_a14_10 = f64x2_mul(self.twiddle15re, x10p21);
7427 let t_a14_11 = f64x2_mul(self.twiddle1re, x11p20);
7428 let t_a14_12 = f64x2_mul(self.twiddle13re, x12p19);
7429 let t_a14_13 = f64x2_mul(self.twiddle4re, x13p18);
7430 let t_a14_14 = f64x2_mul(self.twiddle10re, x14p17);
7431 let t_a14_15 = f64x2_mul(self.twiddle7re, x15p16);
7432 let t_a15_1 = f64x2_mul(self.twiddle15re, x1p30);
7433 let t_a15_2 = f64x2_mul(self.twiddle1re, x2p29);
7434 let t_a15_3 = f64x2_mul(self.twiddle14re, x3p28);
7435 let t_a15_4 = f64x2_mul(self.twiddle2re, x4p27);
7436 let t_a15_5 = f64x2_mul(self.twiddle13re, x5p26);
7437 let t_a15_6 = f64x2_mul(self.twiddle3re, x6p25);
7438 let t_a15_7 = f64x2_mul(self.twiddle12re, x7p24);
7439 let t_a15_8 = f64x2_mul(self.twiddle4re, x8p23);
7440 let t_a15_9 = f64x2_mul(self.twiddle11re, x9p22);
7441 let t_a15_10 = f64x2_mul(self.twiddle5re, x10p21);
7442 let t_a15_11 = f64x2_mul(self.twiddle10re, x11p20);
7443 let t_a15_12 = f64x2_mul(self.twiddle6re, x12p19);
7444 let t_a15_13 = f64x2_mul(self.twiddle9re, x13p18);
7445 let t_a15_14 = f64x2_mul(self.twiddle7re, x14p17);
7446 let t_a15_15 = f64x2_mul(self.twiddle8re, x15p16);
7447
7448 let t_b1_1 = f64x2_mul(self.twiddle1im, x1m30);
7449 let t_b1_2 = f64x2_mul(self.twiddle2im, x2m29);
7450 let t_b1_3 = f64x2_mul(self.twiddle3im, x3m28);
7451 let t_b1_4 = f64x2_mul(self.twiddle4im, x4m27);
7452 let t_b1_5 = f64x2_mul(self.twiddle5im, x5m26);
7453 let t_b1_6 = f64x2_mul(self.twiddle6im, x6m25);
7454 let t_b1_7 = f64x2_mul(self.twiddle7im, x7m24);
7455 let t_b1_8 = f64x2_mul(self.twiddle8im, x8m23);
7456 let t_b1_9 = f64x2_mul(self.twiddle9im, x9m22);
7457 let t_b1_10 = f64x2_mul(self.twiddle10im, x10m21);
7458 let t_b1_11 = f64x2_mul(self.twiddle11im, x11m20);
7459 let t_b1_12 = f64x2_mul(self.twiddle12im, x12m19);
7460 let t_b1_13 = f64x2_mul(self.twiddle13im, x13m18);
7461 let t_b1_14 = f64x2_mul(self.twiddle14im, x14m17);
7462 let t_b1_15 = f64x2_mul(self.twiddle15im, x15m16);
7463 let t_b2_1 = f64x2_mul(self.twiddle2im, x1m30);
7464 let t_b2_2 = f64x2_mul(self.twiddle4im, x2m29);
7465 let t_b2_3 = f64x2_mul(self.twiddle6im, x3m28);
7466 let t_b2_4 = f64x2_mul(self.twiddle8im, x4m27);
7467 let t_b2_5 = f64x2_mul(self.twiddle10im, x5m26);
7468 let t_b2_6 = f64x2_mul(self.twiddle12im, x6m25);
7469 let t_b2_7 = f64x2_mul(self.twiddle14im, x7m24);
7470 let t_b2_8 = f64x2_mul(self.twiddle15im, x8m23);
7471 let t_b2_9 = f64x2_mul(self.twiddle13im, x9m22);
7472 let t_b2_10 = f64x2_mul(self.twiddle11im, x10m21);
7473 let t_b2_11 = f64x2_mul(self.twiddle9im, x11m20);
7474 let t_b2_12 = f64x2_mul(self.twiddle7im, x12m19);
7475 let t_b2_13 = f64x2_mul(self.twiddle5im, x13m18);
7476 let t_b2_14 = f64x2_mul(self.twiddle3im, x14m17);
7477 let t_b2_15 = f64x2_mul(self.twiddle1im, x15m16);
7478 let t_b3_1 = f64x2_mul(self.twiddle3im, x1m30);
7479 let t_b3_2 = f64x2_mul(self.twiddle6im, x2m29);
7480 let t_b3_3 = f64x2_mul(self.twiddle9im, x3m28);
7481 let t_b3_4 = f64x2_mul(self.twiddle12im, x4m27);
7482 let t_b3_5 = f64x2_mul(self.twiddle15im, x5m26);
7483 let t_b3_6 = f64x2_mul(self.twiddle13im, x6m25);
7484 let t_b3_7 = f64x2_mul(self.twiddle10im, x7m24);
7485 let t_b3_8 = f64x2_mul(self.twiddle7im, x8m23);
7486 let t_b3_9 = f64x2_mul(self.twiddle4im, x9m22);
7487 let t_b3_10 = f64x2_mul(self.twiddle1im, x10m21);
7488 let t_b3_11 = f64x2_mul(self.twiddle2im, x11m20);
7489 let t_b3_12 = f64x2_mul(self.twiddle5im, x12m19);
7490 let t_b3_13 = f64x2_mul(self.twiddle8im, x13m18);
7491 let t_b3_14 = f64x2_mul(self.twiddle11im, x14m17);
7492 let t_b3_15 = f64x2_mul(self.twiddle14im, x15m16);
7493 let t_b4_1 = f64x2_mul(self.twiddle4im, x1m30);
7494 let t_b4_2 = f64x2_mul(self.twiddle8im, x2m29);
7495 let t_b4_3 = f64x2_mul(self.twiddle12im, x3m28);
7496 let t_b4_4 = f64x2_mul(self.twiddle15im, x4m27);
7497 let t_b4_5 = f64x2_mul(self.twiddle11im, x5m26);
7498 let t_b4_6 = f64x2_mul(self.twiddle7im, x6m25);
7499 let t_b4_7 = f64x2_mul(self.twiddle3im, x7m24);
7500 let t_b4_8 = f64x2_mul(self.twiddle1im, x8m23);
7501 let t_b4_9 = f64x2_mul(self.twiddle5im, x9m22);
7502 let t_b4_10 = f64x2_mul(self.twiddle9im, x10m21);
7503 let t_b4_11 = f64x2_mul(self.twiddle13im, x11m20);
7504 let t_b4_12 = f64x2_mul(self.twiddle14im, x12m19);
7505 let t_b4_13 = f64x2_mul(self.twiddle10im, x13m18);
7506 let t_b4_14 = f64x2_mul(self.twiddle6im, x14m17);
7507 let t_b4_15 = f64x2_mul(self.twiddle2im, x15m16);
7508 let t_b5_1 = f64x2_mul(self.twiddle5im, x1m30);
7509 let t_b5_2 = f64x2_mul(self.twiddle10im, x2m29);
7510 let t_b5_3 = f64x2_mul(self.twiddle15im, x3m28);
7511 let t_b5_4 = f64x2_mul(self.twiddle11im, x4m27);
7512 let t_b5_5 = f64x2_mul(self.twiddle6im, x5m26);
7513 let t_b5_6 = f64x2_mul(self.twiddle1im, x6m25);
7514 let t_b5_7 = f64x2_mul(self.twiddle4im, x7m24);
7515 let t_b5_8 = f64x2_mul(self.twiddle9im, x8m23);
7516 let t_b5_9 = f64x2_mul(self.twiddle14im, x9m22);
7517 let t_b5_10 = f64x2_mul(self.twiddle12im, x10m21);
7518 let t_b5_11 = f64x2_mul(self.twiddle7im, x11m20);
7519 let t_b5_12 = f64x2_mul(self.twiddle2im, x12m19);
7520 let t_b5_13 = f64x2_mul(self.twiddle3im, x13m18);
7521 let t_b5_14 = f64x2_mul(self.twiddle8im, x14m17);
7522 let t_b5_15 = f64x2_mul(self.twiddle13im, x15m16);
7523 let t_b6_1 = f64x2_mul(self.twiddle6im, x1m30);
7524 let t_b6_2 = f64x2_mul(self.twiddle12im, x2m29);
7525 let t_b6_3 = f64x2_mul(self.twiddle13im, x3m28);
7526 let t_b6_4 = f64x2_mul(self.twiddle7im, x4m27);
7527 let t_b6_5 = f64x2_mul(self.twiddle1im, x5m26);
7528 let t_b6_6 = f64x2_mul(self.twiddle5im, x6m25);
7529 let t_b6_7 = f64x2_mul(self.twiddle11im, x7m24);
7530 let t_b6_8 = f64x2_mul(self.twiddle14im, x8m23);
7531 let t_b6_9 = f64x2_mul(self.twiddle8im, x9m22);
7532 let t_b6_10 = f64x2_mul(self.twiddle2im, x10m21);
7533 let t_b6_11 = f64x2_mul(self.twiddle4im, x11m20);
7534 let t_b6_12 = f64x2_mul(self.twiddle10im, x12m19);
7535 let t_b6_13 = f64x2_mul(self.twiddle15im, x13m18);
7536 let t_b6_14 = f64x2_mul(self.twiddle9im, x14m17);
7537 let t_b6_15 = f64x2_mul(self.twiddle3im, x15m16);
7538 let t_b7_1 = f64x2_mul(self.twiddle7im, x1m30);
7539 let t_b7_2 = f64x2_mul(self.twiddle14im, x2m29);
7540 let t_b7_3 = f64x2_mul(self.twiddle10im, x3m28);
7541 let t_b7_4 = f64x2_mul(self.twiddle3im, x4m27);
7542 let t_b7_5 = f64x2_mul(self.twiddle4im, x5m26);
7543 let t_b7_6 = f64x2_mul(self.twiddle11im, x6m25);
7544 let t_b7_7 = f64x2_mul(self.twiddle13im, x7m24);
7545 let t_b7_8 = f64x2_mul(self.twiddle6im, x8m23);
7546 let t_b7_9 = f64x2_mul(self.twiddle1im, x9m22);
7547 let t_b7_10 = f64x2_mul(self.twiddle8im, x10m21);
7548 let t_b7_11 = f64x2_mul(self.twiddle15im, x11m20);
7549 let t_b7_12 = f64x2_mul(self.twiddle9im, x12m19);
7550 let t_b7_13 = f64x2_mul(self.twiddle2im, x13m18);
7551 let t_b7_14 = f64x2_mul(self.twiddle5im, x14m17);
7552 let t_b7_15 = f64x2_mul(self.twiddle12im, x15m16);
7553 let t_b8_1 = f64x2_mul(self.twiddle8im, x1m30);
7554 let t_b8_2 = f64x2_mul(self.twiddle15im, x2m29);
7555 let t_b8_3 = f64x2_mul(self.twiddle7im, x3m28);
7556 let t_b8_4 = f64x2_mul(self.twiddle1im, x4m27);
7557 let t_b8_5 = f64x2_mul(self.twiddle9im, x5m26);
7558 let t_b8_6 = f64x2_mul(self.twiddle14im, x6m25);
7559 let t_b8_7 = f64x2_mul(self.twiddle6im, x7m24);
7560 let t_b8_8 = f64x2_mul(self.twiddle2im, x8m23);
7561 let t_b8_9 = f64x2_mul(self.twiddle10im, x9m22);
7562 let t_b8_10 = f64x2_mul(self.twiddle13im, x10m21);
7563 let t_b8_11 = f64x2_mul(self.twiddle5im, x11m20);
7564 let t_b8_12 = f64x2_mul(self.twiddle3im, x12m19);
7565 let t_b8_13 = f64x2_mul(self.twiddle11im, x13m18);
7566 let t_b8_14 = f64x2_mul(self.twiddle12im, x14m17);
7567 let t_b8_15 = f64x2_mul(self.twiddle4im, x15m16);
7568 let t_b9_1 = f64x2_mul(self.twiddle9im, x1m30);
7569 let t_b9_2 = f64x2_mul(self.twiddle13im, x2m29);
7570 let t_b9_3 = f64x2_mul(self.twiddle4im, x3m28);
7571 let t_b9_4 = f64x2_mul(self.twiddle5im, x4m27);
7572 let t_b9_5 = f64x2_mul(self.twiddle14im, x5m26);
7573 let t_b9_6 = f64x2_mul(self.twiddle8im, x6m25);
7574 let t_b9_7 = f64x2_mul(self.twiddle1im, x7m24);
7575 let t_b9_8 = f64x2_mul(self.twiddle10im, x8m23);
7576 let t_b9_9 = f64x2_mul(self.twiddle12im, x9m22);
7577 let t_b9_10 = f64x2_mul(self.twiddle3im, x10m21);
7578 let t_b9_11 = f64x2_mul(self.twiddle6im, x11m20);
7579 let t_b9_12 = f64x2_mul(self.twiddle15im, x12m19);
7580 let t_b9_13 = f64x2_mul(self.twiddle7im, x13m18);
7581 let t_b9_14 = f64x2_mul(self.twiddle2im, x14m17);
7582 let t_b9_15 = f64x2_mul(self.twiddle11im, x15m16);
7583 let t_b10_1 = f64x2_mul(self.twiddle10im, x1m30);
7584 let t_b10_2 = f64x2_mul(self.twiddle11im, x2m29);
7585 let t_b10_3 = f64x2_mul(self.twiddle1im, x3m28);
7586 let t_b10_4 = f64x2_mul(self.twiddle9im, x4m27);
7587 let t_b10_5 = f64x2_mul(self.twiddle12im, x5m26);
7588 let t_b10_6 = f64x2_mul(self.twiddle2im, x6m25);
7589 let t_b10_7 = f64x2_mul(self.twiddle8im, x7m24);
7590 let t_b10_8 = f64x2_mul(self.twiddle13im, x8m23);
7591 let t_b10_9 = f64x2_mul(self.twiddle3im, x9m22);
7592 let t_b10_10 = f64x2_mul(self.twiddle7im, x10m21);
7593 let t_b10_11 = f64x2_mul(self.twiddle14im, x11m20);
7594 let t_b10_12 = f64x2_mul(self.twiddle4im, x12m19);
7595 let t_b10_13 = f64x2_mul(self.twiddle6im, x13m18);
7596 let t_b10_14 = f64x2_mul(self.twiddle15im, x14m17);
7597 let t_b10_15 = f64x2_mul(self.twiddle5im, x15m16);
7598 let t_b11_1 = f64x2_mul(self.twiddle11im, x1m30);
7599 let t_b11_2 = f64x2_mul(self.twiddle9im, x2m29);
7600 let t_b11_3 = f64x2_mul(self.twiddle2im, x3m28);
7601 let t_b11_4 = f64x2_mul(self.twiddle13im, x4m27);
7602 let t_b11_5 = f64x2_mul(self.twiddle7im, x5m26);
7603 let t_b11_6 = f64x2_mul(self.twiddle4im, x6m25);
7604 let t_b11_7 = f64x2_mul(self.twiddle15im, x7m24);
7605 let t_b11_8 = f64x2_mul(self.twiddle5im, x8m23);
7606 let t_b11_9 = f64x2_mul(self.twiddle6im, x9m22);
7607 let t_b11_10 = f64x2_mul(self.twiddle14im, x10m21);
7608 let t_b11_11 = f64x2_mul(self.twiddle3im, x11m20);
7609 let t_b11_12 = f64x2_mul(self.twiddle8im, x12m19);
7610 let t_b11_13 = f64x2_mul(self.twiddle12im, x13m18);
7611 let t_b11_14 = f64x2_mul(self.twiddle1im, x14m17);
7612 let t_b11_15 = f64x2_mul(self.twiddle10im, x15m16);
7613 let t_b12_1 = f64x2_mul(self.twiddle12im, x1m30);
7614 let t_b12_2 = f64x2_mul(self.twiddle7im, x2m29);
7615 let t_b12_3 = f64x2_mul(self.twiddle5im, x3m28);
7616 let t_b12_4 = f64x2_mul(self.twiddle14im, x4m27);
7617 let t_b12_5 = f64x2_mul(self.twiddle2im, x5m26);
7618 let t_b12_6 = f64x2_mul(self.twiddle10im, x6m25);
7619 let t_b12_7 = f64x2_mul(self.twiddle9im, x7m24);
7620 let t_b12_8 = f64x2_mul(self.twiddle3im, x8m23);
7621 let t_b12_9 = f64x2_mul(self.twiddle15im, x9m22);
7622 let t_b12_10 = f64x2_mul(self.twiddle4im, x10m21);
7623 let t_b12_11 = f64x2_mul(self.twiddle8im, x11m20);
7624 let t_b12_12 = f64x2_mul(self.twiddle11im, x12m19);
7625 let t_b12_13 = f64x2_mul(self.twiddle1im, x13m18);
7626 let t_b12_14 = f64x2_mul(self.twiddle13im, x14m17);
7627 let t_b12_15 = f64x2_mul(self.twiddle6im, x15m16);
7628 let t_b13_1 = f64x2_mul(self.twiddle13im, x1m30);
7629 let t_b13_2 = f64x2_mul(self.twiddle5im, x2m29);
7630 let t_b13_3 = f64x2_mul(self.twiddle8im, x3m28);
7631 let t_b13_4 = f64x2_mul(self.twiddle10im, x4m27);
7632 let t_b13_5 = f64x2_mul(self.twiddle3im, x5m26);
7633 let t_b13_6 = f64x2_mul(self.twiddle15im, x6m25);
7634 let t_b13_7 = f64x2_mul(self.twiddle2im, x7m24);
7635 let t_b13_8 = f64x2_mul(self.twiddle11im, x8m23);
7636 let t_b13_9 = f64x2_mul(self.twiddle7im, x9m22);
7637 let t_b13_10 = f64x2_mul(self.twiddle6im, x10m21);
7638 let t_b13_11 = f64x2_mul(self.twiddle12im, x11m20);
7639 let t_b13_12 = f64x2_mul(self.twiddle1im, x12m19);
7640 let t_b13_13 = f64x2_mul(self.twiddle14im, x13m18);
7641 let t_b13_14 = f64x2_mul(self.twiddle4im, x14m17);
7642 let t_b13_15 = f64x2_mul(self.twiddle9im, x15m16);
7643 let t_b14_1 = f64x2_mul(self.twiddle14im, x1m30);
7644 let t_b14_2 = f64x2_mul(self.twiddle3im, x2m29);
7645 let t_b14_3 = f64x2_mul(self.twiddle11im, x3m28);
7646 let t_b14_4 = f64x2_mul(self.twiddle6im, x4m27);
7647 let t_b14_5 = f64x2_mul(self.twiddle8im, x5m26);
7648 let t_b14_6 = f64x2_mul(self.twiddle9im, x6m25);
7649 let t_b14_7 = f64x2_mul(self.twiddle5im, x7m24);
7650 let t_b14_8 = f64x2_mul(self.twiddle12im, x8m23);
7651 let t_b14_9 = f64x2_mul(self.twiddle2im, x9m22);
7652 let t_b14_10 = f64x2_mul(self.twiddle15im, x10m21);
7653 let t_b14_11 = f64x2_mul(self.twiddle1im, x11m20);
7654 let t_b14_12 = f64x2_mul(self.twiddle13im, x12m19);
7655 let t_b14_13 = f64x2_mul(self.twiddle4im, x13m18);
7656 let t_b14_14 = f64x2_mul(self.twiddle10im, x14m17);
7657 let t_b14_15 = f64x2_mul(self.twiddle7im, x15m16);
7658 let t_b15_1 = f64x2_mul(self.twiddle15im, x1m30);
7659 let t_b15_2 = f64x2_mul(self.twiddle1im, x2m29);
7660 let t_b15_3 = f64x2_mul(self.twiddle14im, x3m28);
7661 let t_b15_4 = f64x2_mul(self.twiddle2im, x4m27);
7662 let t_b15_5 = f64x2_mul(self.twiddle13im, x5m26);
7663 let t_b15_6 = f64x2_mul(self.twiddle3im, x6m25);
7664 let t_b15_7 = f64x2_mul(self.twiddle12im, x7m24);
7665 let t_b15_8 = f64x2_mul(self.twiddle4im, x8m23);
7666 let t_b15_9 = f64x2_mul(self.twiddle11im, x9m22);
7667 let t_b15_10 = f64x2_mul(self.twiddle5im, x10m21);
7668 let t_b15_11 = f64x2_mul(self.twiddle10im, x11m20);
7669 let t_b15_12 = f64x2_mul(self.twiddle6im, x12m19);
7670 let t_b15_13 = f64x2_mul(self.twiddle9im, x13m18);
7671 let t_b15_14 = f64x2_mul(self.twiddle7im, x14m17);
7672 let t_b15_15 = f64x2_mul(self.twiddle8im, x15m16);
7673
7674 let x0 = values[0];
7675 let t_a1 = calc_f64!(
7676 x0 + t_a1_1
7677 + t_a1_2
7678 + t_a1_3
7679 + t_a1_4
7680 + t_a1_5
7681 + t_a1_6
7682 + t_a1_7
7683 + t_a1_8
7684 + t_a1_9
7685 + t_a1_10
7686 + t_a1_11
7687 + t_a1_12
7688 + t_a1_13
7689 + t_a1_14
7690 + t_a1_15
7691 );
7692 let t_a2 = calc_f64!(
7693 x0 + t_a2_1
7694 + t_a2_2
7695 + t_a2_3
7696 + t_a2_4
7697 + t_a2_5
7698 + t_a2_6
7699 + t_a2_7
7700 + t_a2_8
7701 + t_a2_9
7702 + t_a2_10
7703 + t_a2_11
7704 + t_a2_12
7705 + t_a2_13
7706 + t_a2_14
7707 + t_a2_15
7708 );
7709 let t_a3 = calc_f64!(
7710 x0 + t_a3_1
7711 + t_a3_2
7712 + t_a3_3
7713 + t_a3_4
7714 + t_a3_5
7715 + t_a3_6
7716 + t_a3_7
7717 + t_a3_8
7718 + t_a3_9
7719 + t_a3_10
7720 + t_a3_11
7721 + t_a3_12
7722 + t_a3_13
7723 + t_a3_14
7724 + t_a3_15
7725 );
7726 let t_a4 = calc_f64!(
7727 x0 + t_a4_1
7728 + t_a4_2
7729 + t_a4_3
7730 + t_a4_4
7731 + t_a4_5
7732 + t_a4_6
7733 + t_a4_7
7734 + t_a4_8
7735 + t_a4_9
7736 + t_a4_10
7737 + t_a4_11
7738 + t_a4_12
7739 + t_a4_13
7740 + t_a4_14
7741 + t_a4_15
7742 );
7743 let t_a5 = calc_f64!(
7744 x0 + t_a5_1
7745 + t_a5_2
7746 + t_a5_3
7747 + t_a5_4
7748 + t_a5_5
7749 + t_a5_6
7750 + t_a5_7
7751 + t_a5_8
7752 + t_a5_9
7753 + t_a5_10
7754 + t_a5_11
7755 + t_a5_12
7756 + t_a5_13
7757 + t_a5_14
7758 + t_a5_15
7759 );
7760 let t_a6 = calc_f64!(
7761 x0 + t_a6_1
7762 + t_a6_2
7763 + t_a6_3
7764 + t_a6_4
7765 + t_a6_5
7766 + t_a6_6
7767 + t_a6_7
7768 + t_a6_8
7769 + t_a6_9
7770 + t_a6_10
7771 + t_a6_11
7772 + t_a6_12
7773 + t_a6_13
7774 + t_a6_14
7775 + t_a6_15
7776 );
7777 let t_a7 = calc_f64!(
7778 x0 + t_a7_1
7779 + t_a7_2
7780 + t_a7_3
7781 + t_a7_4
7782 + t_a7_5
7783 + t_a7_6
7784 + t_a7_7
7785 + t_a7_8
7786 + t_a7_9
7787 + t_a7_10
7788 + t_a7_11
7789 + t_a7_12
7790 + t_a7_13
7791 + t_a7_14
7792 + t_a7_15
7793 );
7794 let t_a8 = calc_f64!(
7795 x0 + t_a8_1
7796 + t_a8_2
7797 + t_a8_3
7798 + t_a8_4
7799 + t_a8_5
7800 + t_a8_6
7801 + t_a8_7
7802 + t_a8_8
7803 + t_a8_9
7804 + t_a8_10
7805 + t_a8_11
7806 + t_a8_12
7807 + t_a8_13
7808 + t_a8_14
7809 + t_a8_15
7810 );
7811 let t_a9 = calc_f64!(
7812 x0 + t_a9_1
7813 + t_a9_2
7814 + t_a9_3
7815 + t_a9_4
7816 + t_a9_5
7817 + t_a9_6
7818 + t_a9_7
7819 + t_a9_8
7820 + t_a9_9
7821 + t_a9_10
7822 + t_a9_11
7823 + t_a9_12
7824 + t_a9_13
7825 + t_a9_14
7826 + t_a9_15
7827 );
7828 let t_a10 = calc_f64!(
7829 x0 + t_a10_1
7830 + t_a10_2
7831 + t_a10_3
7832 + t_a10_4
7833 + t_a10_5
7834 + t_a10_6
7835 + t_a10_7
7836 + t_a10_8
7837 + t_a10_9
7838 + t_a10_10
7839 + t_a10_11
7840 + t_a10_12
7841 + t_a10_13
7842 + t_a10_14
7843 + t_a10_15
7844 );
7845 let t_a11 = calc_f64!(
7846 x0 + t_a11_1
7847 + t_a11_2
7848 + t_a11_3
7849 + t_a11_4
7850 + t_a11_5
7851 + t_a11_6
7852 + t_a11_7
7853 + t_a11_8
7854 + t_a11_9
7855 + t_a11_10
7856 + t_a11_11
7857 + t_a11_12
7858 + t_a11_13
7859 + t_a11_14
7860 + t_a11_15
7861 );
7862 let t_a12 = calc_f64!(
7863 x0 + t_a12_1
7864 + t_a12_2
7865 + t_a12_3
7866 + t_a12_4
7867 + t_a12_5
7868 + t_a12_6
7869 + t_a12_7
7870 + t_a12_8
7871 + t_a12_9
7872 + t_a12_10
7873 + t_a12_11
7874 + t_a12_12
7875 + t_a12_13
7876 + t_a12_14
7877 + t_a12_15
7878 );
7879 let t_a13 = calc_f64!(
7880 x0 + t_a13_1
7881 + t_a13_2
7882 + t_a13_3
7883 + t_a13_4
7884 + t_a13_5
7885 + t_a13_6
7886 + t_a13_7
7887 + t_a13_8
7888 + t_a13_9
7889 + t_a13_10
7890 + t_a13_11
7891 + t_a13_12
7892 + t_a13_13
7893 + t_a13_14
7894 + t_a13_15
7895 );
7896 let t_a14 = calc_f64!(
7897 x0 + t_a14_1
7898 + t_a14_2
7899 + t_a14_3
7900 + t_a14_4
7901 + t_a14_5
7902 + t_a14_6
7903 + t_a14_7
7904 + t_a14_8
7905 + t_a14_9
7906 + t_a14_10
7907 + t_a14_11
7908 + t_a14_12
7909 + t_a14_13
7910 + t_a14_14
7911 + t_a14_15
7912 );
7913 let t_a15 = calc_f64!(
7914 x0 + t_a15_1
7915 + t_a15_2
7916 + t_a15_3
7917 + t_a15_4
7918 + t_a15_5
7919 + t_a15_6
7920 + t_a15_7
7921 + t_a15_8
7922 + t_a15_9
7923 + t_a15_10
7924 + t_a15_11
7925 + t_a15_12
7926 + t_a15_13
7927 + t_a15_14
7928 + t_a15_15
7929 );
7930
7931 let t_b1 = calc_f64!(
7932 t_b1_1
7933 + t_b1_2
7934 + t_b1_3
7935 + t_b1_4
7936 + t_b1_5
7937 + t_b1_6
7938 + t_b1_7
7939 + t_b1_8
7940 + t_b1_9
7941 + t_b1_10
7942 + t_b1_11
7943 + t_b1_12
7944 + t_b1_13
7945 + t_b1_14
7946 + t_b1_15
7947 );
7948 let t_b2 = calc_f64!(
7949 t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7
7950 - t_b2_8
7951 - t_b2_9
7952 - t_b2_10
7953 - t_b2_11
7954 - t_b2_12
7955 - t_b2_13
7956 - t_b2_14
7957 - t_b2_15
7958 );
7959 let t_b3 = calc_f64!(
7960 t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 + t_b3_5
7961 - t_b3_6
7962 - t_b3_7
7963 - t_b3_8
7964 - t_b3_9
7965 - t_b3_10
7966 + t_b3_11
7967 + t_b3_12
7968 + t_b3_13
7969 + t_b3_14
7970 + t_b3_15
7971 );
7972 let t_b4 = calc_f64!(
7973 t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7
7974 + t_b4_8
7975 + t_b4_9
7976 + t_b4_10
7977 + t_b4_11
7978 - t_b4_12
7979 - t_b4_13
7980 - t_b4_14
7981 - t_b4_15
7982 );
7983 let t_b5 = calc_f64!(
7984 t_b5_1 + t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8 + t_b5_9
7985 - t_b5_10
7986 - t_b5_11
7987 - t_b5_12
7988 + t_b5_13
7989 + t_b5_14
7990 + t_b5_15
7991 );
7992 let t_b6 = calc_f64!(
7993 t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7
7994 - t_b6_8
7995 - t_b6_9
7996 - t_b6_10
7997 + t_b6_11
7998 + t_b6_12
7999 - t_b6_13
8000 - t_b6_14
8001 - t_b6_15
8002 );
8003 let t_b7 = calc_f64!(
8004 t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8
8005 + t_b7_9
8006 + t_b7_10
8007 + t_b7_11
8008 - t_b7_12
8009 - t_b7_13
8010 + t_b7_14
8011 + t_b7_15
8012 );
8013 let t_b8 = calc_f64!(
8014 t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9
8015 - t_b8_10
8016 - t_b8_11
8017 + t_b8_12
8018 + t_b8_13
8019 - t_b8_14
8020 - t_b8_15
8021 );
8022 let t_b9 = calc_f64!(
8023 t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8
8024 - t_b9_9
8025 - t_b9_10
8026 + t_b9_11
8027 + t_b9_12
8028 - t_b9_13
8029 + t_b9_14
8030 + t_b9_15
8031 );
8032 let t_b10 = calc_f64!(
8033 t_b10_1 - t_b10_2 - t_b10_3 + t_b10_4 - t_b10_5 - t_b10_6 + t_b10_7 - t_b10_8 - t_b10_9
8034 + t_b10_10
8035 - t_b10_11
8036 - t_b10_12
8037 + t_b10_13
8038 - t_b10_14
8039 - t_b10_15
8040 );
8041 let t_b11 = calc_f64!(
8042 t_b11_1 - t_b11_2 + t_b11_3 + t_b11_4 - t_b11_5 + t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9
8043 - t_b11_10
8044 - t_b11_11
8045 + t_b11_12
8046 - t_b11_13
8047 - t_b11_14
8048 + t_b11_15
8049 );
8050 let t_b12 = calc_f64!(
8051 t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 - t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 + t_b12_9
8052 - t_b12_10
8053 + t_b12_11
8054 - t_b12_12
8055 + t_b12_13
8056 + t_b12_14
8057 - t_b12_15
8058 );
8059 let t_b13 = calc_f64!(
8060 t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 - t_b13_7 + t_b13_8 - t_b13_9
8061 + t_b13_10
8062 - t_b13_11
8063 + t_b13_12
8064 + t_b13_13
8065 - t_b13_14
8066 + t_b13_15
8067 );
8068 let t_b14 = calc_f64!(
8069 t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9
8070 - t_b14_10
8071 - t_b14_11
8072 + t_b14_12
8073 - t_b14_13
8074 + t_b14_14
8075 - t_b14_15
8076 );
8077 let t_b15 = calc_f64!(
8078 t_b15_1 - t_b15_2 + t_b15_3 - t_b15_4 + t_b15_5 - t_b15_6 + t_b15_7 - t_b15_8 + t_b15_9
8079 - t_b15_10
8080 + t_b15_11
8081 - t_b15_12
8082 + t_b15_13
8083 - t_b15_14
8084 + t_b15_15
8085 );
8086
8087 let t_b1_rot = self.rotate.rotate(t_b1);
8088 let t_b2_rot = self.rotate.rotate(t_b2);
8089 let t_b3_rot = self.rotate.rotate(t_b3);
8090 let t_b4_rot = self.rotate.rotate(t_b4);
8091 let t_b5_rot = self.rotate.rotate(t_b5);
8092 let t_b6_rot = self.rotate.rotate(t_b6);
8093 let t_b7_rot = self.rotate.rotate(t_b7);
8094 let t_b8_rot = self.rotate.rotate(t_b8);
8095 let t_b9_rot = self.rotate.rotate(t_b9);
8096 let t_b10_rot = self.rotate.rotate(t_b10);
8097 let t_b11_rot = self.rotate.rotate(t_b11);
8098 let t_b12_rot = self.rotate.rotate(t_b12);
8099 let t_b13_rot = self.rotate.rotate(t_b13);
8100 let t_b14_rot = self.rotate.rotate(t_b14);
8101 let t_b15_rot = self.rotate.rotate(t_b15);
8102
8103 let y0 = calc_f64!(
8104 x0 + x1p30
8105 + x2p29
8106 + x3p28
8107 + x4p27
8108 + x5p26
8109 + x6p25
8110 + x7p24
8111 + x8p23
8112 + x9p22
8113 + x10p21
8114 + x11p20
8115 + x12p19
8116 + x13p18
8117 + x14p17
8118 + x15p16
8119 );
8120 let [y1, y30] = solo_fft2_f64(t_a1, t_b1_rot);
8121 let [y2, y29] = solo_fft2_f64(t_a2, t_b2_rot);
8122 let [y3, y28] = solo_fft2_f64(t_a3, t_b3_rot);
8123 let [y4, y27] = solo_fft2_f64(t_a4, t_b4_rot);
8124 let [y5, y26] = solo_fft2_f64(t_a5, t_b5_rot);
8125 let [y6, y25] = solo_fft2_f64(t_a6, t_b6_rot);
8126 let [y7, y24] = solo_fft2_f64(t_a7, t_b7_rot);
8127 let [y8, y23] = solo_fft2_f64(t_a8, t_b8_rot);
8128 let [y9, y22] = solo_fft2_f64(t_a9, t_b9_rot);
8129 let [y10, y21] = solo_fft2_f64(t_a10, t_b10_rot);
8130 let [y11, y20] = solo_fft2_f64(t_a11, t_b11_rot);
8131 let [y12, y19] = solo_fft2_f64(t_a12, t_b12_rot);
8132 let [y13, y18] = solo_fft2_f64(t_a13, t_b13_rot);
8133 let [y14, y17] = solo_fft2_f64(t_a14, t_b14_rot);
8134 let [y15, y16] = solo_fft2_f64(t_a15, t_b15_rot);
8135 [
8136 y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18,
8137 y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30,
8138 ]
8139 }
8140 }
8141
8142 // _____ _____ ____ _____ ____
8143 // |_ _| ____/ ___|_ _/ ___|
8144 // | | | _| \___ \ | | \___ \
8145 // | | | |___ ___) || | ___) |
8146 // |_| |_____|____/ |_| |____/
8147 //
8148
8149 #[cfg(test)]
8150 mod unit_tests {
8151 use super::*;
8152 use crate::test_utils::check_fft_algorithm;
8153 use wasm_bindgen_test::wasm_bindgen_test;
8154
8155 //the tests for all butterflies will be identical except for the identifiers used and size
8156 //so it's ideal for a macro
8157 macro_rules! test_butterfly_32_func {
8158 ($test_name:ident, $struct_name:ident, $size:expr) => {
8159 #[wasm_bindgen_test]
8160 fn $test_name() {
8161 let butterfly = $struct_name::new(FftDirection::Forward);
8162 check_fft_algorithm::<f32>(&butterfly, $size, FftDirection::Forward);
8163
8164 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
8165 check_fft_algorithm::<f32>(&butterfly_direction, $size, FftDirection::Inverse);
8166 }
8167 };
8168 }
8169 test_butterfly_32_func!(test_wasm_simdf32_butterfly7, WasmSimdF32Butterfly7, 7);
8170 test_butterfly_32_func!(test_wasm_simdf32_butterfly11, WasmSimdF32Butterfly11, 11);
8171 test_butterfly_32_func!(test_wasm_simdf32_butterfly13, WasmSimdF32Butterfly13, 13);
8172 test_butterfly_32_func!(test_wasm_simdf32_butterfly17, WasmSimdF32Butterfly17, 17);
8173 test_butterfly_32_func!(test_wasm_simdf32_butterfly19, WasmSimdF32Butterfly19, 19);
8174 test_butterfly_32_func!(test_wasm_simdf32_butterfly23, WasmSimdF32Butterfly23, 23);
8175 test_butterfly_32_func!(test_wasm_simdf32_butterfly29, WasmSimdF32Butterfly29, 29);
8176 test_butterfly_32_func!(test_wasm_simdf32_butterfly31, WasmSimdF32Butterfly31, 31);
8177
8178 //the tests for all butterflies will be identical except for the identifiers used and size
8179 //so it's ideal for a macro
8180 macro_rules! test_butterfly_64_func {
8181 ($test_name:ident, $struct_name:ident, $size:expr) => {
8182 #[test]
8183 fn $test_name() {
8184 let butterfly = $struct_name::new(FftDirection::Forward);
8185 check_fft_algorithm::<f64>(&butterfly, $size, FftDirection::Forward);
8186
8187 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
8188 check_fft_algorithm::<f64>(&butterfly_direction, $size, FftDirection::Inverse);
8189 }
8190 };
8191 }
8192 test_butterfly_64_func!(test_wasm_simdf64_butterfly7, WasmSimdF64Butterfly7, 7);
8193 test_butterfly_64_func!(test_wasm_simdf64_butterfly11, WasmSimdF64Butterfly11, 11);
8194 test_butterfly_64_func!(test_wasm_simdf64_butterfly13, WasmSimdF64Butterfly13, 13);
8195 test_butterfly_64_func!(test_wasm_simdf64_butterfly17, WasmSimdF64Butterfly17, 17);
8196 test_butterfly_64_func!(test_wasm_simdf64_butterfly19, WasmSimdF64Butterfly19, 19);
8197 test_butterfly_64_func!(test_wasm_simdf64_butterfly23, WasmSimdF64Butterfly23, 23);
8198 test_butterfly_64_func!(test_wasm_simdf64_butterfly29, WasmSimdF64Butterfly29, 29);
8199 test_butterfly_64_func!(test_wasm_simdf64_butterfly31, WasmSimdF64Butterfly31, 31);
8200 }