Debian Codegraph @

.cargo/vendor/rustfft/src/sse/sse_prime_butterflies.rs

raw
1 use core::arch::x86_64::*;
2 use num_complex::Complex;
3
4 use crate::{common::FftNum, FftDirection};
5
6 use crate::array_utils;
7 use crate::array_utils::workaround_transmute_mut;
8 use crate::array_utils::DoubleBuf;
9 use crate::common::{fft_error_inplace, fft_error_outofplace};
10 use crate::twiddles;
11 use crate::{Direction, Fft, Length};
12
13 use super::sse_common::{assert_f32, assert_f64};
14 use super::sse_utils::*;
15 use super::sse_vector::{SseArrayMut};
16 use super::sse_butterflies::{parallel_fft2_interleaved_f32, solo_fft2_f64};
17
18 // Auto-generated prime length butterflies
19 // The code here is mostly autogenerated by the python script tools/gen_sse_butterflies.py
20 //
21 // The algorithm is derived directly from the definition of the DFT, by eliminating any repeated calculations.
22 // See the comments in src/algorithm/butterflies.rs for a detailed description.
23 //
24 // The script generates the code for performing a single f64 fft, as well as dual f32 fft.
25 // It also generates the code for reading and writing the input and output.
26 // The single 32-bit ffts reuse the dual ffts.
27
28 // _____ _________ _ _ _
29 // |___ | |___ /___ \| |__ (_) |_
30 // / / _____ |_ \ __) | '_ \| | __|
31 // / / |_____| ___) / __/| |_) | | |_
32 // /_/ |____/_____|_.__/|_|\__|
33 //
34
35 pub struct SseF32Butterfly7<T> {
36 direction: FftDirection,
37 _phantom: std::marker::PhantomData<T>,
38 rotate: Rotate90F32,
39 twiddle1re: __m128,
40 twiddle1im: __m128,
41 twiddle2re: __m128,
42 twiddle2im: __m128,
43 twiddle3re: __m128,
44 twiddle3im: __m128,
45 }
46
47 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly7, 7, |this: &SseF32Butterfly7<_>| this
48 .direction);
49 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly7, 7, |this: &SseF32Butterfly7<_>| this
50 .direction);
51 impl<T: FftNum> SseF32Butterfly7<T> {
52 #[inline(always)]
53 pub fn new(direction: FftDirection) -> Self {
54 assert_f32::<T>();
55 let rotate = Rotate90F32::new(true);
56 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 7, direction);
57 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 7, direction);
58 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 7, direction);
59 let twiddle1re = unsafe { _mm_set_ps(tw1.re, tw1.re, tw1.re, tw1.re) };
60 let twiddle1im = unsafe { _mm_set_ps(tw1.im, tw1.im, tw1.im, tw1.im) };
61 let twiddle2re = unsafe { _mm_set_ps(tw2.re, tw2.re, tw2.re, tw2.re) };
62 let twiddle2im = unsafe { _mm_set_ps(tw2.im, tw2.im, tw2.im, tw2.im) };
63 let twiddle3re = unsafe { _mm_set_ps(tw3.re, tw3.re, tw3.re, tw3.re) };
64 let twiddle3im = unsafe { _mm_set_ps(tw3.im, tw3.im, tw3.im, tw3.im) };
65
66 Self {
67 direction,
68 _phantom: std::marker::PhantomData,
69 rotate,
70 twiddle1re,
71 twiddle1im,
72 twiddle2re,
73 twiddle2im,
74 twiddle3re,
75 twiddle3im,
76 }
77 }
78 #[inline(always)]
79 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
80 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6});
81
82 let out = self.perform_parallel_fft_direct(values);
83
84 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6});
85 }
86
87 #[inline(always)]
88 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
89 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12});
90
91 let values = [
92 extract_lo_hi_f32(input_packed[0], input_packed[3]),
93 extract_hi_lo_f32(input_packed[0], input_packed[4]),
94 extract_lo_hi_f32(input_packed[1], input_packed[4]),
95 extract_hi_lo_f32(input_packed[1], input_packed[5]),
96 extract_lo_hi_f32(input_packed[2], input_packed[5]),
97 extract_hi_lo_f32(input_packed[2], input_packed[6]),
98 extract_lo_hi_f32(input_packed[3], input_packed[6]),
99 ];
100
101 let out = self.perform_parallel_fft_direct(values);
102
103 let out_packed = [
104 extract_lo_lo_f32(out[0], out[1]),
105 extract_lo_lo_f32(out[2], out[3]),
106 extract_lo_lo_f32(out[4], out[5]),
107 extract_lo_hi_f32(out[6], out[0]),
108 extract_hi_hi_f32(out[1], out[2]),
109 extract_hi_hi_f32(out[3], out[4]),
110 extract_hi_hi_f32(out[5], out[6]),
111 ];
112
113 write_complex_to_array_strided!(out_packed, buffer, 2, {0,1,2,3,4,5,6});
114 }
115
116 #[inline(always)]
117 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 7]) -> [__m128; 7] {
118 let [x1p6, x1m6] = parallel_fft2_interleaved_f32(values[1], values[6]);
119 let [x2p5, x2m5] = parallel_fft2_interleaved_f32(values[2], values[5]);
120 let [x3p4, x3m4] = parallel_fft2_interleaved_f32(values[3], values[4]);
121
122 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p6);
123 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p5);
124 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p4);
125 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p6);
126 let t_a2_2 = _mm_mul_ps(self.twiddle3re, x2p5);
127 let t_a2_3 = _mm_mul_ps(self.twiddle1re, x3p4);
128 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p6);
129 let t_a3_2 = _mm_mul_ps(self.twiddle1re, x2p5);
130 let t_a3_3 = _mm_mul_ps(self.twiddle2re, x3p4);
131
132 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m6);
133 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m5);
134 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m4);
135 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m6);
136 let t_b2_2 = _mm_mul_ps(self.twiddle3im, x2m5);
137 let t_b2_3 = _mm_mul_ps(self.twiddle1im, x3m4);
138 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m6);
139 let t_b3_2 = _mm_mul_ps(self.twiddle1im, x2m5);
140 let t_b3_3 = _mm_mul_ps(self.twiddle2im, x3m4);
141
142 let x0 = values[0];
143 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3);
144 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3);
145 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3);
146
147 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3);
148 let t_b2 = calc_f32!(t_b2_1 - t_b2_2 - t_b2_3);
149 let t_b3 = calc_f32!(t_b3_1 - t_b3_2 + t_b3_3);
150
151 let t_b1_rot = self.rotate.rotate_both(t_b1);
152 let t_b2_rot = self.rotate.rotate_both(t_b2);
153 let t_b3_rot = self.rotate.rotate_both(t_b3);
154
155 let y0 = calc_f32!(x0 + x1p6 + x2p5 + x3p4);
156 let [y1, y6] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
157 let [y2, y5] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
158 let [y3, y4] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
159 [y0, y1, y2, y3, y4, y5, y6]
160 }
161 }
162
163 // _____ __ _ _ _ _ _
164 // |___ | / /_ | || | | |__ (_) |_
165 // / / _____ | '_ \| || |_| '_ \| | __|
166 // / / |_____| | (_) |__ _| |_) | | |_
167 // /_/ \___/ |_| |_.__/|_|\__|
168 //
169
170 pub struct SseF64Butterfly7<T> {
171 direction: FftDirection,
172 _phantom: std::marker::PhantomData<T>,
173 rotate: Rotate90F64,
174 twiddle1re: __m128d,
175 twiddle1im: __m128d,
176 twiddle2re: __m128d,
177 twiddle2im: __m128d,
178 twiddle3re: __m128d,
179 twiddle3im: __m128d,
180 }
181
182 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly7, 7, |this: &SseF64Butterfly7<_>| this
183 .direction);
184 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly7, 7, |this: &SseF64Butterfly7<_>| this
185 .direction);
186 impl<T: FftNum> SseF64Butterfly7<T> {
187 #[inline(always)]
188 pub fn new(direction: FftDirection) -> Self {
189 assert_f64::<T>();
190 let rotate = Rotate90F64::new(true);
191 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 7, direction);
192 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 7, direction);
193 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 7, direction);
194 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
195 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
196 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
197 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
198 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
199 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
200
201 Self {
202 direction,
203 _phantom: std::marker::PhantomData,
204 rotate,
205 twiddle1re,
206 twiddle1im,
207 twiddle2re,
208 twiddle2im,
209 twiddle3re,
210 twiddle3im,
211 }
212 }
213
214 #[inline(always)]
215 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
216 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6});
217
218 let out = self.perform_fft_direct(values);
219
220 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6});
221 }
222
223 #[inline(always)]
224 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 7]) -> [__m128d; 7] {
225 let [x1p6, x1m6] = solo_fft2_f64(values[1], values[6]);
226 let [x2p5, x2m5] = solo_fft2_f64(values[2], values[5]);
227 let [x3p4, x3m4] = solo_fft2_f64(values[3], values[4]);
228
229 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p6);
230 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p5);
231 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p4);
232 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p6);
233 let t_a2_2 = _mm_mul_pd(self.twiddle3re, x2p5);
234 let t_a2_3 = _mm_mul_pd(self.twiddle1re, x3p4);
235 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p6);
236 let t_a3_2 = _mm_mul_pd(self.twiddle1re, x2p5);
237 let t_a3_3 = _mm_mul_pd(self.twiddle2re, x3p4);
238
239 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m6);
240 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m5);
241 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m4);
242 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m6);
243 let t_b2_2 = _mm_mul_pd(self.twiddle3im, x2m5);
244 let t_b2_3 = _mm_mul_pd(self.twiddle1im, x3m4);
245 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m6);
246 let t_b3_2 = _mm_mul_pd(self.twiddle1im, x2m5);
247 let t_b3_3 = _mm_mul_pd(self.twiddle2im, x3m4);
248
249 let x0 = values[0];
250 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3);
251 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3);
252 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3);
253
254 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3);
255 let t_b2 = calc_f64!(t_b2_1 - t_b2_2 - t_b2_3);
256 let t_b3 = calc_f64!(t_b3_1 - t_b3_2 + t_b3_3);
257
258 let t_b1_rot = self.rotate.rotate(t_b1);
259 let t_b2_rot = self.rotate.rotate(t_b2);
260 let t_b3_rot = self.rotate.rotate(t_b3);
261
262 let y0 = calc_f64!(x0 + x1p6 + x2p5 + x3p4);
263 let [y1, y6] = solo_fft2_f64(t_a1, t_b1_rot);
264 let [y2, y5] = solo_fft2_f64(t_a2, t_b2_rot);
265 let [y3, y4] = solo_fft2_f64(t_a3, t_b3_rot);
266 [y0, y1, y2, y3, y4, y5, y6]
267
268 }
269 }
270
271 // _ _ _________ _ _ _
272 // / / | |___ /___ \| |__ (_) |_
273 // | | | _____ |_ \ __) | '_ \| | __|
274 // | | | |_____| ___) / __/| |_) | | |_
275 // |_|_| |____/_____|_.__/|_|\__|
276 //
277
278 pub struct SseF32Butterfly11<T> {
279 direction: FftDirection,
280 _phantom: std::marker::PhantomData<T>,
281 rotate: Rotate90F32,
282 twiddle1re: __m128,
283 twiddle1im: __m128,
284 twiddle2re: __m128,
285 twiddle2im: __m128,
286 twiddle3re: __m128,
287 twiddle3im: __m128,
288 twiddle4re: __m128,
289 twiddle4im: __m128,
290 twiddle5re: __m128,
291 twiddle5im: __m128,
292 }
293
294 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly11, 11, |this: &SseF32Butterfly11<_>| this
295 .direction);
296 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly11, 11, |this: &SseF32Butterfly11<_>| this
297 .direction);
298 impl<T: FftNum> SseF32Butterfly11<T> {
299 #[inline(always)]
300 pub fn new(direction: FftDirection) -> Self {
301 assert_f32::<T>();
302 let rotate = Rotate90F32::new(true);
303 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 11, direction);
304 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 11, direction);
305 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 11, direction);
306 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 11, direction);
307 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 11, direction);
308 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
309 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
310 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
311 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
312 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
313 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
314 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
315 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
316 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
317 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
318
319 Self {
320 direction,
321 _phantom: std::marker::PhantomData,
322 rotate,
323 twiddle1re,
324 twiddle1im,
325 twiddle2re,
326 twiddle2im,
327 twiddle3re,
328 twiddle3im,
329 twiddle4re,
330 twiddle4im,
331 twiddle5re,
332 twiddle5im,
333 }
334 }
335
336 #[inline(always)]
337 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
338 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
339
340 let out = self.perform_parallel_fft_direct(values);
341
342 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
343 }
344
345 #[inline(always)]
346 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
347 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20});
348
349 let values = [
350 extract_lo_hi_f32(input_packed[0], input_packed[5]),
351 extract_hi_lo_f32(input_packed[0], input_packed[6]),
352 extract_lo_hi_f32(input_packed[1], input_packed[6]),
353 extract_hi_lo_f32(input_packed[1], input_packed[7]),
354 extract_lo_hi_f32(input_packed[2], input_packed[7]),
355 extract_hi_lo_f32(input_packed[2], input_packed[8]),
356 extract_lo_hi_f32(input_packed[3], input_packed[8]),
357 extract_hi_lo_f32(input_packed[3], input_packed[9]),
358 extract_lo_hi_f32(input_packed[4], input_packed[9]),
359 extract_hi_lo_f32(input_packed[4], input_packed[10]),
360 extract_lo_hi_f32(input_packed[5], input_packed[10]),
361 ];
362
363 let out = self.perform_parallel_fft_direct(values);
364
365 let out_packed = [
366 extract_lo_lo_f32(out[0], out[1]),
367 extract_lo_lo_f32(out[2], out[3]),
368 extract_lo_lo_f32(out[4], out[5]),
369 extract_lo_lo_f32(out[6], out[7]),
370 extract_lo_lo_f32(out[8], out[9]),
371 extract_lo_hi_f32(out[10], out[0]),
372 extract_hi_hi_f32(out[1], out[2]),
373 extract_hi_hi_f32(out[3], out[4]),
374 extract_hi_hi_f32(out[5], out[6]),
375 extract_hi_hi_f32(out[7], out[8]),
376 extract_hi_hi_f32(out[9], out[10]),
377 ];
378
379 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
380 }
381
382 #[inline(always)]
383 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 11]) -> [__m128; 11] {
384 let [x1p10, x1m10] = parallel_fft2_interleaved_f32(values[1], values[10]);
385 let [x2p9, x2m9] = parallel_fft2_interleaved_f32(values[2], values[9]);
386 let [x3p8, x3m8] = parallel_fft2_interleaved_f32(values[3], values[8]);
387 let [x4p7, x4m7] = parallel_fft2_interleaved_f32(values[4], values[7]);
388 let [x5p6, x5m6] = parallel_fft2_interleaved_f32(values[5], values[6]);
389
390 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p10);
391 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p9);
392 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p8);
393 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p7);
394 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p6);
395 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p10);
396 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p9);
397 let t_a2_3 = _mm_mul_ps(self.twiddle5re, x3p8);
398 let t_a2_4 = _mm_mul_ps(self.twiddle3re, x4p7);
399 let t_a2_5 = _mm_mul_ps(self.twiddle1re, x5p6);
400 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p10);
401 let t_a3_2 = _mm_mul_ps(self.twiddle5re, x2p9);
402 let t_a3_3 = _mm_mul_ps(self.twiddle2re, x3p8);
403 let t_a3_4 = _mm_mul_ps(self.twiddle1re, x4p7);
404 let t_a3_5 = _mm_mul_ps(self.twiddle4re, x5p6);
405 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p10);
406 let t_a4_2 = _mm_mul_ps(self.twiddle3re, x2p9);
407 let t_a4_3 = _mm_mul_ps(self.twiddle1re, x3p8);
408 let t_a4_4 = _mm_mul_ps(self.twiddle5re, x4p7);
409 let t_a4_5 = _mm_mul_ps(self.twiddle2re, x5p6);
410 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p10);
411 let t_a5_2 = _mm_mul_ps(self.twiddle1re, x2p9);
412 let t_a5_3 = _mm_mul_ps(self.twiddle4re, x3p8);
413 let t_a5_4 = _mm_mul_ps(self.twiddle2re, x4p7);
414 let t_a5_5 = _mm_mul_ps(self.twiddle3re, x5p6);
415
416 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m10);
417 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m9);
418 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m8);
419 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m7);
420 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m6);
421 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m10);
422 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m9);
423 let t_b2_3 = _mm_mul_ps(self.twiddle5im, x3m8);
424 let t_b2_4 = _mm_mul_ps(self.twiddle3im, x4m7);
425 let t_b2_5 = _mm_mul_ps(self.twiddle1im, x5m6);
426 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m10);
427 let t_b3_2 = _mm_mul_ps(self.twiddle5im, x2m9);
428 let t_b3_3 = _mm_mul_ps(self.twiddle2im, x3m8);
429 let t_b3_4 = _mm_mul_ps(self.twiddle1im, x4m7);
430 let t_b3_5 = _mm_mul_ps(self.twiddle4im, x5m6);
431 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m10);
432 let t_b4_2 = _mm_mul_ps(self.twiddle3im, x2m9);
433 let t_b4_3 = _mm_mul_ps(self.twiddle1im, x3m8);
434 let t_b4_4 = _mm_mul_ps(self.twiddle5im, x4m7);
435 let t_b4_5 = _mm_mul_ps(self.twiddle2im, x5m6);
436 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m10);
437 let t_b5_2 = _mm_mul_ps(self.twiddle1im, x2m9);
438 let t_b5_3 = _mm_mul_ps(self.twiddle4im, x3m8);
439 let t_b5_4 = _mm_mul_ps(self.twiddle2im, x4m7);
440 let t_b5_5 = _mm_mul_ps(self.twiddle3im, x5m6);
441
442 let x0 = values[0];
443 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5);
444 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5);
445 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5);
446 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5);
447 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5);
448
449 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5);
450 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 - t_b2_3 - t_b2_4 - t_b2_5);
451 let t_b3 = calc_f32!(t_b3_1 - t_b3_2 - t_b3_3 + t_b3_4 + t_b3_5);
452 let t_b4 = calc_f32!(t_b4_1 - t_b4_2 + t_b4_3 + t_b4_4 - t_b4_5);
453 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 + t_b5_5);
454
455 let t_b1_rot = self.rotate.rotate_both(t_b1);
456 let t_b2_rot = self.rotate.rotate_both(t_b2);
457 let t_b3_rot = self.rotate.rotate_both(t_b3);
458 let t_b4_rot = self.rotate.rotate_both(t_b4);
459 let t_b5_rot = self.rotate.rotate_both(t_b5);
460
461 let y0 = calc_f32!(x0 + x1p10 + x2p9 + x3p8 + x4p7 + x5p6);
462 let [y1, y10] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
463 let [y2, y9] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
464 let [y3, y8] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
465 let [y4, y7] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
466 let [y5, y6] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
467 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10]
468
469 }
470 }
471
472 // _ _ __ _ _ _ _ _
473 // / / | / /_ | || | | |__ (_) |_
474 // | | | _____ | '_ \| || |_| '_ \| | __|
475 // | | | |_____| | (_) |__ _| |_) | | |_
476 // |_|_| \___/ |_| |_.__/|_|\__|
477 //
478
479 pub struct SseF64Butterfly11<T> {
480 direction: FftDirection,
481 _phantom: std::marker::PhantomData<T>,
482 rotate: Rotate90F64,
483 twiddle1re: __m128d,
484 twiddle1im: __m128d,
485 twiddle2re: __m128d,
486 twiddle2im: __m128d,
487 twiddle3re: __m128d,
488 twiddle3im: __m128d,
489 twiddle4re: __m128d,
490 twiddle4im: __m128d,
491 twiddle5re: __m128d,
492 twiddle5im: __m128d,
493 }
494
495 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly11, 11, |this: &SseF64Butterfly11<_>| this
496 .direction);
497 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly11, 11, |this: &SseF64Butterfly11<_>| this
498 .direction);
499 impl<T: FftNum> SseF64Butterfly11<T> {
500 #[inline(always)]
501 pub fn new(direction: FftDirection) -> Self {
502 assert_f64::<T>();
503 let rotate = Rotate90F64::new(true);
504 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 11, direction);
505 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 11, direction);
506 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 11, direction);
507 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 11, direction);
508 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 11, direction);
509 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
510 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
511 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
512 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
513 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
514 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
515 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
516 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
517 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
518 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
519
520 Self {
521 direction,
522 _phantom: std::marker::PhantomData,
523 rotate,
524 twiddle1re,
525 twiddle1im,
526 twiddle2re,
527 twiddle2im,
528 twiddle3re,
529 twiddle3im,
530 twiddle4re,
531 twiddle4im,
532 twiddle5re,
533 twiddle5im,
534 }
535 }
536
537 #[inline(always)]
538 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
539 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
540
541 let out = self.perform_fft_direct(values);
542
543 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
544 }
545
546 #[inline(always)]
547 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 11]) -> [__m128d; 11] {
548 let [x1p10, x1m10] = solo_fft2_f64(values[1], values[10]);
549 let [x2p9, x2m9] = solo_fft2_f64(values[2], values[9]);
550 let [x3p8, x3m8] = solo_fft2_f64(values[3], values[8]);
551 let [x4p7, x4m7] = solo_fft2_f64(values[4], values[7]);
552 let [x5p6, x5m6] = solo_fft2_f64(values[5], values[6]);
553
554 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p10);
555 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p9);
556 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p8);
557 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p7);
558 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p6);
559 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p10);
560 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p9);
561 let t_a2_3 = _mm_mul_pd(self.twiddle5re, x3p8);
562 let t_a2_4 = _mm_mul_pd(self.twiddle3re, x4p7);
563 let t_a2_5 = _mm_mul_pd(self.twiddle1re, x5p6);
564 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p10);
565 let t_a3_2 = _mm_mul_pd(self.twiddle5re, x2p9);
566 let t_a3_3 = _mm_mul_pd(self.twiddle2re, x3p8);
567 let t_a3_4 = _mm_mul_pd(self.twiddle1re, x4p7);
568 let t_a3_5 = _mm_mul_pd(self.twiddle4re, x5p6);
569 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p10);
570 let t_a4_2 = _mm_mul_pd(self.twiddle3re, x2p9);
571 let t_a4_3 = _mm_mul_pd(self.twiddle1re, x3p8);
572 let t_a4_4 = _mm_mul_pd(self.twiddle5re, x4p7);
573 let t_a4_5 = _mm_mul_pd(self.twiddle2re, x5p6);
574 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p10);
575 let t_a5_2 = _mm_mul_pd(self.twiddle1re, x2p9);
576 let t_a5_3 = _mm_mul_pd(self.twiddle4re, x3p8);
577 let t_a5_4 = _mm_mul_pd(self.twiddle2re, x4p7);
578 let t_a5_5 = _mm_mul_pd(self.twiddle3re, x5p6);
579
580 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m10);
581 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m9);
582 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m8);
583 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m7);
584 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m6);
585 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m10);
586 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m9);
587 let t_b2_3 = _mm_mul_pd(self.twiddle5im, x3m8);
588 let t_b2_4 = _mm_mul_pd(self.twiddle3im, x4m7);
589 let t_b2_5 = _mm_mul_pd(self.twiddle1im, x5m6);
590 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m10);
591 let t_b3_2 = _mm_mul_pd(self.twiddle5im, x2m9);
592 let t_b3_3 = _mm_mul_pd(self.twiddle2im, x3m8);
593 let t_b3_4 = _mm_mul_pd(self.twiddle1im, x4m7);
594 let t_b3_5 = _mm_mul_pd(self.twiddle4im, x5m6);
595 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m10);
596 let t_b4_2 = _mm_mul_pd(self.twiddle3im, x2m9);
597 let t_b4_3 = _mm_mul_pd(self.twiddle1im, x3m8);
598 let t_b4_4 = _mm_mul_pd(self.twiddle5im, x4m7);
599 let t_b4_5 = _mm_mul_pd(self.twiddle2im, x5m6);
600 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m10);
601 let t_b5_2 = _mm_mul_pd(self.twiddle1im, x2m9);
602 let t_b5_3 = _mm_mul_pd(self.twiddle4im, x3m8);
603 let t_b5_4 = _mm_mul_pd(self.twiddle2im, x4m7);
604 let t_b5_5 = _mm_mul_pd(self.twiddle3im, x5m6);
605
606 let x0 = values[0];
607 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5);
608 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5);
609 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5);
610 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5);
611 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5);
612
613 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5);
614 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 - t_b2_3 - t_b2_4 - t_b2_5);
615 let t_b3 = calc_f64!(t_b3_1 - t_b3_2 - t_b3_3 + t_b3_4 + t_b3_5);
616 let t_b4 = calc_f64!(t_b4_1 - t_b4_2 + t_b4_3 + t_b4_4 - t_b4_5);
617 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 + t_b5_5);
618
619 let t_b1_rot = self.rotate.rotate(t_b1);
620 let t_b2_rot = self.rotate.rotate(t_b2);
621 let t_b3_rot = self.rotate.rotate(t_b3);
622 let t_b4_rot = self.rotate.rotate(t_b4);
623 let t_b5_rot = self.rotate.rotate(t_b5);
624
625 let y0 = calc_f64!(x0 + x1p10 + x2p9 + x3p8 + x4p7 + x5p6);
626 let [y1, y10] = solo_fft2_f64(t_a1, t_b1_rot);
627 let [y2, y9] = solo_fft2_f64(t_a2, t_b2_rot);
628 let [y3, y8] = solo_fft2_f64(t_a3, t_b3_rot);
629 let [y4, y7] = solo_fft2_f64(t_a4, t_b4_rot);
630 let [y5, y6] = solo_fft2_f64(t_a5, t_b5_rot);
631 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10]
632 }
633 }
634
635 // _ _____ _________ _ _ _
636 // / |___ / |___ /___ \| |__ (_) |_
637 // | | |_ \ _____ |_ \ __) | '_ \| | __|
638 // | |___) | |_____| ___) / __/| |_) | | |_
639 // |_|____/ |____/_____|_.__/|_|\__|
640 //
641
642 pub struct SseF32Butterfly13<T> {
643 direction: FftDirection,
644 _phantom: std::marker::PhantomData<T>,
645 rotate: Rotate90F32,
646 twiddle1re: __m128,
647 twiddle1im: __m128,
648 twiddle2re: __m128,
649 twiddle2im: __m128,
650 twiddle3re: __m128,
651 twiddle3im: __m128,
652 twiddle4re: __m128,
653 twiddle4im: __m128,
654 twiddle5re: __m128,
655 twiddle5im: __m128,
656 twiddle6re: __m128,
657 twiddle6im: __m128,
658 }
659
660 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly13, 13, |this: &SseF32Butterfly13<_>| this
661 .direction);
662 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly13, 13, |this: &SseF32Butterfly13<_>| this
663 .direction);
664 impl<T: FftNum> SseF32Butterfly13<T> {
665 #[inline(always)]
666 pub fn new(direction: FftDirection) -> Self {
667 assert_f32::<T>();
668 let rotate = Rotate90F32::new(true);
669 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 13, direction);
670 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 13, direction);
671 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 13, direction);
672 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 13, direction);
673 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 13, direction);
674 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 13, direction);
675 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
676 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
677 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
678 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
679 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
680 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
681 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
682 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
683 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
684 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
685 let twiddle6re = unsafe { _mm_load1_ps(&tw6.re) };
686 let twiddle6im = unsafe { _mm_load1_ps(&tw6.im) };
687
688 Self {
689 direction,
690 _phantom: std::marker::PhantomData,
691 rotate,
692 twiddle1re,
693 twiddle1im,
694 twiddle2re,
695 twiddle2im,
696 twiddle3re,
697 twiddle3im,
698 twiddle4re,
699 twiddle4im,
700 twiddle5re,
701 twiddle5im,
702 twiddle6re,
703 twiddle6im,
704 }
705 }
706
707 #[inline(always)]
708 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
709 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
710
711 let out = self.perform_parallel_fft_direct(values);
712
713 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
714 }
715
716 #[inline(always)]
717 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
718 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24});
719
720 let values = [
721 extract_lo_hi_f32(input_packed[0], input_packed[6]),
722 extract_hi_lo_f32(input_packed[0], input_packed[7]),
723 extract_lo_hi_f32(input_packed[1], input_packed[7]),
724 extract_hi_lo_f32(input_packed[1], input_packed[8]),
725 extract_lo_hi_f32(input_packed[2], input_packed[8]),
726 extract_hi_lo_f32(input_packed[2], input_packed[9]),
727 extract_lo_hi_f32(input_packed[3], input_packed[9]),
728 extract_hi_lo_f32(input_packed[3], input_packed[10]),
729 extract_lo_hi_f32(input_packed[4], input_packed[10]),
730 extract_hi_lo_f32(input_packed[4], input_packed[11]),
731 extract_lo_hi_f32(input_packed[5], input_packed[11]),
732 extract_hi_lo_f32(input_packed[5], input_packed[12]),
733 extract_lo_hi_f32(input_packed[6], input_packed[12]),
734 ];
735
736 let out = self.perform_parallel_fft_direct(values);
737
738 let out_packed = [
739 extract_lo_lo_f32(out[0], out[1]),
740 extract_lo_lo_f32(out[2], out[3]),
741 extract_lo_lo_f32(out[4], out[5]),
742 extract_lo_lo_f32(out[6], out[7]),
743 extract_lo_lo_f32(out[8], out[9]),
744 extract_lo_lo_f32(out[10], out[11]),
745 extract_lo_hi_f32(out[12], out[0]),
746 extract_hi_hi_f32(out[1], out[2]),
747 extract_hi_hi_f32(out[3], out[4]),
748 extract_hi_hi_f32(out[5], out[6]),
749 extract_hi_hi_f32(out[7], out[8]),
750 extract_hi_hi_f32(out[9], out[10]),
751 extract_hi_hi_f32(out[11], out[12]),
752 ];
753
754 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
755 }
756
757 #[inline(always)]
758 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 13]) -> [__m128; 13] {
759 let [x1p12, x1m12] = parallel_fft2_interleaved_f32(values[1], values[12]);
760 let [x2p11, x2m11] = parallel_fft2_interleaved_f32(values[2], values[11]);
761 let [x3p10, x3m10] = parallel_fft2_interleaved_f32(values[3], values[10]);
762 let [x4p9, x4m9] = parallel_fft2_interleaved_f32(values[4], values[9]);
763 let [x5p8, x5m8] = parallel_fft2_interleaved_f32(values[5], values[8]);
764 let [x6p7, x6m7] = parallel_fft2_interleaved_f32(values[6], values[7]);
765
766 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p12);
767 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p11);
768 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p10);
769 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p9);
770 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p8);
771 let t_a1_6 = _mm_mul_ps(self.twiddle6re, x6p7);
772 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p12);
773 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p11);
774 let t_a2_3 = _mm_mul_ps(self.twiddle6re, x3p10);
775 let t_a2_4 = _mm_mul_ps(self.twiddle5re, x4p9);
776 let t_a2_5 = _mm_mul_ps(self.twiddle3re, x5p8);
777 let t_a2_6 = _mm_mul_ps(self.twiddle1re, x6p7);
778 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p12);
779 let t_a3_2 = _mm_mul_ps(self.twiddle6re, x2p11);
780 let t_a3_3 = _mm_mul_ps(self.twiddle4re, x3p10);
781 let t_a3_4 = _mm_mul_ps(self.twiddle1re, x4p9);
782 let t_a3_5 = _mm_mul_ps(self.twiddle2re, x5p8);
783 let t_a3_6 = _mm_mul_ps(self.twiddle5re, x6p7);
784 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p12);
785 let t_a4_2 = _mm_mul_ps(self.twiddle5re, x2p11);
786 let t_a4_3 = _mm_mul_ps(self.twiddle1re, x3p10);
787 let t_a4_4 = _mm_mul_ps(self.twiddle3re, x4p9);
788 let t_a4_5 = _mm_mul_ps(self.twiddle6re, x5p8);
789 let t_a4_6 = _mm_mul_ps(self.twiddle2re, x6p7);
790 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p12);
791 let t_a5_2 = _mm_mul_ps(self.twiddle3re, x2p11);
792 let t_a5_3 = _mm_mul_ps(self.twiddle2re, x3p10);
793 let t_a5_4 = _mm_mul_ps(self.twiddle6re, x4p9);
794 let t_a5_5 = _mm_mul_ps(self.twiddle1re, x5p8);
795 let t_a5_6 = _mm_mul_ps(self.twiddle4re, x6p7);
796 let t_a6_1 = _mm_mul_ps(self.twiddle6re, x1p12);
797 let t_a6_2 = _mm_mul_ps(self.twiddle1re, x2p11);
798 let t_a6_3 = _mm_mul_ps(self.twiddle5re, x3p10);
799 let t_a6_4 = _mm_mul_ps(self.twiddle2re, x4p9);
800 let t_a6_5 = _mm_mul_ps(self.twiddle4re, x5p8);
801 let t_a6_6 = _mm_mul_ps(self.twiddle3re, x6p7);
802
803 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m12);
804 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m11);
805 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m10);
806 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m9);
807 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m8);
808 let t_b1_6 = _mm_mul_ps(self.twiddle6im, x6m7);
809 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m12);
810 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m11);
811 let t_b2_3 = _mm_mul_ps(self.twiddle6im, x3m10);
812 let t_b2_4 = _mm_mul_ps(self.twiddle5im, x4m9);
813 let t_b2_5 = _mm_mul_ps(self.twiddle3im, x5m8);
814 let t_b2_6 = _mm_mul_ps(self.twiddle1im, x6m7);
815 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m12);
816 let t_b3_2 = _mm_mul_ps(self.twiddle6im, x2m11);
817 let t_b3_3 = _mm_mul_ps(self.twiddle4im, x3m10);
818 let t_b3_4 = _mm_mul_ps(self.twiddle1im, x4m9);
819 let t_b3_5 = _mm_mul_ps(self.twiddle2im, x5m8);
820 let t_b3_6 = _mm_mul_ps(self.twiddle5im, x6m7);
821 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m12);
822 let t_b4_2 = _mm_mul_ps(self.twiddle5im, x2m11);
823 let t_b4_3 = _mm_mul_ps(self.twiddle1im, x3m10);
824 let t_b4_4 = _mm_mul_ps(self.twiddle3im, x4m9);
825 let t_b4_5 = _mm_mul_ps(self.twiddle6im, x5m8);
826 let t_b4_6 = _mm_mul_ps(self.twiddle2im, x6m7);
827 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m12);
828 let t_b5_2 = _mm_mul_ps(self.twiddle3im, x2m11);
829 let t_b5_3 = _mm_mul_ps(self.twiddle2im, x3m10);
830 let t_b5_4 = _mm_mul_ps(self.twiddle6im, x4m9);
831 let t_b5_5 = _mm_mul_ps(self.twiddle1im, x5m8);
832 let t_b5_6 = _mm_mul_ps(self.twiddle4im, x6m7);
833 let t_b6_1 = _mm_mul_ps(self.twiddle6im, x1m12);
834 let t_b6_2 = _mm_mul_ps(self.twiddle1im, x2m11);
835 let t_b6_3 = _mm_mul_ps(self.twiddle5im, x3m10);
836 let t_b6_4 = _mm_mul_ps(self.twiddle2im, x4m9);
837 let t_b6_5 = _mm_mul_ps(self.twiddle4im, x5m8);
838 let t_b6_6 = _mm_mul_ps(self.twiddle3im, x6m7);
839
840 let x0 = values[0];
841 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6);
842 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6);
843 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6);
844 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6);
845 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6);
846 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6);
847
848 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6);
849 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 - t_b2_4 - t_b2_5 - t_b2_6);
850 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 + t_b3_5 + t_b3_6);
851 let t_b4 = calc_f32!(t_b4_1 - t_b4_2 - t_b4_3 + t_b4_4 - t_b4_5 - t_b4_6);
852 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6);
853 let t_b6 = calc_f32!(t_b6_1 - t_b6_2 + t_b6_3 - t_b6_4 + t_b6_5 - t_b6_6);
854
855 let t_b1_rot = self.rotate.rotate_both(t_b1);
856 let t_b2_rot = self.rotate.rotate_both(t_b2);
857 let t_b3_rot = self.rotate.rotate_both(t_b3);
858 let t_b4_rot = self.rotate.rotate_both(t_b4);
859 let t_b5_rot = self.rotate.rotate_both(t_b5);
860 let t_b6_rot = self.rotate.rotate_both(t_b6);
861
862 let y0 = calc_f32!(x0 + x1p12 + x2p11 + x3p10 + x4p9 + x5p8 + x6p7);
863 let [y1, y12] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
864 let [y2, y11] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
865 let [y3, y10] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
866 let [y4, y9] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
867 let [y5, y8] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
868 let [y6, y7] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
869 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12]
870 }
871 }
872
873 // _ _____ __ _ _ _ _ _
874 // / |___ / / /_ | || | | |__ (_) |_
875 // | | |_ \ _____ | '_ \| || |_| '_ \| | __|
876 // | |___) | |_____| | (_) |__ _| |_) | | |_
877 // |_|____/ \___/ |_| |_.__/|_|\__|
878 //
879
880 pub struct SseF64Butterfly13<T> {
881 direction: FftDirection,
882 _phantom: std::marker::PhantomData<T>,
883 rotate: Rotate90F64,
884 twiddle1re: __m128d,
885 twiddle1im: __m128d,
886 twiddle2re: __m128d,
887 twiddle2im: __m128d,
888 twiddle3re: __m128d,
889 twiddle3im: __m128d,
890 twiddle4re: __m128d,
891 twiddle4im: __m128d,
892 twiddle5re: __m128d,
893 twiddle5im: __m128d,
894 twiddle6re: __m128d,
895 twiddle6im: __m128d,
896 }
897
898 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly13, 13, |this: &SseF64Butterfly13<_>| this
899 .direction);
900 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly13, 13, |this: &SseF64Butterfly13<_>| this
901 .direction);
902 impl<T: FftNum> SseF64Butterfly13<T> {
903 #[inline(always)]
904 pub fn new(direction: FftDirection) -> Self {
905 assert_f64::<T>();
906 let rotate = Rotate90F64::new(true);
907 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 13, direction);
908 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 13, direction);
909 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 13, direction);
910 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 13, direction);
911 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 13, direction);
912 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 13, direction);
913 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
914 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
915 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
916 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
917 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
918 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
919 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
920 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
921 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
922 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
923 let twiddle6re = unsafe { _mm_set_pd(tw6.re, tw6.re) };
924 let twiddle6im = unsafe { _mm_set_pd(tw6.im, tw6.im) };
925
926 Self {
927 direction,
928 _phantom: std::marker::PhantomData,
929 rotate,
930 twiddle1re,
931 twiddle1im,
932 twiddle2re,
933 twiddle2im,
934 twiddle3re,
935 twiddle3im,
936 twiddle4re,
937 twiddle4im,
938 twiddle5re,
939 twiddle5im,
940 twiddle6re,
941 twiddle6im,
942 }
943 }
944
945 #[inline(always)]
946 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
947 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
948
949 let out = self.perform_fft_direct(values);
950
951 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
952 }
953
954 #[inline(always)]
955 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 13]) -> [__m128d; 13] {
956 let [x1p12, x1m12] = solo_fft2_f64(values[1], values[12]);
957 let [x2p11, x2m11] = solo_fft2_f64(values[2], values[11]);
958 let [x3p10, x3m10] = solo_fft2_f64(values[3], values[10]);
959 let [x4p9, x4m9] = solo_fft2_f64(values[4], values[9]);
960 let [x5p8, x5m8] = solo_fft2_f64(values[5], values[8]);
961 let [x6p7, x6m7] = solo_fft2_f64(values[6], values[7]);
962
963 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p12);
964 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p11);
965 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p10);
966 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p9);
967 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p8);
968 let t_a1_6 = _mm_mul_pd(self.twiddle6re, x6p7);
969 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p12);
970 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p11);
971 let t_a2_3 = _mm_mul_pd(self.twiddle6re, x3p10);
972 let t_a2_4 = _mm_mul_pd(self.twiddle5re, x4p9);
973 let t_a2_5 = _mm_mul_pd(self.twiddle3re, x5p8);
974 let t_a2_6 = _mm_mul_pd(self.twiddle1re, x6p7);
975 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p12);
976 let t_a3_2 = _mm_mul_pd(self.twiddle6re, x2p11);
977 let t_a3_3 = _mm_mul_pd(self.twiddle4re, x3p10);
978 let t_a3_4 = _mm_mul_pd(self.twiddle1re, x4p9);
979 let t_a3_5 = _mm_mul_pd(self.twiddle2re, x5p8);
980 let t_a3_6 = _mm_mul_pd(self.twiddle5re, x6p7);
981 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p12);
982 let t_a4_2 = _mm_mul_pd(self.twiddle5re, x2p11);
983 let t_a4_3 = _mm_mul_pd(self.twiddle1re, x3p10);
984 let t_a4_4 = _mm_mul_pd(self.twiddle3re, x4p9);
985 let t_a4_5 = _mm_mul_pd(self.twiddle6re, x5p8);
986 let t_a4_6 = _mm_mul_pd(self.twiddle2re, x6p7);
987 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p12);
988 let t_a5_2 = _mm_mul_pd(self.twiddle3re, x2p11);
989 let t_a5_3 = _mm_mul_pd(self.twiddle2re, x3p10);
990 let t_a5_4 = _mm_mul_pd(self.twiddle6re, x4p9);
991 let t_a5_5 = _mm_mul_pd(self.twiddle1re, x5p8);
992 let t_a5_6 = _mm_mul_pd(self.twiddle4re, x6p7);
993 let t_a6_1 = _mm_mul_pd(self.twiddle6re, x1p12);
994 let t_a6_2 = _mm_mul_pd(self.twiddle1re, x2p11);
995 let t_a6_3 = _mm_mul_pd(self.twiddle5re, x3p10);
996 let t_a6_4 = _mm_mul_pd(self.twiddle2re, x4p9);
997 let t_a6_5 = _mm_mul_pd(self.twiddle4re, x5p8);
998 let t_a6_6 = _mm_mul_pd(self.twiddle3re, x6p7);
999
1000 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m12);
1001 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m11);
1002 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m10);
1003 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m9);
1004 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m8);
1005 let t_b1_6 = _mm_mul_pd(self.twiddle6im, x6m7);
1006 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m12);
1007 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m11);
1008 let t_b2_3 = _mm_mul_pd(self.twiddle6im, x3m10);
1009 let t_b2_4 = _mm_mul_pd(self.twiddle5im, x4m9);
1010 let t_b2_5 = _mm_mul_pd(self.twiddle3im, x5m8);
1011 let t_b2_6 = _mm_mul_pd(self.twiddle1im, x6m7);
1012 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m12);
1013 let t_b3_2 = _mm_mul_pd(self.twiddle6im, x2m11);
1014 let t_b3_3 = _mm_mul_pd(self.twiddle4im, x3m10);
1015 let t_b3_4 = _mm_mul_pd(self.twiddle1im, x4m9);
1016 let t_b3_5 = _mm_mul_pd(self.twiddle2im, x5m8);
1017 let t_b3_6 = _mm_mul_pd(self.twiddle5im, x6m7);
1018 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m12);
1019 let t_b4_2 = _mm_mul_pd(self.twiddle5im, x2m11);
1020 let t_b4_3 = _mm_mul_pd(self.twiddle1im, x3m10);
1021 let t_b4_4 = _mm_mul_pd(self.twiddle3im, x4m9);
1022 let t_b4_5 = _mm_mul_pd(self.twiddle6im, x5m8);
1023 let t_b4_6 = _mm_mul_pd(self.twiddle2im, x6m7);
1024 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m12);
1025 let t_b5_2 = _mm_mul_pd(self.twiddle3im, x2m11);
1026 let t_b5_3 = _mm_mul_pd(self.twiddle2im, x3m10);
1027 let t_b5_4 = _mm_mul_pd(self.twiddle6im, x4m9);
1028 let t_b5_5 = _mm_mul_pd(self.twiddle1im, x5m8);
1029 let t_b5_6 = _mm_mul_pd(self.twiddle4im, x6m7);
1030 let t_b6_1 = _mm_mul_pd(self.twiddle6im, x1m12);
1031 let t_b6_2 = _mm_mul_pd(self.twiddle1im, x2m11);
1032 let t_b6_3 = _mm_mul_pd(self.twiddle5im, x3m10);
1033 let t_b6_4 = _mm_mul_pd(self.twiddle2im, x4m9);
1034 let t_b6_5 = _mm_mul_pd(self.twiddle4im, x5m8);
1035 let t_b6_6 = _mm_mul_pd(self.twiddle3im, x6m7);
1036
1037 let x0 = values[0];
1038 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6);
1039 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6);
1040 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6);
1041 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6);
1042 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6);
1043 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6);
1044
1045 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6);
1046 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 - t_b2_4 - t_b2_5 - t_b2_6);
1047 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 + t_b3_5 + t_b3_6);
1048 let t_b4 = calc_f64!(t_b4_1 - t_b4_2 - t_b4_3 + t_b4_4 - t_b4_5 - t_b4_6);
1049 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6);
1050 let t_b6 = calc_f64!(t_b6_1 - t_b6_2 + t_b6_3 - t_b6_4 + t_b6_5 - t_b6_6);
1051
1052 let t_b1_rot = self.rotate.rotate(t_b1);
1053 let t_b2_rot = self.rotate.rotate(t_b2);
1054 let t_b3_rot = self.rotate.rotate(t_b3);
1055 let t_b4_rot = self.rotate.rotate(t_b4);
1056 let t_b5_rot = self.rotate.rotate(t_b5);
1057 let t_b6_rot = self.rotate.rotate(t_b6);
1058
1059 let y0 = calc_f64!(x0 + x1p12 + x2p11 + x3p10 + x4p9 + x5p8 + x6p7);
1060 let [y1, y12] = solo_fft2_f64(t_a1, t_b1_rot);
1061 let [y2, y11] = solo_fft2_f64(t_a2, t_b2_rot);
1062 let [y3, y10] = solo_fft2_f64(t_a3, t_b3_rot);
1063 let [y4, y9] = solo_fft2_f64(t_a4, t_b4_rot);
1064 let [y5, y8] = solo_fft2_f64(t_a5, t_b5_rot);
1065 let [y6, y7] = solo_fft2_f64(t_a6, t_b6_rot);
1066 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12]
1067 }
1068 }
1069
1070 // _ _____ _________ _ _ _
1071 // / |___ | |___ /___ \| |__ (_) |_
1072 // | | / / _____ |_ \ __) | '_ \| | __|
1073 // | | / / |_____| ___) / __/| |_) | | |_
1074 // |_|/_/ |____/_____|_.__/|_|\__|
1075 //
1076
1077 pub struct SseF32Butterfly17<T> {
1078 direction: FftDirection,
1079 _phantom: std::marker::PhantomData<T>,
1080 rotate: Rotate90F32,
1081 twiddle1re: __m128,
1082 twiddle1im: __m128,
1083 twiddle2re: __m128,
1084 twiddle2im: __m128,
1085 twiddle3re: __m128,
1086 twiddle3im: __m128,
1087 twiddle4re: __m128,
1088 twiddle4im: __m128,
1089 twiddle5re: __m128,
1090 twiddle5im: __m128,
1091 twiddle6re: __m128,
1092 twiddle6im: __m128,
1093 twiddle7re: __m128,
1094 twiddle7im: __m128,
1095 twiddle8re: __m128,
1096 twiddle8im: __m128,
1097 }
1098
1099 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly17, 17, |this: &SseF32Butterfly17<_>| this
1100 .direction);
1101 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly17, 17, |this: &SseF32Butterfly17<_>| this
1102 .direction);
1103 impl<T: FftNum> SseF32Butterfly17<T> {
1104 #[inline(always)]
1105 pub fn new(direction: FftDirection) -> Self {
1106 assert_f32::<T>();
1107 let rotate = Rotate90F32::new(true);
1108 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 17, direction);
1109 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 17, direction);
1110 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 17, direction);
1111 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 17, direction);
1112 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 17, direction);
1113 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 17, direction);
1114 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 17, direction);
1115 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 17, direction);
1116 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
1117 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
1118 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
1119 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
1120 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
1121 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
1122 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
1123 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
1124 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
1125 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
1126 let twiddle6re = unsafe { _mm_load1_ps(&tw6.re) };
1127 let twiddle6im = unsafe { _mm_load1_ps(&tw6.im) };
1128 let twiddle7re = unsafe { _mm_load1_ps(&tw7.re) };
1129 let twiddle7im = unsafe { _mm_load1_ps(&tw7.im) };
1130 let twiddle8re = unsafe { _mm_load1_ps(&tw8.re) };
1131 let twiddle8im = unsafe { _mm_load1_ps(&tw8.im) };
1132
1133 Self {
1134 direction,
1135 _phantom: std::marker::PhantomData,
1136 rotate,
1137 twiddle1re,
1138 twiddle1im,
1139 twiddle2re,
1140 twiddle2im,
1141 twiddle3re,
1142 twiddle3im,
1143 twiddle4re,
1144 twiddle4im,
1145 twiddle5re,
1146 twiddle5im,
1147 twiddle6re,
1148 twiddle6im,
1149 twiddle7re,
1150 twiddle7im,
1151 twiddle8re,
1152 twiddle8im,
1153 }
1154 }
1155
1156 #[inline(always)]
1157 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1158 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1159
1160 let out = self.perform_parallel_fft_direct(values);
1161
1162 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1163 }
1164
1165 #[inline(always)]
1166 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1167 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32});
1168
1169 let values = [
1170 extract_lo_hi_f32(input_packed[0], input_packed[8]),
1171 extract_hi_lo_f32(input_packed[0], input_packed[9]),
1172 extract_lo_hi_f32(input_packed[1], input_packed[9]),
1173 extract_hi_lo_f32(input_packed[1], input_packed[10]),
1174 extract_lo_hi_f32(input_packed[2], input_packed[10]),
1175 extract_hi_lo_f32(input_packed[2], input_packed[11]),
1176 extract_lo_hi_f32(input_packed[3], input_packed[11]),
1177 extract_hi_lo_f32(input_packed[3], input_packed[12]),
1178 extract_lo_hi_f32(input_packed[4], input_packed[12]),
1179 extract_hi_lo_f32(input_packed[4], input_packed[13]),
1180 extract_lo_hi_f32(input_packed[5], input_packed[13]),
1181 extract_hi_lo_f32(input_packed[5], input_packed[14]),
1182 extract_lo_hi_f32(input_packed[6], input_packed[14]),
1183 extract_hi_lo_f32(input_packed[6], input_packed[15]),
1184 extract_lo_hi_f32(input_packed[7], input_packed[15]),
1185 extract_hi_lo_f32(input_packed[7], input_packed[16]),
1186 extract_lo_hi_f32(input_packed[8], input_packed[16]),
1187 ];
1188
1189 let out = self.perform_parallel_fft_direct(values);
1190
1191 let out_packed = [
1192 extract_lo_lo_f32(out[0], out[1]),
1193 extract_lo_lo_f32(out[2], out[3]),
1194 extract_lo_lo_f32(out[4], out[5]),
1195 extract_lo_lo_f32(out[6], out[7]),
1196 extract_lo_lo_f32(out[8], out[9]),
1197 extract_lo_lo_f32(out[10], out[11]),
1198 extract_lo_lo_f32(out[12], out[13]),
1199 extract_lo_lo_f32(out[14], out[15]),
1200 extract_lo_hi_f32(out[16], out[0]),
1201 extract_hi_hi_f32(out[1], out[2]),
1202 extract_hi_hi_f32(out[3], out[4]),
1203 extract_hi_hi_f32(out[5], out[6]),
1204 extract_hi_hi_f32(out[7], out[8]),
1205 extract_hi_hi_f32(out[9], out[10]),
1206 extract_hi_hi_f32(out[11], out[12]),
1207 extract_hi_hi_f32(out[13], out[14]),
1208 extract_hi_hi_f32(out[15], out[16]),
1209 ];
1210
1211 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1212 }
1213
1214 #[inline(always)]
1215 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 17]) -> [__m128; 17] {
1216 let [x1p16, x1m16] = parallel_fft2_interleaved_f32(values[1], values[16]);
1217 let [x2p15, x2m15] = parallel_fft2_interleaved_f32(values[2], values[15]);
1218 let [x3p14, x3m14] = parallel_fft2_interleaved_f32(values[3], values[14]);
1219 let [x4p13, x4m13] = parallel_fft2_interleaved_f32(values[4], values[13]);
1220 let [x5p12, x5m12] = parallel_fft2_interleaved_f32(values[5], values[12]);
1221 let [x6p11, x6m11] = parallel_fft2_interleaved_f32(values[6], values[11]);
1222 let [x7p10, x7m10] = parallel_fft2_interleaved_f32(values[7], values[10]);
1223 let [x8p9, x8m9] = parallel_fft2_interleaved_f32(values[8], values[9]);
1224
1225 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p16);
1226 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p15);
1227 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p14);
1228 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p13);
1229 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p12);
1230 let t_a1_6 = _mm_mul_ps(self.twiddle6re, x6p11);
1231 let t_a1_7 = _mm_mul_ps(self.twiddle7re, x7p10);
1232 let t_a1_8 = _mm_mul_ps(self.twiddle8re, x8p9);
1233 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p16);
1234 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p15);
1235 let t_a2_3 = _mm_mul_ps(self.twiddle6re, x3p14);
1236 let t_a2_4 = _mm_mul_ps(self.twiddle8re, x4p13);
1237 let t_a2_5 = _mm_mul_ps(self.twiddle7re, x5p12);
1238 let t_a2_6 = _mm_mul_ps(self.twiddle5re, x6p11);
1239 let t_a2_7 = _mm_mul_ps(self.twiddle3re, x7p10);
1240 let t_a2_8 = _mm_mul_ps(self.twiddle1re, x8p9);
1241 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p16);
1242 let t_a3_2 = _mm_mul_ps(self.twiddle6re, x2p15);
1243 let t_a3_3 = _mm_mul_ps(self.twiddle8re, x3p14);
1244 let t_a3_4 = _mm_mul_ps(self.twiddle5re, x4p13);
1245 let t_a3_5 = _mm_mul_ps(self.twiddle2re, x5p12);
1246 let t_a3_6 = _mm_mul_ps(self.twiddle1re, x6p11);
1247 let t_a3_7 = _mm_mul_ps(self.twiddle4re, x7p10);
1248 let t_a3_8 = _mm_mul_ps(self.twiddle7re, x8p9);
1249 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p16);
1250 let t_a4_2 = _mm_mul_ps(self.twiddle8re, x2p15);
1251 let t_a4_3 = _mm_mul_ps(self.twiddle5re, x3p14);
1252 let t_a4_4 = _mm_mul_ps(self.twiddle1re, x4p13);
1253 let t_a4_5 = _mm_mul_ps(self.twiddle3re, x5p12);
1254 let t_a4_6 = _mm_mul_ps(self.twiddle7re, x6p11);
1255 let t_a4_7 = _mm_mul_ps(self.twiddle6re, x7p10);
1256 let t_a4_8 = _mm_mul_ps(self.twiddle2re, x8p9);
1257 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p16);
1258 let t_a5_2 = _mm_mul_ps(self.twiddle7re, x2p15);
1259 let t_a5_3 = _mm_mul_ps(self.twiddle2re, x3p14);
1260 let t_a5_4 = _mm_mul_ps(self.twiddle3re, x4p13);
1261 let t_a5_5 = _mm_mul_ps(self.twiddle8re, x5p12);
1262 let t_a5_6 = _mm_mul_ps(self.twiddle4re, x6p11);
1263 let t_a5_7 = _mm_mul_ps(self.twiddle1re, x7p10);
1264 let t_a5_8 = _mm_mul_ps(self.twiddle6re, x8p9);
1265 let t_a6_1 = _mm_mul_ps(self.twiddle6re, x1p16);
1266 let t_a6_2 = _mm_mul_ps(self.twiddle5re, x2p15);
1267 let t_a6_3 = _mm_mul_ps(self.twiddle1re, x3p14);
1268 let t_a6_4 = _mm_mul_ps(self.twiddle7re, x4p13);
1269 let t_a6_5 = _mm_mul_ps(self.twiddle4re, x5p12);
1270 let t_a6_6 = _mm_mul_ps(self.twiddle2re, x6p11);
1271 let t_a6_7 = _mm_mul_ps(self.twiddle8re, x7p10);
1272 let t_a6_8 = _mm_mul_ps(self.twiddle3re, x8p9);
1273 let t_a7_1 = _mm_mul_ps(self.twiddle7re, x1p16);
1274 let t_a7_2 = _mm_mul_ps(self.twiddle3re, x2p15);
1275 let t_a7_3 = _mm_mul_ps(self.twiddle4re, x3p14);
1276 let t_a7_4 = _mm_mul_ps(self.twiddle6re, x4p13);
1277 let t_a7_5 = _mm_mul_ps(self.twiddle1re, x5p12);
1278 let t_a7_6 = _mm_mul_ps(self.twiddle8re, x6p11);
1279 let t_a7_7 = _mm_mul_ps(self.twiddle2re, x7p10);
1280 let t_a7_8 = _mm_mul_ps(self.twiddle5re, x8p9);
1281 let t_a8_1 = _mm_mul_ps(self.twiddle8re, x1p16);
1282 let t_a8_2 = _mm_mul_ps(self.twiddle1re, x2p15);
1283 let t_a8_3 = _mm_mul_ps(self.twiddle7re, x3p14);
1284 let t_a8_4 = _mm_mul_ps(self.twiddle2re, x4p13);
1285 let t_a8_5 = _mm_mul_ps(self.twiddle6re, x5p12);
1286 let t_a8_6 = _mm_mul_ps(self.twiddle3re, x6p11);
1287 let t_a8_7 = _mm_mul_ps(self.twiddle5re, x7p10);
1288 let t_a8_8 = _mm_mul_ps(self.twiddle4re, x8p9);
1289
1290 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m16);
1291 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m15);
1292 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m14);
1293 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m13);
1294 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m12);
1295 let t_b1_6 = _mm_mul_ps(self.twiddle6im, x6m11);
1296 let t_b1_7 = _mm_mul_ps(self.twiddle7im, x7m10);
1297 let t_b1_8 = _mm_mul_ps(self.twiddle8im, x8m9);
1298 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m16);
1299 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m15);
1300 let t_b2_3 = _mm_mul_ps(self.twiddle6im, x3m14);
1301 let t_b2_4 = _mm_mul_ps(self.twiddle8im, x4m13);
1302 let t_b2_5 = _mm_mul_ps(self.twiddle7im, x5m12);
1303 let t_b2_6 = _mm_mul_ps(self.twiddle5im, x6m11);
1304 let t_b2_7 = _mm_mul_ps(self.twiddle3im, x7m10);
1305 let t_b2_8 = _mm_mul_ps(self.twiddle1im, x8m9);
1306 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m16);
1307 let t_b3_2 = _mm_mul_ps(self.twiddle6im, x2m15);
1308 let t_b3_3 = _mm_mul_ps(self.twiddle8im, x3m14);
1309 let t_b3_4 = _mm_mul_ps(self.twiddle5im, x4m13);
1310 let t_b3_5 = _mm_mul_ps(self.twiddle2im, x5m12);
1311 let t_b3_6 = _mm_mul_ps(self.twiddle1im, x6m11);
1312 let t_b3_7 = _mm_mul_ps(self.twiddle4im, x7m10);
1313 let t_b3_8 = _mm_mul_ps(self.twiddle7im, x8m9);
1314 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m16);
1315 let t_b4_2 = _mm_mul_ps(self.twiddle8im, x2m15);
1316 let t_b4_3 = _mm_mul_ps(self.twiddle5im, x3m14);
1317 let t_b4_4 = _mm_mul_ps(self.twiddle1im, x4m13);
1318 let t_b4_5 = _mm_mul_ps(self.twiddle3im, x5m12);
1319 let t_b4_6 = _mm_mul_ps(self.twiddle7im, x6m11);
1320 let t_b4_7 = _mm_mul_ps(self.twiddle6im, x7m10);
1321 let t_b4_8 = _mm_mul_ps(self.twiddle2im, x8m9);
1322 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m16);
1323 let t_b5_2 = _mm_mul_ps(self.twiddle7im, x2m15);
1324 let t_b5_3 = _mm_mul_ps(self.twiddle2im, x3m14);
1325 let t_b5_4 = _mm_mul_ps(self.twiddle3im, x4m13);
1326 let t_b5_5 = _mm_mul_ps(self.twiddle8im, x5m12);
1327 let t_b5_6 = _mm_mul_ps(self.twiddle4im, x6m11);
1328 let t_b5_7 = _mm_mul_ps(self.twiddle1im, x7m10);
1329 let t_b5_8 = _mm_mul_ps(self.twiddle6im, x8m9);
1330 let t_b6_1 = _mm_mul_ps(self.twiddle6im, x1m16);
1331 let t_b6_2 = _mm_mul_ps(self.twiddle5im, x2m15);
1332 let t_b6_3 = _mm_mul_ps(self.twiddle1im, x3m14);
1333 let t_b6_4 = _mm_mul_ps(self.twiddle7im, x4m13);
1334 let t_b6_5 = _mm_mul_ps(self.twiddle4im, x5m12);
1335 let t_b6_6 = _mm_mul_ps(self.twiddle2im, x6m11);
1336 let t_b6_7 = _mm_mul_ps(self.twiddle8im, x7m10);
1337 let t_b6_8 = _mm_mul_ps(self.twiddle3im, x8m9);
1338 let t_b7_1 = _mm_mul_ps(self.twiddle7im, x1m16);
1339 let t_b7_2 = _mm_mul_ps(self.twiddle3im, x2m15);
1340 let t_b7_3 = _mm_mul_ps(self.twiddle4im, x3m14);
1341 let t_b7_4 = _mm_mul_ps(self.twiddle6im, x4m13);
1342 let t_b7_5 = _mm_mul_ps(self.twiddle1im, x5m12);
1343 let t_b7_6 = _mm_mul_ps(self.twiddle8im, x6m11);
1344 let t_b7_7 = _mm_mul_ps(self.twiddle2im, x7m10);
1345 let t_b7_8 = _mm_mul_ps(self.twiddle5im, x8m9);
1346 let t_b8_1 = _mm_mul_ps(self.twiddle8im, x1m16);
1347 let t_b8_2 = _mm_mul_ps(self.twiddle1im, x2m15);
1348 let t_b8_3 = _mm_mul_ps(self.twiddle7im, x3m14);
1349 let t_b8_4 = _mm_mul_ps(self.twiddle2im, x4m13);
1350 let t_b8_5 = _mm_mul_ps(self.twiddle6im, x5m12);
1351 let t_b8_6 = _mm_mul_ps(self.twiddle3im, x6m11);
1352 let t_b8_7 = _mm_mul_ps(self.twiddle5im, x7m10);
1353 let t_b8_8 = _mm_mul_ps(self.twiddle4im, x8m9);
1354
1355 let x0 = values[0];
1356 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8);
1357 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8);
1358 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8);
1359 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8);
1360 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8);
1361 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8);
1362 let t_a7 = calc_f32!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8);
1363 let t_a8 = calc_f32!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8);
1364
1365 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8);
1366 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8);
1367 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 - t_b3_5 + t_b3_6 + t_b3_7 + t_b3_8);
1368 let t_b4 = calc_f32!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 - t_b4_7 - t_b4_8);
1369 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8);
1370 let t_b6 = calc_f32!(t_b6_1 - t_b6_2 + t_b6_3 + t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8);
1371 let t_b7 = calc_f32!(t_b7_1 - t_b7_2 + t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 + t_b7_8);
1372 let t_b8 = calc_f32!(t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 + t_b8_7 - t_b8_8);
1373
1374 let t_b1_rot = self.rotate.rotate_both(t_b1);
1375 let t_b2_rot = self.rotate.rotate_both(t_b2);
1376 let t_b3_rot = self.rotate.rotate_both(t_b3);
1377 let t_b4_rot = self.rotate.rotate_both(t_b4);
1378 let t_b5_rot = self.rotate.rotate_both(t_b5);
1379 let t_b6_rot = self.rotate.rotate_both(t_b6);
1380 let t_b7_rot = self.rotate.rotate_both(t_b7);
1381 let t_b8_rot = self.rotate.rotate_both(t_b8);
1382
1383 let y0 = calc_f32!(x0 + x1p16 + x2p15 + x3p14 + x4p13 + x5p12 + x6p11 + x7p10 + x8p9);
1384 let [y1, y16] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
1385 let [y2, y15] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
1386 let [y3, y14] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
1387 let [y4, y13] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
1388 let [y5, y12] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
1389 let [y6, y11] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
1390 let [y7, y10] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
1391 let [y8, y9] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
1392 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16]
1393 }
1394 }
1395
1396 // _ _____ __ _ _ _ _ _
1397 // / |___ | / /_ | || | | |__ (_) |_
1398 // | | / / _____ | '_ \| || |_| '_ \| | __|
1399 // | | / / |_____| | (_) |__ _| |_) | | |_
1400 // |_|/_/ \___/ |_| |_.__/|_|\__|
1401 //
1402
1403 pub struct SseF64Butterfly17<T> {
1404 direction: FftDirection,
1405 _phantom: std::marker::PhantomData<T>,
1406 rotate: Rotate90F64,
1407 twiddle1re: __m128d,
1408 twiddle1im: __m128d,
1409 twiddle2re: __m128d,
1410 twiddle2im: __m128d,
1411 twiddle3re: __m128d,
1412 twiddle3im: __m128d,
1413 twiddle4re: __m128d,
1414 twiddle4im: __m128d,
1415 twiddle5re: __m128d,
1416 twiddle5im: __m128d,
1417 twiddle6re: __m128d,
1418 twiddle6im: __m128d,
1419 twiddle7re: __m128d,
1420 twiddle7im: __m128d,
1421 twiddle8re: __m128d,
1422 twiddle8im: __m128d,
1423 }
1424
1425 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly17, 17, |this: &SseF64Butterfly17<_>| this
1426 .direction);
1427 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly17, 17, |this: &SseF64Butterfly17<_>| this
1428 .direction);
1429 impl<T: FftNum> SseF64Butterfly17<T> {
1430 #[inline(always)]
1431 pub fn new(direction: FftDirection) -> Self {
1432 assert_f64::<T>();
1433 let rotate = Rotate90F64::new(true);
1434 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 17, direction);
1435 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 17, direction);
1436 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 17, direction);
1437 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 17, direction);
1438 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 17, direction);
1439 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 17, direction);
1440 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 17, direction);
1441 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 17, direction);
1442 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
1443 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
1444 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
1445 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
1446 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
1447 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
1448 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
1449 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
1450 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
1451 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
1452 let twiddle6re = unsafe { _mm_set_pd(tw6.re, tw6.re) };
1453 let twiddle6im = unsafe { _mm_set_pd(tw6.im, tw6.im) };
1454 let twiddle7re = unsafe { _mm_set_pd(tw7.re, tw7.re) };
1455 let twiddle7im = unsafe { _mm_set_pd(tw7.im, tw7.im) };
1456 let twiddle8re = unsafe { _mm_set_pd(tw8.re, tw8.re) };
1457 let twiddle8im = unsafe { _mm_set_pd(tw8.im, tw8.im) };
1458
1459 Self {
1460 direction,
1461 _phantom: std::marker::PhantomData,
1462 rotate,
1463 twiddle1re,
1464 twiddle1im,
1465 twiddle2re,
1466 twiddle2im,
1467 twiddle3re,
1468 twiddle3im,
1469 twiddle4re,
1470 twiddle4im,
1471 twiddle5re,
1472 twiddle5im,
1473 twiddle6re,
1474 twiddle6im,
1475 twiddle7re,
1476 twiddle7im,
1477 twiddle8re,
1478 twiddle8im,
1479 }
1480 }
1481
1482 #[inline(always)]
1483 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
1484 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1485
1486 let out = self.perform_fft_direct(values);
1487
1488 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
1489 }
1490
1491 #[inline(always)]
1492 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 17]) -> [__m128d; 17] {
1493 let [x1p16, x1m16] = solo_fft2_f64(values[1], values[16]);
1494 let [x2p15, x2m15] = solo_fft2_f64(values[2], values[15]);
1495 let [x3p14, x3m14] = solo_fft2_f64(values[3], values[14]);
1496 let [x4p13, x4m13] = solo_fft2_f64(values[4], values[13]);
1497 let [x5p12, x5m12] = solo_fft2_f64(values[5], values[12]);
1498 let [x6p11, x6m11] = solo_fft2_f64(values[6], values[11]);
1499 let [x7p10, x7m10] = solo_fft2_f64(values[7], values[10]);
1500 let [x8p9, x8m9] = solo_fft2_f64(values[8], values[9]);
1501
1502 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p16);
1503 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p15);
1504 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p14);
1505 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p13);
1506 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p12);
1507 let t_a1_6 = _mm_mul_pd(self.twiddle6re, x6p11);
1508 let t_a1_7 = _mm_mul_pd(self.twiddle7re, x7p10);
1509 let t_a1_8 = _mm_mul_pd(self.twiddle8re, x8p9);
1510 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p16);
1511 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p15);
1512 let t_a2_3 = _mm_mul_pd(self.twiddle6re, x3p14);
1513 let t_a2_4 = _mm_mul_pd(self.twiddle8re, x4p13);
1514 let t_a2_5 = _mm_mul_pd(self.twiddle7re, x5p12);
1515 let t_a2_6 = _mm_mul_pd(self.twiddle5re, x6p11);
1516 let t_a2_7 = _mm_mul_pd(self.twiddle3re, x7p10);
1517 let t_a2_8 = _mm_mul_pd(self.twiddle1re, x8p9);
1518 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p16);
1519 let t_a3_2 = _mm_mul_pd(self.twiddle6re, x2p15);
1520 let t_a3_3 = _mm_mul_pd(self.twiddle8re, x3p14);
1521 let t_a3_4 = _mm_mul_pd(self.twiddle5re, x4p13);
1522 let t_a3_5 = _mm_mul_pd(self.twiddle2re, x5p12);
1523 let t_a3_6 = _mm_mul_pd(self.twiddle1re, x6p11);
1524 let t_a3_7 = _mm_mul_pd(self.twiddle4re, x7p10);
1525 let t_a3_8 = _mm_mul_pd(self.twiddle7re, x8p9);
1526 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p16);
1527 let t_a4_2 = _mm_mul_pd(self.twiddle8re, x2p15);
1528 let t_a4_3 = _mm_mul_pd(self.twiddle5re, x3p14);
1529 let t_a4_4 = _mm_mul_pd(self.twiddle1re, x4p13);
1530 let t_a4_5 = _mm_mul_pd(self.twiddle3re, x5p12);
1531 let t_a4_6 = _mm_mul_pd(self.twiddle7re, x6p11);
1532 let t_a4_7 = _mm_mul_pd(self.twiddle6re, x7p10);
1533 let t_a4_8 = _mm_mul_pd(self.twiddle2re, x8p9);
1534 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p16);
1535 let t_a5_2 = _mm_mul_pd(self.twiddle7re, x2p15);
1536 let t_a5_3 = _mm_mul_pd(self.twiddle2re, x3p14);
1537 let t_a5_4 = _mm_mul_pd(self.twiddle3re, x4p13);
1538 let t_a5_5 = _mm_mul_pd(self.twiddle8re, x5p12);
1539 let t_a5_6 = _mm_mul_pd(self.twiddle4re, x6p11);
1540 let t_a5_7 = _mm_mul_pd(self.twiddle1re, x7p10);
1541 let t_a5_8 = _mm_mul_pd(self.twiddle6re, x8p9);
1542 let t_a6_1 = _mm_mul_pd(self.twiddle6re, x1p16);
1543 let t_a6_2 = _mm_mul_pd(self.twiddle5re, x2p15);
1544 let t_a6_3 = _mm_mul_pd(self.twiddle1re, x3p14);
1545 let t_a6_4 = _mm_mul_pd(self.twiddle7re, x4p13);
1546 let t_a6_5 = _mm_mul_pd(self.twiddle4re, x5p12);
1547 let t_a6_6 = _mm_mul_pd(self.twiddle2re, x6p11);
1548 let t_a6_7 = _mm_mul_pd(self.twiddle8re, x7p10);
1549 let t_a6_8 = _mm_mul_pd(self.twiddle3re, x8p9);
1550 let t_a7_1 = _mm_mul_pd(self.twiddle7re, x1p16);
1551 let t_a7_2 = _mm_mul_pd(self.twiddle3re, x2p15);
1552 let t_a7_3 = _mm_mul_pd(self.twiddle4re, x3p14);
1553 let t_a7_4 = _mm_mul_pd(self.twiddle6re, x4p13);
1554 let t_a7_5 = _mm_mul_pd(self.twiddle1re, x5p12);
1555 let t_a7_6 = _mm_mul_pd(self.twiddle8re, x6p11);
1556 let t_a7_7 = _mm_mul_pd(self.twiddle2re, x7p10);
1557 let t_a7_8 = _mm_mul_pd(self.twiddle5re, x8p9);
1558 let t_a8_1 = _mm_mul_pd(self.twiddle8re, x1p16);
1559 let t_a8_2 = _mm_mul_pd(self.twiddle1re, x2p15);
1560 let t_a8_3 = _mm_mul_pd(self.twiddle7re, x3p14);
1561 let t_a8_4 = _mm_mul_pd(self.twiddle2re, x4p13);
1562 let t_a8_5 = _mm_mul_pd(self.twiddle6re, x5p12);
1563 let t_a8_6 = _mm_mul_pd(self.twiddle3re, x6p11);
1564 let t_a8_7 = _mm_mul_pd(self.twiddle5re, x7p10);
1565 let t_a8_8 = _mm_mul_pd(self.twiddle4re, x8p9);
1566
1567 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m16);
1568 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m15);
1569 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m14);
1570 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m13);
1571 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m12);
1572 let t_b1_6 = _mm_mul_pd(self.twiddle6im, x6m11);
1573 let t_b1_7 = _mm_mul_pd(self.twiddle7im, x7m10);
1574 let t_b1_8 = _mm_mul_pd(self.twiddle8im, x8m9);
1575 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m16);
1576 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m15);
1577 let t_b2_3 = _mm_mul_pd(self.twiddle6im, x3m14);
1578 let t_b2_4 = _mm_mul_pd(self.twiddle8im, x4m13);
1579 let t_b2_5 = _mm_mul_pd(self.twiddle7im, x5m12);
1580 let t_b2_6 = _mm_mul_pd(self.twiddle5im, x6m11);
1581 let t_b2_7 = _mm_mul_pd(self.twiddle3im, x7m10);
1582 let t_b2_8 = _mm_mul_pd(self.twiddle1im, x8m9);
1583 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m16);
1584 let t_b3_2 = _mm_mul_pd(self.twiddle6im, x2m15);
1585 let t_b3_3 = _mm_mul_pd(self.twiddle8im, x3m14);
1586 let t_b3_4 = _mm_mul_pd(self.twiddle5im, x4m13);
1587 let t_b3_5 = _mm_mul_pd(self.twiddle2im, x5m12);
1588 let t_b3_6 = _mm_mul_pd(self.twiddle1im, x6m11);
1589 let t_b3_7 = _mm_mul_pd(self.twiddle4im, x7m10);
1590 let t_b3_8 = _mm_mul_pd(self.twiddle7im, x8m9);
1591 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m16);
1592 let t_b4_2 = _mm_mul_pd(self.twiddle8im, x2m15);
1593 let t_b4_3 = _mm_mul_pd(self.twiddle5im, x3m14);
1594 let t_b4_4 = _mm_mul_pd(self.twiddle1im, x4m13);
1595 let t_b4_5 = _mm_mul_pd(self.twiddle3im, x5m12);
1596 let t_b4_6 = _mm_mul_pd(self.twiddle7im, x6m11);
1597 let t_b4_7 = _mm_mul_pd(self.twiddle6im, x7m10);
1598 let t_b4_8 = _mm_mul_pd(self.twiddle2im, x8m9);
1599 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m16);
1600 let t_b5_2 = _mm_mul_pd(self.twiddle7im, x2m15);
1601 let t_b5_3 = _mm_mul_pd(self.twiddle2im, x3m14);
1602 let t_b5_4 = _mm_mul_pd(self.twiddle3im, x4m13);
1603 let t_b5_5 = _mm_mul_pd(self.twiddle8im, x5m12);
1604 let t_b5_6 = _mm_mul_pd(self.twiddle4im, x6m11);
1605 let t_b5_7 = _mm_mul_pd(self.twiddle1im, x7m10);
1606 let t_b5_8 = _mm_mul_pd(self.twiddle6im, x8m9);
1607 let t_b6_1 = _mm_mul_pd(self.twiddle6im, x1m16);
1608 let t_b6_2 = _mm_mul_pd(self.twiddle5im, x2m15);
1609 let t_b6_3 = _mm_mul_pd(self.twiddle1im, x3m14);
1610 let t_b6_4 = _mm_mul_pd(self.twiddle7im, x4m13);
1611 let t_b6_5 = _mm_mul_pd(self.twiddle4im, x5m12);
1612 let t_b6_6 = _mm_mul_pd(self.twiddle2im, x6m11);
1613 let t_b6_7 = _mm_mul_pd(self.twiddle8im, x7m10);
1614 let t_b6_8 = _mm_mul_pd(self.twiddle3im, x8m9);
1615 let t_b7_1 = _mm_mul_pd(self.twiddle7im, x1m16);
1616 let t_b7_2 = _mm_mul_pd(self.twiddle3im, x2m15);
1617 let t_b7_3 = _mm_mul_pd(self.twiddle4im, x3m14);
1618 let t_b7_4 = _mm_mul_pd(self.twiddle6im, x4m13);
1619 let t_b7_5 = _mm_mul_pd(self.twiddle1im, x5m12);
1620 let t_b7_6 = _mm_mul_pd(self.twiddle8im, x6m11);
1621 let t_b7_7 = _mm_mul_pd(self.twiddle2im, x7m10);
1622 let t_b7_8 = _mm_mul_pd(self.twiddle5im, x8m9);
1623 let t_b8_1 = _mm_mul_pd(self.twiddle8im, x1m16);
1624 let t_b8_2 = _mm_mul_pd(self.twiddle1im, x2m15);
1625 let t_b8_3 = _mm_mul_pd(self.twiddle7im, x3m14);
1626 let t_b8_4 = _mm_mul_pd(self.twiddle2im, x4m13);
1627 let t_b8_5 = _mm_mul_pd(self.twiddle6im, x5m12);
1628 let t_b8_6 = _mm_mul_pd(self.twiddle3im, x6m11);
1629 let t_b8_7 = _mm_mul_pd(self.twiddle5im, x7m10);
1630 let t_b8_8 = _mm_mul_pd(self.twiddle4im, x8m9);
1631
1632 let x0 = values[0];
1633 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8);
1634 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8);
1635 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8);
1636 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8);
1637 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8);
1638 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8);
1639 let t_a7 = calc_f64!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8);
1640 let t_a8 = calc_f64!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8);
1641
1642 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8);
1643 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8);
1644 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 - t_b3_3 - t_b3_4 - t_b3_5 + t_b3_6 + t_b3_7 + t_b3_8);
1645 let t_b4 = calc_f64!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 - t_b4_7 - t_b4_8);
1646 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8);
1647 let t_b6 = calc_f64!(t_b6_1 - t_b6_2 + t_b6_3 + t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8);
1648 let t_b7 = calc_f64!(t_b7_1 - t_b7_2 + t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 + t_b7_8);
1649 let t_b8 = calc_f64!(t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 + t_b8_7 - t_b8_8);
1650
1651 let t_b1_rot = self.rotate.rotate(t_b1);
1652 let t_b2_rot = self.rotate.rotate(t_b2);
1653 let t_b3_rot = self.rotate.rotate(t_b3);
1654 let t_b4_rot = self.rotate.rotate(t_b4);
1655 let t_b5_rot = self.rotate.rotate(t_b5);
1656 let t_b6_rot = self.rotate.rotate(t_b6);
1657 let t_b7_rot = self.rotate.rotate(t_b7);
1658 let t_b8_rot = self.rotate.rotate(t_b8);
1659
1660 let y0 = calc_f64!(x0 + x1p16 + x2p15 + x3p14 + x4p13 + x5p12 + x6p11 + x7p10 + x8p9);
1661 let [y1, y16] = solo_fft2_f64(t_a1, t_b1_rot);
1662 let [y2, y15] = solo_fft2_f64(t_a2, t_b2_rot);
1663 let [y3, y14] = solo_fft2_f64(t_a3, t_b3_rot);
1664 let [y4, y13] = solo_fft2_f64(t_a4, t_b4_rot);
1665 let [y5, y12] = solo_fft2_f64(t_a5, t_b5_rot);
1666 let [y6, y11] = solo_fft2_f64(t_a6, t_b6_rot);
1667 let [y7, y10] = solo_fft2_f64(t_a7, t_b7_rot);
1668 let [y8, y9] = solo_fft2_f64(t_a8, t_b8_rot);
1669 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16]
1670 }
1671 }
1672
1673 // _ ___ _________ _ _ _
1674 // / |/ _ \ |___ /___ \| |__ (_) |_
1675 // | | (_) | _____ |_ \ __) | '_ \| | __|
1676 // | |\__, | |_____| ___) / __/| |_) | | |_
1677 // |_| /_/ |____/_____|_.__/|_|\__|
1678 //
1679 pub struct SseF32Butterfly19<T> {
1680 direction: FftDirection,
1681 _phantom: std::marker::PhantomData<T>,
1682 rotate: Rotate90F32,
1683 twiddle1re: __m128,
1684 twiddle1im: __m128,
1685 twiddle2re: __m128,
1686 twiddle2im: __m128,
1687 twiddle3re: __m128,
1688 twiddle3im: __m128,
1689 twiddle4re: __m128,
1690 twiddle4im: __m128,
1691 twiddle5re: __m128,
1692 twiddle5im: __m128,
1693 twiddle6re: __m128,
1694 twiddle6im: __m128,
1695 twiddle7re: __m128,
1696 twiddle7im: __m128,
1697 twiddle8re: __m128,
1698 twiddle8im: __m128,
1699 twiddle9re: __m128,
1700 twiddle9im: __m128,
1701 }
1702
1703 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly19, 19, |this: &SseF32Butterfly19<_>| this
1704 .direction);
1705 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly19, 19, |this: &SseF32Butterfly19<_>| this
1706 .direction);
1707 impl<T: FftNum> SseF32Butterfly19<T> {
1708 #[inline(always)]
1709 pub fn new(direction: FftDirection) -> Self {
1710 assert_f32::<T>();
1711 let rotate = Rotate90F32::new(true);
1712 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 19, direction);
1713 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 19, direction);
1714 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 19, direction);
1715 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 19, direction);
1716 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 19, direction);
1717 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 19, direction);
1718 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 19, direction);
1719 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 19, direction);
1720 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 19, direction);
1721 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
1722 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
1723 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
1724 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
1725 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
1726 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
1727 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
1728 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
1729 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
1730 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
1731 let twiddle6re = unsafe { _mm_load1_ps(&tw6.re) };
1732 let twiddle6im = unsafe { _mm_load1_ps(&tw6.im) };
1733 let twiddle7re = unsafe { _mm_load1_ps(&tw7.re) };
1734 let twiddle7im = unsafe { _mm_load1_ps(&tw7.im) };
1735 let twiddle8re = unsafe { _mm_load1_ps(&tw8.re) };
1736 let twiddle8im = unsafe { _mm_load1_ps(&tw8.im) };
1737 let twiddle9re = unsafe { _mm_load1_ps(&tw9.re) };
1738 let twiddle9im = unsafe { _mm_load1_ps(&tw9.im) };
1739
1740 Self {
1741 direction,
1742 _phantom: std::marker::PhantomData,
1743 rotate,
1744 twiddle1re,
1745 twiddle1im,
1746 twiddle2re,
1747 twiddle2im,
1748 twiddle3re,
1749 twiddle3im,
1750 twiddle4re,
1751 twiddle4im,
1752 twiddle5re,
1753 twiddle5im,
1754 twiddle6re,
1755 twiddle6im,
1756 twiddle7re,
1757 twiddle7im,
1758 twiddle8re,
1759 twiddle8im,
1760 twiddle9re,
1761 twiddle9im,
1762 }
1763 }
1764
1765 #[inline(always)]
1766 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1767 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
1768
1769 let out = self.perform_parallel_fft_direct(values);
1770
1771 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
1772 }
1773
1774 #[inline(always)]
1775 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
1776 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36});
1777
1778 let values = [
1779 extract_lo_hi_f32(input_packed[0], input_packed[9]),
1780 extract_hi_lo_f32(input_packed[0], input_packed[10]),
1781 extract_lo_hi_f32(input_packed[1], input_packed[10]),
1782 extract_hi_lo_f32(input_packed[1], input_packed[11]),
1783 extract_lo_hi_f32(input_packed[2], input_packed[11]),
1784 extract_hi_lo_f32(input_packed[2], input_packed[12]),
1785 extract_lo_hi_f32(input_packed[3], input_packed[12]),
1786 extract_hi_lo_f32(input_packed[3], input_packed[13]),
1787 extract_lo_hi_f32(input_packed[4], input_packed[13]),
1788 extract_hi_lo_f32(input_packed[4], input_packed[14]),
1789 extract_lo_hi_f32(input_packed[5], input_packed[14]),
1790 extract_hi_lo_f32(input_packed[5], input_packed[15]),
1791 extract_lo_hi_f32(input_packed[6], input_packed[15]),
1792 extract_hi_lo_f32(input_packed[6], input_packed[16]),
1793 extract_lo_hi_f32(input_packed[7], input_packed[16]),
1794 extract_hi_lo_f32(input_packed[7], input_packed[17]),
1795 extract_lo_hi_f32(input_packed[8], input_packed[17]),
1796 extract_hi_lo_f32(input_packed[8], input_packed[18]),
1797 extract_lo_hi_f32(input_packed[9], input_packed[18]),
1798 ];
1799
1800 let out = self.perform_parallel_fft_direct(values);
1801
1802 let out_packed = [
1803 extract_lo_lo_f32(out[0], out[1]),
1804 extract_lo_lo_f32(out[2], out[3]),
1805 extract_lo_lo_f32(out[4], out[5]),
1806 extract_lo_lo_f32(out[6], out[7]),
1807 extract_lo_lo_f32(out[8], out[9]),
1808 extract_lo_lo_f32(out[10], out[11]),
1809 extract_lo_lo_f32(out[12], out[13]),
1810 extract_lo_lo_f32(out[14], out[15]),
1811 extract_lo_lo_f32(out[16], out[17]),
1812 extract_lo_hi_f32(out[18], out[0]),
1813 extract_hi_hi_f32(out[1], out[2]),
1814 extract_hi_hi_f32(out[3], out[4]),
1815 extract_hi_hi_f32(out[5], out[6]),
1816 extract_hi_hi_f32(out[7], out[8]),
1817 extract_hi_hi_f32(out[9], out[10]),
1818 extract_hi_hi_f32(out[11], out[12]),
1819 extract_hi_hi_f32(out[13], out[14]),
1820 extract_hi_hi_f32(out[15], out[16]),
1821 extract_hi_hi_f32(out[17], out[18]),
1822 ];
1823
1824 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
1825 }
1826
1827 #[inline(always)]
1828 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 19]) -> [__m128; 19] {
1829 let [x1p18, x1m18] = parallel_fft2_interleaved_f32(values[1], values[18]);
1830 let [x2p17, x2m17] = parallel_fft2_interleaved_f32(values[2], values[17]);
1831 let [x3p16, x3m16] = parallel_fft2_interleaved_f32(values[3], values[16]);
1832 let [x4p15, x4m15] = parallel_fft2_interleaved_f32(values[4], values[15]);
1833 let [x5p14, x5m14] = parallel_fft2_interleaved_f32(values[5], values[14]);
1834 let [x6p13, x6m13] = parallel_fft2_interleaved_f32(values[6], values[13]);
1835 let [x7p12, x7m12] = parallel_fft2_interleaved_f32(values[7], values[12]);
1836 let [x8p11, x8m11] = parallel_fft2_interleaved_f32(values[8], values[11]);
1837 let [x9p10, x9m10] = parallel_fft2_interleaved_f32(values[9], values[10]);
1838
1839 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p18);
1840 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p17);
1841 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p16);
1842 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p15);
1843 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p14);
1844 let t_a1_6 = _mm_mul_ps(self.twiddle6re, x6p13);
1845 let t_a1_7 = _mm_mul_ps(self.twiddle7re, x7p12);
1846 let t_a1_8 = _mm_mul_ps(self.twiddle8re, x8p11);
1847 let t_a1_9 = _mm_mul_ps(self.twiddle9re, x9p10);
1848 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p18);
1849 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p17);
1850 let t_a2_3 = _mm_mul_ps(self.twiddle6re, x3p16);
1851 let t_a2_4 = _mm_mul_ps(self.twiddle8re, x4p15);
1852 let t_a2_5 = _mm_mul_ps(self.twiddle9re, x5p14);
1853 let t_a2_6 = _mm_mul_ps(self.twiddle7re, x6p13);
1854 let t_a2_7 = _mm_mul_ps(self.twiddle5re, x7p12);
1855 let t_a2_8 = _mm_mul_ps(self.twiddle3re, x8p11);
1856 let t_a2_9 = _mm_mul_ps(self.twiddle1re, x9p10);
1857 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p18);
1858 let t_a3_2 = _mm_mul_ps(self.twiddle6re, x2p17);
1859 let t_a3_3 = _mm_mul_ps(self.twiddle9re, x3p16);
1860 let t_a3_4 = _mm_mul_ps(self.twiddle7re, x4p15);
1861 let t_a3_5 = _mm_mul_ps(self.twiddle4re, x5p14);
1862 let t_a3_6 = _mm_mul_ps(self.twiddle1re, x6p13);
1863 let t_a3_7 = _mm_mul_ps(self.twiddle2re, x7p12);
1864 let t_a3_8 = _mm_mul_ps(self.twiddle5re, x8p11);
1865 let t_a3_9 = _mm_mul_ps(self.twiddle8re, x9p10);
1866 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p18);
1867 let t_a4_2 = _mm_mul_ps(self.twiddle8re, x2p17);
1868 let t_a4_3 = _mm_mul_ps(self.twiddle7re, x3p16);
1869 let t_a4_4 = _mm_mul_ps(self.twiddle3re, x4p15);
1870 let t_a4_5 = _mm_mul_ps(self.twiddle1re, x5p14);
1871 let t_a4_6 = _mm_mul_ps(self.twiddle5re, x6p13);
1872 let t_a4_7 = _mm_mul_ps(self.twiddle9re, x7p12);
1873 let t_a4_8 = _mm_mul_ps(self.twiddle6re, x8p11);
1874 let t_a4_9 = _mm_mul_ps(self.twiddle2re, x9p10);
1875 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p18);
1876 let t_a5_2 = _mm_mul_ps(self.twiddle9re, x2p17);
1877 let t_a5_3 = _mm_mul_ps(self.twiddle4re, x3p16);
1878 let t_a5_4 = _mm_mul_ps(self.twiddle1re, x4p15);
1879 let t_a5_5 = _mm_mul_ps(self.twiddle6re, x5p14);
1880 let t_a5_6 = _mm_mul_ps(self.twiddle8re, x6p13);
1881 let t_a5_7 = _mm_mul_ps(self.twiddle3re, x7p12);
1882 let t_a5_8 = _mm_mul_ps(self.twiddle2re, x8p11);
1883 let t_a5_9 = _mm_mul_ps(self.twiddle7re, x9p10);
1884 let t_a6_1 = _mm_mul_ps(self.twiddle6re, x1p18);
1885 let t_a6_2 = _mm_mul_ps(self.twiddle7re, x2p17);
1886 let t_a6_3 = _mm_mul_ps(self.twiddle1re, x3p16);
1887 let t_a6_4 = _mm_mul_ps(self.twiddle5re, x4p15);
1888 let t_a6_5 = _mm_mul_ps(self.twiddle8re, x5p14);
1889 let t_a6_6 = _mm_mul_ps(self.twiddle2re, x6p13);
1890 let t_a6_7 = _mm_mul_ps(self.twiddle4re, x7p12);
1891 let t_a6_8 = _mm_mul_ps(self.twiddle9re, x8p11);
1892 let t_a6_9 = _mm_mul_ps(self.twiddle3re, x9p10);
1893 let t_a7_1 = _mm_mul_ps(self.twiddle7re, x1p18);
1894 let t_a7_2 = _mm_mul_ps(self.twiddle5re, x2p17);
1895 let t_a7_3 = _mm_mul_ps(self.twiddle2re, x3p16);
1896 let t_a7_4 = _mm_mul_ps(self.twiddle9re, x4p15);
1897 let t_a7_5 = _mm_mul_ps(self.twiddle3re, x5p14);
1898 let t_a7_6 = _mm_mul_ps(self.twiddle4re, x6p13);
1899 let t_a7_7 = _mm_mul_ps(self.twiddle8re, x7p12);
1900 let t_a7_8 = _mm_mul_ps(self.twiddle1re, x8p11);
1901 let t_a7_9 = _mm_mul_ps(self.twiddle6re, x9p10);
1902 let t_a8_1 = _mm_mul_ps(self.twiddle8re, x1p18);
1903 let t_a8_2 = _mm_mul_ps(self.twiddle3re, x2p17);
1904 let t_a8_3 = _mm_mul_ps(self.twiddle5re, x3p16);
1905 let t_a8_4 = _mm_mul_ps(self.twiddle6re, x4p15);
1906 let t_a8_5 = _mm_mul_ps(self.twiddle2re, x5p14);
1907 let t_a8_6 = _mm_mul_ps(self.twiddle9re, x6p13);
1908 let t_a8_7 = _mm_mul_ps(self.twiddle1re, x7p12);
1909 let t_a8_8 = _mm_mul_ps(self.twiddle7re, x8p11);
1910 let t_a8_9 = _mm_mul_ps(self.twiddle4re, x9p10);
1911 let t_a9_1 = _mm_mul_ps(self.twiddle9re, x1p18);
1912 let t_a9_2 = _mm_mul_ps(self.twiddle1re, x2p17);
1913 let t_a9_3 = _mm_mul_ps(self.twiddle8re, x3p16);
1914 let t_a9_4 = _mm_mul_ps(self.twiddle2re, x4p15);
1915 let t_a9_5 = _mm_mul_ps(self.twiddle7re, x5p14);
1916 let t_a9_6 = _mm_mul_ps(self.twiddle3re, x6p13);
1917 let t_a9_7 = _mm_mul_ps(self.twiddle6re, x7p12);
1918 let t_a9_8 = _mm_mul_ps(self.twiddle4re, x8p11);
1919 let t_a9_9 = _mm_mul_ps(self.twiddle5re, x9p10);
1920
1921 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m18);
1922 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m17);
1923 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m16);
1924 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m15);
1925 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m14);
1926 let t_b1_6 = _mm_mul_ps(self.twiddle6im, x6m13);
1927 let t_b1_7 = _mm_mul_ps(self.twiddle7im, x7m12);
1928 let t_b1_8 = _mm_mul_ps(self.twiddle8im, x8m11);
1929 let t_b1_9 = _mm_mul_ps(self.twiddle9im, x9m10);
1930 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m18);
1931 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m17);
1932 let t_b2_3 = _mm_mul_ps(self.twiddle6im, x3m16);
1933 let t_b2_4 = _mm_mul_ps(self.twiddle8im, x4m15);
1934 let t_b2_5 = _mm_mul_ps(self.twiddle9im, x5m14);
1935 let t_b2_6 = _mm_mul_ps(self.twiddle7im, x6m13);
1936 let t_b2_7 = _mm_mul_ps(self.twiddle5im, x7m12);
1937 let t_b2_8 = _mm_mul_ps(self.twiddle3im, x8m11);
1938 let t_b2_9 = _mm_mul_ps(self.twiddle1im, x9m10);
1939 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m18);
1940 let t_b3_2 = _mm_mul_ps(self.twiddle6im, x2m17);
1941 let t_b3_3 = _mm_mul_ps(self.twiddle9im, x3m16);
1942 let t_b3_4 = _mm_mul_ps(self.twiddle7im, x4m15);
1943 let t_b3_5 = _mm_mul_ps(self.twiddle4im, x5m14);
1944 let t_b3_6 = _mm_mul_ps(self.twiddle1im, x6m13);
1945 let t_b3_7 = _mm_mul_ps(self.twiddle2im, x7m12);
1946 let t_b3_8 = _mm_mul_ps(self.twiddle5im, x8m11);
1947 let t_b3_9 = _mm_mul_ps(self.twiddle8im, x9m10);
1948 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m18);
1949 let t_b4_2 = _mm_mul_ps(self.twiddle8im, x2m17);
1950 let t_b4_3 = _mm_mul_ps(self.twiddle7im, x3m16);
1951 let t_b4_4 = _mm_mul_ps(self.twiddle3im, x4m15);
1952 let t_b4_5 = _mm_mul_ps(self.twiddle1im, x5m14);
1953 let t_b4_6 = _mm_mul_ps(self.twiddle5im, x6m13);
1954 let t_b4_7 = _mm_mul_ps(self.twiddle9im, x7m12);
1955 let t_b4_8 = _mm_mul_ps(self.twiddle6im, x8m11);
1956 let t_b4_9 = _mm_mul_ps(self.twiddle2im, x9m10);
1957 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m18);
1958 let t_b5_2 = _mm_mul_ps(self.twiddle9im, x2m17);
1959 let t_b5_3 = _mm_mul_ps(self.twiddle4im, x3m16);
1960 let t_b5_4 = _mm_mul_ps(self.twiddle1im, x4m15);
1961 let t_b5_5 = _mm_mul_ps(self.twiddle6im, x5m14);
1962 let t_b5_6 = _mm_mul_ps(self.twiddle8im, x6m13);
1963 let t_b5_7 = _mm_mul_ps(self.twiddle3im, x7m12);
1964 let t_b5_8 = _mm_mul_ps(self.twiddle2im, x8m11);
1965 let t_b5_9 = _mm_mul_ps(self.twiddle7im, x9m10);
1966 let t_b6_1 = _mm_mul_ps(self.twiddle6im, x1m18);
1967 let t_b6_2 = _mm_mul_ps(self.twiddle7im, x2m17);
1968 let t_b6_3 = _mm_mul_ps(self.twiddle1im, x3m16);
1969 let t_b6_4 = _mm_mul_ps(self.twiddle5im, x4m15);
1970 let t_b6_5 = _mm_mul_ps(self.twiddle8im, x5m14);
1971 let t_b6_6 = _mm_mul_ps(self.twiddle2im, x6m13);
1972 let t_b6_7 = _mm_mul_ps(self.twiddle4im, x7m12);
1973 let t_b6_8 = _mm_mul_ps(self.twiddle9im, x8m11);
1974 let t_b6_9 = _mm_mul_ps(self.twiddle3im, x9m10);
1975 let t_b7_1 = _mm_mul_ps(self.twiddle7im, x1m18);
1976 let t_b7_2 = _mm_mul_ps(self.twiddle5im, x2m17);
1977 let t_b7_3 = _mm_mul_ps(self.twiddle2im, x3m16);
1978 let t_b7_4 = _mm_mul_ps(self.twiddle9im, x4m15);
1979 let t_b7_5 = _mm_mul_ps(self.twiddle3im, x5m14);
1980 let t_b7_6 = _mm_mul_ps(self.twiddle4im, x6m13);
1981 let t_b7_7 = _mm_mul_ps(self.twiddle8im, x7m12);
1982 let t_b7_8 = _mm_mul_ps(self.twiddle1im, x8m11);
1983 let t_b7_9 = _mm_mul_ps(self.twiddle6im, x9m10);
1984 let t_b8_1 = _mm_mul_ps(self.twiddle8im, x1m18);
1985 let t_b8_2 = _mm_mul_ps(self.twiddle3im, x2m17);
1986 let t_b8_3 = _mm_mul_ps(self.twiddle5im, x3m16);
1987 let t_b8_4 = _mm_mul_ps(self.twiddle6im, x4m15);
1988 let t_b8_5 = _mm_mul_ps(self.twiddle2im, x5m14);
1989 let t_b8_6 = _mm_mul_ps(self.twiddle9im, x6m13);
1990 let t_b8_7 = _mm_mul_ps(self.twiddle1im, x7m12);
1991 let t_b8_8 = _mm_mul_ps(self.twiddle7im, x8m11);
1992 let t_b8_9 = _mm_mul_ps(self.twiddle4im, x9m10);
1993 let t_b9_1 = _mm_mul_ps(self.twiddle9im, x1m18);
1994 let t_b9_2 = _mm_mul_ps(self.twiddle1im, x2m17);
1995 let t_b9_3 = _mm_mul_ps(self.twiddle8im, x3m16);
1996 let t_b9_4 = _mm_mul_ps(self.twiddle2im, x4m15);
1997 let t_b9_5 = _mm_mul_ps(self.twiddle7im, x5m14);
1998 let t_b9_6 = _mm_mul_ps(self.twiddle3im, x6m13);
1999 let t_b9_7 = _mm_mul_ps(self.twiddle6im, x7m12);
2000 let t_b9_8 = _mm_mul_ps(self.twiddle4im, x8m11);
2001 let t_b9_9 = _mm_mul_ps(self.twiddle5im, x9m10);
2002
2003 let x0 = values[0];
2004 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9);
2005 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9);
2006 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9);
2007 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9);
2008 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9);
2009 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9);
2010 let t_a7 = calc_f32!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9);
2011 let t_a8 = calc_f32!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9);
2012 let t_a9 = calc_f32!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9);
2013
2014 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9);
2015 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8 - t_b2_9);
2016 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 + t_b3_7 + t_b3_8 + t_b3_9);
2017 let t_b4 = calc_f32!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 + t_b4_7 - t_b4_8 - t_b4_9);
2018 let t_b5 = calc_f32!(t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 - t_b5_7 + t_b5_8 + t_b5_9);
2019 let t_b6 = calc_f32!(t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 - t_b6_5 - t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9);
2020 let t_b7 = calc_f32!(t_b7_1 - t_b7_2 + t_b7_3 + t_b7_4 - t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9);
2021 let t_b8 = calc_f32!(t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 - t_b8_9);
2022 let t_b9 = calc_f32!(t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 - t_b9_8 + t_b9_9);
2023
2024 let t_b1_rot = self.rotate.rotate_both(t_b1);
2025 let t_b2_rot = self.rotate.rotate_both(t_b2);
2026 let t_b3_rot = self.rotate.rotate_both(t_b3);
2027 let t_b4_rot = self.rotate.rotate_both(t_b4);
2028 let t_b5_rot = self.rotate.rotate_both(t_b5);
2029 let t_b6_rot = self.rotate.rotate_both(t_b6);
2030 let t_b7_rot = self.rotate.rotate_both(t_b7);
2031 let t_b8_rot = self.rotate.rotate_both(t_b8);
2032 let t_b9_rot = self.rotate.rotate_both(t_b9);
2033
2034 let y0 = calc_f32!(x0 + x1p18 + x2p17 + x3p16 + x4p15 + x5p14 + x6p13 + x7p12 + x8p11 + x9p10);
2035 let [y1, y18] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
2036 let [y2, y17] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
2037 let [y3, y16] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
2038 let [y4, y15] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
2039 let [y5, y14] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
2040 let [y6, y13] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
2041 let [y7, y12] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
2042 let [y8, y11] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
2043 let [y9, y10] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
2044 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18]
2045 }
2046 }
2047
2048 // _ ___ __ _ _ _ _ _
2049 // / |/ _ \ / /_ | || | | |__ (_) |_
2050 // | | (_) | _____ | '_ \| || |_| '_ \| | __|
2051 // | |\__, | |_____| | (_) |__ _| |_) | | |_
2052 // |_| /_/ \___/ |_| |_.__/|_|\__|
2053 //
2054
2055 pub struct SseF64Butterfly19<T> {
2056 direction: FftDirection,
2057 _phantom: std::marker::PhantomData<T>,
2058 rotate: Rotate90F64,
2059 twiddle1re: __m128d,
2060 twiddle1im: __m128d,
2061 twiddle2re: __m128d,
2062 twiddle2im: __m128d,
2063 twiddle3re: __m128d,
2064 twiddle3im: __m128d,
2065 twiddle4re: __m128d,
2066 twiddle4im: __m128d,
2067 twiddle5re: __m128d,
2068 twiddle5im: __m128d,
2069 twiddle6re: __m128d,
2070 twiddle6im: __m128d,
2071 twiddle7re: __m128d,
2072 twiddle7im: __m128d,
2073 twiddle8re: __m128d,
2074 twiddle8im: __m128d,
2075 twiddle9re: __m128d,
2076 twiddle9im: __m128d,
2077 }
2078
2079 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly19, 19, |this: &SseF64Butterfly19<_>| this
2080 .direction);
2081 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly19, 19, |this: &SseF64Butterfly19<_>| this
2082 .direction);
2083 impl<T: FftNum> SseF64Butterfly19<T> {
2084 #[inline(always)]
2085 pub fn new(direction: FftDirection) -> Self {
2086 assert_f64::<T>();
2087 let rotate = Rotate90F64::new(true);
2088 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 19, direction);
2089 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 19, direction);
2090 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 19, direction);
2091 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 19, direction);
2092 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 19, direction);
2093 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 19, direction);
2094 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 19, direction);
2095 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 19, direction);
2096 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 19, direction);
2097 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
2098 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
2099 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
2100 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
2101 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
2102 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
2103 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
2104 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
2105 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
2106 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
2107 let twiddle6re = unsafe { _mm_set_pd(tw6.re, tw6.re) };
2108 let twiddle6im = unsafe { _mm_set_pd(tw6.im, tw6.im) };
2109 let twiddle7re = unsafe { _mm_set_pd(tw7.re, tw7.re) };
2110 let twiddle7im = unsafe { _mm_set_pd(tw7.im, tw7.im) };
2111 let twiddle8re = unsafe { _mm_set_pd(tw8.re, tw8.re) };
2112 let twiddle8im = unsafe { _mm_set_pd(tw8.im, tw8.im) };
2113 let twiddle9re = unsafe { _mm_set_pd(tw9.re, tw9.re) };
2114 let twiddle9im = unsafe { _mm_set_pd(tw9.im, tw9.im) };
2115
2116 Self {
2117 direction,
2118 _phantom: std::marker::PhantomData,
2119 rotate,
2120 twiddle1re,
2121 twiddle1im,
2122 twiddle2re,
2123 twiddle2im,
2124 twiddle3re,
2125 twiddle3im,
2126 twiddle4re,
2127 twiddle4im,
2128 twiddle5re,
2129 twiddle5im,
2130 twiddle6re,
2131 twiddle6im,
2132 twiddle7re,
2133 twiddle7im,
2134 twiddle8re,
2135 twiddle8im,
2136 twiddle9re,
2137 twiddle9im,
2138 }
2139 }
2140
2141 #[inline(always)]
2142 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
2143 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
2144
2145 let out = self.perform_fft_direct(values);
2146
2147 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18});
2148 }
2149
2150 #[inline(always)]
2151 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 19]) -> [__m128d; 19] {
2152 let [x1p18, x1m18] = solo_fft2_f64(values[1], values[18]);
2153 let [x2p17, x2m17] = solo_fft2_f64(values[2], values[17]);
2154 let [x3p16, x3m16] = solo_fft2_f64(values[3], values[16]);
2155 let [x4p15, x4m15] = solo_fft2_f64(values[4], values[15]);
2156 let [x5p14, x5m14] = solo_fft2_f64(values[5], values[14]);
2157 let [x6p13, x6m13] = solo_fft2_f64(values[6], values[13]);
2158 let [x7p12, x7m12] = solo_fft2_f64(values[7], values[12]);
2159 let [x8p11, x8m11] = solo_fft2_f64(values[8], values[11]);
2160 let [x9p10, x9m10] = solo_fft2_f64(values[9], values[10]);
2161
2162 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p18);
2163 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p17);
2164 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p16);
2165 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p15);
2166 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p14);
2167 let t_a1_6 = _mm_mul_pd(self.twiddle6re, x6p13);
2168 let t_a1_7 = _mm_mul_pd(self.twiddle7re, x7p12);
2169 let t_a1_8 = _mm_mul_pd(self.twiddle8re, x8p11);
2170 let t_a1_9 = _mm_mul_pd(self.twiddle9re, x9p10);
2171 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p18);
2172 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p17);
2173 let t_a2_3 = _mm_mul_pd(self.twiddle6re, x3p16);
2174 let t_a2_4 = _mm_mul_pd(self.twiddle8re, x4p15);
2175 let t_a2_5 = _mm_mul_pd(self.twiddle9re, x5p14);
2176 let t_a2_6 = _mm_mul_pd(self.twiddle7re, x6p13);
2177 let t_a2_7 = _mm_mul_pd(self.twiddle5re, x7p12);
2178 let t_a2_8 = _mm_mul_pd(self.twiddle3re, x8p11);
2179 let t_a2_9 = _mm_mul_pd(self.twiddle1re, x9p10);
2180 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p18);
2181 let t_a3_2 = _mm_mul_pd(self.twiddle6re, x2p17);
2182 let t_a3_3 = _mm_mul_pd(self.twiddle9re, x3p16);
2183 let t_a3_4 = _mm_mul_pd(self.twiddle7re, x4p15);
2184 let t_a3_5 = _mm_mul_pd(self.twiddle4re, x5p14);
2185 let t_a3_6 = _mm_mul_pd(self.twiddle1re, x6p13);
2186 let t_a3_7 = _mm_mul_pd(self.twiddle2re, x7p12);
2187 let t_a3_8 = _mm_mul_pd(self.twiddle5re, x8p11);
2188 let t_a3_9 = _mm_mul_pd(self.twiddle8re, x9p10);
2189 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p18);
2190 let t_a4_2 = _mm_mul_pd(self.twiddle8re, x2p17);
2191 let t_a4_3 = _mm_mul_pd(self.twiddle7re, x3p16);
2192 let t_a4_4 = _mm_mul_pd(self.twiddle3re, x4p15);
2193 let t_a4_5 = _mm_mul_pd(self.twiddle1re, x5p14);
2194 let t_a4_6 = _mm_mul_pd(self.twiddle5re, x6p13);
2195 let t_a4_7 = _mm_mul_pd(self.twiddle9re, x7p12);
2196 let t_a4_8 = _mm_mul_pd(self.twiddle6re, x8p11);
2197 let t_a4_9 = _mm_mul_pd(self.twiddle2re, x9p10);
2198 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p18);
2199 let t_a5_2 = _mm_mul_pd(self.twiddle9re, x2p17);
2200 let t_a5_3 = _mm_mul_pd(self.twiddle4re, x3p16);
2201 let t_a5_4 = _mm_mul_pd(self.twiddle1re, x4p15);
2202 let t_a5_5 = _mm_mul_pd(self.twiddle6re, x5p14);
2203 let t_a5_6 = _mm_mul_pd(self.twiddle8re, x6p13);
2204 let t_a5_7 = _mm_mul_pd(self.twiddle3re, x7p12);
2205 let t_a5_8 = _mm_mul_pd(self.twiddle2re, x8p11);
2206 let t_a5_9 = _mm_mul_pd(self.twiddle7re, x9p10);
2207 let t_a6_1 = _mm_mul_pd(self.twiddle6re, x1p18);
2208 let t_a6_2 = _mm_mul_pd(self.twiddle7re, x2p17);
2209 let t_a6_3 = _mm_mul_pd(self.twiddle1re, x3p16);
2210 let t_a6_4 = _mm_mul_pd(self.twiddle5re, x4p15);
2211 let t_a6_5 = _mm_mul_pd(self.twiddle8re, x5p14);
2212 let t_a6_6 = _mm_mul_pd(self.twiddle2re, x6p13);
2213 let t_a6_7 = _mm_mul_pd(self.twiddle4re, x7p12);
2214 let t_a6_8 = _mm_mul_pd(self.twiddle9re, x8p11);
2215 let t_a6_9 = _mm_mul_pd(self.twiddle3re, x9p10);
2216 let t_a7_1 = _mm_mul_pd(self.twiddle7re, x1p18);
2217 let t_a7_2 = _mm_mul_pd(self.twiddle5re, x2p17);
2218 let t_a7_3 = _mm_mul_pd(self.twiddle2re, x3p16);
2219 let t_a7_4 = _mm_mul_pd(self.twiddle9re, x4p15);
2220 let t_a7_5 = _mm_mul_pd(self.twiddle3re, x5p14);
2221 let t_a7_6 = _mm_mul_pd(self.twiddle4re, x6p13);
2222 let t_a7_7 = _mm_mul_pd(self.twiddle8re, x7p12);
2223 let t_a7_8 = _mm_mul_pd(self.twiddle1re, x8p11);
2224 let t_a7_9 = _mm_mul_pd(self.twiddle6re, x9p10);
2225 let t_a8_1 = _mm_mul_pd(self.twiddle8re, x1p18);
2226 let t_a8_2 = _mm_mul_pd(self.twiddle3re, x2p17);
2227 let t_a8_3 = _mm_mul_pd(self.twiddle5re, x3p16);
2228 let t_a8_4 = _mm_mul_pd(self.twiddle6re, x4p15);
2229 let t_a8_5 = _mm_mul_pd(self.twiddle2re, x5p14);
2230 let t_a8_6 = _mm_mul_pd(self.twiddle9re, x6p13);
2231 let t_a8_7 = _mm_mul_pd(self.twiddle1re, x7p12);
2232 let t_a8_8 = _mm_mul_pd(self.twiddle7re, x8p11);
2233 let t_a8_9 = _mm_mul_pd(self.twiddle4re, x9p10);
2234 let t_a9_1 = _mm_mul_pd(self.twiddle9re, x1p18);
2235 let t_a9_2 = _mm_mul_pd(self.twiddle1re, x2p17);
2236 let t_a9_3 = _mm_mul_pd(self.twiddle8re, x3p16);
2237 let t_a9_4 = _mm_mul_pd(self.twiddle2re, x4p15);
2238 let t_a9_5 = _mm_mul_pd(self.twiddle7re, x5p14);
2239 let t_a9_6 = _mm_mul_pd(self.twiddle3re, x6p13);
2240 let t_a9_7 = _mm_mul_pd(self.twiddle6re, x7p12);
2241 let t_a9_8 = _mm_mul_pd(self.twiddle4re, x8p11);
2242 let t_a9_9 = _mm_mul_pd(self.twiddle5re, x9p10);
2243
2244 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m18);
2245 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m17);
2246 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m16);
2247 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m15);
2248 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m14);
2249 let t_b1_6 = _mm_mul_pd(self.twiddle6im, x6m13);
2250 let t_b1_7 = _mm_mul_pd(self.twiddle7im, x7m12);
2251 let t_b1_8 = _mm_mul_pd(self.twiddle8im, x8m11);
2252 let t_b1_9 = _mm_mul_pd(self.twiddle9im, x9m10);
2253 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m18);
2254 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m17);
2255 let t_b2_3 = _mm_mul_pd(self.twiddle6im, x3m16);
2256 let t_b2_4 = _mm_mul_pd(self.twiddle8im, x4m15);
2257 let t_b2_5 = _mm_mul_pd(self.twiddle9im, x5m14);
2258 let t_b2_6 = _mm_mul_pd(self.twiddle7im, x6m13);
2259 let t_b2_7 = _mm_mul_pd(self.twiddle5im, x7m12);
2260 let t_b2_8 = _mm_mul_pd(self.twiddle3im, x8m11);
2261 let t_b2_9 = _mm_mul_pd(self.twiddle1im, x9m10);
2262 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m18);
2263 let t_b3_2 = _mm_mul_pd(self.twiddle6im, x2m17);
2264 let t_b3_3 = _mm_mul_pd(self.twiddle9im, x3m16);
2265 let t_b3_4 = _mm_mul_pd(self.twiddle7im, x4m15);
2266 let t_b3_5 = _mm_mul_pd(self.twiddle4im, x5m14);
2267 let t_b3_6 = _mm_mul_pd(self.twiddle1im, x6m13);
2268 let t_b3_7 = _mm_mul_pd(self.twiddle2im, x7m12);
2269 let t_b3_8 = _mm_mul_pd(self.twiddle5im, x8m11);
2270 let t_b3_9 = _mm_mul_pd(self.twiddle8im, x9m10);
2271 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m18);
2272 let t_b4_2 = _mm_mul_pd(self.twiddle8im, x2m17);
2273 let t_b4_3 = _mm_mul_pd(self.twiddle7im, x3m16);
2274 let t_b4_4 = _mm_mul_pd(self.twiddle3im, x4m15);
2275 let t_b4_5 = _mm_mul_pd(self.twiddle1im, x5m14);
2276 let t_b4_6 = _mm_mul_pd(self.twiddle5im, x6m13);
2277 let t_b4_7 = _mm_mul_pd(self.twiddle9im, x7m12);
2278 let t_b4_8 = _mm_mul_pd(self.twiddle6im, x8m11);
2279 let t_b4_9 = _mm_mul_pd(self.twiddle2im, x9m10);
2280 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m18);
2281 let t_b5_2 = _mm_mul_pd(self.twiddle9im, x2m17);
2282 let t_b5_3 = _mm_mul_pd(self.twiddle4im, x3m16);
2283 let t_b5_4 = _mm_mul_pd(self.twiddle1im, x4m15);
2284 let t_b5_5 = _mm_mul_pd(self.twiddle6im, x5m14);
2285 let t_b5_6 = _mm_mul_pd(self.twiddle8im, x6m13);
2286 let t_b5_7 = _mm_mul_pd(self.twiddle3im, x7m12);
2287 let t_b5_8 = _mm_mul_pd(self.twiddle2im, x8m11);
2288 let t_b5_9 = _mm_mul_pd(self.twiddle7im, x9m10);
2289 let t_b6_1 = _mm_mul_pd(self.twiddle6im, x1m18);
2290 let t_b6_2 = _mm_mul_pd(self.twiddle7im, x2m17);
2291 let t_b6_3 = _mm_mul_pd(self.twiddle1im, x3m16);
2292 let t_b6_4 = _mm_mul_pd(self.twiddle5im, x4m15);
2293 let t_b6_5 = _mm_mul_pd(self.twiddle8im, x5m14);
2294 let t_b6_6 = _mm_mul_pd(self.twiddle2im, x6m13);
2295 let t_b6_7 = _mm_mul_pd(self.twiddle4im, x7m12);
2296 let t_b6_8 = _mm_mul_pd(self.twiddle9im, x8m11);
2297 let t_b6_9 = _mm_mul_pd(self.twiddle3im, x9m10);
2298 let t_b7_1 = _mm_mul_pd(self.twiddle7im, x1m18);
2299 let t_b7_2 = _mm_mul_pd(self.twiddle5im, x2m17);
2300 let t_b7_3 = _mm_mul_pd(self.twiddle2im, x3m16);
2301 let t_b7_4 = _mm_mul_pd(self.twiddle9im, x4m15);
2302 let t_b7_5 = _mm_mul_pd(self.twiddle3im, x5m14);
2303 let t_b7_6 = _mm_mul_pd(self.twiddle4im, x6m13);
2304 let t_b7_7 = _mm_mul_pd(self.twiddle8im, x7m12);
2305 let t_b7_8 = _mm_mul_pd(self.twiddle1im, x8m11);
2306 let t_b7_9 = _mm_mul_pd(self.twiddle6im, x9m10);
2307 let t_b8_1 = _mm_mul_pd(self.twiddle8im, x1m18);
2308 let t_b8_2 = _mm_mul_pd(self.twiddle3im, x2m17);
2309 let t_b8_3 = _mm_mul_pd(self.twiddle5im, x3m16);
2310 let t_b8_4 = _mm_mul_pd(self.twiddle6im, x4m15);
2311 let t_b8_5 = _mm_mul_pd(self.twiddle2im, x5m14);
2312 let t_b8_6 = _mm_mul_pd(self.twiddle9im, x6m13);
2313 let t_b8_7 = _mm_mul_pd(self.twiddle1im, x7m12);
2314 let t_b8_8 = _mm_mul_pd(self.twiddle7im, x8m11);
2315 let t_b8_9 = _mm_mul_pd(self.twiddle4im, x9m10);
2316 let t_b9_1 = _mm_mul_pd(self.twiddle9im, x1m18);
2317 let t_b9_2 = _mm_mul_pd(self.twiddle1im, x2m17);
2318 let t_b9_3 = _mm_mul_pd(self.twiddle8im, x3m16);
2319 let t_b9_4 = _mm_mul_pd(self.twiddle2im, x4m15);
2320 let t_b9_5 = _mm_mul_pd(self.twiddle7im, x5m14);
2321 let t_b9_6 = _mm_mul_pd(self.twiddle3im, x6m13);
2322 let t_b9_7 = _mm_mul_pd(self.twiddle6im, x7m12);
2323 let t_b9_8 = _mm_mul_pd(self.twiddle4im, x8m11);
2324 let t_b9_9 = _mm_mul_pd(self.twiddle5im, x9m10);
2325
2326 let x0 = values[0];
2327 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9);
2328 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9);
2329 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9);
2330 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9);
2331 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9);
2332 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9);
2333 let t_a7 = calc_f64!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9);
2334 let t_a8 = calc_f64!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9);
2335 let t_a9 = calc_f64!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9);
2336
2337 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9);
2338 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 - t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8 - t_b2_9);
2339 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 + t_b3_7 + t_b3_8 + t_b3_9);
2340 let t_b4 = calc_f64!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 + t_b4_5 + t_b4_6 + t_b4_7 - t_b4_8 - t_b4_9);
2341 let t_b5 = calc_f64!(t_b5_1 - t_b5_2 - t_b5_3 + t_b5_4 + t_b5_5 - t_b5_6 - t_b5_7 + t_b5_8 + t_b5_9);
2342 let t_b6 = calc_f64!(t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 - t_b6_5 - t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9);
2343 let t_b7 = calc_f64!(t_b7_1 - t_b7_2 + t_b7_3 + t_b7_4 - t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9);
2344 let t_b8 = calc_f64!(t_b8_1 - t_b8_2 + t_b8_3 - t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 - t_b8_9);
2345 let t_b9 = calc_f64!(t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 - t_b9_8 + t_b9_9);
2346
2347 let t_b1_rot = self.rotate.rotate(t_b1);
2348 let t_b2_rot = self.rotate.rotate(t_b2);
2349 let t_b3_rot = self.rotate.rotate(t_b3);
2350 let t_b4_rot = self.rotate.rotate(t_b4);
2351 let t_b5_rot = self.rotate.rotate(t_b5);
2352 let t_b6_rot = self.rotate.rotate(t_b6);
2353 let t_b7_rot = self.rotate.rotate(t_b7);
2354 let t_b8_rot = self.rotate.rotate(t_b8);
2355 let t_b9_rot = self.rotate.rotate(t_b9);
2356
2357 let y0 = calc_f64!(x0 + x1p18 + x2p17 + x3p16 + x4p15 + x5p14 + x6p13 + x7p12 + x8p11 + x9p10);
2358 let [y1, y18] = solo_fft2_f64(t_a1, t_b1_rot);
2359 let [y2, y17] = solo_fft2_f64(t_a2, t_b2_rot);
2360 let [y3, y16] = solo_fft2_f64(t_a3, t_b3_rot);
2361 let [y4, y15] = solo_fft2_f64(t_a4, t_b4_rot);
2362 let [y5, y14] = solo_fft2_f64(t_a5, t_b5_rot);
2363 let [y6, y13] = solo_fft2_f64(t_a6, t_b6_rot);
2364 let [y7, y12] = solo_fft2_f64(t_a7, t_b7_rot);
2365 let [y8, y11] = solo_fft2_f64(t_a8, t_b8_rot);
2366 let [y9, y10] = solo_fft2_f64(t_a9, t_b9_rot);
2367 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18]
2368 }
2369 }
2370
2371 // ____ _____ _________ _ _ _
2372 // |___ \|___ / |___ /___ \| |__ (_) |_
2373 // __) | |_ \ _____ |_ \ __) | '_ \| | __|
2374 // / __/ ___) | |_____| ___) / __/| |_) | | |_
2375 // |_____|____/ |____/_____|_.__/|_|\__|
2376 //
2377
2378 pub struct SseF32Butterfly23<T> {
2379 direction: FftDirection,
2380 _phantom: std::marker::PhantomData<T>,
2381 rotate: Rotate90F32,
2382 twiddle1re: __m128,
2383 twiddle1im: __m128,
2384 twiddle2re: __m128,
2385 twiddle2im: __m128,
2386 twiddle3re: __m128,
2387 twiddle3im: __m128,
2388 twiddle4re: __m128,
2389 twiddle4im: __m128,
2390 twiddle5re: __m128,
2391 twiddle5im: __m128,
2392 twiddle6re: __m128,
2393 twiddle6im: __m128,
2394 twiddle7re: __m128,
2395 twiddle7im: __m128,
2396 twiddle8re: __m128,
2397 twiddle8im: __m128,
2398 twiddle9re: __m128,
2399 twiddle9im: __m128,
2400 twiddle10re: __m128,
2401 twiddle10im: __m128,
2402 twiddle11re: __m128,
2403 twiddle11im: __m128,
2404 }
2405
2406 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly23, 23, |this: &SseF32Butterfly23<_>| this
2407 .direction);
2408 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly23, 23, |this: &SseF32Butterfly23<_>| this
2409 .direction);
2410 impl<T: FftNum> SseF32Butterfly23<T> {
2411 #[inline(always)]
2412 pub fn new(direction: FftDirection) -> Self {
2413 assert_f32::<T>();
2414 let rotate = Rotate90F32::new(true);
2415 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 23, direction);
2416 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 23, direction);
2417 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 23, direction);
2418 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 23, direction);
2419 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 23, direction);
2420 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 23, direction);
2421 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 23, direction);
2422 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 23, direction);
2423 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 23, direction);
2424 let tw10: Complex<f32> = twiddles::compute_twiddle(10, 23, direction);
2425 let tw11: Complex<f32> = twiddles::compute_twiddle(11, 23, direction);
2426 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
2427 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
2428 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
2429 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
2430 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
2431 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
2432 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
2433 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
2434 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
2435 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
2436 let twiddle6re = unsafe { _mm_load1_ps(&tw6.re) };
2437 let twiddle6im = unsafe { _mm_load1_ps(&tw6.im) };
2438 let twiddle7re = unsafe { _mm_load1_ps(&tw7.re) };
2439 let twiddle7im = unsafe { _mm_load1_ps(&tw7.im) };
2440 let twiddle8re = unsafe { _mm_load1_ps(&tw8.re) };
2441 let twiddle8im = unsafe { _mm_load1_ps(&tw8.im) };
2442 let twiddle9re = unsafe { _mm_load1_ps(&tw9.re) };
2443 let twiddle9im = unsafe { _mm_load1_ps(&tw9.im) };
2444 let twiddle10re = unsafe { _mm_load1_ps(&tw10.re) };
2445 let twiddle10im = unsafe { _mm_load1_ps(&tw10.im) };
2446 let twiddle11re = unsafe { _mm_load1_ps(&tw11.re) };
2447 let twiddle11im = unsafe { _mm_load1_ps(&tw11.im) };
2448
2449 Self {
2450 direction,
2451 _phantom: std::marker::PhantomData,
2452 rotate,
2453 twiddle1re,
2454 twiddle1im,
2455 twiddle2re,
2456 twiddle2im,
2457 twiddle3re,
2458 twiddle3im,
2459 twiddle4re,
2460 twiddle4im,
2461 twiddle5re,
2462 twiddle5im,
2463 twiddle6re,
2464 twiddle6im,
2465 twiddle7re,
2466 twiddle7im,
2467 twiddle8re,
2468 twiddle8im,
2469 twiddle9re,
2470 twiddle9im,
2471 twiddle10re,
2472 twiddle10im,
2473 twiddle11re,
2474 twiddle11im,
2475 }
2476 }
2477
2478 #[inline(always)]
2479 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
2480 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2481
2482 let out = self.perform_parallel_fft_direct(values);
2483
2484 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2485 }
2486
2487 #[inline(always)]
2488 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
2489 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44});
2490
2491 let values = [
2492 extract_lo_hi_f32(input_packed[0], input_packed[11]),
2493 extract_hi_lo_f32(input_packed[0], input_packed[12]),
2494 extract_lo_hi_f32(input_packed[1], input_packed[12]),
2495 extract_hi_lo_f32(input_packed[1], input_packed[13]),
2496 extract_lo_hi_f32(input_packed[2], input_packed[13]),
2497 extract_hi_lo_f32(input_packed[2], input_packed[14]),
2498 extract_lo_hi_f32(input_packed[3], input_packed[14]),
2499 extract_hi_lo_f32(input_packed[3], input_packed[15]),
2500 extract_lo_hi_f32(input_packed[4], input_packed[15]),
2501 extract_hi_lo_f32(input_packed[4], input_packed[16]),
2502 extract_lo_hi_f32(input_packed[5], input_packed[16]),
2503 extract_hi_lo_f32(input_packed[5], input_packed[17]),
2504 extract_lo_hi_f32(input_packed[6], input_packed[17]),
2505 extract_hi_lo_f32(input_packed[6], input_packed[18]),
2506 extract_lo_hi_f32(input_packed[7], input_packed[18]),
2507 extract_hi_lo_f32(input_packed[7], input_packed[19]),
2508 extract_lo_hi_f32(input_packed[8], input_packed[19]),
2509 extract_hi_lo_f32(input_packed[8], input_packed[20]),
2510 extract_lo_hi_f32(input_packed[9], input_packed[20]),
2511 extract_hi_lo_f32(input_packed[9], input_packed[21]),
2512 extract_lo_hi_f32(input_packed[10], input_packed[21]),
2513 extract_hi_lo_f32(input_packed[10], input_packed[22]),
2514 extract_lo_hi_f32(input_packed[11], input_packed[22]),
2515 ];
2516
2517 let out = self.perform_parallel_fft_direct(values);
2518
2519 let out_packed = [
2520 extract_lo_lo_f32(out[0], out[1]),
2521 extract_lo_lo_f32(out[2], out[3]),
2522 extract_lo_lo_f32(out[4], out[5]),
2523 extract_lo_lo_f32(out[6], out[7]),
2524 extract_lo_lo_f32(out[8], out[9]),
2525 extract_lo_lo_f32(out[10], out[11]),
2526 extract_lo_lo_f32(out[12], out[13]),
2527 extract_lo_lo_f32(out[14], out[15]),
2528 extract_lo_lo_f32(out[16], out[17]),
2529 extract_lo_lo_f32(out[18], out[19]),
2530 extract_lo_lo_f32(out[20], out[21]),
2531 extract_lo_hi_f32(out[22], out[0]),
2532 extract_hi_hi_f32(out[1], out[2]),
2533 extract_hi_hi_f32(out[3], out[4]),
2534 extract_hi_hi_f32(out[5], out[6]),
2535 extract_hi_hi_f32(out[7], out[8]),
2536 extract_hi_hi_f32(out[9], out[10]),
2537 extract_hi_hi_f32(out[11], out[12]),
2538 extract_hi_hi_f32(out[13], out[14]),
2539 extract_hi_hi_f32(out[15], out[16]),
2540 extract_hi_hi_f32(out[17], out[18]),
2541 extract_hi_hi_f32(out[19], out[20]),
2542 extract_hi_hi_f32(out[21], out[22]),
2543 ];
2544
2545 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2546 }
2547
2548 #[inline(always)]
2549 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 23]) -> [__m128; 23] {
2550 let [x1p22, x1m22] = parallel_fft2_interleaved_f32(values[1], values[22]);
2551 let [x2p21, x2m21] = parallel_fft2_interleaved_f32(values[2], values[21]);
2552 let [x3p20, x3m20] = parallel_fft2_interleaved_f32(values[3], values[20]);
2553 let [x4p19, x4m19] = parallel_fft2_interleaved_f32(values[4], values[19]);
2554 let [x5p18, x5m18] = parallel_fft2_interleaved_f32(values[5], values[18]);
2555 let [x6p17, x6m17] = parallel_fft2_interleaved_f32(values[6], values[17]);
2556 let [x7p16, x7m16] = parallel_fft2_interleaved_f32(values[7], values[16]);
2557 let [x8p15, x8m15] = parallel_fft2_interleaved_f32(values[8], values[15]);
2558 let [x9p14, x9m14] = parallel_fft2_interleaved_f32(values[9], values[14]);
2559 let [x10p13, x10m13] = parallel_fft2_interleaved_f32(values[10], values[13]);
2560 let [x11p12, x11m12] = parallel_fft2_interleaved_f32(values[11], values[12]);
2561
2562 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p22);
2563 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p21);
2564 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p20);
2565 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p19);
2566 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p18);
2567 let t_a1_6 = _mm_mul_ps(self.twiddle6re, x6p17);
2568 let t_a1_7 = _mm_mul_ps(self.twiddle7re, x7p16);
2569 let t_a1_8 = _mm_mul_ps(self.twiddle8re, x8p15);
2570 let t_a1_9 = _mm_mul_ps(self.twiddle9re, x9p14);
2571 let t_a1_10 = _mm_mul_ps(self.twiddle10re, x10p13);
2572 let t_a1_11 = _mm_mul_ps(self.twiddle11re, x11p12);
2573 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p22);
2574 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p21);
2575 let t_a2_3 = _mm_mul_ps(self.twiddle6re, x3p20);
2576 let t_a2_4 = _mm_mul_ps(self.twiddle8re, x4p19);
2577 let t_a2_5 = _mm_mul_ps(self.twiddle10re, x5p18);
2578 let t_a2_6 = _mm_mul_ps(self.twiddle11re, x6p17);
2579 let t_a2_7 = _mm_mul_ps(self.twiddle9re, x7p16);
2580 let t_a2_8 = _mm_mul_ps(self.twiddle7re, x8p15);
2581 let t_a2_9 = _mm_mul_ps(self.twiddle5re, x9p14);
2582 let t_a2_10 = _mm_mul_ps(self.twiddle3re, x10p13);
2583 let t_a2_11 = _mm_mul_ps(self.twiddle1re, x11p12);
2584 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p22);
2585 let t_a3_2 = _mm_mul_ps(self.twiddle6re, x2p21);
2586 let t_a3_3 = _mm_mul_ps(self.twiddle9re, x3p20);
2587 let t_a3_4 = _mm_mul_ps(self.twiddle11re, x4p19);
2588 let t_a3_5 = _mm_mul_ps(self.twiddle8re, x5p18);
2589 let t_a3_6 = _mm_mul_ps(self.twiddle5re, x6p17);
2590 let t_a3_7 = _mm_mul_ps(self.twiddle2re, x7p16);
2591 let t_a3_8 = _mm_mul_ps(self.twiddle1re, x8p15);
2592 let t_a3_9 = _mm_mul_ps(self.twiddle4re, x9p14);
2593 let t_a3_10 = _mm_mul_ps(self.twiddle7re, x10p13);
2594 let t_a3_11 = _mm_mul_ps(self.twiddle10re, x11p12);
2595 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p22);
2596 let t_a4_2 = _mm_mul_ps(self.twiddle8re, x2p21);
2597 let t_a4_3 = _mm_mul_ps(self.twiddle11re, x3p20);
2598 let t_a4_4 = _mm_mul_ps(self.twiddle7re, x4p19);
2599 let t_a4_5 = _mm_mul_ps(self.twiddle3re, x5p18);
2600 let t_a4_6 = _mm_mul_ps(self.twiddle1re, x6p17);
2601 let t_a4_7 = _mm_mul_ps(self.twiddle5re, x7p16);
2602 let t_a4_8 = _mm_mul_ps(self.twiddle9re, x8p15);
2603 let t_a4_9 = _mm_mul_ps(self.twiddle10re, x9p14);
2604 let t_a4_10 = _mm_mul_ps(self.twiddle6re, x10p13);
2605 let t_a4_11 = _mm_mul_ps(self.twiddle2re, x11p12);
2606 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p22);
2607 let t_a5_2 = _mm_mul_ps(self.twiddle10re, x2p21);
2608 let t_a5_3 = _mm_mul_ps(self.twiddle8re, x3p20);
2609 let t_a5_4 = _mm_mul_ps(self.twiddle3re, x4p19);
2610 let t_a5_5 = _mm_mul_ps(self.twiddle2re, x5p18);
2611 let t_a5_6 = _mm_mul_ps(self.twiddle7re, x6p17);
2612 let t_a5_7 = _mm_mul_ps(self.twiddle11re, x7p16);
2613 let t_a5_8 = _mm_mul_ps(self.twiddle6re, x8p15);
2614 let t_a5_9 = _mm_mul_ps(self.twiddle1re, x9p14);
2615 let t_a5_10 = _mm_mul_ps(self.twiddle4re, x10p13);
2616 let t_a5_11 = _mm_mul_ps(self.twiddle9re, x11p12);
2617 let t_a6_1 = _mm_mul_ps(self.twiddle6re, x1p22);
2618 let t_a6_2 = _mm_mul_ps(self.twiddle11re, x2p21);
2619 let t_a6_3 = _mm_mul_ps(self.twiddle5re, x3p20);
2620 let t_a6_4 = _mm_mul_ps(self.twiddle1re, x4p19);
2621 let t_a6_5 = _mm_mul_ps(self.twiddle7re, x5p18);
2622 let t_a6_6 = _mm_mul_ps(self.twiddle10re, x6p17);
2623 let t_a6_7 = _mm_mul_ps(self.twiddle4re, x7p16);
2624 let t_a6_8 = _mm_mul_ps(self.twiddle2re, x8p15);
2625 let t_a6_9 = _mm_mul_ps(self.twiddle8re, x9p14);
2626 let t_a6_10 = _mm_mul_ps(self.twiddle9re, x10p13);
2627 let t_a6_11 = _mm_mul_ps(self.twiddle3re, x11p12);
2628 let t_a7_1 = _mm_mul_ps(self.twiddle7re, x1p22);
2629 let t_a7_2 = _mm_mul_ps(self.twiddle9re, x2p21);
2630 let t_a7_3 = _mm_mul_ps(self.twiddle2re, x3p20);
2631 let t_a7_4 = _mm_mul_ps(self.twiddle5re, x4p19);
2632 let t_a7_5 = _mm_mul_ps(self.twiddle11re, x5p18);
2633 let t_a7_6 = _mm_mul_ps(self.twiddle4re, x6p17);
2634 let t_a7_7 = _mm_mul_ps(self.twiddle3re, x7p16);
2635 let t_a7_8 = _mm_mul_ps(self.twiddle10re, x8p15);
2636 let t_a7_9 = _mm_mul_ps(self.twiddle6re, x9p14);
2637 let t_a7_10 = _mm_mul_ps(self.twiddle1re, x10p13);
2638 let t_a7_11 = _mm_mul_ps(self.twiddle8re, x11p12);
2639 let t_a8_1 = _mm_mul_ps(self.twiddle8re, x1p22);
2640 let t_a8_2 = _mm_mul_ps(self.twiddle7re, x2p21);
2641 let t_a8_3 = _mm_mul_ps(self.twiddle1re, x3p20);
2642 let t_a8_4 = _mm_mul_ps(self.twiddle9re, x4p19);
2643 let t_a8_5 = _mm_mul_ps(self.twiddle6re, x5p18);
2644 let t_a8_6 = _mm_mul_ps(self.twiddle2re, x6p17);
2645 let t_a8_7 = _mm_mul_ps(self.twiddle10re, x7p16);
2646 let t_a8_8 = _mm_mul_ps(self.twiddle5re, x8p15);
2647 let t_a8_9 = _mm_mul_ps(self.twiddle3re, x9p14);
2648 let t_a8_10 = _mm_mul_ps(self.twiddle11re, x10p13);
2649 let t_a8_11 = _mm_mul_ps(self.twiddle4re, x11p12);
2650 let t_a9_1 = _mm_mul_ps(self.twiddle9re, x1p22);
2651 let t_a9_2 = _mm_mul_ps(self.twiddle5re, x2p21);
2652 let t_a9_3 = _mm_mul_ps(self.twiddle4re, x3p20);
2653 let t_a9_4 = _mm_mul_ps(self.twiddle10re, x4p19);
2654 let t_a9_5 = _mm_mul_ps(self.twiddle1re, x5p18);
2655 let t_a9_6 = _mm_mul_ps(self.twiddle8re, x6p17);
2656 let t_a9_7 = _mm_mul_ps(self.twiddle6re, x7p16);
2657 let t_a9_8 = _mm_mul_ps(self.twiddle3re, x8p15);
2658 let t_a9_9 = _mm_mul_ps(self.twiddle11re, x9p14);
2659 let t_a9_10 = _mm_mul_ps(self.twiddle2re, x10p13);
2660 let t_a9_11 = _mm_mul_ps(self.twiddle7re, x11p12);
2661 let t_a10_1 = _mm_mul_ps(self.twiddle10re, x1p22);
2662 let t_a10_2 = _mm_mul_ps(self.twiddle3re, x2p21);
2663 let t_a10_3 = _mm_mul_ps(self.twiddle7re, x3p20);
2664 let t_a10_4 = _mm_mul_ps(self.twiddle6re, x4p19);
2665 let t_a10_5 = _mm_mul_ps(self.twiddle4re, x5p18);
2666 let t_a10_6 = _mm_mul_ps(self.twiddle9re, x6p17);
2667 let t_a10_7 = _mm_mul_ps(self.twiddle1re, x7p16);
2668 let t_a10_8 = _mm_mul_ps(self.twiddle11re, x8p15);
2669 let t_a10_9 = _mm_mul_ps(self.twiddle2re, x9p14);
2670 let t_a10_10 = _mm_mul_ps(self.twiddle8re, x10p13);
2671 let t_a10_11 = _mm_mul_ps(self.twiddle5re, x11p12);
2672 let t_a11_1 = _mm_mul_ps(self.twiddle11re, x1p22);
2673 let t_a11_2 = _mm_mul_ps(self.twiddle1re, x2p21);
2674 let t_a11_3 = _mm_mul_ps(self.twiddle10re, x3p20);
2675 let t_a11_4 = _mm_mul_ps(self.twiddle2re, x4p19);
2676 let t_a11_5 = _mm_mul_ps(self.twiddle9re, x5p18);
2677 let t_a11_6 = _mm_mul_ps(self.twiddle3re, x6p17);
2678 let t_a11_7 = _mm_mul_ps(self.twiddle8re, x7p16);
2679 let t_a11_8 = _mm_mul_ps(self.twiddle4re, x8p15);
2680 let t_a11_9 = _mm_mul_ps(self.twiddle7re, x9p14);
2681 let t_a11_10 = _mm_mul_ps(self.twiddle5re, x10p13);
2682 let t_a11_11 = _mm_mul_ps(self.twiddle6re, x11p12);
2683
2684 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m22);
2685 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m21);
2686 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m20);
2687 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m19);
2688 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m18);
2689 let t_b1_6 = _mm_mul_ps(self.twiddle6im, x6m17);
2690 let t_b1_7 = _mm_mul_ps(self.twiddle7im, x7m16);
2691 let t_b1_8 = _mm_mul_ps(self.twiddle8im, x8m15);
2692 let t_b1_9 = _mm_mul_ps(self.twiddle9im, x9m14);
2693 let t_b1_10 = _mm_mul_ps(self.twiddle10im, x10m13);
2694 let t_b1_11 = _mm_mul_ps(self.twiddle11im, x11m12);
2695 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m22);
2696 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m21);
2697 let t_b2_3 = _mm_mul_ps(self.twiddle6im, x3m20);
2698 let t_b2_4 = _mm_mul_ps(self.twiddle8im, x4m19);
2699 let t_b2_5 = _mm_mul_ps(self.twiddle10im, x5m18);
2700 let t_b2_6 = _mm_mul_ps(self.twiddle11im, x6m17);
2701 let t_b2_7 = _mm_mul_ps(self.twiddle9im, x7m16);
2702 let t_b2_8 = _mm_mul_ps(self.twiddle7im, x8m15);
2703 let t_b2_9 = _mm_mul_ps(self.twiddle5im, x9m14);
2704 let t_b2_10 = _mm_mul_ps(self.twiddle3im, x10m13);
2705 let t_b2_11 = _mm_mul_ps(self.twiddle1im, x11m12);
2706 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m22);
2707 let t_b3_2 = _mm_mul_ps(self.twiddle6im, x2m21);
2708 let t_b3_3 = _mm_mul_ps(self.twiddle9im, x3m20);
2709 let t_b3_4 = _mm_mul_ps(self.twiddle11im, x4m19);
2710 let t_b3_5 = _mm_mul_ps(self.twiddle8im, x5m18);
2711 let t_b3_6 = _mm_mul_ps(self.twiddle5im, x6m17);
2712 let t_b3_7 = _mm_mul_ps(self.twiddle2im, x7m16);
2713 let t_b3_8 = _mm_mul_ps(self.twiddle1im, x8m15);
2714 let t_b3_9 = _mm_mul_ps(self.twiddle4im, x9m14);
2715 let t_b3_10 = _mm_mul_ps(self.twiddle7im, x10m13);
2716 let t_b3_11 = _mm_mul_ps(self.twiddle10im, x11m12);
2717 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m22);
2718 let t_b4_2 = _mm_mul_ps(self.twiddle8im, x2m21);
2719 let t_b4_3 = _mm_mul_ps(self.twiddle11im, x3m20);
2720 let t_b4_4 = _mm_mul_ps(self.twiddle7im, x4m19);
2721 let t_b4_5 = _mm_mul_ps(self.twiddle3im, x5m18);
2722 let t_b4_6 = _mm_mul_ps(self.twiddle1im, x6m17);
2723 let t_b4_7 = _mm_mul_ps(self.twiddle5im, x7m16);
2724 let t_b4_8 = _mm_mul_ps(self.twiddle9im, x8m15);
2725 let t_b4_9 = _mm_mul_ps(self.twiddle10im, x9m14);
2726 let t_b4_10 = _mm_mul_ps(self.twiddle6im, x10m13);
2727 let t_b4_11 = _mm_mul_ps(self.twiddle2im, x11m12);
2728 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m22);
2729 let t_b5_2 = _mm_mul_ps(self.twiddle10im, x2m21);
2730 let t_b5_3 = _mm_mul_ps(self.twiddle8im, x3m20);
2731 let t_b5_4 = _mm_mul_ps(self.twiddle3im, x4m19);
2732 let t_b5_5 = _mm_mul_ps(self.twiddle2im, x5m18);
2733 let t_b5_6 = _mm_mul_ps(self.twiddle7im, x6m17);
2734 let t_b5_7 = _mm_mul_ps(self.twiddle11im, x7m16);
2735 let t_b5_8 = _mm_mul_ps(self.twiddle6im, x8m15);
2736 let t_b5_9 = _mm_mul_ps(self.twiddle1im, x9m14);
2737 let t_b5_10 = _mm_mul_ps(self.twiddle4im, x10m13);
2738 let t_b5_11 = _mm_mul_ps(self.twiddle9im, x11m12);
2739 let t_b6_1 = _mm_mul_ps(self.twiddle6im, x1m22);
2740 let t_b6_2 = _mm_mul_ps(self.twiddle11im, x2m21);
2741 let t_b6_3 = _mm_mul_ps(self.twiddle5im, x3m20);
2742 let t_b6_4 = _mm_mul_ps(self.twiddle1im, x4m19);
2743 let t_b6_5 = _mm_mul_ps(self.twiddle7im, x5m18);
2744 let t_b6_6 = _mm_mul_ps(self.twiddle10im, x6m17);
2745 let t_b6_7 = _mm_mul_ps(self.twiddle4im, x7m16);
2746 let t_b6_8 = _mm_mul_ps(self.twiddle2im, x8m15);
2747 let t_b6_9 = _mm_mul_ps(self.twiddle8im, x9m14);
2748 let t_b6_10 = _mm_mul_ps(self.twiddle9im, x10m13);
2749 let t_b6_11 = _mm_mul_ps(self.twiddle3im, x11m12);
2750 let t_b7_1 = _mm_mul_ps(self.twiddle7im, x1m22);
2751 let t_b7_2 = _mm_mul_ps(self.twiddle9im, x2m21);
2752 let t_b7_3 = _mm_mul_ps(self.twiddle2im, x3m20);
2753 let t_b7_4 = _mm_mul_ps(self.twiddle5im, x4m19);
2754 let t_b7_5 = _mm_mul_ps(self.twiddle11im, x5m18);
2755 let t_b7_6 = _mm_mul_ps(self.twiddle4im, x6m17);
2756 let t_b7_7 = _mm_mul_ps(self.twiddle3im, x7m16);
2757 let t_b7_8 = _mm_mul_ps(self.twiddle10im, x8m15);
2758 let t_b7_9 = _mm_mul_ps(self.twiddle6im, x9m14);
2759 let t_b7_10 = _mm_mul_ps(self.twiddle1im, x10m13);
2760 let t_b7_11 = _mm_mul_ps(self.twiddle8im, x11m12);
2761 let t_b8_1 = _mm_mul_ps(self.twiddle8im, x1m22);
2762 let t_b8_2 = _mm_mul_ps(self.twiddle7im, x2m21);
2763 let t_b8_3 = _mm_mul_ps(self.twiddle1im, x3m20);
2764 let t_b8_4 = _mm_mul_ps(self.twiddle9im, x4m19);
2765 let t_b8_5 = _mm_mul_ps(self.twiddle6im, x5m18);
2766 let t_b8_6 = _mm_mul_ps(self.twiddle2im, x6m17);
2767 let t_b8_7 = _mm_mul_ps(self.twiddle10im, x7m16);
2768 let t_b8_8 = _mm_mul_ps(self.twiddle5im, x8m15);
2769 let t_b8_9 = _mm_mul_ps(self.twiddle3im, x9m14);
2770 let t_b8_10 = _mm_mul_ps(self.twiddle11im, x10m13);
2771 let t_b8_11 = _mm_mul_ps(self.twiddle4im, x11m12);
2772 let t_b9_1 = _mm_mul_ps(self.twiddle9im, x1m22);
2773 let t_b9_2 = _mm_mul_ps(self.twiddle5im, x2m21);
2774 let t_b9_3 = _mm_mul_ps(self.twiddle4im, x3m20);
2775 let t_b9_4 = _mm_mul_ps(self.twiddle10im, x4m19);
2776 let t_b9_5 = _mm_mul_ps(self.twiddle1im, x5m18);
2777 let t_b9_6 = _mm_mul_ps(self.twiddle8im, x6m17);
2778 let t_b9_7 = _mm_mul_ps(self.twiddle6im, x7m16);
2779 let t_b9_8 = _mm_mul_ps(self.twiddle3im, x8m15);
2780 let t_b9_9 = _mm_mul_ps(self.twiddle11im, x9m14);
2781 let t_b9_10 = _mm_mul_ps(self.twiddle2im, x10m13);
2782 let t_b9_11 = _mm_mul_ps(self.twiddle7im, x11m12);
2783 let t_b10_1 = _mm_mul_ps(self.twiddle10im, x1m22);
2784 let t_b10_2 = _mm_mul_ps(self.twiddle3im, x2m21);
2785 let t_b10_3 = _mm_mul_ps(self.twiddle7im, x3m20);
2786 let t_b10_4 = _mm_mul_ps(self.twiddle6im, x4m19);
2787 let t_b10_5 = _mm_mul_ps(self.twiddle4im, x5m18);
2788 let t_b10_6 = _mm_mul_ps(self.twiddle9im, x6m17);
2789 let t_b10_7 = _mm_mul_ps(self.twiddle1im, x7m16);
2790 let t_b10_8 = _mm_mul_ps(self.twiddle11im, x8m15);
2791 let t_b10_9 = _mm_mul_ps(self.twiddle2im, x9m14);
2792 let t_b10_10 = _mm_mul_ps(self.twiddle8im, x10m13);
2793 let t_b10_11 = _mm_mul_ps(self.twiddle5im, x11m12);
2794 let t_b11_1 = _mm_mul_ps(self.twiddle11im, x1m22);
2795 let t_b11_2 = _mm_mul_ps(self.twiddle1im, x2m21);
2796 let t_b11_3 = _mm_mul_ps(self.twiddle10im, x3m20);
2797 let t_b11_4 = _mm_mul_ps(self.twiddle2im, x4m19);
2798 let t_b11_5 = _mm_mul_ps(self.twiddle9im, x5m18);
2799 let t_b11_6 = _mm_mul_ps(self.twiddle3im, x6m17);
2800 let t_b11_7 = _mm_mul_ps(self.twiddle8im, x7m16);
2801 let t_b11_8 = _mm_mul_ps(self.twiddle4im, x8m15);
2802 let t_b11_9 = _mm_mul_ps(self.twiddle7im, x9m14);
2803 let t_b11_10 = _mm_mul_ps(self.twiddle5im, x10m13);
2804 let t_b11_11 = _mm_mul_ps(self.twiddle6im, x11m12);
2805
2806 let x0 = values[0];
2807 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9 + t_a1_10 + t_a1_11);
2808 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9 + t_a2_10 + t_a2_11);
2809 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9 + t_a3_10 + t_a3_11);
2810 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9 + t_a4_10 + t_a4_11);
2811 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9 + t_a5_10 + t_a5_11);
2812 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9 + t_a6_10 + t_a6_11);
2813 let t_a7 = calc_f32!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9 + t_a7_10 + t_a7_11);
2814 let t_a8 = calc_f32!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9 + t_a8_10 + t_a8_11);
2815 let t_a9 = calc_f32!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9 + t_a9_10 + t_a9_11);
2816 let t_a10 = calc_f32!(x0 + t_a10_1 + t_a10_2 + t_a10_3 + t_a10_4 + t_a10_5 + t_a10_6 + t_a10_7 + t_a10_8 + t_a10_9 + t_a10_10 + t_a10_11);
2817 let t_a11 = calc_f32!(x0 + t_a11_1 + t_a11_2 + t_a11_3 + t_a11_4 + t_a11_5 + t_a11_6 + t_a11_7 + t_a11_8 + t_a11_9 + t_a11_10 + t_a11_11);
2818
2819 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9 + t_b1_10 + t_b1_11);
2820 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8 - t_b2_9 - t_b2_10 - t_b2_11);
2821 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7 + t_b3_8 + t_b3_9 + t_b3_10 + t_b3_11);
2822 let t_b4 = calc_f32!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 - t_b4_5 + t_b4_6 + t_b4_7 + t_b4_8 - t_b4_9 - t_b4_10 - t_b4_11);
2823 let t_b5 = calc_f32!(t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 + t_b5_5 + t_b5_6 - t_b5_7 - t_b5_8 - t_b5_9 + t_b5_10 + t_b5_11);
2824 let t_b6 = calc_f32!(t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 + t_b6_5 - t_b6_6 - t_b6_7 + t_b6_8 + t_b6_9 - t_b6_10 - t_b6_11);
2825 let t_b7 = calc_f32!(t_b7_1 - t_b7_2 - t_b7_3 + t_b7_4 - t_b7_5 - t_b7_6 + t_b7_7 + t_b7_8 - t_b7_9 + t_b7_10 + t_b7_11);
2826 let t_b8 = calc_f32!(t_b8_1 - t_b8_2 + t_b8_3 + t_b8_4 - t_b8_5 + t_b8_6 + t_b8_7 - t_b8_8 + t_b8_9 + t_b8_10 - t_b8_11);
2827 let t_b9 = calc_f32!(t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 - t_b9_5 + t_b9_6 - t_b9_7 + t_b9_8 - t_b9_9 - t_b9_10 + t_b9_11);
2828 let t_b10 = calc_f32!(t_b10_1 - t_b10_2 + t_b10_3 - t_b10_4 + t_b10_5 - t_b10_6 + t_b10_7 + t_b10_8 - t_b10_9 + t_b10_10 - t_b10_11);
2829 let t_b11 = calc_f32!(t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 + t_b11_5 - t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9 - t_b11_10 + t_b11_11);
2830
2831 let t_b1_rot = self.rotate.rotate_both(t_b1);
2832 let t_b2_rot = self.rotate.rotate_both(t_b2);
2833 let t_b3_rot = self.rotate.rotate_both(t_b3);
2834 let t_b4_rot = self.rotate.rotate_both(t_b4);
2835 let t_b5_rot = self.rotate.rotate_both(t_b5);
2836 let t_b6_rot = self.rotate.rotate_both(t_b6);
2837 let t_b7_rot = self.rotate.rotate_both(t_b7);
2838 let t_b8_rot = self.rotate.rotate_both(t_b8);
2839 let t_b9_rot = self.rotate.rotate_both(t_b9);
2840 let t_b10_rot = self.rotate.rotate_both(t_b10);
2841 let t_b11_rot = self.rotate.rotate_both(t_b11);
2842
2843 let y0 = calc_f32!(x0 + x1p22 + x2p21 + x3p20 + x4p19 + x5p18 + x6p17 + x7p16 + x8p15 + x9p14 + x10p13 + x11p12);
2844 let [y1, y22] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
2845 let [y2, y21] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
2846 let [y3, y20] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
2847 let [y4, y19] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
2848 let [y5, y18] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
2849 let [y6, y17] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
2850 let [y7, y16] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
2851 let [y8, y15] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
2852 let [y9, y14] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
2853 let [y10, y13] = parallel_fft2_interleaved_f32(t_a10, t_b10_rot);
2854 let [y11, y12] = parallel_fft2_interleaved_f32(t_a11, t_b11_rot);
2855 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22]
2856 }
2857 }
2858
2859 // ____ _____ __ _ _ _ _ _
2860 // |___ \|___ / / /_ | || | | |__ (_) |_
2861 // __) | |_ \ _____ | '_ \| || |_| '_ \| | __|
2862 // / __/ ___) | |_____| | (_) |__ _| |_) | | |_
2863 // |_____|____/ \___/ |_| |_.__/|_|\__|
2864 //
2865
2866 pub struct SseF64Butterfly23<T> {
2867 direction: FftDirection,
2868 _phantom: std::marker::PhantomData<T>,
2869 rotate: Rotate90F64,
2870 twiddle1re: __m128d,
2871 twiddle1im: __m128d,
2872 twiddle2re: __m128d,
2873 twiddle2im: __m128d,
2874 twiddle3re: __m128d,
2875 twiddle3im: __m128d,
2876 twiddle4re: __m128d,
2877 twiddle4im: __m128d,
2878 twiddle5re: __m128d,
2879 twiddle5im: __m128d,
2880 twiddle6re: __m128d,
2881 twiddle6im: __m128d,
2882 twiddle7re: __m128d,
2883 twiddle7im: __m128d,
2884 twiddle8re: __m128d,
2885 twiddle8im: __m128d,
2886 twiddle9re: __m128d,
2887 twiddle9im: __m128d,
2888 twiddle10re: __m128d,
2889 twiddle10im: __m128d,
2890 twiddle11re: __m128d,
2891 twiddle11im: __m128d,
2892 }
2893
2894 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly23, 23, |this: &SseF64Butterfly23<_>| this
2895 .direction);
2896 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly23, 23, |this: &SseF64Butterfly23<_>| this
2897 .direction);
2898 impl<T: FftNum> SseF64Butterfly23<T> {
2899 #[inline(always)]
2900 pub fn new(direction: FftDirection) -> Self {
2901 assert_f64::<T>();
2902 let rotate = Rotate90F64::new(true);
2903 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 23, direction);
2904 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 23, direction);
2905 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 23, direction);
2906 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 23, direction);
2907 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 23, direction);
2908 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 23, direction);
2909 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 23, direction);
2910 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 23, direction);
2911 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 23, direction);
2912 let tw10: Complex<f64> = twiddles::compute_twiddle(10, 23, direction);
2913 let tw11: Complex<f64> = twiddles::compute_twiddle(11, 23, direction);
2914 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
2915 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
2916 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
2917 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
2918 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
2919 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
2920 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
2921 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
2922 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
2923 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
2924 let twiddle6re = unsafe { _mm_set_pd(tw6.re, tw6.re) };
2925 let twiddle6im = unsafe { _mm_set_pd(tw6.im, tw6.im) };
2926 let twiddle7re = unsafe { _mm_set_pd(tw7.re, tw7.re) };
2927 let twiddle7im = unsafe { _mm_set_pd(tw7.im, tw7.im) };
2928 let twiddle8re = unsafe { _mm_set_pd(tw8.re, tw8.re) };
2929 let twiddle8im = unsafe { _mm_set_pd(tw8.im, tw8.im) };
2930 let twiddle9re = unsafe { _mm_set_pd(tw9.re, tw9.re) };
2931 let twiddle9im = unsafe { _mm_set_pd(tw9.im, tw9.im) };
2932 let twiddle10re = unsafe { _mm_set_pd(tw10.re, tw10.re) };
2933 let twiddle10im = unsafe { _mm_set_pd(tw10.im, tw10.im) };
2934 let twiddle11re = unsafe { _mm_set_pd(tw11.re, tw11.re) };
2935 let twiddle11im = unsafe { _mm_set_pd(tw11.im, tw11.im) };
2936
2937 Self {
2938 direction,
2939 _phantom: std::marker::PhantomData,
2940 rotate,
2941 twiddle1re,
2942 twiddle1im,
2943 twiddle2re,
2944 twiddle2im,
2945 twiddle3re,
2946 twiddle3im,
2947 twiddle4re,
2948 twiddle4im,
2949 twiddle5re,
2950 twiddle5im,
2951 twiddle6re,
2952 twiddle6im,
2953 twiddle7re,
2954 twiddle7im,
2955 twiddle8re,
2956 twiddle8im,
2957 twiddle9re,
2958 twiddle9im,
2959 twiddle10re,
2960 twiddle10im,
2961 twiddle11re,
2962 twiddle11im,
2963 }
2964 }
2965
2966 #[inline(always)]
2967 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
2968 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2969
2970 let out = self.perform_fft_direct(values);
2971
2972 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22});
2973 }
2974
2975 #[inline(always)]
2976 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 23]) -> [__m128d; 23] {
2977 let [x1p22, x1m22] = solo_fft2_f64(values[1], values[22]);
2978 let [x2p21, x2m21] = solo_fft2_f64(values[2], values[21]);
2979 let [x3p20, x3m20] = solo_fft2_f64(values[3], values[20]);
2980 let [x4p19, x4m19] = solo_fft2_f64(values[4], values[19]);
2981 let [x5p18, x5m18] = solo_fft2_f64(values[5], values[18]);
2982 let [x6p17, x6m17] = solo_fft2_f64(values[6], values[17]);
2983 let [x7p16, x7m16] = solo_fft2_f64(values[7], values[16]);
2984 let [x8p15, x8m15] = solo_fft2_f64(values[8], values[15]);
2985 let [x9p14, x9m14] = solo_fft2_f64(values[9], values[14]);
2986 let [x10p13, x10m13] = solo_fft2_f64(values[10], values[13]);
2987 let [x11p12, x11m12] = solo_fft2_f64(values[11], values[12]);
2988
2989 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p22);
2990 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p21);
2991 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p20);
2992 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p19);
2993 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p18);
2994 let t_a1_6 = _mm_mul_pd(self.twiddle6re, x6p17);
2995 let t_a1_7 = _mm_mul_pd(self.twiddle7re, x7p16);
2996 let t_a1_8 = _mm_mul_pd(self.twiddle8re, x8p15);
2997 let t_a1_9 = _mm_mul_pd(self.twiddle9re, x9p14);
2998 let t_a1_10 = _mm_mul_pd(self.twiddle10re, x10p13);
2999 let t_a1_11 = _mm_mul_pd(self.twiddle11re, x11p12);
3000 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p22);
3001 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p21);
3002 let t_a2_3 = _mm_mul_pd(self.twiddle6re, x3p20);
3003 let t_a2_4 = _mm_mul_pd(self.twiddle8re, x4p19);
3004 let t_a2_5 = _mm_mul_pd(self.twiddle10re, x5p18);
3005 let t_a2_6 = _mm_mul_pd(self.twiddle11re, x6p17);
3006 let t_a2_7 = _mm_mul_pd(self.twiddle9re, x7p16);
3007 let t_a2_8 = _mm_mul_pd(self.twiddle7re, x8p15);
3008 let t_a2_9 = _mm_mul_pd(self.twiddle5re, x9p14);
3009 let t_a2_10 = _mm_mul_pd(self.twiddle3re, x10p13);
3010 let t_a2_11 = _mm_mul_pd(self.twiddle1re, x11p12);
3011 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p22);
3012 let t_a3_2 = _mm_mul_pd(self.twiddle6re, x2p21);
3013 let t_a3_3 = _mm_mul_pd(self.twiddle9re, x3p20);
3014 let t_a3_4 = _mm_mul_pd(self.twiddle11re, x4p19);
3015 let t_a3_5 = _mm_mul_pd(self.twiddle8re, x5p18);
3016 let t_a3_6 = _mm_mul_pd(self.twiddle5re, x6p17);
3017 let t_a3_7 = _mm_mul_pd(self.twiddle2re, x7p16);
3018 let t_a3_8 = _mm_mul_pd(self.twiddle1re, x8p15);
3019 let t_a3_9 = _mm_mul_pd(self.twiddle4re, x9p14);
3020 let t_a3_10 = _mm_mul_pd(self.twiddle7re, x10p13);
3021 let t_a3_11 = _mm_mul_pd(self.twiddle10re, x11p12);
3022 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p22);
3023 let t_a4_2 = _mm_mul_pd(self.twiddle8re, x2p21);
3024 let t_a4_3 = _mm_mul_pd(self.twiddle11re, x3p20);
3025 let t_a4_4 = _mm_mul_pd(self.twiddle7re, x4p19);
3026 let t_a4_5 = _mm_mul_pd(self.twiddle3re, x5p18);
3027 let t_a4_6 = _mm_mul_pd(self.twiddle1re, x6p17);
3028 let t_a4_7 = _mm_mul_pd(self.twiddle5re, x7p16);
3029 let t_a4_8 = _mm_mul_pd(self.twiddle9re, x8p15);
3030 let t_a4_9 = _mm_mul_pd(self.twiddle10re, x9p14);
3031 let t_a4_10 = _mm_mul_pd(self.twiddle6re, x10p13);
3032 let t_a4_11 = _mm_mul_pd(self.twiddle2re, x11p12);
3033 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p22);
3034 let t_a5_2 = _mm_mul_pd(self.twiddle10re, x2p21);
3035 let t_a5_3 = _mm_mul_pd(self.twiddle8re, x3p20);
3036 let t_a5_4 = _mm_mul_pd(self.twiddle3re, x4p19);
3037 let t_a5_5 = _mm_mul_pd(self.twiddle2re, x5p18);
3038 let t_a5_6 = _mm_mul_pd(self.twiddle7re, x6p17);
3039 let t_a5_7 = _mm_mul_pd(self.twiddle11re, x7p16);
3040 let t_a5_8 = _mm_mul_pd(self.twiddle6re, x8p15);
3041 let t_a5_9 = _mm_mul_pd(self.twiddle1re, x9p14);
3042 let t_a5_10 = _mm_mul_pd(self.twiddle4re, x10p13);
3043 let t_a5_11 = _mm_mul_pd(self.twiddle9re, x11p12);
3044 let t_a6_1 = _mm_mul_pd(self.twiddle6re, x1p22);
3045 let t_a6_2 = _mm_mul_pd(self.twiddle11re, x2p21);
3046 let t_a6_3 = _mm_mul_pd(self.twiddle5re, x3p20);
3047 let t_a6_4 = _mm_mul_pd(self.twiddle1re, x4p19);
3048 let t_a6_5 = _mm_mul_pd(self.twiddle7re, x5p18);
3049 let t_a6_6 = _mm_mul_pd(self.twiddle10re, x6p17);
3050 let t_a6_7 = _mm_mul_pd(self.twiddle4re, x7p16);
3051 let t_a6_8 = _mm_mul_pd(self.twiddle2re, x8p15);
3052 let t_a6_9 = _mm_mul_pd(self.twiddle8re, x9p14);
3053 let t_a6_10 = _mm_mul_pd(self.twiddle9re, x10p13);
3054 let t_a6_11 = _mm_mul_pd(self.twiddle3re, x11p12);
3055 let t_a7_1 = _mm_mul_pd(self.twiddle7re, x1p22);
3056 let t_a7_2 = _mm_mul_pd(self.twiddle9re, x2p21);
3057 let t_a7_3 = _mm_mul_pd(self.twiddle2re, x3p20);
3058 let t_a7_4 = _mm_mul_pd(self.twiddle5re, x4p19);
3059 let t_a7_5 = _mm_mul_pd(self.twiddle11re, x5p18);
3060 let t_a7_6 = _mm_mul_pd(self.twiddle4re, x6p17);
3061 let t_a7_7 = _mm_mul_pd(self.twiddle3re, x7p16);
3062 let t_a7_8 = _mm_mul_pd(self.twiddle10re, x8p15);
3063 let t_a7_9 = _mm_mul_pd(self.twiddle6re, x9p14);
3064 let t_a7_10 = _mm_mul_pd(self.twiddle1re, x10p13);
3065 let t_a7_11 = _mm_mul_pd(self.twiddle8re, x11p12);
3066 let t_a8_1 = _mm_mul_pd(self.twiddle8re, x1p22);
3067 let t_a8_2 = _mm_mul_pd(self.twiddle7re, x2p21);
3068 let t_a8_3 = _mm_mul_pd(self.twiddle1re, x3p20);
3069 let t_a8_4 = _mm_mul_pd(self.twiddle9re, x4p19);
3070 let t_a8_5 = _mm_mul_pd(self.twiddle6re, x5p18);
3071 let t_a8_6 = _mm_mul_pd(self.twiddle2re, x6p17);
3072 let t_a8_7 = _mm_mul_pd(self.twiddle10re, x7p16);
3073 let t_a8_8 = _mm_mul_pd(self.twiddle5re, x8p15);
3074 let t_a8_9 = _mm_mul_pd(self.twiddle3re, x9p14);
3075 let t_a8_10 = _mm_mul_pd(self.twiddle11re, x10p13);
3076 let t_a8_11 = _mm_mul_pd(self.twiddle4re, x11p12);
3077 let t_a9_1 = _mm_mul_pd(self.twiddle9re, x1p22);
3078 let t_a9_2 = _mm_mul_pd(self.twiddle5re, x2p21);
3079 let t_a9_3 = _mm_mul_pd(self.twiddle4re, x3p20);
3080 let t_a9_4 = _mm_mul_pd(self.twiddle10re, x4p19);
3081 let t_a9_5 = _mm_mul_pd(self.twiddle1re, x5p18);
3082 let t_a9_6 = _mm_mul_pd(self.twiddle8re, x6p17);
3083 let t_a9_7 = _mm_mul_pd(self.twiddle6re, x7p16);
3084 let t_a9_8 = _mm_mul_pd(self.twiddle3re, x8p15);
3085 let t_a9_9 = _mm_mul_pd(self.twiddle11re, x9p14);
3086 let t_a9_10 = _mm_mul_pd(self.twiddle2re, x10p13);
3087 let t_a9_11 = _mm_mul_pd(self.twiddle7re, x11p12);
3088 let t_a10_1 = _mm_mul_pd(self.twiddle10re, x1p22);
3089 let t_a10_2 = _mm_mul_pd(self.twiddle3re, x2p21);
3090 let t_a10_3 = _mm_mul_pd(self.twiddle7re, x3p20);
3091 let t_a10_4 = _mm_mul_pd(self.twiddle6re, x4p19);
3092 let t_a10_5 = _mm_mul_pd(self.twiddle4re, x5p18);
3093 let t_a10_6 = _mm_mul_pd(self.twiddle9re, x6p17);
3094 let t_a10_7 = _mm_mul_pd(self.twiddle1re, x7p16);
3095 let t_a10_8 = _mm_mul_pd(self.twiddle11re, x8p15);
3096 let t_a10_9 = _mm_mul_pd(self.twiddle2re, x9p14);
3097 let t_a10_10 = _mm_mul_pd(self.twiddle8re, x10p13);
3098 let t_a10_11 = _mm_mul_pd(self.twiddle5re, x11p12);
3099 let t_a11_1 = _mm_mul_pd(self.twiddle11re, x1p22);
3100 let t_a11_2 = _mm_mul_pd(self.twiddle1re, x2p21);
3101 let t_a11_3 = _mm_mul_pd(self.twiddle10re, x3p20);
3102 let t_a11_4 = _mm_mul_pd(self.twiddle2re, x4p19);
3103 let t_a11_5 = _mm_mul_pd(self.twiddle9re, x5p18);
3104 let t_a11_6 = _mm_mul_pd(self.twiddle3re, x6p17);
3105 let t_a11_7 = _mm_mul_pd(self.twiddle8re, x7p16);
3106 let t_a11_8 = _mm_mul_pd(self.twiddle4re, x8p15);
3107 let t_a11_9 = _mm_mul_pd(self.twiddle7re, x9p14);
3108 let t_a11_10 = _mm_mul_pd(self.twiddle5re, x10p13);
3109 let t_a11_11 = _mm_mul_pd(self.twiddle6re, x11p12);
3110
3111 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m22);
3112 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m21);
3113 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m20);
3114 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m19);
3115 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m18);
3116 let t_b1_6 = _mm_mul_pd(self.twiddle6im, x6m17);
3117 let t_b1_7 = _mm_mul_pd(self.twiddle7im, x7m16);
3118 let t_b1_8 = _mm_mul_pd(self.twiddle8im, x8m15);
3119 let t_b1_9 = _mm_mul_pd(self.twiddle9im, x9m14);
3120 let t_b1_10 = _mm_mul_pd(self.twiddle10im, x10m13);
3121 let t_b1_11 = _mm_mul_pd(self.twiddle11im, x11m12);
3122 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m22);
3123 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m21);
3124 let t_b2_3 = _mm_mul_pd(self.twiddle6im, x3m20);
3125 let t_b2_4 = _mm_mul_pd(self.twiddle8im, x4m19);
3126 let t_b2_5 = _mm_mul_pd(self.twiddle10im, x5m18);
3127 let t_b2_6 = _mm_mul_pd(self.twiddle11im, x6m17);
3128 let t_b2_7 = _mm_mul_pd(self.twiddle9im, x7m16);
3129 let t_b2_8 = _mm_mul_pd(self.twiddle7im, x8m15);
3130 let t_b2_9 = _mm_mul_pd(self.twiddle5im, x9m14);
3131 let t_b2_10 = _mm_mul_pd(self.twiddle3im, x10m13);
3132 let t_b2_11 = _mm_mul_pd(self.twiddle1im, x11m12);
3133 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m22);
3134 let t_b3_2 = _mm_mul_pd(self.twiddle6im, x2m21);
3135 let t_b3_3 = _mm_mul_pd(self.twiddle9im, x3m20);
3136 let t_b3_4 = _mm_mul_pd(self.twiddle11im, x4m19);
3137 let t_b3_5 = _mm_mul_pd(self.twiddle8im, x5m18);
3138 let t_b3_6 = _mm_mul_pd(self.twiddle5im, x6m17);
3139 let t_b3_7 = _mm_mul_pd(self.twiddle2im, x7m16);
3140 let t_b3_8 = _mm_mul_pd(self.twiddle1im, x8m15);
3141 let t_b3_9 = _mm_mul_pd(self.twiddle4im, x9m14);
3142 let t_b3_10 = _mm_mul_pd(self.twiddle7im, x10m13);
3143 let t_b3_11 = _mm_mul_pd(self.twiddle10im, x11m12);
3144 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m22);
3145 let t_b4_2 = _mm_mul_pd(self.twiddle8im, x2m21);
3146 let t_b4_3 = _mm_mul_pd(self.twiddle11im, x3m20);
3147 let t_b4_4 = _mm_mul_pd(self.twiddle7im, x4m19);
3148 let t_b4_5 = _mm_mul_pd(self.twiddle3im, x5m18);
3149 let t_b4_6 = _mm_mul_pd(self.twiddle1im, x6m17);
3150 let t_b4_7 = _mm_mul_pd(self.twiddle5im, x7m16);
3151 let t_b4_8 = _mm_mul_pd(self.twiddle9im, x8m15);
3152 let t_b4_9 = _mm_mul_pd(self.twiddle10im, x9m14);
3153 let t_b4_10 = _mm_mul_pd(self.twiddle6im, x10m13);
3154 let t_b4_11 = _mm_mul_pd(self.twiddle2im, x11m12);
3155 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m22);
3156 let t_b5_2 = _mm_mul_pd(self.twiddle10im, x2m21);
3157 let t_b5_3 = _mm_mul_pd(self.twiddle8im, x3m20);
3158 let t_b5_4 = _mm_mul_pd(self.twiddle3im, x4m19);
3159 let t_b5_5 = _mm_mul_pd(self.twiddle2im, x5m18);
3160 let t_b5_6 = _mm_mul_pd(self.twiddle7im, x6m17);
3161 let t_b5_7 = _mm_mul_pd(self.twiddle11im, x7m16);
3162 let t_b5_8 = _mm_mul_pd(self.twiddle6im, x8m15);
3163 let t_b5_9 = _mm_mul_pd(self.twiddle1im, x9m14);
3164 let t_b5_10 = _mm_mul_pd(self.twiddle4im, x10m13);
3165 let t_b5_11 = _mm_mul_pd(self.twiddle9im, x11m12);
3166 let t_b6_1 = _mm_mul_pd(self.twiddle6im, x1m22);
3167 let t_b6_2 = _mm_mul_pd(self.twiddle11im, x2m21);
3168 let t_b6_3 = _mm_mul_pd(self.twiddle5im, x3m20);
3169 let t_b6_4 = _mm_mul_pd(self.twiddle1im, x4m19);
3170 let t_b6_5 = _mm_mul_pd(self.twiddle7im, x5m18);
3171 let t_b6_6 = _mm_mul_pd(self.twiddle10im, x6m17);
3172 let t_b6_7 = _mm_mul_pd(self.twiddle4im, x7m16);
3173 let t_b6_8 = _mm_mul_pd(self.twiddle2im, x8m15);
3174 let t_b6_9 = _mm_mul_pd(self.twiddle8im, x9m14);
3175 let t_b6_10 = _mm_mul_pd(self.twiddle9im, x10m13);
3176 let t_b6_11 = _mm_mul_pd(self.twiddle3im, x11m12);
3177 let t_b7_1 = _mm_mul_pd(self.twiddle7im, x1m22);
3178 let t_b7_2 = _mm_mul_pd(self.twiddle9im, x2m21);
3179 let t_b7_3 = _mm_mul_pd(self.twiddle2im, x3m20);
3180 let t_b7_4 = _mm_mul_pd(self.twiddle5im, x4m19);
3181 let t_b7_5 = _mm_mul_pd(self.twiddle11im, x5m18);
3182 let t_b7_6 = _mm_mul_pd(self.twiddle4im, x6m17);
3183 let t_b7_7 = _mm_mul_pd(self.twiddle3im, x7m16);
3184 let t_b7_8 = _mm_mul_pd(self.twiddle10im, x8m15);
3185 let t_b7_9 = _mm_mul_pd(self.twiddle6im, x9m14);
3186 let t_b7_10 = _mm_mul_pd(self.twiddle1im, x10m13);
3187 let t_b7_11 = _mm_mul_pd(self.twiddle8im, x11m12);
3188 let t_b8_1 = _mm_mul_pd(self.twiddle8im, x1m22);
3189 let t_b8_2 = _mm_mul_pd(self.twiddle7im, x2m21);
3190 let t_b8_3 = _mm_mul_pd(self.twiddle1im, x3m20);
3191 let t_b8_4 = _mm_mul_pd(self.twiddle9im, x4m19);
3192 let t_b8_5 = _mm_mul_pd(self.twiddle6im, x5m18);
3193 let t_b8_6 = _mm_mul_pd(self.twiddle2im, x6m17);
3194 let t_b8_7 = _mm_mul_pd(self.twiddle10im, x7m16);
3195 let t_b8_8 = _mm_mul_pd(self.twiddle5im, x8m15);
3196 let t_b8_9 = _mm_mul_pd(self.twiddle3im, x9m14);
3197 let t_b8_10 = _mm_mul_pd(self.twiddle11im, x10m13);
3198 let t_b8_11 = _mm_mul_pd(self.twiddle4im, x11m12);
3199 let t_b9_1 = _mm_mul_pd(self.twiddle9im, x1m22);
3200 let t_b9_2 = _mm_mul_pd(self.twiddle5im, x2m21);
3201 let t_b9_3 = _mm_mul_pd(self.twiddle4im, x3m20);
3202 let t_b9_4 = _mm_mul_pd(self.twiddle10im, x4m19);
3203 let t_b9_5 = _mm_mul_pd(self.twiddle1im, x5m18);
3204 let t_b9_6 = _mm_mul_pd(self.twiddle8im, x6m17);
3205 let t_b9_7 = _mm_mul_pd(self.twiddle6im, x7m16);
3206 let t_b9_8 = _mm_mul_pd(self.twiddle3im, x8m15);
3207 let t_b9_9 = _mm_mul_pd(self.twiddle11im, x9m14);
3208 let t_b9_10 = _mm_mul_pd(self.twiddle2im, x10m13);
3209 let t_b9_11 = _mm_mul_pd(self.twiddle7im, x11m12);
3210 let t_b10_1 = _mm_mul_pd(self.twiddle10im, x1m22);
3211 let t_b10_2 = _mm_mul_pd(self.twiddle3im, x2m21);
3212 let t_b10_3 = _mm_mul_pd(self.twiddle7im, x3m20);
3213 let t_b10_4 = _mm_mul_pd(self.twiddle6im, x4m19);
3214 let t_b10_5 = _mm_mul_pd(self.twiddle4im, x5m18);
3215 let t_b10_6 = _mm_mul_pd(self.twiddle9im, x6m17);
3216 let t_b10_7 = _mm_mul_pd(self.twiddle1im, x7m16);
3217 let t_b10_8 = _mm_mul_pd(self.twiddle11im, x8m15);
3218 let t_b10_9 = _mm_mul_pd(self.twiddle2im, x9m14);
3219 let t_b10_10 = _mm_mul_pd(self.twiddle8im, x10m13);
3220 let t_b10_11 = _mm_mul_pd(self.twiddle5im, x11m12);
3221 let t_b11_1 = _mm_mul_pd(self.twiddle11im, x1m22);
3222 let t_b11_2 = _mm_mul_pd(self.twiddle1im, x2m21);
3223 let t_b11_3 = _mm_mul_pd(self.twiddle10im, x3m20);
3224 let t_b11_4 = _mm_mul_pd(self.twiddle2im, x4m19);
3225 let t_b11_5 = _mm_mul_pd(self.twiddle9im, x5m18);
3226 let t_b11_6 = _mm_mul_pd(self.twiddle3im, x6m17);
3227 let t_b11_7 = _mm_mul_pd(self.twiddle8im, x7m16);
3228 let t_b11_8 = _mm_mul_pd(self.twiddle4im, x8m15);
3229 let t_b11_9 = _mm_mul_pd(self.twiddle7im, x9m14);
3230 let t_b11_10 = _mm_mul_pd(self.twiddle5im, x10m13);
3231 let t_b11_11 = _mm_mul_pd(self.twiddle6im, x11m12);
3232
3233 let x0 = values[0];
3234 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9 + t_a1_10 + t_a1_11);
3235 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9 + t_a2_10 + t_a2_11);
3236 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9 + t_a3_10 + t_a3_11);
3237 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9 + t_a4_10 + t_a4_11);
3238 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9 + t_a5_10 + t_a5_11);
3239 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9 + t_a6_10 + t_a6_11);
3240 let t_a7 = calc_f64!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9 + t_a7_10 + t_a7_11);
3241 let t_a8 = calc_f64!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9 + t_a8_10 + t_a8_11);
3242 let t_a9 = calc_f64!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9 + t_a9_10 + t_a9_11);
3243 let t_a10 = calc_f64!(x0 + t_a10_1 + t_a10_2 + t_a10_3 + t_a10_4 + t_a10_5 + t_a10_6 + t_a10_7 + t_a10_8 + t_a10_9 + t_a10_10 + t_a10_11);
3244 let t_a11 = calc_f64!(x0 + t_a11_1 + t_a11_2 + t_a11_3 + t_a11_4 + t_a11_5 + t_a11_6 + t_a11_7 + t_a11_8 + t_a11_9 + t_a11_10 + t_a11_11);
3245
3246 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9 + t_b1_10 + t_b1_11);
3247 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 - t_b2_6 - t_b2_7 - t_b2_8 - t_b2_9 - t_b2_10 - t_b2_11);
3248 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 + t_b3_3 - t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7 + t_b3_8 + t_b3_9 + t_b3_10 + t_b3_11);
3249 let t_b4 = calc_f64!(t_b4_1 + t_b4_2 - t_b4_3 - t_b4_4 - t_b4_5 + t_b4_6 + t_b4_7 + t_b4_8 - t_b4_9 - t_b4_10 - t_b4_11);
3250 let t_b5 = calc_f64!(t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 + t_b5_5 + t_b5_6 - t_b5_7 - t_b5_8 - t_b5_9 + t_b5_10 + t_b5_11);
3251 let t_b6 = calc_f64!(t_b6_1 - t_b6_2 - t_b6_3 + t_b6_4 + t_b6_5 - t_b6_6 - t_b6_7 + t_b6_8 + t_b6_9 - t_b6_10 - t_b6_11);
3252 let t_b7 = calc_f64!(t_b7_1 - t_b7_2 - t_b7_3 + t_b7_4 - t_b7_5 - t_b7_6 + t_b7_7 + t_b7_8 - t_b7_9 + t_b7_10 + t_b7_11);
3253 let t_b8 = calc_f64!(t_b8_1 - t_b8_2 + t_b8_3 + t_b8_4 - t_b8_5 + t_b8_6 + t_b8_7 - t_b8_8 + t_b8_9 + t_b8_10 - t_b8_11);
3254 let t_b9 = calc_f64!(t_b9_1 - t_b9_2 + t_b9_3 - t_b9_4 - t_b9_5 + t_b9_6 - t_b9_7 + t_b9_8 - t_b9_9 - t_b9_10 + t_b9_11);
3255 let t_b10 = calc_f64!(t_b10_1 - t_b10_2 + t_b10_3 - t_b10_4 + t_b10_5 - t_b10_6 + t_b10_7 + t_b10_8 - t_b10_9 + t_b10_10 - t_b10_11);
3256 let t_b11 = calc_f64!(t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 + t_b11_5 - t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9 - t_b11_10 + t_b11_11);
3257
3258 let t_b1_rot = self.rotate.rotate(t_b1);
3259 let t_b2_rot = self.rotate.rotate(t_b2);
3260 let t_b3_rot = self.rotate.rotate(t_b3);
3261 let t_b4_rot = self.rotate.rotate(t_b4);
3262 let t_b5_rot = self.rotate.rotate(t_b5);
3263 let t_b6_rot = self.rotate.rotate(t_b6);
3264 let t_b7_rot = self.rotate.rotate(t_b7);
3265 let t_b8_rot = self.rotate.rotate(t_b8);
3266 let t_b9_rot = self.rotate.rotate(t_b9);
3267 let t_b10_rot = self.rotate.rotate(t_b10);
3268 let t_b11_rot = self.rotate.rotate(t_b11);
3269
3270 let y0 = calc_f64!(x0 + x1p22 + x2p21 + x3p20 + x4p19 + x5p18 + x6p17 + x7p16 + x8p15 + x9p14 + x10p13 + x11p12);
3271 let [y1, y22] = solo_fft2_f64(t_a1, t_b1_rot);
3272 let [y2, y21] = solo_fft2_f64(t_a2, t_b2_rot);
3273 let [y3, y20] = solo_fft2_f64(t_a3, t_b3_rot);
3274 let [y4, y19] = solo_fft2_f64(t_a4, t_b4_rot);
3275 let [y5, y18] = solo_fft2_f64(t_a5, t_b5_rot);
3276 let [y6, y17] = solo_fft2_f64(t_a6, t_b6_rot);
3277 let [y7, y16] = solo_fft2_f64(t_a7, t_b7_rot);
3278 let [y8, y15] = solo_fft2_f64(t_a8, t_b8_rot);
3279 let [y9, y14] = solo_fft2_f64(t_a9, t_b9_rot);
3280 let [y10, y13] = solo_fft2_f64(t_a10, t_b10_rot);
3281 let [y11, y12] = solo_fft2_f64(t_a11, t_b11_rot);
3282 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22]
3283 }
3284 }
3285
3286 // ____ ___ _________ _ _ _
3287 // |___ \ / _ \ |___ /___ \| |__ (_) |_
3288 // __) | (_) | _____ |_ \ __) | '_ \| | __|
3289 // / __/ \__, | |_____| ___) / __/| |_) | | |_
3290 // |_____| /_/ |____/_____|_.__/|_|\__|
3291 //
3292
3293 pub struct SseF32Butterfly29<T> {
3294 direction: FftDirection,
3295 _phantom: std::marker::PhantomData<T>,
3296 rotate: Rotate90F32,
3297 twiddle1re: __m128,
3298 twiddle1im: __m128,
3299 twiddle2re: __m128,
3300 twiddle2im: __m128,
3301 twiddle3re: __m128,
3302 twiddle3im: __m128,
3303 twiddle4re: __m128,
3304 twiddle4im: __m128,
3305 twiddle5re: __m128,
3306 twiddle5im: __m128,
3307 twiddle6re: __m128,
3308 twiddle6im: __m128,
3309 twiddle7re: __m128,
3310 twiddle7im: __m128,
3311 twiddle8re: __m128,
3312 twiddle8im: __m128,
3313 twiddle9re: __m128,
3314 twiddle9im: __m128,
3315 twiddle10re: __m128,
3316 twiddle10im: __m128,
3317 twiddle11re: __m128,
3318 twiddle11im: __m128,
3319 twiddle12re: __m128,
3320 twiddle12im: __m128,
3321 twiddle13re: __m128,
3322 twiddle13im: __m128,
3323 twiddle14re: __m128,
3324 twiddle14im: __m128,
3325 }
3326
3327 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly29, 29, |this: &SseF32Butterfly29<_>| this
3328 .direction);
3329 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly29, 29, |this: &SseF32Butterfly29<_>| this
3330 .direction);
3331 impl<T: FftNum> SseF32Butterfly29<T> {
3332 #[inline(always)]
3333 pub fn new(direction: FftDirection) -> Self {
3334 assert_f32::<T>();
3335 let rotate = Rotate90F32::new(true);
3336 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 29, direction);
3337 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 29, direction);
3338 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 29, direction);
3339 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 29, direction);
3340 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 29, direction);
3341 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 29, direction);
3342 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 29, direction);
3343 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 29, direction);
3344 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 29, direction);
3345 let tw10: Complex<f32> = twiddles::compute_twiddle(10, 29, direction);
3346 let tw11: Complex<f32> = twiddles::compute_twiddle(11, 29, direction);
3347 let tw12: Complex<f32> = twiddles::compute_twiddle(12, 29, direction);
3348 let tw13: Complex<f32> = twiddles::compute_twiddle(13, 29, direction);
3349 let tw14: Complex<f32> = twiddles::compute_twiddle(14, 29, direction);
3350 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
3351 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
3352 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
3353 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
3354 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
3355 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
3356 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
3357 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
3358 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
3359 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
3360 let twiddle6re = unsafe { _mm_load1_ps(&tw6.re) };
3361 let twiddle6im = unsafe { _mm_load1_ps(&tw6.im) };
3362 let twiddle7re = unsafe { _mm_load1_ps(&tw7.re) };
3363 let twiddle7im = unsafe { _mm_load1_ps(&tw7.im) };
3364 let twiddle8re = unsafe { _mm_load1_ps(&tw8.re) };
3365 let twiddle8im = unsafe { _mm_load1_ps(&tw8.im) };
3366 let twiddle9re = unsafe { _mm_load1_ps(&tw9.re) };
3367 let twiddle9im = unsafe { _mm_load1_ps(&tw9.im) };
3368 let twiddle10re = unsafe { _mm_load1_ps(&tw10.re) };
3369 let twiddle10im = unsafe { _mm_load1_ps(&tw10.im) };
3370 let twiddle11re = unsafe { _mm_load1_ps(&tw11.re) };
3371 let twiddle11im = unsafe { _mm_load1_ps(&tw11.im) };
3372 let twiddle12re = unsafe { _mm_load1_ps(&tw12.re) };
3373 let twiddle12im = unsafe { _mm_load1_ps(&tw12.im) };
3374 let twiddle13re = unsafe { _mm_load1_ps(&tw13.re) };
3375 let twiddle13im = unsafe { _mm_load1_ps(&tw13.im) };
3376 let twiddle14re = unsafe { _mm_load1_ps(&tw14.re) };
3377 let twiddle14im = unsafe { _mm_load1_ps(&tw14.im) };
3378
3379 Self {
3380 direction,
3381 _phantom: std::marker::PhantomData,
3382 rotate,
3383 twiddle1re,
3384 twiddle1im,
3385 twiddle2re,
3386 twiddle2im,
3387 twiddle3re,
3388 twiddle3im,
3389 twiddle4re,
3390 twiddle4im,
3391 twiddle5re,
3392 twiddle5im,
3393 twiddle6re,
3394 twiddle6im,
3395 twiddle7re,
3396 twiddle7im,
3397 twiddle8re,
3398 twiddle8im,
3399 twiddle9re,
3400 twiddle9im,
3401 twiddle10re,
3402 twiddle10im,
3403 twiddle11re,
3404 twiddle11im,
3405 twiddle12re,
3406 twiddle12im,
3407 twiddle13re,
3408 twiddle13im,
3409 twiddle14re,
3410 twiddle14im,
3411 }
3412 }
3413
3414 #[inline(always)]
3415 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
3416 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
3417
3418 let out = self.perform_parallel_fft_direct(values);
3419
3420 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
3421
3422 }
3423
3424 #[inline(always)]
3425 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
3426 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56});
3427
3428 let values = [
3429 extract_lo_hi_f32(input_packed[0], input_packed[14]),
3430 extract_hi_lo_f32(input_packed[0], input_packed[15]),
3431 extract_lo_hi_f32(input_packed[1], input_packed[15]),
3432 extract_hi_lo_f32(input_packed[1], input_packed[16]),
3433 extract_lo_hi_f32(input_packed[2], input_packed[16]),
3434 extract_hi_lo_f32(input_packed[2], input_packed[17]),
3435 extract_lo_hi_f32(input_packed[3], input_packed[17]),
3436 extract_hi_lo_f32(input_packed[3], input_packed[18]),
3437 extract_lo_hi_f32(input_packed[4], input_packed[18]),
3438 extract_hi_lo_f32(input_packed[4], input_packed[19]),
3439 extract_lo_hi_f32(input_packed[5], input_packed[19]),
3440 extract_hi_lo_f32(input_packed[5], input_packed[20]),
3441 extract_lo_hi_f32(input_packed[6], input_packed[20]),
3442 extract_hi_lo_f32(input_packed[6], input_packed[21]),
3443 extract_lo_hi_f32(input_packed[7], input_packed[21]),
3444 extract_hi_lo_f32(input_packed[7], input_packed[22]),
3445 extract_lo_hi_f32(input_packed[8], input_packed[22]),
3446 extract_hi_lo_f32(input_packed[8], input_packed[23]),
3447 extract_lo_hi_f32(input_packed[9], input_packed[23]),
3448 extract_hi_lo_f32(input_packed[9], input_packed[24]),
3449 extract_lo_hi_f32(input_packed[10], input_packed[24]),
3450 extract_hi_lo_f32(input_packed[10], input_packed[25]),
3451 extract_lo_hi_f32(input_packed[11], input_packed[25]),
3452 extract_hi_lo_f32(input_packed[11], input_packed[26]),
3453 extract_lo_hi_f32(input_packed[12], input_packed[26]),
3454 extract_hi_lo_f32(input_packed[12], input_packed[27]),
3455 extract_lo_hi_f32(input_packed[13], input_packed[27]),
3456 extract_hi_lo_f32(input_packed[13], input_packed[28]),
3457 extract_lo_hi_f32(input_packed[14], input_packed[28]),
3458 ];
3459
3460 let out = self.perform_parallel_fft_direct(values);
3461
3462 let out_packed = [
3463 extract_lo_lo_f32(out[0], out[1]),
3464 extract_lo_lo_f32(out[2], out[3]),
3465 extract_lo_lo_f32(out[4], out[5]),
3466 extract_lo_lo_f32(out[6], out[7]),
3467 extract_lo_lo_f32(out[8], out[9]),
3468 extract_lo_lo_f32(out[10], out[11]),
3469 extract_lo_lo_f32(out[12], out[13]),
3470 extract_lo_lo_f32(out[14], out[15]),
3471 extract_lo_lo_f32(out[16], out[17]),
3472 extract_lo_lo_f32(out[18], out[19]),
3473 extract_lo_lo_f32(out[20], out[21]),
3474 extract_lo_lo_f32(out[22], out[23]),
3475 extract_lo_lo_f32(out[24], out[25]),
3476 extract_lo_lo_f32(out[26], out[27]),
3477 extract_lo_hi_f32(out[28], out[0]),
3478 extract_hi_hi_f32(out[1], out[2]),
3479 extract_hi_hi_f32(out[3], out[4]),
3480 extract_hi_hi_f32(out[5], out[6]),
3481 extract_hi_hi_f32(out[7], out[8]),
3482 extract_hi_hi_f32(out[9], out[10]),
3483 extract_hi_hi_f32(out[11], out[12]),
3484 extract_hi_hi_f32(out[13], out[14]),
3485 extract_hi_hi_f32(out[15], out[16]),
3486 extract_hi_hi_f32(out[17], out[18]),
3487 extract_hi_hi_f32(out[19], out[20]),
3488 extract_hi_hi_f32(out[21], out[22]),
3489 extract_hi_hi_f32(out[23], out[24]),
3490 extract_hi_hi_f32(out[25], out[26]),
3491 extract_hi_hi_f32(out[27], out[28]),
3492 ];
3493
3494 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
3495 }
3496
3497 #[inline(always)]
3498 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 29]) -> [__m128; 29] {
3499 let [x1p28, x1m28] = parallel_fft2_interleaved_f32(values[1], values[28]);
3500 let [x2p27, x2m27] = parallel_fft2_interleaved_f32(values[2], values[27]);
3501 let [x3p26, x3m26] = parallel_fft2_interleaved_f32(values[3], values[26]);
3502 let [x4p25, x4m25] = parallel_fft2_interleaved_f32(values[4], values[25]);
3503 let [x5p24, x5m24] = parallel_fft2_interleaved_f32(values[5], values[24]);
3504 let [x6p23, x6m23] = parallel_fft2_interleaved_f32(values[6], values[23]);
3505 let [x7p22, x7m22] = parallel_fft2_interleaved_f32(values[7], values[22]);
3506 let [x8p21, x8m21] = parallel_fft2_interleaved_f32(values[8], values[21]);
3507 let [x9p20, x9m20] = parallel_fft2_interleaved_f32(values[9], values[20]);
3508 let [x10p19, x10m19] = parallel_fft2_interleaved_f32(values[10], values[19]);
3509 let [x11p18, x11m18] = parallel_fft2_interleaved_f32(values[11], values[18]);
3510 let [x12p17, x12m17] = parallel_fft2_interleaved_f32(values[12], values[17]);
3511 let [x13p16, x13m16] = parallel_fft2_interleaved_f32(values[13], values[16]);
3512 let [x14p15, x14m15] = parallel_fft2_interleaved_f32(values[14], values[15]);
3513
3514 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p28);
3515 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p27);
3516 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p26);
3517 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p25);
3518 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p24);
3519 let t_a1_6 = _mm_mul_ps(self.twiddle6re, x6p23);
3520 let t_a1_7 = _mm_mul_ps(self.twiddle7re, x7p22);
3521 let t_a1_8 = _mm_mul_ps(self.twiddle8re, x8p21);
3522 let t_a1_9 = _mm_mul_ps(self.twiddle9re, x9p20);
3523 let t_a1_10 = _mm_mul_ps(self.twiddle10re, x10p19);
3524 let t_a1_11 = _mm_mul_ps(self.twiddle11re, x11p18);
3525 let t_a1_12 = _mm_mul_ps(self.twiddle12re, x12p17);
3526 let t_a1_13 = _mm_mul_ps(self.twiddle13re, x13p16);
3527 let t_a1_14 = _mm_mul_ps(self.twiddle14re, x14p15);
3528 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p28);
3529 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p27);
3530 let t_a2_3 = _mm_mul_ps(self.twiddle6re, x3p26);
3531 let t_a2_4 = _mm_mul_ps(self.twiddle8re, x4p25);
3532 let t_a2_5 = _mm_mul_ps(self.twiddle10re, x5p24);
3533 let t_a2_6 = _mm_mul_ps(self.twiddle12re, x6p23);
3534 let t_a2_7 = _mm_mul_ps(self.twiddle14re, x7p22);
3535 let t_a2_8 = _mm_mul_ps(self.twiddle13re, x8p21);
3536 let t_a2_9 = _mm_mul_ps(self.twiddle11re, x9p20);
3537 let t_a2_10 = _mm_mul_ps(self.twiddle9re, x10p19);
3538 let t_a2_11 = _mm_mul_ps(self.twiddle7re, x11p18);
3539 let t_a2_12 = _mm_mul_ps(self.twiddle5re, x12p17);
3540 let t_a2_13 = _mm_mul_ps(self.twiddle3re, x13p16);
3541 let t_a2_14 = _mm_mul_ps(self.twiddle1re, x14p15);
3542 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p28);
3543 let t_a3_2 = _mm_mul_ps(self.twiddle6re, x2p27);
3544 let t_a3_3 = _mm_mul_ps(self.twiddle9re, x3p26);
3545 let t_a3_4 = _mm_mul_ps(self.twiddle12re, x4p25);
3546 let t_a3_5 = _mm_mul_ps(self.twiddle14re, x5p24);
3547 let t_a3_6 = _mm_mul_ps(self.twiddle11re, x6p23);
3548 let t_a3_7 = _mm_mul_ps(self.twiddle8re, x7p22);
3549 let t_a3_8 = _mm_mul_ps(self.twiddle5re, x8p21);
3550 let t_a3_9 = _mm_mul_ps(self.twiddle2re, x9p20);
3551 let t_a3_10 = _mm_mul_ps(self.twiddle1re, x10p19);
3552 let t_a3_11 = _mm_mul_ps(self.twiddle4re, x11p18);
3553 let t_a3_12 = _mm_mul_ps(self.twiddle7re, x12p17);
3554 let t_a3_13 = _mm_mul_ps(self.twiddle10re, x13p16);
3555 let t_a3_14 = _mm_mul_ps(self.twiddle13re, x14p15);
3556 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p28);
3557 let t_a4_2 = _mm_mul_ps(self.twiddle8re, x2p27);
3558 let t_a4_3 = _mm_mul_ps(self.twiddle12re, x3p26);
3559 let t_a4_4 = _mm_mul_ps(self.twiddle13re, x4p25);
3560 let t_a4_5 = _mm_mul_ps(self.twiddle9re, x5p24);
3561 let t_a4_6 = _mm_mul_ps(self.twiddle5re, x6p23);
3562 let t_a4_7 = _mm_mul_ps(self.twiddle1re, x7p22);
3563 let t_a4_8 = _mm_mul_ps(self.twiddle3re, x8p21);
3564 let t_a4_9 = _mm_mul_ps(self.twiddle7re, x9p20);
3565 let t_a4_10 = _mm_mul_ps(self.twiddle11re, x10p19);
3566 let t_a4_11 = _mm_mul_ps(self.twiddle14re, x11p18);
3567 let t_a4_12 = _mm_mul_ps(self.twiddle10re, x12p17);
3568 let t_a4_13 = _mm_mul_ps(self.twiddle6re, x13p16);
3569 let t_a4_14 = _mm_mul_ps(self.twiddle2re, x14p15);
3570 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p28);
3571 let t_a5_2 = _mm_mul_ps(self.twiddle10re, x2p27);
3572 let t_a5_3 = _mm_mul_ps(self.twiddle14re, x3p26);
3573 let t_a5_4 = _mm_mul_ps(self.twiddle9re, x4p25);
3574 let t_a5_5 = _mm_mul_ps(self.twiddle4re, x5p24);
3575 let t_a5_6 = _mm_mul_ps(self.twiddle1re, x6p23);
3576 let t_a5_7 = _mm_mul_ps(self.twiddle6re, x7p22);
3577 let t_a5_8 = _mm_mul_ps(self.twiddle11re, x8p21);
3578 let t_a5_9 = _mm_mul_ps(self.twiddle13re, x9p20);
3579 let t_a5_10 = _mm_mul_ps(self.twiddle8re, x10p19);
3580 let t_a5_11 = _mm_mul_ps(self.twiddle3re, x11p18);
3581 let t_a5_12 = _mm_mul_ps(self.twiddle2re, x12p17);
3582 let t_a5_13 = _mm_mul_ps(self.twiddle7re, x13p16);
3583 let t_a5_14 = _mm_mul_ps(self.twiddle12re, x14p15);
3584 let t_a6_1 = _mm_mul_ps(self.twiddle6re, x1p28);
3585 let t_a6_2 = _mm_mul_ps(self.twiddle12re, x2p27);
3586 let t_a6_3 = _mm_mul_ps(self.twiddle11re, x3p26);
3587 let t_a6_4 = _mm_mul_ps(self.twiddle5re, x4p25);
3588 let t_a6_5 = _mm_mul_ps(self.twiddle1re, x5p24);
3589 let t_a6_6 = _mm_mul_ps(self.twiddle7re, x6p23);
3590 let t_a6_7 = _mm_mul_ps(self.twiddle13re, x7p22);
3591 let t_a6_8 = _mm_mul_ps(self.twiddle10re, x8p21);
3592 let t_a6_9 = _mm_mul_ps(self.twiddle4re, x9p20);
3593 let t_a6_10 = _mm_mul_ps(self.twiddle2re, x10p19);
3594 let t_a6_11 = _mm_mul_ps(self.twiddle8re, x11p18);
3595 let t_a6_12 = _mm_mul_ps(self.twiddle14re, x12p17);
3596 let t_a6_13 = _mm_mul_ps(self.twiddle9re, x13p16);
3597 let t_a6_14 = _mm_mul_ps(self.twiddle3re, x14p15);
3598 let t_a7_1 = _mm_mul_ps(self.twiddle7re, x1p28);
3599 let t_a7_2 = _mm_mul_ps(self.twiddle14re, x2p27);
3600 let t_a7_3 = _mm_mul_ps(self.twiddle8re, x3p26);
3601 let t_a7_4 = _mm_mul_ps(self.twiddle1re, x4p25);
3602 let t_a7_5 = _mm_mul_ps(self.twiddle6re, x5p24);
3603 let t_a7_6 = _mm_mul_ps(self.twiddle13re, x6p23);
3604 let t_a7_7 = _mm_mul_ps(self.twiddle9re, x7p22);
3605 let t_a7_8 = _mm_mul_ps(self.twiddle2re, x8p21);
3606 let t_a7_9 = _mm_mul_ps(self.twiddle5re, x9p20);
3607 let t_a7_10 = _mm_mul_ps(self.twiddle12re, x10p19);
3608 let t_a7_11 = _mm_mul_ps(self.twiddle10re, x11p18);
3609 let t_a7_12 = _mm_mul_ps(self.twiddle3re, x12p17);
3610 let t_a7_13 = _mm_mul_ps(self.twiddle4re, x13p16);
3611 let t_a7_14 = _mm_mul_ps(self.twiddle11re, x14p15);
3612 let t_a8_1 = _mm_mul_ps(self.twiddle8re, x1p28);
3613 let t_a8_2 = _mm_mul_ps(self.twiddle13re, x2p27);
3614 let t_a8_3 = _mm_mul_ps(self.twiddle5re, x3p26);
3615 let t_a8_4 = _mm_mul_ps(self.twiddle3re, x4p25);
3616 let t_a8_5 = _mm_mul_ps(self.twiddle11re, x5p24);
3617 let t_a8_6 = _mm_mul_ps(self.twiddle10re, x6p23);
3618 let t_a8_7 = _mm_mul_ps(self.twiddle2re, x7p22);
3619 let t_a8_8 = _mm_mul_ps(self.twiddle6re, x8p21);
3620 let t_a8_9 = _mm_mul_ps(self.twiddle14re, x9p20);
3621 let t_a8_10 = _mm_mul_ps(self.twiddle7re, x10p19);
3622 let t_a8_11 = _mm_mul_ps(self.twiddle1re, x11p18);
3623 let t_a8_12 = _mm_mul_ps(self.twiddle9re, x12p17);
3624 let t_a8_13 = _mm_mul_ps(self.twiddle12re, x13p16);
3625 let t_a8_14 = _mm_mul_ps(self.twiddle4re, x14p15);
3626 let t_a9_1 = _mm_mul_ps(self.twiddle9re, x1p28);
3627 let t_a9_2 = _mm_mul_ps(self.twiddle11re, x2p27);
3628 let t_a9_3 = _mm_mul_ps(self.twiddle2re, x3p26);
3629 let t_a9_4 = _mm_mul_ps(self.twiddle7re, x4p25);
3630 let t_a9_5 = _mm_mul_ps(self.twiddle13re, x5p24);
3631 let t_a9_6 = _mm_mul_ps(self.twiddle4re, x6p23);
3632 let t_a9_7 = _mm_mul_ps(self.twiddle5re, x7p22);
3633 let t_a9_8 = _mm_mul_ps(self.twiddle14re, x8p21);
3634 let t_a9_9 = _mm_mul_ps(self.twiddle6re, x9p20);
3635 let t_a9_10 = _mm_mul_ps(self.twiddle3re, x10p19);
3636 let t_a9_11 = _mm_mul_ps(self.twiddle12re, x11p18);
3637 let t_a9_12 = _mm_mul_ps(self.twiddle8re, x12p17);
3638 let t_a9_13 = _mm_mul_ps(self.twiddle1re, x13p16);
3639 let t_a9_14 = _mm_mul_ps(self.twiddle10re, x14p15);
3640 let t_a10_1 = _mm_mul_ps(self.twiddle10re, x1p28);
3641 let t_a10_2 = _mm_mul_ps(self.twiddle9re, x2p27);
3642 let t_a10_3 = _mm_mul_ps(self.twiddle1re, x3p26);
3643 let t_a10_4 = _mm_mul_ps(self.twiddle11re, x4p25);
3644 let t_a10_5 = _mm_mul_ps(self.twiddle8re, x5p24);
3645 let t_a10_6 = _mm_mul_ps(self.twiddle2re, x6p23);
3646 let t_a10_7 = _mm_mul_ps(self.twiddle12re, x7p22);
3647 let t_a10_8 = _mm_mul_ps(self.twiddle7re, x8p21);
3648 let t_a10_9 = _mm_mul_ps(self.twiddle3re, x9p20);
3649 let t_a10_10 = _mm_mul_ps(self.twiddle13re, x10p19);
3650 let t_a10_11 = _mm_mul_ps(self.twiddle6re, x11p18);
3651 let t_a10_12 = _mm_mul_ps(self.twiddle4re, x12p17);
3652 let t_a10_13 = _mm_mul_ps(self.twiddle14re, x13p16);
3653 let t_a10_14 = _mm_mul_ps(self.twiddle5re, x14p15);
3654 let t_a11_1 = _mm_mul_ps(self.twiddle11re, x1p28);
3655 let t_a11_2 = _mm_mul_ps(self.twiddle7re, x2p27);
3656 let t_a11_3 = _mm_mul_ps(self.twiddle4re, x3p26);
3657 let t_a11_4 = _mm_mul_ps(self.twiddle14re, x4p25);
3658 let t_a11_5 = _mm_mul_ps(self.twiddle3re, x5p24);
3659 let t_a11_6 = _mm_mul_ps(self.twiddle8re, x6p23);
3660 let t_a11_7 = _mm_mul_ps(self.twiddle10re, x7p22);
3661 let t_a11_8 = _mm_mul_ps(self.twiddle1re, x8p21);
3662 let t_a11_9 = _mm_mul_ps(self.twiddle12re, x9p20);
3663 let t_a11_10 = _mm_mul_ps(self.twiddle6re, x10p19);
3664 let t_a11_11 = _mm_mul_ps(self.twiddle5re, x11p18);
3665 let t_a11_12 = _mm_mul_ps(self.twiddle13re, x12p17);
3666 let t_a11_13 = _mm_mul_ps(self.twiddle2re, x13p16);
3667 let t_a11_14 = _mm_mul_ps(self.twiddle9re, x14p15);
3668 let t_a12_1 = _mm_mul_ps(self.twiddle12re, x1p28);
3669 let t_a12_2 = _mm_mul_ps(self.twiddle5re, x2p27);
3670 let t_a12_3 = _mm_mul_ps(self.twiddle7re, x3p26);
3671 let t_a12_4 = _mm_mul_ps(self.twiddle10re, x4p25);
3672 let t_a12_5 = _mm_mul_ps(self.twiddle2re, x5p24);
3673 let t_a12_6 = _mm_mul_ps(self.twiddle14re, x6p23);
3674 let t_a12_7 = _mm_mul_ps(self.twiddle3re, x7p22);
3675 let t_a12_8 = _mm_mul_ps(self.twiddle9re, x8p21);
3676 let t_a12_9 = _mm_mul_ps(self.twiddle8re, x9p20);
3677 let t_a12_10 = _mm_mul_ps(self.twiddle4re, x10p19);
3678 let t_a12_11 = _mm_mul_ps(self.twiddle13re, x11p18);
3679 let t_a12_12 = _mm_mul_ps(self.twiddle1re, x12p17);
3680 let t_a12_13 = _mm_mul_ps(self.twiddle11re, x13p16);
3681 let t_a12_14 = _mm_mul_ps(self.twiddle6re, x14p15);
3682 let t_a13_1 = _mm_mul_ps(self.twiddle13re, x1p28);
3683 let t_a13_2 = _mm_mul_ps(self.twiddle3re, x2p27);
3684 let t_a13_3 = _mm_mul_ps(self.twiddle10re, x3p26);
3685 let t_a13_4 = _mm_mul_ps(self.twiddle6re, x4p25);
3686 let t_a13_5 = _mm_mul_ps(self.twiddle7re, x5p24);
3687 let t_a13_6 = _mm_mul_ps(self.twiddle9re, x6p23);
3688 let t_a13_7 = _mm_mul_ps(self.twiddle4re, x7p22);
3689 let t_a13_8 = _mm_mul_ps(self.twiddle12re, x8p21);
3690 let t_a13_9 = _mm_mul_ps(self.twiddle1re, x9p20);
3691 let t_a13_10 = _mm_mul_ps(self.twiddle14re, x10p19);
3692 let t_a13_11 = _mm_mul_ps(self.twiddle2re, x11p18);
3693 let t_a13_12 = _mm_mul_ps(self.twiddle11re, x12p17);
3694 let t_a13_13 = _mm_mul_ps(self.twiddle5re, x13p16);
3695 let t_a13_14 = _mm_mul_ps(self.twiddle8re, x14p15);
3696 let t_a14_1 = _mm_mul_ps(self.twiddle14re, x1p28);
3697 let t_a14_2 = _mm_mul_ps(self.twiddle1re, x2p27);
3698 let t_a14_3 = _mm_mul_ps(self.twiddle13re, x3p26);
3699 let t_a14_4 = _mm_mul_ps(self.twiddle2re, x4p25);
3700 let t_a14_5 = _mm_mul_ps(self.twiddle12re, x5p24);
3701 let t_a14_6 = _mm_mul_ps(self.twiddle3re, x6p23);
3702 let t_a14_7 = _mm_mul_ps(self.twiddle11re, x7p22);
3703 let t_a14_8 = _mm_mul_ps(self.twiddle4re, x8p21);
3704 let t_a14_9 = _mm_mul_ps(self.twiddle10re, x9p20);
3705 let t_a14_10 = _mm_mul_ps(self.twiddle5re, x10p19);
3706 let t_a14_11 = _mm_mul_ps(self.twiddle9re, x11p18);
3707 let t_a14_12 = _mm_mul_ps(self.twiddle6re, x12p17);
3708 let t_a14_13 = _mm_mul_ps(self.twiddle8re, x13p16);
3709 let t_a14_14 = _mm_mul_ps(self.twiddle7re, x14p15);
3710
3711 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m28);
3712 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m27);
3713 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m26);
3714 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m25);
3715 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m24);
3716 let t_b1_6 = _mm_mul_ps(self.twiddle6im, x6m23);
3717 let t_b1_7 = _mm_mul_ps(self.twiddle7im, x7m22);
3718 let t_b1_8 = _mm_mul_ps(self.twiddle8im, x8m21);
3719 let t_b1_9 = _mm_mul_ps(self.twiddle9im, x9m20);
3720 let t_b1_10 = _mm_mul_ps(self.twiddle10im, x10m19);
3721 let t_b1_11 = _mm_mul_ps(self.twiddle11im, x11m18);
3722 let t_b1_12 = _mm_mul_ps(self.twiddle12im, x12m17);
3723 let t_b1_13 = _mm_mul_ps(self.twiddle13im, x13m16);
3724 let t_b1_14 = _mm_mul_ps(self.twiddle14im, x14m15);
3725 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m28);
3726 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m27);
3727 let t_b2_3 = _mm_mul_ps(self.twiddle6im, x3m26);
3728 let t_b2_4 = _mm_mul_ps(self.twiddle8im, x4m25);
3729 let t_b2_5 = _mm_mul_ps(self.twiddle10im, x5m24);
3730 let t_b2_6 = _mm_mul_ps(self.twiddle12im, x6m23);
3731 let t_b2_7 = _mm_mul_ps(self.twiddle14im, x7m22);
3732 let t_b2_8 = _mm_mul_ps(self.twiddle13im, x8m21);
3733 let t_b2_9 = _mm_mul_ps(self.twiddle11im, x9m20);
3734 let t_b2_10 = _mm_mul_ps(self.twiddle9im, x10m19);
3735 let t_b2_11 = _mm_mul_ps(self.twiddle7im, x11m18);
3736 let t_b2_12 = _mm_mul_ps(self.twiddle5im, x12m17);
3737 let t_b2_13 = _mm_mul_ps(self.twiddle3im, x13m16);
3738 let t_b2_14 = _mm_mul_ps(self.twiddle1im, x14m15);
3739 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m28);
3740 let t_b3_2 = _mm_mul_ps(self.twiddle6im, x2m27);
3741 let t_b3_3 = _mm_mul_ps(self.twiddle9im, x3m26);
3742 let t_b3_4 = _mm_mul_ps(self.twiddle12im, x4m25);
3743 let t_b3_5 = _mm_mul_ps(self.twiddle14im, x5m24);
3744 let t_b3_6 = _mm_mul_ps(self.twiddle11im, x6m23);
3745 let t_b3_7 = _mm_mul_ps(self.twiddle8im, x7m22);
3746 let t_b3_8 = _mm_mul_ps(self.twiddle5im, x8m21);
3747 let t_b3_9 = _mm_mul_ps(self.twiddle2im, x9m20);
3748 let t_b3_10 = _mm_mul_ps(self.twiddle1im, x10m19);
3749 let t_b3_11 = _mm_mul_ps(self.twiddle4im, x11m18);
3750 let t_b3_12 = _mm_mul_ps(self.twiddle7im, x12m17);
3751 let t_b3_13 = _mm_mul_ps(self.twiddle10im, x13m16);
3752 let t_b3_14 = _mm_mul_ps(self.twiddle13im, x14m15);
3753 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m28);
3754 let t_b4_2 = _mm_mul_ps(self.twiddle8im, x2m27);
3755 let t_b4_3 = _mm_mul_ps(self.twiddle12im, x3m26);
3756 let t_b4_4 = _mm_mul_ps(self.twiddle13im, x4m25);
3757 let t_b4_5 = _mm_mul_ps(self.twiddle9im, x5m24);
3758 let t_b4_6 = _mm_mul_ps(self.twiddle5im, x6m23);
3759 let t_b4_7 = _mm_mul_ps(self.twiddle1im, x7m22);
3760 let t_b4_8 = _mm_mul_ps(self.twiddle3im, x8m21);
3761 let t_b4_9 = _mm_mul_ps(self.twiddle7im, x9m20);
3762 let t_b4_10 = _mm_mul_ps(self.twiddle11im, x10m19);
3763 let t_b4_11 = _mm_mul_ps(self.twiddle14im, x11m18);
3764 let t_b4_12 = _mm_mul_ps(self.twiddle10im, x12m17);
3765 let t_b4_13 = _mm_mul_ps(self.twiddle6im, x13m16);
3766 let t_b4_14 = _mm_mul_ps(self.twiddle2im, x14m15);
3767 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m28);
3768 let t_b5_2 = _mm_mul_ps(self.twiddle10im, x2m27);
3769 let t_b5_3 = _mm_mul_ps(self.twiddle14im, x3m26);
3770 let t_b5_4 = _mm_mul_ps(self.twiddle9im, x4m25);
3771 let t_b5_5 = _mm_mul_ps(self.twiddle4im, x5m24);
3772 let t_b5_6 = _mm_mul_ps(self.twiddle1im, x6m23);
3773 let t_b5_7 = _mm_mul_ps(self.twiddle6im, x7m22);
3774 let t_b5_8 = _mm_mul_ps(self.twiddle11im, x8m21);
3775 let t_b5_9 = _mm_mul_ps(self.twiddle13im, x9m20);
3776 let t_b5_10 = _mm_mul_ps(self.twiddle8im, x10m19);
3777 let t_b5_11 = _mm_mul_ps(self.twiddle3im, x11m18);
3778 let t_b5_12 = _mm_mul_ps(self.twiddle2im, x12m17);
3779 let t_b5_13 = _mm_mul_ps(self.twiddle7im, x13m16);
3780 let t_b5_14 = _mm_mul_ps(self.twiddle12im, x14m15);
3781 let t_b6_1 = _mm_mul_ps(self.twiddle6im, x1m28);
3782 let t_b6_2 = _mm_mul_ps(self.twiddle12im, x2m27);
3783 let t_b6_3 = _mm_mul_ps(self.twiddle11im, x3m26);
3784 let t_b6_4 = _mm_mul_ps(self.twiddle5im, x4m25);
3785 let t_b6_5 = _mm_mul_ps(self.twiddle1im, x5m24);
3786 let t_b6_6 = _mm_mul_ps(self.twiddle7im, x6m23);
3787 let t_b6_7 = _mm_mul_ps(self.twiddle13im, x7m22);
3788 let t_b6_8 = _mm_mul_ps(self.twiddle10im, x8m21);
3789 let t_b6_9 = _mm_mul_ps(self.twiddle4im, x9m20);
3790 let t_b6_10 = _mm_mul_ps(self.twiddle2im, x10m19);
3791 let t_b6_11 = _mm_mul_ps(self.twiddle8im, x11m18);
3792 let t_b6_12 = _mm_mul_ps(self.twiddle14im, x12m17);
3793 let t_b6_13 = _mm_mul_ps(self.twiddle9im, x13m16);
3794 let t_b6_14 = _mm_mul_ps(self.twiddle3im, x14m15);
3795 let t_b7_1 = _mm_mul_ps(self.twiddle7im, x1m28);
3796 let t_b7_2 = _mm_mul_ps(self.twiddle14im, x2m27);
3797 let t_b7_3 = _mm_mul_ps(self.twiddle8im, x3m26);
3798 let t_b7_4 = _mm_mul_ps(self.twiddle1im, x4m25);
3799 let t_b7_5 = _mm_mul_ps(self.twiddle6im, x5m24);
3800 let t_b7_6 = _mm_mul_ps(self.twiddle13im, x6m23);
3801 let t_b7_7 = _mm_mul_ps(self.twiddle9im, x7m22);
3802 let t_b7_8 = _mm_mul_ps(self.twiddle2im, x8m21);
3803 let t_b7_9 = _mm_mul_ps(self.twiddle5im, x9m20);
3804 let t_b7_10 = _mm_mul_ps(self.twiddle12im, x10m19);
3805 let t_b7_11 = _mm_mul_ps(self.twiddle10im, x11m18);
3806 let t_b7_12 = _mm_mul_ps(self.twiddle3im, x12m17);
3807 let t_b7_13 = _mm_mul_ps(self.twiddle4im, x13m16);
3808 let t_b7_14 = _mm_mul_ps(self.twiddle11im, x14m15);
3809 let t_b8_1 = _mm_mul_ps(self.twiddle8im, x1m28);
3810 let t_b8_2 = _mm_mul_ps(self.twiddle13im, x2m27);
3811 let t_b8_3 = _mm_mul_ps(self.twiddle5im, x3m26);
3812 let t_b8_4 = _mm_mul_ps(self.twiddle3im, x4m25);
3813 let t_b8_5 = _mm_mul_ps(self.twiddle11im, x5m24);
3814 let t_b8_6 = _mm_mul_ps(self.twiddle10im, x6m23);
3815 let t_b8_7 = _mm_mul_ps(self.twiddle2im, x7m22);
3816 let t_b8_8 = _mm_mul_ps(self.twiddle6im, x8m21);
3817 let t_b8_9 = _mm_mul_ps(self.twiddle14im, x9m20);
3818 let t_b8_10 = _mm_mul_ps(self.twiddle7im, x10m19);
3819 let t_b8_11 = _mm_mul_ps(self.twiddle1im, x11m18);
3820 let t_b8_12 = _mm_mul_ps(self.twiddle9im, x12m17);
3821 let t_b8_13 = _mm_mul_ps(self.twiddle12im, x13m16);
3822 let t_b8_14 = _mm_mul_ps(self.twiddle4im, x14m15);
3823 let t_b9_1 = _mm_mul_ps(self.twiddle9im, x1m28);
3824 let t_b9_2 = _mm_mul_ps(self.twiddle11im, x2m27);
3825 let t_b9_3 = _mm_mul_ps(self.twiddle2im, x3m26);
3826 let t_b9_4 = _mm_mul_ps(self.twiddle7im, x4m25);
3827 let t_b9_5 = _mm_mul_ps(self.twiddle13im, x5m24);
3828 let t_b9_6 = _mm_mul_ps(self.twiddle4im, x6m23);
3829 let t_b9_7 = _mm_mul_ps(self.twiddle5im, x7m22);
3830 let t_b9_8 = _mm_mul_ps(self.twiddle14im, x8m21);
3831 let t_b9_9 = _mm_mul_ps(self.twiddle6im, x9m20);
3832 let t_b9_10 = _mm_mul_ps(self.twiddle3im, x10m19);
3833 let t_b9_11 = _mm_mul_ps(self.twiddle12im, x11m18);
3834 let t_b9_12 = _mm_mul_ps(self.twiddle8im, x12m17);
3835 let t_b9_13 = _mm_mul_ps(self.twiddle1im, x13m16);
3836 let t_b9_14 = _mm_mul_ps(self.twiddle10im, x14m15);
3837 let t_b10_1 = _mm_mul_ps(self.twiddle10im, x1m28);
3838 let t_b10_2 = _mm_mul_ps(self.twiddle9im, x2m27);
3839 let t_b10_3 = _mm_mul_ps(self.twiddle1im, x3m26);
3840 let t_b10_4 = _mm_mul_ps(self.twiddle11im, x4m25);
3841 let t_b10_5 = _mm_mul_ps(self.twiddle8im, x5m24);
3842 let t_b10_6 = _mm_mul_ps(self.twiddle2im, x6m23);
3843 let t_b10_7 = _mm_mul_ps(self.twiddle12im, x7m22);
3844 let t_b10_8 = _mm_mul_ps(self.twiddle7im, x8m21);
3845 let t_b10_9 = _mm_mul_ps(self.twiddle3im, x9m20);
3846 let t_b10_10 = _mm_mul_ps(self.twiddle13im, x10m19);
3847 let t_b10_11 = _mm_mul_ps(self.twiddle6im, x11m18);
3848 let t_b10_12 = _mm_mul_ps(self.twiddle4im, x12m17);
3849 let t_b10_13 = _mm_mul_ps(self.twiddle14im, x13m16);
3850 let t_b10_14 = _mm_mul_ps(self.twiddle5im, x14m15);
3851 let t_b11_1 = _mm_mul_ps(self.twiddle11im, x1m28);
3852 let t_b11_2 = _mm_mul_ps(self.twiddle7im, x2m27);
3853 let t_b11_3 = _mm_mul_ps(self.twiddle4im, x3m26);
3854 let t_b11_4 = _mm_mul_ps(self.twiddle14im, x4m25);
3855 let t_b11_5 = _mm_mul_ps(self.twiddle3im, x5m24);
3856 let t_b11_6 = _mm_mul_ps(self.twiddle8im, x6m23);
3857 let t_b11_7 = _mm_mul_ps(self.twiddle10im, x7m22);
3858 let t_b11_8 = _mm_mul_ps(self.twiddle1im, x8m21);
3859 let t_b11_9 = _mm_mul_ps(self.twiddle12im, x9m20);
3860 let t_b11_10 = _mm_mul_ps(self.twiddle6im, x10m19);
3861 let t_b11_11 = _mm_mul_ps(self.twiddle5im, x11m18);
3862 let t_b11_12 = _mm_mul_ps(self.twiddle13im, x12m17);
3863 let t_b11_13 = _mm_mul_ps(self.twiddle2im, x13m16);
3864 let t_b11_14 = _mm_mul_ps(self.twiddle9im, x14m15);
3865 let t_b12_1 = _mm_mul_ps(self.twiddle12im, x1m28);
3866 let t_b12_2 = _mm_mul_ps(self.twiddle5im, x2m27);
3867 let t_b12_3 = _mm_mul_ps(self.twiddle7im, x3m26);
3868 let t_b12_4 = _mm_mul_ps(self.twiddle10im, x4m25);
3869 let t_b12_5 = _mm_mul_ps(self.twiddle2im, x5m24);
3870 let t_b12_6 = _mm_mul_ps(self.twiddle14im, x6m23);
3871 let t_b12_7 = _mm_mul_ps(self.twiddle3im, x7m22);
3872 let t_b12_8 = _mm_mul_ps(self.twiddle9im, x8m21);
3873 let t_b12_9 = _mm_mul_ps(self.twiddle8im, x9m20);
3874 let t_b12_10 = _mm_mul_ps(self.twiddle4im, x10m19);
3875 let t_b12_11 = _mm_mul_ps(self.twiddle13im, x11m18);
3876 let t_b12_12 = _mm_mul_ps(self.twiddle1im, x12m17);
3877 let t_b12_13 = _mm_mul_ps(self.twiddle11im, x13m16);
3878 let t_b12_14 = _mm_mul_ps(self.twiddle6im, x14m15);
3879 let t_b13_1 = _mm_mul_ps(self.twiddle13im, x1m28);
3880 let t_b13_2 = _mm_mul_ps(self.twiddle3im, x2m27);
3881 let t_b13_3 = _mm_mul_ps(self.twiddle10im, x3m26);
3882 let t_b13_4 = _mm_mul_ps(self.twiddle6im, x4m25);
3883 let t_b13_5 = _mm_mul_ps(self.twiddle7im, x5m24);
3884 let t_b13_6 = _mm_mul_ps(self.twiddle9im, x6m23);
3885 let t_b13_7 = _mm_mul_ps(self.twiddle4im, x7m22);
3886 let t_b13_8 = _mm_mul_ps(self.twiddle12im, x8m21);
3887 let t_b13_9 = _mm_mul_ps(self.twiddle1im, x9m20);
3888 let t_b13_10 = _mm_mul_ps(self.twiddle14im, x10m19);
3889 let t_b13_11 = _mm_mul_ps(self.twiddle2im, x11m18);
3890 let t_b13_12 = _mm_mul_ps(self.twiddle11im, x12m17);
3891 let t_b13_13 = _mm_mul_ps(self.twiddle5im, x13m16);
3892 let t_b13_14 = _mm_mul_ps(self.twiddle8im, x14m15);
3893 let t_b14_1 = _mm_mul_ps(self.twiddle14im, x1m28);
3894 let t_b14_2 = _mm_mul_ps(self.twiddle1im, x2m27);
3895 let t_b14_3 = _mm_mul_ps(self.twiddle13im, x3m26);
3896 let t_b14_4 = _mm_mul_ps(self.twiddle2im, x4m25);
3897 let t_b14_5 = _mm_mul_ps(self.twiddle12im, x5m24);
3898 let t_b14_6 = _mm_mul_ps(self.twiddle3im, x6m23);
3899 let t_b14_7 = _mm_mul_ps(self.twiddle11im, x7m22);
3900 let t_b14_8 = _mm_mul_ps(self.twiddle4im, x8m21);
3901 let t_b14_9 = _mm_mul_ps(self.twiddle10im, x9m20);
3902 let t_b14_10 = _mm_mul_ps(self.twiddle5im, x10m19);
3903 let t_b14_11 = _mm_mul_ps(self.twiddle9im, x11m18);
3904 let t_b14_12 = _mm_mul_ps(self.twiddle6im, x12m17);
3905 let t_b14_13 = _mm_mul_ps(self.twiddle8im, x13m16);
3906 let t_b14_14 = _mm_mul_ps(self.twiddle7im, x14m15);
3907
3908 let x0 = values[0];
3909 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9 + t_a1_10 + t_a1_11 + t_a1_12 + t_a1_13 + t_a1_14);
3910 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9 + t_a2_10 + t_a2_11 + t_a2_12 + t_a2_13 + t_a2_14);
3911 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9 + t_a3_10 + t_a3_11 + t_a3_12 + t_a3_13 + t_a3_14);
3912 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9 + t_a4_10 + t_a4_11 + t_a4_12 + t_a4_13 + t_a4_14);
3913 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9 + t_a5_10 + t_a5_11 + t_a5_12 + t_a5_13 + t_a5_14);
3914 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9 + t_a6_10 + t_a6_11 + t_a6_12 + t_a6_13 + t_a6_14);
3915 let t_a7 = calc_f32!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9 + t_a7_10 + t_a7_11 + t_a7_12 + t_a7_13 + t_a7_14);
3916 let t_a8 = calc_f32!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9 + t_a8_10 + t_a8_11 + t_a8_12 + t_a8_13 + t_a8_14);
3917 let t_a9 = calc_f32!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9 + t_a9_10 + t_a9_11 + t_a9_12 + t_a9_13 + t_a9_14);
3918 let t_a10 = calc_f32!(x0 + t_a10_1 + t_a10_2 + t_a10_3 + t_a10_4 + t_a10_5 + t_a10_6 + t_a10_7 + t_a10_8 + t_a10_9 + t_a10_10 + t_a10_11 + t_a10_12 + t_a10_13 + t_a10_14);
3919 let t_a11 = calc_f32!(x0 + t_a11_1 + t_a11_2 + t_a11_3 + t_a11_4 + t_a11_5 + t_a11_6 + t_a11_7 + t_a11_8 + t_a11_9 + t_a11_10 + t_a11_11 + t_a11_12 + t_a11_13 + t_a11_14);
3920 let t_a12 = calc_f32!(x0 + t_a12_1 + t_a12_2 + t_a12_3 + t_a12_4 + t_a12_5 + t_a12_6 + t_a12_7 + t_a12_8 + t_a12_9 + t_a12_10 + t_a12_11 + t_a12_12 + t_a12_13 + t_a12_14);
3921 let t_a13 = calc_f32!(x0 + t_a13_1 + t_a13_2 + t_a13_3 + t_a13_4 + t_a13_5 + t_a13_6 + t_a13_7 + t_a13_8 + t_a13_9 + t_a13_10 + t_a13_11 + t_a13_12 + t_a13_13 + t_a13_14);
3922 let t_a14 = calc_f32!(x0 + t_a14_1 + t_a14_2 + t_a14_3 + t_a14_4 + t_a14_5 + t_a14_6 + t_a14_7 + t_a14_8 + t_a14_9 + t_a14_10 + t_a14_11 + t_a14_12 + t_a14_13 + t_a14_14);
3923
3924 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9 + t_b1_10 + t_b1_11 + t_b1_12 + t_b1_13 + t_b1_14);
3925 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7 - t_b2_8 - t_b2_9 - t_b2_10 - t_b2_11 - t_b2_12 - t_b2_13 - t_b2_14);
3926 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7 - t_b3_8 - t_b3_9 + t_b3_10 + t_b3_11 + t_b3_12 + t_b3_13 + t_b3_14);
3927 let t_b4 = calc_f32!(t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7 + t_b4_8 + t_b4_9 + t_b4_10 - t_b4_11 - t_b4_12 - t_b4_13 - t_b4_14);
3928 let t_b5 = calc_f32!(t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6 + t_b5_7 + t_b5_8 - t_b5_9 - t_b5_10 - t_b5_11 + t_b5_12 + t_b5_13 + t_b5_14);
3929 let t_b6 = calc_f32!(t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 + t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9 + t_b6_10 + t_b6_11 + t_b6_12 - t_b6_13 - t_b6_14);
3930 let t_b7 = calc_f32!(t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9 + t_b7_10 - t_b7_11 - t_b7_12 + t_b7_13 + t_b7_14);
3931 let t_b8 = calc_f32!(t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9 - t_b8_10 + t_b8_11 + t_b8_12 - t_b8_13 - t_b8_14);
3932 let t_b9 = calc_f32!(t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 - t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8 - t_b9_9 + t_b9_10 + t_b9_11 - t_b9_12 + t_b9_13 + t_b9_14);
3933 let t_b10 = calc_f32!(t_b10_1 - t_b10_2 + t_b10_3 + t_b10_4 - t_b10_5 + t_b10_6 + t_b10_7 - t_b10_8 + t_b10_9 + t_b10_10 - t_b10_11 + t_b10_12 + t_b10_13 - t_b10_14);
3934 let t_b11 = calc_f32!(t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 - t_b11_5 + t_b11_6 - t_b11_7 + t_b11_8 + t_b11_9 - t_b11_10 + t_b11_11 - t_b11_12 - t_b11_13 + t_b11_14);
3935 let t_b12 = calc_f32!(t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 + t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 - t_b12_9 + t_b12_10 - t_b12_11 - t_b12_12 + t_b12_13 - t_b12_14);
3936 let t_b13 = calc_f32!(t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 + t_b13_7 - t_b13_8 + t_b13_9 + t_b13_10 - t_b13_11 + t_b13_12 - t_b13_13 + t_b13_14);
3937 let t_b14 = calc_f32!(t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9 - t_b14_10 + t_b14_11 - t_b14_12 + t_b14_13 - t_b14_14);
3938
3939 let t_b1_rot = self.rotate.rotate_both(t_b1);
3940 let t_b2_rot = self.rotate.rotate_both(t_b2);
3941 let t_b3_rot = self.rotate.rotate_both(t_b3);
3942 let t_b4_rot = self.rotate.rotate_both(t_b4);
3943 let t_b5_rot = self.rotate.rotate_both(t_b5);
3944 let t_b6_rot = self.rotate.rotate_both(t_b6);
3945 let t_b7_rot = self.rotate.rotate_both(t_b7);
3946 let t_b8_rot = self.rotate.rotate_both(t_b8);
3947 let t_b9_rot = self.rotate.rotate_both(t_b9);
3948 let t_b10_rot = self.rotate.rotate_both(t_b10);
3949 let t_b11_rot = self.rotate.rotate_both(t_b11);
3950 let t_b12_rot = self.rotate.rotate_both(t_b12);
3951 let t_b13_rot = self.rotate.rotate_both(t_b13);
3952 let t_b14_rot = self.rotate.rotate_both(t_b14);
3953
3954 let y0 = calc_f32!(x0 + x1p28 + x2p27 + x3p26 + x4p25 + x5p24 + x6p23 + x7p22 + x8p21 + x9p20 + x10p19 + x11p18 + x12p17 + x13p16 + x14p15);
3955 let [y1, y28] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
3956 let [y2, y27] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
3957 let [y3, y26] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
3958 let [y4, y25] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
3959 let [y5, y24] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
3960 let [y6, y23] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
3961 let [y7, y22] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
3962 let [y8, y21] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
3963 let [y9, y20] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
3964 let [y10, y19] = parallel_fft2_interleaved_f32(t_a10, t_b10_rot);
3965 let [y11, y18] = parallel_fft2_interleaved_f32(t_a11, t_b11_rot);
3966 let [y12, y17] = parallel_fft2_interleaved_f32(t_a12, t_b12_rot);
3967 let [y13, y16] = parallel_fft2_interleaved_f32(t_a13, t_b13_rot);
3968 let [y14, y15] = parallel_fft2_interleaved_f32(t_a14, t_b14_rot);
3969 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28]
3970 }
3971 }
3972
3973 // ____ ___ __ _ _ _ _ _
3974 // |___ \ / _ \ / /_ | || | | |__ (_) |_
3975 // __) | (_) | _____ | '_ \| || |_| '_ \| | __|
3976 // / __/ \__, | |_____| | (_) |__ _| |_) | | |_
3977 // |_____| /_/ \___/ |_| |_.__/|_|\__|
3978 //
3979
3980 pub struct SseF64Butterfly29<T> {
3981 direction: FftDirection,
3982 _phantom: std::marker::PhantomData<T>,
3983 rotate: Rotate90F64,
3984 twiddle1re: __m128d,
3985 twiddle1im: __m128d,
3986 twiddle2re: __m128d,
3987 twiddle2im: __m128d,
3988 twiddle3re: __m128d,
3989 twiddle3im: __m128d,
3990 twiddle4re: __m128d,
3991 twiddle4im: __m128d,
3992 twiddle5re: __m128d,
3993 twiddle5im: __m128d,
3994 twiddle6re: __m128d,
3995 twiddle6im: __m128d,
3996 twiddle7re: __m128d,
3997 twiddle7im: __m128d,
3998 twiddle8re: __m128d,
3999 twiddle8im: __m128d,
4000 twiddle9re: __m128d,
4001 twiddle9im: __m128d,
4002 twiddle10re: __m128d,
4003 twiddle10im: __m128d,
4004 twiddle11re: __m128d,
4005 twiddle11im: __m128d,
4006 twiddle12re: __m128d,
4007 twiddle12im: __m128d,
4008 twiddle13re: __m128d,
4009 twiddle13im: __m128d,
4010 twiddle14re: __m128d,
4011 twiddle14im: __m128d,
4012 }
4013
4014 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly29, 29, |this: &SseF64Butterfly29<_>| this
4015 .direction);
4016 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly29, 29, |this: &SseF64Butterfly29<_>| this
4017 .direction);
4018 impl<T: FftNum> SseF64Butterfly29<T> {
4019 #[inline(always)]
4020 pub fn new(direction: FftDirection) -> Self {
4021 assert_f64::<T>();
4022 let rotate = Rotate90F64::new(true);
4023 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 29, direction);
4024 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 29, direction);
4025 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 29, direction);
4026 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 29, direction);
4027 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 29, direction);
4028 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 29, direction);
4029 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 29, direction);
4030 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 29, direction);
4031 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 29, direction);
4032 let tw10: Complex<f64> = twiddles::compute_twiddle(10, 29, direction);
4033 let tw11: Complex<f64> = twiddles::compute_twiddle(11, 29, direction);
4034 let tw12: Complex<f64> = twiddles::compute_twiddle(12, 29, direction);
4035 let tw13: Complex<f64> = twiddles::compute_twiddle(13, 29, direction);
4036 let tw14: Complex<f64> = twiddles::compute_twiddle(14, 29, direction);
4037 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
4038 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
4039 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
4040 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
4041 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
4042 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
4043 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
4044 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
4045 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
4046 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
4047 let twiddle6re = unsafe { _mm_set_pd(tw6.re, tw6.re) };
4048 let twiddle6im = unsafe { _mm_set_pd(tw6.im, tw6.im) };
4049 let twiddle7re = unsafe { _mm_set_pd(tw7.re, tw7.re) };
4050 let twiddle7im = unsafe { _mm_set_pd(tw7.im, tw7.im) };
4051 let twiddle8re = unsafe { _mm_set_pd(tw8.re, tw8.re) };
4052 let twiddle8im = unsafe { _mm_set_pd(tw8.im, tw8.im) };
4053 let twiddle9re = unsafe { _mm_set_pd(tw9.re, tw9.re) };
4054 let twiddle9im = unsafe { _mm_set_pd(tw9.im, tw9.im) };
4055 let twiddle10re = unsafe { _mm_set_pd(tw10.re, tw10.re) };
4056 let twiddle10im = unsafe { _mm_set_pd(tw10.im, tw10.im) };
4057 let twiddle11re = unsafe { _mm_set_pd(tw11.re, tw11.re) };
4058 let twiddle11im = unsafe { _mm_set_pd(tw11.im, tw11.im) };
4059 let twiddle12re = unsafe { _mm_set_pd(tw12.re, tw12.re) };
4060 let twiddle12im = unsafe { _mm_set_pd(tw12.im, tw12.im) };
4061 let twiddle13re = unsafe { _mm_set_pd(tw13.re, tw13.re) };
4062 let twiddle13im = unsafe { _mm_set_pd(tw13.im, tw13.im) };
4063 let twiddle14re = unsafe { _mm_set_pd(tw14.re, tw14.re) };
4064 let twiddle14im = unsafe { _mm_set_pd(tw14.im, tw14.im) };
4065
4066 Self {
4067 direction,
4068 _phantom: std::marker::PhantomData,
4069 rotate,
4070 twiddle1re,
4071 twiddle1im,
4072 twiddle2re,
4073 twiddle2im,
4074 twiddle3re,
4075 twiddle3im,
4076 twiddle4re,
4077 twiddle4im,
4078 twiddle5re,
4079 twiddle5im,
4080 twiddle6re,
4081 twiddle6im,
4082 twiddle7re,
4083 twiddle7im,
4084 twiddle8re,
4085 twiddle8im,
4086 twiddle9re,
4087 twiddle9im,
4088 twiddle10re,
4089 twiddle10im,
4090 twiddle11re,
4091 twiddle11im,
4092 twiddle12re,
4093 twiddle12im,
4094 twiddle13re,
4095 twiddle13im,
4096 twiddle14re,
4097 twiddle14im,
4098 }
4099 }
4100
4101 #[inline(always)]
4102 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
4103 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
4104
4105 let out = self.perform_fft_direct(values);
4106
4107 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28});
4108 }
4109
4110 #[inline(always)]
4111 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 29]) -> [__m128d; 29] {
4112 let [x1p28, x1m28] = solo_fft2_f64(values[1], values[28]);
4113 let [x2p27, x2m27] = solo_fft2_f64(values[2], values[27]);
4114 let [x3p26, x3m26] = solo_fft2_f64(values[3], values[26]);
4115 let [x4p25, x4m25] = solo_fft2_f64(values[4], values[25]);
4116 let [x5p24, x5m24] = solo_fft2_f64(values[5], values[24]);
4117 let [x6p23, x6m23] = solo_fft2_f64(values[6], values[23]);
4118 let [x7p22, x7m22] = solo_fft2_f64(values[7], values[22]);
4119 let [x8p21, x8m21] = solo_fft2_f64(values[8], values[21]);
4120 let [x9p20, x9m20] = solo_fft2_f64(values[9], values[20]);
4121 let [x10p19, x10m19] = solo_fft2_f64(values[10], values[19]);
4122 let [x11p18, x11m18] = solo_fft2_f64(values[11], values[18]);
4123 let [x12p17, x12m17] = solo_fft2_f64(values[12], values[17]);
4124 let [x13p16, x13m16] = solo_fft2_f64(values[13], values[16]);
4125 let [x14p15, x14m15] = solo_fft2_f64(values[14], values[15]);
4126
4127 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p28);
4128 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p27);
4129 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p26);
4130 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p25);
4131 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p24);
4132 let t_a1_6 = _mm_mul_pd(self.twiddle6re, x6p23);
4133 let t_a1_7 = _mm_mul_pd(self.twiddle7re, x7p22);
4134 let t_a1_8 = _mm_mul_pd(self.twiddle8re, x8p21);
4135 let t_a1_9 = _mm_mul_pd(self.twiddle9re, x9p20);
4136 let t_a1_10 = _mm_mul_pd(self.twiddle10re, x10p19);
4137 let t_a1_11 = _mm_mul_pd(self.twiddle11re, x11p18);
4138 let t_a1_12 = _mm_mul_pd(self.twiddle12re, x12p17);
4139 let t_a1_13 = _mm_mul_pd(self.twiddle13re, x13p16);
4140 let t_a1_14 = _mm_mul_pd(self.twiddle14re, x14p15);
4141 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p28);
4142 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p27);
4143 let t_a2_3 = _mm_mul_pd(self.twiddle6re, x3p26);
4144 let t_a2_4 = _mm_mul_pd(self.twiddle8re, x4p25);
4145 let t_a2_5 = _mm_mul_pd(self.twiddle10re, x5p24);
4146 let t_a2_6 = _mm_mul_pd(self.twiddle12re, x6p23);
4147 let t_a2_7 = _mm_mul_pd(self.twiddle14re, x7p22);
4148 let t_a2_8 = _mm_mul_pd(self.twiddle13re, x8p21);
4149 let t_a2_9 = _mm_mul_pd(self.twiddle11re, x9p20);
4150 let t_a2_10 = _mm_mul_pd(self.twiddle9re, x10p19);
4151 let t_a2_11 = _mm_mul_pd(self.twiddle7re, x11p18);
4152 let t_a2_12 = _mm_mul_pd(self.twiddle5re, x12p17);
4153 let t_a2_13 = _mm_mul_pd(self.twiddle3re, x13p16);
4154 let t_a2_14 = _mm_mul_pd(self.twiddle1re, x14p15);
4155 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p28);
4156 let t_a3_2 = _mm_mul_pd(self.twiddle6re, x2p27);
4157 let t_a3_3 = _mm_mul_pd(self.twiddle9re, x3p26);
4158 let t_a3_4 = _mm_mul_pd(self.twiddle12re, x4p25);
4159 let t_a3_5 = _mm_mul_pd(self.twiddle14re, x5p24);
4160 let t_a3_6 = _mm_mul_pd(self.twiddle11re, x6p23);
4161 let t_a3_7 = _mm_mul_pd(self.twiddle8re, x7p22);
4162 let t_a3_8 = _mm_mul_pd(self.twiddle5re, x8p21);
4163 let t_a3_9 = _mm_mul_pd(self.twiddle2re, x9p20);
4164 let t_a3_10 = _mm_mul_pd(self.twiddle1re, x10p19);
4165 let t_a3_11 = _mm_mul_pd(self.twiddle4re, x11p18);
4166 let t_a3_12 = _mm_mul_pd(self.twiddle7re, x12p17);
4167 let t_a3_13 = _mm_mul_pd(self.twiddle10re, x13p16);
4168 let t_a3_14 = _mm_mul_pd(self.twiddle13re, x14p15);
4169 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p28);
4170 let t_a4_2 = _mm_mul_pd(self.twiddle8re, x2p27);
4171 let t_a4_3 = _mm_mul_pd(self.twiddle12re, x3p26);
4172 let t_a4_4 = _mm_mul_pd(self.twiddle13re, x4p25);
4173 let t_a4_5 = _mm_mul_pd(self.twiddle9re, x5p24);
4174 let t_a4_6 = _mm_mul_pd(self.twiddle5re, x6p23);
4175 let t_a4_7 = _mm_mul_pd(self.twiddle1re, x7p22);
4176 let t_a4_8 = _mm_mul_pd(self.twiddle3re, x8p21);
4177 let t_a4_9 = _mm_mul_pd(self.twiddle7re, x9p20);
4178 let t_a4_10 = _mm_mul_pd(self.twiddle11re, x10p19);
4179 let t_a4_11 = _mm_mul_pd(self.twiddle14re, x11p18);
4180 let t_a4_12 = _mm_mul_pd(self.twiddle10re, x12p17);
4181 let t_a4_13 = _mm_mul_pd(self.twiddle6re, x13p16);
4182 let t_a4_14 = _mm_mul_pd(self.twiddle2re, x14p15);
4183 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p28);
4184 let t_a5_2 = _mm_mul_pd(self.twiddle10re, x2p27);
4185 let t_a5_3 = _mm_mul_pd(self.twiddle14re, x3p26);
4186 let t_a5_4 = _mm_mul_pd(self.twiddle9re, x4p25);
4187 let t_a5_5 = _mm_mul_pd(self.twiddle4re, x5p24);
4188 let t_a5_6 = _mm_mul_pd(self.twiddle1re, x6p23);
4189 let t_a5_7 = _mm_mul_pd(self.twiddle6re, x7p22);
4190 let t_a5_8 = _mm_mul_pd(self.twiddle11re, x8p21);
4191 let t_a5_9 = _mm_mul_pd(self.twiddle13re, x9p20);
4192 let t_a5_10 = _mm_mul_pd(self.twiddle8re, x10p19);
4193 let t_a5_11 = _mm_mul_pd(self.twiddle3re, x11p18);
4194 let t_a5_12 = _mm_mul_pd(self.twiddle2re, x12p17);
4195 let t_a5_13 = _mm_mul_pd(self.twiddle7re, x13p16);
4196 let t_a5_14 = _mm_mul_pd(self.twiddle12re, x14p15);
4197 let t_a6_1 = _mm_mul_pd(self.twiddle6re, x1p28);
4198 let t_a6_2 = _mm_mul_pd(self.twiddle12re, x2p27);
4199 let t_a6_3 = _mm_mul_pd(self.twiddle11re, x3p26);
4200 let t_a6_4 = _mm_mul_pd(self.twiddle5re, x4p25);
4201 let t_a6_5 = _mm_mul_pd(self.twiddle1re, x5p24);
4202 let t_a6_6 = _mm_mul_pd(self.twiddle7re, x6p23);
4203 let t_a6_7 = _mm_mul_pd(self.twiddle13re, x7p22);
4204 let t_a6_8 = _mm_mul_pd(self.twiddle10re, x8p21);
4205 let t_a6_9 = _mm_mul_pd(self.twiddle4re, x9p20);
4206 let t_a6_10 = _mm_mul_pd(self.twiddle2re, x10p19);
4207 let t_a6_11 = _mm_mul_pd(self.twiddle8re, x11p18);
4208 let t_a6_12 = _mm_mul_pd(self.twiddle14re, x12p17);
4209 let t_a6_13 = _mm_mul_pd(self.twiddle9re, x13p16);
4210 let t_a6_14 = _mm_mul_pd(self.twiddle3re, x14p15);
4211 let t_a7_1 = _mm_mul_pd(self.twiddle7re, x1p28);
4212 let t_a7_2 = _mm_mul_pd(self.twiddle14re, x2p27);
4213 let t_a7_3 = _mm_mul_pd(self.twiddle8re, x3p26);
4214 let t_a7_4 = _mm_mul_pd(self.twiddle1re, x4p25);
4215 let t_a7_5 = _mm_mul_pd(self.twiddle6re, x5p24);
4216 let t_a7_6 = _mm_mul_pd(self.twiddle13re, x6p23);
4217 let t_a7_7 = _mm_mul_pd(self.twiddle9re, x7p22);
4218 let t_a7_8 = _mm_mul_pd(self.twiddle2re, x8p21);
4219 let t_a7_9 = _mm_mul_pd(self.twiddle5re, x9p20);
4220 let t_a7_10 = _mm_mul_pd(self.twiddle12re, x10p19);
4221 let t_a7_11 = _mm_mul_pd(self.twiddle10re, x11p18);
4222 let t_a7_12 = _mm_mul_pd(self.twiddle3re, x12p17);
4223 let t_a7_13 = _mm_mul_pd(self.twiddle4re, x13p16);
4224 let t_a7_14 = _mm_mul_pd(self.twiddle11re, x14p15);
4225 let t_a8_1 = _mm_mul_pd(self.twiddle8re, x1p28);
4226 let t_a8_2 = _mm_mul_pd(self.twiddle13re, x2p27);
4227 let t_a8_3 = _mm_mul_pd(self.twiddle5re, x3p26);
4228 let t_a8_4 = _mm_mul_pd(self.twiddle3re, x4p25);
4229 let t_a8_5 = _mm_mul_pd(self.twiddle11re, x5p24);
4230 let t_a8_6 = _mm_mul_pd(self.twiddle10re, x6p23);
4231 let t_a8_7 = _mm_mul_pd(self.twiddle2re, x7p22);
4232 let t_a8_8 = _mm_mul_pd(self.twiddle6re, x8p21);
4233 let t_a8_9 = _mm_mul_pd(self.twiddle14re, x9p20);
4234 let t_a8_10 = _mm_mul_pd(self.twiddle7re, x10p19);
4235 let t_a8_11 = _mm_mul_pd(self.twiddle1re, x11p18);
4236 let t_a8_12 = _mm_mul_pd(self.twiddle9re, x12p17);
4237 let t_a8_13 = _mm_mul_pd(self.twiddle12re, x13p16);
4238 let t_a8_14 = _mm_mul_pd(self.twiddle4re, x14p15);
4239 let t_a9_1 = _mm_mul_pd(self.twiddle9re, x1p28);
4240 let t_a9_2 = _mm_mul_pd(self.twiddle11re, x2p27);
4241 let t_a9_3 = _mm_mul_pd(self.twiddle2re, x3p26);
4242 let t_a9_4 = _mm_mul_pd(self.twiddle7re, x4p25);
4243 let t_a9_5 = _mm_mul_pd(self.twiddle13re, x5p24);
4244 let t_a9_6 = _mm_mul_pd(self.twiddle4re, x6p23);
4245 let t_a9_7 = _mm_mul_pd(self.twiddle5re, x7p22);
4246 let t_a9_8 = _mm_mul_pd(self.twiddle14re, x8p21);
4247 let t_a9_9 = _mm_mul_pd(self.twiddle6re, x9p20);
4248 let t_a9_10 = _mm_mul_pd(self.twiddle3re, x10p19);
4249 let t_a9_11 = _mm_mul_pd(self.twiddle12re, x11p18);
4250 let t_a9_12 = _mm_mul_pd(self.twiddle8re, x12p17);
4251 let t_a9_13 = _mm_mul_pd(self.twiddle1re, x13p16);
4252 let t_a9_14 = _mm_mul_pd(self.twiddle10re, x14p15);
4253 let t_a10_1 = _mm_mul_pd(self.twiddle10re, x1p28);
4254 let t_a10_2 = _mm_mul_pd(self.twiddle9re, x2p27);
4255 let t_a10_3 = _mm_mul_pd(self.twiddle1re, x3p26);
4256 let t_a10_4 = _mm_mul_pd(self.twiddle11re, x4p25);
4257 let t_a10_5 = _mm_mul_pd(self.twiddle8re, x5p24);
4258 let t_a10_6 = _mm_mul_pd(self.twiddle2re, x6p23);
4259 let t_a10_7 = _mm_mul_pd(self.twiddle12re, x7p22);
4260 let t_a10_8 = _mm_mul_pd(self.twiddle7re, x8p21);
4261 let t_a10_9 = _mm_mul_pd(self.twiddle3re, x9p20);
4262 let t_a10_10 = _mm_mul_pd(self.twiddle13re, x10p19);
4263 let t_a10_11 = _mm_mul_pd(self.twiddle6re, x11p18);
4264 let t_a10_12 = _mm_mul_pd(self.twiddle4re, x12p17);
4265 let t_a10_13 = _mm_mul_pd(self.twiddle14re, x13p16);
4266 let t_a10_14 = _mm_mul_pd(self.twiddle5re, x14p15);
4267 let t_a11_1 = _mm_mul_pd(self.twiddle11re, x1p28);
4268 let t_a11_2 = _mm_mul_pd(self.twiddle7re, x2p27);
4269 let t_a11_3 = _mm_mul_pd(self.twiddle4re, x3p26);
4270 let t_a11_4 = _mm_mul_pd(self.twiddle14re, x4p25);
4271 let t_a11_5 = _mm_mul_pd(self.twiddle3re, x5p24);
4272 let t_a11_6 = _mm_mul_pd(self.twiddle8re, x6p23);
4273 let t_a11_7 = _mm_mul_pd(self.twiddle10re, x7p22);
4274 let t_a11_8 = _mm_mul_pd(self.twiddle1re, x8p21);
4275 let t_a11_9 = _mm_mul_pd(self.twiddle12re, x9p20);
4276 let t_a11_10 = _mm_mul_pd(self.twiddle6re, x10p19);
4277 let t_a11_11 = _mm_mul_pd(self.twiddle5re, x11p18);
4278 let t_a11_12 = _mm_mul_pd(self.twiddle13re, x12p17);
4279 let t_a11_13 = _mm_mul_pd(self.twiddle2re, x13p16);
4280 let t_a11_14 = _mm_mul_pd(self.twiddle9re, x14p15);
4281 let t_a12_1 = _mm_mul_pd(self.twiddle12re, x1p28);
4282 let t_a12_2 = _mm_mul_pd(self.twiddle5re, x2p27);
4283 let t_a12_3 = _mm_mul_pd(self.twiddle7re, x3p26);
4284 let t_a12_4 = _mm_mul_pd(self.twiddle10re, x4p25);
4285 let t_a12_5 = _mm_mul_pd(self.twiddle2re, x5p24);
4286 let t_a12_6 = _mm_mul_pd(self.twiddle14re, x6p23);
4287 let t_a12_7 = _mm_mul_pd(self.twiddle3re, x7p22);
4288 let t_a12_8 = _mm_mul_pd(self.twiddle9re, x8p21);
4289 let t_a12_9 = _mm_mul_pd(self.twiddle8re, x9p20);
4290 let t_a12_10 = _mm_mul_pd(self.twiddle4re, x10p19);
4291 let t_a12_11 = _mm_mul_pd(self.twiddle13re, x11p18);
4292 let t_a12_12 = _mm_mul_pd(self.twiddle1re, x12p17);
4293 let t_a12_13 = _mm_mul_pd(self.twiddle11re, x13p16);
4294 let t_a12_14 = _mm_mul_pd(self.twiddle6re, x14p15);
4295 let t_a13_1 = _mm_mul_pd(self.twiddle13re, x1p28);
4296 let t_a13_2 = _mm_mul_pd(self.twiddle3re, x2p27);
4297 let t_a13_3 = _mm_mul_pd(self.twiddle10re, x3p26);
4298 let t_a13_4 = _mm_mul_pd(self.twiddle6re, x4p25);
4299 let t_a13_5 = _mm_mul_pd(self.twiddle7re, x5p24);
4300 let t_a13_6 = _mm_mul_pd(self.twiddle9re, x6p23);
4301 let t_a13_7 = _mm_mul_pd(self.twiddle4re, x7p22);
4302 let t_a13_8 = _mm_mul_pd(self.twiddle12re, x8p21);
4303 let t_a13_9 = _mm_mul_pd(self.twiddle1re, x9p20);
4304 let t_a13_10 = _mm_mul_pd(self.twiddle14re, x10p19);
4305 let t_a13_11 = _mm_mul_pd(self.twiddle2re, x11p18);
4306 let t_a13_12 = _mm_mul_pd(self.twiddle11re, x12p17);
4307 let t_a13_13 = _mm_mul_pd(self.twiddle5re, x13p16);
4308 let t_a13_14 = _mm_mul_pd(self.twiddle8re, x14p15);
4309 let t_a14_1 = _mm_mul_pd(self.twiddle14re, x1p28);
4310 let t_a14_2 = _mm_mul_pd(self.twiddle1re, x2p27);
4311 let t_a14_3 = _mm_mul_pd(self.twiddle13re, x3p26);
4312 let t_a14_4 = _mm_mul_pd(self.twiddle2re, x4p25);
4313 let t_a14_5 = _mm_mul_pd(self.twiddle12re, x5p24);
4314 let t_a14_6 = _mm_mul_pd(self.twiddle3re, x6p23);
4315 let t_a14_7 = _mm_mul_pd(self.twiddle11re, x7p22);
4316 let t_a14_8 = _mm_mul_pd(self.twiddle4re, x8p21);
4317 let t_a14_9 = _mm_mul_pd(self.twiddle10re, x9p20);
4318 let t_a14_10 = _mm_mul_pd(self.twiddle5re, x10p19);
4319 let t_a14_11 = _mm_mul_pd(self.twiddle9re, x11p18);
4320 let t_a14_12 = _mm_mul_pd(self.twiddle6re, x12p17);
4321 let t_a14_13 = _mm_mul_pd(self.twiddle8re, x13p16);
4322 let t_a14_14 = _mm_mul_pd(self.twiddle7re, x14p15);
4323
4324 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m28);
4325 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m27);
4326 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m26);
4327 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m25);
4328 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m24);
4329 let t_b1_6 = _mm_mul_pd(self.twiddle6im, x6m23);
4330 let t_b1_7 = _mm_mul_pd(self.twiddle7im, x7m22);
4331 let t_b1_8 = _mm_mul_pd(self.twiddle8im, x8m21);
4332 let t_b1_9 = _mm_mul_pd(self.twiddle9im, x9m20);
4333 let t_b1_10 = _mm_mul_pd(self.twiddle10im, x10m19);
4334 let t_b1_11 = _mm_mul_pd(self.twiddle11im, x11m18);
4335 let t_b1_12 = _mm_mul_pd(self.twiddle12im, x12m17);
4336 let t_b1_13 = _mm_mul_pd(self.twiddle13im, x13m16);
4337 let t_b1_14 = _mm_mul_pd(self.twiddle14im, x14m15);
4338 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m28);
4339 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m27);
4340 let t_b2_3 = _mm_mul_pd(self.twiddle6im, x3m26);
4341 let t_b2_4 = _mm_mul_pd(self.twiddle8im, x4m25);
4342 let t_b2_5 = _mm_mul_pd(self.twiddle10im, x5m24);
4343 let t_b2_6 = _mm_mul_pd(self.twiddle12im, x6m23);
4344 let t_b2_7 = _mm_mul_pd(self.twiddle14im, x7m22);
4345 let t_b2_8 = _mm_mul_pd(self.twiddle13im, x8m21);
4346 let t_b2_9 = _mm_mul_pd(self.twiddle11im, x9m20);
4347 let t_b2_10 = _mm_mul_pd(self.twiddle9im, x10m19);
4348 let t_b2_11 = _mm_mul_pd(self.twiddle7im, x11m18);
4349 let t_b2_12 = _mm_mul_pd(self.twiddle5im, x12m17);
4350 let t_b2_13 = _mm_mul_pd(self.twiddle3im, x13m16);
4351 let t_b2_14 = _mm_mul_pd(self.twiddle1im, x14m15);
4352 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m28);
4353 let t_b3_2 = _mm_mul_pd(self.twiddle6im, x2m27);
4354 let t_b3_3 = _mm_mul_pd(self.twiddle9im, x3m26);
4355 let t_b3_4 = _mm_mul_pd(self.twiddle12im, x4m25);
4356 let t_b3_5 = _mm_mul_pd(self.twiddle14im, x5m24);
4357 let t_b3_6 = _mm_mul_pd(self.twiddle11im, x6m23);
4358 let t_b3_7 = _mm_mul_pd(self.twiddle8im, x7m22);
4359 let t_b3_8 = _mm_mul_pd(self.twiddle5im, x8m21);
4360 let t_b3_9 = _mm_mul_pd(self.twiddle2im, x9m20);
4361 let t_b3_10 = _mm_mul_pd(self.twiddle1im, x10m19);
4362 let t_b3_11 = _mm_mul_pd(self.twiddle4im, x11m18);
4363 let t_b3_12 = _mm_mul_pd(self.twiddle7im, x12m17);
4364 let t_b3_13 = _mm_mul_pd(self.twiddle10im, x13m16);
4365 let t_b3_14 = _mm_mul_pd(self.twiddle13im, x14m15);
4366 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m28);
4367 let t_b4_2 = _mm_mul_pd(self.twiddle8im, x2m27);
4368 let t_b4_3 = _mm_mul_pd(self.twiddle12im, x3m26);
4369 let t_b4_4 = _mm_mul_pd(self.twiddle13im, x4m25);
4370 let t_b4_5 = _mm_mul_pd(self.twiddle9im, x5m24);
4371 let t_b4_6 = _mm_mul_pd(self.twiddle5im, x6m23);
4372 let t_b4_7 = _mm_mul_pd(self.twiddle1im, x7m22);
4373 let t_b4_8 = _mm_mul_pd(self.twiddle3im, x8m21);
4374 let t_b4_9 = _mm_mul_pd(self.twiddle7im, x9m20);
4375 let t_b4_10 = _mm_mul_pd(self.twiddle11im, x10m19);
4376 let t_b4_11 = _mm_mul_pd(self.twiddle14im, x11m18);
4377 let t_b4_12 = _mm_mul_pd(self.twiddle10im, x12m17);
4378 let t_b4_13 = _mm_mul_pd(self.twiddle6im, x13m16);
4379 let t_b4_14 = _mm_mul_pd(self.twiddle2im, x14m15);
4380 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m28);
4381 let t_b5_2 = _mm_mul_pd(self.twiddle10im, x2m27);
4382 let t_b5_3 = _mm_mul_pd(self.twiddle14im, x3m26);
4383 let t_b5_4 = _mm_mul_pd(self.twiddle9im, x4m25);
4384 let t_b5_5 = _mm_mul_pd(self.twiddle4im, x5m24);
4385 let t_b5_6 = _mm_mul_pd(self.twiddle1im, x6m23);
4386 let t_b5_7 = _mm_mul_pd(self.twiddle6im, x7m22);
4387 let t_b5_8 = _mm_mul_pd(self.twiddle11im, x8m21);
4388 let t_b5_9 = _mm_mul_pd(self.twiddle13im, x9m20);
4389 let t_b5_10 = _mm_mul_pd(self.twiddle8im, x10m19);
4390 let t_b5_11 = _mm_mul_pd(self.twiddle3im, x11m18);
4391 let t_b5_12 = _mm_mul_pd(self.twiddle2im, x12m17);
4392 let t_b5_13 = _mm_mul_pd(self.twiddle7im, x13m16);
4393 let t_b5_14 = _mm_mul_pd(self.twiddle12im, x14m15);
4394 let t_b6_1 = _mm_mul_pd(self.twiddle6im, x1m28);
4395 let t_b6_2 = _mm_mul_pd(self.twiddle12im, x2m27);
4396 let t_b6_3 = _mm_mul_pd(self.twiddle11im, x3m26);
4397 let t_b6_4 = _mm_mul_pd(self.twiddle5im, x4m25);
4398 let t_b6_5 = _mm_mul_pd(self.twiddle1im, x5m24);
4399 let t_b6_6 = _mm_mul_pd(self.twiddle7im, x6m23);
4400 let t_b6_7 = _mm_mul_pd(self.twiddle13im, x7m22);
4401 let t_b6_8 = _mm_mul_pd(self.twiddle10im, x8m21);
4402 let t_b6_9 = _mm_mul_pd(self.twiddle4im, x9m20);
4403 let t_b6_10 = _mm_mul_pd(self.twiddle2im, x10m19);
4404 let t_b6_11 = _mm_mul_pd(self.twiddle8im, x11m18);
4405 let t_b6_12 = _mm_mul_pd(self.twiddle14im, x12m17);
4406 let t_b6_13 = _mm_mul_pd(self.twiddle9im, x13m16);
4407 let t_b6_14 = _mm_mul_pd(self.twiddle3im, x14m15);
4408 let t_b7_1 = _mm_mul_pd(self.twiddle7im, x1m28);
4409 let t_b7_2 = _mm_mul_pd(self.twiddle14im, x2m27);
4410 let t_b7_3 = _mm_mul_pd(self.twiddle8im, x3m26);
4411 let t_b7_4 = _mm_mul_pd(self.twiddle1im, x4m25);
4412 let t_b7_5 = _mm_mul_pd(self.twiddle6im, x5m24);
4413 let t_b7_6 = _mm_mul_pd(self.twiddle13im, x6m23);
4414 let t_b7_7 = _mm_mul_pd(self.twiddle9im, x7m22);
4415 let t_b7_8 = _mm_mul_pd(self.twiddle2im, x8m21);
4416 let t_b7_9 = _mm_mul_pd(self.twiddle5im, x9m20);
4417 let t_b7_10 = _mm_mul_pd(self.twiddle12im, x10m19);
4418 let t_b7_11 = _mm_mul_pd(self.twiddle10im, x11m18);
4419 let t_b7_12 = _mm_mul_pd(self.twiddle3im, x12m17);
4420 let t_b7_13 = _mm_mul_pd(self.twiddle4im, x13m16);
4421 let t_b7_14 = _mm_mul_pd(self.twiddle11im, x14m15);
4422 let t_b8_1 = _mm_mul_pd(self.twiddle8im, x1m28);
4423 let t_b8_2 = _mm_mul_pd(self.twiddle13im, x2m27);
4424 let t_b8_3 = _mm_mul_pd(self.twiddle5im, x3m26);
4425 let t_b8_4 = _mm_mul_pd(self.twiddle3im, x4m25);
4426 let t_b8_5 = _mm_mul_pd(self.twiddle11im, x5m24);
4427 let t_b8_6 = _mm_mul_pd(self.twiddle10im, x6m23);
4428 let t_b8_7 = _mm_mul_pd(self.twiddle2im, x7m22);
4429 let t_b8_8 = _mm_mul_pd(self.twiddle6im, x8m21);
4430 let t_b8_9 = _mm_mul_pd(self.twiddle14im, x9m20);
4431 let t_b8_10 = _mm_mul_pd(self.twiddle7im, x10m19);
4432 let t_b8_11 = _mm_mul_pd(self.twiddle1im, x11m18);
4433 let t_b8_12 = _mm_mul_pd(self.twiddle9im, x12m17);
4434 let t_b8_13 = _mm_mul_pd(self.twiddle12im, x13m16);
4435 let t_b8_14 = _mm_mul_pd(self.twiddle4im, x14m15);
4436 let t_b9_1 = _mm_mul_pd(self.twiddle9im, x1m28);
4437 let t_b9_2 = _mm_mul_pd(self.twiddle11im, x2m27);
4438 let t_b9_3 = _mm_mul_pd(self.twiddle2im, x3m26);
4439 let t_b9_4 = _mm_mul_pd(self.twiddle7im, x4m25);
4440 let t_b9_5 = _mm_mul_pd(self.twiddle13im, x5m24);
4441 let t_b9_6 = _mm_mul_pd(self.twiddle4im, x6m23);
4442 let t_b9_7 = _mm_mul_pd(self.twiddle5im, x7m22);
4443 let t_b9_8 = _mm_mul_pd(self.twiddle14im, x8m21);
4444 let t_b9_9 = _mm_mul_pd(self.twiddle6im, x9m20);
4445 let t_b9_10 = _mm_mul_pd(self.twiddle3im, x10m19);
4446 let t_b9_11 = _mm_mul_pd(self.twiddle12im, x11m18);
4447 let t_b9_12 = _mm_mul_pd(self.twiddle8im, x12m17);
4448 let t_b9_13 = _mm_mul_pd(self.twiddle1im, x13m16);
4449 let t_b9_14 = _mm_mul_pd(self.twiddle10im, x14m15);
4450 let t_b10_1 = _mm_mul_pd(self.twiddle10im, x1m28);
4451 let t_b10_2 = _mm_mul_pd(self.twiddle9im, x2m27);
4452 let t_b10_3 = _mm_mul_pd(self.twiddle1im, x3m26);
4453 let t_b10_4 = _mm_mul_pd(self.twiddle11im, x4m25);
4454 let t_b10_5 = _mm_mul_pd(self.twiddle8im, x5m24);
4455 let t_b10_6 = _mm_mul_pd(self.twiddle2im, x6m23);
4456 let t_b10_7 = _mm_mul_pd(self.twiddle12im, x7m22);
4457 let t_b10_8 = _mm_mul_pd(self.twiddle7im, x8m21);
4458 let t_b10_9 = _mm_mul_pd(self.twiddle3im, x9m20);
4459 let t_b10_10 = _mm_mul_pd(self.twiddle13im, x10m19);
4460 let t_b10_11 = _mm_mul_pd(self.twiddle6im, x11m18);
4461 let t_b10_12 = _mm_mul_pd(self.twiddle4im, x12m17);
4462 let t_b10_13 = _mm_mul_pd(self.twiddle14im, x13m16);
4463 let t_b10_14 = _mm_mul_pd(self.twiddle5im, x14m15);
4464 let t_b11_1 = _mm_mul_pd(self.twiddle11im, x1m28);
4465 let t_b11_2 = _mm_mul_pd(self.twiddle7im, x2m27);
4466 let t_b11_3 = _mm_mul_pd(self.twiddle4im, x3m26);
4467 let t_b11_4 = _mm_mul_pd(self.twiddle14im, x4m25);
4468 let t_b11_5 = _mm_mul_pd(self.twiddle3im, x5m24);
4469 let t_b11_6 = _mm_mul_pd(self.twiddle8im, x6m23);
4470 let t_b11_7 = _mm_mul_pd(self.twiddle10im, x7m22);
4471 let t_b11_8 = _mm_mul_pd(self.twiddle1im, x8m21);
4472 let t_b11_9 = _mm_mul_pd(self.twiddle12im, x9m20);
4473 let t_b11_10 = _mm_mul_pd(self.twiddle6im, x10m19);
4474 let t_b11_11 = _mm_mul_pd(self.twiddle5im, x11m18);
4475 let t_b11_12 = _mm_mul_pd(self.twiddle13im, x12m17);
4476 let t_b11_13 = _mm_mul_pd(self.twiddle2im, x13m16);
4477 let t_b11_14 = _mm_mul_pd(self.twiddle9im, x14m15);
4478 let t_b12_1 = _mm_mul_pd(self.twiddle12im, x1m28);
4479 let t_b12_2 = _mm_mul_pd(self.twiddle5im, x2m27);
4480 let t_b12_3 = _mm_mul_pd(self.twiddle7im, x3m26);
4481 let t_b12_4 = _mm_mul_pd(self.twiddle10im, x4m25);
4482 let t_b12_5 = _mm_mul_pd(self.twiddle2im, x5m24);
4483 let t_b12_6 = _mm_mul_pd(self.twiddle14im, x6m23);
4484 let t_b12_7 = _mm_mul_pd(self.twiddle3im, x7m22);
4485 let t_b12_8 = _mm_mul_pd(self.twiddle9im, x8m21);
4486 let t_b12_9 = _mm_mul_pd(self.twiddle8im, x9m20);
4487 let t_b12_10 = _mm_mul_pd(self.twiddle4im, x10m19);
4488 let t_b12_11 = _mm_mul_pd(self.twiddle13im, x11m18);
4489 let t_b12_12 = _mm_mul_pd(self.twiddle1im, x12m17);
4490 let t_b12_13 = _mm_mul_pd(self.twiddle11im, x13m16);
4491 let t_b12_14 = _mm_mul_pd(self.twiddle6im, x14m15);
4492 let t_b13_1 = _mm_mul_pd(self.twiddle13im, x1m28);
4493 let t_b13_2 = _mm_mul_pd(self.twiddle3im, x2m27);
4494 let t_b13_3 = _mm_mul_pd(self.twiddle10im, x3m26);
4495 let t_b13_4 = _mm_mul_pd(self.twiddle6im, x4m25);
4496 let t_b13_5 = _mm_mul_pd(self.twiddle7im, x5m24);
4497 let t_b13_6 = _mm_mul_pd(self.twiddle9im, x6m23);
4498 let t_b13_7 = _mm_mul_pd(self.twiddle4im, x7m22);
4499 let t_b13_8 = _mm_mul_pd(self.twiddle12im, x8m21);
4500 let t_b13_9 = _mm_mul_pd(self.twiddle1im, x9m20);
4501 let t_b13_10 = _mm_mul_pd(self.twiddle14im, x10m19);
4502 let t_b13_11 = _mm_mul_pd(self.twiddle2im, x11m18);
4503 let t_b13_12 = _mm_mul_pd(self.twiddle11im, x12m17);
4504 let t_b13_13 = _mm_mul_pd(self.twiddle5im, x13m16);
4505 let t_b13_14 = _mm_mul_pd(self.twiddle8im, x14m15);
4506 let t_b14_1 = _mm_mul_pd(self.twiddle14im, x1m28);
4507 let t_b14_2 = _mm_mul_pd(self.twiddle1im, x2m27);
4508 let t_b14_3 = _mm_mul_pd(self.twiddle13im, x3m26);
4509 let t_b14_4 = _mm_mul_pd(self.twiddle2im, x4m25);
4510 let t_b14_5 = _mm_mul_pd(self.twiddle12im, x5m24);
4511 let t_b14_6 = _mm_mul_pd(self.twiddle3im, x6m23);
4512 let t_b14_7 = _mm_mul_pd(self.twiddle11im, x7m22);
4513 let t_b14_8 = _mm_mul_pd(self.twiddle4im, x8m21);
4514 let t_b14_9 = _mm_mul_pd(self.twiddle10im, x9m20);
4515 let t_b14_10 = _mm_mul_pd(self.twiddle5im, x10m19);
4516 let t_b14_11 = _mm_mul_pd(self.twiddle9im, x11m18);
4517 let t_b14_12 = _mm_mul_pd(self.twiddle6im, x12m17);
4518 let t_b14_13 = _mm_mul_pd(self.twiddle8im, x13m16);
4519 let t_b14_14 = _mm_mul_pd(self.twiddle7im, x14m15);
4520
4521 let x0 = values[0];
4522 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9 + t_a1_10 + t_a1_11 + t_a1_12 + t_a1_13 + t_a1_14);
4523 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9 + t_a2_10 + t_a2_11 + t_a2_12 + t_a2_13 + t_a2_14);
4524 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9 + t_a3_10 + t_a3_11 + t_a3_12 + t_a3_13 + t_a3_14);
4525 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9 + t_a4_10 + t_a4_11 + t_a4_12 + t_a4_13 + t_a4_14);
4526 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9 + t_a5_10 + t_a5_11 + t_a5_12 + t_a5_13 + t_a5_14);
4527 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9 + t_a6_10 + t_a6_11 + t_a6_12 + t_a6_13 + t_a6_14);
4528 let t_a7 = calc_f64!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9 + t_a7_10 + t_a7_11 + t_a7_12 + t_a7_13 + t_a7_14);
4529 let t_a8 = calc_f64!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9 + t_a8_10 + t_a8_11 + t_a8_12 + t_a8_13 + t_a8_14);
4530 let t_a9 = calc_f64!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9 + t_a9_10 + t_a9_11 + t_a9_12 + t_a9_13 + t_a9_14);
4531 let t_a10 = calc_f64!(x0 + t_a10_1 + t_a10_2 + t_a10_3 + t_a10_4 + t_a10_5 + t_a10_6 + t_a10_7 + t_a10_8 + t_a10_9 + t_a10_10 + t_a10_11 + t_a10_12 + t_a10_13 + t_a10_14);
4532 let t_a11 = calc_f64!(x0 + t_a11_1 + t_a11_2 + t_a11_3 + t_a11_4 + t_a11_5 + t_a11_6 + t_a11_7 + t_a11_8 + t_a11_9 + t_a11_10 + t_a11_11 + t_a11_12 + t_a11_13 + t_a11_14);
4533 let t_a12 = calc_f64!(x0 + t_a12_1 + t_a12_2 + t_a12_3 + t_a12_4 + t_a12_5 + t_a12_6 + t_a12_7 + t_a12_8 + t_a12_9 + t_a12_10 + t_a12_11 + t_a12_12 + t_a12_13 + t_a12_14);
4534 let t_a13 = calc_f64!(x0 + t_a13_1 + t_a13_2 + t_a13_3 + t_a13_4 + t_a13_5 + t_a13_6 + t_a13_7 + t_a13_8 + t_a13_9 + t_a13_10 + t_a13_11 + t_a13_12 + t_a13_13 + t_a13_14);
4535 let t_a14 = calc_f64!(x0 + t_a14_1 + t_a14_2 + t_a14_3 + t_a14_4 + t_a14_5 + t_a14_6 + t_a14_7 + t_a14_8 + t_a14_9 + t_a14_10 + t_a14_11 + t_a14_12 + t_a14_13 + t_a14_14);
4536
4537 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9 + t_b1_10 + t_b1_11 + t_b1_12 + t_b1_13 + t_b1_14);
4538 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7 - t_b2_8 - t_b2_9 - t_b2_10 - t_b2_11 - t_b2_12 - t_b2_13 - t_b2_14);
4539 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 - t_b3_5 - t_b3_6 - t_b3_7 - t_b3_8 - t_b3_9 + t_b3_10 + t_b3_11 + t_b3_12 + t_b3_13 + t_b3_14);
4540 let t_b4 = calc_f64!(t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7 + t_b4_8 + t_b4_9 + t_b4_10 - t_b4_11 - t_b4_12 - t_b4_13 - t_b4_14);
4541 let t_b5 = calc_f64!(t_b5_1 + t_b5_2 - t_b5_3 - t_b5_4 - t_b5_5 + t_b5_6 + t_b5_7 + t_b5_8 - t_b5_9 - t_b5_10 - t_b5_11 + t_b5_12 + t_b5_13 + t_b5_14);
4542 let t_b6 = calc_f64!(t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 + t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9 + t_b6_10 + t_b6_11 + t_b6_12 - t_b6_13 - t_b6_14);
4543 let t_b7 = calc_f64!(t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9 + t_b7_10 - t_b7_11 - t_b7_12 + t_b7_13 + t_b7_14);
4544 let t_b8 = calc_f64!(t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9 - t_b8_10 + t_b8_11 + t_b8_12 - t_b8_13 - t_b8_14);
4545 let t_b9 = calc_f64!(t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 - t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8 - t_b9_9 + t_b9_10 + t_b9_11 - t_b9_12 + t_b9_13 + t_b9_14);
4546 let t_b10 = calc_f64!(t_b10_1 - t_b10_2 + t_b10_3 + t_b10_4 - t_b10_5 + t_b10_6 + t_b10_7 - t_b10_8 + t_b10_9 + t_b10_10 - t_b10_11 + t_b10_12 + t_b10_13 - t_b10_14);
4547 let t_b11 = calc_f64!(t_b11_1 - t_b11_2 + t_b11_3 - t_b11_4 - t_b11_5 + t_b11_6 - t_b11_7 + t_b11_8 + t_b11_9 - t_b11_10 + t_b11_11 - t_b11_12 - t_b11_13 + t_b11_14);
4548 let t_b12 = calc_f64!(t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 + t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 - t_b12_9 + t_b12_10 - t_b12_11 - t_b12_12 + t_b12_13 - t_b12_14);
4549 let t_b13 = calc_f64!(t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 + t_b13_7 - t_b13_8 + t_b13_9 + t_b13_10 - t_b13_11 + t_b13_12 - t_b13_13 + t_b13_14);
4550 let t_b14 = calc_f64!(t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9 - t_b14_10 + t_b14_11 - t_b14_12 + t_b14_13 - t_b14_14);
4551
4552 let t_b1_rot = self.rotate.rotate(t_b1);
4553 let t_b2_rot = self.rotate.rotate(t_b2);
4554 let t_b3_rot = self.rotate.rotate(t_b3);
4555 let t_b4_rot = self.rotate.rotate(t_b4);
4556 let t_b5_rot = self.rotate.rotate(t_b5);
4557 let t_b6_rot = self.rotate.rotate(t_b6);
4558 let t_b7_rot = self.rotate.rotate(t_b7);
4559 let t_b8_rot = self.rotate.rotate(t_b8);
4560 let t_b9_rot = self.rotate.rotate(t_b9);
4561 let t_b10_rot = self.rotate.rotate(t_b10);
4562 let t_b11_rot = self.rotate.rotate(t_b11);
4563 let t_b12_rot = self.rotate.rotate(t_b12);
4564 let t_b13_rot = self.rotate.rotate(t_b13);
4565 let t_b14_rot = self.rotate.rotate(t_b14);
4566
4567 let y0 = calc_f64!(x0 + x1p28 + x2p27 + x3p26 + x4p25 + x5p24 + x6p23 + x7p22 + x8p21 + x9p20 + x10p19 + x11p18 + x12p17 + x13p16 + x14p15);
4568 let [y1, y28] = solo_fft2_f64(t_a1, t_b1_rot);
4569 let [y2, y27] = solo_fft2_f64(t_a2, t_b2_rot);
4570 let [y3, y26] = solo_fft2_f64(t_a3, t_b3_rot);
4571 let [y4, y25] = solo_fft2_f64(t_a4, t_b4_rot);
4572 let [y5, y24] = solo_fft2_f64(t_a5, t_b5_rot);
4573 let [y6, y23] = solo_fft2_f64(t_a6, t_b6_rot);
4574 let [y7, y22] = solo_fft2_f64(t_a7, t_b7_rot);
4575 let [y8, y21] = solo_fft2_f64(t_a8, t_b8_rot);
4576 let [y9, y20] = solo_fft2_f64(t_a9, t_b9_rot);
4577 let [y10, y19] = solo_fft2_f64(t_a10, t_b10_rot);
4578 let [y11, y18] = solo_fft2_f64(t_a11, t_b11_rot);
4579 let [y12, y17] = solo_fft2_f64(t_a12, t_b12_rot);
4580 let [y13, y16] = solo_fft2_f64(t_a13, t_b13_rot);
4581 let [y14, y15] = solo_fft2_f64(t_a14, t_b14_rot);
4582 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28]
4583 }
4584 }
4585
4586 // _____ _ _________ _ _ _
4587 // |___ // | |___ /___ \| |__ (_) |_
4588 // |_ \| | _____ |_ \ __) | '_ \| | __|
4589 // ___) | | |_____| ___) / __/| |_) | | |_
4590 // |____/|_| |____/_____|_.__/|_|\__|
4591 //
4592 pub struct SseF32Butterfly31<T> {
4593 direction: FftDirection,
4594 _phantom: std::marker::PhantomData<T>,
4595 rotate: Rotate90F32,
4596 twiddle1re: __m128,
4597 twiddle1im: __m128,
4598 twiddle2re: __m128,
4599 twiddle2im: __m128,
4600 twiddle3re: __m128,
4601 twiddle3im: __m128,
4602 twiddle4re: __m128,
4603 twiddle4im: __m128,
4604 twiddle5re: __m128,
4605 twiddle5im: __m128,
4606 twiddle6re: __m128,
4607 twiddle6im: __m128,
4608 twiddle7re: __m128,
4609 twiddle7im: __m128,
4610 twiddle8re: __m128,
4611 twiddle8im: __m128,
4612 twiddle9re: __m128,
4613 twiddle9im: __m128,
4614 twiddle10re: __m128,
4615 twiddle10im: __m128,
4616 twiddle11re: __m128,
4617 twiddle11im: __m128,
4618 twiddle12re: __m128,
4619 twiddle12im: __m128,
4620 twiddle13re: __m128,
4621 twiddle13im: __m128,
4622 twiddle14re: __m128,
4623 twiddle14im: __m128,
4624 twiddle15re: __m128,
4625 twiddle15im: __m128,
4626 }
4627
4628 boilerplate_fft_sse_f32_butterfly!(SseF32Butterfly31, 31, |this: &SseF32Butterfly31<_>| this
4629 .direction);
4630 boilerplate_fft_sse_common_butterfly!(SseF32Butterfly31, 31, |this: &SseF32Butterfly31<_>| this
4631 .direction);
4632 impl<T: FftNum> SseF32Butterfly31<T> {
4633 #[inline(always)]
4634 pub fn new(direction: FftDirection) -> Self {
4635 assert_f32::<T>();
4636 let rotate = Rotate90F32::new(true);
4637 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 31, direction);
4638 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 31, direction);
4639 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 31, direction);
4640 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 31, direction);
4641 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 31, direction);
4642 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 31, direction);
4643 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 31, direction);
4644 let tw8: Complex<f32> = twiddles::compute_twiddle(8, 31, direction);
4645 let tw9: Complex<f32> = twiddles::compute_twiddle(9, 31, direction);
4646 let tw10: Complex<f32> = twiddles::compute_twiddle(10, 31, direction);
4647 let tw11: Complex<f32> = twiddles::compute_twiddle(11, 31, direction);
4648 let tw12: Complex<f32> = twiddles::compute_twiddle(12, 31, direction);
4649 let tw13: Complex<f32> = twiddles::compute_twiddle(13, 31, direction);
4650 let tw14: Complex<f32> = twiddles::compute_twiddle(14, 31, direction);
4651 let tw15: Complex<f32> = twiddles::compute_twiddle(15, 31, direction);
4652 let twiddle1re = unsafe { _mm_load1_ps(&tw1.re) };
4653 let twiddle1im = unsafe { _mm_load1_ps(&tw1.im) };
4654 let twiddle2re = unsafe { _mm_load1_ps(&tw2.re) };
4655 let twiddle2im = unsafe { _mm_load1_ps(&tw2.im) };
4656 let twiddle3re = unsafe { _mm_load1_ps(&tw3.re) };
4657 let twiddle3im = unsafe { _mm_load1_ps(&tw3.im) };
4658 let twiddle4re = unsafe { _mm_load1_ps(&tw4.re) };
4659 let twiddle4im = unsafe { _mm_load1_ps(&tw4.im) };
4660 let twiddle5re = unsafe { _mm_load1_ps(&tw5.re) };
4661 let twiddle5im = unsafe { _mm_load1_ps(&tw5.im) };
4662 let twiddle6re = unsafe { _mm_load1_ps(&tw6.re) };
4663 let twiddle6im = unsafe { _mm_load1_ps(&tw6.im) };
4664 let twiddle7re = unsafe { _mm_load1_ps(&tw7.re) };
4665 let twiddle7im = unsafe { _mm_load1_ps(&tw7.im) };
4666 let twiddle8re = unsafe { _mm_load1_ps(&tw8.re) };
4667 let twiddle8im = unsafe { _mm_load1_ps(&tw8.im) };
4668 let twiddle9re = unsafe { _mm_load1_ps(&tw9.re) };
4669 let twiddle9im = unsafe { _mm_load1_ps(&tw9.im) };
4670 let twiddle10re = unsafe { _mm_load1_ps(&tw10.re) };
4671 let twiddle10im = unsafe { _mm_load1_ps(&tw10.im) };
4672 let twiddle11re = unsafe { _mm_load1_ps(&tw11.re) };
4673 let twiddle11im = unsafe { _mm_load1_ps(&tw11.im) };
4674 let twiddle12re = unsafe { _mm_load1_ps(&tw12.re) };
4675 let twiddle12im = unsafe { _mm_load1_ps(&tw12.im) };
4676 let twiddle13re = unsafe { _mm_load1_ps(&tw13.re) };
4677 let twiddle13im = unsafe { _mm_load1_ps(&tw13.im) };
4678 let twiddle14re = unsafe { _mm_load1_ps(&tw14.re) };
4679 let twiddle14im = unsafe { _mm_load1_ps(&tw14.im) };
4680 let twiddle15re = unsafe { _mm_load1_ps(&tw15.re) };
4681 let twiddle15im = unsafe { _mm_load1_ps(&tw15.im) };
4682
4683 Self {
4684 direction,
4685 _phantom: std::marker::PhantomData,
4686 rotate,
4687 twiddle1re,
4688 twiddle1im,
4689 twiddle2re,
4690 twiddle2im,
4691 twiddle3re,
4692 twiddle3im,
4693 twiddle4re,
4694 twiddle4im,
4695 twiddle5re,
4696 twiddle5im,
4697 twiddle6re,
4698 twiddle6im,
4699 twiddle7re,
4700 twiddle7im,
4701 twiddle8re,
4702 twiddle8im,
4703 twiddle9re,
4704 twiddle9im,
4705 twiddle10re,
4706 twiddle10im,
4707 twiddle11re,
4708 twiddle11im,
4709 twiddle12re,
4710 twiddle12im,
4711 twiddle13re,
4712 twiddle13im,
4713 twiddle14re,
4714 twiddle14im,
4715 twiddle15re,
4716 twiddle15im,
4717 }
4718 }
4719
4720 #[inline(always)]
4721 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
4722 let values = read_partial1_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
4723
4724 let out = self.perform_parallel_fft_direct(values);
4725
4726 write_partial_lo_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
4727 }
4728
4729 #[inline(always)]
4730 pub(crate) unsafe fn perform_parallel_fft_contiguous(&self, mut buffer: impl SseArrayMut<f32>) {
4731 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60});
4732
4733 let values = [
4734 extract_lo_hi_f32(input_packed[0], input_packed[15]),
4735 extract_hi_lo_f32(input_packed[0], input_packed[16]),
4736 extract_lo_hi_f32(input_packed[1], input_packed[16]),
4737 extract_hi_lo_f32(input_packed[1], input_packed[17]),
4738 extract_lo_hi_f32(input_packed[2], input_packed[17]),
4739 extract_hi_lo_f32(input_packed[2], input_packed[18]),
4740 extract_lo_hi_f32(input_packed[3], input_packed[18]),
4741 extract_hi_lo_f32(input_packed[3], input_packed[19]),
4742 extract_lo_hi_f32(input_packed[4], input_packed[19]),
4743 extract_hi_lo_f32(input_packed[4], input_packed[20]),
4744 extract_lo_hi_f32(input_packed[5], input_packed[20]),
4745 extract_hi_lo_f32(input_packed[5], input_packed[21]),
4746 extract_lo_hi_f32(input_packed[6], input_packed[21]),
4747 extract_hi_lo_f32(input_packed[6], input_packed[22]),
4748 extract_lo_hi_f32(input_packed[7], input_packed[22]),
4749 extract_hi_lo_f32(input_packed[7], input_packed[23]),
4750 extract_lo_hi_f32(input_packed[8], input_packed[23]),
4751 extract_hi_lo_f32(input_packed[8], input_packed[24]),
4752 extract_lo_hi_f32(input_packed[9], input_packed[24]),
4753 extract_hi_lo_f32(input_packed[9], input_packed[25]),
4754 extract_lo_hi_f32(input_packed[10], input_packed[25]),
4755 extract_hi_lo_f32(input_packed[10], input_packed[26]),
4756 extract_lo_hi_f32(input_packed[11], input_packed[26]),
4757 extract_hi_lo_f32(input_packed[11], input_packed[27]),
4758 extract_lo_hi_f32(input_packed[12], input_packed[27]),
4759 extract_hi_lo_f32(input_packed[12], input_packed[28]),
4760 extract_lo_hi_f32(input_packed[13], input_packed[28]),
4761 extract_hi_lo_f32(input_packed[13], input_packed[29]),
4762 extract_lo_hi_f32(input_packed[14], input_packed[29]),
4763 extract_hi_lo_f32(input_packed[14], input_packed[30]),
4764 extract_lo_hi_f32(input_packed[15], input_packed[30]),
4765 ];
4766
4767 let out = self.perform_parallel_fft_direct(values);
4768
4769 let out_packed = [
4770 extract_lo_lo_f32(out[0], out[1]),
4771 extract_lo_lo_f32(out[2], out[3]),
4772 extract_lo_lo_f32(out[4], out[5]),
4773 extract_lo_lo_f32(out[6], out[7]),
4774 extract_lo_lo_f32(out[8], out[9]),
4775 extract_lo_lo_f32(out[10], out[11]),
4776 extract_lo_lo_f32(out[12], out[13]),
4777 extract_lo_lo_f32(out[14], out[15]),
4778 extract_lo_lo_f32(out[16], out[17]),
4779 extract_lo_lo_f32(out[18], out[19]),
4780 extract_lo_lo_f32(out[20], out[21]),
4781 extract_lo_lo_f32(out[22], out[23]),
4782 extract_lo_lo_f32(out[24], out[25]),
4783 extract_lo_lo_f32(out[26], out[27]),
4784 extract_lo_lo_f32(out[28], out[29]),
4785 extract_lo_hi_f32(out[30], out[0]),
4786 extract_hi_hi_f32(out[1], out[2]),
4787 extract_hi_hi_f32(out[3], out[4]),
4788 extract_hi_hi_f32(out[5], out[6]),
4789 extract_hi_hi_f32(out[7], out[8]),
4790 extract_hi_hi_f32(out[9], out[10]),
4791 extract_hi_hi_f32(out[11], out[12]),
4792 extract_hi_hi_f32(out[13], out[14]),
4793 extract_hi_hi_f32(out[15], out[16]),
4794 extract_hi_hi_f32(out[17], out[18]),
4795 extract_hi_hi_f32(out[19], out[20]),
4796 extract_hi_hi_f32(out[21], out[22]),
4797 extract_hi_hi_f32(out[23], out[24]),
4798 extract_hi_hi_f32(out[25], out[26]),
4799 extract_hi_hi_f32(out[27], out[28]),
4800 extract_hi_hi_f32(out[29], out[30]),
4801 ];
4802
4803 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
4804 }
4805
4806 #[inline(always)]
4807 pub(crate) unsafe fn perform_parallel_fft_direct(&self, values: [__m128; 31]) -> [__m128; 31] {
4808 let [x1p30, x1m30] = parallel_fft2_interleaved_f32(values[1], values[30]);
4809 let [x2p29, x2m29] = parallel_fft2_interleaved_f32(values[2], values[29]);
4810 let [x3p28, x3m28] = parallel_fft2_interleaved_f32(values[3], values[28]);
4811 let [x4p27, x4m27] = parallel_fft2_interleaved_f32(values[4], values[27]);
4812 let [x5p26, x5m26] = parallel_fft2_interleaved_f32(values[5], values[26]);
4813 let [x6p25, x6m25] = parallel_fft2_interleaved_f32(values[6], values[25]);
4814 let [x7p24, x7m24] = parallel_fft2_interleaved_f32(values[7], values[24]);
4815 let [x8p23, x8m23] = parallel_fft2_interleaved_f32(values[8], values[23]);
4816 let [x9p22, x9m22] = parallel_fft2_interleaved_f32(values[9], values[22]);
4817 let [x10p21, x10m21] = parallel_fft2_interleaved_f32(values[10], values[21]);
4818 let [x11p20, x11m20] = parallel_fft2_interleaved_f32(values[11], values[20]);
4819 let [x12p19, x12m19] = parallel_fft2_interleaved_f32(values[12], values[19]);
4820 let [x13p18, x13m18] = parallel_fft2_interleaved_f32(values[13], values[18]);
4821 let [x14p17, x14m17] = parallel_fft2_interleaved_f32(values[14], values[17]);
4822 let [x15p16, x15m16] = parallel_fft2_interleaved_f32(values[15], values[16]);
4823
4824 let t_a1_1 = _mm_mul_ps(self.twiddle1re, x1p30);
4825 let t_a1_2 = _mm_mul_ps(self.twiddle2re, x2p29);
4826 let t_a1_3 = _mm_mul_ps(self.twiddle3re, x3p28);
4827 let t_a1_4 = _mm_mul_ps(self.twiddle4re, x4p27);
4828 let t_a1_5 = _mm_mul_ps(self.twiddle5re, x5p26);
4829 let t_a1_6 = _mm_mul_ps(self.twiddle6re, x6p25);
4830 let t_a1_7 = _mm_mul_ps(self.twiddle7re, x7p24);
4831 let t_a1_8 = _mm_mul_ps(self.twiddle8re, x8p23);
4832 let t_a1_9 = _mm_mul_ps(self.twiddle9re, x9p22);
4833 let t_a1_10 = _mm_mul_ps(self.twiddle10re, x10p21);
4834 let t_a1_11 = _mm_mul_ps(self.twiddle11re, x11p20);
4835 let t_a1_12 = _mm_mul_ps(self.twiddle12re, x12p19);
4836 let t_a1_13 = _mm_mul_ps(self.twiddle13re, x13p18);
4837 let t_a1_14 = _mm_mul_ps(self.twiddle14re, x14p17);
4838 let t_a1_15 = _mm_mul_ps(self.twiddle15re, x15p16);
4839 let t_a2_1 = _mm_mul_ps(self.twiddle2re, x1p30);
4840 let t_a2_2 = _mm_mul_ps(self.twiddle4re, x2p29);
4841 let t_a2_3 = _mm_mul_ps(self.twiddle6re, x3p28);
4842 let t_a2_4 = _mm_mul_ps(self.twiddle8re, x4p27);
4843 let t_a2_5 = _mm_mul_ps(self.twiddle10re, x5p26);
4844 let t_a2_6 = _mm_mul_ps(self.twiddle12re, x6p25);
4845 let t_a2_7 = _mm_mul_ps(self.twiddle14re, x7p24);
4846 let t_a2_8 = _mm_mul_ps(self.twiddle15re, x8p23);
4847 let t_a2_9 = _mm_mul_ps(self.twiddle13re, x9p22);
4848 let t_a2_10 = _mm_mul_ps(self.twiddle11re, x10p21);
4849 let t_a2_11 = _mm_mul_ps(self.twiddle9re, x11p20);
4850 let t_a2_12 = _mm_mul_ps(self.twiddle7re, x12p19);
4851 let t_a2_13 = _mm_mul_ps(self.twiddle5re, x13p18);
4852 let t_a2_14 = _mm_mul_ps(self.twiddle3re, x14p17);
4853 let t_a2_15 = _mm_mul_ps(self.twiddle1re, x15p16);
4854 let t_a3_1 = _mm_mul_ps(self.twiddle3re, x1p30);
4855 let t_a3_2 = _mm_mul_ps(self.twiddle6re, x2p29);
4856 let t_a3_3 = _mm_mul_ps(self.twiddle9re, x3p28);
4857 let t_a3_4 = _mm_mul_ps(self.twiddle12re, x4p27);
4858 let t_a3_5 = _mm_mul_ps(self.twiddle15re, x5p26);
4859 let t_a3_6 = _mm_mul_ps(self.twiddle13re, x6p25);
4860 let t_a3_7 = _mm_mul_ps(self.twiddle10re, x7p24);
4861 let t_a3_8 = _mm_mul_ps(self.twiddle7re, x8p23);
4862 let t_a3_9 = _mm_mul_ps(self.twiddle4re, x9p22);
4863 let t_a3_10 = _mm_mul_ps(self.twiddle1re, x10p21);
4864 let t_a3_11 = _mm_mul_ps(self.twiddle2re, x11p20);
4865 let t_a3_12 = _mm_mul_ps(self.twiddle5re, x12p19);
4866 let t_a3_13 = _mm_mul_ps(self.twiddle8re, x13p18);
4867 let t_a3_14 = _mm_mul_ps(self.twiddle11re, x14p17);
4868 let t_a3_15 = _mm_mul_ps(self.twiddle14re, x15p16);
4869 let t_a4_1 = _mm_mul_ps(self.twiddle4re, x1p30);
4870 let t_a4_2 = _mm_mul_ps(self.twiddle8re, x2p29);
4871 let t_a4_3 = _mm_mul_ps(self.twiddle12re, x3p28);
4872 let t_a4_4 = _mm_mul_ps(self.twiddle15re, x4p27);
4873 let t_a4_5 = _mm_mul_ps(self.twiddle11re, x5p26);
4874 let t_a4_6 = _mm_mul_ps(self.twiddle7re, x6p25);
4875 let t_a4_7 = _mm_mul_ps(self.twiddle3re, x7p24);
4876 let t_a4_8 = _mm_mul_ps(self.twiddle1re, x8p23);
4877 let t_a4_9 = _mm_mul_ps(self.twiddle5re, x9p22);
4878 let t_a4_10 = _mm_mul_ps(self.twiddle9re, x10p21);
4879 let t_a4_11 = _mm_mul_ps(self.twiddle13re, x11p20);
4880 let t_a4_12 = _mm_mul_ps(self.twiddle14re, x12p19);
4881 let t_a4_13 = _mm_mul_ps(self.twiddle10re, x13p18);
4882 let t_a4_14 = _mm_mul_ps(self.twiddle6re, x14p17);
4883 let t_a4_15 = _mm_mul_ps(self.twiddle2re, x15p16);
4884 let t_a5_1 = _mm_mul_ps(self.twiddle5re, x1p30);
4885 let t_a5_2 = _mm_mul_ps(self.twiddle10re, x2p29);
4886 let t_a5_3 = _mm_mul_ps(self.twiddle15re, x3p28);
4887 let t_a5_4 = _mm_mul_ps(self.twiddle11re, x4p27);
4888 let t_a5_5 = _mm_mul_ps(self.twiddle6re, x5p26);
4889 let t_a5_6 = _mm_mul_ps(self.twiddle1re, x6p25);
4890 let t_a5_7 = _mm_mul_ps(self.twiddle4re, x7p24);
4891 let t_a5_8 = _mm_mul_ps(self.twiddle9re, x8p23);
4892 let t_a5_9 = _mm_mul_ps(self.twiddle14re, x9p22);
4893 let t_a5_10 = _mm_mul_ps(self.twiddle12re, x10p21);
4894 let t_a5_11 = _mm_mul_ps(self.twiddle7re, x11p20);
4895 let t_a5_12 = _mm_mul_ps(self.twiddle2re, x12p19);
4896 let t_a5_13 = _mm_mul_ps(self.twiddle3re, x13p18);
4897 let t_a5_14 = _mm_mul_ps(self.twiddle8re, x14p17);
4898 let t_a5_15 = _mm_mul_ps(self.twiddle13re, x15p16);
4899 let t_a6_1 = _mm_mul_ps(self.twiddle6re, x1p30);
4900 let t_a6_2 = _mm_mul_ps(self.twiddle12re, x2p29);
4901 let t_a6_3 = _mm_mul_ps(self.twiddle13re, x3p28);
4902 let t_a6_4 = _mm_mul_ps(self.twiddle7re, x4p27);
4903 let t_a6_5 = _mm_mul_ps(self.twiddle1re, x5p26);
4904 let t_a6_6 = _mm_mul_ps(self.twiddle5re, x6p25);
4905 let t_a6_7 = _mm_mul_ps(self.twiddle11re, x7p24);
4906 let t_a6_8 = _mm_mul_ps(self.twiddle14re, x8p23);
4907 let t_a6_9 = _mm_mul_ps(self.twiddle8re, x9p22);
4908 let t_a6_10 = _mm_mul_ps(self.twiddle2re, x10p21);
4909 let t_a6_11 = _mm_mul_ps(self.twiddle4re, x11p20);
4910 let t_a6_12 = _mm_mul_ps(self.twiddle10re, x12p19);
4911 let t_a6_13 = _mm_mul_ps(self.twiddle15re, x13p18);
4912 let t_a6_14 = _mm_mul_ps(self.twiddle9re, x14p17);
4913 let t_a6_15 = _mm_mul_ps(self.twiddle3re, x15p16);
4914 let t_a7_1 = _mm_mul_ps(self.twiddle7re, x1p30);
4915 let t_a7_2 = _mm_mul_ps(self.twiddle14re, x2p29);
4916 let t_a7_3 = _mm_mul_ps(self.twiddle10re, x3p28);
4917 let t_a7_4 = _mm_mul_ps(self.twiddle3re, x4p27);
4918 let t_a7_5 = _mm_mul_ps(self.twiddle4re, x5p26);
4919 let t_a7_6 = _mm_mul_ps(self.twiddle11re, x6p25);
4920 let t_a7_7 = _mm_mul_ps(self.twiddle13re, x7p24);
4921 let t_a7_8 = _mm_mul_ps(self.twiddle6re, x8p23);
4922 let t_a7_9 = _mm_mul_ps(self.twiddle1re, x9p22);
4923 let t_a7_10 = _mm_mul_ps(self.twiddle8re, x10p21);
4924 let t_a7_11 = _mm_mul_ps(self.twiddle15re, x11p20);
4925 let t_a7_12 = _mm_mul_ps(self.twiddle9re, x12p19);
4926 let t_a7_13 = _mm_mul_ps(self.twiddle2re, x13p18);
4927 let t_a7_14 = _mm_mul_ps(self.twiddle5re, x14p17);
4928 let t_a7_15 = _mm_mul_ps(self.twiddle12re, x15p16);
4929 let t_a8_1 = _mm_mul_ps(self.twiddle8re, x1p30);
4930 let t_a8_2 = _mm_mul_ps(self.twiddle15re, x2p29);
4931 let t_a8_3 = _mm_mul_ps(self.twiddle7re, x3p28);
4932 let t_a8_4 = _mm_mul_ps(self.twiddle1re, x4p27);
4933 let t_a8_5 = _mm_mul_ps(self.twiddle9re, x5p26);
4934 let t_a8_6 = _mm_mul_ps(self.twiddle14re, x6p25);
4935 let t_a8_7 = _mm_mul_ps(self.twiddle6re, x7p24);
4936 let t_a8_8 = _mm_mul_ps(self.twiddle2re, x8p23);
4937 let t_a8_9 = _mm_mul_ps(self.twiddle10re, x9p22);
4938 let t_a8_10 = _mm_mul_ps(self.twiddle13re, x10p21);
4939 let t_a8_11 = _mm_mul_ps(self.twiddle5re, x11p20);
4940 let t_a8_12 = _mm_mul_ps(self.twiddle3re, x12p19);
4941 let t_a8_13 = _mm_mul_ps(self.twiddle11re, x13p18);
4942 let t_a8_14 = _mm_mul_ps(self.twiddle12re, x14p17);
4943 let t_a8_15 = _mm_mul_ps(self.twiddle4re, x15p16);
4944 let t_a9_1 = _mm_mul_ps(self.twiddle9re, x1p30);
4945 let t_a9_2 = _mm_mul_ps(self.twiddle13re, x2p29);
4946 let t_a9_3 = _mm_mul_ps(self.twiddle4re, x3p28);
4947 let t_a9_4 = _mm_mul_ps(self.twiddle5re, x4p27);
4948 let t_a9_5 = _mm_mul_ps(self.twiddle14re, x5p26);
4949 let t_a9_6 = _mm_mul_ps(self.twiddle8re, x6p25);
4950 let t_a9_7 = _mm_mul_ps(self.twiddle1re, x7p24);
4951 let t_a9_8 = _mm_mul_ps(self.twiddle10re, x8p23);
4952 let t_a9_9 = _mm_mul_ps(self.twiddle12re, x9p22);
4953 let t_a9_10 = _mm_mul_ps(self.twiddle3re, x10p21);
4954 let t_a9_11 = _mm_mul_ps(self.twiddle6re, x11p20);
4955 let t_a9_12 = _mm_mul_ps(self.twiddle15re, x12p19);
4956 let t_a9_13 = _mm_mul_ps(self.twiddle7re, x13p18);
4957 let t_a9_14 = _mm_mul_ps(self.twiddle2re, x14p17);
4958 let t_a9_15 = _mm_mul_ps(self.twiddle11re, x15p16);
4959 let t_a10_1 = _mm_mul_ps(self.twiddle10re, x1p30);
4960 let t_a10_2 = _mm_mul_ps(self.twiddle11re, x2p29);
4961 let t_a10_3 = _mm_mul_ps(self.twiddle1re, x3p28);
4962 let t_a10_4 = _mm_mul_ps(self.twiddle9re, x4p27);
4963 let t_a10_5 = _mm_mul_ps(self.twiddle12re, x5p26);
4964 let t_a10_6 = _mm_mul_ps(self.twiddle2re, x6p25);
4965 let t_a10_7 = _mm_mul_ps(self.twiddle8re, x7p24);
4966 let t_a10_8 = _mm_mul_ps(self.twiddle13re, x8p23);
4967 let t_a10_9 = _mm_mul_ps(self.twiddle3re, x9p22);
4968 let t_a10_10 = _mm_mul_ps(self.twiddle7re, x10p21);
4969 let t_a10_11 = _mm_mul_ps(self.twiddle14re, x11p20);
4970 let t_a10_12 = _mm_mul_ps(self.twiddle4re, x12p19);
4971 let t_a10_13 = _mm_mul_ps(self.twiddle6re, x13p18);
4972 let t_a10_14 = _mm_mul_ps(self.twiddle15re, x14p17);
4973 let t_a10_15 = _mm_mul_ps(self.twiddle5re, x15p16);
4974 let t_a11_1 = _mm_mul_ps(self.twiddle11re, x1p30);
4975 let t_a11_2 = _mm_mul_ps(self.twiddle9re, x2p29);
4976 let t_a11_3 = _mm_mul_ps(self.twiddle2re, x3p28);
4977 let t_a11_4 = _mm_mul_ps(self.twiddle13re, x4p27);
4978 let t_a11_5 = _mm_mul_ps(self.twiddle7re, x5p26);
4979 let t_a11_6 = _mm_mul_ps(self.twiddle4re, x6p25);
4980 let t_a11_7 = _mm_mul_ps(self.twiddle15re, x7p24);
4981 let t_a11_8 = _mm_mul_ps(self.twiddle5re, x8p23);
4982 let t_a11_9 = _mm_mul_ps(self.twiddle6re, x9p22);
4983 let t_a11_10 = _mm_mul_ps(self.twiddle14re, x10p21);
4984 let t_a11_11 = _mm_mul_ps(self.twiddle3re, x11p20);
4985 let t_a11_12 = _mm_mul_ps(self.twiddle8re, x12p19);
4986 let t_a11_13 = _mm_mul_ps(self.twiddle12re, x13p18);
4987 let t_a11_14 = _mm_mul_ps(self.twiddle1re, x14p17);
4988 let t_a11_15 = _mm_mul_ps(self.twiddle10re, x15p16);
4989 let t_a12_1 = _mm_mul_ps(self.twiddle12re, x1p30);
4990 let t_a12_2 = _mm_mul_ps(self.twiddle7re, x2p29);
4991 let t_a12_3 = _mm_mul_ps(self.twiddle5re, x3p28);
4992 let t_a12_4 = _mm_mul_ps(self.twiddle14re, x4p27);
4993 let t_a12_5 = _mm_mul_ps(self.twiddle2re, x5p26);
4994 let t_a12_6 = _mm_mul_ps(self.twiddle10re, x6p25);
4995 let t_a12_7 = _mm_mul_ps(self.twiddle9re, x7p24);
4996 let t_a12_8 = _mm_mul_ps(self.twiddle3re, x8p23);
4997 let t_a12_9 = _mm_mul_ps(self.twiddle15re, x9p22);
4998 let t_a12_10 = _mm_mul_ps(self.twiddle4re, x10p21);
4999 let t_a12_11 = _mm_mul_ps(self.twiddle8re, x11p20);
5000 let t_a12_12 = _mm_mul_ps(self.twiddle11re, x12p19);
5001 let t_a12_13 = _mm_mul_ps(self.twiddle1re, x13p18);
5002 let t_a12_14 = _mm_mul_ps(self.twiddle13re, x14p17);
5003 let t_a12_15 = _mm_mul_ps(self.twiddle6re, x15p16);
5004 let t_a13_1 = _mm_mul_ps(self.twiddle13re, x1p30);
5005 let t_a13_2 = _mm_mul_ps(self.twiddle5re, x2p29);
5006 let t_a13_3 = _mm_mul_ps(self.twiddle8re, x3p28);
5007 let t_a13_4 = _mm_mul_ps(self.twiddle10re, x4p27);
5008 let t_a13_5 = _mm_mul_ps(self.twiddle3re, x5p26);
5009 let t_a13_6 = _mm_mul_ps(self.twiddle15re, x6p25);
5010 let t_a13_7 = _mm_mul_ps(self.twiddle2re, x7p24);
5011 let t_a13_8 = _mm_mul_ps(self.twiddle11re, x8p23);
5012 let t_a13_9 = _mm_mul_ps(self.twiddle7re, x9p22);
5013 let t_a13_10 = _mm_mul_ps(self.twiddle6re, x10p21);
5014 let t_a13_11 = _mm_mul_ps(self.twiddle12re, x11p20);
5015 let t_a13_12 = _mm_mul_ps(self.twiddle1re, x12p19);
5016 let t_a13_13 = _mm_mul_ps(self.twiddle14re, x13p18);
5017 let t_a13_14 = _mm_mul_ps(self.twiddle4re, x14p17);
5018 let t_a13_15 = _mm_mul_ps(self.twiddle9re, x15p16);
5019 let t_a14_1 = _mm_mul_ps(self.twiddle14re, x1p30);
5020 let t_a14_2 = _mm_mul_ps(self.twiddle3re, x2p29);
5021 let t_a14_3 = _mm_mul_ps(self.twiddle11re, x3p28);
5022 let t_a14_4 = _mm_mul_ps(self.twiddle6re, x4p27);
5023 let t_a14_5 = _mm_mul_ps(self.twiddle8re, x5p26);
5024 let t_a14_6 = _mm_mul_ps(self.twiddle9re, x6p25);
5025 let t_a14_7 = _mm_mul_ps(self.twiddle5re, x7p24);
5026 let t_a14_8 = _mm_mul_ps(self.twiddle12re, x8p23);
5027 let t_a14_9 = _mm_mul_ps(self.twiddle2re, x9p22);
5028 let t_a14_10 = _mm_mul_ps(self.twiddle15re, x10p21);
5029 let t_a14_11 = _mm_mul_ps(self.twiddle1re, x11p20);
5030 let t_a14_12 = _mm_mul_ps(self.twiddle13re, x12p19);
5031 let t_a14_13 = _mm_mul_ps(self.twiddle4re, x13p18);
5032 let t_a14_14 = _mm_mul_ps(self.twiddle10re, x14p17);
5033 let t_a14_15 = _mm_mul_ps(self.twiddle7re, x15p16);
5034 let t_a15_1 = _mm_mul_ps(self.twiddle15re, x1p30);
5035 let t_a15_2 = _mm_mul_ps(self.twiddle1re, x2p29);
5036 let t_a15_3 = _mm_mul_ps(self.twiddle14re, x3p28);
5037 let t_a15_4 = _mm_mul_ps(self.twiddle2re, x4p27);
5038 let t_a15_5 = _mm_mul_ps(self.twiddle13re, x5p26);
5039 let t_a15_6 = _mm_mul_ps(self.twiddle3re, x6p25);
5040 let t_a15_7 = _mm_mul_ps(self.twiddle12re, x7p24);
5041 let t_a15_8 = _mm_mul_ps(self.twiddle4re, x8p23);
5042 let t_a15_9 = _mm_mul_ps(self.twiddle11re, x9p22);
5043 let t_a15_10 = _mm_mul_ps(self.twiddle5re, x10p21);
5044 let t_a15_11 = _mm_mul_ps(self.twiddle10re, x11p20);
5045 let t_a15_12 = _mm_mul_ps(self.twiddle6re, x12p19);
5046 let t_a15_13 = _mm_mul_ps(self.twiddle9re, x13p18);
5047 let t_a15_14 = _mm_mul_ps(self.twiddle7re, x14p17);
5048 let t_a15_15 = _mm_mul_ps(self.twiddle8re, x15p16);
5049
5050 let t_b1_1 = _mm_mul_ps(self.twiddle1im, x1m30);
5051 let t_b1_2 = _mm_mul_ps(self.twiddle2im, x2m29);
5052 let t_b1_3 = _mm_mul_ps(self.twiddle3im, x3m28);
5053 let t_b1_4 = _mm_mul_ps(self.twiddle4im, x4m27);
5054 let t_b1_5 = _mm_mul_ps(self.twiddle5im, x5m26);
5055 let t_b1_6 = _mm_mul_ps(self.twiddle6im, x6m25);
5056 let t_b1_7 = _mm_mul_ps(self.twiddle7im, x7m24);
5057 let t_b1_8 = _mm_mul_ps(self.twiddle8im, x8m23);
5058 let t_b1_9 = _mm_mul_ps(self.twiddle9im, x9m22);
5059 let t_b1_10 = _mm_mul_ps(self.twiddle10im, x10m21);
5060 let t_b1_11 = _mm_mul_ps(self.twiddle11im, x11m20);
5061 let t_b1_12 = _mm_mul_ps(self.twiddle12im, x12m19);
5062 let t_b1_13 = _mm_mul_ps(self.twiddle13im, x13m18);
5063 let t_b1_14 = _mm_mul_ps(self.twiddle14im, x14m17);
5064 let t_b1_15 = _mm_mul_ps(self.twiddle15im, x15m16);
5065 let t_b2_1 = _mm_mul_ps(self.twiddle2im, x1m30);
5066 let t_b2_2 = _mm_mul_ps(self.twiddle4im, x2m29);
5067 let t_b2_3 = _mm_mul_ps(self.twiddle6im, x3m28);
5068 let t_b2_4 = _mm_mul_ps(self.twiddle8im, x4m27);
5069 let t_b2_5 = _mm_mul_ps(self.twiddle10im, x5m26);
5070 let t_b2_6 = _mm_mul_ps(self.twiddle12im, x6m25);
5071 let t_b2_7 = _mm_mul_ps(self.twiddle14im, x7m24);
5072 let t_b2_8 = _mm_mul_ps(self.twiddle15im, x8m23);
5073 let t_b2_9 = _mm_mul_ps(self.twiddle13im, x9m22);
5074 let t_b2_10 = _mm_mul_ps(self.twiddle11im, x10m21);
5075 let t_b2_11 = _mm_mul_ps(self.twiddle9im, x11m20);
5076 let t_b2_12 = _mm_mul_ps(self.twiddle7im, x12m19);
5077 let t_b2_13 = _mm_mul_ps(self.twiddle5im, x13m18);
5078 let t_b2_14 = _mm_mul_ps(self.twiddle3im, x14m17);
5079 let t_b2_15 = _mm_mul_ps(self.twiddle1im, x15m16);
5080 let t_b3_1 = _mm_mul_ps(self.twiddle3im, x1m30);
5081 let t_b3_2 = _mm_mul_ps(self.twiddle6im, x2m29);
5082 let t_b3_3 = _mm_mul_ps(self.twiddle9im, x3m28);
5083 let t_b3_4 = _mm_mul_ps(self.twiddle12im, x4m27);
5084 let t_b3_5 = _mm_mul_ps(self.twiddle15im, x5m26);
5085 let t_b3_6 = _mm_mul_ps(self.twiddle13im, x6m25);
5086 let t_b3_7 = _mm_mul_ps(self.twiddle10im, x7m24);
5087 let t_b3_8 = _mm_mul_ps(self.twiddle7im, x8m23);
5088 let t_b3_9 = _mm_mul_ps(self.twiddle4im, x9m22);
5089 let t_b3_10 = _mm_mul_ps(self.twiddle1im, x10m21);
5090 let t_b3_11 = _mm_mul_ps(self.twiddle2im, x11m20);
5091 let t_b3_12 = _mm_mul_ps(self.twiddle5im, x12m19);
5092 let t_b3_13 = _mm_mul_ps(self.twiddle8im, x13m18);
5093 let t_b3_14 = _mm_mul_ps(self.twiddle11im, x14m17);
5094 let t_b3_15 = _mm_mul_ps(self.twiddle14im, x15m16);
5095 let t_b4_1 = _mm_mul_ps(self.twiddle4im, x1m30);
5096 let t_b4_2 = _mm_mul_ps(self.twiddle8im, x2m29);
5097 let t_b4_3 = _mm_mul_ps(self.twiddle12im, x3m28);
5098 let t_b4_4 = _mm_mul_ps(self.twiddle15im, x4m27);
5099 let t_b4_5 = _mm_mul_ps(self.twiddle11im, x5m26);
5100 let t_b4_6 = _mm_mul_ps(self.twiddle7im, x6m25);
5101 let t_b4_7 = _mm_mul_ps(self.twiddle3im, x7m24);
5102 let t_b4_8 = _mm_mul_ps(self.twiddle1im, x8m23);
5103 let t_b4_9 = _mm_mul_ps(self.twiddle5im, x9m22);
5104 let t_b4_10 = _mm_mul_ps(self.twiddle9im, x10m21);
5105 let t_b4_11 = _mm_mul_ps(self.twiddle13im, x11m20);
5106 let t_b4_12 = _mm_mul_ps(self.twiddle14im, x12m19);
5107 let t_b4_13 = _mm_mul_ps(self.twiddle10im, x13m18);
5108 let t_b4_14 = _mm_mul_ps(self.twiddle6im, x14m17);
5109 let t_b4_15 = _mm_mul_ps(self.twiddle2im, x15m16);
5110 let t_b5_1 = _mm_mul_ps(self.twiddle5im, x1m30);
5111 let t_b5_2 = _mm_mul_ps(self.twiddle10im, x2m29);
5112 let t_b5_3 = _mm_mul_ps(self.twiddle15im, x3m28);
5113 let t_b5_4 = _mm_mul_ps(self.twiddle11im, x4m27);
5114 let t_b5_5 = _mm_mul_ps(self.twiddle6im, x5m26);
5115 let t_b5_6 = _mm_mul_ps(self.twiddle1im, x6m25);
5116 let t_b5_7 = _mm_mul_ps(self.twiddle4im, x7m24);
5117 let t_b5_8 = _mm_mul_ps(self.twiddle9im, x8m23);
5118 let t_b5_9 = _mm_mul_ps(self.twiddle14im, x9m22);
5119 let t_b5_10 = _mm_mul_ps(self.twiddle12im, x10m21);
5120 let t_b5_11 = _mm_mul_ps(self.twiddle7im, x11m20);
5121 let t_b5_12 = _mm_mul_ps(self.twiddle2im, x12m19);
5122 let t_b5_13 = _mm_mul_ps(self.twiddle3im, x13m18);
5123 let t_b5_14 = _mm_mul_ps(self.twiddle8im, x14m17);
5124 let t_b5_15 = _mm_mul_ps(self.twiddle13im, x15m16);
5125 let t_b6_1 = _mm_mul_ps(self.twiddle6im, x1m30);
5126 let t_b6_2 = _mm_mul_ps(self.twiddle12im, x2m29);
5127 let t_b6_3 = _mm_mul_ps(self.twiddle13im, x3m28);
5128 let t_b6_4 = _mm_mul_ps(self.twiddle7im, x4m27);
5129 let t_b6_5 = _mm_mul_ps(self.twiddle1im, x5m26);
5130 let t_b6_6 = _mm_mul_ps(self.twiddle5im, x6m25);
5131 let t_b6_7 = _mm_mul_ps(self.twiddle11im, x7m24);
5132 let t_b6_8 = _mm_mul_ps(self.twiddle14im, x8m23);
5133 let t_b6_9 = _mm_mul_ps(self.twiddle8im, x9m22);
5134 let t_b6_10 = _mm_mul_ps(self.twiddle2im, x10m21);
5135 let t_b6_11 = _mm_mul_ps(self.twiddle4im, x11m20);
5136 let t_b6_12 = _mm_mul_ps(self.twiddle10im, x12m19);
5137 let t_b6_13 = _mm_mul_ps(self.twiddle15im, x13m18);
5138 let t_b6_14 = _mm_mul_ps(self.twiddle9im, x14m17);
5139 let t_b6_15 = _mm_mul_ps(self.twiddle3im, x15m16);
5140 let t_b7_1 = _mm_mul_ps(self.twiddle7im, x1m30);
5141 let t_b7_2 = _mm_mul_ps(self.twiddle14im, x2m29);
5142 let t_b7_3 = _mm_mul_ps(self.twiddle10im, x3m28);
5143 let t_b7_4 = _mm_mul_ps(self.twiddle3im, x4m27);
5144 let t_b7_5 = _mm_mul_ps(self.twiddle4im, x5m26);
5145 let t_b7_6 = _mm_mul_ps(self.twiddle11im, x6m25);
5146 let t_b7_7 = _mm_mul_ps(self.twiddle13im, x7m24);
5147 let t_b7_8 = _mm_mul_ps(self.twiddle6im, x8m23);
5148 let t_b7_9 = _mm_mul_ps(self.twiddle1im, x9m22);
5149 let t_b7_10 = _mm_mul_ps(self.twiddle8im, x10m21);
5150 let t_b7_11 = _mm_mul_ps(self.twiddle15im, x11m20);
5151 let t_b7_12 = _mm_mul_ps(self.twiddle9im, x12m19);
5152 let t_b7_13 = _mm_mul_ps(self.twiddle2im, x13m18);
5153 let t_b7_14 = _mm_mul_ps(self.twiddle5im, x14m17);
5154 let t_b7_15 = _mm_mul_ps(self.twiddle12im, x15m16);
5155 let t_b8_1 = _mm_mul_ps(self.twiddle8im, x1m30);
5156 let t_b8_2 = _mm_mul_ps(self.twiddle15im, x2m29);
5157 let t_b8_3 = _mm_mul_ps(self.twiddle7im, x3m28);
5158 let t_b8_4 = _mm_mul_ps(self.twiddle1im, x4m27);
5159 let t_b8_5 = _mm_mul_ps(self.twiddle9im, x5m26);
5160 let t_b8_6 = _mm_mul_ps(self.twiddle14im, x6m25);
5161 let t_b8_7 = _mm_mul_ps(self.twiddle6im, x7m24);
5162 let t_b8_8 = _mm_mul_ps(self.twiddle2im, x8m23);
5163 let t_b8_9 = _mm_mul_ps(self.twiddle10im, x9m22);
5164 let t_b8_10 = _mm_mul_ps(self.twiddle13im, x10m21);
5165 let t_b8_11 = _mm_mul_ps(self.twiddle5im, x11m20);
5166 let t_b8_12 = _mm_mul_ps(self.twiddle3im, x12m19);
5167 let t_b8_13 = _mm_mul_ps(self.twiddle11im, x13m18);
5168 let t_b8_14 = _mm_mul_ps(self.twiddle12im, x14m17);
5169 let t_b8_15 = _mm_mul_ps(self.twiddle4im, x15m16);
5170 let t_b9_1 = _mm_mul_ps(self.twiddle9im, x1m30);
5171 let t_b9_2 = _mm_mul_ps(self.twiddle13im, x2m29);
5172 let t_b9_3 = _mm_mul_ps(self.twiddle4im, x3m28);
5173 let t_b9_4 = _mm_mul_ps(self.twiddle5im, x4m27);
5174 let t_b9_5 = _mm_mul_ps(self.twiddle14im, x5m26);
5175 let t_b9_6 = _mm_mul_ps(self.twiddle8im, x6m25);
5176 let t_b9_7 = _mm_mul_ps(self.twiddle1im, x7m24);
5177 let t_b9_8 = _mm_mul_ps(self.twiddle10im, x8m23);
5178 let t_b9_9 = _mm_mul_ps(self.twiddle12im, x9m22);
5179 let t_b9_10 = _mm_mul_ps(self.twiddle3im, x10m21);
5180 let t_b9_11 = _mm_mul_ps(self.twiddle6im, x11m20);
5181 let t_b9_12 = _mm_mul_ps(self.twiddle15im, x12m19);
5182 let t_b9_13 = _mm_mul_ps(self.twiddle7im, x13m18);
5183 let t_b9_14 = _mm_mul_ps(self.twiddle2im, x14m17);
5184 let t_b9_15 = _mm_mul_ps(self.twiddle11im, x15m16);
5185 let t_b10_1 = _mm_mul_ps(self.twiddle10im, x1m30);
5186 let t_b10_2 = _mm_mul_ps(self.twiddle11im, x2m29);
5187 let t_b10_3 = _mm_mul_ps(self.twiddle1im, x3m28);
5188 let t_b10_4 = _mm_mul_ps(self.twiddle9im, x4m27);
5189 let t_b10_5 = _mm_mul_ps(self.twiddle12im, x5m26);
5190 let t_b10_6 = _mm_mul_ps(self.twiddle2im, x6m25);
5191 let t_b10_7 = _mm_mul_ps(self.twiddle8im, x7m24);
5192 let t_b10_8 = _mm_mul_ps(self.twiddle13im, x8m23);
5193 let t_b10_9 = _mm_mul_ps(self.twiddle3im, x9m22);
5194 let t_b10_10 = _mm_mul_ps(self.twiddle7im, x10m21);
5195 let t_b10_11 = _mm_mul_ps(self.twiddle14im, x11m20);
5196 let t_b10_12 = _mm_mul_ps(self.twiddle4im, x12m19);
5197 let t_b10_13 = _mm_mul_ps(self.twiddle6im, x13m18);
5198 let t_b10_14 = _mm_mul_ps(self.twiddle15im, x14m17);
5199 let t_b10_15 = _mm_mul_ps(self.twiddle5im, x15m16);
5200 let t_b11_1 = _mm_mul_ps(self.twiddle11im, x1m30);
5201 let t_b11_2 = _mm_mul_ps(self.twiddle9im, x2m29);
5202 let t_b11_3 = _mm_mul_ps(self.twiddle2im, x3m28);
5203 let t_b11_4 = _mm_mul_ps(self.twiddle13im, x4m27);
5204 let t_b11_5 = _mm_mul_ps(self.twiddle7im, x5m26);
5205 let t_b11_6 = _mm_mul_ps(self.twiddle4im, x6m25);
5206 let t_b11_7 = _mm_mul_ps(self.twiddle15im, x7m24);
5207 let t_b11_8 = _mm_mul_ps(self.twiddle5im, x8m23);
5208 let t_b11_9 = _mm_mul_ps(self.twiddle6im, x9m22);
5209 let t_b11_10 = _mm_mul_ps(self.twiddle14im, x10m21);
5210 let t_b11_11 = _mm_mul_ps(self.twiddle3im, x11m20);
5211 let t_b11_12 = _mm_mul_ps(self.twiddle8im, x12m19);
5212 let t_b11_13 = _mm_mul_ps(self.twiddle12im, x13m18);
5213 let t_b11_14 = _mm_mul_ps(self.twiddle1im, x14m17);
5214 let t_b11_15 = _mm_mul_ps(self.twiddle10im, x15m16);
5215 let t_b12_1 = _mm_mul_ps(self.twiddle12im, x1m30);
5216 let t_b12_2 = _mm_mul_ps(self.twiddle7im, x2m29);
5217 let t_b12_3 = _mm_mul_ps(self.twiddle5im, x3m28);
5218 let t_b12_4 = _mm_mul_ps(self.twiddle14im, x4m27);
5219 let t_b12_5 = _mm_mul_ps(self.twiddle2im, x5m26);
5220 let t_b12_6 = _mm_mul_ps(self.twiddle10im, x6m25);
5221 let t_b12_7 = _mm_mul_ps(self.twiddle9im, x7m24);
5222 let t_b12_8 = _mm_mul_ps(self.twiddle3im, x8m23);
5223 let t_b12_9 = _mm_mul_ps(self.twiddle15im, x9m22);
5224 let t_b12_10 = _mm_mul_ps(self.twiddle4im, x10m21);
5225 let t_b12_11 = _mm_mul_ps(self.twiddle8im, x11m20);
5226 let t_b12_12 = _mm_mul_ps(self.twiddle11im, x12m19);
5227 let t_b12_13 = _mm_mul_ps(self.twiddle1im, x13m18);
5228 let t_b12_14 = _mm_mul_ps(self.twiddle13im, x14m17);
5229 let t_b12_15 = _mm_mul_ps(self.twiddle6im, x15m16);
5230 let t_b13_1 = _mm_mul_ps(self.twiddle13im, x1m30);
5231 let t_b13_2 = _mm_mul_ps(self.twiddle5im, x2m29);
5232 let t_b13_3 = _mm_mul_ps(self.twiddle8im, x3m28);
5233 let t_b13_4 = _mm_mul_ps(self.twiddle10im, x4m27);
5234 let t_b13_5 = _mm_mul_ps(self.twiddle3im, x5m26);
5235 let t_b13_6 = _mm_mul_ps(self.twiddle15im, x6m25);
5236 let t_b13_7 = _mm_mul_ps(self.twiddle2im, x7m24);
5237 let t_b13_8 = _mm_mul_ps(self.twiddle11im, x8m23);
5238 let t_b13_9 = _mm_mul_ps(self.twiddle7im, x9m22);
5239 let t_b13_10 = _mm_mul_ps(self.twiddle6im, x10m21);
5240 let t_b13_11 = _mm_mul_ps(self.twiddle12im, x11m20);
5241 let t_b13_12 = _mm_mul_ps(self.twiddle1im, x12m19);
5242 let t_b13_13 = _mm_mul_ps(self.twiddle14im, x13m18);
5243 let t_b13_14 = _mm_mul_ps(self.twiddle4im, x14m17);
5244 let t_b13_15 = _mm_mul_ps(self.twiddle9im, x15m16);
5245 let t_b14_1 = _mm_mul_ps(self.twiddle14im, x1m30);
5246 let t_b14_2 = _mm_mul_ps(self.twiddle3im, x2m29);
5247 let t_b14_3 = _mm_mul_ps(self.twiddle11im, x3m28);
5248 let t_b14_4 = _mm_mul_ps(self.twiddle6im, x4m27);
5249 let t_b14_5 = _mm_mul_ps(self.twiddle8im, x5m26);
5250 let t_b14_6 = _mm_mul_ps(self.twiddle9im, x6m25);
5251 let t_b14_7 = _mm_mul_ps(self.twiddle5im, x7m24);
5252 let t_b14_8 = _mm_mul_ps(self.twiddle12im, x8m23);
5253 let t_b14_9 = _mm_mul_ps(self.twiddle2im, x9m22);
5254 let t_b14_10 = _mm_mul_ps(self.twiddle15im, x10m21);
5255 let t_b14_11 = _mm_mul_ps(self.twiddle1im, x11m20);
5256 let t_b14_12 = _mm_mul_ps(self.twiddle13im, x12m19);
5257 let t_b14_13 = _mm_mul_ps(self.twiddle4im, x13m18);
5258 let t_b14_14 = _mm_mul_ps(self.twiddle10im, x14m17);
5259 let t_b14_15 = _mm_mul_ps(self.twiddle7im, x15m16);
5260 let t_b15_1 = _mm_mul_ps(self.twiddle15im, x1m30);
5261 let t_b15_2 = _mm_mul_ps(self.twiddle1im, x2m29);
5262 let t_b15_3 = _mm_mul_ps(self.twiddle14im, x3m28);
5263 let t_b15_4 = _mm_mul_ps(self.twiddle2im, x4m27);
5264 let t_b15_5 = _mm_mul_ps(self.twiddle13im, x5m26);
5265 let t_b15_6 = _mm_mul_ps(self.twiddle3im, x6m25);
5266 let t_b15_7 = _mm_mul_ps(self.twiddle12im, x7m24);
5267 let t_b15_8 = _mm_mul_ps(self.twiddle4im, x8m23);
5268 let t_b15_9 = _mm_mul_ps(self.twiddle11im, x9m22);
5269 let t_b15_10 = _mm_mul_ps(self.twiddle5im, x10m21);
5270 let t_b15_11 = _mm_mul_ps(self.twiddle10im, x11m20);
5271 let t_b15_12 = _mm_mul_ps(self.twiddle6im, x12m19);
5272 let t_b15_13 = _mm_mul_ps(self.twiddle9im, x13m18);
5273 let t_b15_14 = _mm_mul_ps(self.twiddle7im, x14m17);
5274 let t_b15_15 = _mm_mul_ps(self.twiddle8im, x15m16);
5275
5276 let x0 = values[0];
5277 let t_a1 = calc_f32!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9 + t_a1_10 + t_a1_11 + t_a1_12 + t_a1_13 + t_a1_14 + t_a1_15);
5278 let t_a2 = calc_f32!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9 + t_a2_10 + t_a2_11 + t_a2_12 + t_a2_13 + t_a2_14 + t_a2_15);
5279 let t_a3 = calc_f32!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9 + t_a3_10 + t_a3_11 + t_a3_12 + t_a3_13 + t_a3_14 + t_a3_15);
5280 let t_a4 = calc_f32!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9 + t_a4_10 + t_a4_11 + t_a4_12 + t_a4_13 + t_a4_14 + t_a4_15);
5281 let t_a5 = calc_f32!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9 + t_a5_10 + t_a5_11 + t_a5_12 + t_a5_13 + t_a5_14 + t_a5_15);
5282 let t_a6 = calc_f32!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9 + t_a6_10 + t_a6_11 + t_a6_12 + t_a6_13 + t_a6_14 + t_a6_15);
5283 let t_a7 = calc_f32!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9 + t_a7_10 + t_a7_11 + t_a7_12 + t_a7_13 + t_a7_14 + t_a7_15);
5284 let t_a8 = calc_f32!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9 + t_a8_10 + t_a8_11 + t_a8_12 + t_a8_13 + t_a8_14 + t_a8_15);
5285 let t_a9 = calc_f32!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9 + t_a9_10 + t_a9_11 + t_a9_12 + t_a9_13 + t_a9_14 + t_a9_15);
5286 let t_a10 = calc_f32!(x0 + t_a10_1 + t_a10_2 + t_a10_3 + t_a10_4 + t_a10_5 + t_a10_6 + t_a10_7 + t_a10_8 + t_a10_9 + t_a10_10 + t_a10_11 + t_a10_12 + t_a10_13 + t_a10_14 + t_a10_15);
5287 let t_a11 = calc_f32!(x0 + t_a11_1 + t_a11_2 + t_a11_3 + t_a11_4 + t_a11_5 + t_a11_6 + t_a11_7 + t_a11_8 + t_a11_9 + t_a11_10 + t_a11_11 + t_a11_12 + t_a11_13 + t_a11_14 + t_a11_15);
5288 let t_a12 = calc_f32!(x0 + t_a12_1 + t_a12_2 + t_a12_3 + t_a12_4 + t_a12_5 + t_a12_6 + t_a12_7 + t_a12_8 + t_a12_9 + t_a12_10 + t_a12_11 + t_a12_12 + t_a12_13 + t_a12_14 + t_a12_15);
5289 let t_a13 = calc_f32!(x0 + t_a13_1 + t_a13_2 + t_a13_3 + t_a13_4 + t_a13_5 + t_a13_6 + t_a13_7 + t_a13_8 + t_a13_9 + t_a13_10 + t_a13_11 + t_a13_12 + t_a13_13 + t_a13_14 + t_a13_15);
5290 let t_a14 = calc_f32!(x0 + t_a14_1 + t_a14_2 + t_a14_3 + t_a14_4 + t_a14_5 + t_a14_6 + t_a14_7 + t_a14_8 + t_a14_9 + t_a14_10 + t_a14_11 + t_a14_12 + t_a14_13 + t_a14_14 + t_a14_15);
5291 let t_a15 = calc_f32!(x0 + t_a15_1 + t_a15_2 + t_a15_3 + t_a15_4 + t_a15_5 + t_a15_6 + t_a15_7 + t_a15_8 + t_a15_9 + t_a15_10 + t_a15_11 + t_a15_12 + t_a15_13 + t_a15_14 + t_a15_15);
5292
5293 let t_b1 = calc_f32!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9 + t_b1_10 + t_b1_11 + t_b1_12 + t_b1_13 + t_b1_14 + t_b1_15);
5294 let t_b2 = calc_f32!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7 - t_b2_8 - t_b2_9 - t_b2_10 - t_b2_11 - t_b2_12 - t_b2_13 - t_b2_14 - t_b2_15);
5295 let t_b3 = calc_f32!(t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 + t_b3_5 - t_b3_6 - t_b3_7 - t_b3_8 - t_b3_9 - t_b3_10 + t_b3_11 + t_b3_12 + t_b3_13 + t_b3_14 + t_b3_15);
5296 let t_b4 = calc_f32!(t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7 + t_b4_8 + t_b4_9 + t_b4_10 + t_b4_11 - t_b4_12 - t_b4_13 - t_b4_14 - t_b4_15);
5297 let t_b5 = calc_f32!(t_b5_1 + t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8 + t_b5_9 - t_b5_10 - t_b5_11 - t_b5_12 + t_b5_13 + t_b5_14 + t_b5_15);
5298 let t_b6 = calc_f32!(t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9 - t_b6_10 + t_b6_11 + t_b6_12 - t_b6_13 - t_b6_14 - t_b6_15);
5299 let t_b7 = calc_f32!(t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9 + t_b7_10 + t_b7_11 - t_b7_12 - t_b7_13 + t_b7_14 + t_b7_15);
5300 let t_b8 = calc_f32!(t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9 - t_b8_10 - t_b8_11 + t_b8_12 + t_b8_13 - t_b8_14 - t_b8_15);
5301 let t_b9 = calc_f32!(t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8 - t_b9_9 - t_b9_10 + t_b9_11 + t_b9_12 - t_b9_13 + t_b9_14 + t_b9_15);
5302 let t_b10 = calc_f32!(t_b10_1 - t_b10_2 - t_b10_3 + t_b10_4 - t_b10_5 - t_b10_6 + t_b10_7 - t_b10_8 - t_b10_9 + t_b10_10 - t_b10_11 - t_b10_12 + t_b10_13 - t_b10_14 - t_b10_15);
5303 let t_b11 = calc_f32!(t_b11_1 - t_b11_2 + t_b11_3 + t_b11_4 - t_b11_5 + t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9 - t_b11_10 - t_b11_11 + t_b11_12 - t_b11_13 - t_b11_14 + t_b11_15);
5304 let t_b12 = calc_f32!(t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 - t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 + t_b12_9 - t_b12_10 + t_b12_11 - t_b12_12 + t_b12_13 + t_b12_14 - t_b12_15);
5305 let t_b13 = calc_f32!(t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 - t_b13_7 + t_b13_8 - t_b13_9 + t_b13_10 - t_b13_11 + t_b13_12 + t_b13_13 - t_b13_14 + t_b13_15);
5306 let t_b14 = calc_f32!(t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9 - t_b14_10 - t_b14_11 + t_b14_12 - t_b14_13 + t_b14_14 - t_b14_15);
5307 let t_b15 = calc_f32!(t_b15_1 - t_b15_2 + t_b15_3 - t_b15_4 + t_b15_5 - t_b15_6 + t_b15_7 - t_b15_8 + t_b15_9 - t_b15_10 + t_b15_11 - t_b15_12 + t_b15_13 - t_b15_14 + t_b15_15);
5308
5309 let t_b1_rot = self.rotate.rotate_both(t_b1);
5310 let t_b2_rot = self.rotate.rotate_both(t_b2);
5311 let t_b3_rot = self.rotate.rotate_both(t_b3);
5312 let t_b4_rot = self.rotate.rotate_both(t_b4);
5313 let t_b5_rot = self.rotate.rotate_both(t_b5);
5314 let t_b6_rot = self.rotate.rotate_both(t_b6);
5315 let t_b7_rot = self.rotate.rotate_both(t_b7);
5316 let t_b8_rot = self.rotate.rotate_both(t_b8);
5317 let t_b9_rot = self.rotate.rotate_both(t_b9);
5318 let t_b10_rot = self.rotate.rotate_both(t_b10);
5319 let t_b11_rot = self.rotate.rotate_both(t_b11);
5320 let t_b12_rot = self.rotate.rotate_both(t_b12);
5321 let t_b13_rot = self.rotate.rotate_both(t_b13);
5322 let t_b14_rot = self.rotate.rotate_both(t_b14);
5323 let t_b15_rot = self.rotate.rotate_both(t_b15);
5324
5325 let y0 = calc_f32!(x0 + x1p30 + x2p29 + x3p28 + x4p27 + x5p26 + x6p25 + x7p24 + x8p23 + x9p22 + x10p21 + x11p20 + x12p19 + x13p18 + x14p17 + x15p16);
5326 let [y1, y30] = parallel_fft2_interleaved_f32(t_a1, t_b1_rot);
5327 let [y2, y29] = parallel_fft2_interleaved_f32(t_a2, t_b2_rot);
5328 let [y3, y28] = parallel_fft2_interleaved_f32(t_a3, t_b3_rot);
5329 let [y4, y27] = parallel_fft2_interleaved_f32(t_a4, t_b4_rot);
5330 let [y5, y26] = parallel_fft2_interleaved_f32(t_a5, t_b5_rot);
5331 let [y6, y25] = parallel_fft2_interleaved_f32(t_a6, t_b6_rot);
5332 let [y7, y24] = parallel_fft2_interleaved_f32(t_a7, t_b7_rot);
5333 let [y8, y23] = parallel_fft2_interleaved_f32(t_a8, t_b8_rot);
5334 let [y9, y22] = parallel_fft2_interleaved_f32(t_a9, t_b9_rot);
5335 let [y10, y21] = parallel_fft2_interleaved_f32(t_a10, t_b10_rot);
5336 let [y11, y20] = parallel_fft2_interleaved_f32(t_a11, t_b11_rot);
5337 let [y12, y19] = parallel_fft2_interleaved_f32(t_a12, t_b12_rot);
5338 let [y13, y18] = parallel_fft2_interleaved_f32(t_a13, t_b13_rot);
5339 let [y14, y17] = parallel_fft2_interleaved_f32(t_a14, t_b14_rot);
5340 let [y15, y16] = parallel_fft2_interleaved_f32(t_a15, t_b15_rot);
5341 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30]
5342 }
5343 }
5344
5345 // _____ _ __ _ _ _ _ _
5346 // |___ // | / /_ | || | | |__ (_) |_
5347 // |_ \| | _____ | '_ \| || |_| '_ \| | __|
5348 // ___) | | |_____| | (_) |__ _| |_) | | |_
5349 // |____/|_| \___/ |_| |_.__/|_|\__|
5350 //
5351
5352 pub struct SseF64Butterfly31<T> {
5353 direction: FftDirection,
5354 _phantom: std::marker::PhantomData<T>,
5355 rotate: Rotate90F64,
5356 twiddle1re: __m128d,
5357 twiddle1im: __m128d,
5358 twiddle2re: __m128d,
5359 twiddle2im: __m128d,
5360 twiddle3re: __m128d,
5361 twiddle3im: __m128d,
5362 twiddle4re: __m128d,
5363 twiddle4im: __m128d,
5364 twiddle5re: __m128d,
5365 twiddle5im: __m128d,
5366 twiddle6re: __m128d,
5367 twiddle6im: __m128d,
5368 twiddle7re: __m128d,
5369 twiddle7im: __m128d,
5370 twiddle8re: __m128d,
5371 twiddle8im: __m128d,
5372 twiddle9re: __m128d,
5373 twiddle9im: __m128d,
5374 twiddle10re: __m128d,
5375 twiddle10im: __m128d,
5376 twiddle11re: __m128d,
5377 twiddle11im: __m128d,
5378 twiddle12re: __m128d,
5379 twiddle12im: __m128d,
5380 twiddle13re: __m128d,
5381 twiddle13im: __m128d,
5382 twiddle14re: __m128d,
5383 twiddle14im: __m128d,
5384 twiddle15re: __m128d,
5385 twiddle15im: __m128d,
5386 }
5387
5388 boilerplate_fft_sse_f64_butterfly!(SseF64Butterfly31, 31, |this: &SseF64Butterfly31<_>| this
5389 .direction);
5390 boilerplate_fft_sse_common_butterfly!(SseF64Butterfly31, 31, |this: &SseF64Butterfly31<_>| this
5391 .direction);
5392 impl<T: FftNum> SseF64Butterfly31<T> {
5393 #[inline(always)]
5394 pub fn new(direction: FftDirection) -> Self {
5395 assert_f64::<T>();
5396 let rotate = Rotate90F64::new(true);
5397 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 31, direction);
5398 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 31, direction);
5399 let tw3: Complex<f64> = twiddles::compute_twiddle(3, 31, direction);
5400 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 31, direction);
5401 let tw5: Complex<f64> = twiddles::compute_twiddle(5, 31, direction);
5402 let tw6: Complex<f64> = twiddles::compute_twiddle(6, 31, direction);
5403 let tw7: Complex<f64> = twiddles::compute_twiddle(7, 31, direction);
5404 let tw8: Complex<f64> = twiddles::compute_twiddle(8, 31, direction);
5405 let tw9: Complex<f64> = twiddles::compute_twiddle(9, 31, direction);
5406 let tw10: Complex<f64> = twiddles::compute_twiddle(10, 31, direction);
5407 let tw11: Complex<f64> = twiddles::compute_twiddle(11, 31, direction);
5408 let tw12: Complex<f64> = twiddles::compute_twiddle(12, 31, direction);
5409 let tw13: Complex<f64> = twiddles::compute_twiddle(13, 31, direction);
5410 let tw14: Complex<f64> = twiddles::compute_twiddle(14, 31, direction);
5411 let tw15: Complex<f64> = twiddles::compute_twiddle(15, 31, direction);
5412 let twiddle1re = unsafe { _mm_set_pd(tw1.re, tw1.re) };
5413 let twiddle1im = unsafe { _mm_set_pd(tw1.im, tw1.im) };
5414 let twiddle2re = unsafe { _mm_set_pd(tw2.re, tw2.re) };
5415 let twiddle2im = unsafe { _mm_set_pd(tw2.im, tw2.im) };
5416 let twiddle3re = unsafe { _mm_set_pd(tw3.re, tw3.re) };
5417 let twiddle3im = unsafe { _mm_set_pd(tw3.im, tw3.im) };
5418 let twiddle4re = unsafe { _mm_set_pd(tw4.re, tw4.re) };
5419 let twiddle4im = unsafe { _mm_set_pd(tw4.im, tw4.im) };
5420 let twiddle5re = unsafe { _mm_set_pd(tw5.re, tw5.re) };
5421 let twiddle5im = unsafe { _mm_set_pd(tw5.im, tw5.im) };
5422 let twiddle6re = unsafe { _mm_set_pd(tw6.re, tw6.re) };
5423 let twiddle6im = unsafe { _mm_set_pd(tw6.im, tw6.im) };
5424 let twiddle7re = unsafe { _mm_set_pd(tw7.re, tw7.re) };
5425 let twiddle7im = unsafe { _mm_set_pd(tw7.im, tw7.im) };
5426 let twiddle8re = unsafe { _mm_set_pd(tw8.re, tw8.re) };
5427 let twiddle8im = unsafe { _mm_set_pd(tw8.im, tw8.im) };
5428 let twiddle9re = unsafe { _mm_set_pd(tw9.re, tw9.re) };
5429 let twiddle9im = unsafe { _mm_set_pd(tw9.im, tw9.im) };
5430 let twiddle10re = unsafe { _mm_set_pd(tw10.re, tw10.re) };
5431 let twiddle10im = unsafe { _mm_set_pd(tw10.im, tw10.im) };
5432 let twiddle11re = unsafe { _mm_set_pd(tw11.re, tw11.re) };
5433 let twiddle11im = unsafe { _mm_set_pd(tw11.im, tw11.im) };
5434 let twiddle12re = unsafe { _mm_set_pd(tw12.re, tw12.re) };
5435 let twiddle12im = unsafe { _mm_set_pd(tw12.im, tw12.im) };
5436 let twiddle13re = unsafe { _mm_set_pd(tw13.re, tw13.re) };
5437 let twiddle13im = unsafe { _mm_set_pd(tw13.im, tw13.im) };
5438 let twiddle14re = unsafe { _mm_set_pd(tw14.re, tw14.re) };
5439 let twiddle14im = unsafe { _mm_set_pd(tw14.im, tw14.im) };
5440 let twiddle15re = unsafe { _mm_set_pd(tw15.re, tw15.re) };
5441 let twiddle15im = unsafe { _mm_set_pd(tw15.im, tw15.im) };
5442
5443 Self {
5444 direction,
5445 _phantom: std::marker::PhantomData,
5446 rotate,
5447 twiddle1re,
5448 twiddle1im,
5449 twiddle2re,
5450 twiddle2im,
5451 twiddle3re,
5452 twiddle3im,
5453 twiddle4re,
5454 twiddle4im,
5455 twiddle5re,
5456 twiddle5im,
5457 twiddle6re,
5458 twiddle6im,
5459 twiddle7re,
5460 twiddle7im,
5461 twiddle8re,
5462 twiddle8im,
5463 twiddle9re,
5464 twiddle9im,
5465 twiddle10re,
5466 twiddle10im,
5467 twiddle11re,
5468 twiddle11im,
5469 twiddle12re,
5470 twiddle12im,
5471 twiddle13re,
5472 twiddle13im,
5473 twiddle14re,
5474 twiddle14im,
5475 twiddle15re,
5476 twiddle15im,
5477 }
5478 }
5479
5480 #[inline(always)]
5481 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl SseArrayMut<f64>) {
5482 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
5483
5484 let out = self.perform_fft_direct(values);
5485
5486 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30});
5487 }
5488
5489 #[inline(always)]
5490 pub(crate) unsafe fn perform_fft_direct(&self, values: [__m128d; 31]) -> [__m128d; 31] {
5491 let [x1p30, x1m30] = solo_fft2_f64(values[1], values[30]);
5492 let [x2p29, x2m29] = solo_fft2_f64(values[2], values[29]);
5493 let [x3p28, x3m28] = solo_fft2_f64(values[3], values[28]);
5494 let [x4p27, x4m27] = solo_fft2_f64(values[4], values[27]);
5495 let [x5p26, x5m26] = solo_fft2_f64(values[5], values[26]);
5496 let [x6p25, x6m25] = solo_fft2_f64(values[6], values[25]);
5497 let [x7p24, x7m24] = solo_fft2_f64(values[7], values[24]);
5498 let [x8p23, x8m23] = solo_fft2_f64(values[8], values[23]);
5499 let [x9p22, x9m22] = solo_fft2_f64(values[9], values[22]);
5500 let [x10p21, x10m21] = solo_fft2_f64(values[10], values[21]);
5501 let [x11p20, x11m20] = solo_fft2_f64(values[11], values[20]);
5502 let [x12p19, x12m19] = solo_fft2_f64(values[12], values[19]);
5503 let [x13p18, x13m18] = solo_fft2_f64(values[13], values[18]);
5504 let [x14p17, x14m17] = solo_fft2_f64(values[14], values[17]);
5505 let [x15p16, x15m16] = solo_fft2_f64(values[15], values[16]);
5506
5507 let t_a1_1 = _mm_mul_pd(self.twiddle1re, x1p30);
5508 let t_a1_2 = _mm_mul_pd(self.twiddle2re, x2p29);
5509 let t_a1_3 = _mm_mul_pd(self.twiddle3re, x3p28);
5510 let t_a1_4 = _mm_mul_pd(self.twiddle4re, x4p27);
5511 let t_a1_5 = _mm_mul_pd(self.twiddle5re, x5p26);
5512 let t_a1_6 = _mm_mul_pd(self.twiddle6re, x6p25);
5513 let t_a1_7 = _mm_mul_pd(self.twiddle7re, x7p24);
5514 let t_a1_8 = _mm_mul_pd(self.twiddle8re, x8p23);
5515 let t_a1_9 = _mm_mul_pd(self.twiddle9re, x9p22);
5516 let t_a1_10 = _mm_mul_pd(self.twiddle10re, x10p21);
5517 let t_a1_11 = _mm_mul_pd(self.twiddle11re, x11p20);
5518 let t_a1_12 = _mm_mul_pd(self.twiddle12re, x12p19);
5519 let t_a1_13 = _mm_mul_pd(self.twiddle13re, x13p18);
5520 let t_a1_14 = _mm_mul_pd(self.twiddle14re, x14p17);
5521 let t_a1_15 = _mm_mul_pd(self.twiddle15re, x15p16);
5522 let t_a2_1 = _mm_mul_pd(self.twiddle2re, x1p30);
5523 let t_a2_2 = _mm_mul_pd(self.twiddle4re, x2p29);
5524 let t_a2_3 = _mm_mul_pd(self.twiddle6re, x3p28);
5525 let t_a2_4 = _mm_mul_pd(self.twiddle8re, x4p27);
5526 let t_a2_5 = _mm_mul_pd(self.twiddle10re, x5p26);
5527 let t_a2_6 = _mm_mul_pd(self.twiddle12re, x6p25);
5528 let t_a2_7 = _mm_mul_pd(self.twiddle14re, x7p24);
5529 let t_a2_8 = _mm_mul_pd(self.twiddle15re, x8p23);
5530 let t_a2_9 = _mm_mul_pd(self.twiddle13re, x9p22);
5531 let t_a2_10 = _mm_mul_pd(self.twiddle11re, x10p21);
5532 let t_a2_11 = _mm_mul_pd(self.twiddle9re, x11p20);
5533 let t_a2_12 = _mm_mul_pd(self.twiddle7re, x12p19);
5534 let t_a2_13 = _mm_mul_pd(self.twiddle5re, x13p18);
5535 let t_a2_14 = _mm_mul_pd(self.twiddle3re, x14p17);
5536 let t_a2_15 = _mm_mul_pd(self.twiddle1re, x15p16);
5537 let t_a3_1 = _mm_mul_pd(self.twiddle3re, x1p30);
5538 let t_a3_2 = _mm_mul_pd(self.twiddle6re, x2p29);
5539 let t_a3_3 = _mm_mul_pd(self.twiddle9re, x3p28);
5540 let t_a3_4 = _mm_mul_pd(self.twiddle12re, x4p27);
5541 let t_a3_5 = _mm_mul_pd(self.twiddle15re, x5p26);
5542 let t_a3_6 = _mm_mul_pd(self.twiddle13re, x6p25);
5543 let t_a3_7 = _mm_mul_pd(self.twiddle10re, x7p24);
5544 let t_a3_8 = _mm_mul_pd(self.twiddle7re, x8p23);
5545 let t_a3_9 = _mm_mul_pd(self.twiddle4re, x9p22);
5546 let t_a3_10 = _mm_mul_pd(self.twiddle1re, x10p21);
5547 let t_a3_11 = _mm_mul_pd(self.twiddle2re, x11p20);
5548 let t_a3_12 = _mm_mul_pd(self.twiddle5re, x12p19);
5549 let t_a3_13 = _mm_mul_pd(self.twiddle8re, x13p18);
5550 let t_a3_14 = _mm_mul_pd(self.twiddle11re, x14p17);
5551 let t_a3_15 = _mm_mul_pd(self.twiddle14re, x15p16);
5552 let t_a4_1 = _mm_mul_pd(self.twiddle4re, x1p30);
5553 let t_a4_2 = _mm_mul_pd(self.twiddle8re, x2p29);
5554 let t_a4_3 = _mm_mul_pd(self.twiddle12re, x3p28);
5555 let t_a4_4 = _mm_mul_pd(self.twiddle15re, x4p27);
5556 let t_a4_5 = _mm_mul_pd(self.twiddle11re, x5p26);
5557 let t_a4_6 = _mm_mul_pd(self.twiddle7re, x6p25);
5558 let t_a4_7 = _mm_mul_pd(self.twiddle3re, x7p24);
5559 let t_a4_8 = _mm_mul_pd(self.twiddle1re, x8p23);
5560 let t_a4_9 = _mm_mul_pd(self.twiddle5re, x9p22);
5561 let t_a4_10 = _mm_mul_pd(self.twiddle9re, x10p21);
5562 let t_a4_11 = _mm_mul_pd(self.twiddle13re, x11p20);
5563 let t_a4_12 = _mm_mul_pd(self.twiddle14re, x12p19);
5564 let t_a4_13 = _mm_mul_pd(self.twiddle10re, x13p18);
5565 let t_a4_14 = _mm_mul_pd(self.twiddle6re, x14p17);
5566 let t_a4_15 = _mm_mul_pd(self.twiddle2re, x15p16);
5567 let t_a5_1 = _mm_mul_pd(self.twiddle5re, x1p30);
5568 let t_a5_2 = _mm_mul_pd(self.twiddle10re, x2p29);
5569 let t_a5_3 = _mm_mul_pd(self.twiddle15re, x3p28);
5570 let t_a5_4 = _mm_mul_pd(self.twiddle11re, x4p27);
5571 let t_a5_5 = _mm_mul_pd(self.twiddle6re, x5p26);
5572 let t_a5_6 = _mm_mul_pd(self.twiddle1re, x6p25);
5573 let t_a5_7 = _mm_mul_pd(self.twiddle4re, x7p24);
5574 let t_a5_8 = _mm_mul_pd(self.twiddle9re, x8p23);
5575 let t_a5_9 = _mm_mul_pd(self.twiddle14re, x9p22);
5576 let t_a5_10 = _mm_mul_pd(self.twiddle12re, x10p21);
5577 let t_a5_11 = _mm_mul_pd(self.twiddle7re, x11p20);
5578 let t_a5_12 = _mm_mul_pd(self.twiddle2re, x12p19);
5579 let t_a5_13 = _mm_mul_pd(self.twiddle3re, x13p18);
5580 let t_a5_14 = _mm_mul_pd(self.twiddle8re, x14p17);
5581 let t_a5_15 = _mm_mul_pd(self.twiddle13re, x15p16);
5582 let t_a6_1 = _mm_mul_pd(self.twiddle6re, x1p30);
5583 let t_a6_2 = _mm_mul_pd(self.twiddle12re, x2p29);
5584 let t_a6_3 = _mm_mul_pd(self.twiddle13re, x3p28);
5585 let t_a6_4 = _mm_mul_pd(self.twiddle7re, x4p27);
5586 let t_a6_5 = _mm_mul_pd(self.twiddle1re, x5p26);
5587 let t_a6_6 = _mm_mul_pd(self.twiddle5re, x6p25);
5588 let t_a6_7 = _mm_mul_pd(self.twiddle11re, x7p24);
5589 let t_a6_8 = _mm_mul_pd(self.twiddle14re, x8p23);
5590 let t_a6_9 = _mm_mul_pd(self.twiddle8re, x9p22);
5591 let t_a6_10 = _mm_mul_pd(self.twiddle2re, x10p21);
5592 let t_a6_11 = _mm_mul_pd(self.twiddle4re, x11p20);
5593 let t_a6_12 = _mm_mul_pd(self.twiddle10re, x12p19);
5594 let t_a6_13 = _mm_mul_pd(self.twiddle15re, x13p18);
5595 let t_a6_14 = _mm_mul_pd(self.twiddle9re, x14p17);
5596 let t_a6_15 = _mm_mul_pd(self.twiddle3re, x15p16);
5597 let t_a7_1 = _mm_mul_pd(self.twiddle7re, x1p30);
5598 let t_a7_2 = _mm_mul_pd(self.twiddle14re, x2p29);
5599 let t_a7_3 = _mm_mul_pd(self.twiddle10re, x3p28);
5600 let t_a7_4 = _mm_mul_pd(self.twiddle3re, x4p27);
5601 let t_a7_5 = _mm_mul_pd(self.twiddle4re, x5p26);
5602 let t_a7_6 = _mm_mul_pd(self.twiddle11re, x6p25);
5603 let t_a7_7 = _mm_mul_pd(self.twiddle13re, x7p24);
5604 let t_a7_8 = _mm_mul_pd(self.twiddle6re, x8p23);
5605 let t_a7_9 = _mm_mul_pd(self.twiddle1re, x9p22);
5606 let t_a7_10 = _mm_mul_pd(self.twiddle8re, x10p21);
5607 let t_a7_11 = _mm_mul_pd(self.twiddle15re, x11p20);
5608 let t_a7_12 = _mm_mul_pd(self.twiddle9re, x12p19);
5609 let t_a7_13 = _mm_mul_pd(self.twiddle2re, x13p18);
5610 let t_a7_14 = _mm_mul_pd(self.twiddle5re, x14p17);
5611 let t_a7_15 = _mm_mul_pd(self.twiddle12re, x15p16);
5612 let t_a8_1 = _mm_mul_pd(self.twiddle8re, x1p30);
5613 let t_a8_2 = _mm_mul_pd(self.twiddle15re, x2p29);
5614 let t_a8_3 = _mm_mul_pd(self.twiddle7re, x3p28);
5615 let t_a8_4 = _mm_mul_pd(self.twiddle1re, x4p27);
5616 let t_a8_5 = _mm_mul_pd(self.twiddle9re, x5p26);
5617 let t_a8_6 = _mm_mul_pd(self.twiddle14re, x6p25);
5618 let t_a8_7 = _mm_mul_pd(self.twiddle6re, x7p24);
5619 let t_a8_8 = _mm_mul_pd(self.twiddle2re, x8p23);
5620 let t_a8_9 = _mm_mul_pd(self.twiddle10re, x9p22);
5621 let t_a8_10 = _mm_mul_pd(self.twiddle13re, x10p21);
5622 let t_a8_11 = _mm_mul_pd(self.twiddle5re, x11p20);
5623 let t_a8_12 = _mm_mul_pd(self.twiddle3re, x12p19);
5624 let t_a8_13 = _mm_mul_pd(self.twiddle11re, x13p18);
5625 let t_a8_14 = _mm_mul_pd(self.twiddle12re, x14p17);
5626 let t_a8_15 = _mm_mul_pd(self.twiddle4re, x15p16);
5627 let t_a9_1 = _mm_mul_pd(self.twiddle9re, x1p30);
5628 let t_a9_2 = _mm_mul_pd(self.twiddle13re, x2p29);
5629 let t_a9_3 = _mm_mul_pd(self.twiddle4re, x3p28);
5630 let t_a9_4 = _mm_mul_pd(self.twiddle5re, x4p27);
5631 let t_a9_5 = _mm_mul_pd(self.twiddle14re, x5p26);
5632 let t_a9_6 = _mm_mul_pd(self.twiddle8re, x6p25);
5633 let t_a9_7 = _mm_mul_pd(self.twiddle1re, x7p24);
5634 let t_a9_8 = _mm_mul_pd(self.twiddle10re, x8p23);
5635 let t_a9_9 = _mm_mul_pd(self.twiddle12re, x9p22);
5636 let t_a9_10 = _mm_mul_pd(self.twiddle3re, x10p21);
5637 let t_a9_11 = _mm_mul_pd(self.twiddle6re, x11p20);
5638 let t_a9_12 = _mm_mul_pd(self.twiddle15re, x12p19);
5639 let t_a9_13 = _mm_mul_pd(self.twiddle7re, x13p18);
5640 let t_a9_14 = _mm_mul_pd(self.twiddle2re, x14p17);
5641 let t_a9_15 = _mm_mul_pd(self.twiddle11re, x15p16);
5642 let t_a10_1 = _mm_mul_pd(self.twiddle10re, x1p30);
5643 let t_a10_2 = _mm_mul_pd(self.twiddle11re, x2p29);
5644 let t_a10_3 = _mm_mul_pd(self.twiddle1re, x3p28);
5645 let t_a10_4 = _mm_mul_pd(self.twiddle9re, x4p27);
5646 let t_a10_5 = _mm_mul_pd(self.twiddle12re, x5p26);
5647 let t_a10_6 = _mm_mul_pd(self.twiddle2re, x6p25);
5648 let t_a10_7 = _mm_mul_pd(self.twiddle8re, x7p24);
5649 let t_a10_8 = _mm_mul_pd(self.twiddle13re, x8p23);
5650 let t_a10_9 = _mm_mul_pd(self.twiddle3re, x9p22);
5651 let t_a10_10 = _mm_mul_pd(self.twiddle7re, x10p21);
5652 let t_a10_11 = _mm_mul_pd(self.twiddle14re, x11p20);
5653 let t_a10_12 = _mm_mul_pd(self.twiddle4re, x12p19);
5654 let t_a10_13 = _mm_mul_pd(self.twiddle6re, x13p18);
5655 let t_a10_14 = _mm_mul_pd(self.twiddle15re, x14p17);
5656 let t_a10_15 = _mm_mul_pd(self.twiddle5re, x15p16);
5657 let t_a11_1 = _mm_mul_pd(self.twiddle11re, x1p30);
5658 let t_a11_2 = _mm_mul_pd(self.twiddle9re, x2p29);
5659 let t_a11_3 = _mm_mul_pd(self.twiddle2re, x3p28);
5660 let t_a11_4 = _mm_mul_pd(self.twiddle13re, x4p27);
5661 let t_a11_5 = _mm_mul_pd(self.twiddle7re, x5p26);
5662 let t_a11_6 = _mm_mul_pd(self.twiddle4re, x6p25);
5663 let t_a11_7 = _mm_mul_pd(self.twiddle15re, x7p24);
5664 let t_a11_8 = _mm_mul_pd(self.twiddle5re, x8p23);
5665 let t_a11_9 = _mm_mul_pd(self.twiddle6re, x9p22);
5666 let t_a11_10 = _mm_mul_pd(self.twiddle14re, x10p21);
5667 let t_a11_11 = _mm_mul_pd(self.twiddle3re, x11p20);
5668 let t_a11_12 = _mm_mul_pd(self.twiddle8re, x12p19);
5669 let t_a11_13 = _mm_mul_pd(self.twiddle12re, x13p18);
5670 let t_a11_14 = _mm_mul_pd(self.twiddle1re, x14p17);
5671 let t_a11_15 = _mm_mul_pd(self.twiddle10re, x15p16);
5672 let t_a12_1 = _mm_mul_pd(self.twiddle12re, x1p30);
5673 let t_a12_2 = _mm_mul_pd(self.twiddle7re, x2p29);
5674 let t_a12_3 = _mm_mul_pd(self.twiddle5re, x3p28);
5675 let t_a12_4 = _mm_mul_pd(self.twiddle14re, x4p27);
5676 let t_a12_5 = _mm_mul_pd(self.twiddle2re, x5p26);
5677 let t_a12_6 = _mm_mul_pd(self.twiddle10re, x6p25);
5678 let t_a12_7 = _mm_mul_pd(self.twiddle9re, x7p24);
5679 let t_a12_8 = _mm_mul_pd(self.twiddle3re, x8p23);
5680 let t_a12_9 = _mm_mul_pd(self.twiddle15re, x9p22);
5681 let t_a12_10 = _mm_mul_pd(self.twiddle4re, x10p21);
5682 let t_a12_11 = _mm_mul_pd(self.twiddle8re, x11p20);
5683 let t_a12_12 = _mm_mul_pd(self.twiddle11re, x12p19);
5684 let t_a12_13 = _mm_mul_pd(self.twiddle1re, x13p18);
5685 let t_a12_14 = _mm_mul_pd(self.twiddle13re, x14p17);
5686 let t_a12_15 = _mm_mul_pd(self.twiddle6re, x15p16);
5687 let t_a13_1 = _mm_mul_pd(self.twiddle13re, x1p30);
5688 let t_a13_2 = _mm_mul_pd(self.twiddle5re, x2p29);
5689 let t_a13_3 = _mm_mul_pd(self.twiddle8re, x3p28);
5690 let t_a13_4 = _mm_mul_pd(self.twiddle10re, x4p27);
5691 let t_a13_5 = _mm_mul_pd(self.twiddle3re, x5p26);
5692 let t_a13_6 = _mm_mul_pd(self.twiddle15re, x6p25);
5693 let t_a13_7 = _mm_mul_pd(self.twiddle2re, x7p24);
5694 let t_a13_8 = _mm_mul_pd(self.twiddle11re, x8p23);
5695 let t_a13_9 = _mm_mul_pd(self.twiddle7re, x9p22);
5696 let t_a13_10 = _mm_mul_pd(self.twiddle6re, x10p21);
5697 let t_a13_11 = _mm_mul_pd(self.twiddle12re, x11p20);
5698 let t_a13_12 = _mm_mul_pd(self.twiddle1re, x12p19);
5699 let t_a13_13 = _mm_mul_pd(self.twiddle14re, x13p18);
5700 let t_a13_14 = _mm_mul_pd(self.twiddle4re, x14p17);
5701 let t_a13_15 = _mm_mul_pd(self.twiddle9re, x15p16);
5702 let t_a14_1 = _mm_mul_pd(self.twiddle14re, x1p30);
5703 let t_a14_2 = _mm_mul_pd(self.twiddle3re, x2p29);
5704 let t_a14_3 = _mm_mul_pd(self.twiddle11re, x3p28);
5705 let t_a14_4 = _mm_mul_pd(self.twiddle6re, x4p27);
5706 let t_a14_5 = _mm_mul_pd(self.twiddle8re, x5p26);
5707 let t_a14_6 = _mm_mul_pd(self.twiddle9re, x6p25);
5708 let t_a14_7 = _mm_mul_pd(self.twiddle5re, x7p24);
5709 let t_a14_8 = _mm_mul_pd(self.twiddle12re, x8p23);
5710 let t_a14_9 = _mm_mul_pd(self.twiddle2re, x9p22);
5711 let t_a14_10 = _mm_mul_pd(self.twiddle15re, x10p21);
5712 let t_a14_11 = _mm_mul_pd(self.twiddle1re, x11p20);
5713 let t_a14_12 = _mm_mul_pd(self.twiddle13re, x12p19);
5714 let t_a14_13 = _mm_mul_pd(self.twiddle4re, x13p18);
5715 let t_a14_14 = _mm_mul_pd(self.twiddle10re, x14p17);
5716 let t_a14_15 = _mm_mul_pd(self.twiddle7re, x15p16);
5717 let t_a15_1 = _mm_mul_pd(self.twiddle15re, x1p30);
5718 let t_a15_2 = _mm_mul_pd(self.twiddle1re, x2p29);
5719 let t_a15_3 = _mm_mul_pd(self.twiddle14re, x3p28);
5720 let t_a15_4 = _mm_mul_pd(self.twiddle2re, x4p27);
5721 let t_a15_5 = _mm_mul_pd(self.twiddle13re, x5p26);
5722 let t_a15_6 = _mm_mul_pd(self.twiddle3re, x6p25);
5723 let t_a15_7 = _mm_mul_pd(self.twiddle12re, x7p24);
5724 let t_a15_8 = _mm_mul_pd(self.twiddle4re, x8p23);
5725 let t_a15_9 = _mm_mul_pd(self.twiddle11re, x9p22);
5726 let t_a15_10 = _mm_mul_pd(self.twiddle5re, x10p21);
5727 let t_a15_11 = _mm_mul_pd(self.twiddle10re, x11p20);
5728 let t_a15_12 = _mm_mul_pd(self.twiddle6re, x12p19);
5729 let t_a15_13 = _mm_mul_pd(self.twiddle9re, x13p18);
5730 let t_a15_14 = _mm_mul_pd(self.twiddle7re, x14p17);
5731 let t_a15_15 = _mm_mul_pd(self.twiddle8re, x15p16);
5732
5733 let t_b1_1 = _mm_mul_pd(self.twiddle1im, x1m30);
5734 let t_b1_2 = _mm_mul_pd(self.twiddle2im, x2m29);
5735 let t_b1_3 = _mm_mul_pd(self.twiddle3im, x3m28);
5736 let t_b1_4 = _mm_mul_pd(self.twiddle4im, x4m27);
5737 let t_b1_5 = _mm_mul_pd(self.twiddle5im, x5m26);
5738 let t_b1_6 = _mm_mul_pd(self.twiddle6im, x6m25);
5739 let t_b1_7 = _mm_mul_pd(self.twiddle7im, x7m24);
5740 let t_b1_8 = _mm_mul_pd(self.twiddle8im, x8m23);
5741 let t_b1_9 = _mm_mul_pd(self.twiddle9im, x9m22);
5742 let t_b1_10 = _mm_mul_pd(self.twiddle10im, x10m21);
5743 let t_b1_11 = _mm_mul_pd(self.twiddle11im, x11m20);
5744 let t_b1_12 = _mm_mul_pd(self.twiddle12im, x12m19);
5745 let t_b1_13 = _mm_mul_pd(self.twiddle13im, x13m18);
5746 let t_b1_14 = _mm_mul_pd(self.twiddle14im, x14m17);
5747 let t_b1_15 = _mm_mul_pd(self.twiddle15im, x15m16);
5748 let t_b2_1 = _mm_mul_pd(self.twiddle2im, x1m30);
5749 let t_b2_2 = _mm_mul_pd(self.twiddle4im, x2m29);
5750 let t_b2_3 = _mm_mul_pd(self.twiddle6im, x3m28);
5751 let t_b2_4 = _mm_mul_pd(self.twiddle8im, x4m27);
5752 let t_b2_5 = _mm_mul_pd(self.twiddle10im, x5m26);
5753 let t_b2_6 = _mm_mul_pd(self.twiddle12im, x6m25);
5754 let t_b2_7 = _mm_mul_pd(self.twiddle14im, x7m24);
5755 let t_b2_8 = _mm_mul_pd(self.twiddle15im, x8m23);
5756 let t_b2_9 = _mm_mul_pd(self.twiddle13im, x9m22);
5757 let t_b2_10 = _mm_mul_pd(self.twiddle11im, x10m21);
5758 let t_b2_11 = _mm_mul_pd(self.twiddle9im, x11m20);
5759 let t_b2_12 = _mm_mul_pd(self.twiddle7im, x12m19);
5760 let t_b2_13 = _mm_mul_pd(self.twiddle5im, x13m18);
5761 let t_b2_14 = _mm_mul_pd(self.twiddle3im, x14m17);
5762 let t_b2_15 = _mm_mul_pd(self.twiddle1im, x15m16);
5763 let t_b3_1 = _mm_mul_pd(self.twiddle3im, x1m30);
5764 let t_b3_2 = _mm_mul_pd(self.twiddle6im, x2m29);
5765 let t_b3_3 = _mm_mul_pd(self.twiddle9im, x3m28);
5766 let t_b3_4 = _mm_mul_pd(self.twiddle12im, x4m27);
5767 let t_b3_5 = _mm_mul_pd(self.twiddle15im, x5m26);
5768 let t_b3_6 = _mm_mul_pd(self.twiddle13im, x6m25);
5769 let t_b3_7 = _mm_mul_pd(self.twiddle10im, x7m24);
5770 let t_b3_8 = _mm_mul_pd(self.twiddle7im, x8m23);
5771 let t_b3_9 = _mm_mul_pd(self.twiddle4im, x9m22);
5772 let t_b3_10 = _mm_mul_pd(self.twiddle1im, x10m21);
5773 let t_b3_11 = _mm_mul_pd(self.twiddle2im, x11m20);
5774 let t_b3_12 = _mm_mul_pd(self.twiddle5im, x12m19);
5775 let t_b3_13 = _mm_mul_pd(self.twiddle8im, x13m18);
5776 let t_b3_14 = _mm_mul_pd(self.twiddle11im, x14m17);
5777 let t_b3_15 = _mm_mul_pd(self.twiddle14im, x15m16);
5778 let t_b4_1 = _mm_mul_pd(self.twiddle4im, x1m30);
5779 let t_b4_2 = _mm_mul_pd(self.twiddle8im, x2m29);
5780 let t_b4_3 = _mm_mul_pd(self.twiddle12im, x3m28);
5781 let t_b4_4 = _mm_mul_pd(self.twiddle15im, x4m27);
5782 let t_b4_5 = _mm_mul_pd(self.twiddle11im, x5m26);
5783 let t_b4_6 = _mm_mul_pd(self.twiddle7im, x6m25);
5784 let t_b4_7 = _mm_mul_pd(self.twiddle3im, x7m24);
5785 let t_b4_8 = _mm_mul_pd(self.twiddle1im, x8m23);
5786 let t_b4_9 = _mm_mul_pd(self.twiddle5im, x9m22);
5787 let t_b4_10 = _mm_mul_pd(self.twiddle9im, x10m21);
5788 let t_b4_11 = _mm_mul_pd(self.twiddle13im, x11m20);
5789 let t_b4_12 = _mm_mul_pd(self.twiddle14im, x12m19);
5790 let t_b4_13 = _mm_mul_pd(self.twiddle10im, x13m18);
5791 let t_b4_14 = _mm_mul_pd(self.twiddle6im, x14m17);
5792 let t_b4_15 = _mm_mul_pd(self.twiddle2im, x15m16);
5793 let t_b5_1 = _mm_mul_pd(self.twiddle5im, x1m30);
5794 let t_b5_2 = _mm_mul_pd(self.twiddle10im, x2m29);
5795 let t_b5_3 = _mm_mul_pd(self.twiddle15im, x3m28);
5796 let t_b5_4 = _mm_mul_pd(self.twiddle11im, x4m27);
5797 let t_b5_5 = _mm_mul_pd(self.twiddle6im, x5m26);
5798 let t_b5_6 = _mm_mul_pd(self.twiddle1im, x6m25);
5799 let t_b5_7 = _mm_mul_pd(self.twiddle4im, x7m24);
5800 let t_b5_8 = _mm_mul_pd(self.twiddle9im, x8m23);
5801 let t_b5_9 = _mm_mul_pd(self.twiddle14im, x9m22);
5802 let t_b5_10 = _mm_mul_pd(self.twiddle12im, x10m21);
5803 let t_b5_11 = _mm_mul_pd(self.twiddle7im, x11m20);
5804 let t_b5_12 = _mm_mul_pd(self.twiddle2im, x12m19);
5805 let t_b5_13 = _mm_mul_pd(self.twiddle3im, x13m18);
5806 let t_b5_14 = _mm_mul_pd(self.twiddle8im, x14m17);
5807 let t_b5_15 = _mm_mul_pd(self.twiddle13im, x15m16);
5808 let t_b6_1 = _mm_mul_pd(self.twiddle6im, x1m30);
5809 let t_b6_2 = _mm_mul_pd(self.twiddle12im, x2m29);
5810 let t_b6_3 = _mm_mul_pd(self.twiddle13im, x3m28);
5811 let t_b6_4 = _mm_mul_pd(self.twiddle7im, x4m27);
5812 let t_b6_5 = _mm_mul_pd(self.twiddle1im, x5m26);
5813 let t_b6_6 = _mm_mul_pd(self.twiddle5im, x6m25);
5814 let t_b6_7 = _mm_mul_pd(self.twiddle11im, x7m24);
5815 let t_b6_8 = _mm_mul_pd(self.twiddle14im, x8m23);
5816 let t_b6_9 = _mm_mul_pd(self.twiddle8im, x9m22);
5817 let t_b6_10 = _mm_mul_pd(self.twiddle2im, x10m21);
5818 let t_b6_11 = _mm_mul_pd(self.twiddle4im, x11m20);
5819 let t_b6_12 = _mm_mul_pd(self.twiddle10im, x12m19);
5820 let t_b6_13 = _mm_mul_pd(self.twiddle15im, x13m18);
5821 let t_b6_14 = _mm_mul_pd(self.twiddle9im, x14m17);
5822 let t_b6_15 = _mm_mul_pd(self.twiddle3im, x15m16);
5823 let t_b7_1 = _mm_mul_pd(self.twiddle7im, x1m30);
5824 let t_b7_2 = _mm_mul_pd(self.twiddle14im, x2m29);
5825 let t_b7_3 = _mm_mul_pd(self.twiddle10im, x3m28);
5826 let t_b7_4 = _mm_mul_pd(self.twiddle3im, x4m27);
5827 let t_b7_5 = _mm_mul_pd(self.twiddle4im, x5m26);
5828 let t_b7_6 = _mm_mul_pd(self.twiddle11im, x6m25);
5829 let t_b7_7 = _mm_mul_pd(self.twiddle13im, x7m24);
5830 let t_b7_8 = _mm_mul_pd(self.twiddle6im, x8m23);
5831 let t_b7_9 = _mm_mul_pd(self.twiddle1im, x9m22);
5832 let t_b7_10 = _mm_mul_pd(self.twiddle8im, x10m21);
5833 let t_b7_11 = _mm_mul_pd(self.twiddle15im, x11m20);
5834 let t_b7_12 = _mm_mul_pd(self.twiddle9im, x12m19);
5835 let t_b7_13 = _mm_mul_pd(self.twiddle2im, x13m18);
5836 let t_b7_14 = _mm_mul_pd(self.twiddle5im, x14m17);
5837 let t_b7_15 = _mm_mul_pd(self.twiddle12im, x15m16);
5838 let t_b8_1 = _mm_mul_pd(self.twiddle8im, x1m30);
5839 let t_b8_2 = _mm_mul_pd(self.twiddle15im, x2m29);
5840 let t_b8_3 = _mm_mul_pd(self.twiddle7im, x3m28);
5841 let t_b8_4 = _mm_mul_pd(self.twiddle1im, x4m27);
5842 let t_b8_5 = _mm_mul_pd(self.twiddle9im, x5m26);
5843 let t_b8_6 = _mm_mul_pd(self.twiddle14im, x6m25);
5844 let t_b8_7 = _mm_mul_pd(self.twiddle6im, x7m24);
5845 let t_b8_8 = _mm_mul_pd(self.twiddle2im, x8m23);
5846 let t_b8_9 = _mm_mul_pd(self.twiddle10im, x9m22);
5847 let t_b8_10 = _mm_mul_pd(self.twiddle13im, x10m21);
5848 let t_b8_11 = _mm_mul_pd(self.twiddle5im, x11m20);
5849 let t_b8_12 = _mm_mul_pd(self.twiddle3im, x12m19);
5850 let t_b8_13 = _mm_mul_pd(self.twiddle11im, x13m18);
5851 let t_b8_14 = _mm_mul_pd(self.twiddle12im, x14m17);
5852 let t_b8_15 = _mm_mul_pd(self.twiddle4im, x15m16);
5853 let t_b9_1 = _mm_mul_pd(self.twiddle9im, x1m30);
5854 let t_b9_2 = _mm_mul_pd(self.twiddle13im, x2m29);
5855 let t_b9_3 = _mm_mul_pd(self.twiddle4im, x3m28);
5856 let t_b9_4 = _mm_mul_pd(self.twiddle5im, x4m27);
5857 let t_b9_5 = _mm_mul_pd(self.twiddle14im, x5m26);
5858 let t_b9_6 = _mm_mul_pd(self.twiddle8im, x6m25);
5859 let t_b9_7 = _mm_mul_pd(self.twiddle1im, x7m24);
5860 let t_b9_8 = _mm_mul_pd(self.twiddle10im, x8m23);
5861 let t_b9_9 = _mm_mul_pd(self.twiddle12im, x9m22);
5862 let t_b9_10 = _mm_mul_pd(self.twiddle3im, x10m21);
5863 let t_b9_11 = _mm_mul_pd(self.twiddle6im, x11m20);
5864 let t_b9_12 = _mm_mul_pd(self.twiddle15im, x12m19);
5865 let t_b9_13 = _mm_mul_pd(self.twiddle7im, x13m18);
5866 let t_b9_14 = _mm_mul_pd(self.twiddle2im, x14m17);
5867 let t_b9_15 = _mm_mul_pd(self.twiddle11im, x15m16);
5868 let t_b10_1 = _mm_mul_pd(self.twiddle10im, x1m30);
5869 let t_b10_2 = _mm_mul_pd(self.twiddle11im, x2m29);
5870 let t_b10_3 = _mm_mul_pd(self.twiddle1im, x3m28);
5871 let t_b10_4 = _mm_mul_pd(self.twiddle9im, x4m27);
5872 let t_b10_5 = _mm_mul_pd(self.twiddle12im, x5m26);
5873 let t_b10_6 = _mm_mul_pd(self.twiddle2im, x6m25);
5874 let t_b10_7 = _mm_mul_pd(self.twiddle8im, x7m24);
5875 let t_b10_8 = _mm_mul_pd(self.twiddle13im, x8m23);
5876 let t_b10_9 = _mm_mul_pd(self.twiddle3im, x9m22);
5877 let t_b10_10 = _mm_mul_pd(self.twiddle7im, x10m21);
5878 let t_b10_11 = _mm_mul_pd(self.twiddle14im, x11m20);
5879 let t_b10_12 = _mm_mul_pd(self.twiddle4im, x12m19);
5880 let t_b10_13 = _mm_mul_pd(self.twiddle6im, x13m18);
5881 let t_b10_14 = _mm_mul_pd(self.twiddle15im, x14m17);
5882 let t_b10_15 = _mm_mul_pd(self.twiddle5im, x15m16);
5883 let t_b11_1 = _mm_mul_pd(self.twiddle11im, x1m30);
5884 let t_b11_2 = _mm_mul_pd(self.twiddle9im, x2m29);
5885 let t_b11_3 = _mm_mul_pd(self.twiddle2im, x3m28);
5886 let t_b11_4 = _mm_mul_pd(self.twiddle13im, x4m27);
5887 let t_b11_5 = _mm_mul_pd(self.twiddle7im, x5m26);
5888 let t_b11_6 = _mm_mul_pd(self.twiddle4im, x6m25);
5889 let t_b11_7 = _mm_mul_pd(self.twiddle15im, x7m24);
5890 let t_b11_8 = _mm_mul_pd(self.twiddle5im, x8m23);
5891 let t_b11_9 = _mm_mul_pd(self.twiddle6im, x9m22);
5892 let t_b11_10 = _mm_mul_pd(self.twiddle14im, x10m21);
5893 let t_b11_11 = _mm_mul_pd(self.twiddle3im, x11m20);
5894 let t_b11_12 = _mm_mul_pd(self.twiddle8im, x12m19);
5895 let t_b11_13 = _mm_mul_pd(self.twiddle12im, x13m18);
5896 let t_b11_14 = _mm_mul_pd(self.twiddle1im, x14m17);
5897 let t_b11_15 = _mm_mul_pd(self.twiddle10im, x15m16);
5898 let t_b12_1 = _mm_mul_pd(self.twiddle12im, x1m30);
5899 let t_b12_2 = _mm_mul_pd(self.twiddle7im, x2m29);
5900 let t_b12_3 = _mm_mul_pd(self.twiddle5im, x3m28);
5901 let t_b12_4 = _mm_mul_pd(self.twiddle14im, x4m27);
5902 let t_b12_5 = _mm_mul_pd(self.twiddle2im, x5m26);
5903 let t_b12_6 = _mm_mul_pd(self.twiddle10im, x6m25);
5904 let t_b12_7 = _mm_mul_pd(self.twiddle9im, x7m24);
5905 let t_b12_8 = _mm_mul_pd(self.twiddle3im, x8m23);
5906 let t_b12_9 = _mm_mul_pd(self.twiddle15im, x9m22);
5907 let t_b12_10 = _mm_mul_pd(self.twiddle4im, x10m21);
5908 let t_b12_11 = _mm_mul_pd(self.twiddle8im, x11m20);
5909 let t_b12_12 = _mm_mul_pd(self.twiddle11im, x12m19);
5910 let t_b12_13 = _mm_mul_pd(self.twiddle1im, x13m18);
5911 let t_b12_14 = _mm_mul_pd(self.twiddle13im, x14m17);
5912 let t_b12_15 = _mm_mul_pd(self.twiddle6im, x15m16);
5913 let t_b13_1 = _mm_mul_pd(self.twiddle13im, x1m30);
5914 let t_b13_2 = _mm_mul_pd(self.twiddle5im, x2m29);
5915 let t_b13_3 = _mm_mul_pd(self.twiddle8im, x3m28);
5916 let t_b13_4 = _mm_mul_pd(self.twiddle10im, x4m27);
5917 let t_b13_5 = _mm_mul_pd(self.twiddle3im, x5m26);
5918 let t_b13_6 = _mm_mul_pd(self.twiddle15im, x6m25);
5919 let t_b13_7 = _mm_mul_pd(self.twiddle2im, x7m24);
5920 let t_b13_8 = _mm_mul_pd(self.twiddle11im, x8m23);
5921 let t_b13_9 = _mm_mul_pd(self.twiddle7im, x9m22);
5922 let t_b13_10 = _mm_mul_pd(self.twiddle6im, x10m21);
5923 let t_b13_11 = _mm_mul_pd(self.twiddle12im, x11m20);
5924 let t_b13_12 = _mm_mul_pd(self.twiddle1im, x12m19);
5925 let t_b13_13 = _mm_mul_pd(self.twiddle14im, x13m18);
5926 let t_b13_14 = _mm_mul_pd(self.twiddle4im, x14m17);
5927 let t_b13_15 = _mm_mul_pd(self.twiddle9im, x15m16);
5928 let t_b14_1 = _mm_mul_pd(self.twiddle14im, x1m30);
5929 let t_b14_2 = _mm_mul_pd(self.twiddle3im, x2m29);
5930 let t_b14_3 = _mm_mul_pd(self.twiddle11im, x3m28);
5931 let t_b14_4 = _mm_mul_pd(self.twiddle6im, x4m27);
5932 let t_b14_5 = _mm_mul_pd(self.twiddle8im, x5m26);
5933 let t_b14_6 = _mm_mul_pd(self.twiddle9im, x6m25);
5934 let t_b14_7 = _mm_mul_pd(self.twiddle5im, x7m24);
5935 let t_b14_8 = _mm_mul_pd(self.twiddle12im, x8m23);
5936 let t_b14_9 = _mm_mul_pd(self.twiddle2im, x9m22);
5937 let t_b14_10 = _mm_mul_pd(self.twiddle15im, x10m21);
5938 let t_b14_11 = _mm_mul_pd(self.twiddle1im, x11m20);
5939 let t_b14_12 = _mm_mul_pd(self.twiddle13im, x12m19);
5940 let t_b14_13 = _mm_mul_pd(self.twiddle4im, x13m18);
5941 let t_b14_14 = _mm_mul_pd(self.twiddle10im, x14m17);
5942 let t_b14_15 = _mm_mul_pd(self.twiddle7im, x15m16);
5943 let t_b15_1 = _mm_mul_pd(self.twiddle15im, x1m30);
5944 let t_b15_2 = _mm_mul_pd(self.twiddle1im, x2m29);
5945 let t_b15_3 = _mm_mul_pd(self.twiddle14im, x3m28);
5946 let t_b15_4 = _mm_mul_pd(self.twiddle2im, x4m27);
5947 let t_b15_5 = _mm_mul_pd(self.twiddle13im, x5m26);
5948 let t_b15_6 = _mm_mul_pd(self.twiddle3im, x6m25);
5949 let t_b15_7 = _mm_mul_pd(self.twiddle12im, x7m24);
5950 let t_b15_8 = _mm_mul_pd(self.twiddle4im, x8m23);
5951 let t_b15_9 = _mm_mul_pd(self.twiddle11im, x9m22);
5952 let t_b15_10 = _mm_mul_pd(self.twiddle5im, x10m21);
5953 let t_b15_11 = _mm_mul_pd(self.twiddle10im, x11m20);
5954 let t_b15_12 = _mm_mul_pd(self.twiddle6im, x12m19);
5955 let t_b15_13 = _mm_mul_pd(self.twiddle9im, x13m18);
5956 let t_b15_14 = _mm_mul_pd(self.twiddle7im, x14m17);
5957 let t_b15_15 = _mm_mul_pd(self.twiddle8im, x15m16);
5958
5959 let x0 = values[0];
5960 let t_a1 = calc_f64!(x0 + t_a1_1 + t_a1_2 + t_a1_3 + t_a1_4 + t_a1_5 + t_a1_6 + t_a1_7 + t_a1_8 + t_a1_9 + t_a1_10 + t_a1_11 + t_a1_12 + t_a1_13 + t_a1_14 + t_a1_15);
5961 let t_a2 = calc_f64!(x0 + t_a2_1 + t_a2_2 + t_a2_3 + t_a2_4 + t_a2_5 + t_a2_6 + t_a2_7 + t_a2_8 + t_a2_9 + t_a2_10 + t_a2_11 + t_a2_12 + t_a2_13 + t_a2_14 + t_a2_15);
5962 let t_a3 = calc_f64!(x0 + t_a3_1 + t_a3_2 + t_a3_3 + t_a3_4 + t_a3_5 + t_a3_6 + t_a3_7 + t_a3_8 + t_a3_9 + t_a3_10 + t_a3_11 + t_a3_12 + t_a3_13 + t_a3_14 + t_a3_15);
5963 let t_a4 = calc_f64!(x0 + t_a4_1 + t_a4_2 + t_a4_3 + t_a4_4 + t_a4_5 + t_a4_6 + t_a4_7 + t_a4_8 + t_a4_9 + t_a4_10 + t_a4_11 + t_a4_12 + t_a4_13 + t_a4_14 + t_a4_15);
5964 let t_a5 = calc_f64!(x0 + t_a5_1 + t_a5_2 + t_a5_3 + t_a5_4 + t_a5_5 + t_a5_6 + t_a5_7 + t_a5_8 + t_a5_9 + t_a5_10 + t_a5_11 + t_a5_12 + t_a5_13 + t_a5_14 + t_a5_15);
5965 let t_a6 = calc_f64!(x0 + t_a6_1 + t_a6_2 + t_a6_3 + t_a6_4 + t_a6_5 + t_a6_6 + t_a6_7 + t_a6_8 + t_a6_9 + t_a6_10 + t_a6_11 + t_a6_12 + t_a6_13 + t_a6_14 + t_a6_15);
5966 let t_a7 = calc_f64!(x0 + t_a7_1 + t_a7_2 + t_a7_3 + t_a7_4 + t_a7_5 + t_a7_6 + t_a7_7 + t_a7_8 + t_a7_9 + t_a7_10 + t_a7_11 + t_a7_12 + t_a7_13 + t_a7_14 + t_a7_15);
5967 let t_a8 = calc_f64!(x0 + t_a8_1 + t_a8_2 + t_a8_3 + t_a8_4 + t_a8_5 + t_a8_6 + t_a8_7 + t_a8_8 + t_a8_9 + t_a8_10 + t_a8_11 + t_a8_12 + t_a8_13 + t_a8_14 + t_a8_15);
5968 let t_a9 = calc_f64!(x0 + t_a9_1 + t_a9_2 + t_a9_3 + t_a9_4 + t_a9_5 + t_a9_6 + t_a9_7 + t_a9_8 + t_a9_9 + t_a9_10 + t_a9_11 + t_a9_12 + t_a9_13 + t_a9_14 + t_a9_15);
5969 let t_a10 = calc_f64!(x0 + t_a10_1 + t_a10_2 + t_a10_3 + t_a10_4 + t_a10_5 + t_a10_6 + t_a10_7 + t_a10_8 + t_a10_9 + t_a10_10 + t_a10_11 + t_a10_12 + t_a10_13 + t_a10_14 + t_a10_15);
5970 let t_a11 = calc_f64!(x0 + t_a11_1 + t_a11_2 + t_a11_3 + t_a11_4 + t_a11_5 + t_a11_6 + t_a11_7 + t_a11_8 + t_a11_9 + t_a11_10 + t_a11_11 + t_a11_12 + t_a11_13 + t_a11_14 + t_a11_15);
5971 let t_a12 = calc_f64!(x0 + t_a12_1 + t_a12_2 + t_a12_3 + t_a12_4 + t_a12_5 + t_a12_6 + t_a12_7 + t_a12_8 + t_a12_9 + t_a12_10 + t_a12_11 + t_a12_12 + t_a12_13 + t_a12_14 + t_a12_15);
5972 let t_a13 = calc_f64!(x0 + t_a13_1 + t_a13_2 + t_a13_3 + t_a13_4 + t_a13_5 + t_a13_6 + t_a13_7 + t_a13_8 + t_a13_9 + t_a13_10 + t_a13_11 + t_a13_12 + t_a13_13 + t_a13_14 + t_a13_15);
5973 let t_a14 = calc_f64!(x0 + t_a14_1 + t_a14_2 + t_a14_3 + t_a14_4 + t_a14_5 + t_a14_6 + t_a14_7 + t_a14_8 + t_a14_9 + t_a14_10 + t_a14_11 + t_a14_12 + t_a14_13 + t_a14_14 + t_a14_15);
5974 let t_a15 = calc_f64!(x0 + t_a15_1 + t_a15_2 + t_a15_3 + t_a15_4 + t_a15_5 + t_a15_6 + t_a15_7 + t_a15_8 + t_a15_9 + t_a15_10 + t_a15_11 + t_a15_12 + t_a15_13 + t_a15_14 + t_a15_15);
5975
5976 let t_b1 = calc_f64!(t_b1_1 + t_b1_2 + t_b1_3 + t_b1_4 + t_b1_5 + t_b1_6 + t_b1_7 + t_b1_8 + t_b1_9 + t_b1_10 + t_b1_11 + t_b1_12 + t_b1_13 + t_b1_14 + t_b1_15);
5977 let t_b2 = calc_f64!(t_b2_1 + t_b2_2 + t_b2_3 + t_b2_4 + t_b2_5 + t_b2_6 + t_b2_7 - t_b2_8 - t_b2_9 - t_b2_10 - t_b2_11 - t_b2_12 - t_b2_13 - t_b2_14 - t_b2_15);
5978 let t_b3 = calc_f64!(t_b3_1 + t_b3_2 + t_b3_3 + t_b3_4 + t_b3_5 - t_b3_6 - t_b3_7 - t_b3_8 - t_b3_9 - t_b3_10 + t_b3_11 + t_b3_12 + t_b3_13 + t_b3_14 + t_b3_15);
5979 let t_b4 = calc_f64!(t_b4_1 + t_b4_2 + t_b4_3 - t_b4_4 - t_b4_5 - t_b4_6 - t_b4_7 + t_b4_8 + t_b4_9 + t_b4_10 + t_b4_11 - t_b4_12 - t_b4_13 - t_b4_14 - t_b4_15);
5980 let t_b5 = calc_f64!(t_b5_1 + t_b5_2 + t_b5_3 - t_b5_4 - t_b5_5 - t_b5_6 + t_b5_7 + t_b5_8 + t_b5_9 - t_b5_10 - t_b5_11 - t_b5_12 + t_b5_13 + t_b5_14 + t_b5_15);
5981 let t_b6 = calc_f64!(t_b6_1 + t_b6_2 - t_b6_3 - t_b6_4 - t_b6_5 + t_b6_6 + t_b6_7 - t_b6_8 - t_b6_9 - t_b6_10 + t_b6_11 + t_b6_12 - t_b6_13 - t_b6_14 - t_b6_15);
5982 let t_b7 = calc_f64!(t_b7_1 + t_b7_2 - t_b7_3 - t_b7_4 + t_b7_5 + t_b7_6 - t_b7_7 - t_b7_8 + t_b7_9 + t_b7_10 + t_b7_11 - t_b7_12 - t_b7_13 + t_b7_14 + t_b7_15);
5983 let t_b8 = calc_f64!(t_b8_1 - t_b8_2 - t_b8_3 + t_b8_4 + t_b8_5 - t_b8_6 - t_b8_7 + t_b8_8 + t_b8_9 - t_b8_10 - t_b8_11 + t_b8_12 + t_b8_13 - t_b8_14 - t_b8_15);
5984 let t_b9 = calc_f64!(t_b9_1 - t_b9_2 - t_b9_3 + t_b9_4 + t_b9_5 - t_b9_6 + t_b9_7 + t_b9_8 - t_b9_9 - t_b9_10 + t_b9_11 + t_b9_12 - t_b9_13 + t_b9_14 + t_b9_15);
5985 let t_b10 = calc_f64!(t_b10_1 - t_b10_2 - t_b10_3 + t_b10_4 - t_b10_5 - t_b10_6 + t_b10_7 - t_b10_8 - t_b10_9 + t_b10_10 - t_b10_11 - t_b10_12 + t_b10_13 - t_b10_14 - t_b10_15);
5986 let t_b11 = calc_f64!(t_b11_1 - t_b11_2 + t_b11_3 + t_b11_4 - t_b11_5 + t_b11_6 + t_b11_7 - t_b11_8 + t_b11_9 - t_b11_10 - t_b11_11 + t_b11_12 - t_b11_13 - t_b11_14 + t_b11_15);
5987 let t_b12 = calc_f64!(t_b12_1 - t_b12_2 + t_b12_3 - t_b12_4 - t_b12_5 + t_b12_6 - t_b12_7 + t_b12_8 + t_b12_9 - t_b12_10 + t_b12_11 - t_b12_12 + t_b12_13 + t_b12_14 - t_b12_15);
5988 let t_b13 = calc_f64!(t_b13_1 - t_b13_2 + t_b13_3 - t_b13_4 + t_b13_5 - t_b13_6 - t_b13_7 + t_b13_8 - t_b13_9 + t_b13_10 - t_b13_11 + t_b13_12 + t_b13_13 - t_b13_14 + t_b13_15);
5989 let t_b14 = calc_f64!(t_b14_1 - t_b14_2 + t_b14_3 - t_b14_4 + t_b14_5 - t_b14_6 + t_b14_7 - t_b14_8 + t_b14_9 - t_b14_10 - t_b14_11 + t_b14_12 - t_b14_13 + t_b14_14 - t_b14_15);
5990 let t_b15 = calc_f64!(t_b15_1 - t_b15_2 + t_b15_3 - t_b15_4 + t_b15_5 - t_b15_6 + t_b15_7 - t_b15_8 + t_b15_9 - t_b15_10 + t_b15_11 - t_b15_12 + t_b15_13 - t_b15_14 + t_b15_15);
5991
5992 let t_b1_rot = self.rotate.rotate(t_b1);
5993 let t_b2_rot = self.rotate.rotate(t_b2);
5994 let t_b3_rot = self.rotate.rotate(t_b3);
5995 let t_b4_rot = self.rotate.rotate(t_b4);
5996 let t_b5_rot = self.rotate.rotate(t_b5);
5997 let t_b6_rot = self.rotate.rotate(t_b6);
5998 let t_b7_rot = self.rotate.rotate(t_b7);
5999 let t_b8_rot = self.rotate.rotate(t_b8);
6000 let t_b9_rot = self.rotate.rotate(t_b9);
6001 let t_b10_rot = self.rotate.rotate(t_b10);
6002 let t_b11_rot = self.rotate.rotate(t_b11);
6003 let t_b12_rot = self.rotate.rotate(t_b12);
6004 let t_b13_rot = self.rotate.rotate(t_b13);
6005 let t_b14_rot = self.rotate.rotate(t_b14);
6006 let t_b15_rot = self.rotate.rotate(t_b15);
6007
6008 let y0 = calc_f64!(x0 + x1p30 + x2p29 + x3p28 + x4p27 + x5p26 + x6p25 + x7p24 + x8p23 + x9p22 + x10p21 + x11p20 + x12p19 + x13p18 + x14p17 + x15p16);
6009 let [y1, y30] = solo_fft2_f64(t_a1, t_b1_rot);
6010 let [y2, y29] = solo_fft2_f64(t_a2, t_b2_rot);
6011 let [y3, y28] = solo_fft2_f64(t_a3, t_b3_rot);
6012 let [y4, y27] = solo_fft2_f64(t_a4, t_b4_rot);
6013 let [y5, y26] = solo_fft2_f64(t_a5, t_b5_rot);
6014 let [y6, y25] = solo_fft2_f64(t_a6, t_b6_rot);
6015 let [y7, y24] = solo_fft2_f64(t_a7, t_b7_rot);
6016 let [y8, y23] = solo_fft2_f64(t_a8, t_b8_rot);
6017 let [y9, y22] = solo_fft2_f64(t_a9, t_b9_rot);
6018 let [y10, y21] = solo_fft2_f64(t_a10, t_b10_rot);
6019 let [y11, y20] = solo_fft2_f64(t_a11, t_b11_rot);
6020 let [y12, y19] = solo_fft2_f64(t_a12, t_b12_rot);
6021 let [y13, y18] = solo_fft2_f64(t_a13, t_b13_rot);
6022 let [y14, y17] = solo_fft2_f64(t_a14, t_b14_rot);
6023 let [y15, y16] = solo_fft2_f64(t_a15, t_b15_rot);
6024 [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15, y16, y17, y18, y19, y20, y21, y22, y23, y24, y25, y26, y27, y28, y29, y30]
6025 }
6026 }
6027
6028 // _____ _____ ____ _____ ____
6029 // |_ _| ____/ ___|_ _/ ___|
6030 // | | | _| \___ \ | | \___ \
6031 // | | | |___ ___) || | ___) |
6032 // |_| |_____|____/ |_| |____/
6033 //
6034
6035 #[cfg(test)]
6036 mod unit_tests {
6037 use super::*;
6038 use crate::test_utils::check_fft_algorithm;
6039
6040 //the tests for all butterflies will be identical except for the identifiers used and size
6041 //so it's ideal for a macro
6042 macro_rules! test_butterfly_32_func {
6043 ($test_name:ident, $struct_name:ident, $size:expr) => {
6044 #[test]
6045 fn $test_name() {
6046 let butterfly = $struct_name::new(FftDirection::Forward);
6047 check_fft_algorithm::<f32>(&butterfly, $size, FftDirection::Forward);
6048
6049 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
6050 check_fft_algorithm::<f32>(&butterfly_direction, $size, FftDirection::Inverse);
6051 }
6052 };
6053 }
6054 test_butterfly_32_func!(test_ssef32_butterfly7, SseF32Butterfly7, 7);
6055 test_butterfly_32_func!(test_ssef32_butterfly11, SseF32Butterfly11, 11);
6056 test_butterfly_32_func!(test_ssef32_butterfly13, SseF32Butterfly13, 13);
6057 test_butterfly_32_func!(test_ssef32_butterfly17, SseF32Butterfly17, 17);
6058 test_butterfly_32_func!(test_ssef32_butterfly19, SseF32Butterfly19, 19);
6059 test_butterfly_32_func!(test_ssef32_butterfly23, SseF32Butterfly23, 23);
6060 test_butterfly_32_func!(test_ssef32_butterfly29, SseF32Butterfly29, 29);
6061 test_butterfly_32_func!(test_ssef32_butterfly31, SseF32Butterfly31, 31);
6062
6063 //the tests for all butterflies will be identical except for the identifiers used and size
6064 //so it's ideal for a macro
6065 macro_rules! test_butterfly_64_func {
6066 ($test_name:ident, $struct_name:ident, $size:expr) => {
6067 #[test]
6068 fn $test_name() {
6069 let butterfly = $struct_name::new(FftDirection::Forward);
6070 check_fft_algorithm::<f64>(&butterfly, $size, FftDirection::Forward);
6071
6072 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
6073 check_fft_algorithm::<f64>(&butterfly_direction, $size, FftDirection::Inverse);
6074 }
6075 };
6076 }
6077 test_butterfly_64_func!(test_ssef64_butterfly7, SseF64Butterfly7, 7);
6078 test_butterfly_64_func!(test_ssef64_butterfly11, SseF64Butterfly11, 11);
6079 test_butterfly_64_func!(test_ssef64_butterfly13, SseF64Butterfly13, 13);
6080 test_butterfly_64_func!(test_ssef64_butterfly17, SseF64Butterfly17, 17);
6081 test_butterfly_64_func!(test_ssef64_butterfly19, SseF64Butterfly19, 19);
6082 test_butterfly_64_func!(test_ssef64_butterfly23, SseF64Butterfly23, 23);
6083 test_butterfly_64_func!(test_ssef64_butterfly29, SseF64Butterfly29, 29);
6084 test_butterfly_64_func!(test_ssef64_butterfly31, SseF64Butterfly31, 31);
6085 }