Debian Codegraph @

.cargo/vendor/rustfft/src/algorithm/butterflies.rs

raw
1 use num_complex::Complex;
2
3 use crate::{common::FftNum, FftDirection};
4
5 use crate::array_utils::{self, DoubleBuf, LoadStore};
6 use crate::common::{fft_error_inplace, fft_error_outofplace};
7 use crate::twiddles;
8 use crate::{Direction, Fft, Length};
9
10 #[allow(unused)]
11 macro_rules! boilerplate_fft_butterfly {
12 ($struct_name:ident, $len:expr, $direction_fn:expr) => {
13 impl<T: FftNum> $struct_name<T> {
14 #[inline(always)]
15 pub(crate) unsafe fn perform_fft_butterfly(&self, buffer: impl LoadStore<T>) {
16 self.perform_fft_contiguous(buffer);
17 }
18 }
19 impl<T: FftNum> Fft<T> for $struct_name<T> {
20 fn process_outofplace_with_scratch(
21 &self,
22 input: &mut [Complex<T>],
23 output: &mut [Complex<T>],
24 _scratch: &mut [Complex<T>],
25 ) {
26 if input.len() < self.len() || output.len() != input.len() {
27 // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
28 fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
29 return; // Unreachable, because fft_error_outofplace asserts, but it helps codegen to put it here
30 }
31
32 let result = array_utils::iter_chunks_zipped(
33 input,
34 output,
35 self.len(),
36 |in_chunk, out_chunk| {
37 unsafe {
38 self.perform_fft_butterfly(DoubleBuf {
39 input: in_chunk,
40 output: out_chunk,
41 })
42 };
43 },
44 );
45
46 if result.is_err() {
47 // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
48 // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
49 fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
50 }
51 }
52 fn process_with_scratch(&self, buffer: &mut [Complex<T>], _scratch: &mut [Complex<T>]) {
53 if buffer.len() < self.len() {
54 // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
55 fft_error_inplace(self.len(), buffer.len(), 0, 0);
56 return; // Unreachable, because fft_error_inplace asserts, but it helps codegen to put it here
57 }
58
59 let result = array_utils::iter_chunks(buffer, self.len(), |chunk| unsafe {
60 self.perform_fft_butterfly(chunk)
61 });
62
63 if result.is_err() {
64 // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
65 // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
66 fft_error_inplace(self.len(), buffer.len(), 0, 0);
67 }
68 }
69 #[inline(always)]
70 fn get_inplace_scratch_len(&self) -> usize {
71 0
72 }
73 #[inline(always)]
74 fn get_outofplace_scratch_len(&self) -> usize {
75 0
76 }
77 }
78 impl<T> Length for $struct_name<T> {
79 #[inline(always)]
80 fn len(&self) -> usize {
81 $len
82 }
83 }
84 impl<T> Direction for $struct_name<T> {
85 #[inline(always)]
86 fn fft_direction(&self) -> FftDirection {
87 $direction_fn(self)
88 }
89 }
90 };
91 }
92
93 pub struct Butterfly1<T> {
94 direction: FftDirection,
95 _phantom: std::marker::PhantomData<T>,
96 }
97 impl<T: FftNum> Butterfly1<T> {
98 #[inline(always)]
99 pub fn new(direction: FftDirection) -> Self {
100 Self {
101 direction,
102 _phantom: std::marker::PhantomData,
103 }
104 }
105 }
106 impl<T: FftNum> Fft<T> for Butterfly1<T> {
107 fn process_outofplace_with_scratch(
108 &self,
109 input: &mut [Complex<T>],
110 output: &mut [Complex<T>],
111 _scratch: &mut [Complex<T>],
112 ) {
113 output.copy_from_slice(&input);
114 }
115
116 fn process_with_scratch(&self, _buffer: &mut [Complex<T>], _scratch: &mut [Complex<T>]) {}
117
118 fn get_inplace_scratch_len(&self) -> usize {
119 0
120 }
121
122 fn get_outofplace_scratch_len(&self) -> usize {
123 0
124 }
125 }
126 impl<T> Length for Butterfly1<T> {
127 fn len(&self) -> usize {
128 1
129 }
130 }
131 impl<T> Direction for Butterfly1<T> {
132 fn fft_direction(&self) -> FftDirection {
133 self.direction
134 }
135 }
136
137 pub struct Butterfly2<T> {
138 direction: FftDirection,
139 _phantom: std::marker::PhantomData<T>,
140 }
141 boilerplate_fft_butterfly!(Butterfly2, 2, |this: &Butterfly2<_>| this.direction);
142 impl<T: FftNum> Butterfly2<T> {
143 #[inline(always)]
144 pub fn new(direction: FftDirection) -> Self {
145 Self {
146 direction,
147 _phantom: std::marker::PhantomData,
148 }
149 }
150 #[inline(always)]
151 unsafe fn perform_fft_strided(left: &mut Complex<T>, right: &mut Complex<T>) {
152 let temp = *left + *right;
153
154 *right = *left - *right;
155 *left = temp;
156 }
157 #[inline(always)]
158 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
159 let value0 = buffer.load(0);
160 let value1 = buffer.load(1);
161 buffer.store(value0 + value1, 0);
162 buffer.store(value0 - value1, 1);
163 }
164 }
165
166 pub struct Butterfly3<T> {
167 pub twiddle: Complex<T>,
168 direction: FftDirection,
169 }
170 boilerplate_fft_butterfly!(Butterfly3, 3, |this: &Butterfly3<_>| this.direction);
171 impl<T: FftNum> Butterfly3<T> {
172 #[inline(always)]
173 pub fn new(direction: FftDirection) -> Self {
174 Self {
175 twiddle: twiddles::compute_twiddle(1, 3, direction),
176 direction,
177 }
178 }
179 #[inline(always)]
180 pub fn direction_of(fft: &Butterfly3<T>) -> Self {
181 Self {
182 twiddle: fft.twiddle.conj(),
183 direction: fft.direction.opposite_direction(),
184 }
185 }
186 #[inline(always)]
187 unsafe fn perform_fft_strided(
188 &self,
189 val0: &mut Complex<T>,
190 val1: &mut Complex<T>,
191 val2: &mut Complex<T>,
192 ) {
193 let xp = *val1 + *val2;
194 let xn = *val1 - *val2;
195 let sum = *val0 + xp;
196
197 let temp_a = *val0
198 + Complex {
199 re: self.twiddle.re * xp.re,
200 im: self.twiddle.re * xp.im,
201 };
202 let temp_b = Complex {
203 re: -self.twiddle.im * xn.im,
204 im: self.twiddle.im * xn.re,
205 };
206
207 *val0 = sum;
208 *val1 = temp_a + temp_b;
209 *val2 = temp_a - temp_b;
210 }
211
212 #[inline(always)]
213 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
214 let xp = buffer.load(1) + buffer.load(2);
215 let xn = buffer.load(1) - buffer.load(2);
216 let sum = buffer.load(0) + xp;
217
218 let temp_a = buffer.load(0)
219 + Complex {
220 re: self.twiddle.re * xp.re,
221 im: self.twiddle.re * xp.im,
222 };
223 let temp_b = Complex {
224 re: -self.twiddle.im * xn.im,
225 im: self.twiddle.im * xn.re,
226 };
227
228 buffer.store(sum, 0);
229 buffer.store(temp_a + temp_b, 1);
230 buffer.store(temp_a - temp_b, 2);
231 }
232 }
233
234 pub struct Butterfly4<T> {
235 direction: FftDirection,
236 _phantom: std::marker::PhantomData<T>,
237 }
238 boilerplate_fft_butterfly!(Butterfly4, 4, |this: &Butterfly4<_>| this.direction);
239 impl<T: FftNum> Butterfly4<T> {
240 #[inline(always)]
241 pub fn new(direction: FftDirection) -> Self {
242 Self {
243 direction,
244 _phantom: std::marker::PhantomData,
245 }
246 }
247 #[inline(always)]
248 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
249 //we're going to hardcode a step of mixed radix
250 //aka we're going to do the six step algorithm
251
252 // step 1: transpose, which we're skipping because we're just going to perform non-contiguous FFTs
253 let mut value0 = buffer.load(0);
254 let mut value1 = buffer.load(1);
255 let mut value2 = buffer.load(2);
256 let mut value3 = buffer.load(3);
257
258 // step 2: column FFTs
259 Butterfly2::perform_fft_strided(&mut value0, &mut value2);
260 Butterfly2::perform_fft_strided(&mut value1, &mut value3);
261
262 // step 3: apply twiddle factors (only one in this case, and it's either 0 + i or 0 - i)
263 value3 = twiddles::rotate_90(value3, self.direction);
264
265 // step 4: transpose, which we're skipping because we're the previous FFTs were non-contiguous
266
267 // step 5: row FFTs
268 Butterfly2::perform_fft_strided(&mut value0, &mut value1);
269 Butterfly2::perform_fft_strided(&mut value2, &mut value3);
270
271 // step 6: transpose by swapping index 1 and 2
272 buffer.store(value0, 0);
273 buffer.store(value2, 1);
274 buffer.store(value1, 2);
275 buffer.store(value3, 3);
276 }
277 }
278
279 pub struct Butterfly5<T> {
280 twiddle1: Complex<T>,
281 twiddle2: Complex<T>,
282 direction: FftDirection,
283 }
284 boilerplate_fft_butterfly!(Butterfly5, 5, |this: &Butterfly5<_>| this.direction);
285 impl<T: FftNum> Butterfly5<T> {
286 pub fn new(direction: FftDirection) -> Self {
287 Self {
288 twiddle1: twiddles::compute_twiddle(1, 5, direction),
289 twiddle2: twiddles::compute_twiddle(2, 5, direction),
290 direction,
291 }
292 }
293
294 #[inline(never)] // refusing to inline this code reduces code size, and doesn't hurt performance
295 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
296 // let mut outer = Butterfly2::perform_fft_array([buffer.load(1), buffer.load(4)]);
297 // let mut inner = Butterfly2::perform_fft_array([buffer.load(2), buffer.load(3)]);
298 // let input0 = buffer.load(0);
299
300 // buffer.store(input0 + outer[0] + inner[0], 0);
301
302 // inner[1] = twiddles::rotate_90(inner[1], true);
303 // outer[1] = twiddles::rotate_90(outer[1], true);
304
305 // {
306 // let twiddled1 = outer[0] * self.twiddles[0].re;
307 // let twiddled2 = inner[0] * self.twiddles[1].re;
308 // let twiddled3 = inner[1] * self.twiddles[1].im;
309 // let twiddled4 = outer[1] * self.twiddles[0].im;
310
311 // let sum12 = twiddled1 + twiddled2;
312 // let sum34 = twiddled4 + twiddled3;
313
314 // let output1 = sum12 + sum34;
315 // let output4 = sum12 - sum34;
316
317 // buffer.store(input0 + output1, 1);
318 // buffer.store(input0 + output4, 4);
319 // }
320
321 // {
322 // let twiddled1 = outer[0] * self.twiddles[1].re;
323 // let twiddled2 = inner[0] * self.twiddles[0].re;
324 // let twiddled3 = inner[1] * self.twiddles[0].im;
325 // let twiddled4 = outer[1] * self.twiddles[1].im;
326 // }
327
328 // Let's do a plain 5-point Dft
329 // |X0| | W0 W0 W0 W0 W0 | |x0|
330 // |X1| | W0 W1 W2 W3 W4 | |x1|
331 // |X2| = | W0 W2 W4 W6 W8 | * |x2|
332 // |X3| | W0 W3 W6 W9 W12 | |x3|
333 // |X4| | W0 W4 W8 W12 W16 | |x4|
334 //
335 // where Wn = exp(-2*pi*n/5) for a forward transform, and exp(+2*pi*n/5) for an direction.
336 //
337 // This can be simplified a bit since exp(-2*pi*n/5) = exp(-2*pi*n/5 + m*2*pi)
338 // |X0| | W0 W0 W0 W0 W0 | |x0|
339 // |X1| | W0 W1 W2 W3 W4 | |x1|
340 // |X2| = | W0 W2 W4 W1 W3 | * |x2|
341 // |X3| | W0 W3 W1 W4 W2 | |x3|
342 // |X4| | W0 W4 W3 W2 W1 | |x4|
343 //
344 // Next we can use the symmetry that W3 = W2* and W4 = W1* (where * means complex conjugate), and W0 = 1
345 // |X0| | 1 1 1 1 1 | |x0|
346 // |X1| | 1 W1 W2 W2* W1* | |x1|
347 // |X2| = | 1 W2 W1* W1 W2* | * |x2|
348 // |X3| | 1 W2* W1 W1* W2 | |x3|
349 // |X4| | 1 W1* W2* W2 W1 | |x4|
350 //
351 // Next, we write out the whole expression with real and imaginary parts.
352 // X0 = x0 + x1 + x2 + x3 + x4
353 // X1 = x0 + (W1.re + j*W1.im)*x1 + (W2.re + j*W2.im)*x2 + (W2.re - j*W2.im)*x3 + (W1.re - j*W1.im)*x4
354 // X2 = x0 + (W2.re + j*W2.im)*x1 + (W1.re - j*W1.im)*x2 + (W1.re + j*W1.im)*x3 + (W2.re - j*W2.im)*x4
355 // X3 = x0 + (W2.re - j*W2.im)*x1 + (W1.re + j*W1.im)*x2 + (W1.re - j*W1.im)*x3 + (W2.re + j*W2.im)*x4
356 // X4 = x0 + (W1.re - j*W1.im)*x1 + (W2.re - j*W2.im)*x2 + (W2.re + j*W2.im)*x3 + (W1.re + j*W1.im)*x4
357 //
358 // Then we rearrange and sort terms.
359 // X0 = x0 + x1 + x2 + x3 + x4
360 // X1 = x0 + W1.re*(x1+x4) + W2.re*(x2+x3) + j*(W1.im*(x1-x4) + W2.im*(x2-x3))
361 // X2 = x0 + W1.re*(x2+x3) + W2.re*(x1+x4) - j*(W1.im*(x2-x3) - W2.im*(x1-x4))
362 // X3 = x0 + W1.re*(x2+x3) + W2.re*(x1+x4) + j*(W1.im*(x2-x3) - W2.im*(x1-x4))
363 // X4 = x0 + W1.re*(x1+x4) + W2.re*(x2+x3) - j*(W1.im*(x1-x4) + W2.im*(x2-x3))
364 //
365 // Now we define x14p=x1+x4 x14n=x1-x4, x23p=x2+x3, x23n=x2-x3
366 // X0 = x0 + x1 + x2 + x3 + x4
367 // X1 = x0 + W1.re*(x14p) + W2.re*(x23p) + j*(W1.im*(x14n) + W2.im*(x23n))
368 // X2 = x0 + W1.re*(x23p) + W2.re*(x14p) - j*(W1.im*(x23n) - W2.im*(x14n))
369 // X3 = x0 + W1.re*(x23p) + W2.re*(x14p) + j*(W1.im*(x23n) - W2.im*(x14n))
370 // X4 = x0 + W1.re*(x14p) + W2.re*(x23p) - j*(W1.im*(x14n) + W2.im*(x23n))
371 //
372 // The final step is to write out real and imaginary parts of x14n etc, and replacing using j*j=-1
373 // After this it's easy to remove any repeated calculation of the same values.
374
375 let x14p = buffer.load(1) + buffer.load(4);
376 let x14n = buffer.load(1) - buffer.load(4);
377 let x23p = buffer.load(2) + buffer.load(3);
378 let x23n = buffer.load(2) - buffer.load(3);
379 let sum = buffer.load(0) + x14p + x23p;
380 let b14re_a = buffer.load(0).re + self.twiddle1.re * x14p.re + self.twiddle2.re * x23p.re;
381 let b14re_b = self.twiddle1.im * x14n.im + self.twiddle2.im * x23n.im;
382 let b23re_a = buffer.load(0).re + self.twiddle2.re * x14p.re + self.twiddle1.re * x23p.re;
383 let b23re_b = self.twiddle2.im * x14n.im + -self.twiddle1.im * x23n.im;
384
385 let b14im_a = buffer.load(0).im + self.twiddle1.re * x14p.im + self.twiddle2.re * x23p.im;
386 let b14im_b = self.twiddle1.im * x14n.re + self.twiddle2.im * x23n.re;
387 let b23im_a = buffer.load(0).im + self.twiddle2.re * x14p.im + self.twiddle1.re * x23p.im;
388 let b23im_b = self.twiddle2.im * x14n.re + -self.twiddle1.im * x23n.re;
389
390 let out1re = b14re_a - b14re_b;
391 let out1im = b14im_a + b14im_b;
392 let out2re = b23re_a - b23re_b;
393 let out2im = b23im_a + b23im_b;
394 let out3re = b23re_a + b23re_b;
395 let out3im = b23im_a - b23im_b;
396 let out4re = b14re_a + b14re_b;
397 let out4im = b14im_a - b14im_b;
398 buffer.store(sum, 0);
399 buffer.store(
400 Complex {
401 re: out1re,
402 im: out1im,
403 },
404 1,
405 );
406 buffer.store(
407 Complex {
408 re: out2re,
409 im: out2im,
410 },
411 2,
412 );
413 buffer.store(
414 Complex {
415 re: out3re,
416 im: out3im,
417 },
418 3,
419 );
420 buffer.store(
421 Complex {
422 re: out4re,
423 im: out4im,
424 },
425 4,
426 );
427 }
428 }
429
430 pub struct Butterfly6<T> {
431 butterfly3: Butterfly3<T>,
432 }
433 boilerplate_fft_butterfly!(Butterfly6, 6, |this: &Butterfly6<_>| this
434 .butterfly3
435 .fft_direction());
436 impl<T: FftNum> Butterfly6<T> {
437 #[inline(always)]
438 pub fn new(direction: FftDirection) -> Self {
439 Self {
440 butterfly3: Butterfly3::new(direction),
441 }
442 }
443 #[inline(always)]
444 pub fn direction_of(fft: &Butterfly6<T>) -> Self {
445 Self {
446 butterfly3: Butterfly3::direction_of(&fft.butterfly3),
447 }
448 }
449 #[inline(always)]
450 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
451 //since GCD(2,3) == 1 we're going to hardcode a step of the Good-Thomas algorithm to avoid twiddle factors
452
453 // step 1: reorder the input directly into the scratch. normally there's a whole thing to compute this ordering
454 //but thankfully we can just precompute it and hardcode it
455 let mut scratch_a = [buffer.load(0), buffer.load(2), buffer.load(4)];
456
457 let mut scratch_b = [buffer.load(3), buffer.load(5), buffer.load(1)];
458
459 // step 2: column FFTs
460 self.butterfly3.perform_fft_contiguous(&mut scratch_a);
461 self.butterfly3.perform_fft_contiguous(&mut scratch_b);
462
463 // step 3: apply twiddle factors -- SKIPPED because good-thomas doesn't have twiddle factors :)
464
465 // step 4: SKIPPED because the next FFTs will be non-contiguous
466
467 // step 5: row FFTs
468 Butterfly2::perform_fft_strided(&mut scratch_a[0], &mut scratch_b[0]);
469 Butterfly2::perform_fft_strided(&mut scratch_a[1], &mut scratch_b[1]);
470 Butterfly2::perform_fft_strided(&mut scratch_a[2], &mut scratch_b[2]);
471
472 // step 6: reorder the result back into the buffer. again we would normally have to do an expensive computation
473 // but instead we can precompute and hardcode the ordering
474 // note that we're also rolling a transpose step into this reorder
475 buffer.store(scratch_a[0], 0);
476 buffer.store(scratch_b[1], 1);
477 buffer.store(scratch_a[2], 2);
478 buffer.store(scratch_b[0], 3);
479 buffer.store(scratch_a[1], 4);
480 buffer.store(scratch_b[2], 5);
481 }
482 }
483
484 pub struct Butterfly7<T> {
485 twiddle1: Complex<T>,
486 twiddle2: Complex<T>,
487 twiddle3: Complex<T>,
488 direction: FftDirection,
489 }
490 boilerplate_fft_butterfly!(Butterfly7, 7, |this: &Butterfly7<_>| this.direction);
491 impl<T: FftNum> Butterfly7<T> {
492 pub fn new(direction: FftDirection) -> Self {
493 Self {
494 twiddle1: twiddles::compute_twiddle(1, 7, direction),
495 twiddle2: twiddles::compute_twiddle(2, 7, direction),
496 twiddle3: twiddles::compute_twiddle(3, 7, direction),
497 direction,
498 }
499 }
500 #[inline(never)]
501 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
502 // let mut outer = Butterfly2::perform_fft_array([buffer.load(1), buffer.load(6)]);
503 // let mut mid = Butterfly2::perform_fft_array([buffer.load(2), buffer.load(5)]);
504 // let mut inner = Butterfly2::perform_fft_array([buffer.load(3), buffer.load(4)]);
505 // let input0 = buffer.load(0);
506
507 // buffer.store(input0 + outer[0] + mid[0] + inner[0], 0);
508
509 // inner[1] = twiddles::rotate_90(inner[1], true);
510 // mid[1] = twiddles::rotate_90(mid[1], true);
511 // outer[1] = twiddles::rotate_90(outer[1], true);
512
513 // {
514 // let twiddled1 = outer[0] * self.twiddles[0].re;
515 // let twiddled2 = mid[0] * self.twiddles[1].re;
516 // let twiddled3 = inner[0] * self.twiddles[2].re;
517 // let twiddled4 = inner[1] * self.twiddles[2].im;
518 // let twiddled5 = mid[1] * self.twiddles[1].im;
519 // let twiddled6 = outer[1] * self.twiddles[0].im;
520
521 // let sum123 = twiddled1 + twiddled2 + twiddled3;
522 // let sum456 = twiddled4 + twiddled5 + twiddled6;
523
524 // let output1 = sum123 + sum456;
525 // let output6 = sum123 - sum456;
526
527 // buffer.store(input0 + output1, 1);
528 // buffer.store(input0 + output6, 6);
529 // }
530
531 // {
532 // let twiddled1 = outer[0] * self.twiddles[1].re;
533 // let twiddled2 = mid[0] * self.twiddles[2].re;
534 // let twiddled3 = inner[0] * self.twiddles[0].re;
535 // let twiddled4 = inner[1] * self.twiddles[0].im;
536 // let twiddled5 = mid[1] * self.twiddles[2].im;
537 // let twiddled6 = outer[1] * self.twiddles[1].im;
538
539 // let sum123 = twiddled1 + twiddled2 + twiddled3;
540 // let sum456 = twiddled6 - twiddled4 - twiddled5;
541
542 // let output2 = sum123 + sum456;
543 // let output5 = sum123 - sum456;
544
545 // buffer.store(input0 + output2, 2);
546 // buffer.store(input0 + output5, 5);
547 // }
548
549 // Let's do a plain 7-point Dft
550 // |X0| | W0 W0 W0 W0 W0 W0 W0 | |x0|
551 // |X1| | W0 W1 W2 W3 W4 W5 W6 | |x1|
552 // |X2| | W0 W2 W4 W6 W8 W10 W12 | |x2|
553 // |X3| = | W0 W3 W6 W9 W12 W15 W18 | * |x3|
554 // |X4| | W0 W4 W8 W12 W16 W20 W24 | |x4|
555 // |X5| | W0 W5 W10 W15 W20 W25 W30 | |x4|
556 // |X6| | W0 W6 W12 W18 W24 W30 W36 | |x4|
557 //
558 // where Wn = exp(-2*pi*n/7) for a forward transform, and exp(+2*pi*n/7) for an direction.
559 //
560 // Using the same logic as for the 5-point butterfly, this can be simplified to:
561 // |X0| | 1 1 1 1 1 1 1 | |x0|
562 // |X1| | 1 W1 W2 W3 W3* W2* W1* | |x1|
563 // |X2| | 1 W2 W3* W1* W1 W3 W2* | |x2|
564 // |X3| = | 1 W3 W1* W2 W2* W1 W3* | * |x3|
565 // |X4| | 1 W3* W1 W2* W2 W1* W3 | |x4|
566 // |X5| | 1 W2* W3 W1 W1* W3* W2 | |x5|
567 // |X6| | 1 W1* W2* W3* W3 W2 W1 | |x6|
568 //
569 // From here it's just about eliminating repeated calculations, following the same procedure as for the 5-point butterfly.
570
571 let x16p = buffer.load(1) + buffer.load(6);
572 let x16n = buffer.load(1) - buffer.load(6);
573 let x25p = buffer.load(2) + buffer.load(5);
574 let x25n = buffer.load(2) - buffer.load(5);
575 let x34p = buffer.load(3) + buffer.load(4);
576 let x34n = buffer.load(3) - buffer.load(4);
577 let sum = buffer.load(0) + x16p + x25p + x34p;
578
579 let x16re_a = buffer.load(0).re
580 + self.twiddle1.re * x16p.re
581 + self.twiddle2.re * x25p.re
582 + self.twiddle3.re * x34p.re;
583 let x16re_b =
584 self.twiddle1.im * x16n.im + self.twiddle2.im * x25n.im + self.twiddle3.im * x34n.im;
585 let x25re_a = buffer.load(0).re
586 + self.twiddle1.re * x34p.re
587 + self.twiddle2.re * x16p.re
588 + self.twiddle3.re * x25p.re;
589 let x25re_b =
590 -self.twiddle1.im * x34n.im + self.twiddle2.im * x16n.im - self.twiddle3.im * x25n.im;
591 let x34re_a = buffer.load(0).re
592 + self.twiddle1.re * x25p.re
593 + self.twiddle2.re * x34p.re
594 + self.twiddle3.re * x16p.re;
595 let x34re_b =
596 -self.twiddle1.im * x25n.im + self.twiddle2.im * x34n.im + self.twiddle3.im * x16n.im;
597 let x16im_a = buffer.load(0).im
598 + self.twiddle1.re * x16p.im
599 + self.twiddle2.re * x25p.im
600 + self.twiddle3.re * x34p.im;
601 let x16im_b =
602 self.twiddle1.im * x16n.re + self.twiddle2.im * x25n.re + self.twiddle3.im * x34n.re;
603 let x25im_a = buffer.load(0).im
604 + self.twiddle1.re * x34p.im
605 + self.twiddle2.re * x16p.im
606 + self.twiddle3.re * x25p.im;
607 let x25im_b =
608 -self.twiddle1.im * x34n.re + self.twiddle2.im * x16n.re - self.twiddle3.im * x25n.re;
609 let x34im_a = buffer.load(0).im
610 + self.twiddle1.re * x25p.im
611 + self.twiddle2.re * x34p.im
612 + self.twiddle3.re * x16p.im;
613 let x34im_b =
614 self.twiddle1.im * x25n.re - self.twiddle2.im * x34n.re - self.twiddle3.im * x16n.re;
615
616 let out1re = x16re_a - x16re_b;
617 let out1im = x16im_a + x16im_b;
618 let out2re = x25re_a - x25re_b;
619 let out2im = x25im_a + x25im_b;
620 let out3re = x34re_a - x34re_b;
621 let out3im = x34im_a - x34im_b;
622 let out4re = x34re_a + x34re_b;
623 let out4im = x34im_a + x34im_b;
624 let out5re = x25re_a + x25re_b;
625 let out5im = x25im_a - x25im_b;
626 let out6re = x16re_a + x16re_b;
627 let out6im = x16im_a - x16im_b;
628
629 buffer.store(sum, 0);
630 buffer.store(
631 Complex {
632 re: out1re,
633 im: out1im,
634 },
635 1,
636 );
637 buffer.store(
638 Complex {
639 re: out2re,
640 im: out2im,
641 },
642 2,
643 );
644 buffer.store(
645 Complex {
646 re: out3re,
647 im: out3im,
648 },
649 3,
650 );
651 buffer.store(
652 Complex {
653 re: out4re,
654 im: out4im,
655 },
656 4,
657 );
658 buffer.store(
659 Complex {
660 re: out5re,
661 im: out5im,
662 },
663 5,
664 );
665 buffer.store(
666 Complex {
667 re: out6re,
668 im: out6im,
669 },
670 6,
671 );
672 }
673 }
674
675 pub struct Butterfly8<T> {
676 root2: T,
677 direction: FftDirection,
678 }
679 boilerplate_fft_butterfly!(Butterfly8, 8, |this: &Butterfly8<_>| this.direction);
680 impl<T: FftNum> Butterfly8<T> {
681 #[inline(always)]
682 pub fn new(direction: FftDirection) -> Self {
683 Self {
684 root2: T::from_f64(0.5f64.sqrt()).unwrap(),
685 direction,
686 }
687 }
688
689 #[inline(always)]
690 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
691 let butterfly4 = Butterfly4::new(self.direction);
692
693 //we're going to hardcode a step of mixed radix
694 //aka we're going to do the six step algorithm
695
696 // step 1: transpose the input into the scratch
697 let mut scratch0 = [
698 buffer.load(0),
699 buffer.load(2),
700 buffer.load(4),
701 buffer.load(6),
702 ];
703 let mut scratch1 = [
704 buffer.load(1),
705 buffer.load(3),
706 buffer.load(5),
707 buffer.load(7),
708 ];
709
710 // step 2: column FFTs
711 butterfly4.perform_fft_contiguous(&mut scratch0);
712 butterfly4.perform_fft_contiguous(&mut scratch1);
713
714 // step 3: apply twiddle factors
715 scratch1[1] = (twiddles::rotate_90(scratch1[1], self.direction) + scratch1[1]) * self.root2;
716 scratch1[2] = twiddles::rotate_90(scratch1[2], self.direction);
717 scratch1[3] = (twiddles::rotate_90(scratch1[3], self.direction) - scratch1[3]) * self.root2;
718
719 // step 4: transpose -- skipped because we're going to do the next FFTs non-contiguously
720
721 // step 5: row FFTs
722 for i in 0..4 {
723 Butterfly2::perform_fft_strided(&mut scratch0[i], &mut scratch1[i]);
724 }
725
726 // step 6: copy data to the output. we don't need to transpose, because we skipped the step 4 transpose
727 for i in 0..4 {
728 buffer.store(scratch0[i], i);
729 }
730 for i in 0..4 {
731 buffer.store(scratch1[i], i + 4);
732 }
733 }
734 }
735
736 pub struct Butterfly9<T> {
737 butterfly3: Butterfly3<T>,
738 twiddle1: Complex<T>,
739 twiddle2: Complex<T>,
740 twiddle4: Complex<T>,
741 }
742 boilerplate_fft_butterfly!(Butterfly9, 9, |this: &Butterfly9<_>| this
743 .butterfly3
744 .fft_direction());
745 impl<T: FftNum> Butterfly9<T> {
746 #[inline(always)]
747 pub fn new(direction: FftDirection) -> Self {
748 Self {
749 butterfly3: Butterfly3::new(direction),
750 twiddle1: twiddles::compute_twiddle(1, 9, direction),
751 twiddle2: twiddles::compute_twiddle(2, 9, direction),
752 twiddle4: twiddles::compute_twiddle(4, 9, direction),
753 }
754 }
755 #[inline(always)]
756 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
757 // algorithm: mixed radix with width=3 and height=3
758
759 // step 1: transpose the input into the scratch
760 let mut scratch0 = [buffer.load(0), buffer.load(3), buffer.load(6)];
761 let mut scratch1 = [buffer.load(1), buffer.load(4), buffer.load(7)];
762 let mut scratch2 = [buffer.load(2), buffer.load(5), buffer.load(8)];
763
764 // step 2: column FFTs
765 self.butterfly3.perform_fft_contiguous(&mut scratch0);
766 self.butterfly3.perform_fft_contiguous(&mut scratch1);
767 self.butterfly3.perform_fft_contiguous(&mut scratch2);
768
769 // step 3: apply twiddle factors
770 scratch1[1] = scratch1[1] * self.twiddle1;
771 scratch1[2] = scratch1[2] * self.twiddle2;
772 scratch2[1] = scratch2[1] * self.twiddle2;
773 scratch2[2] = scratch2[2] * self.twiddle4;
774
775 // step 4: SKIPPED because the next FFTs will be non-contiguous
776
777 // step 5: row FFTs
778 self.butterfly3
779 .perform_fft_strided(&mut scratch0[0], &mut scratch1[0], &mut scratch2[0]);
780 self.butterfly3
781 .perform_fft_strided(&mut scratch0[1], &mut scratch1[1], &mut scratch2[1]);
782 self.butterfly3
783 .perform_fft_strided(&mut scratch0[2], &mut scratch1[2], &mut scratch2[2]);
784
785 // step 6: copy the result into the output. normally we'd need to do a transpose here, but we can skip it because we skipped the transpose in step 4
786 buffer.store(scratch0[0], 0);
787 buffer.store(scratch0[1], 1);
788 buffer.store(scratch0[2], 2);
789 buffer.store(scratch1[0], 3);
790 buffer.store(scratch1[1], 4);
791 buffer.store(scratch1[2], 5);
792 buffer.store(scratch2[0], 6);
793 buffer.store(scratch2[1], 7);
794 buffer.store(scratch2[2], 8);
795 }
796 }
797
798 pub struct Butterfly11<T> {
799 twiddle1: Complex<T>,
800 twiddle2: Complex<T>,
801 twiddle3: Complex<T>,
802 twiddle4: Complex<T>,
803 twiddle5: Complex<T>,
804 direction: FftDirection,
805 }
806 boilerplate_fft_butterfly!(Butterfly11, 11, |this: &Butterfly11<_>| this.direction);
807 impl<T: FftNum> Butterfly11<T> {
808 pub fn new(direction: FftDirection) -> Self {
809 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 11, direction);
810 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 11, direction);
811 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 11, direction);
812 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 11, direction);
813 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 11, direction);
814 Self {
815 twiddle1,
816 twiddle2,
817 twiddle3,
818 twiddle4,
819 twiddle5,
820 direction,
821 }
822 }
823
824 #[inline(never)]
825 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
826 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
827 // However, instead of doing it by hand the actual code is autogenerated
828 // with the `genbutterflies.py` script in the `tools` directory.
829
830 let x110p = buffer.load(1) + buffer.load(10);
831 let x110n = buffer.load(1) - buffer.load(10);
832 let x29p = buffer.load(2) + buffer.load(9);
833 let x29n = buffer.load(2) - buffer.load(9);
834 let x38p = buffer.load(3) + buffer.load(8);
835 let x38n = buffer.load(3) - buffer.load(8);
836 let x47p = buffer.load(4) + buffer.load(7);
837 let x47n = buffer.load(4) - buffer.load(7);
838 let x56p = buffer.load(5) + buffer.load(6);
839 let x56n = buffer.load(5) - buffer.load(6);
840 let sum = buffer.load(0) + x110p + x29p + x38p + x47p + x56p;
841 let b110re_a = buffer.load(0).re
842 + self.twiddle1.re * x110p.re
843 + self.twiddle2.re * x29p.re
844 + self.twiddle3.re * x38p.re
845 + self.twiddle4.re * x47p.re
846 + self.twiddle5.re * x56p.re;
847 let b110re_b = self.twiddle1.im * x110n.im
848 + self.twiddle2.im * x29n.im
849 + self.twiddle3.im * x38n.im
850 + self.twiddle4.im * x47n.im
851 + self.twiddle5.im * x56n.im;
852 let b29re_a = buffer.load(0).re
853 + self.twiddle2.re * x110p.re
854 + self.twiddle4.re * x29p.re
855 + self.twiddle5.re * x38p.re
856 + self.twiddle3.re * x47p.re
857 + self.twiddle1.re * x56p.re;
858 let b29re_b = self.twiddle2.im * x110n.im
859 + self.twiddle4.im * x29n.im
860 + -self.twiddle5.im * x38n.im
861 + -self.twiddle3.im * x47n.im
862 + -self.twiddle1.im * x56n.im;
863 let b38re_a = buffer.load(0).re
864 + self.twiddle3.re * x110p.re
865 + self.twiddle5.re * x29p.re
866 + self.twiddle2.re * x38p.re
867 + self.twiddle1.re * x47p.re
868 + self.twiddle4.re * x56p.re;
869 let b38re_b = self.twiddle3.im * x110n.im
870 + -self.twiddle5.im * x29n.im
871 + -self.twiddle2.im * x38n.im
872 + self.twiddle1.im * x47n.im
873 + self.twiddle4.im * x56n.im;
874 let b47re_a = buffer.load(0).re
875 + self.twiddle4.re * x110p.re
876 + self.twiddle3.re * x29p.re
877 + self.twiddle1.re * x38p.re
878 + self.twiddle5.re * x47p.re
879 + self.twiddle2.re * x56p.re;
880 let b47re_b = self.twiddle4.im * x110n.im
881 + -self.twiddle3.im * x29n.im
882 + self.twiddle1.im * x38n.im
883 + self.twiddle5.im * x47n.im
884 + -self.twiddle2.im * x56n.im;
885 let b56re_a = buffer.load(0).re
886 + self.twiddle5.re * x110p.re
887 + self.twiddle1.re * x29p.re
888 + self.twiddle4.re * x38p.re
889 + self.twiddle2.re * x47p.re
890 + self.twiddle3.re * x56p.re;
891 let b56re_b = self.twiddle5.im * x110n.im
892 + -self.twiddle1.im * x29n.im
893 + self.twiddle4.im * x38n.im
894 + -self.twiddle2.im * x47n.im
895 + self.twiddle3.im * x56n.im;
896
897 let b110im_a = buffer.load(0).im
898 + self.twiddle1.re * x110p.im
899 + self.twiddle2.re * x29p.im
900 + self.twiddle3.re * x38p.im
901 + self.twiddle4.re * x47p.im
902 + self.twiddle5.re * x56p.im;
903 let b110im_b = self.twiddle1.im * x110n.re
904 + self.twiddle2.im * x29n.re
905 + self.twiddle3.im * x38n.re
906 + self.twiddle4.im * x47n.re
907 + self.twiddle5.im * x56n.re;
908 let b29im_a = buffer.load(0).im
909 + self.twiddle2.re * x110p.im
910 + self.twiddle4.re * x29p.im
911 + self.twiddle5.re * x38p.im
912 + self.twiddle3.re * x47p.im
913 + self.twiddle1.re * x56p.im;
914 let b29im_b = self.twiddle2.im * x110n.re
915 + self.twiddle4.im * x29n.re
916 + -self.twiddle5.im * x38n.re
917 + -self.twiddle3.im * x47n.re
918 + -self.twiddle1.im * x56n.re;
919 let b38im_a = buffer.load(0).im
920 + self.twiddle3.re * x110p.im
921 + self.twiddle5.re * x29p.im
922 + self.twiddle2.re * x38p.im
923 + self.twiddle1.re * x47p.im
924 + self.twiddle4.re * x56p.im;
925 let b38im_b = self.twiddle3.im * x110n.re
926 + -self.twiddle5.im * x29n.re
927 + -self.twiddle2.im * x38n.re
928 + self.twiddle1.im * x47n.re
929 + self.twiddle4.im * x56n.re;
930 let b47im_a = buffer.load(0).im
931 + self.twiddle4.re * x110p.im
932 + self.twiddle3.re * x29p.im
933 + self.twiddle1.re * x38p.im
934 + self.twiddle5.re * x47p.im
935 + self.twiddle2.re * x56p.im;
936 let b47im_b = self.twiddle4.im * x110n.re
937 + -self.twiddle3.im * x29n.re
938 + self.twiddle1.im * x38n.re
939 + self.twiddle5.im * x47n.re
940 + -self.twiddle2.im * x56n.re;
941 let b56im_a = buffer.load(0).im
942 + self.twiddle5.re * x110p.im
943 + self.twiddle1.re * x29p.im
944 + self.twiddle4.re * x38p.im
945 + self.twiddle2.re * x47p.im
946 + self.twiddle3.re * x56p.im;
947 let b56im_b = self.twiddle5.im * x110n.re
948 + -self.twiddle1.im * x29n.re
949 + self.twiddle4.im * x38n.re
950 + -self.twiddle2.im * x47n.re
951 + self.twiddle3.im * x56n.re;
952
953 let out1re = b110re_a - b110re_b;
954 let out1im = b110im_a + b110im_b;
955 let out2re = b29re_a - b29re_b;
956 let out2im = b29im_a + b29im_b;
957 let out3re = b38re_a - b38re_b;
958 let out3im = b38im_a + b38im_b;
959 let out4re = b47re_a - b47re_b;
960 let out4im = b47im_a + b47im_b;
961 let out5re = b56re_a - b56re_b;
962 let out5im = b56im_a + b56im_b;
963 let out6re = b56re_a + b56re_b;
964 let out6im = b56im_a - b56im_b;
965 let out7re = b47re_a + b47re_b;
966 let out7im = b47im_a - b47im_b;
967 let out8re = b38re_a + b38re_b;
968 let out8im = b38im_a - b38im_b;
969 let out9re = b29re_a + b29re_b;
970 let out9im = b29im_a - b29im_b;
971 let out10re = b110re_a + b110re_b;
972 let out10im = b110im_a - b110im_b;
973 buffer.store(sum, 0);
974 buffer.store(
975 Complex {
976 re: out1re,
977 im: out1im,
978 },
979 1,
980 );
981 buffer.store(
982 Complex {
983 re: out2re,
984 im: out2im,
985 },
986 2,
987 );
988 buffer.store(
989 Complex {
990 re: out3re,
991 im: out3im,
992 },
993 3,
994 );
995 buffer.store(
996 Complex {
997 re: out4re,
998 im: out4im,
999 },
1000 4,
1001 );
1002 buffer.store(
1003 Complex {
1004 re: out5re,
1005 im: out5im,
1006 },
1007 5,
1008 );
1009 buffer.store(
1010 Complex {
1011 re: out6re,
1012 im: out6im,
1013 },
1014 6,
1015 );
1016 buffer.store(
1017 Complex {
1018 re: out7re,
1019 im: out7im,
1020 },
1021 7,
1022 );
1023 buffer.store(
1024 Complex {
1025 re: out8re,
1026 im: out8im,
1027 },
1028 8,
1029 );
1030 buffer.store(
1031 Complex {
1032 re: out9re,
1033 im: out9im,
1034 },
1035 9,
1036 );
1037 buffer.store(
1038 Complex {
1039 re: out10re,
1040 im: out10im,
1041 },
1042 10,
1043 );
1044 }
1045 }
1046
1047 pub struct Butterfly13<T> {
1048 twiddle1: Complex<T>,
1049 twiddle2: Complex<T>,
1050 twiddle3: Complex<T>,
1051 twiddle4: Complex<T>,
1052 twiddle5: Complex<T>,
1053 twiddle6: Complex<T>,
1054 direction: FftDirection,
1055 }
1056 boilerplate_fft_butterfly!(Butterfly13, 13, |this: &Butterfly13<_>| this.direction);
1057 impl<T: FftNum> Butterfly13<T> {
1058 pub fn new(direction: FftDirection) -> Self {
1059 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 13, direction);
1060 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 13, direction);
1061 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 13, direction);
1062 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 13, direction);
1063 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 13, direction);
1064 let twiddle6: Complex<T> = twiddles::compute_twiddle(6, 13, direction);
1065 Self {
1066 twiddle1,
1067 twiddle2,
1068 twiddle3,
1069 twiddle4,
1070 twiddle5,
1071 twiddle6,
1072 direction,
1073 }
1074 }
1075
1076 #[inline(never)]
1077 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
1078 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
1079 // However, instead of doing it by hand the actual code is autogenerated
1080 // with the `genbutterflies.py` script in the `tools` directory.
1081 let x112p = buffer.load(1) + buffer.load(12);
1082 let x112n = buffer.load(1) - buffer.load(12);
1083 let x211p = buffer.load(2) + buffer.load(11);
1084 let x211n = buffer.load(2) - buffer.load(11);
1085 let x310p = buffer.load(3) + buffer.load(10);
1086 let x310n = buffer.load(3) - buffer.load(10);
1087 let x49p = buffer.load(4) + buffer.load(9);
1088 let x49n = buffer.load(4) - buffer.load(9);
1089 let x58p = buffer.load(5) + buffer.load(8);
1090 let x58n = buffer.load(5) - buffer.load(8);
1091 let x67p = buffer.load(6) + buffer.load(7);
1092 let x67n = buffer.load(6) - buffer.load(7);
1093 let sum = buffer.load(0) + x112p + x211p + x310p + x49p + x58p + x67p;
1094 let b112re_a = buffer.load(0).re
1095 + self.twiddle1.re * x112p.re
1096 + self.twiddle2.re * x211p.re
1097 + self.twiddle3.re * x310p.re
1098 + self.twiddle4.re * x49p.re
1099 + self.twiddle5.re * x58p.re
1100 + self.twiddle6.re * x67p.re;
1101 let b112re_b = self.twiddle1.im * x112n.im
1102 + self.twiddle2.im * x211n.im
1103 + self.twiddle3.im * x310n.im
1104 + self.twiddle4.im * x49n.im
1105 + self.twiddle5.im * x58n.im
1106 + self.twiddle6.im * x67n.im;
1107 let b211re_a = buffer.load(0).re
1108 + self.twiddle2.re * x112p.re
1109 + self.twiddle4.re * x211p.re
1110 + self.twiddle6.re * x310p.re
1111 + self.twiddle5.re * x49p.re
1112 + self.twiddle3.re * x58p.re
1113 + self.twiddle1.re * x67p.re;
1114 let b211re_b = self.twiddle2.im * x112n.im
1115 + self.twiddle4.im * x211n.im
1116 + self.twiddle6.im * x310n.im
1117 + -self.twiddle5.im * x49n.im
1118 + -self.twiddle3.im * x58n.im
1119 + -self.twiddle1.im * x67n.im;
1120 let b310re_a = buffer.load(0).re
1121 + self.twiddle3.re * x112p.re
1122 + self.twiddle6.re * x211p.re
1123 + self.twiddle4.re * x310p.re
1124 + self.twiddle1.re * x49p.re
1125 + self.twiddle2.re * x58p.re
1126 + self.twiddle5.re * x67p.re;
1127 let b310re_b = self.twiddle3.im * x112n.im
1128 + self.twiddle6.im * x211n.im
1129 + -self.twiddle4.im * x310n.im
1130 + -self.twiddle1.im * x49n.im
1131 + self.twiddle2.im * x58n.im
1132 + self.twiddle5.im * x67n.im;
1133 let b49re_a = buffer.load(0).re
1134 + self.twiddle4.re * x112p.re
1135 + self.twiddle5.re * x211p.re
1136 + self.twiddle1.re * x310p.re
1137 + self.twiddle3.re * x49p.re
1138 + self.twiddle6.re * x58p.re
1139 + self.twiddle2.re * x67p.re;
1140 let b49re_b = self.twiddle4.im * x112n.im
1141 + -self.twiddle5.im * x211n.im
1142 + -self.twiddle1.im * x310n.im
1143 + self.twiddle3.im * x49n.im
1144 + -self.twiddle6.im * x58n.im
1145 + -self.twiddle2.im * x67n.im;
1146 let b58re_a = buffer.load(0).re
1147 + self.twiddle5.re * x112p.re
1148 + self.twiddle3.re * x211p.re
1149 + self.twiddle2.re * x310p.re
1150 + self.twiddle6.re * x49p.re
1151 + self.twiddle1.re * x58p.re
1152 + self.twiddle4.re * x67p.re;
1153 let b58re_b = self.twiddle5.im * x112n.im
1154 + -self.twiddle3.im * x211n.im
1155 + self.twiddle2.im * x310n.im
1156 + -self.twiddle6.im * x49n.im
1157 + -self.twiddle1.im * x58n.im
1158 + self.twiddle4.im * x67n.im;
1159 let b67re_a = buffer.load(0).re
1160 + self.twiddle6.re * x112p.re
1161 + self.twiddle1.re * x211p.re
1162 + self.twiddle5.re * x310p.re
1163 + self.twiddle2.re * x49p.re
1164 + self.twiddle4.re * x58p.re
1165 + self.twiddle3.re * x67p.re;
1166 let b67re_b = self.twiddle6.im * x112n.im
1167 + -self.twiddle1.im * x211n.im
1168 + self.twiddle5.im * x310n.im
1169 + -self.twiddle2.im * x49n.im
1170 + self.twiddle4.im * x58n.im
1171 + -self.twiddle3.im * x67n.im;
1172
1173 let b112im_a = buffer.load(0).im
1174 + self.twiddle1.re * x112p.im
1175 + self.twiddle2.re * x211p.im
1176 + self.twiddle3.re * x310p.im
1177 + self.twiddle4.re * x49p.im
1178 + self.twiddle5.re * x58p.im
1179 + self.twiddle6.re * x67p.im;
1180 let b112im_b = self.twiddle1.im * x112n.re
1181 + self.twiddle2.im * x211n.re
1182 + self.twiddle3.im * x310n.re
1183 + self.twiddle4.im * x49n.re
1184 + self.twiddle5.im * x58n.re
1185 + self.twiddle6.im * x67n.re;
1186 let b211im_a = buffer.load(0).im
1187 + self.twiddle2.re * x112p.im
1188 + self.twiddle4.re * x211p.im
1189 + self.twiddle6.re * x310p.im
1190 + self.twiddle5.re * x49p.im
1191 + self.twiddle3.re * x58p.im
1192 + self.twiddle1.re * x67p.im;
1193 let b211im_b = self.twiddle2.im * x112n.re
1194 + self.twiddle4.im * x211n.re
1195 + self.twiddle6.im * x310n.re
1196 + -self.twiddle5.im * x49n.re
1197 + -self.twiddle3.im * x58n.re
1198 + -self.twiddle1.im * x67n.re;
1199 let b310im_a = buffer.load(0).im
1200 + self.twiddle3.re * x112p.im
1201 + self.twiddle6.re * x211p.im
1202 + self.twiddle4.re * x310p.im
1203 + self.twiddle1.re * x49p.im
1204 + self.twiddle2.re * x58p.im
1205 + self.twiddle5.re * x67p.im;
1206 let b310im_b = self.twiddle3.im * x112n.re
1207 + self.twiddle6.im * x211n.re
1208 + -self.twiddle4.im * x310n.re
1209 + -self.twiddle1.im * x49n.re
1210 + self.twiddle2.im * x58n.re
1211 + self.twiddle5.im * x67n.re;
1212 let b49im_a = buffer.load(0).im
1213 + self.twiddle4.re * x112p.im
1214 + self.twiddle5.re * x211p.im
1215 + self.twiddle1.re * x310p.im
1216 + self.twiddle3.re * x49p.im
1217 + self.twiddle6.re * x58p.im
1218 + self.twiddle2.re * x67p.im;
1219 let b49im_b = self.twiddle4.im * x112n.re
1220 + -self.twiddle5.im * x211n.re
1221 + -self.twiddle1.im * x310n.re
1222 + self.twiddle3.im * x49n.re
1223 + -self.twiddle6.im * x58n.re
1224 + -self.twiddle2.im * x67n.re;
1225 let b58im_a = buffer.load(0).im
1226 + self.twiddle5.re * x112p.im
1227 + self.twiddle3.re * x211p.im
1228 + self.twiddle2.re * x310p.im
1229 + self.twiddle6.re * x49p.im
1230 + self.twiddle1.re * x58p.im
1231 + self.twiddle4.re * x67p.im;
1232 let b58im_b = self.twiddle5.im * x112n.re
1233 + -self.twiddle3.im * x211n.re
1234 + self.twiddle2.im * x310n.re
1235 + -self.twiddle6.im * x49n.re
1236 + -self.twiddle1.im * x58n.re
1237 + self.twiddle4.im * x67n.re;
1238 let b67im_a = buffer.load(0).im
1239 + self.twiddle6.re * x112p.im
1240 + self.twiddle1.re * x211p.im
1241 + self.twiddle5.re * x310p.im
1242 + self.twiddle2.re * x49p.im
1243 + self.twiddle4.re * x58p.im
1244 + self.twiddle3.re * x67p.im;
1245 let b67im_b = self.twiddle6.im * x112n.re
1246 + -self.twiddle1.im * x211n.re
1247 + self.twiddle5.im * x310n.re
1248 + -self.twiddle2.im * x49n.re
1249 + self.twiddle4.im * x58n.re
1250 + -self.twiddle3.im * x67n.re;
1251
1252 let out1re = b112re_a - b112re_b;
1253 let out1im = b112im_a + b112im_b;
1254 let out2re = b211re_a - b211re_b;
1255 let out2im = b211im_a + b211im_b;
1256 let out3re = b310re_a - b310re_b;
1257 let out3im = b310im_a + b310im_b;
1258 let out4re = b49re_a - b49re_b;
1259 let out4im = b49im_a + b49im_b;
1260 let out5re = b58re_a - b58re_b;
1261 let out5im = b58im_a + b58im_b;
1262 let out6re = b67re_a - b67re_b;
1263 let out6im = b67im_a + b67im_b;
1264 let out7re = b67re_a + b67re_b;
1265 let out7im = b67im_a - b67im_b;
1266 let out8re = b58re_a + b58re_b;
1267 let out8im = b58im_a - b58im_b;
1268 let out9re = b49re_a + b49re_b;
1269 let out9im = b49im_a - b49im_b;
1270 let out10re = b310re_a + b310re_b;
1271 let out10im = b310im_a - b310im_b;
1272 let out11re = b211re_a + b211re_b;
1273 let out11im = b211im_a - b211im_b;
1274 let out12re = b112re_a + b112re_b;
1275 let out12im = b112im_a - b112im_b;
1276 buffer.store(sum, 0);
1277 buffer.store(
1278 Complex {
1279 re: out1re,
1280 im: out1im,
1281 },
1282 1,
1283 );
1284 buffer.store(
1285 Complex {
1286 re: out2re,
1287 im: out2im,
1288 },
1289 2,
1290 );
1291 buffer.store(
1292 Complex {
1293 re: out3re,
1294 im: out3im,
1295 },
1296 3,
1297 );
1298 buffer.store(
1299 Complex {
1300 re: out4re,
1301 im: out4im,
1302 },
1303 4,
1304 );
1305 buffer.store(
1306 Complex {
1307 re: out5re,
1308 im: out5im,
1309 },
1310 5,
1311 );
1312 buffer.store(
1313 Complex {
1314 re: out6re,
1315 im: out6im,
1316 },
1317 6,
1318 );
1319 buffer.store(
1320 Complex {
1321 re: out7re,
1322 im: out7im,
1323 },
1324 7,
1325 );
1326 buffer.store(
1327 Complex {
1328 re: out8re,
1329 im: out8im,
1330 },
1331 8,
1332 );
1333 buffer.store(
1334 Complex {
1335 re: out9re,
1336 im: out9im,
1337 },
1338 9,
1339 );
1340 buffer.store(
1341 Complex {
1342 re: out10re,
1343 im: out10im,
1344 },
1345 10,
1346 );
1347 buffer.store(
1348 Complex {
1349 re: out11re,
1350 im: out11im,
1351 },
1352 11,
1353 );
1354 buffer.store(
1355 Complex {
1356 re: out12re,
1357 im: out12im,
1358 },
1359 12,
1360 );
1361 }
1362 }
1363
1364 pub struct Butterfly16<T> {
1365 butterfly8: Butterfly8<T>,
1366 twiddle1: Complex<T>,
1367 twiddle2: Complex<T>,
1368 twiddle3: Complex<T>,
1369 }
1370 boilerplate_fft_butterfly!(Butterfly16, 16, |this: &Butterfly16<_>| this
1371 .butterfly8
1372 .fft_direction());
1373 impl<T: FftNum> Butterfly16<T> {
1374 #[inline(always)]
1375 pub fn new(direction: FftDirection) -> Self {
1376 Self {
1377 butterfly8: Butterfly8::new(direction),
1378 twiddle1: twiddles::compute_twiddle(1, 16, direction),
1379 twiddle2: twiddles::compute_twiddle(2, 16, direction),
1380 twiddle3: twiddles::compute_twiddle(3, 16, direction),
1381 }
1382 }
1383
1384 #[inline(never)]
1385 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
1386 let butterfly4 = Butterfly4::new(self.fft_direction());
1387
1388 // we're going to hardcode a step of split radix
1389 // step 1: copy and reorder the input into the scratch
1390 let mut scratch_evens = [
1391 buffer.load(0),
1392 buffer.load(2),
1393 buffer.load(4),
1394 buffer.load(6),
1395 buffer.load(8),
1396 buffer.load(10),
1397 buffer.load(12),
1398 buffer.load(14),
1399 ];
1400
1401 let mut scratch_odds_n1 = [
1402 buffer.load(1),
1403 buffer.load(5),
1404 buffer.load(9),
1405 buffer.load(13),
1406 ];
1407 let mut scratch_odds_n3 = [
1408 buffer.load(15),
1409 buffer.load(3),
1410 buffer.load(7),
1411 buffer.load(11),
1412 ];
1413
1414 // step 2: column FFTs
1415 self.butterfly8.perform_fft_contiguous(&mut scratch_evens);
1416 butterfly4.perform_fft_contiguous(&mut scratch_odds_n1);
1417 butterfly4.perform_fft_contiguous(&mut scratch_odds_n3);
1418
1419 // step 3: apply twiddle factors
1420 scratch_odds_n1[1] = scratch_odds_n1[1] * self.twiddle1;
1421 scratch_odds_n3[1] = scratch_odds_n3[1] * self.twiddle1.conj();
1422
1423 scratch_odds_n1[2] = scratch_odds_n1[2] * self.twiddle2;
1424 scratch_odds_n3[2] = scratch_odds_n3[2] * self.twiddle2.conj();
1425
1426 scratch_odds_n1[3] = scratch_odds_n1[3] * self.twiddle3;
1427 scratch_odds_n3[3] = scratch_odds_n3[3] * self.twiddle3.conj();
1428
1429 // step 4: cross FFTs
1430 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[0], &mut scratch_odds_n3[0]);
1431 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[1], &mut scratch_odds_n3[1]);
1432 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[2], &mut scratch_odds_n3[2]);
1433 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[3], &mut scratch_odds_n3[3]);
1434
1435 // apply the butterfly 4 twiddle factor, which is just a rotation
1436 scratch_odds_n3[0] = twiddles::rotate_90(scratch_odds_n3[0], self.fft_direction());
1437 scratch_odds_n3[1] = twiddles::rotate_90(scratch_odds_n3[1], self.fft_direction());
1438 scratch_odds_n3[2] = twiddles::rotate_90(scratch_odds_n3[2], self.fft_direction());
1439 scratch_odds_n3[3] = twiddles::rotate_90(scratch_odds_n3[3], self.fft_direction());
1440
1441 //step 5: copy/add/subtract data back to buffer
1442 buffer.store(scratch_evens[0] + scratch_odds_n1[0], 0);
1443 buffer.store(scratch_evens[1] + scratch_odds_n1[1], 1);
1444 buffer.store(scratch_evens[2] + scratch_odds_n1[2], 2);
1445 buffer.store(scratch_evens[3] + scratch_odds_n1[3], 3);
1446 buffer.store(scratch_evens[4] + scratch_odds_n3[0], 4);
1447 buffer.store(scratch_evens[5] + scratch_odds_n3[1], 5);
1448 buffer.store(scratch_evens[6] + scratch_odds_n3[2], 6);
1449 buffer.store(scratch_evens[7] + scratch_odds_n3[3], 7);
1450 buffer.store(scratch_evens[0] - scratch_odds_n1[0], 8);
1451 buffer.store(scratch_evens[1] - scratch_odds_n1[1], 9);
1452 buffer.store(scratch_evens[2] - scratch_odds_n1[2], 10);
1453 buffer.store(scratch_evens[3] - scratch_odds_n1[3], 11);
1454 buffer.store(scratch_evens[4] - scratch_odds_n3[0], 12);
1455 buffer.store(scratch_evens[5] - scratch_odds_n3[1], 13);
1456 buffer.store(scratch_evens[6] - scratch_odds_n3[2], 14);
1457 buffer.store(scratch_evens[7] - scratch_odds_n3[3], 15);
1458 }
1459 }
1460
1461 pub struct Butterfly17<T> {
1462 twiddle1: Complex<T>,
1463 twiddle2: Complex<T>,
1464 twiddle3: Complex<T>,
1465 twiddle4: Complex<T>,
1466 twiddle5: Complex<T>,
1467 twiddle6: Complex<T>,
1468 twiddle7: Complex<T>,
1469 twiddle8: Complex<T>,
1470 direction: FftDirection,
1471 }
1472 boilerplate_fft_butterfly!(Butterfly17, 17, |this: &Butterfly17<_>| this.direction);
1473 impl<T: FftNum> Butterfly17<T> {
1474 pub fn new(direction: FftDirection) -> Self {
1475 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 17, direction);
1476 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 17, direction);
1477 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 17, direction);
1478 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 17, direction);
1479 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 17, direction);
1480 let twiddle6: Complex<T> = twiddles::compute_twiddle(6, 17, direction);
1481 let twiddle7: Complex<T> = twiddles::compute_twiddle(7, 17, direction);
1482 let twiddle8: Complex<T> = twiddles::compute_twiddle(8, 17, direction);
1483 Self {
1484 twiddle1,
1485 twiddle2,
1486 twiddle3,
1487 twiddle4,
1488 twiddle5,
1489 twiddle6,
1490 twiddle7,
1491 twiddle8,
1492 direction,
1493 }
1494 }
1495
1496 #[inline(never)]
1497 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
1498 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
1499 // However, instead of doing it by hand the actual code is autogenerated
1500 // with the `genbutterflies.py` script in the `tools` directory.
1501 let x116p = buffer.load(1) + buffer.load(16);
1502 let x116n = buffer.load(1) - buffer.load(16);
1503 let x215p = buffer.load(2) + buffer.load(15);
1504 let x215n = buffer.load(2) - buffer.load(15);
1505 let x314p = buffer.load(3) + buffer.load(14);
1506 let x314n = buffer.load(3) - buffer.load(14);
1507 let x413p = buffer.load(4) + buffer.load(13);
1508 let x413n = buffer.load(4) - buffer.load(13);
1509 let x512p = buffer.load(5) + buffer.load(12);
1510 let x512n = buffer.load(5) - buffer.load(12);
1511 let x611p = buffer.load(6) + buffer.load(11);
1512 let x611n = buffer.load(6) - buffer.load(11);
1513 let x710p = buffer.load(7) + buffer.load(10);
1514 let x710n = buffer.load(7) - buffer.load(10);
1515 let x89p = buffer.load(8) + buffer.load(9);
1516 let x89n = buffer.load(8) - buffer.load(9);
1517 let sum = buffer.load(0) + x116p + x215p + x314p + x413p + x512p + x611p + x710p + x89p;
1518 let b116re_a = buffer.load(0).re
1519 + self.twiddle1.re * x116p.re
1520 + self.twiddle2.re * x215p.re
1521 + self.twiddle3.re * x314p.re
1522 + self.twiddle4.re * x413p.re
1523 + self.twiddle5.re * x512p.re
1524 + self.twiddle6.re * x611p.re
1525 + self.twiddle7.re * x710p.re
1526 + self.twiddle8.re * x89p.re;
1527 let b116re_b = self.twiddle1.im * x116n.im
1528 + self.twiddle2.im * x215n.im
1529 + self.twiddle3.im * x314n.im
1530 + self.twiddle4.im * x413n.im
1531 + self.twiddle5.im * x512n.im
1532 + self.twiddle6.im * x611n.im
1533 + self.twiddle7.im * x710n.im
1534 + self.twiddle8.im * x89n.im;
1535 let b215re_a = buffer.load(0).re
1536 + self.twiddle2.re * x116p.re
1537 + self.twiddle4.re * x215p.re
1538 + self.twiddle6.re * x314p.re
1539 + self.twiddle8.re * x413p.re
1540 + self.twiddle7.re * x512p.re
1541 + self.twiddle5.re * x611p.re
1542 + self.twiddle3.re * x710p.re
1543 + self.twiddle1.re * x89p.re;
1544 let b215re_b = self.twiddle2.im * x116n.im
1545 + self.twiddle4.im * x215n.im
1546 + self.twiddle6.im * x314n.im
1547 + self.twiddle8.im * x413n.im
1548 + -self.twiddle7.im * x512n.im
1549 + -self.twiddle5.im * x611n.im
1550 + -self.twiddle3.im * x710n.im
1551 + -self.twiddle1.im * x89n.im;
1552 let b314re_a = buffer.load(0).re
1553 + self.twiddle3.re * x116p.re
1554 + self.twiddle6.re * x215p.re
1555 + self.twiddle8.re * x314p.re
1556 + self.twiddle5.re * x413p.re
1557 + self.twiddle2.re * x512p.re
1558 + self.twiddle1.re * x611p.re
1559 + self.twiddle4.re * x710p.re
1560 + self.twiddle7.re * x89p.re;
1561 let b314re_b = self.twiddle3.im * x116n.im
1562 + self.twiddle6.im * x215n.im
1563 + -self.twiddle8.im * x314n.im
1564 + -self.twiddle5.im * x413n.im
1565 + -self.twiddle2.im * x512n.im
1566 + self.twiddle1.im * x611n.im
1567 + self.twiddle4.im * x710n.im
1568 + self.twiddle7.im * x89n.im;
1569 let b413re_a = buffer.load(0).re
1570 + self.twiddle4.re * x116p.re
1571 + self.twiddle8.re * x215p.re
1572 + self.twiddle5.re * x314p.re
1573 + self.twiddle1.re * x413p.re
1574 + self.twiddle3.re * x512p.re
1575 + self.twiddle7.re * x611p.re
1576 + self.twiddle6.re * x710p.re
1577 + self.twiddle2.re * x89p.re;
1578 let b413re_b = self.twiddle4.im * x116n.im
1579 + self.twiddle8.im * x215n.im
1580 + -self.twiddle5.im * x314n.im
1581 + -self.twiddle1.im * x413n.im
1582 + self.twiddle3.im * x512n.im
1583 + self.twiddle7.im * x611n.im
1584 + -self.twiddle6.im * x710n.im
1585 + -self.twiddle2.im * x89n.im;
1586 let b512re_a = buffer.load(0).re
1587 + self.twiddle5.re * x116p.re
1588 + self.twiddle7.re * x215p.re
1589 + self.twiddle2.re * x314p.re
1590 + self.twiddle3.re * x413p.re
1591 + self.twiddle8.re * x512p.re
1592 + self.twiddle4.re * x611p.re
1593 + self.twiddle1.re * x710p.re
1594 + self.twiddle6.re * x89p.re;
1595 let b512re_b = self.twiddle5.im * x116n.im
1596 + -self.twiddle7.im * x215n.im
1597 + -self.twiddle2.im * x314n.im
1598 + self.twiddle3.im * x413n.im
1599 + self.twiddle8.im * x512n.im
1600 + -self.twiddle4.im * x611n.im
1601 + self.twiddle1.im * x710n.im
1602 + self.twiddle6.im * x89n.im;
1603 let b611re_a = buffer.load(0).re
1604 + self.twiddle6.re * x116p.re
1605 + self.twiddle5.re * x215p.re
1606 + self.twiddle1.re * x314p.re
1607 + self.twiddle7.re * x413p.re
1608 + self.twiddle4.re * x512p.re
1609 + self.twiddle2.re * x611p.re
1610 + self.twiddle8.re * x710p.re
1611 + self.twiddle3.re * x89p.re;
1612 let b611re_b = self.twiddle6.im * x116n.im
1613 + -self.twiddle5.im * x215n.im
1614 + self.twiddle1.im * x314n.im
1615 + self.twiddle7.im * x413n.im
1616 + -self.twiddle4.im * x512n.im
1617 + self.twiddle2.im * x611n.im
1618 + self.twiddle8.im * x710n.im
1619 + -self.twiddle3.im * x89n.im;
1620 let b710re_a = buffer.load(0).re
1621 + self.twiddle7.re * x116p.re
1622 + self.twiddle3.re * x215p.re
1623 + self.twiddle4.re * x314p.re
1624 + self.twiddle6.re * x413p.re
1625 + self.twiddle1.re * x512p.re
1626 + self.twiddle8.re * x611p.re
1627 + self.twiddle2.re * x710p.re
1628 + self.twiddle5.re * x89p.re;
1629 let b710re_b = self.twiddle7.im * x116n.im
1630 + -self.twiddle3.im * x215n.im
1631 + self.twiddle4.im * x314n.im
1632 + -self.twiddle6.im * x413n.im
1633 + self.twiddle1.im * x512n.im
1634 + self.twiddle8.im * x611n.im
1635 + -self.twiddle2.im * x710n.im
1636 + self.twiddle5.im * x89n.im;
1637 let b89re_a = buffer.load(0).re
1638 + self.twiddle8.re * x116p.re
1639 + self.twiddle1.re * x215p.re
1640 + self.twiddle7.re * x314p.re
1641 + self.twiddle2.re * x413p.re
1642 + self.twiddle6.re * x512p.re
1643 + self.twiddle3.re * x611p.re
1644 + self.twiddle5.re * x710p.re
1645 + self.twiddle4.re * x89p.re;
1646 let b89re_b = self.twiddle8.im * x116n.im
1647 + -self.twiddle1.im * x215n.im
1648 + self.twiddle7.im * x314n.im
1649 + -self.twiddle2.im * x413n.im
1650 + self.twiddle6.im * x512n.im
1651 + -self.twiddle3.im * x611n.im
1652 + self.twiddle5.im * x710n.im
1653 + -self.twiddle4.im * x89n.im;
1654
1655 let b116im_a = buffer.load(0).im
1656 + self.twiddle1.re * x116p.im
1657 + self.twiddle2.re * x215p.im
1658 + self.twiddle3.re * x314p.im
1659 + self.twiddle4.re * x413p.im
1660 + self.twiddle5.re * x512p.im
1661 + self.twiddle6.re * x611p.im
1662 + self.twiddle7.re * x710p.im
1663 + self.twiddle8.re * x89p.im;
1664 let b116im_b = self.twiddle1.im * x116n.re
1665 + self.twiddle2.im * x215n.re
1666 + self.twiddle3.im * x314n.re
1667 + self.twiddle4.im * x413n.re
1668 + self.twiddle5.im * x512n.re
1669 + self.twiddle6.im * x611n.re
1670 + self.twiddle7.im * x710n.re
1671 + self.twiddle8.im * x89n.re;
1672 let b215im_a = buffer.load(0).im
1673 + self.twiddle2.re * x116p.im
1674 + self.twiddle4.re * x215p.im
1675 + self.twiddle6.re * x314p.im
1676 + self.twiddle8.re * x413p.im
1677 + self.twiddle7.re * x512p.im
1678 + self.twiddle5.re * x611p.im
1679 + self.twiddle3.re * x710p.im
1680 + self.twiddle1.re * x89p.im;
1681 let b215im_b = self.twiddle2.im * x116n.re
1682 + self.twiddle4.im * x215n.re
1683 + self.twiddle6.im * x314n.re
1684 + self.twiddle8.im * x413n.re
1685 + -self.twiddle7.im * x512n.re
1686 + -self.twiddle5.im * x611n.re
1687 + -self.twiddle3.im * x710n.re
1688 + -self.twiddle1.im * x89n.re;
1689 let b314im_a = buffer.load(0).im
1690 + self.twiddle3.re * x116p.im
1691 + self.twiddle6.re * x215p.im
1692 + self.twiddle8.re * x314p.im
1693 + self.twiddle5.re * x413p.im
1694 + self.twiddle2.re * x512p.im
1695 + self.twiddle1.re * x611p.im
1696 + self.twiddle4.re * x710p.im
1697 + self.twiddle7.re * x89p.im;
1698 let b314im_b = self.twiddle3.im * x116n.re
1699 + self.twiddle6.im * x215n.re
1700 + -self.twiddle8.im * x314n.re
1701 + -self.twiddle5.im * x413n.re
1702 + -self.twiddle2.im * x512n.re
1703 + self.twiddle1.im * x611n.re
1704 + self.twiddle4.im * x710n.re
1705 + self.twiddle7.im * x89n.re;
1706 let b413im_a = buffer.load(0).im
1707 + self.twiddle4.re * x116p.im
1708 + self.twiddle8.re * x215p.im
1709 + self.twiddle5.re * x314p.im
1710 + self.twiddle1.re * x413p.im
1711 + self.twiddle3.re * x512p.im
1712 + self.twiddle7.re * x611p.im
1713 + self.twiddle6.re * x710p.im
1714 + self.twiddle2.re * x89p.im;
1715 let b413im_b = self.twiddle4.im * x116n.re
1716 + self.twiddle8.im * x215n.re
1717 + -self.twiddle5.im * x314n.re
1718 + -self.twiddle1.im * x413n.re
1719 + self.twiddle3.im * x512n.re
1720 + self.twiddle7.im * x611n.re
1721 + -self.twiddle6.im * x710n.re
1722 + -self.twiddle2.im * x89n.re;
1723 let b512im_a = buffer.load(0).im
1724 + self.twiddle5.re * x116p.im
1725 + self.twiddle7.re * x215p.im
1726 + self.twiddle2.re * x314p.im
1727 + self.twiddle3.re * x413p.im
1728 + self.twiddle8.re * x512p.im
1729 + self.twiddle4.re * x611p.im
1730 + self.twiddle1.re * x710p.im
1731 + self.twiddle6.re * x89p.im;
1732 let b512im_b = self.twiddle5.im * x116n.re
1733 + -self.twiddle7.im * x215n.re
1734 + -self.twiddle2.im * x314n.re
1735 + self.twiddle3.im * x413n.re
1736 + self.twiddle8.im * x512n.re
1737 + -self.twiddle4.im * x611n.re
1738 + self.twiddle1.im * x710n.re
1739 + self.twiddle6.im * x89n.re;
1740 let b611im_a = buffer.load(0).im
1741 + self.twiddle6.re * x116p.im
1742 + self.twiddle5.re * x215p.im
1743 + self.twiddle1.re * x314p.im
1744 + self.twiddle7.re * x413p.im
1745 + self.twiddle4.re * x512p.im
1746 + self.twiddle2.re * x611p.im
1747 + self.twiddle8.re * x710p.im
1748 + self.twiddle3.re * x89p.im;
1749 let b611im_b = self.twiddle6.im * x116n.re
1750 + -self.twiddle5.im * x215n.re
1751 + self.twiddle1.im * x314n.re
1752 + self.twiddle7.im * x413n.re
1753 + -self.twiddle4.im * x512n.re
1754 + self.twiddle2.im * x611n.re
1755 + self.twiddle8.im * x710n.re
1756 + -self.twiddle3.im * x89n.re;
1757 let b710im_a = buffer.load(0).im
1758 + self.twiddle7.re * x116p.im
1759 + self.twiddle3.re * x215p.im
1760 + self.twiddle4.re * x314p.im
1761 + self.twiddle6.re * x413p.im
1762 + self.twiddle1.re * x512p.im
1763 + self.twiddle8.re * x611p.im
1764 + self.twiddle2.re * x710p.im
1765 + self.twiddle5.re * x89p.im;
1766 let b710im_b = self.twiddle7.im * x116n.re
1767 + -self.twiddle3.im * x215n.re
1768 + self.twiddle4.im * x314n.re
1769 + -self.twiddle6.im * x413n.re
1770 + self.twiddle1.im * x512n.re
1771 + self.twiddle8.im * x611n.re
1772 + -self.twiddle2.im * x710n.re
1773 + self.twiddle5.im * x89n.re;
1774 let b89im_a = buffer.load(0).im
1775 + self.twiddle8.re * x116p.im
1776 + self.twiddle1.re * x215p.im
1777 + self.twiddle7.re * x314p.im
1778 + self.twiddle2.re * x413p.im
1779 + self.twiddle6.re * x512p.im
1780 + self.twiddle3.re * x611p.im
1781 + self.twiddle5.re * x710p.im
1782 + self.twiddle4.re * x89p.im;
1783 let b89im_b = self.twiddle8.im * x116n.re
1784 + -self.twiddle1.im * x215n.re
1785 + self.twiddle7.im * x314n.re
1786 + -self.twiddle2.im * x413n.re
1787 + self.twiddle6.im * x512n.re
1788 + -self.twiddle3.im * x611n.re
1789 + self.twiddle5.im * x710n.re
1790 + -self.twiddle4.im * x89n.re;
1791
1792 let out1re = b116re_a - b116re_b;
1793 let out1im = b116im_a + b116im_b;
1794 let out2re = b215re_a - b215re_b;
1795 let out2im = b215im_a + b215im_b;
1796 let out3re = b314re_a - b314re_b;
1797 let out3im = b314im_a + b314im_b;
1798 let out4re = b413re_a - b413re_b;
1799 let out4im = b413im_a + b413im_b;
1800 let out5re = b512re_a - b512re_b;
1801 let out5im = b512im_a + b512im_b;
1802 let out6re = b611re_a - b611re_b;
1803 let out6im = b611im_a + b611im_b;
1804 let out7re = b710re_a - b710re_b;
1805 let out7im = b710im_a + b710im_b;
1806 let out8re = b89re_a - b89re_b;
1807 let out8im = b89im_a + b89im_b;
1808 let out9re = b89re_a + b89re_b;
1809 let out9im = b89im_a - b89im_b;
1810 let out10re = b710re_a + b710re_b;
1811 let out10im = b710im_a - b710im_b;
1812 let out11re = b611re_a + b611re_b;
1813 let out11im = b611im_a - b611im_b;
1814 let out12re = b512re_a + b512re_b;
1815 let out12im = b512im_a - b512im_b;
1816 let out13re = b413re_a + b413re_b;
1817 let out13im = b413im_a - b413im_b;
1818 let out14re = b314re_a + b314re_b;
1819 let out14im = b314im_a - b314im_b;
1820 let out15re = b215re_a + b215re_b;
1821 let out15im = b215im_a - b215im_b;
1822 let out16re = b116re_a + b116re_b;
1823 let out16im = b116im_a - b116im_b;
1824 buffer.store(sum, 0);
1825 buffer.store(
1826 Complex {
1827 re: out1re,
1828 im: out1im,
1829 },
1830 1,
1831 );
1832 buffer.store(
1833 Complex {
1834 re: out2re,
1835 im: out2im,
1836 },
1837 2,
1838 );
1839 buffer.store(
1840 Complex {
1841 re: out3re,
1842 im: out3im,
1843 },
1844 3,
1845 );
1846 buffer.store(
1847 Complex {
1848 re: out4re,
1849 im: out4im,
1850 },
1851 4,
1852 );
1853 buffer.store(
1854 Complex {
1855 re: out5re,
1856 im: out5im,
1857 },
1858 5,
1859 );
1860 buffer.store(
1861 Complex {
1862 re: out6re,
1863 im: out6im,
1864 },
1865 6,
1866 );
1867 buffer.store(
1868 Complex {
1869 re: out7re,
1870 im: out7im,
1871 },
1872 7,
1873 );
1874 buffer.store(
1875 Complex {
1876 re: out8re,
1877 im: out8im,
1878 },
1879 8,
1880 );
1881 buffer.store(
1882 Complex {
1883 re: out9re,
1884 im: out9im,
1885 },
1886 9,
1887 );
1888 buffer.store(
1889 Complex {
1890 re: out10re,
1891 im: out10im,
1892 },
1893 10,
1894 );
1895 buffer.store(
1896 Complex {
1897 re: out11re,
1898 im: out11im,
1899 },
1900 11,
1901 );
1902 buffer.store(
1903 Complex {
1904 re: out12re,
1905 im: out12im,
1906 },
1907 12,
1908 );
1909 buffer.store(
1910 Complex {
1911 re: out13re,
1912 im: out13im,
1913 },
1914 13,
1915 );
1916 buffer.store(
1917 Complex {
1918 re: out14re,
1919 im: out14im,
1920 },
1921 14,
1922 );
1923 buffer.store(
1924 Complex {
1925 re: out15re,
1926 im: out15im,
1927 },
1928 15,
1929 );
1930 buffer.store(
1931 Complex {
1932 re: out16re,
1933 im: out16im,
1934 },
1935 16,
1936 );
1937 }
1938 }
1939
1940 pub struct Butterfly19<T> {
1941 twiddle1: Complex<T>,
1942 twiddle2: Complex<T>,
1943 twiddle3: Complex<T>,
1944 twiddle4: Complex<T>,
1945 twiddle5: Complex<T>,
1946 twiddle6: Complex<T>,
1947 twiddle7: Complex<T>,
1948 twiddle8: Complex<T>,
1949 twiddle9: Complex<T>,
1950 direction: FftDirection,
1951 }
1952 boilerplate_fft_butterfly!(Butterfly19, 19, |this: &Butterfly19<_>| this.direction);
1953 impl<T: FftNum> Butterfly19<T> {
1954 pub fn new(direction: FftDirection) -> Self {
1955 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 19, direction);
1956 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 19, direction);
1957 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 19, direction);
1958 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 19, direction);
1959 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 19, direction);
1960 let twiddle6: Complex<T> = twiddles::compute_twiddle(6, 19, direction);
1961 let twiddle7: Complex<T> = twiddles::compute_twiddle(7, 19, direction);
1962 let twiddle8: Complex<T> = twiddles::compute_twiddle(8, 19, direction);
1963 let twiddle9: Complex<T> = twiddles::compute_twiddle(9, 19, direction);
1964 Self {
1965 twiddle1,
1966 twiddle2,
1967 twiddle3,
1968 twiddle4,
1969 twiddle5,
1970 twiddle6,
1971 twiddle7,
1972 twiddle8,
1973 twiddle9,
1974 direction,
1975 }
1976 }
1977
1978 #[inline(never)]
1979 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
1980 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
1981 // However, instead of doing it by hand the actual code is autogenerated
1982 // with the `genbutterflies.py` script in the `tools` directory.
1983 let x118p = buffer.load(1) + buffer.load(18);
1984 let x118n = buffer.load(1) - buffer.load(18);
1985 let x217p = buffer.load(2) + buffer.load(17);
1986 let x217n = buffer.load(2) - buffer.load(17);
1987 let x316p = buffer.load(3) + buffer.load(16);
1988 let x316n = buffer.load(3) - buffer.load(16);
1989 let x415p = buffer.load(4) + buffer.load(15);
1990 let x415n = buffer.load(4) - buffer.load(15);
1991 let x514p = buffer.load(5) + buffer.load(14);
1992 let x514n = buffer.load(5) - buffer.load(14);
1993 let x613p = buffer.load(6) + buffer.load(13);
1994 let x613n = buffer.load(6) - buffer.load(13);
1995 let x712p = buffer.load(7) + buffer.load(12);
1996 let x712n = buffer.load(7) - buffer.load(12);
1997 let x811p = buffer.load(8) + buffer.load(11);
1998 let x811n = buffer.load(8) - buffer.load(11);
1999 let x910p = buffer.load(9) + buffer.load(10);
2000 let x910n = buffer.load(9) - buffer.load(10);
2001 let sum =
2002 buffer.load(0) + x118p + x217p + x316p + x415p + x514p + x613p + x712p + x811p + x910p;
2003 let b118re_a = buffer.load(0).re
2004 + self.twiddle1.re * x118p.re
2005 + self.twiddle2.re * x217p.re
2006 + self.twiddle3.re * x316p.re
2007 + self.twiddle4.re * x415p.re
2008 + self.twiddle5.re * x514p.re
2009 + self.twiddle6.re * x613p.re
2010 + self.twiddle7.re * x712p.re
2011 + self.twiddle8.re * x811p.re
2012 + self.twiddle9.re * x910p.re;
2013 let b118re_b = self.twiddle1.im * x118n.im
2014 + self.twiddle2.im * x217n.im
2015 + self.twiddle3.im * x316n.im
2016 + self.twiddle4.im * x415n.im
2017 + self.twiddle5.im * x514n.im
2018 + self.twiddle6.im * x613n.im
2019 + self.twiddle7.im * x712n.im
2020 + self.twiddle8.im * x811n.im
2021 + self.twiddle9.im * x910n.im;
2022 let b217re_a = buffer.load(0).re
2023 + self.twiddle2.re * x118p.re
2024 + self.twiddle4.re * x217p.re
2025 + self.twiddle6.re * x316p.re
2026 + self.twiddle8.re * x415p.re
2027 + self.twiddle9.re * x514p.re
2028 + self.twiddle7.re * x613p.re
2029 + self.twiddle5.re * x712p.re
2030 + self.twiddle3.re * x811p.re
2031 + self.twiddle1.re * x910p.re;
2032 let b217re_b = self.twiddle2.im * x118n.im
2033 + self.twiddle4.im * x217n.im
2034 + self.twiddle6.im * x316n.im
2035 + self.twiddle8.im * x415n.im
2036 + -self.twiddle9.im * x514n.im
2037 + -self.twiddle7.im * x613n.im
2038 + -self.twiddle5.im * x712n.im
2039 + -self.twiddle3.im * x811n.im
2040 + -self.twiddle1.im * x910n.im;
2041 let b316re_a = buffer.load(0).re
2042 + self.twiddle3.re * x118p.re
2043 + self.twiddle6.re * x217p.re
2044 + self.twiddle9.re * x316p.re
2045 + self.twiddle7.re * x415p.re
2046 + self.twiddle4.re * x514p.re
2047 + self.twiddle1.re * x613p.re
2048 + self.twiddle2.re * x712p.re
2049 + self.twiddle5.re * x811p.re
2050 + self.twiddle8.re * x910p.re;
2051 let b316re_b = self.twiddle3.im * x118n.im
2052 + self.twiddle6.im * x217n.im
2053 + self.twiddle9.im * x316n.im
2054 + -self.twiddle7.im * x415n.im
2055 + -self.twiddle4.im * x514n.im
2056 + -self.twiddle1.im * x613n.im
2057 + self.twiddle2.im * x712n.im
2058 + self.twiddle5.im * x811n.im
2059 + self.twiddle8.im * x910n.im;
2060 let b415re_a = buffer.load(0).re
2061 + self.twiddle4.re * x118p.re
2062 + self.twiddle8.re * x217p.re
2063 + self.twiddle7.re * x316p.re
2064 + self.twiddle3.re * x415p.re
2065 + self.twiddle1.re * x514p.re
2066 + self.twiddle5.re * x613p.re
2067 + self.twiddle9.re * x712p.re
2068 + self.twiddle6.re * x811p.re
2069 + self.twiddle2.re * x910p.re;
2070 let b415re_b = self.twiddle4.im * x118n.im
2071 + self.twiddle8.im * x217n.im
2072 + -self.twiddle7.im * x316n.im
2073 + -self.twiddle3.im * x415n.im
2074 + self.twiddle1.im * x514n.im
2075 + self.twiddle5.im * x613n.im
2076 + self.twiddle9.im * x712n.im
2077 + -self.twiddle6.im * x811n.im
2078 + -self.twiddle2.im * x910n.im;
2079 let b514re_a = buffer.load(0).re
2080 + self.twiddle5.re * x118p.re
2081 + self.twiddle9.re * x217p.re
2082 + self.twiddle4.re * x316p.re
2083 + self.twiddle1.re * x415p.re
2084 + self.twiddle6.re * x514p.re
2085 + self.twiddle8.re * x613p.re
2086 + self.twiddle3.re * x712p.re
2087 + self.twiddle2.re * x811p.re
2088 + self.twiddle7.re * x910p.re;
2089 let b514re_b = self.twiddle5.im * x118n.im
2090 + -self.twiddle9.im * x217n.im
2091 + -self.twiddle4.im * x316n.im
2092 + self.twiddle1.im * x415n.im
2093 + self.twiddle6.im * x514n.im
2094 + -self.twiddle8.im * x613n.im
2095 + -self.twiddle3.im * x712n.im
2096 + self.twiddle2.im * x811n.im
2097 + self.twiddle7.im * x910n.im;
2098 let b613re_a = buffer.load(0).re
2099 + self.twiddle6.re * x118p.re
2100 + self.twiddle7.re * x217p.re
2101 + self.twiddle1.re * x316p.re
2102 + self.twiddle5.re * x415p.re
2103 + self.twiddle8.re * x514p.re
2104 + self.twiddle2.re * x613p.re
2105 + self.twiddle4.re * x712p.re
2106 + self.twiddle9.re * x811p.re
2107 + self.twiddle3.re * x910p.re;
2108 let b613re_b = self.twiddle6.im * x118n.im
2109 + -self.twiddle7.im * x217n.im
2110 + -self.twiddle1.im * x316n.im
2111 + self.twiddle5.im * x415n.im
2112 + -self.twiddle8.im * x514n.im
2113 + -self.twiddle2.im * x613n.im
2114 + self.twiddle4.im * x712n.im
2115 + -self.twiddle9.im * x811n.im
2116 + -self.twiddle3.im * x910n.im;
2117 let b712re_a = buffer.load(0).re
2118 + self.twiddle7.re * x118p.re
2119 + self.twiddle5.re * x217p.re
2120 + self.twiddle2.re * x316p.re
2121 + self.twiddle9.re * x415p.re
2122 + self.twiddle3.re * x514p.re
2123 + self.twiddle4.re * x613p.re
2124 + self.twiddle8.re * x712p.re
2125 + self.twiddle1.re * x811p.re
2126 + self.twiddle6.re * x910p.re;
2127 let b712re_b = self.twiddle7.im * x118n.im
2128 + -self.twiddle5.im * x217n.im
2129 + self.twiddle2.im * x316n.im
2130 + self.twiddle9.im * x415n.im
2131 + -self.twiddle3.im * x514n.im
2132 + self.twiddle4.im * x613n.im
2133 + -self.twiddle8.im * x712n.im
2134 + -self.twiddle1.im * x811n.im
2135 + self.twiddle6.im * x910n.im;
2136 let b811re_a = buffer.load(0).re
2137 + self.twiddle8.re * x118p.re
2138 + self.twiddle3.re * x217p.re
2139 + self.twiddle5.re * x316p.re
2140 + self.twiddle6.re * x415p.re
2141 + self.twiddle2.re * x514p.re
2142 + self.twiddle9.re * x613p.re
2143 + self.twiddle1.re * x712p.re
2144 + self.twiddle7.re * x811p.re
2145 + self.twiddle4.re * x910p.re;
2146 let b811re_b = self.twiddle8.im * x118n.im
2147 + -self.twiddle3.im * x217n.im
2148 + self.twiddle5.im * x316n.im
2149 + -self.twiddle6.im * x415n.im
2150 + self.twiddle2.im * x514n.im
2151 + -self.twiddle9.im * x613n.im
2152 + -self.twiddle1.im * x712n.im
2153 + self.twiddle7.im * x811n.im
2154 + -self.twiddle4.im * x910n.im;
2155 let b910re_a = buffer.load(0).re
2156 + self.twiddle9.re * x118p.re
2157 + self.twiddle1.re * x217p.re
2158 + self.twiddle8.re * x316p.re
2159 + self.twiddle2.re * x415p.re
2160 + self.twiddle7.re * x514p.re
2161 + self.twiddle3.re * x613p.re
2162 + self.twiddle6.re * x712p.re
2163 + self.twiddle4.re * x811p.re
2164 + self.twiddle5.re * x910p.re;
2165 let b910re_b = self.twiddle9.im * x118n.im
2166 + -self.twiddle1.im * x217n.im
2167 + self.twiddle8.im * x316n.im
2168 + -self.twiddle2.im * x415n.im
2169 + self.twiddle7.im * x514n.im
2170 + -self.twiddle3.im * x613n.im
2171 + self.twiddle6.im * x712n.im
2172 + -self.twiddle4.im * x811n.im
2173 + self.twiddle5.im * x910n.im;
2174
2175 let b118im_a = buffer.load(0).im
2176 + self.twiddle1.re * x118p.im
2177 + self.twiddle2.re * x217p.im
2178 + self.twiddle3.re * x316p.im
2179 + self.twiddle4.re * x415p.im
2180 + self.twiddle5.re * x514p.im
2181 + self.twiddle6.re * x613p.im
2182 + self.twiddle7.re * x712p.im
2183 + self.twiddle8.re * x811p.im
2184 + self.twiddle9.re * x910p.im;
2185 let b118im_b = self.twiddle1.im * x118n.re
2186 + self.twiddle2.im * x217n.re
2187 + self.twiddle3.im * x316n.re
2188 + self.twiddle4.im * x415n.re
2189 + self.twiddle5.im * x514n.re
2190 + self.twiddle6.im * x613n.re
2191 + self.twiddle7.im * x712n.re
2192 + self.twiddle8.im * x811n.re
2193 + self.twiddle9.im * x910n.re;
2194 let b217im_a = buffer.load(0).im
2195 + self.twiddle2.re * x118p.im
2196 + self.twiddle4.re * x217p.im
2197 + self.twiddle6.re * x316p.im
2198 + self.twiddle8.re * x415p.im
2199 + self.twiddle9.re * x514p.im
2200 + self.twiddle7.re * x613p.im
2201 + self.twiddle5.re * x712p.im
2202 + self.twiddle3.re * x811p.im
2203 + self.twiddle1.re * x910p.im;
2204 let b217im_b = self.twiddle2.im * x118n.re
2205 + self.twiddle4.im * x217n.re
2206 + self.twiddle6.im * x316n.re
2207 + self.twiddle8.im * x415n.re
2208 + -self.twiddle9.im * x514n.re
2209 + -self.twiddle7.im * x613n.re
2210 + -self.twiddle5.im * x712n.re
2211 + -self.twiddle3.im * x811n.re
2212 + -self.twiddle1.im * x910n.re;
2213 let b316im_a = buffer.load(0).im
2214 + self.twiddle3.re * x118p.im
2215 + self.twiddle6.re * x217p.im
2216 + self.twiddle9.re * x316p.im
2217 + self.twiddle7.re * x415p.im
2218 + self.twiddle4.re * x514p.im
2219 + self.twiddle1.re * x613p.im
2220 + self.twiddle2.re * x712p.im
2221 + self.twiddle5.re * x811p.im
2222 + self.twiddle8.re * x910p.im;
2223 let b316im_b = self.twiddle3.im * x118n.re
2224 + self.twiddle6.im * x217n.re
2225 + self.twiddle9.im * x316n.re
2226 + -self.twiddle7.im * x415n.re
2227 + -self.twiddle4.im * x514n.re
2228 + -self.twiddle1.im * x613n.re
2229 + self.twiddle2.im * x712n.re
2230 + self.twiddle5.im * x811n.re
2231 + self.twiddle8.im * x910n.re;
2232 let b415im_a = buffer.load(0).im
2233 + self.twiddle4.re * x118p.im
2234 + self.twiddle8.re * x217p.im
2235 + self.twiddle7.re * x316p.im
2236 + self.twiddle3.re * x415p.im
2237 + self.twiddle1.re * x514p.im
2238 + self.twiddle5.re * x613p.im
2239 + self.twiddle9.re * x712p.im
2240 + self.twiddle6.re * x811p.im
2241 + self.twiddle2.re * x910p.im;
2242 let b415im_b = self.twiddle4.im * x118n.re
2243 + self.twiddle8.im * x217n.re
2244 + -self.twiddle7.im * x316n.re
2245 + -self.twiddle3.im * x415n.re
2246 + self.twiddle1.im * x514n.re
2247 + self.twiddle5.im * x613n.re
2248 + self.twiddle9.im * x712n.re
2249 + -self.twiddle6.im * x811n.re
2250 + -self.twiddle2.im * x910n.re;
2251 let b514im_a = buffer.load(0).im
2252 + self.twiddle5.re * x118p.im
2253 + self.twiddle9.re * x217p.im
2254 + self.twiddle4.re * x316p.im
2255 + self.twiddle1.re * x415p.im
2256 + self.twiddle6.re * x514p.im
2257 + self.twiddle8.re * x613p.im
2258 + self.twiddle3.re * x712p.im
2259 + self.twiddle2.re * x811p.im
2260 + self.twiddle7.re * x910p.im;
2261 let b514im_b = self.twiddle5.im * x118n.re
2262 + -self.twiddle9.im * x217n.re
2263 + -self.twiddle4.im * x316n.re
2264 + self.twiddle1.im * x415n.re
2265 + self.twiddle6.im * x514n.re
2266 + -self.twiddle8.im * x613n.re
2267 + -self.twiddle3.im * x712n.re
2268 + self.twiddle2.im * x811n.re
2269 + self.twiddle7.im * x910n.re;
2270 let b613im_a = buffer.load(0).im
2271 + self.twiddle6.re * x118p.im
2272 + self.twiddle7.re * x217p.im
2273 + self.twiddle1.re * x316p.im
2274 + self.twiddle5.re * x415p.im
2275 + self.twiddle8.re * x514p.im
2276 + self.twiddle2.re * x613p.im
2277 + self.twiddle4.re * x712p.im
2278 + self.twiddle9.re * x811p.im
2279 + self.twiddle3.re * x910p.im;
2280 let b613im_b = self.twiddle6.im * x118n.re
2281 + -self.twiddle7.im * x217n.re
2282 + -self.twiddle1.im * x316n.re
2283 + self.twiddle5.im * x415n.re
2284 + -self.twiddle8.im * x514n.re
2285 + -self.twiddle2.im * x613n.re
2286 + self.twiddle4.im * x712n.re
2287 + -self.twiddle9.im * x811n.re
2288 + -self.twiddle3.im * x910n.re;
2289 let b712im_a = buffer.load(0).im
2290 + self.twiddle7.re * x118p.im
2291 + self.twiddle5.re * x217p.im
2292 + self.twiddle2.re * x316p.im
2293 + self.twiddle9.re * x415p.im
2294 + self.twiddle3.re * x514p.im
2295 + self.twiddle4.re * x613p.im
2296 + self.twiddle8.re * x712p.im
2297 + self.twiddle1.re * x811p.im
2298 + self.twiddle6.re * x910p.im;
2299 let b712im_b = self.twiddle7.im * x118n.re
2300 + -self.twiddle5.im * x217n.re
2301 + self.twiddle2.im * x316n.re
2302 + self.twiddle9.im * x415n.re
2303 + -self.twiddle3.im * x514n.re
2304 + self.twiddle4.im * x613n.re
2305 + -self.twiddle8.im * x712n.re
2306 + -self.twiddle1.im * x811n.re
2307 + self.twiddle6.im * x910n.re;
2308 let b811im_a = buffer.load(0).im
2309 + self.twiddle8.re * x118p.im
2310 + self.twiddle3.re * x217p.im
2311 + self.twiddle5.re * x316p.im
2312 + self.twiddle6.re * x415p.im
2313 + self.twiddle2.re * x514p.im
2314 + self.twiddle9.re * x613p.im
2315 + self.twiddle1.re * x712p.im
2316 + self.twiddle7.re * x811p.im
2317 + self.twiddle4.re * x910p.im;
2318 let b811im_b = self.twiddle8.im * x118n.re
2319 + -self.twiddle3.im * x217n.re
2320 + self.twiddle5.im * x316n.re
2321 + -self.twiddle6.im * x415n.re
2322 + self.twiddle2.im * x514n.re
2323 + -self.twiddle9.im * x613n.re
2324 + -self.twiddle1.im * x712n.re
2325 + self.twiddle7.im * x811n.re
2326 + -self.twiddle4.im * x910n.re;
2327 let b910im_a = buffer.load(0).im
2328 + self.twiddle9.re * x118p.im
2329 + self.twiddle1.re * x217p.im
2330 + self.twiddle8.re * x316p.im
2331 + self.twiddle2.re * x415p.im
2332 + self.twiddle7.re * x514p.im
2333 + self.twiddle3.re * x613p.im
2334 + self.twiddle6.re * x712p.im
2335 + self.twiddle4.re * x811p.im
2336 + self.twiddle5.re * x910p.im;
2337 let b910im_b = self.twiddle9.im * x118n.re
2338 + -self.twiddle1.im * x217n.re
2339 + self.twiddle8.im * x316n.re
2340 + -self.twiddle2.im * x415n.re
2341 + self.twiddle7.im * x514n.re
2342 + -self.twiddle3.im * x613n.re
2343 + self.twiddle6.im * x712n.re
2344 + -self.twiddle4.im * x811n.re
2345 + self.twiddle5.im * x910n.re;
2346
2347 let out1re = b118re_a - b118re_b;
2348 let out1im = b118im_a + b118im_b;
2349 let out2re = b217re_a - b217re_b;
2350 let out2im = b217im_a + b217im_b;
2351 let out3re = b316re_a - b316re_b;
2352 let out3im = b316im_a + b316im_b;
2353 let out4re = b415re_a - b415re_b;
2354 let out4im = b415im_a + b415im_b;
2355 let out5re = b514re_a - b514re_b;
2356 let out5im = b514im_a + b514im_b;
2357 let out6re = b613re_a - b613re_b;
2358 let out6im = b613im_a + b613im_b;
2359 let out7re = b712re_a - b712re_b;
2360 let out7im = b712im_a + b712im_b;
2361 let out8re = b811re_a - b811re_b;
2362 let out8im = b811im_a + b811im_b;
2363 let out9re = b910re_a - b910re_b;
2364 let out9im = b910im_a + b910im_b;
2365 let out10re = b910re_a + b910re_b;
2366 let out10im = b910im_a - b910im_b;
2367 let out11re = b811re_a + b811re_b;
2368 let out11im = b811im_a - b811im_b;
2369 let out12re = b712re_a + b712re_b;
2370 let out12im = b712im_a - b712im_b;
2371 let out13re = b613re_a + b613re_b;
2372 let out13im = b613im_a - b613im_b;
2373 let out14re = b514re_a + b514re_b;
2374 let out14im = b514im_a - b514im_b;
2375 let out15re = b415re_a + b415re_b;
2376 let out15im = b415im_a - b415im_b;
2377 let out16re = b316re_a + b316re_b;
2378 let out16im = b316im_a - b316im_b;
2379 let out17re = b217re_a + b217re_b;
2380 let out17im = b217im_a - b217im_b;
2381 let out18re = b118re_a + b118re_b;
2382 let out18im = b118im_a - b118im_b;
2383 buffer.store(sum, 0);
2384 buffer.store(
2385 Complex {
2386 re: out1re,
2387 im: out1im,
2388 },
2389 1,
2390 );
2391 buffer.store(
2392 Complex {
2393 re: out2re,
2394 im: out2im,
2395 },
2396 2,
2397 );
2398 buffer.store(
2399 Complex {
2400 re: out3re,
2401 im: out3im,
2402 },
2403 3,
2404 );
2405 buffer.store(
2406 Complex {
2407 re: out4re,
2408 im: out4im,
2409 },
2410 4,
2411 );
2412 buffer.store(
2413 Complex {
2414 re: out5re,
2415 im: out5im,
2416 },
2417 5,
2418 );
2419 buffer.store(
2420 Complex {
2421 re: out6re,
2422 im: out6im,
2423 },
2424 6,
2425 );
2426 buffer.store(
2427 Complex {
2428 re: out7re,
2429 im: out7im,
2430 },
2431 7,
2432 );
2433 buffer.store(
2434 Complex {
2435 re: out8re,
2436 im: out8im,
2437 },
2438 8,
2439 );
2440 buffer.store(
2441 Complex {
2442 re: out9re,
2443 im: out9im,
2444 },
2445 9,
2446 );
2447 buffer.store(
2448 Complex {
2449 re: out10re,
2450 im: out10im,
2451 },
2452 10,
2453 );
2454 buffer.store(
2455 Complex {
2456 re: out11re,
2457 im: out11im,
2458 },
2459 11,
2460 );
2461 buffer.store(
2462 Complex {
2463 re: out12re,
2464 im: out12im,
2465 },
2466 12,
2467 );
2468 buffer.store(
2469 Complex {
2470 re: out13re,
2471 im: out13im,
2472 },
2473 13,
2474 );
2475 buffer.store(
2476 Complex {
2477 re: out14re,
2478 im: out14im,
2479 },
2480 14,
2481 );
2482 buffer.store(
2483 Complex {
2484 re: out15re,
2485 im: out15im,
2486 },
2487 15,
2488 );
2489 buffer.store(
2490 Complex {
2491 re: out16re,
2492 im: out16im,
2493 },
2494 16,
2495 );
2496 buffer.store(
2497 Complex {
2498 re: out17re,
2499 im: out17im,
2500 },
2501 17,
2502 );
2503 buffer.store(
2504 Complex {
2505 re: out18re,
2506 im: out18im,
2507 },
2508 18,
2509 );
2510 }
2511 }
2512
2513 pub struct Butterfly23<T> {
2514 twiddle1: Complex<T>,
2515 twiddle2: Complex<T>,
2516 twiddle3: Complex<T>,
2517 twiddle4: Complex<T>,
2518 twiddle5: Complex<T>,
2519 twiddle6: Complex<T>,
2520 twiddle7: Complex<T>,
2521 twiddle8: Complex<T>,
2522 twiddle9: Complex<T>,
2523 twiddle10: Complex<T>,
2524 twiddle11: Complex<T>,
2525 direction: FftDirection,
2526 }
2527 boilerplate_fft_butterfly!(Butterfly23, 23, |this: &Butterfly23<_>| this.direction);
2528 impl<T: FftNum> Butterfly23<T> {
2529 pub fn new(direction: FftDirection) -> Self {
2530 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 23, direction);
2531 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 23, direction);
2532 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 23, direction);
2533 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 23, direction);
2534 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 23, direction);
2535 let twiddle6: Complex<T> = twiddles::compute_twiddle(6, 23, direction);
2536 let twiddle7: Complex<T> = twiddles::compute_twiddle(7, 23, direction);
2537 let twiddle8: Complex<T> = twiddles::compute_twiddle(8, 23, direction);
2538 let twiddle9: Complex<T> = twiddles::compute_twiddle(9, 23, direction);
2539 let twiddle10: Complex<T> = twiddles::compute_twiddle(10, 23, direction);
2540 let twiddle11: Complex<T> = twiddles::compute_twiddle(11, 23, direction);
2541 Self {
2542 twiddle1,
2543 twiddle2,
2544 twiddle3,
2545 twiddle4,
2546 twiddle5,
2547 twiddle6,
2548 twiddle7,
2549 twiddle8,
2550 twiddle9,
2551 twiddle10,
2552 twiddle11,
2553 direction,
2554 }
2555 }
2556
2557 #[inline(never)]
2558 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
2559 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
2560 // However, instead of doing it by hand the actual code is autogenerated
2561 // with the `genbutterflies.py` script in the `tools` directory.
2562 let x122p = buffer.load(1) + buffer.load(22);
2563 let x122n = buffer.load(1) - buffer.load(22);
2564 let x221p = buffer.load(2) + buffer.load(21);
2565 let x221n = buffer.load(2) - buffer.load(21);
2566 let x320p = buffer.load(3) + buffer.load(20);
2567 let x320n = buffer.load(3) - buffer.load(20);
2568 let x419p = buffer.load(4) + buffer.load(19);
2569 let x419n = buffer.load(4) - buffer.load(19);
2570 let x518p = buffer.load(5) + buffer.load(18);
2571 let x518n = buffer.load(5) - buffer.load(18);
2572 let x617p = buffer.load(6) + buffer.load(17);
2573 let x617n = buffer.load(6) - buffer.load(17);
2574 let x716p = buffer.load(7) + buffer.load(16);
2575 let x716n = buffer.load(7) - buffer.load(16);
2576 let x815p = buffer.load(8) + buffer.load(15);
2577 let x815n = buffer.load(8) - buffer.load(15);
2578 let x914p = buffer.load(9) + buffer.load(14);
2579 let x914n = buffer.load(9) - buffer.load(14);
2580 let x1013p = buffer.load(10) + buffer.load(13);
2581 let x1013n = buffer.load(10) - buffer.load(13);
2582 let x1112p = buffer.load(11) + buffer.load(12);
2583 let x1112n = buffer.load(11) - buffer.load(12);
2584 let sum = buffer.load(0)
2585 + x122p
2586 + x221p
2587 + x320p
2588 + x419p
2589 + x518p
2590 + x617p
2591 + x716p
2592 + x815p
2593 + x914p
2594 + x1013p
2595 + x1112p;
2596 let b122re_a = buffer.load(0).re
2597 + self.twiddle1.re * x122p.re
2598 + self.twiddle2.re * x221p.re
2599 + self.twiddle3.re * x320p.re
2600 + self.twiddle4.re * x419p.re
2601 + self.twiddle5.re * x518p.re
2602 + self.twiddle6.re * x617p.re
2603 + self.twiddle7.re * x716p.re
2604 + self.twiddle8.re * x815p.re
2605 + self.twiddle9.re * x914p.re
2606 + self.twiddle10.re * x1013p.re
2607 + self.twiddle11.re * x1112p.re;
2608 let b122re_b = self.twiddle1.im * x122n.im
2609 + self.twiddle2.im * x221n.im
2610 + self.twiddle3.im * x320n.im
2611 + self.twiddle4.im * x419n.im
2612 + self.twiddle5.im * x518n.im
2613 + self.twiddle6.im * x617n.im
2614 + self.twiddle7.im * x716n.im
2615 + self.twiddle8.im * x815n.im
2616 + self.twiddle9.im * x914n.im
2617 + self.twiddle10.im * x1013n.im
2618 + self.twiddle11.im * x1112n.im;
2619 let b221re_a = buffer.load(0).re
2620 + self.twiddle2.re * x122p.re
2621 + self.twiddle4.re * x221p.re
2622 + self.twiddle6.re * x320p.re
2623 + self.twiddle8.re * x419p.re
2624 + self.twiddle10.re * x518p.re
2625 + self.twiddle11.re * x617p.re
2626 + self.twiddle9.re * x716p.re
2627 + self.twiddle7.re * x815p.re
2628 + self.twiddle5.re * x914p.re
2629 + self.twiddle3.re * x1013p.re
2630 + self.twiddle1.re * x1112p.re;
2631 let b221re_b = self.twiddle2.im * x122n.im
2632 + self.twiddle4.im * x221n.im
2633 + self.twiddle6.im * x320n.im
2634 + self.twiddle8.im * x419n.im
2635 + self.twiddle10.im * x518n.im
2636 + -self.twiddle11.im * x617n.im
2637 + -self.twiddle9.im * x716n.im
2638 + -self.twiddle7.im * x815n.im
2639 + -self.twiddle5.im * x914n.im
2640 + -self.twiddle3.im * x1013n.im
2641 + -self.twiddle1.im * x1112n.im;
2642 let b320re_a = buffer.load(0).re
2643 + self.twiddle3.re * x122p.re
2644 + self.twiddle6.re * x221p.re
2645 + self.twiddle9.re * x320p.re
2646 + self.twiddle11.re * x419p.re
2647 + self.twiddle8.re * x518p.re
2648 + self.twiddle5.re * x617p.re
2649 + self.twiddle2.re * x716p.re
2650 + self.twiddle1.re * x815p.re
2651 + self.twiddle4.re * x914p.re
2652 + self.twiddle7.re * x1013p.re
2653 + self.twiddle10.re * x1112p.re;
2654 let b320re_b = self.twiddle3.im * x122n.im
2655 + self.twiddle6.im * x221n.im
2656 + self.twiddle9.im * x320n.im
2657 + -self.twiddle11.im * x419n.im
2658 + -self.twiddle8.im * x518n.im
2659 + -self.twiddle5.im * x617n.im
2660 + -self.twiddle2.im * x716n.im
2661 + self.twiddle1.im * x815n.im
2662 + self.twiddle4.im * x914n.im
2663 + self.twiddle7.im * x1013n.im
2664 + self.twiddle10.im * x1112n.im;
2665 let b419re_a = buffer.load(0).re
2666 + self.twiddle4.re * x122p.re
2667 + self.twiddle8.re * x221p.re
2668 + self.twiddle11.re * x320p.re
2669 + self.twiddle7.re * x419p.re
2670 + self.twiddle3.re * x518p.re
2671 + self.twiddle1.re * x617p.re
2672 + self.twiddle5.re * x716p.re
2673 + self.twiddle9.re * x815p.re
2674 + self.twiddle10.re * x914p.re
2675 + self.twiddle6.re * x1013p.re
2676 + self.twiddle2.re * x1112p.re;
2677 let b419re_b = self.twiddle4.im * x122n.im
2678 + self.twiddle8.im * x221n.im
2679 + -self.twiddle11.im * x320n.im
2680 + -self.twiddle7.im * x419n.im
2681 + -self.twiddle3.im * x518n.im
2682 + self.twiddle1.im * x617n.im
2683 + self.twiddle5.im * x716n.im
2684 + self.twiddle9.im * x815n.im
2685 + -self.twiddle10.im * x914n.im
2686 + -self.twiddle6.im * x1013n.im
2687 + -self.twiddle2.im * x1112n.im;
2688 let b518re_a = buffer.load(0).re
2689 + self.twiddle5.re * x122p.re
2690 + self.twiddle10.re * x221p.re
2691 + self.twiddle8.re * x320p.re
2692 + self.twiddle3.re * x419p.re
2693 + self.twiddle2.re * x518p.re
2694 + self.twiddle7.re * x617p.re
2695 + self.twiddle11.re * x716p.re
2696 + self.twiddle6.re * x815p.re
2697 + self.twiddle1.re * x914p.re
2698 + self.twiddle4.re * x1013p.re
2699 + self.twiddle9.re * x1112p.re;
2700 let b518re_b = self.twiddle5.im * x122n.im
2701 + self.twiddle10.im * x221n.im
2702 + -self.twiddle8.im * x320n.im
2703 + -self.twiddle3.im * x419n.im
2704 + self.twiddle2.im * x518n.im
2705 + self.twiddle7.im * x617n.im
2706 + -self.twiddle11.im * x716n.im
2707 + -self.twiddle6.im * x815n.im
2708 + -self.twiddle1.im * x914n.im
2709 + self.twiddle4.im * x1013n.im
2710 + self.twiddle9.im * x1112n.im;
2711 let b617re_a = buffer.load(0).re
2712 + self.twiddle6.re * x122p.re
2713 + self.twiddle11.re * x221p.re
2714 + self.twiddle5.re * x320p.re
2715 + self.twiddle1.re * x419p.re
2716 + self.twiddle7.re * x518p.re
2717 + self.twiddle10.re * x617p.re
2718 + self.twiddle4.re * x716p.re
2719 + self.twiddle2.re * x815p.re
2720 + self.twiddle8.re * x914p.re
2721 + self.twiddle9.re * x1013p.re
2722 + self.twiddle3.re * x1112p.re;
2723 let b617re_b = self.twiddle6.im * x122n.im
2724 + -self.twiddle11.im * x221n.im
2725 + -self.twiddle5.im * x320n.im
2726 + self.twiddle1.im * x419n.im
2727 + self.twiddle7.im * x518n.im
2728 + -self.twiddle10.im * x617n.im
2729 + -self.twiddle4.im * x716n.im
2730 + self.twiddle2.im * x815n.im
2731 + self.twiddle8.im * x914n.im
2732 + -self.twiddle9.im * x1013n.im
2733 + -self.twiddle3.im * x1112n.im;
2734 let b716re_a = buffer.load(0).re
2735 + self.twiddle7.re * x122p.re
2736 + self.twiddle9.re * x221p.re
2737 + self.twiddle2.re * x320p.re
2738 + self.twiddle5.re * x419p.re
2739 + self.twiddle11.re * x518p.re
2740 + self.twiddle4.re * x617p.re
2741 + self.twiddle3.re * x716p.re
2742 + self.twiddle10.re * x815p.re
2743 + self.twiddle6.re * x914p.re
2744 + self.twiddle1.re * x1013p.re
2745 + self.twiddle8.re * x1112p.re;
2746 let b716re_b = self.twiddle7.im * x122n.im
2747 + -self.twiddle9.im * x221n.im
2748 + -self.twiddle2.im * x320n.im
2749 + self.twiddle5.im * x419n.im
2750 + -self.twiddle11.im * x518n.im
2751 + -self.twiddle4.im * x617n.im
2752 + self.twiddle3.im * x716n.im
2753 + self.twiddle10.im * x815n.im
2754 + -self.twiddle6.im * x914n.im
2755 + self.twiddle1.im * x1013n.im
2756 + self.twiddle8.im * x1112n.im;
2757 let b815re_a = buffer.load(0).re
2758 + self.twiddle8.re * x122p.re
2759 + self.twiddle7.re * x221p.re
2760 + self.twiddle1.re * x320p.re
2761 + self.twiddle9.re * x419p.re
2762 + self.twiddle6.re * x518p.re
2763 + self.twiddle2.re * x617p.re
2764 + self.twiddle10.re * x716p.re
2765 + self.twiddle5.re * x815p.re
2766 + self.twiddle3.re * x914p.re
2767 + self.twiddle11.re * x1013p.re
2768 + self.twiddle4.re * x1112p.re;
2769 let b815re_b = self.twiddle8.im * x122n.im
2770 + -self.twiddle7.im * x221n.im
2771 + self.twiddle1.im * x320n.im
2772 + self.twiddle9.im * x419n.im
2773 + -self.twiddle6.im * x518n.im
2774 + self.twiddle2.im * x617n.im
2775 + self.twiddle10.im * x716n.im
2776 + -self.twiddle5.im * x815n.im
2777 + self.twiddle3.im * x914n.im
2778 + self.twiddle11.im * x1013n.im
2779 + -self.twiddle4.im * x1112n.im;
2780 let b914re_a = buffer.load(0).re
2781 + self.twiddle9.re * x122p.re
2782 + self.twiddle5.re * x221p.re
2783 + self.twiddle4.re * x320p.re
2784 + self.twiddle10.re * x419p.re
2785 + self.twiddle1.re * x518p.re
2786 + self.twiddle8.re * x617p.re
2787 + self.twiddle6.re * x716p.re
2788 + self.twiddle3.re * x815p.re
2789 + self.twiddle11.re * x914p.re
2790 + self.twiddle2.re * x1013p.re
2791 + self.twiddle7.re * x1112p.re;
2792 let b914re_b = self.twiddle9.im * x122n.im
2793 + -self.twiddle5.im * x221n.im
2794 + self.twiddle4.im * x320n.im
2795 + -self.twiddle10.im * x419n.im
2796 + -self.twiddle1.im * x518n.im
2797 + self.twiddle8.im * x617n.im
2798 + -self.twiddle6.im * x716n.im
2799 + self.twiddle3.im * x815n.im
2800 + -self.twiddle11.im * x914n.im
2801 + -self.twiddle2.im * x1013n.im
2802 + self.twiddle7.im * x1112n.im;
2803 let b1013re_a = buffer.load(0).re
2804 + self.twiddle10.re * x122p.re
2805 + self.twiddle3.re * x221p.re
2806 + self.twiddle7.re * x320p.re
2807 + self.twiddle6.re * x419p.re
2808 + self.twiddle4.re * x518p.re
2809 + self.twiddle9.re * x617p.re
2810 + self.twiddle1.re * x716p.re
2811 + self.twiddle11.re * x815p.re
2812 + self.twiddle2.re * x914p.re
2813 + self.twiddle8.re * x1013p.re
2814 + self.twiddle5.re * x1112p.re;
2815 let b1013re_b = self.twiddle10.im * x122n.im
2816 + -self.twiddle3.im * x221n.im
2817 + self.twiddle7.im * x320n.im
2818 + -self.twiddle6.im * x419n.im
2819 + self.twiddle4.im * x518n.im
2820 + -self.twiddle9.im * x617n.im
2821 + self.twiddle1.im * x716n.im
2822 + self.twiddle11.im * x815n.im
2823 + -self.twiddle2.im * x914n.im
2824 + self.twiddle8.im * x1013n.im
2825 + -self.twiddle5.im * x1112n.im;
2826 let b1112re_a = buffer.load(0).re
2827 + self.twiddle11.re * x122p.re
2828 + self.twiddle1.re * x221p.re
2829 + self.twiddle10.re * x320p.re
2830 + self.twiddle2.re * x419p.re
2831 + self.twiddle9.re * x518p.re
2832 + self.twiddle3.re * x617p.re
2833 + self.twiddle8.re * x716p.re
2834 + self.twiddle4.re * x815p.re
2835 + self.twiddle7.re * x914p.re
2836 + self.twiddle5.re * x1013p.re
2837 + self.twiddle6.re * x1112p.re;
2838 let b1112re_b = self.twiddle11.im * x122n.im
2839 + -self.twiddle1.im * x221n.im
2840 + self.twiddle10.im * x320n.im
2841 + -self.twiddle2.im * x419n.im
2842 + self.twiddle9.im * x518n.im
2843 + -self.twiddle3.im * x617n.im
2844 + self.twiddle8.im * x716n.im
2845 + -self.twiddle4.im * x815n.im
2846 + self.twiddle7.im * x914n.im
2847 + -self.twiddle5.im * x1013n.im
2848 + self.twiddle6.im * x1112n.im;
2849
2850 let b122im_a = buffer.load(0).im
2851 + self.twiddle1.re * x122p.im
2852 + self.twiddle2.re * x221p.im
2853 + self.twiddle3.re * x320p.im
2854 + self.twiddle4.re * x419p.im
2855 + self.twiddle5.re * x518p.im
2856 + self.twiddle6.re * x617p.im
2857 + self.twiddle7.re * x716p.im
2858 + self.twiddle8.re * x815p.im
2859 + self.twiddle9.re * x914p.im
2860 + self.twiddle10.re * x1013p.im
2861 + self.twiddle11.re * x1112p.im;
2862 let b122im_b = self.twiddle1.im * x122n.re
2863 + self.twiddle2.im * x221n.re
2864 + self.twiddle3.im * x320n.re
2865 + self.twiddle4.im * x419n.re
2866 + self.twiddle5.im * x518n.re
2867 + self.twiddle6.im * x617n.re
2868 + self.twiddle7.im * x716n.re
2869 + self.twiddle8.im * x815n.re
2870 + self.twiddle9.im * x914n.re
2871 + self.twiddle10.im * x1013n.re
2872 + self.twiddle11.im * x1112n.re;
2873 let b221im_a = buffer.load(0).im
2874 + self.twiddle2.re * x122p.im
2875 + self.twiddle4.re * x221p.im
2876 + self.twiddle6.re * x320p.im
2877 + self.twiddle8.re * x419p.im
2878 + self.twiddle10.re * x518p.im
2879 + self.twiddle11.re * x617p.im
2880 + self.twiddle9.re * x716p.im
2881 + self.twiddle7.re * x815p.im
2882 + self.twiddle5.re * x914p.im
2883 + self.twiddle3.re * x1013p.im
2884 + self.twiddle1.re * x1112p.im;
2885 let b221im_b = self.twiddle2.im * x122n.re
2886 + self.twiddle4.im * x221n.re
2887 + self.twiddle6.im * x320n.re
2888 + self.twiddle8.im * x419n.re
2889 + self.twiddle10.im * x518n.re
2890 + -self.twiddle11.im * x617n.re
2891 + -self.twiddle9.im * x716n.re
2892 + -self.twiddle7.im * x815n.re
2893 + -self.twiddle5.im * x914n.re
2894 + -self.twiddle3.im * x1013n.re
2895 + -self.twiddle1.im * x1112n.re;
2896 let b320im_a = buffer.load(0).im
2897 + self.twiddle3.re * x122p.im
2898 + self.twiddle6.re * x221p.im
2899 + self.twiddle9.re * x320p.im
2900 + self.twiddle11.re * x419p.im
2901 + self.twiddle8.re * x518p.im
2902 + self.twiddle5.re * x617p.im
2903 + self.twiddle2.re * x716p.im
2904 + self.twiddle1.re * x815p.im
2905 + self.twiddle4.re * x914p.im
2906 + self.twiddle7.re * x1013p.im
2907 + self.twiddle10.re * x1112p.im;
2908 let b320im_b = self.twiddle3.im * x122n.re
2909 + self.twiddle6.im * x221n.re
2910 + self.twiddle9.im * x320n.re
2911 + -self.twiddle11.im * x419n.re
2912 + -self.twiddle8.im * x518n.re
2913 + -self.twiddle5.im * x617n.re
2914 + -self.twiddle2.im * x716n.re
2915 + self.twiddle1.im * x815n.re
2916 + self.twiddle4.im * x914n.re
2917 + self.twiddle7.im * x1013n.re
2918 + self.twiddle10.im * x1112n.re;
2919 let b419im_a = buffer.load(0).im
2920 + self.twiddle4.re * x122p.im
2921 + self.twiddle8.re * x221p.im
2922 + self.twiddle11.re * x320p.im
2923 + self.twiddle7.re * x419p.im
2924 + self.twiddle3.re * x518p.im
2925 + self.twiddle1.re * x617p.im
2926 + self.twiddle5.re * x716p.im
2927 + self.twiddle9.re * x815p.im
2928 + self.twiddle10.re * x914p.im
2929 + self.twiddle6.re * x1013p.im
2930 + self.twiddle2.re * x1112p.im;
2931 let b419im_b = self.twiddle4.im * x122n.re
2932 + self.twiddle8.im * x221n.re
2933 + -self.twiddle11.im * x320n.re
2934 + -self.twiddle7.im * x419n.re
2935 + -self.twiddle3.im * x518n.re
2936 + self.twiddle1.im * x617n.re
2937 + self.twiddle5.im * x716n.re
2938 + self.twiddle9.im * x815n.re
2939 + -self.twiddle10.im * x914n.re
2940 + -self.twiddle6.im * x1013n.re
2941 + -self.twiddle2.im * x1112n.re;
2942 let b518im_a = buffer.load(0).im
2943 + self.twiddle5.re * x122p.im
2944 + self.twiddle10.re * x221p.im
2945 + self.twiddle8.re * x320p.im
2946 + self.twiddle3.re * x419p.im
2947 + self.twiddle2.re * x518p.im
2948 + self.twiddle7.re * x617p.im
2949 + self.twiddle11.re * x716p.im
2950 + self.twiddle6.re * x815p.im
2951 + self.twiddle1.re * x914p.im
2952 + self.twiddle4.re * x1013p.im
2953 + self.twiddle9.re * x1112p.im;
2954 let b518im_b = self.twiddle5.im * x122n.re
2955 + self.twiddle10.im * x221n.re
2956 + -self.twiddle8.im * x320n.re
2957 + -self.twiddle3.im * x419n.re
2958 + self.twiddle2.im * x518n.re
2959 + self.twiddle7.im * x617n.re
2960 + -self.twiddle11.im * x716n.re
2961 + -self.twiddle6.im * x815n.re
2962 + -self.twiddle1.im * x914n.re
2963 + self.twiddle4.im * x1013n.re
2964 + self.twiddle9.im * x1112n.re;
2965 let b617im_a = buffer.load(0).im
2966 + self.twiddle6.re * x122p.im
2967 + self.twiddle11.re * x221p.im
2968 + self.twiddle5.re * x320p.im
2969 + self.twiddle1.re * x419p.im
2970 + self.twiddle7.re * x518p.im
2971 + self.twiddle10.re * x617p.im
2972 + self.twiddle4.re * x716p.im
2973 + self.twiddle2.re * x815p.im
2974 + self.twiddle8.re * x914p.im
2975 + self.twiddle9.re * x1013p.im
2976 + self.twiddle3.re * x1112p.im;
2977 let b617im_b = self.twiddle6.im * x122n.re
2978 + -self.twiddle11.im * x221n.re
2979 + -self.twiddle5.im * x320n.re
2980 + self.twiddle1.im * x419n.re
2981 + self.twiddle7.im * x518n.re
2982 + -self.twiddle10.im * x617n.re
2983 + -self.twiddle4.im * x716n.re
2984 + self.twiddle2.im * x815n.re
2985 + self.twiddle8.im * x914n.re
2986 + -self.twiddle9.im * x1013n.re
2987 + -self.twiddle3.im * x1112n.re;
2988 let b716im_a = buffer.load(0).im
2989 + self.twiddle7.re * x122p.im
2990 + self.twiddle9.re * x221p.im
2991 + self.twiddle2.re * x320p.im
2992 + self.twiddle5.re * x419p.im
2993 + self.twiddle11.re * x518p.im
2994 + self.twiddle4.re * x617p.im
2995 + self.twiddle3.re * x716p.im
2996 + self.twiddle10.re * x815p.im
2997 + self.twiddle6.re * x914p.im
2998 + self.twiddle1.re * x1013p.im
2999 + self.twiddle8.re * x1112p.im;
3000 let b716im_b = self.twiddle7.im * x122n.re
3001 + -self.twiddle9.im * x221n.re
3002 + -self.twiddle2.im * x320n.re
3003 + self.twiddle5.im * x419n.re
3004 + -self.twiddle11.im * x518n.re
3005 + -self.twiddle4.im * x617n.re
3006 + self.twiddle3.im * x716n.re
3007 + self.twiddle10.im * x815n.re
3008 + -self.twiddle6.im * x914n.re
3009 + self.twiddle1.im * x1013n.re
3010 + self.twiddle8.im * x1112n.re;
3011 let b815im_a = buffer.load(0).im
3012 + self.twiddle8.re * x122p.im
3013 + self.twiddle7.re * x221p.im
3014 + self.twiddle1.re * x320p.im
3015 + self.twiddle9.re * x419p.im
3016 + self.twiddle6.re * x518p.im
3017 + self.twiddle2.re * x617p.im
3018 + self.twiddle10.re * x716p.im
3019 + self.twiddle5.re * x815p.im
3020 + self.twiddle3.re * x914p.im
3021 + self.twiddle11.re * x1013p.im
3022 + self.twiddle4.re * x1112p.im;
3023 let b815im_b = self.twiddle8.im * x122n.re
3024 + -self.twiddle7.im * x221n.re
3025 + self.twiddle1.im * x320n.re
3026 + self.twiddle9.im * x419n.re
3027 + -self.twiddle6.im * x518n.re
3028 + self.twiddle2.im * x617n.re
3029 + self.twiddle10.im * x716n.re
3030 + -self.twiddle5.im * x815n.re
3031 + self.twiddle3.im * x914n.re
3032 + self.twiddle11.im * x1013n.re
3033 + -self.twiddle4.im * x1112n.re;
3034 let b914im_a = buffer.load(0).im
3035 + self.twiddle9.re * x122p.im
3036 + self.twiddle5.re * x221p.im
3037 + self.twiddle4.re * x320p.im
3038 + self.twiddle10.re * x419p.im
3039 + self.twiddle1.re * x518p.im
3040 + self.twiddle8.re * x617p.im
3041 + self.twiddle6.re * x716p.im
3042 + self.twiddle3.re * x815p.im
3043 + self.twiddle11.re * x914p.im
3044 + self.twiddle2.re * x1013p.im
3045 + self.twiddle7.re * x1112p.im;
3046 let b914im_b = self.twiddle9.im * x122n.re
3047 + -self.twiddle5.im * x221n.re
3048 + self.twiddle4.im * x320n.re
3049 + -self.twiddle10.im * x419n.re
3050 + -self.twiddle1.im * x518n.re
3051 + self.twiddle8.im * x617n.re
3052 + -self.twiddle6.im * x716n.re
3053 + self.twiddle3.im * x815n.re
3054 + -self.twiddle11.im * x914n.re
3055 + -self.twiddle2.im * x1013n.re
3056 + self.twiddle7.im * x1112n.re;
3057 let b1013im_a = buffer.load(0).im
3058 + self.twiddle10.re * x122p.im
3059 + self.twiddle3.re * x221p.im
3060 + self.twiddle7.re * x320p.im
3061 + self.twiddle6.re * x419p.im
3062 + self.twiddle4.re * x518p.im
3063 + self.twiddle9.re * x617p.im
3064 + self.twiddle1.re * x716p.im
3065 + self.twiddle11.re * x815p.im
3066 + self.twiddle2.re * x914p.im
3067 + self.twiddle8.re * x1013p.im
3068 + self.twiddle5.re * x1112p.im;
3069 let b1013im_b = self.twiddle10.im * x122n.re
3070 + -self.twiddle3.im * x221n.re
3071 + self.twiddle7.im * x320n.re
3072 + -self.twiddle6.im * x419n.re
3073 + self.twiddle4.im * x518n.re
3074 + -self.twiddle9.im * x617n.re
3075 + self.twiddle1.im * x716n.re
3076 + self.twiddle11.im * x815n.re
3077 + -self.twiddle2.im * x914n.re
3078 + self.twiddle8.im * x1013n.re
3079 + -self.twiddle5.im * x1112n.re;
3080 let b1112im_a = buffer.load(0).im
3081 + self.twiddle11.re * x122p.im
3082 + self.twiddle1.re * x221p.im
3083 + self.twiddle10.re * x320p.im
3084 + self.twiddle2.re * x419p.im
3085 + self.twiddle9.re * x518p.im
3086 + self.twiddle3.re * x617p.im
3087 + self.twiddle8.re * x716p.im
3088 + self.twiddle4.re * x815p.im
3089 + self.twiddle7.re * x914p.im
3090 + self.twiddle5.re * x1013p.im
3091 + self.twiddle6.re * x1112p.im;
3092 let b1112im_b = self.twiddle11.im * x122n.re
3093 + -self.twiddle1.im * x221n.re
3094 + self.twiddle10.im * x320n.re
3095 + -self.twiddle2.im * x419n.re
3096 + self.twiddle9.im * x518n.re
3097 + -self.twiddle3.im * x617n.re
3098 + self.twiddle8.im * x716n.re
3099 + -self.twiddle4.im * x815n.re
3100 + self.twiddle7.im * x914n.re
3101 + -self.twiddle5.im * x1013n.re
3102 + self.twiddle6.im * x1112n.re;
3103
3104 let out1re = b122re_a - b122re_b;
3105 let out1im = b122im_a + b122im_b;
3106 let out2re = b221re_a - b221re_b;
3107 let out2im = b221im_a + b221im_b;
3108 let out3re = b320re_a - b320re_b;
3109 let out3im = b320im_a + b320im_b;
3110 let out4re = b419re_a - b419re_b;
3111 let out4im = b419im_a + b419im_b;
3112 let out5re = b518re_a - b518re_b;
3113 let out5im = b518im_a + b518im_b;
3114 let out6re = b617re_a - b617re_b;
3115 let out6im = b617im_a + b617im_b;
3116 let out7re = b716re_a - b716re_b;
3117 let out7im = b716im_a + b716im_b;
3118 let out8re = b815re_a - b815re_b;
3119 let out8im = b815im_a + b815im_b;
3120 let out9re = b914re_a - b914re_b;
3121 let out9im = b914im_a + b914im_b;
3122 let out10re = b1013re_a - b1013re_b;
3123 let out10im = b1013im_a + b1013im_b;
3124 let out11re = b1112re_a - b1112re_b;
3125 let out11im = b1112im_a + b1112im_b;
3126 let out12re = b1112re_a + b1112re_b;
3127 let out12im = b1112im_a - b1112im_b;
3128 let out13re = b1013re_a + b1013re_b;
3129 let out13im = b1013im_a - b1013im_b;
3130 let out14re = b914re_a + b914re_b;
3131 let out14im = b914im_a - b914im_b;
3132 let out15re = b815re_a + b815re_b;
3133 let out15im = b815im_a - b815im_b;
3134 let out16re = b716re_a + b716re_b;
3135 let out16im = b716im_a - b716im_b;
3136 let out17re = b617re_a + b617re_b;
3137 let out17im = b617im_a - b617im_b;
3138 let out18re = b518re_a + b518re_b;
3139 let out18im = b518im_a - b518im_b;
3140 let out19re = b419re_a + b419re_b;
3141 let out19im = b419im_a - b419im_b;
3142 let out20re = b320re_a + b320re_b;
3143 let out20im = b320im_a - b320im_b;
3144 let out21re = b221re_a + b221re_b;
3145 let out21im = b221im_a - b221im_b;
3146 let out22re = b122re_a + b122re_b;
3147 let out22im = b122im_a - b122im_b;
3148 buffer.store(sum, 0);
3149 buffer.store(
3150 Complex {
3151 re: out1re,
3152 im: out1im,
3153 },
3154 1,
3155 );
3156 buffer.store(
3157 Complex {
3158 re: out2re,
3159 im: out2im,
3160 },
3161 2,
3162 );
3163 buffer.store(
3164 Complex {
3165 re: out3re,
3166 im: out3im,
3167 },
3168 3,
3169 );
3170 buffer.store(
3171 Complex {
3172 re: out4re,
3173 im: out4im,
3174 },
3175 4,
3176 );
3177 buffer.store(
3178 Complex {
3179 re: out5re,
3180 im: out5im,
3181 },
3182 5,
3183 );
3184 buffer.store(
3185 Complex {
3186 re: out6re,
3187 im: out6im,
3188 },
3189 6,
3190 );
3191 buffer.store(
3192 Complex {
3193 re: out7re,
3194 im: out7im,
3195 },
3196 7,
3197 );
3198 buffer.store(
3199 Complex {
3200 re: out8re,
3201 im: out8im,
3202 },
3203 8,
3204 );
3205 buffer.store(
3206 Complex {
3207 re: out9re,
3208 im: out9im,
3209 },
3210 9,
3211 );
3212 buffer.store(
3213 Complex {
3214 re: out10re,
3215 im: out10im,
3216 },
3217 10,
3218 );
3219 buffer.store(
3220 Complex {
3221 re: out11re,
3222 im: out11im,
3223 },
3224 11,
3225 );
3226 buffer.store(
3227 Complex {
3228 re: out12re,
3229 im: out12im,
3230 },
3231 12,
3232 );
3233 buffer.store(
3234 Complex {
3235 re: out13re,
3236 im: out13im,
3237 },
3238 13,
3239 );
3240 buffer.store(
3241 Complex {
3242 re: out14re,
3243 im: out14im,
3244 },
3245 14,
3246 );
3247 buffer.store(
3248 Complex {
3249 re: out15re,
3250 im: out15im,
3251 },
3252 15,
3253 );
3254 buffer.store(
3255 Complex {
3256 re: out16re,
3257 im: out16im,
3258 },
3259 16,
3260 );
3261 buffer.store(
3262 Complex {
3263 re: out17re,
3264 im: out17im,
3265 },
3266 17,
3267 );
3268 buffer.store(
3269 Complex {
3270 re: out18re,
3271 im: out18im,
3272 },
3273 18,
3274 );
3275 buffer.store(
3276 Complex {
3277 re: out19re,
3278 im: out19im,
3279 },
3280 19,
3281 );
3282 buffer.store(
3283 Complex {
3284 re: out20re,
3285 im: out20im,
3286 },
3287 20,
3288 );
3289 buffer.store(
3290 Complex {
3291 re: out21re,
3292 im: out21im,
3293 },
3294 21,
3295 );
3296 buffer.store(
3297 Complex {
3298 re: out22re,
3299 im: out22im,
3300 },
3301 22,
3302 );
3303 }
3304 }
3305
3306 pub struct Butterfly27<T> {
3307 butterfly9: Butterfly9<T>,
3308 twiddles: [Complex<T>; 12],
3309 }
3310 boilerplate_fft_butterfly!(Butterfly27, 27, |this: &Butterfly27<_>| this
3311 .butterfly9
3312 .fft_direction());
3313 impl<T: FftNum> Butterfly27<T> {
3314 #[inline(always)]
3315 pub fn new(direction: FftDirection) -> Self {
3316 Self {
3317 butterfly9: Butterfly9::new(direction),
3318 twiddles: [
3319 twiddles::compute_twiddle(1, 27, direction),
3320 twiddles::compute_twiddle(2, 27, direction),
3321 twiddles::compute_twiddle(3, 27, direction),
3322 twiddles::compute_twiddle(4, 27, direction),
3323 twiddles::compute_twiddle(5, 27, direction),
3324 twiddles::compute_twiddle(6, 27, direction),
3325 twiddles::compute_twiddle(7, 27, direction),
3326 twiddles::compute_twiddle(8, 27, direction),
3327 twiddles::compute_twiddle(10, 27, direction),
3328 twiddles::compute_twiddle(12, 27, direction),
3329 twiddles::compute_twiddle(14, 27, direction),
3330 twiddles::compute_twiddle(16, 27, direction),
3331 ],
3332 }
3333 }
3334
3335 #[inline(always)]
3336 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
3337 // algorithm: mixed radix with width=9 and height=3
3338
3339 // step 1: transpose the input into the scratch
3340 let mut scratch0 = [
3341 buffer.load(0),
3342 buffer.load(3),
3343 buffer.load(6),
3344 buffer.load(9),
3345 buffer.load(12),
3346 buffer.load(15),
3347 buffer.load(18),
3348 buffer.load(21),
3349 buffer.load(24),
3350 ];
3351 let mut scratch1 = [
3352 buffer.load(1 + 0),
3353 buffer.load(1 + 3),
3354 buffer.load(1 + 6),
3355 buffer.load(1 + 9),
3356 buffer.load(1 + 12),
3357 buffer.load(1 + 15),
3358 buffer.load(1 + 18),
3359 buffer.load(1 + 21),
3360 buffer.load(1 + 24),
3361 ];
3362 let mut scratch2 = [
3363 buffer.load(2 + 0),
3364 buffer.load(2 + 3),
3365 buffer.load(2 + 6),
3366 buffer.load(2 + 9),
3367 buffer.load(2 + 12),
3368 buffer.load(2 + 15),
3369 buffer.load(2 + 18),
3370 buffer.load(2 + 21),
3371 buffer.load(2 + 24),
3372 ];
3373
3374 // step 2: column FFTs
3375 self.butterfly9.perform_fft_contiguous(&mut scratch0);
3376 self.butterfly9.perform_fft_contiguous(&mut scratch1);
3377 self.butterfly9.perform_fft_contiguous(&mut scratch2);
3378
3379 // step 3: apply twiddle factors
3380 scratch1[1] = scratch1[1] * self.twiddles[0];
3381 scratch1[2] = scratch1[2] * self.twiddles[1];
3382 scratch1[3] = scratch1[3] * self.twiddles[2];
3383 scratch1[4] = scratch1[4] * self.twiddles[3];
3384 scratch1[5] = scratch1[5] * self.twiddles[4];
3385 scratch1[6] = scratch1[6] * self.twiddles[5];
3386 scratch1[7] = scratch1[7] * self.twiddles[6];
3387 scratch1[8] = scratch1[8] * self.twiddles[7];
3388 scratch2[1] = scratch2[1] * self.twiddles[1];
3389 scratch2[2] = scratch2[2] * self.twiddles[3];
3390 scratch2[3] = scratch2[3] * self.twiddles[5];
3391 scratch2[4] = scratch2[4] * self.twiddles[7];
3392 scratch2[5] = scratch2[5] * self.twiddles[8];
3393 scratch2[6] = scratch2[6] * self.twiddles[9];
3394 scratch2[7] = scratch2[7] * self.twiddles[10];
3395 scratch2[8] = scratch2[8] * self.twiddles[11];
3396
3397 // step 4: SKIPPED because the next FFTs will be non-contiguous
3398
3399 // step 5: row FFTs
3400 self.butterfly9.butterfly3.perform_fft_strided(
3401 &mut scratch0[0],
3402 &mut scratch1[0],
3403 &mut scratch2[0],
3404 );
3405 self.butterfly9.butterfly3.perform_fft_strided(
3406 &mut scratch0[1],
3407 &mut scratch1[1],
3408 &mut scratch2[1],
3409 );
3410 self.butterfly9.butterfly3.perform_fft_strided(
3411 &mut scratch0[2],
3412 &mut scratch1[2],
3413 &mut scratch2[2],
3414 );
3415 self.butterfly9.butterfly3.perform_fft_strided(
3416 &mut scratch0[3],
3417 &mut scratch1[3],
3418 &mut scratch2[3],
3419 );
3420 self.butterfly9.butterfly3.perform_fft_strided(
3421 &mut scratch0[4],
3422 &mut scratch1[4],
3423 &mut scratch2[4],
3424 );
3425 self.butterfly9.butterfly3.perform_fft_strided(
3426 &mut scratch0[5],
3427 &mut scratch1[5],
3428 &mut scratch2[5],
3429 );
3430 self.butterfly9.butterfly3.perform_fft_strided(
3431 &mut scratch0[6],
3432 &mut scratch1[6],
3433 &mut scratch2[6],
3434 );
3435 self.butterfly9.butterfly3.perform_fft_strided(
3436 &mut scratch0[7],
3437 &mut scratch1[7],
3438 &mut scratch2[7],
3439 );
3440 self.butterfly9.butterfly3.perform_fft_strided(
3441 &mut scratch0[8],
3442 &mut scratch1[8],
3443 &mut scratch2[8],
3444 );
3445
3446 // step 6: copy the result into the output. normally we'd need to do a transpose here, but we can skip it because we skipped the transpose in step 4
3447 buffer.store(scratch0[0], 0);
3448 buffer.store(scratch0[1], 1);
3449 buffer.store(scratch0[2], 2);
3450 buffer.store(scratch0[3], 3);
3451 buffer.store(scratch0[4], 4);
3452 buffer.store(scratch0[5], 5);
3453 buffer.store(scratch0[6], 6);
3454 buffer.store(scratch0[7], 7);
3455 buffer.store(scratch0[8], 8);
3456
3457 buffer.store(scratch1[0], 9 + 0);
3458 buffer.store(scratch1[1], 9 + 1);
3459 buffer.store(scratch1[2], 9 + 2);
3460 buffer.store(scratch1[3], 9 + 3);
3461 buffer.store(scratch1[4], 9 + 4);
3462 buffer.store(scratch1[5], 9 + 5);
3463 buffer.store(scratch1[6], 9 + 6);
3464 buffer.store(scratch1[7], 9 + 7);
3465 buffer.store(scratch1[8], 9 + 8);
3466
3467 buffer.store(scratch2[0], 18 + 0);
3468 buffer.store(scratch2[1], 18 + 1);
3469 buffer.store(scratch2[2], 18 + 2);
3470 buffer.store(scratch2[3], 18 + 3);
3471 buffer.store(scratch2[4], 18 + 4);
3472 buffer.store(scratch2[5], 18 + 5);
3473 buffer.store(scratch2[6], 18 + 6);
3474 buffer.store(scratch2[7], 18 + 7);
3475 buffer.store(scratch2[8], 18 + 8);
3476 }
3477 }
3478
3479 pub struct Butterfly29<T> {
3480 twiddle1: Complex<T>,
3481 twiddle2: Complex<T>,
3482 twiddle3: Complex<T>,
3483 twiddle4: Complex<T>,
3484 twiddle5: Complex<T>,
3485 twiddle6: Complex<T>,
3486 twiddle7: Complex<T>,
3487 twiddle8: Complex<T>,
3488 twiddle9: Complex<T>,
3489 twiddle10: Complex<T>,
3490 twiddle11: Complex<T>,
3491 twiddle12: Complex<T>,
3492 twiddle13: Complex<T>,
3493 twiddle14: Complex<T>,
3494 direction: FftDirection,
3495 }
3496 boilerplate_fft_butterfly!(Butterfly29, 29, |this: &Butterfly29<_>| this.direction);
3497 impl<T: FftNum> Butterfly29<T> {
3498 pub fn new(direction: FftDirection) -> Self {
3499 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 29, direction);
3500 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 29, direction);
3501 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 29, direction);
3502 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 29, direction);
3503 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 29, direction);
3504 let twiddle6: Complex<T> = twiddles::compute_twiddle(6, 29, direction);
3505 let twiddle7: Complex<T> = twiddles::compute_twiddle(7, 29, direction);
3506 let twiddle8: Complex<T> = twiddles::compute_twiddle(8, 29, direction);
3507 let twiddle9: Complex<T> = twiddles::compute_twiddle(9, 29, direction);
3508 let twiddle10: Complex<T> = twiddles::compute_twiddle(10, 29, direction);
3509 let twiddle11: Complex<T> = twiddles::compute_twiddle(11, 29, direction);
3510 let twiddle12: Complex<T> = twiddles::compute_twiddle(12, 29, direction);
3511 let twiddle13: Complex<T> = twiddles::compute_twiddle(13, 29, direction);
3512 let twiddle14: Complex<T> = twiddles::compute_twiddle(14, 29, direction);
3513 Self {
3514 twiddle1,
3515 twiddle2,
3516 twiddle3,
3517 twiddle4,
3518 twiddle5,
3519 twiddle6,
3520 twiddle7,
3521 twiddle8,
3522 twiddle9,
3523 twiddle10,
3524 twiddle11,
3525 twiddle12,
3526 twiddle13,
3527 twiddle14,
3528 direction,
3529 }
3530 }
3531
3532 #[inline(never)]
3533 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
3534 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
3535 // However, instead of doing it by hand the actual code is autogenerated
3536 // with the `genbutterflies.py` script in the `tools` directory.
3537 let x128p = buffer.load(1) + buffer.load(28);
3538 let x128n = buffer.load(1) - buffer.load(28);
3539 let x227p = buffer.load(2) + buffer.load(27);
3540 let x227n = buffer.load(2) - buffer.load(27);
3541 let x326p = buffer.load(3) + buffer.load(26);
3542 let x326n = buffer.load(3) - buffer.load(26);
3543 let x425p = buffer.load(4) + buffer.load(25);
3544 let x425n = buffer.load(4) - buffer.load(25);
3545 let x524p = buffer.load(5) + buffer.load(24);
3546 let x524n = buffer.load(5) - buffer.load(24);
3547 let x623p = buffer.load(6) + buffer.load(23);
3548 let x623n = buffer.load(6) - buffer.load(23);
3549 let x722p = buffer.load(7) + buffer.load(22);
3550 let x722n = buffer.load(7) - buffer.load(22);
3551 let x821p = buffer.load(8) + buffer.load(21);
3552 let x821n = buffer.load(8) - buffer.load(21);
3553 let x920p = buffer.load(9) + buffer.load(20);
3554 let x920n = buffer.load(9) - buffer.load(20);
3555 let x1019p = buffer.load(10) + buffer.load(19);
3556 let x1019n = buffer.load(10) - buffer.load(19);
3557 let x1118p = buffer.load(11) + buffer.load(18);
3558 let x1118n = buffer.load(11) - buffer.load(18);
3559 let x1217p = buffer.load(12) + buffer.load(17);
3560 let x1217n = buffer.load(12) - buffer.load(17);
3561 let x1316p = buffer.load(13) + buffer.load(16);
3562 let x1316n = buffer.load(13) - buffer.load(16);
3563 let x1415p = buffer.load(14) + buffer.load(15);
3564 let x1415n = buffer.load(14) - buffer.load(15);
3565 let sum = buffer.load(0)
3566 + x128p
3567 + x227p
3568 + x326p
3569 + x425p
3570 + x524p
3571 + x623p
3572 + x722p
3573 + x821p
3574 + x920p
3575 + x1019p
3576 + x1118p
3577 + x1217p
3578 + x1316p
3579 + x1415p;
3580 let b128re_a = buffer.load(0).re
3581 + self.twiddle1.re * x128p.re
3582 + self.twiddle2.re * x227p.re
3583 + self.twiddle3.re * x326p.re
3584 + self.twiddle4.re * x425p.re
3585 + self.twiddle5.re * x524p.re
3586 + self.twiddle6.re * x623p.re
3587 + self.twiddle7.re * x722p.re
3588 + self.twiddle8.re * x821p.re
3589 + self.twiddle9.re * x920p.re
3590 + self.twiddle10.re * x1019p.re
3591 + self.twiddle11.re * x1118p.re
3592 + self.twiddle12.re * x1217p.re
3593 + self.twiddle13.re * x1316p.re
3594 + self.twiddle14.re * x1415p.re;
3595 let b128re_b = self.twiddle1.im * x128n.im
3596 + self.twiddle2.im * x227n.im
3597 + self.twiddle3.im * x326n.im
3598 + self.twiddle4.im * x425n.im
3599 + self.twiddle5.im * x524n.im
3600 + self.twiddle6.im * x623n.im
3601 + self.twiddle7.im * x722n.im
3602 + self.twiddle8.im * x821n.im
3603 + self.twiddle9.im * x920n.im
3604 + self.twiddle10.im * x1019n.im
3605 + self.twiddle11.im * x1118n.im
3606 + self.twiddle12.im * x1217n.im
3607 + self.twiddle13.im * x1316n.im
3608 + self.twiddle14.im * x1415n.im;
3609 let b227re_a = buffer.load(0).re
3610 + self.twiddle2.re * x128p.re
3611 + self.twiddle4.re * x227p.re
3612 + self.twiddle6.re * x326p.re
3613 + self.twiddle8.re * x425p.re
3614 + self.twiddle10.re * x524p.re
3615 + self.twiddle12.re * x623p.re
3616 + self.twiddle14.re * x722p.re
3617 + self.twiddle13.re * x821p.re
3618 + self.twiddle11.re * x920p.re
3619 + self.twiddle9.re * x1019p.re
3620 + self.twiddle7.re * x1118p.re
3621 + self.twiddle5.re * x1217p.re
3622 + self.twiddle3.re * x1316p.re
3623 + self.twiddle1.re * x1415p.re;
3624 let b227re_b = self.twiddle2.im * x128n.im
3625 + self.twiddle4.im * x227n.im
3626 + self.twiddle6.im * x326n.im
3627 + self.twiddle8.im * x425n.im
3628 + self.twiddle10.im * x524n.im
3629 + self.twiddle12.im * x623n.im
3630 + self.twiddle14.im * x722n.im
3631 + -self.twiddle13.im * x821n.im
3632 + -self.twiddle11.im * x920n.im
3633 + -self.twiddle9.im * x1019n.im
3634 + -self.twiddle7.im * x1118n.im
3635 + -self.twiddle5.im * x1217n.im
3636 + -self.twiddle3.im * x1316n.im
3637 + -self.twiddle1.im * x1415n.im;
3638 let b326re_a = buffer.load(0).re
3639 + self.twiddle3.re * x128p.re
3640 + self.twiddle6.re * x227p.re
3641 + self.twiddle9.re * x326p.re
3642 + self.twiddle12.re * x425p.re
3643 + self.twiddle14.re * x524p.re
3644 + self.twiddle11.re * x623p.re
3645 + self.twiddle8.re * x722p.re
3646 + self.twiddle5.re * x821p.re
3647 + self.twiddle2.re * x920p.re
3648 + self.twiddle1.re * x1019p.re
3649 + self.twiddle4.re * x1118p.re
3650 + self.twiddle7.re * x1217p.re
3651 + self.twiddle10.re * x1316p.re
3652 + self.twiddle13.re * x1415p.re;
3653 let b326re_b = self.twiddle3.im * x128n.im
3654 + self.twiddle6.im * x227n.im
3655 + self.twiddle9.im * x326n.im
3656 + self.twiddle12.im * x425n.im
3657 + -self.twiddle14.im * x524n.im
3658 + -self.twiddle11.im * x623n.im
3659 + -self.twiddle8.im * x722n.im
3660 + -self.twiddle5.im * x821n.im
3661 + -self.twiddle2.im * x920n.im
3662 + self.twiddle1.im * x1019n.im
3663 + self.twiddle4.im * x1118n.im
3664 + self.twiddle7.im * x1217n.im
3665 + self.twiddle10.im * x1316n.im
3666 + self.twiddle13.im * x1415n.im;
3667 let b425re_a = buffer.load(0).re
3668 + self.twiddle4.re * x128p.re
3669 + self.twiddle8.re * x227p.re
3670 + self.twiddle12.re * x326p.re
3671 + self.twiddle13.re * x425p.re
3672 + self.twiddle9.re * x524p.re
3673 + self.twiddle5.re * x623p.re
3674 + self.twiddle1.re * x722p.re
3675 + self.twiddle3.re * x821p.re
3676 + self.twiddle7.re * x920p.re
3677 + self.twiddle11.re * x1019p.re
3678 + self.twiddle14.re * x1118p.re
3679 + self.twiddle10.re * x1217p.re
3680 + self.twiddle6.re * x1316p.re
3681 + self.twiddle2.re * x1415p.re;
3682 let b425re_b = self.twiddle4.im * x128n.im
3683 + self.twiddle8.im * x227n.im
3684 + self.twiddle12.im * x326n.im
3685 + -self.twiddle13.im * x425n.im
3686 + -self.twiddle9.im * x524n.im
3687 + -self.twiddle5.im * x623n.im
3688 + -self.twiddle1.im * x722n.im
3689 + self.twiddle3.im * x821n.im
3690 + self.twiddle7.im * x920n.im
3691 + self.twiddle11.im * x1019n.im
3692 + -self.twiddle14.im * x1118n.im
3693 + -self.twiddle10.im * x1217n.im
3694 + -self.twiddle6.im * x1316n.im
3695 + -self.twiddle2.im * x1415n.im;
3696 let b524re_a = buffer.load(0).re
3697 + self.twiddle5.re * x128p.re
3698 + self.twiddle10.re * x227p.re
3699 + self.twiddle14.re * x326p.re
3700 + self.twiddle9.re * x425p.re
3701 + self.twiddle4.re * x524p.re
3702 + self.twiddle1.re * x623p.re
3703 + self.twiddle6.re * x722p.re
3704 + self.twiddle11.re * x821p.re
3705 + self.twiddle13.re * x920p.re
3706 + self.twiddle8.re * x1019p.re
3707 + self.twiddle3.re * x1118p.re
3708 + self.twiddle2.re * x1217p.re
3709 + self.twiddle7.re * x1316p.re
3710 + self.twiddle12.re * x1415p.re;
3711 let b524re_b = self.twiddle5.im * x128n.im
3712 + self.twiddle10.im * x227n.im
3713 + -self.twiddle14.im * x326n.im
3714 + -self.twiddle9.im * x425n.im
3715 + -self.twiddle4.im * x524n.im
3716 + self.twiddle1.im * x623n.im
3717 + self.twiddle6.im * x722n.im
3718 + self.twiddle11.im * x821n.im
3719 + -self.twiddle13.im * x920n.im
3720 + -self.twiddle8.im * x1019n.im
3721 + -self.twiddle3.im * x1118n.im
3722 + self.twiddle2.im * x1217n.im
3723 + self.twiddle7.im * x1316n.im
3724 + self.twiddle12.im * x1415n.im;
3725 let b623re_a = buffer.load(0).re
3726 + self.twiddle6.re * x128p.re
3727 + self.twiddle12.re * x227p.re
3728 + self.twiddle11.re * x326p.re
3729 + self.twiddle5.re * x425p.re
3730 + self.twiddle1.re * x524p.re
3731 + self.twiddle7.re * x623p.re
3732 + self.twiddle13.re * x722p.re
3733 + self.twiddle10.re * x821p.re
3734 + self.twiddle4.re * x920p.re
3735 + self.twiddle2.re * x1019p.re
3736 + self.twiddle8.re * x1118p.re
3737 + self.twiddle14.re * x1217p.re
3738 + self.twiddle9.re * x1316p.re
3739 + self.twiddle3.re * x1415p.re;
3740 let b623re_b = self.twiddle6.im * x128n.im
3741 + self.twiddle12.im * x227n.im
3742 + -self.twiddle11.im * x326n.im
3743 + -self.twiddle5.im * x425n.im
3744 + self.twiddle1.im * x524n.im
3745 + self.twiddle7.im * x623n.im
3746 + self.twiddle13.im * x722n.im
3747 + -self.twiddle10.im * x821n.im
3748 + -self.twiddle4.im * x920n.im
3749 + self.twiddle2.im * x1019n.im
3750 + self.twiddle8.im * x1118n.im
3751 + self.twiddle14.im * x1217n.im
3752 + -self.twiddle9.im * x1316n.im
3753 + -self.twiddle3.im * x1415n.im;
3754 let b722re_a = buffer.load(0).re
3755 + self.twiddle7.re * x128p.re
3756 + self.twiddle14.re * x227p.re
3757 + self.twiddle8.re * x326p.re
3758 + self.twiddle1.re * x425p.re
3759 + self.twiddle6.re * x524p.re
3760 + self.twiddle13.re * x623p.re
3761 + self.twiddle9.re * x722p.re
3762 + self.twiddle2.re * x821p.re
3763 + self.twiddle5.re * x920p.re
3764 + self.twiddle12.re * x1019p.re
3765 + self.twiddle10.re * x1118p.re
3766 + self.twiddle3.re * x1217p.re
3767 + self.twiddle4.re * x1316p.re
3768 + self.twiddle11.re * x1415p.re;
3769 let b722re_b = self.twiddle7.im * x128n.im
3770 + self.twiddle14.im * x227n.im
3771 + -self.twiddle8.im * x326n.im
3772 + -self.twiddle1.im * x425n.im
3773 + self.twiddle6.im * x524n.im
3774 + self.twiddle13.im * x623n.im
3775 + -self.twiddle9.im * x722n.im
3776 + -self.twiddle2.im * x821n.im
3777 + self.twiddle5.im * x920n.im
3778 + self.twiddle12.im * x1019n.im
3779 + -self.twiddle10.im * x1118n.im
3780 + -self.twiddle3.im * x1217n.im
3781 + self.twiddle4.im * x1316n.im
3782 + self.twiddle11.im * x1415n.im;
3783 let b821re_a = buffer.load(0).re
3784 + self.twiddle8.re * x128p.re
3785 + self.twiddle13.re * x227p.re
3786 + self.twiddle5.re * x326p.re
3787 + self.twiddle3.re * x425p.re
3788 + self.twiddle11.re * x524p.re
3789 + self.twiddle10.re * x623p.re
3790 + self.twiddle2.re * x722p.re
3791 + self.twiddle6.re * x821p.re
3792 + self.twiddle14.re * x920p.re
3793 + self.twiddle7.re * x1019p.re
3794 + self.twiddle1.re * x1118p.re
3795 + self.twiddle9.re * x1217p.re
3796 + self.twiddle12.re * x1316p.re
3797 + self.twiddle4.re * x1415p.re;
3798 let b821re_b = self.twiddle8.im * x128n.im
3799 + -self.twiddle13.im * x227n.im
3800 + -self.twiddle5.im * x326n.im
3801 + self.twiddle3.im * x425n.im
3802 + self.twiddle11.im * x524n.im
3803 + -self.twiddle10.im * x623n.im
3804 + -self.twiddle2.im * x722n.im
3805 + self.twiddle6.im * x821n.im
3806 + self.twiddle14.im * x920n.im
3807 + -self.twiddle7.im * x1019n.im
3808 + self.twiddle1.im * x1118n.im
3809 + self.twiddle9.im * x1217n.im
3810 + -self.twiddle12.im * x1316n.im
3811 + -self.twiddle4.im * x1415n.im;
3812 let b920re_a = buffer.load(0).re
3813 + self.twiddle9.re * x128p.re
3814 + self.twiddle11.re * x227p.re
3815 + self.twiddle2.re * x326p.re
3816 + self.twiddle7.re * x425p.re
3817 + self.twiddle13.re * x524p.re
3818 + self.twiddle4.re * x623p.re
3819 + self.twiddle5.re * x722p.re
3820 + self.twiddle14.re * x821p.re
3821 + self.twiddle6.re * x920p.re
3822 + self.twiddle3.re * x1019p.re
3823 + self.twiddle12.re * x1118p.re
3824 + self.twiddle8.re * x1217p.re
3825 + self.twiddle1.re * x1316p.re
3826 + self.twiddle10.re * x1415p.re;
3827 let b920re_b = self.twiddle9.im * x128n.im
3828 + -self.twiddle11.im * x227n.im
3829 + -self.twiddle2.im * x326n.im
3830 + self.twiddle7.im * x425n.im
3831 + -self.twiddle13.im * x524n.im
3832 + -self.twiddle4.im * x623n.im
3833 + self.twiddle5.im * x722n.im
3834 + self.twiddle14.im * x821n.im
3835 + -self.twiddle6.im * x920n.im
3836 + self.twiddle3.im * x1019n.im
3837 + self.twiddle12.im * x1118n.im
3838 + -self.twiddle8.im * x1217n.im
3839 + self.twiddle1.im * x1316n.im
3840 + self.twiddle10.im * x1415n.im;
3841 let b1019re_a = buffer.load(0).re
3842 + self.twiddle10.re * x128p.re
3843 + self.twiddle9.re * x227p.re
3844 + self.twiddle1.re * x326p.re
3845 + self.twiddle11.re * x425p.re
3846 + self.twiddle8.re * x524p.re
3847 + self.twiddle2.re * x623p.re
3848 + self.twiddle12.re * x722p.re
3849 + self.twiddle7.re * x821p.re
3850 + self.twiddle3.re * x920p.re
3851 + self.twiddle13.re * x1019p.re
3852 + self.twiddle6.re * x1118p.re
3853 + self.twiddle4.re * x1217p.re
3854 + self.twiddle14.re * x1316p.re
3855 + self.twiddle5.re * x1415p.re;
3856 let b1019re_b = self.twiddle10.im * x128n.im
3857 + -self.twiddle9.im * x227n.im
3858 + self.twiddle1.im * x326n.im
3859 + self.twiddle11.im * x425n.im
3860 + -self.twiddle8.im * x524n.im
3861 + self.twiddle2.im * x623n.im
3862 + self.twiddle12.im * x722n.im
3863 + -self.twiddle7.im * x821n.im
3864 + self.twiddle3.im * x920n.im
3865 + self.twiddle13.im * x1019n.im
3866 + -self.twiddle6.im * x1118n.im
3867 + self.twiddle4.im * x1217n.im
3868 + self.twiddle14.im * x1316n.im
3869 + -self.twiddle5.im * x1415n.im;
3870 let b1118re_a = buffer.load(0).re
3871 + self.twiddle11.re * x128p.re
3872 + self.twiddle7.re * x227p.re
3873 + self.twiddle4.re * x326p.re
3874 + self.twiddle14.re * x425p.re
3875 + self.twiddle3.re * x524p.re
3876 + self.twiddle8.re * x623p.re
3877 + self.twiddle10.re * x722p.re
3878 + self.twiddle1.re * x821p.re
3879 + self.twiddle12.re * x920p.re
3880 + self.twiddle6.re * x1019p.re
3881 + self.twiddle5.re * x1118p.re
3882 + self.twiddle13.re * x1217p.re
3883 + self.twiddle2.re * x1316p.re
3884 + self.twiddle9.re * x1415p.re;
3885 let b1118re_b = self.twiddle11.im * x128n.im
3886 + -self.twiddle7.im * x227n.im
3887 + self.twiddle4.im * x326n.im
3888 + -self.twiddle14.im * x425n.im
3889 + -self.twiddle3.im * x524n.im
3890 + self.twiddle8.im * x623n.im
3891 + -self.twiddle10.im * x722n.im
3892 + self.twiddle1.im * x821n.im
3893 + self.twiddle12.im * x920n.im
3894 + -self.twiddle6.im * x1019n.im
3895 + self.twiddle5.im * x1118n.im
3896 + -self.twiddle13.im * x1217n.im
3897 + -self.twiddle2.im * x1316n.im
3898 + self.twiddle9.im * x1415n.im;
3899 let b1217re_a = buffer.load(0).re
3900 + self.twiddle12.re * x128p.re
3901 + self.twiddle5.re * x227p.re
3902 + self.twiddle7.re * x326p.re
3903 + self.twiddle10.re * x425p.re
3904 + self.twiddle2.re * x524p.re
3905 + self.twiddle14.re * x623p.re
3906 + self.twiddle3.re * x722p.re
3907 + self.twiddle9.re * x821p.re
3908 + self.twiddle8.re * x920p.re
3909 + self.twiddle4.re * x1019p.re
3910 + self.twiddle13.re * x1118p.re
3911 + self.twiddle1.re * x1217p.re
3912 + self.twiddle11.re * x1316p.re
3913 + self.twiddle6.re * x1415p.re;
3914 let b1217re_b = self.twiddle12.im * x128n.im
3915 + -self.twiddle5.im * x227n.im
3916 + self.twiddle7.im * x326n.im
3917 + -self.twiddle10.im * x425n.im
3918 + self.twiddle2.im * x524n.im
3919 + self.twiddle14.im * x623n.im
3920 + -self.twiddle3.im * x722n.im
3921 + self.twiddle9.im * x821n.im
3922 + -self.twiddle8.im * x920n.im
3923 + self.twiddle4.im * x1019n.im
3924 + -self.twiddle13.im * x1118n.im
3925 + -self.twiddle1.im * x1217n.im
3926 + self.twiddle11.im * x1316n.im
3927 + -self.twiddle6.im * x1415n.im;
3928 let b1316re_a = buffer.load(0).re
3929 + self.twiddle13.re * x128p.re
3930 + self.twiddle3.re * x227p.re
3931 + self.twiddle10.re * x326p.re
3932 + self.twiddle6.re * x425p.re
3933 + self.twiddle7.re * x524p.re
3934 + self.twiddle9.re * x623p.re
3935 + self.twiddle4.re * x722p.re
3936 + self.twiddle12.re * x821p.re
3937 + self.twiddle1.re * x920p.re
3938 + self.twiddle14.re * x1019p.re
3939 + self.twiddle2.re * x1118p.re
3940 + self.twiddle11.re * x1217p.re
3941 + self.twiddle5.re * x1316p.re
3942 + self.twiddle8.re * x1415p.re;
3943 let b1316re_b = self.twiddle13.im * x128n.im
3944 + -self.twiddle3.im * x227n.im
3945 + self.twiddle10.im * x326n.im
3946 + -self.twiddle6.im * x425n.im
3947 + self.twiddle7.im * x524n.im
3948 + -self.twiddle9.im * x623n.im
3949 + self.twiddle4.im * x722n.im
3950 + -self.twiddle12.im * x821n.im
3951 + self.twiddle1.im * x920n.im
3952 + self.twiddle14.im * x1019n.im
3953 + -self.twiddle2.im * x1118n.im
3954 + self.twiddle11.im * x1217n.im
3955 + -self.twiddle5.im * x1316n.im
3956 + self.twiddle8.im * x1415n.im;
3957 let b1415re_a = buffer.load(0).re
3958 + self.twiddle14.re * x128p.re
3959 + self.twiddle1.re * x227p.re
3960 + self.twiddle13.re * x326p.re
3961 + self.twiddle2.re * x425p.re
3962 + self.twiddle12.re * x524p.re
3963 + self.twiddle3.re * x623p.re
3964 + self.twiddle11.re * x722p.re
3965 + self.twiddle4.re * x821p.re
3966 + self.twiddle10.re * x920p.re
3967 + self.twiddle5.re * x1019p.re
3968 + self.twiddle9.re * x1118p.re
3969 + self.twiddle6.re * x1217p.re
3970 + self.twiddle8.re * x1316p.re
3971 + self.twiddle7.re * x1415p.re;
3972 let b1415re_b = self.twiddle14.im * x128n.im
3973 + -self.twiddle1.im * x227n.im
3974 + self.twiddle13.im * x326n.im
3975 + -self.twiddle2.im * x425n.im
3976 + self.twiddle12.im * x524n.im
3977 + -self.twiddle3.im * x623n.im
3978 + self.twiddle11.im * x722n.im
3979 + -self.twiddle4.im * x821n.im
3980 + self.twiddle10.im * x920n.im
3981 + -self.twiddle5.im * x1019n.im
3982 + self.twiddle9.im * x1118n.im
3983 + -self.twiddle6.im * x1217n.im
3984 + self.twiddle8.im * x1316n.im
3985 + -self.twiddle7.im * x1415n.im;
3986
3987 let b128im_a = buffer.load(0).im
3988 + self.twiddle1.re * x128p.im
3989 + self.twiddle2.re * x227p.im
3990 + self.twiddle3.re * x326p.im
3991 + self.twiddle4.re * x425p.im
3992 + self.twiddle5.re * x524p.im
3993 + self.twiddle6.re * x623p.im
3994 + self.twiddle7.re * x722p.im
3995 + self.twiddle8.re * x821p.im
3996 + self.twiddle9.re * x920p.im
3997 + self.twiddle10.re * x1019p.im
3998 + self.twiddle11.re * x1118p.im
3999 + self.twiddle12.re * x1217p.im
4000 + self.twiddle13.re * x1316p.im
4001 + self.twiddle14.re * x1415p.im;
4002 let b128im_b = self.twiddle1.im * x128n.re
4003 + self.twiddle2.im * x227n.re
4004 + self.twiddle3.im * x326n.re
4005 + self.twiddle4.im * x425n.re
4006 + self.twiddle5.im * x524n.re
4007 + self.twiddle6.im * x623n.re
4008 + self.twiddle7.im * x722n.re
4009 + self.twiddle8.im * x821n.re
4010 + self.twiddle9.im * x920n.re
4011 + self.twiddle10.im * x1019n.re
4012 + self.twiddle11.im * x1118n.re
4013 + self.twiddle12.im * x1217n.re
4014 + self.twiddle13.im * x1316n.re
4015 + self.twiddle14.im * x1415n.re;
4016 let b227im_a = buffer.load(0).im
4017 + self.twiddle2.re * x128p.im
4018 + self.twiddle4.re * x227p.im
4019 + self.twiddle6.re * x326p.im
4020 + self.twiddle8.re * x425p.im
4021 + self.twiddle10.re * x524p.im
4022 + self.twiddle12.re * x623p.im
4023 + self.twiddle14.re * x722p.im
4024 + self.twiddle13.re * x821p.im
4025 + self.twiddle11.re * x920p.im
4026 + self.twiddle9.re * x1019p.im
4027 + self.twiddle7.re * x1118p.im
4028 + self.twiddle5.re * x1217p.im
4029 + self.twiddle3.re * x1316p.im
4030 + self.twiddle1.re * x1415p.im;
4031 let b227im_b = self.twiddle2.im * x128n.re
4032 + self.twiddle4.im * x227n.re
4033 + self.twiddle6.im * x326n.re
4034 + self.twiddle8.im * x425n.re
4035 + self.twiddle10.im * x524n.re
4036 + self.twiddle12.im * x623n.re
4037 + self.twiddle14.im * x722n.re
4038 + -self.twiddle13.im * x821n.re
4039 + -self.twiddle11.im * x920n.re
4040 + -self.twiddle9.im * x1019n.re
4041 + -self.twiddle7.im * x1118n.re
4042 + -self.twiddle5.im * x1217n.re
4043 + -self.twiddle3.im * x1316n.re
4044 + -self.twiddle1.im * x1415n.re;
4045 let b326im_a = buffer.load(0).im
4046 + self.twiddle3.re * x128p.im
4047 + self.twiddle6.re * x227p.im
4048 + self.twiddle9.re * x326p.im
4049 + self.twiddle12.re * x425p.im
4050 + self.twiddle14.re * x524p.im
4051 + self.twiddle11.re * x623p.im
4052 + self.twiddle8.re * x722p.im
4053 + self.twiddle5.re * x821p.im
4054 + self.twiddle2.re * x920p.im
4055 + self.twiddle1.re * x1019p.im
4056 + self.twiddle4.re * x1118p.im
4057 + self.twiddle7.re * x1217p.im
4058 + self.twiddle10.re * x1316p.im
4059 + self.twiddle13.re * x1415p.im;
4060 let b326im_b = self.twiddle3.im * x128n.re
4061 + self.twiddle6.im * x227n.re
4062 + self.twiddle9.im * x326n.re
4063 + self.twiddle12.im * x425n.re
4064 + -self.twiddle14.im * x524n.re
4065 + -self.twiddle11.im * x623n.re
4066 + -self.twiddle8.im * x722n.re
4067 + -self.twiddle5.im * x821n.re
4068 + -self.twiddle2.im * x920n.re
4069 + self.twiddle1.im * x1019n.re
4070 + self.twiddle4.im * x1118n.re
4071 + self.twiddle7.im * x1217n.re
4072 + self.twiddle10.im * x1316n.re
4073 + self.twiddle13.im * x1415n.re;
4074 let b425im_a = buffer.load(0).im
4075 + self.twiddle4.re * x128p.im
4076 + self.twiddle8.re * x227p.im
4077 + self.twiddle12.re * x326p.im
4078 + self.twiddle13.re * x425p.im
4079 + self.twiddle9.re * x524p.im
4080 + self.twiddle5.re * x623p.im
4081 + self.twiddle1.re * x722p.im
4082 + self.twiddle3.re * x821p.im
4083 + self.twiddle7.re * x920p.im
4084 + self.twiddle11.re * x1019p.im
4085 + self.twiddle14.re * x1118p.im
4086 + self.twiddle10.re * x1217p.im
4087 + self.twiddle6.re * x1316p.im
4088 + self.twiddle2.re * x1415p.im;
4089 let b425im_b = self.twiddle4.im * x128n.re
4090 + self.twiddle8.im * x227n.re
4091 + self.twiddle12.im * x326n.re
4092 + -self.twiddle13.im * x425n.re
4093 + -self.twiddle9.im * x524n.re
4094 + -self.twiddle5.im * x623n.re
4095 + -self.twiddle1.im * x722n.re
4096 + self.twiddle3.im * x821n.re
4097 + self.twiddle7.im * x920n.re
4098 + self.twiddle11.im * x1019n.re
4099 + -self.twiddle14.im * x1118n.re
4100 + -self.twiddle10.im * x1217n.re
4101 + -self.twiddle6.im * x1316n.re
4102 + -self.twiddle2.im * x1415n.re;
4103 let b524im_a = buffer.load(0).im
4104 + self.twiddle5.re * x128p.im
4105 + self.twiddle10.re * x227p.im
4106 + self.twiddle14.re * x326p.im
4107 + self.twiddle9.re * x425p.im
4108 + self.twiddle4.re * x524p.im
4109 + self.twiddle1.re * x623p.im
4110 + self.twiddle6.re * x722p.im
4111 + self.twiddle11.re * x821p.im
4112 + self.twiddle13.re * x920p.im
4113 + self.twiddle8.re * x1019p.im
4114 + self.twiddle3.re * x1118p.im
4115 + self.twiddle2.re * x1217p.im
4116 + self.twiddle7.re * x1316p.im
4117 + self.twiddle12.re * x1415p.im;
4118 let b524im_b = self.twiddle5.im * x128n.re
4119 + self.twiddle10.im * x227n.re
4120 + -self.twiddle14.im * x326n.re
4121 + -self.twiddle9.im * x425n.re
4122 + -self.twiddle4.im * x524n.re
4123 + self.twiddle1.im * x623n.re
4124 + self.twiddle6.im * x722n.re
4125 + self.twiddle11.im * x821n.re
4126 + -self.twiddle13.im * x920n.re
4127 + -self.twiddle8.im * x1019n.re
4128 + -self.twiddle3.im * x1118n.re
4129 + self.twiddle2.im * x1217n.re
4130 + self.twiddle7.im * x1316n.re
4131 + self.twiddle12.im * x1415n.re;
4132 let b623im_a = buffer.load(0).im
4133 + self.twiddle6.re * x128p.im
4134 + self.twiddle12.re * x227p.im
4135 + self.twiddle11.re * x326p.im
4136 + self.twiddle5.re * x425p.im
4137 + self.twiddle1.re * x524p.im
4138 + self.twiddle7.re * x623p.im
4139 + self.twiddle13.re * x722p.im
4140 + self.twiddle10.re * x821p.im
4141 + self.twiddle4.re * x920p.im
4142 + self.twiddle2.re * x1019p.im
4143 + self.twiddle8.re * x1118p.im
4144 + self.twiddle14.re * x1217p.im
4145 + self.twiddle9.re * x1316p.im
4146 + self.twiddle3.re * x1415p.im;
4147 let b623im_b = self.twiddle6.im * x128n.re
4148 + self.twiddle12.im * x227n.re
4149 + -self.twiddle11.im * x326n.re
4150 + -self.twiddle5.im * x425n.re
4151 + self.twiddle1.im * x524n.re
4152 + self.twiddle7.im * x623n.re
4153 + self.twiddle13.im * x722n.re
4154 + -self.twiddle10.im * x821n.re
4155 + -self.twiddle4.im * x920n.re
4156 + self.twiddle2.im * x1019n.re
4157 + self.twiddle8.im * x1118n.re
4158 + self.twiddle14.im * x1217n.re
4159 + -self.twiddle9.im * x1316n.re
4160 + -self.twiddle3.im * x1415n.re;
4161 let b722im_a = buffer.load(0).im
4162 + self.twiddle7.re * x128p.im
4163 + self.twiddle14.re * x227p.im
4164 + self.twiddle8.re * x326p.im
4165 + self.twiddle1.re * x425p.im
4166 + self.twiddle6.re * x524p.im
4167 + self.twiddle13.re * x623p.im
4168 + self.twiddle9.re * x722p.im
4169 + self.twiddle2.re * x821p.im
4170 + self.twiddle5.re * x920p.im
4171 + self.twiddle12.re * x1019p.im
4172 + self.twiddle10.re * x1118p.im
4173 + self.twiddle3.re * x1217p.im
4174 + self.twiddle4.re * x1316p.im
4175 + self.twiddle11.re * x1415p.im;
4176 let b722im_b = self.twiddle7.im * x128n.re
4177 + self.twiddle14.im * x227n.re
4178 + -self.twiddle8.im * x326n.re
4179 + -self.twiddle1.im * x425n.re
4180 + self.twiddle6.im * x524n.re
4181 + self.twiddle13.im * x623n.re
4182 + -self.twiddle9.im * x722n.re
4183 + -self.twiddle2.im * x821n.re
4184 + self.twiddle5.im * x920n.re
4185 + self.twiddle12.im * x1019n.re
4186 + -self.twiddle10.im * x1118n.re
4187 + -self.twiddle3.im * x1217n.re
4188 + self.twiddle4.im * x1316n.re
4189 + self.twiddle11.im * x1415n.re;
4190 let b821im_a = buffer.load(0).im
4191 + self.twiddle8.re * x128p.im
4192 + self.twiddle13.re * x227p.im
4193 + self.twiddle5.re * x326p.im
4194 + self.twiddle3.re * x425p.im
4195 + self.twiddle11.re * x524p.im
4196 + self.twiddle10.re * x623p.im
4197 + self.twiddle2.re * x722p.im
4198 + self.twiddle6.re * x821p.im
4199 + self.twiddle14.re * x920p.im
4200 + self.twiddle7.re * x1019p.im
4201 + self.twiddle1.re * x1118p.im
4202 + self.twiddle9.re * x1217p.im
4203 + self.twiddle12.re * x1316p.im
4204 + self.twiddle4.re * x1415p.im;
4205 let b821im_b = self.twiddle8.im * x128n.re
4206 + -self.twiddle13.im * x227n.re
4207 + -self.twiddle5.im * x326n.re
4208 + self.twiddle3.im * x425n.re
4209 + self.twiddle11.im * x524n.re
4210 + -self.twiddle10.im * x623n.re
4211 + -self.twiddle2.im * x722n.re
4212 + self.twiddle6.im * x821n.re
4213 + self.twiddle14.im * x920n.re
4214 + -self.twiddle7.im * x1019n.re
4215 + self.twiddle1.im * x1118n.re
4216 + self.twiddle9.im * x1217n.re
4217 + -self.twiddle12.im * x1316n.re
4218 + -self.twiddle4.im * x1415n.re;
4219 let b920im_a = buffer.load(0).im
4220 + self.twiddle9.re * x128p.im
4221 + self.twiddle11.re * x227p.im
4222 + self.twiddle2.re * x326p.im
4223 + self.twiddle7.re * x425p.im
4224 + self.twiddle13.re * x524p.im
4225 + self.twiddle4.re * x623p.im
4226 + self.twiddle5.re * x722p.im
4227 + self.twiddle14.re * x821p.im
4228 + self.twiddle6.re * x920p.im
4229 + self.twiddle3.re * x1019p.im
4230 + self.twiddle12.re * x1118p.im
4231 + self.twiddle8.re * x1217p.im
4232 + self.twiddle1.re * x1316p.im
4233 + self.twiddle10.re * x1415p.im;
4234 let b920im_b = self.twiddle9.im * x128n.re
4235 + -self.twiddle11.im * x227n.re
4236 + -self.twiddle2.im * x326n.re
4237 + self.twiddle7.im * x425n.re
4238 + -self.twiddle13.im * x524n.re
4239 + -self.twiddle4.im * x623n.re
4240 + self.twiddle5.im * x722n.re
4241 + self.twiddle14.im * x821n.re
4242 + -self.twiddle6.im * x920n.re
4243 + self.twiddle3.im * x1019n.re
4244 + self.twiddle12.im * x1118n.re
4245 + -self.twiddle8.im * x1217n.re
4246 + self.twiddle1.im * x1316n.re
4247 + self.twiddle10.im * x1415n.re;
4248 let b1019im_a = buffer.load(0).im
4249 + self.twiddle10.re * x128p.im
4250 + self.twiddle9.re * x227p.im
4251 + self.twiddle1.re * x326p.im
4252 + self.twiddle11.re * x425p.im
4253 + self.twiddle8.re * x524p.im
4254 + self.twiddle2.re * x623p.im
4255 + self.twiddle12.re * x722p.im
4256 + self.twiddle7.re * x821p.im
4257 + self.twiddle3.re * x920p.im
4258 + self.twiddle13.re * x1019p.im
4259 + self.twiddle6.re * x1118p.im
4260 + self.twiddle4.re * x1217p.im
4261 + self.twiddle14.re * x1316p.im
4262 + self.twiddle5.re * x1415p.im;
4263 let b1019im_b = self.twiddle10.im * x128n.re
4264 + -self.twiddle9.im * x227n.re
4265 + self.twiddle1.im * x326n.re
4266 + self.twiddle11.im * x425n.re
4267 + -self.twiddle8.im * x524n.re
4268 + self.twiddle2.im * x623n.re
4269 + self.twiddle12.im * x722n.re
4270 + -self.twiddle7.im * x821n.re
4271 + self.twiddle3.im * x920n.re
4272 + self.twiddle13.im * x1019n.re
4273 + -self.twiddle6.im * x1118n.re
4274 + self.twiddle4.im * x1217n.re
4275 + self.twiddle14.im * x1316n.re
4276 + -self.twiddle5.im * x1415n.re;
4277 let b1118im_a = buffer.load(0).im
4278 + self.twiddle11.re * x128p.im
4279 + self.twiddle7.re * x227p.im
4280 + self.twiddle4.re * x326p.im
4281 + self.twiddle14.re * x425p.im
4282 + self.twiddle3.re * x524p.im
4283 + self.twiddle8.re * x623p.im
4284 + self.twiddle10.re * x722p.im
4285 + self.twiddle1.re * x821p.im
4286 + self.twiddle12.re * x920p.im
4287 + self.twiddle6.re * x1019p.im
4288 + self.twiddle5.re * x1118p.im
4289 + self.twiddle13.re * x1217p.im
4290 + self.twiddle2.re * x1316p.im
4291 + self.twiddle9.re * x1415p.im;
4292 let b1118im_b = self.twiddle11.im * x128n.re
4293 + -self.twiddle7.im * x227n.re
4294 + self.twiddle4.im * x326n.re
4295 + -self.twiddle14.im * x425n.re
4296 + -self.twiddle3.im * x524n.re
4297 + self.twiddle8.im * x623n.re
4298 + -self.twiddle10.im * x722n.re
4299 + self.twiddle1.im * x821n.re
4300 + self.twiddle12.im * x920n.re
4301 + -self.twiddle6.im * x1019n.re
4302 + self.twiddle5.im * x1118n.re
4303 + -self.twiddle13.im * x1217n.re
4304 + -self.twiddle2.im * x1316n.re
4305 + self.twiddle9.im * x1415n.re;
4306 let b1217im_a = buffer.load(0).im
4307 + self.twiddle12.re * x128p.im
4308 + self.twiddle5.re * x227p.im
4309 + self.twiddle7.re * x326p.im
4310 + self.twiddle10.re * x425p.im
4311 + self.twiddle2.re * x524p.im
4312 + self.twiddle14.re * x623p.im
4313 + self.twiddle3.re * x722p.im
4314 + self.twiddle9.re * x821p.im
4315 + self.twiddle8.re * x920p.im
4316 + self.twiddle4.re * x1019p.im
4317 + self.twiddle13.re * x1118p.im
4318 + self.twiddle1.re * x1217p.im
4319 + self.twiddle11.re * x1316p.im
4320 + self.twiddle6.re * x1415p.im;
4321 let b1217im_b = self.twiddle12.im * x128n.re
4322 + -self.twiddle5.im * x227n.re
4323 + self.twiddle7.im * x326n.re
4324 + -self.twiddle10.im * x425n.re
4325 + self.twiddle2.im * x524n.re
4326 + self.twiddle14.im * x623n.re
4327 + -self.twiddle3.im * x722n.re
4328 + self.twiddle9.im * x821n.re
4329 + -self.twiddle8.im * x920n.re
4330 + self.twiddle4.im * x1019n.re
4331 + -self.twiddle13.im * x1118n.re
4332 + -self.twiddle1.im * x1217n.re
4333 + self.twiddle11.im * x1316n.re
4334 + -self.twiddle6.im * x1415n.re;
4335 let b1316im_a = buffer.load(0).im
4336 + self.twiddle13.re * x128p.im
4337 + self.twiddle3.re * x227p.im
4338 + self.twiddle10.re * x326p.im
4339 + self.twiddle6.re * x425p.im
4340 + self.twiddle7.re * x524p.im
4341 + self.twiddle9.re * x623p.im
4342 + self.twiddle4.re * x722p.im
4343 + self.twiddle12.re * x821p.im
4344 + self.twiddle1.re * x920p.im
4345 + self.twiddle14.re * x1019p.im
4346 + self.twiddle2.re * x1118p.im
4347 + self.twiddle11.re * x1217p.im
4348 + self.twiddle5.re * x1316p.im
4349 + self.twiddle8.re * x1415p.im;
4350 let b1316im_b = self.twiddle13.im * x128n.re
4351 + -self.twiddle3.im * x227n.re
4352 + self.twiddle10.im * x326n.re
4353 + -self.twiddle6.im * x425n.re
4354 + self.twiddle7.im * x524n.re
4355 + -self.twiddle9.im * x623n.re
4356 + self.twiddle4.im * x722n.re
4357 + -self.twiddle12.im * x821n.re
4358 + self.twiddle1.im * x920n.re
4359 + self.twiddle14.im * x1019n.re
4360 + -self.twiddle2.im * x1118n.re
4361 + self.twiddle11.im * x1217n.re
4362 + -self.twiddle5.im * x1316n.re
4363 + self.twiddle8.im * x1415n.re;
4364 let b1415im_a = buffer.load(0).im
4365 + self.twiddle14.re * x128p.im
4366 + self.twiddle1.re * x227p.im
4367 + self.twiddle13.re * x326p.im
4368 + self.twiddle2.re * x425p.im
4369 + self.twiddle12.re * x524p.im
4370 + self.twiddle3.re * x623p.im
4371 + self.twiddle11.re * x722p.im
4372 + self.twiddle4.re * x821p.im
4373 + self.twiddle10.re * x920p.im
4374 + self.twiddle5.re * x1019p.im
4375 + self.twiddle9.re * x1118p.im
4376 + self.twiddle6.re * x1217p.im
4377 + self.twiddle8.re * x1316p.im
4378 + self.twiddle7.re * x1415p.im;
4379 let b1415im_b = self.twiddle14.im * x128n.re
4380 + -self.twiddle1.im * x227n.re
4381 + self.twiddle13.im * x326n.re
4382 + -self.twiddle2.im * x425n.re
4383 + self.twiddle12.im * x524n.re
4384 + -self.twiddle3.im * x623n.re
4385 + self.twiddle11.im * x722n.re
4386 + -self.twiddle4.im * x821n.re
4387 + self.twiddle10.im * x920n.re
4388 + -self.twiddle5.im * x1019n.re
4389 + self.twiddle9.im * x1118n.re
4390 + -self.twiddle6.im * x1217n.re
4391 + self.twiddle8.im * x1316n.re
4392 + -self.twiddle7.im * x1415n.re;
4393
4394 let out1re = b128re_a - b128re_b;
4395 let out1im = b128im_a + b128im_b;
4396 let out2re = b227re_a - b227re_b;
4397 let out2im = b227im_a + b227im_b;
4398 let out3re = b326re_a - b326re_b;
4399 let out3im = b326im_a + b326im_b;
4400 let out4re = b425re_a - b425re_b;
4401 let out4im = b425im_a + b425im_b;
4402 let out5re = b524re_a - b524re_b;
4403 let out5im = b524im_a + b524im_b;
4404 let out6re = b623re_a - b623re_b;
4405 let out6im = b623im_a + b623im_b;
4406 let out7re = b722re_a - b722re_b;
4407 let out7im = b722im_a + b722im_b;
4408 let out8re = b821re_a - b821re_b;
4409 let out8im = b821im_a + b821im_b;
4410 let out9re = b920re_a - b920re_b;
4411 let out9im = b920im_a + b920im_b;
4412 let out10re = b1019re_a - b1019re_b;
4413 let out10im = b1019im_a + b1019im_b;
4414 let out11re = b1118re_a - b1118re_b;
4415 let out11im = b1118im_a + b1118im_b;
4416 let out12re = b1217re_a - b1217re_b;
4417 let out12im = b1217im_a + b1217im_b;
4418 let out13re = b1316re_a - b1316re_b;
4419 let out13im = b1316im_a + b1316im_b;
4420 let out14re = b1415re_a - b1415re_b;
4421 let out14im = b1415im_a + b1415im_b;
4422 let out15re = b1415re_a + b1415re_b;
4423 let out15im = b1415im_a - b1415im_b;
4424 let out16re = b1316re_a + b1316re_b;
4425 let out16im = b1316im_a - b1316im_b;
4426 let out17re = b1217re_a + b1217re_b;
4427 let out17im = b1217im_a - b1217im_b;
4428 let out18re = b1118re_a + b1118re_b;
4429 let out18im = b1118im_a - b1118im_b;
4430 let out19re = b1019re_a + b1019re_b;
4431 let out19im = b1019im_a - b1019im_b;
4432 let out20re = b920re_a + b920re_b;
4433 let out20im = b920im_a - b920im_b;
4434 let out21re = b821re_a + b821re_b;
4435 let out21im = b821im_a - b821im_b;
4436 let out22re = b722re_a + b722re_b;
4437 let out22im = b722im_a - b722im_b;
4438 let out23re = b623re_a + b623re_b;
4439 let out23im = b623im_a - b623im_b;
4440 let out24re = b524re_a + b524re_b;
4441 let out24im = b524im_a - b524im_b;
4442 let out25re = b425re_a + b425re_b;
4443 let out25im = b425im_a - b425im_b;
4444 let out26re = b326re_a + b326re_b;
4445 let out26im = b326im_a - b326im_b;
4446 let out27re = b227re_a + b227re_b;
4447 let out27im = b227im_a - b227im_b;
4448 let out28re = b128re_a + b128re_b;
4449 let out28im = b128im_a - b128im_b;
4450 buffer.store(sum, 0);
4451 buffer.store(
4452 Complex {
4453 re: out1re,
4454 im: out1im,
4455 },
4456 1,
4457 );
4458 buffer.store(
4459 Complex {
4460 re: out2re,
4461 im: out2im,
4462 },
4463 2,
4464 );
4465 buffer.store(
4466 Complex {
4467 re: out3re,
4468 im: out3im,
4469 },
4470 3,
4471 );
4472 buffer.store(
4473 Complex {
4474 re: out4re,
4475 im: out4im,
4476 },
4477 4,
4478 );
4479 buffer.store(
4480 Complex {
4481 re: out5re,
4482 im: out5im,
4483 },
4484 5,
4485 );
4486 buffer.store(
4487 Complex {
4488 re: out6re,
4489 im: out6im,
4490 },
4491 6,
4492 );
4493 buffer.store(
4494 Complex {
4495 re: out7re,
4496 im: out7im,
4497 },
4498 7,
4499 );
4500 buffer.store(
4501 Complex {
4502 re: out8re,
4503 im: out8im,
4504 },
4505 8,
4506 );
4507 buffer.store(
4508 Complex {
4509 re: out9re,
4510 im: out9im,
4511 },
4512 9,
4513 );
4514 buffer.store(
4515 Complex {
4516 re: out10re,
4517 im: out10im,
4518 },
4519 10,
4520 );
4521 buffer.store(
4522 Complex {
4523 re: out11re,
4524 im: out11im,
4525 },
4526 11,
4527 );
4528 buffer.store(
4529 Complex {
4530 re: out12re,
4531 im: out12im,
4532 },
4533 12,
4534 );
4535 buffer.store(
4536 Complex {
4537 re: out13re,
4538 im: out13im,
4539 },
4540 13,
4541 );
4542 buffer.store(
4543 Complex {
4544 re: out14re,
4545 im: out14im,
4546 },
4547 14,
4548 );
4549 buffer.store(
4550 Complex {
4551 re: out15re,
4552 im: out15im,
4553 },
4554 15,
4555 );
4556 buffer.store(
4557 Complex {
4558 re: out16re,
4559 im: out16im,
4560 },
4561 16,
4562 );
4563 buffer.store(
4564 Complex {
4565 re: out17re,
4566 im: out17im,
4567 },
4568 17,
4569 );
4570 buffer.store(
4571 Complex {
4572 re: out18re,
4573 im: out18im,
4574 },
4575 18,
4576 );
4577 buffer.store(
4578 Complex {
4579 re: out19re,
4580 im: out19im,
4581 },
4582 19,
4583 );
4584 buffer.store(
4585 Complex {
4586 re: out20re,
4587 im: out20im,
4588 },
4589 20,
4590 );
4591 buffer.store(
4592 Complex {
4593 re: out21re,
4594 im: out21im,
4595 },
4596 21,
4597 );
4598 buffer.store(
4599 Complex {
4600 re: out22re,
4601 im: out22im,
4602 },
4603 22,
4604 );
4605 buffer.store(
4606 Complex {
4607 re: out23re,
4608 im: out23im,
4609 },
4610 23,
4611 );
4612 buffer.store(
4613 Complex {
4614 re: out24re,
4615 im: out24im,
4616 },
4617 24,
4618 );
4619 buffer.store(
4620 Complex {
4621 re: out25re,
4622 im: out25im,
4623 },
4624 25,
4625 );
4626 buffer.store(
4627 Complex {
4628 re: out26re,
4629 im: out26im,
4630 },
4631 26,
4632 );
4633 buffer.store(
4634 Complex {
4635 re: out27re,
4636 im: out27im,
4637 },
4638 27,
4639 );
4640 buffer.store(
4641 Complex {
4642 re: out28re,
4643 im: out28im,
4644 },
4645 28,
4646 );
4647 }
4648 }
4649 pub struct Butterfly31<T> {
4650 twiddle1: Complex<T>,
4651 twiddle2: Complex<T>,
4652 twiddle3: Complex<T>,
4653 twiddle4: Complex<T>,
4654 twiddle5: Complex<T>,
4655 twiddle6: Complex<T>,
4656 twiddle7: Complex<T>,
4657 twiddle8: Complex<T>,
4658 twiddle9: Complex<T>,
4659 twiddle10: Complex<T>,
4660 twiddle11: Complex<T>,
4661 twiddle12: Complex<T>,
4662 twiddle13: Complex<T>,
4663 twiddle14: Complex<T>,
4664 twiddle15: Complex<T>,
4665 direction: FftDirection,
4666 }
4667 boilerplate_fft_butterfly!(Butterfly31, 31, |this: &Butterfly31<_>| this.direction);
4668 impl<T: FftNum> Butterfly31<T> {
4669 pub fn new(direction: FftDirection) -> Self {
4670 let twiddle1: Complex<T> = twiddles::compute_twiddle(1, 31, direction);
4671 let twiddle2: Complex<T> = twiddles::compute_twiddle(2, 31, direction);
4672 let twiddle3: Complex<T> = twiddles::compute_twiddle(3, 31, direction);
4673 let twiddle4: Complex<T> = twiddles::compute_twiddle(4, 31, direction);
4674 let twiddle5: Complex<T> = twiddles::compute_twiddle(5, 31, direction);
4675 let twiddle6: Complex<T> = twiddles::compute_twiddle(6, 31, direction);
4676 let twiddle7: Complex<T> = twiddles::compute_twiddle(7, 31, direction);
4677 let twiddle8: Complex<T> = twiddles::compute_twiddle(8, 31, direction);
4678 let twiddle9: Complex<T> = twiddles::compute_twiddle(9, 31, direction);
4679 let twiddle10: Complex<T> = twiddles::compute_twiddle(10, 31, direction);
4680 let twiddle11: Complex<T> = twiddles::compute_twiddle(11, 31, direction);
4681 let twiddle12: Complex<T> = twiddles::compute_twiddle(12, 31, direction);
4682 let twiddle13: Complex<T> = twiddles::compute_twiddle(13, 31, direction);
4683 let twiddle14: Complex<T> = twiddles::compute_twiddle(14, 31, direction);
4684 let twiddle15: Complex<T> = twiddles::compute_twiddle(15, 31, direction);
4685 Self {
4686 twiddle1,
4687 twiddle2,
4688 twiddle3,
4689 twiddle4,
4690 twiddle5,
4691 twiddle6,
4692 twiddle7,
4693 twiddle8,
4694 twiddle9,
4695 twiddle10,
4696 twiddle11,
4697 twiddle12,
4698 twiddle13,
4699 twiddle14,
4700 twiddle15,
4701 direction,
4702 }
4703 }
4704
4705 #[inline(never)]
4706 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
4707 // This function was derived in the same manner as the butterflies for length 3, 5 and 7.
4708 // However, instead of doing it by hand the actual code is autogenerated
4709 // with the `genbutterflies.py` script in the `tools` directory.
4710 let x130p = buffer.load(1) + buffer.load(30);
4711 let x130n = buffer.load(1) - buffer.load(30);
4712 let x229p = buffer.load(2) + buffer.load(29);
4713 let x229n = buffer.load(2) - buffer.load(29);
4714 let x328p = buffer.load(3) + buffer.load(28);
4715 let x328n = buffer.load(3) - buffer.load(28);
4716 let x427p = buffer.load(4) + buffer.load(27);
4717 let x427n = buffer.load(4) - buffer.load(27);
4718 let x526p = buffer.load(5) + buffer.load(26);
4719 let x526n = buffer.load(5) - buffer.load(26);
4720 let x625p = buffer.load(6) + buffer.load(25);
4721 let x625n = buffer.load(6) - buffer.load(25);
4722 let x724p = buffer.load(7) + buffer.load(24);
4723 let x724n = buffer.load(7) - buffer.load(24);
4724 let x823p = buffer.load(8) + buffer.load(23);
4725 let x823n = buffer.load(8) - buffer.load(23);
4726 let x922p = buffer.load(9) + buffer.load(22);
4727 let x922n = buffer.load(9) - buffer.load(22);
4728 let x1021p = buffer.load(10) + buffer.load(21);
4729 let x1021n = buffer.load(10) - buffer.load(21);
4730 let x1120p = buffer.load(11) + buffer.load(20);
4731 let x1120n = buffer.load(11) - buffer.load(20);
4732 let x1219p = buffer.load(12) + buffer.load(19);
4733 let x1219n = buffer.load(12) - buffer.load(19);
4734 let x1318p = buffer.load(13) + buffer.load(18);
4735 let x1318n = buffer.load(13) - buffer.load(18);
4736 let x1417p = buffer.load(14) + buffer.load(17);
4737 let x1417n = buffer.load(14) - buffer.load(17);
4738 let x1516p = buffer.load(15) + buffer.load(16);
4739 let x1516n = buffer.load(15) - buffer.load(16);
4740 let sum = buffer.load(0)
4741 + x130p
4742 + x229p
4743 + x328p
4744 + x427p
4745 + x526p
4746 + x625p
4747 + x724p
4748 + x823p
4749 + x922p
4750 + x1021p
4751 + x1120p
4752 + x1219p
4753 + x1318p
4754 + x1417p
4755 + x1516p;
4756 let b130re_a = buffer.load(0).re
4757 + self.twiddle1.re * x130p.re
4758 + self.twiddle2.re * x229p.re
4759 + self.twiddle3.re * x328p.re
4760 + self.twiddle4.re * x427p.re
4761 + self.twiddle5.re * x526p.re
4762 + self.twiddle6.re * x625p.re
4763 + self.twiddle7.re * x724p.re
4764 + self.twiddle8.re * x823p.re
4765 + self.twiddle9.re * x922p.re
4766 + self.twiddle10.re * x1021p.re
4767 + self.twiddle11.re * x1120p.re
4768 + self.twiddle12.re * x1219p.re
4769 + self.twiddle13.re * x1318p.re
4770 + self.twiddle14.re * x1417p.re
4771 + self.twiddle15.re * x1516p.re;
4772 let b130re_b = self.twiddle1.im * x130n.im
4773 + self.twiddle2.im * x229n.im
4774 + self.twiddle3.im * x328n.im
4775 + self.twiddle4.im * x427n.im
4776 + self.twiddle5.im * x526n.im
4777 + self.twiddle6.im * x625n.im
4778 + self.twiddle7.im * x724n.im
4779 + self.twiddle8.im * x823n.im
4780 + self.twiddle9.im * x922n.im
4781 + self.twiddle10.im * x1021n.im
4782 + self.twiddle11.im * x1120n.im
4783 + self.twiddle12.im * x1219n.im
4784 + self.twiddle13.im * x1318n.im
4785 + self.twiddle14.im * x1417n.im
4786 + self.twiddle15.im * x1516n.im;
4787 let b229re_a = buffer.load(0).re
4788 + self.twiddle2.re * x130p.re
4789 + self.twiddle4.re * x229p.re
4790 + self.twiddle6.re * x328p.re
4791 + self.twiddle8.re * x427p.re
4792 + self.twiddle10.re * x526p.re
4793 + self.twiddle12.re * x625p.re
4794 + self.twiddle14.re * x724p.re
4795 + self.twiddle15.re * x823p.re
4796 + self.twiddle13.re * x922p.re
4797 + self.twiddle11.re * x1021p.re
4798 + self.twiddle9.re * x1120p.re
4799 + self.twiddle7.re * x1219p.re
4800 + self.twiddle5.re * x1318p.re
4801 + self.twiddle3.re * x1417p.re
4802 + self.twiddle1.re * x1516p.re;
4803 let b229re_b = self.twiddle2.im * x130n.im
4804 + self.twiddle4.im * x229n.im
4805 + self.twiddle6.im * x328n.im
4806 + self.twiddle8.im * x427n.im
4807 + self.twiddle10.im * x526n.im
4808 + self.twiddle12.im * x625n.im
4809 + self.twiddle14.im * x724n.im
4810 + -self.twiddle15.im * x823n.im
4811 + -self.twiddle13.im * x922n.im
4812 + -self.twiddle11.im * x1021n.im
4813 + -self.twiddle9.im * x1120n.im
4814 + -self.twiddle7.im * x1219n.im
4815 + -self.twiddle5.im * x1318n.im
4816 + -self.twiddle3.im * x1417n.im
4817 + -self.twiddle1.im * x1516n.im;
4818 let b328re_a = buffer.load(0).re
4819 + self.twiddle3.re * x130p.re
4820 + self.twiddle6.re * x229p.re
4821 + self.twiddle9.re * x328p.re
4822 + self.twiddle12.re * x427p.re
4823 + self.twiddle15.re * x526p.re
4824 + self.twiddle13.re * x625p.re
4825 + self.twiddle10.re * x724p.re
4826 + self.twiddle7.re * x823p.re
4827 + self.twiddle4.re * x922p.re
4828 + self.twiddle1.re * x1021p.re
4829 + self.twiddle2.re * x1120p.re
4830 + self.twiddle5.re * x1219p.re
4831 + self.twiddle8.re * x1318p.re
4832 + self.twiddle11.re * x1417p.re
4833 + self.twiddle14.re * x1516p.re;
4834 let b328re_b = self.twiddle3.im * x130n.im
4835 + self.twiddle6.im * x229n.im
4836 + self.twiddle9.im * x328n.im
4837 + self.twiddle12.im * x427n.im
4838 + self.twiddle15.im * x526n.im
4839 + -self.twiddle13.im * x625n.im
4840 + -self.twiddle10.im * x724n.im
4841 + -self.twiddle7.im * x823n.im
4842 + -self.twiddle4.im * x922n.im
4843 + -self.twiddle1.im * x1021n.im
4844 + self.twiddle2.im * x1120n.im
4845 + self.twiddle5.im * x1219n.im
4846 + self.twiddle8.im * x1318n.im
4847 + self.twiddle11.im * x1417n.im
4848 + self.twiddle14.im * x1516n.im;
4849 let b427re_a = buffer.load(0).re
4850 + self.twiddle4.re * x130p.re
4851 + self.twiddle8.re * x229p.re
4852 + self.twiddle12.re * x328p.re
4853 + self.twiddle15.re * x427p.re
4854 + self.twiddle11.re * x526p.re
4855 + self.twiddle7.re * x625p.re
4856 + self.twiddle3.re * x724p.re
4857 + self.twiddle1.re * x823p.re
4858 + self.twiddle5.re * x922p.re
4859 + self.twiddle9.re * x1021p.re
4860 + self.twiddle13.re * x1120p.re
4861 + self.twiddle14.re * x1219p.re
4862 + self.twiddle10.re * x1318p.re
4863 + self.twiddle6.re * x1417p.re
4864 + self.twiddle2.re * x1516p.re;
4865 let b427re_b = self.twiddle4.im * x130n.im
4866 + self.twiddle8.im * x229n.im
4867 + self.twiddle12.im * x328n.im
4868 + -self.twiddle15.im * x427n.im
4869 + -self.twiddle11.im * x526n.im
4870 + -self.twiddle7.im * x625n.im
4871 + -self.twiddle3.im * x724n.im
4872 + self.twiddle1.im * x823n.im
4873 + self.twiddle5.im * x922n.im
4874 + self.twiddle9.im * x1021n.im
4875 + self.twiddle13.im * x1120n.im
4876 + -self.twiddle14.im * x1219n.im
4877 + -self.twiddle10.im * x1318n.im
4878 + -self.twiddle6.im * x1417n.im
4879 + -self.twiddle2.im * x1516n.im;
4880 let b526re_a = buffer.load(0).re
4881 + self.twiddle5.re * x130p.re
4882 + self.twiddle10.re * x229p.re
4883 + self.twiddle15.re * x328p.re
4884 + self.twiddle11.re * x427p.re
4885 + self.twiddle6.re * x526p.re
4886 + self.twiddle1.re * x625p.re
4887 + self.twiddle4.re * x724p.re
4888 + self.twiddle9.re * x823p.re
4889 + self.twiddle14.re * x922p.re
4890 + self.twiddle12.re * x1021p.re
4891 + self.twiddle7.re * x1120p.re
4892 + self.twiddle2.re * x1219p.re
4893 + self.twiddle3.re * x1318p.re
4894 + self.twiddle8.re * x1417p.re
4895 + self.twiddle13.re * x1516p.re;
4896 let b526re_b = self.twiddle5.im * x130n.im
4897 + self.twiddle10.im * x229n.im
4898 + self.twiddle15.im * x328n.im
4899 + -self.twiddle11.im * x427n.im
4900 + -self.twiddle6.im * x526n.im
4901 + -self.twiddle1.im * x625n.im
4902 + self.twiddle4.im * x724n.im
4903 + self.twiddle9.im * x823n.im
4904 + self.twiddle14.im * x922n.im
4905 + -self.twiddle12.im * x1021n.im
4906 + -self.twiddle7.im * x1120n.im
4907 + -self.twiddle2.im * x1219n.im
4908 + self.twiddle3.im * x1318n.im
4909 + self.twiddle8.im * x1417n.im
4910 + self.twiddle13.im * x1516n.im;
4911 let b625re_a = buffer.load(0).re
4912 + self.twiddle6.re * x130p.re
4913 + self.twiddle12.re * x229p.re
4914 + self.twiddle13.re * x328p.re
4915 + self.twiddle7.re * x427p.re
4916 + self.twiddle1.re * x526p.re
4917 + self.twiddle5.re * x625p.re
4918 + self.twiddle11.re * x724p.re
4919 + self.twiddle14.re * x823p.re
4920 + self.twiddle8.re * x922p.re
4921 + self.twiddle2.re * x1021p.re
4922 + self.twiddle4.re * x1120p.re
4923 + self.twiddle10.re * x1219p.re
4924 + self.twiddle15.re * x1318p.re
4925 + self.twiddle9.re * x1417p.re
4926 + self.twiddle3.re * x1516p.re;
4927 let b625re_b = self.twiddle6.im * x130n.im
4928 + self.twiddle12.im * x229n.im
4929 + -self.twiddle13.im * x328n.im
4930 + -self.twiddle7.im * x427n.im
4931 + -self.twiddle1.im * x526n.im
4932 + self.twiddle5.im * x625n.im
4933 + self.twiddle11.im * x724n.im
4934 + -self.twiddle14.im * x823n.im
4935 + -self.twiddle8.im * x922n.im
4936 + -self.twiddle2.im * x1021n.im
4937 + self.twiddle4.im * x1120n.im
4938 + self.twiddle10.im * x1219n.im
4939 + -self.twiddle15.im * x1318n.im
4940 + -self.twiddle9.im * x1417n.im
4941 + -self.twiddle3.im * x1516n.im;
4942 let b724re_a = buffer.load(0).re
4943 + self.twiddle7.re * x130p.re
4944 + self.twiddle14.re * x229p.re
4945 + self.twiddle10.re * x328p.re
4946 + self.twiddle3.re * x427p.re
4947 + self.twiddle4.re * x526p.re
4948 + self.twiddle11.re * x625p.re
4949 + self.twiddle13.re * x724p.re
4950 + self.twiddle6.re * x823p.re
4951 + self.twiddle1.re * x922p.re
4952 + self.twiddle8.re * x1021p.re
4953 + self.twiddle15.re * x1120p.re
4954 + self.twiddle9.re * x1219p.re
4955 + self.twiddle2.re * x1318p.re
4956 + self.twiddle5.re * x1417p.re
4957 + self.twiddle12.re * x1516p.re;
4958 let b724re_b = self.twiddle7.im * x130n.im
4959 + self.twiddle14.im * x229n.im
4960 + -self.twiddle10.im * x328n.im
4961 + -self.twiddle3.im * x427n.im
4962 + self.twiddle4.im * x526n.im
4963 + self.twiddle11.im * x625n.im
4964 + -self.twiddle13.im * x724n.im
4965 + -self.twiddle6.im * x823n.im
4966 + self.twiddle1.im * x922n.im
4967 + self.twiddle8.im * x1021n.im
4968 + self.twiddle15.im * x1120n.im
4969 + -self.twiddle9.im * x1219n.im
4970 + -self.twiddle2.im * x1318n.im
4971 + self.twiddle5.im * x1417n.im
4972 + self.twiddle12.im * x1516n.im;
4973 let b823re_a = buffer.load(0).re
4974 + self.twiddle8.re * x130p.re
4975 + self.twiddle15.re * x229p.re
4976 + self.twiddle7.re * x328p.re
4977 + self.twiddle1.re * x427p.re
4978 + self.twiddle9.re * x526p.re
4979 + self.twiddle14.re * x625p.re
4980 + self.twiddle6.re * x724p.re
4981 + self.twiddle2.re * x823p.re
4982 + self.twiddle10.re * x922p.re
4983 + self.twiddle13.re * x1021p.re
4984 + self.twiddle5.re * x1120p.re
4985 + self.twiddle3.re * x1219p.re
4986 + self.twiddle11.re * x1318p.re
4987 + self.twiddle12.re * x1417p.re
4988 + self.twiddle4.re * x1516p.re;
4989 let b823re_b = self.twiddle8.im * x130n.im
4990 + -self.twiddle15.im * x229n.im
4991 + -self.twiddle7.im * x328n.im
4992 + self.twiddle1.im * x427n.im
4993 + self.twiddle9.im * x526n.im
4994 + -self.twiddle14.im * x625n.im
4995 + -self.twiddle6.im * x724n.im
4996 + self.twiddle2.im * x823n.im
4997 + self.twiddle10.im * x922n.im
4998 + -self.twiddle13.im * x1021n.im
4999 + -self.twiddle5.im * x1120n.im
5000 + self.twiddle3.im * x1219n.im
5001 + self.twiddle11.im * x1318n.im
5002 + -self.twiddle12.im * x1417n.im
5003 + -self.twiddle4.im * x1516n.im;
5004 let b922re_a = buffer.load(0).re
5005 + self.twiddle9.re * x130p.re
5006 + self.twiddle13.re * x229p.re
5007 + self.twiddle4.re * x328p.re
5008 + self.twiddle5.re * x427p.re
5009 + self.twiddle14.re * x526p.re
5010 + self.twiddle8.re * x625p.re
5011 + self.twiddle1.re * x724p.re
5012 + self.twiddle10.re * x823p.re
5013 + self.twiddle12.re * x922p.re
5014 + self.twiddle3.re * x1021p.re
5015 + self.twiddle6.re * x1120p.re
5016 + self.twiddle15.re * x1219p.re
5017 + self.twiddle7.re * x1318p.re
5018 + self.twiddle2.re * x1417p.re
5019 + self.twiddle11.re * x1516p.re;
5020 let b922re_b = self.twiddle9.im * x130n.im
5021 + -self.twiddle13.im * x229n.im
5022 + -self.twiddle4.im * x328n.im
5023 + self.twiddle5.im * x427n.im
5024 + self.twiddle14.im * x526n.im
5025 + -self.twiddle8.im * x625n.im
5026 + self.twiddle1.im * x724n.im
5027 + self.twiddle10.im * x823n.im
5028 + -self.twiddle12.im * x922n.im
5029 + -self.twiddle3.im * x1021n.im
5030 + self.twiddle6.im * x1120n.im
5031 + self.twiddle15.im * x1219n.im
5032 + -self.twiddle7.im * x1318n.im
5033 + self.twiddle2.im * x1417n.im
5034 + self.twiddle11.im * x1516n.im;
5035 let b1021re_a = buffer.load(0).re
5036 + self.twiddle10.re * x130p.re
5037 + self.twiddle11.re * x229p.re
5038 + self.twiddle1.re * x328p.re
5039 + self.twiddle9.re * x427p.re
5040 + self.twiddle12.re * x526p.re
5041 + self.twiddle2.re * x625p.re
5042 + self.twiddle8.re * x724p.re
5043 + self.twiddle13.re * x823p.re
5044 + self.twiddle3.re * x922p.re
5045 + self.twiddle7.re * x1021p.re
5046 + self.twiddle14.re * x1120p.re
5047 + self.twiddle4.re * x1219p.re
5048 + self.twiddle6.re * x1318p.re
5049 + self.twiddle15.re * x1417p.re
5050 + self.twiddle5.re * x1516p.re;
5051 let b1021re_b = self.twiddle10.im * x130n.im
5052 + -self.twiddle11.im * x229n.im
5053 + -self.twiddle1.im * x328n.im
5054 + self.twiddle9.im * x427n.im
5055 + -self.twiddle12.im * x526n.im
5056 + -self.twiddle2.im * x625n.im
5057 + self.twiddle8.im * x724n.im
5058 + -self.twiddle13.im * x823n.im
5059 + -self.twiddle3.im * x922n.im
5060 + self.twiddle7.im * x1021n.im
5061 + -self.twiddle14.im * x1120n.im
5062 + -self.twiddle4.im * x1219n.im
5063 + self.twiddle6.im * x1318n.im
5064 + -self.twiddle15.im * x1417n.im
5065 + -self.twiddle5.im * x1516n.im;
5066 let b1120re_a = buffer.load(0).re
5067 + self.twiddle11.re * x130p.re
5068 + self.twiddle9.re * x229p.re
5069 + self.twiddle2.re * x328p.re
5070 + self.twiddle13.re * x427p.re
5071 + self.twiddle7.re * x526p.re
5072 + self.twiddle4.re * x625p.re
5073 + self.twiddle15.re * x724p.re
5074 + self.twiddle5.re * x823p.re
5075 + self.twiddle6.re * x922p.re
5076 + self.twiddle14.re * x1021p.re
5077 + self.twiddle3.re * x1120p.re
5078 + self.twiddle8.re * x1219p.re
5079 + self.twiddle12.re * x1318p.re
5080 + self.twiddle1.re * x1417p.re
5081 + self.twiddle10.re * x1516p.re;
5082 let b1120re_b = self.twiddle11.im * x130n.im
5083 + -self.twiddle9.im * x229n.im
5084 + self.twiddle2.im * x328n.im
5085 + self.twiddle13.im * x427n.im
5086 + -self.twiddle7.im * x526n.im
5087 + self.twiddle4.im * x625n.im
5088 + self.twiddle15.im * x724n.im
5089 + -self.twiddle5.im * x823n.im
5090 + self.twiddle6.im * x922n.im
5091 + -self.twiddle14.im * x1021n.im
5092 + -self.twiddle3.im * x1120n.im
5093 + self.twiddle8.im * x1219n.im
5094 + -self.twiddle12.im * x1318n.im
5095 + -self.twiddle1.im * x1417n.im
5096 + self.twiddle10.im * x1516n.im;
5097 let b1219re_a = buffer.load(0).re
5098 + self.twiddle12.re * x130p.re
5099 + self.twiddle7.re * x229p.re
5100 + self.twiddle5.re * x328p.re
5101 + self.twiddle14.re * x427p.re
5102 + self.twiddle2.re * x526p.re
5103 + self.twiddle10.re * x625p.re
5104 + self.twiddle9.re * x724p.re
5105 + self.twiddle3.re * x823p.re
5106 + self.twiddle15.re * x922p.re
5107 + self.twiddle4.re * x1021p.re
5108 + self.twiddle8.re * x1120p.re
5109 + self.twiddle11.re * x1219p.re
5110 + self.twiddle1.re * x1318p.re
5111 + self.twiddle13.re * x1417p.re
5112 + self.twiddle6.re * x1516p.re;
5113 let b1219re_b = self.twiddle12.im * x130n.im
5114 + -self.twiddle7.im * x229n.im
5115 + self.twiddle5.im * x328n.im
5116 + -self.twiddle14.im * x427n.im
5117 + -self.twiddle2.im * x526n.im
5118 + self.twiddle10.im * x625n.im
5119 + -self.twiddle9.im * x724n.im
5120 + self.twiddle3.im * x823n.im
5121 + self.twiddle15.im * x922n.im
5122 + -self.twiddle4.im * x1021n.im
5123 + self.twiddle8.im * x1120n.im
5124 + -self.twiddle11.im * x1219n.im
5125 + self.twiddle1.im * x1318n.im
5126 + self.twiddle13.im * x1417n.im
5127 + -self.twiddle6.im * x1516n.im;
5128 let b1318re_a = buffer.load(0).re
5129 + self.twiddle13.re * x130p.re
5130 + self.twiddle5.re * x229p.re
5131 + self.twiddle8.re * x328p.re
5132 + self.twiddle10.re * x427p.re
5133 + self.twiddle3.re * x526p.re
5134 + self.twiddle15.re * x625p.re
5135 + self.twiddle2.re * x724p.re
5136 + self.twiddle11.re * x823p.re
5137 + self.twiddle7.re * x922p.re
5138 + self.twiddle6.re * x1021p.re
5139 + self.twiddle12.re * x1120p.re
5140 + self.twiddle1.re * x1219p.re
5141 + self.twiddle14.re * x1318p.re
5142 + self.twiddle4.re * x1417p.re
5143 + self.twiddle9.re * x1516p.re;
5144 let b1318re_b = self.twiddle13.im * x130n.im
5145 + -self.twiddle5.im * x229n.im
5146 + self.twiddle8.im * x328n.im
5147 + -self.twiddle10.im * x427n.im
5148 + self.twiddle3.im * x526n.im
5149 + -self.twiddle15.im * x625n.im
5150 + -self.twiddle2.im * x724n.im
5151 + self.twiddle11.im * x823n.im
5152 + -self.twiddle7.im * x922n.im
5153 + self.twiddle6.im * x1021n.im
5154 + -self.twiddle12.im * x1120n.im
5155 + self.twiddle1.im * x1219n.im
5156 + self.twiddle14.im * x1318n.im
5157 + -self.twiddle4.im * x1417n.im
5158 + self.twiddle9.im * x1516n.im;
5159 let b1417re_a = buffer.load(0).re
5160 + self.twiddle14.re * x130p.re
5161 + self.twiddle3.re * x229p.re
5162 + self.twiddle11.re * x328p.re
5163 + self.twiddle6.re * x427p.re
5164 + self.twiddle8.re * x526p.re
5165 + self.twiddle9.re * x625p.re
5166 + self.twiddle5.re * x724p.re
5167 + self.twiddle12.re * x823p.re
5168 + self.twiddle2.re * x922p.re
5169 + self.twiddle15.re * x1021p.re
5170 + self.twiddle1.re * x1120p.re
5171 + self.twiddle13.re * x1219p.re
5172 + self.twiddle4.re * x1318p.re
5173 + self.twiddle10.re * x1417p.re
5174 + self.twiddle7.re * x1516p.re;
5175 let b1417re_b = self.twiddle14.im * x130n.im
5176 + -self.twiddle3.im * x229n.im
5177 + self.twiddle11.im * x328n.im
5178 + -self.twiddle6.im * x427n.im
5179 + self.twiddle8.im * x526n.im
5180 + -self.twiddle9.im * x625n.im
5181 + self.twiddle5.im * x724n.im
5182 + -self.twiddle12.im * x823n.im
5183 + self.twiddle2.im * x922n.im
5184 + -self.twiddle15.im * x1021n.im
5185 + -self.twiddle1.im * x1120n.im
5186 + self.twiddle13.im * x1219n.im
5187 + -self.twiddle4.im * x1318n.im
5188 + self.twiddle10.im * x1417n.im
5189 + -self.twiddle7.im * x1516n.im;
5190 let b1516re_a = buffer.load(0).re
5191 + self.twiddle15.re * x130p.re
5192 + self.twiddle1.re * x229p.re
5193 + self.twiddle14.re * x328p.re
5194 + self.twiddle2.re * x427p.re
5195 + self.twiddle13.re * x526p.re
5196 + self.twiddle3.re * x625p.re
5197 + self.twiddle12.re * x724p.re
5198 + self.twiddle4.re * x823p.re
5199 + self.twiddle11.re * x922p.re
5200 + self.twiddle5.re * x1021p.re
5201 + self.twiddle10.re * x1120p.re
5202 + self.twiddle6.re * x1219p.re
5203 + self.twiddle9.re * x1318p.re
5204 + self.twiddle7.re * x1417p.re
5205 + self.twiddle8.re * x1516p.re;
5206 let b1516re_b = self.twiddle15.im * x130n.im
5207 + -self.twiddle1.im * x229n.im
5208 + self.twiddle14.im * x328n.im
5209 + -self.twiddle2.im * x427n.im
5210 + self.twiddle13.im * x526n.im
5211 + -self.twiddle3.im * x625n.im
5212 + self.twiddle12.im * x724n.im
5213 + -self.twiddle4.im * x823n.im
5214 + self.twiddle11.im * x922n.im
5215 + -self.twiddle5.im * x1021n.im
5216 + self.twiddle10.im * x1120n.im
5217 + -self.twiddle6.im * x1219n.im
5218 + self.twiddle9.im * x1318n.im
5219 + -self.twiddle7.im * x1417n.im
5220 + self.twiddle8.im * x1516n.im;
5221
5222 let b130im_a = buffer.load(0).im
5223 + self.twiddle1.re * x130p.im
5224 + self.twiddle2.re * x229p.im
5225 + self.twiddle3.re * x328p.im
5226 + self.twiddle4.re * x427p.im
5227 + self.twiddle5.re * x526p.im
5228 + self.twiddle6.re * x625p.im
5229 + self.twiddle7.re * x724p.im
5230 + self.twiddle8.re * x823p.im
5231 + self.twiddle9.re * x922p.im
5232 + self.twiddle10.re * x1021p.im
5233 + self.twiddle11.re * x1120p.im
5234 + self.twiddle12.re * x1219p.im
5235 + self.twiddle13.re * x1318p.im
5236 + self.twiddle14.re * x1417p.im
5237 + self.twiddle15.re * x1516p.im;
5238 let b130im_b = self.twiddle1.im * x130n.re
5239 + self.twiddle2.im * x229n.re
5240 + self.twiddle3.im * x328n.re
5241 + self.twiddle4.im * x427n.re
5242 + self.twiddle5.im * x526n.re
5243 + self.twiddle6.im * x625n.re
5244 + self.twiddle7.im * x724n.re
5245 + self.twiddle8.im * x823n.re
5246 + self.twiddle9.im * x922n.re
5247 + self.twiddle10.im * x1021n.re
5248 + self.twiddle11.im * x1120n.re
5249 + self.twiddle12.im * x1219n.re
5250 + self.twiddle13.im * x1318n.re
5251 + self.twiddle14.im * x1417n.re
5252 + self.twiddle15.im * x1516n.re;
5253 let b229im_a = buffer.load(0).im
5254 + self.twiddle2.re * x130p.im
5255 + self.twiddle4.re * x229p.im
5256 + self.twiddle6.re * x328p.im
5257 + self.twiddle8.re * x427p.im
5258 + self.twiddle10.re * x526p.im
5259 + self.twiddle12.re * x625p.im
5260 + self.twiddle14.re * x724p.im
5261 + self.twiddle15.re * x823p.im
5262 + self.twiddle13.re * x922p.im
5263 + self.twiddle11.re * x1021p.im
5264 + self.twiddle9.re * x1120p.im
5265 + self.twiddle7.re * x1219p.im
5266 + self.twiddle5.re * x1318p.im
5267 + self.twiddle3.re * x1417p.im
5268 + self.twiddle1.re * x1516p.im;
5269 let b229im_b = self.twiddle2.im * x130n.re
5270 + self.twiddle4.im * x229n.re
5271 + self.twiddle6.im * x328n.re
5272 + self.twiddle8.im * x427n.re
5273 + self.twiddle10.im * x526n.re
5274 + self.twiddle12.im * x625n.re
5275 + self.twiddle14.im * x724n.re
5276 + -self.twiddle15.im * x823n.re
5277 + -self.twiddle13.im * x922n.re
5278 + -self.twiddle11.im * x1021n.re
5279 + -self.twiddle9.im * x1120n.re
5280 + -self.twiddle7.im * x1219n.re
5281 + -self.twiddle5.im * x1318n.re
5282 + -self.twiddle3.im * x1417n.re
5283 + -self.twiddle1.im * x1516n.re;
5284 let b328im_a = buffer.load(0).im
5285 + self.twiddle3.re * x130p.im
5286 + self.twiddle6.re * x229p.im
5287 + self.twiddle9.re * x328p.im
5288 + self.twiddle12.re * x427p.im
5289 + self.twiddle15.re * x526p.im
5290 + self.twiddle13.re * x625p.im
5291 + self.twiddle10.re * x724p.im
5292 + self.twiddle7.re * x823p.im
5293 + self.twiddle4.re * x922p.im
5294 + self.twiddle1.re * x1021p.im
5295 + self.twiddle2.re * x1120p.im
5296 + self.twiddle5.re * x1219p.im
5297 + self.twiddle8.re * x1318p.im
5298 + self.twiddle11.re * x1417p.im
5299 + self.twiddle14.re * x1516p.im;
5300 let b328im_b = self.twiddle3.im * x130n.re
5301 + self.twiddle6.im * x229n.re
5302 + self.twiddle9.im * x328n.re
5303 + self.twiddle12.im * x427n.re
5304 + self.twiddle15.im * x526n.re
5305 + -self.twiddle13.im * x625n.re
5306 + -self.twiddle10.im * x724n.re
5307 + -self.twiddle7.im * x823n.re
5308 + -self.twiddle4.im * x922n.re
5309 + -self.twiddle1.im * x1021n.re
5310 + self.twiddle2.im * x1120n.re
5311 + self.twiddle5.im * x1219n.re
5312 + self.twiddle8.im * x1318n.re
5313 + self.twiddle11.im * x1417n.re
5314 + self.twiddle14.im * x1516n.re;
5315 let b427im_a = buffer.load(0).im
5316 + self.twiddle4.re * x130p.im
5317 + self.twiddle8.re * x229p.im
5318 + self.twiddle12.re * x328p.im
5319 + self.twiddle15.re * x427p.im
5320 + self.twiddle11.re * x526p.im
5321 + self.twiddle7.re * x625p.im
5322 + self.twiddle3.re * x724p.im
5323 + self.twiddle1.re * x823p.im
5324 + self.twiddle5.re * x922p.im
5325 + self.twiddle9.re * x1021p.im
5326 + self.twiddle13.re * x1120p.im
5327 + self.twiddle14.re * x1219p.im
5328 + self.twiddle10.re * x1318p.im
5329 + self.twiddle6.re * x1417p.im
5330 + self.twiddle2.re * x1516p.im;
5331 let b427im_b = self.twiddle4.im * x130n.re
5332 + self.twiddle8.im * x229n.re
5333 + self.twiddle12.im * x328n.re
5334 + -self.twiddle15.im * x427n.re
5335 + -self.twiddle11.im * x526n.re
5336 + -self.twiddle7.im * x625n.re
5337 + -self.twiddle3.im * x724n.re
5338 + self.twiddle1.im * x823n.re
5339 + self.twiddle5.im * x922n.re
5340 + self.twiddle9.im * x1021n.re
5341 + self.twiddle13.im * x1120n.re
5342 + -self.twiddle14.im * x1219n.re
5343 + -self.twiddle10.im * x1318n.re
5344 + -self.twiddle6.im * x1417n.re
5345 + -self.twiddle2.im * x1516n.re;
5346 let b526im_a = buffer.load(0).im
5347 + self.twiddle5.re * x130p.im
5348 + self.twiddle10.re * x229p.im
5349 + self.twiddle15.re * x328p.im
5350 + self.twiddle11.re * x427p.im
5351 + self.twiddle6.re * x526p.im
5352 + self.twiddle1.re * x625p.im
5353 + self.twiddle4.re * x724p.im
5354 + self.twiddle9.re * x823p.im
5355 + self.twiddle14.re * x922p.im
5356 + self.twiddle12.re * x1021p.im
5357 + self.twiddle7.re * x1120p.im
5358 + self.twiddle2.re * x1219p.im
5359 + self.twiddle3.re * x1318p.im
5360 + self.twiddle8.re * x1417p.im
5361 + self.twiddle13.re * x1516p.im;
5362 let b526im_b = self.twiddle5.im * x130n.re
5363 + self.twiddle10.im * x229n.re
5364 + self.twiddle15.im * x328n.re
5365 + -self.twiddle11.im * x427n.re
5366 + -self.twiddle6.im * x526n.re
5367 + -self.twiddle1.im * x625n.re
5368 + self.twiddle4.im * x724n.re
5369 + self.twiddle9.im * x823n.re
5370 + self.twiddle14.im * x922n.re
5371 + -self.twiddle12.im * x1021n.re
5372 + -self.twiddle7.im * x1120n.re
5373 + -self.twiddle2.im * x1219n.re
5374 + self.twiddle3.im * x1318n.re
5375 + self.twiddle8.im * x1417n.re
5376 + self.twiddle13.im * x1516n.re;
5377 let b625im_a = buffer.load(0).im
5378 + self.twiddle6.re * x130p.im
5379 + self.twiddle12.re * x229p.im
5380 + self.twiddle13.re * x328p.im
5381 + self.twiddle7.re * x427p.im
5382 + self.twiddle1.re * x526p.im
5383 + self.twiddle5.re * x625p.im
5384 + self.twiddle11.re * x724p.im
5385 + self.twiddle14.re * x823p.im
5386 + self.twiddle8.re * x922p.im
5387 + self.twiddle2.re * x1021p.im
5388 + self.twiddle4.re * x1120p.im
5389 + self.twiddle10.re * x1219p.im
5390 + self.twiddle15.re * x1318p.im
5391 + self.twiddle9.re * x1417p.im
5392 + self.twiddle3.re * x1516p.im;
5393 let b625im_b = self.twiddle6.im * x130n.re
5394 + self.twiddle12.im * x229n.re
5395 + -self.twiddle13.im * x328n.re
5396 + -self.twiddle7.im * x427n.re
5397 + -self.twiddle1.im * x526n.re
5398 + self.twiddle5.im * x625n.re
5399 + self.twiddle11.im * x724n.re
5400 + -self.twiddle14.im * x823n.re
5401 + -self.twiddle8.im * x922n.re
5402 + -self.twiddle2.im * x1021n.re
5403 + self.twiddle4.im * x1120n.re
5404 + self.twiddle10.im * x1219n.re
5405 + -self.twiddle15.im * x1318n.re
5406 + -self.twiddle9.im * x1417n.re
5407 + -self.twiddle3.im * x1516n.re;
5408 let b724im_a = buffer.load(0).im
5409 + self.twiddle7.re * x130p.im
5410 + self.twiddle14.re * x229p.im
5411 + self.twiddle10.re * x328p.im
5412 + self.twiddle3.re * x427p.im
5413 + self.twiddle4.re * x526p.im
5414 + self.twiddle11.re * x625p.im
5415 + self.twiddle13.re * x724p.im
5416 + self.twiddle6.re * x823p.im
5417 + self.twiddle1.re * x922p.im
5418 + self.twiddle8.re * x1021p.im
5419 + self.twiddle15.re * x1120p.im
5420 + self.twiddle9.re * x1219p.im
5421 + self.twiddle2.re * x1318p.im
5422 + self.twiddle5.re * x1417p.im
5423 + self.twiddle12.re * x1516p.im;
5424 let b724im_b = self.twiddle7.im * x130n.re
5425 + self.twiddle14.im * x229n.re
5426 + -self.twiddle10.im * x328n.re
5427 + -self.twiddle3.im * x427n.re
5428 + self.twiddle4.im * x526n.re
5429 + self.twiddle11.im * x625n.re
5430 + -self.twiddle13.im * x724n.re
5431 + -self.twiddle6.im * x823n.re
5432 + self.twiddle1.im * x922n.re
5433 + self.twiddle8.im * x1021n.re
5434 + self.twiddle15.im * x1120n.re
5435 + -self.twiddle9.im * x1219n.re
5436 + -self.twiddle2.im * x1318n.re
5437 + self.twiddle5.im * x1417n.re
5438 + self.twiddle12.im * x1516n.re;
5439 let b823im_a = buffer.load(0).im
5440 + self.twiddle8.re * x130p.im
5441 + self.twiddle15.re * x229p.im
5442 + self.twiddle7.re * x328p.im
5443 + self.twiddle1.re * x427p.im
5444 + self.twiddle9.re * x526p.im
5445 + self.twiddle14.re * x625p.im
5446 + self.twiddle6.re * x724p.im
5447 + self.twiddle2.re * x823p.im
5448 + self.twiddle10.re * x922p.im
5449 + self.twiddle13.re * x1021p.im
5450 + self.twiddle5.re * x1120p.im
5451 + self.twiddle3.re * x1219p.im
5452 + self.twiddle11.re * x1318p.im
5453 + self.twiddle12.re * x1417p.im
5454 + self.twiddle4.re * x1516p.im;
5455 let b823im_b = self.twiddle8.im * x130n.re
5456 + -self.twiddle15.im * x229n.re
5457 + -self.twiddle7.im * x328n.re
5458 + self.twiddle1.im * x427n.re
5459 + self.twiddle9.im * x526n.re
5460 + -self.twiddle14.im * x625n.re
5461 + -self.twiddle6.im * x724n.re
5462 + self.twiddle2.im * x823n.re
5463 + self.twiddle10.im * x922n.re
5464 + -self.twiddle13.im * x1021n.re
5465 + -self.twiddle5.im * x1120n.re
5466 + self.twiddle3.im * x1219n.re
5467 + self.twiddle11.im * x1318n.re
5468 + -self.twiddle12.im * x1417n.re
5469 + -self.twiddle4.im * x1516n.re;
5470 let b922im_a = buffer.load(0).im
5471 + self.twiddle9.re * x130p.im
5472 + self.twiddle13.re * x229p.im
5473 + self.twiddle4.re * x328p.im
5474 + self.twiddle5.re * x427p.im
5475 + self.twiddle14.re * x526p.im
5476 + self.twiddle8.re * x625p.im
5477 + self.twiddle1.re * x724p.im
5478 + self.twiddle10.re * x823p.im
5479 + self.twiddle12.re * x922p.im
5480 + self.twiddle3.re * x1021p.im
5481 + self.twiddle6.re * x1120p.im
5482 + self.twiddle15.re * x1219p.im
5483 + self.twiddle7.re * x1318p.im
5484 + self.twiddle2.re * x1417p.im
5485 + self.twiddle11.re * x1516p.im;
5486 let b922im_b = self.twiddle9.im * x130n.re
5487 + -self.twiddle13.im * x229n.re
5488 + -self.twiddle4.im * x328n.re
5489 + self.twiddle5.im * x427n.re
5490 + self.twiddle14.im * x526n.re
5491 + -self.twiddle8.im * x625n.re
5492 + self.twiddle1.im * x724n.re
5493 + self.twiddle10.im * x823n.re
5494 + -self.twiddle12.im * x922n.re
5495 + -self.twiddle3.im * x1021n.re
5496 + self.twiddle6.im * x1120n.re
5497 + self.twiddle15.im * x1219n.re
5498 + -self.twiddle7.im * x1318n.re
5499 + self.twiddle2.im * x1417n.re
5500 + self.twiddle11.im * x1516n.re;
5501 let b1021im_a = buffer.load(0).im
5502 + self.twiddle10.re * x130p.im
5503 + self.twiddle11.re * x229p.im
5504 + self.twiddle1.re * x328p.im
5505 + self.twiddle9.re * x427p.im
5506 + self.twiddle12.re * x526p.im
5507 + self.twiddle2.re * x625p.im
5508 + self.twiddle8.re * x724p.im
5509 + self.twiddle13.re * x823p.im
5510 + self.twiddle3.re * x922p.im
5511 + self.twiddle7.re * x1021p.im
5512 + self.twiddle14.re * x1120p.im
5513 + self.twiddle4.re * x1219p.im
5514 + self.twiddle6.re * x1318p.im
5515 + self.twiddle15.re * x1417p.im
5516 + self.twiddle5.re * x1516p.im;
5517 let b1021im_b = self.twiddle10.im * x130n.re
5518 + -self.twiddle11.im * x229n.re
5519 + -self.twiddle1.im * x328n.re
5520 + self.twiddle9.im * x427n.re
5521 + -self.twiddle12.im * x526n.re
5522 + -self.twiddle2.im * x625n.re
5523 + self.twiddle8.im * x724n.re
5524 + -self.twiddle13.im * x823n.re
5525 + -self.twiddle3.im * x922n.re
5526 + self.twiddle7.im * x1021n.re
5527 + -self.twiddle14.im * x1120n.re
5528 + -self.twiddle4.im * x1219n.re
5529 + self.twiddle6.im * x1318n.re
5530 + -self.twiddle15.im * x1417n.re
5531 + -self.twiddle5.im * x1516n.re;
5532 let b1120im_a = buffer.load(0).im
5533 + self.twiddle11.re * x130p.im
5534 + self.twiddle9.re * x229p.im
5535 + self.twiddle2.re * x328p.im
5536 + self.twiddle13.re * x427p.im
5537 + self.twiddle7.re * x526p.im
5538 + self.twiddle4.re * x625p.im
5539 + self.twiddle15.re * x724p.im
5540 + self.twiddle5.re * x823p.im
5541 + self.twiddle6.re * x922p.im
5542 + self.twiddle14.re * x1021p.im
5543 + self.twiddle3.re * x1120p.im
5544 + self.twiddle8.re * x1219p.im
5545 + self.twiddle12.re * x1318p.im
5546 + self.twiddle1.re * x1417p.im
5547 + self.twiddle10.re * x1516p.im;
5548 let b1120im_b = self.twiddle11.im * x130n.re
5549 + -self.twiddle9.im * x229n.re
5550 + self.twiddle2.im * x328n.re
5551 + self.twiddle13.im * x427n.re
5552 + -self.twiddle7.im * x526n.re
5553 + self.twiddle4.im * x625n.re
5554 + self.twiddle15.im * x724n.re
5555 + -self.twiddle5.im * x823n.re
5556 + self.twiddle6.im * x922n.re
5557 + -self.twiddle14.im * x1021n.re
5558 + -self.twiddle3.im * x1120n.re
5559 + self.twiddle8.im * x1219n.re
5560 + -self.twiddle12.im * x1318n.re
5561 + -self.twiddle1.im * x1417n.re
5562 + self.twiddle10.im * x1516n.re;
5563 let b1219im_a = buffer.load(0).im
5564 + self.twiddle12.re * x130p.im
5565 + self.twiddle7.re * x229p.im
5566 + self.twiddle5.re * x328p.im
5567 + self.twiddle14.re * x427p.im
5568 + self.twiddle2.re * x526p.im
5569 + self.twiddle10.re * x625p.im
5570 + self.twiddle9.re * x724p.im
5571 + self.twiddle3.re * x823p.im
5572 + self.twiddle15.re * x922p.im
5573 + self.twiddle4.re * x1021p.im
5574 + self.twiddle8.re * x1120p.im
5575 + self.twiddle11.re * x1219p.im
5576 + self.twiddle1.re * x1318p.im
5577 + self.twiddle13.re * x1417p.im
5578 + self.twiddle6.re * x1516p.im;
5579 let b1219im_b = self.twiddle12.im * x130n.re
5580 + -self.twiddle7.im * x229n.re
5581 + self.twiddle5.im * x328n.re
5582 + -self.twiddle14.im * x427n.re
5583 + -self.twiddle2.im * x526n.re
5584 + self.twiddle10.im * x625n.re
5585 + -self.twiddle9.im * x724n.re
5586 + self.twiddle3.im * x823n.re
5587 + self.twiddle15.im * x922n.re
5588 + -self.twiddle4.im * x1021n.re
5589 + self.twiddle8.im * x1120n.re
5590 + -self.twiddle11.im * x1219n.re
5591 + self.twiddle1.im * x1318n.re
5592 + self.twiddle13.im * x1417n.re
5593 + -self.twiddle6.im * x1516n.re;
5594 let b1318im_a = buffer.load(0).im
5595 + self.twiddle13.re * x130p.im
5596 + self.twiddle5.re * x229p.im
5597 + self.twiddle8.re * x328p.im
5598 + self.twiddle10.re * x427p.im
5599 + self.twiddle3.re * x526p.im
5600 + self.twiddle15.re * x625p.im
5601 + self.twiddle2.re * x724p.im
5602 + self.twiddle11.re * x823p.im
5603 + self.twiddle7.re * x922p.im
5604 + self.twiddle6.re * x1021p.im
5605 + self.twiddle12.re * x1120p.im
5606 + self.twiddle1.re * x1219p.im
5607 + self.twiddle14.re * x1318p.im
5608 + self.twiddle4.re * x1417p.im
5609 + self.twiddle9.re * x1516p.im;
5610 let b1318im_b = self.twiddle13.im * x130n.re
5611 + -self.twiddle5.im * x229n.re
5612 + self.twiddle8.im * x328n.re
5613 + -self.twiddle10.im * x427n.re
5614 + self.twiddle3.im * x526n.re
5615 + -self.twiddle15.im * x625n.re
5616 + -self.twiddle2.im * x724n.re
5617 + self.twiddle11.im * x823n.re
5618 + -self.twiddle7.im * x922n.re
5619 + self.twiddle6.im * x1021n.re
5620 + -self.twiddle12.im * x1120n.re
5621 + self.twiddle1.im * x1219n.re
5622 + self.twiddle14.im * x1318n.re
5623 + -self.twiddle4.im * x1417n.re
5624 + self.twiddle9.im * x1516n.re;
5625 let b1417im_a = buffer.load(0).im
5626 + self.twiddle14.re * x130p.im
5627 + self.twiddle3.re * x229p.im
5628 + self.twiddle11.re * x328p.im
5629 + self.twiddle6.re * x427p.im
5630 + self.twiddle8.re * x526p.im
5631 + self.twiddle9.re * x625p.im
5632 + self.twiddle5.re * x724p.im
5633 + self.twiddle12.re * x823p.im
5634 + self.twiddle2.re * x922p.im
5635 + self.twiddle15.re * x1021p.im
5636 + self.twiddle1.re * x1120p.im
5637 + self.twiddle13.re * x1219p.im
5638 + self.twiddle4.re * x1318p.im
5639 + self.twiddle10.re * x1417p.im
5640 + self.twiddle7.re * x1516p.im;
5641 let b1417im_b = self.twiddle14.im * x130n.re
5642 + -self.twiddle3.im * x229n.re
5643 + self.twiddle11.im * x328n.re
5644 + -self.twiddle6.im * x427n.re
5645 + self.twiddle8.im * x526n.re
5646 + -self.twiddle9.im * x625n.re
5647 + self.twiddle5.im * x724n.re
5648 + -self.twiddle12.im * x823n.re
5649 + self.twiddle2.im * x922n.re
5650 + -self.twiddle15.im * x1021n.re
5651 + -self.twiddle1.im * x1120n.re
5652 + self.twiddle13.im * x1219n.re
5653 + -self.twiddle4.im * x1318n.re
5654 + self.twiddle10.im * x1417n.re
5655 + -self.twiddle7.im * x1516n.re;
5656 let b1516im_a = buffer.load(0).im
5657 + self.twiddle15.re * x130p.im
5658 + self.twiddle1.re * x229p.im
5659 + self.twiddle14.re * x328p.im
5660 + self.twiddle2.re * x427p.im
5661 + self.twiddle13.re * x526p.im
5662 + self.twiddle3.re * x625p.im
5663 + self.twiddle12.re * x724p.im
5664 + self.twiddle4.re * x823p.im
5665 + self.twiddle11.re * x922p.im
5666 + self.twiddle5.re * x1021p.im
5667 + self.twiddle10.re * x1120p.im
5668 + self.twiddle6.re * x1219p.im
5669 + self.twiddle9.re * x1318p.im
5670 + self.twiddle7.re * x1417p.im
5671 + self.twiddle8.re * x1516p.im;
5672 let b1516im_b = self.twiddle15.im * x130n.re
5673 + -self.twiddle1.im * x229n.re
5674 + self.twiddle14.im * x328n.re
5675 + -self.twiddle2.im * x427n.re
5676 + self.twiddle13.im * x526n.re
5677 + -self.twiddle3.im * x625n.re
5678 + self.twiddle12.im * x724n.re
5679 + -self.twiddle4.im * x823n.re
5680 + self.twiddle11.im * x922n.re
5681 + -self.twiddle5.im * x1021n.re
5682 + self.twiddle10.im * x1120n.re
5683 + -self.twiddle6.im * x1219n.re
5684 + self.twiddle9.im * x1318n.re
5685 + -self.twiddle7.im * x1417n.re
5686 + self.twiddle8.im * x1516n.re;
5687
5688 let out1re = b130re_a - b130re_b;
5689 let out1im = b130im_a + b130im_b;
5690 let out2re = b229re_a - b229re_b;
5691 let out2im = b229im_a + b229im_b;
5692 let out3re = b328re_a - b328re_b;
5693 let out3im = b328im_a + b328im_b;
5694 let out4re = b427re_a - b427re_b;
5695 let out4im = b427im_a + b427im_b;
5696 let out5re = b526re_a - b526re_b;
5697 let out5im = b526im_a + b526im_b;
5698 let out6re = b625re_a - b625re_b;
5699 let out6im = b625im_a + b625im_b;
5700 let out7re = b724re_a - b724re_b;
5701 let out7im = b724im_a + b724im_b;
5702 let out8re = b823re_a - b823re_b;
5703 let out8im = b823im_a + b823im_b;
5704 let out9re = b922re_a - b922re_b;
5705 let out9im = b922im_a + b922im_b;
5706 let out10re = b1021re_a - b1021re_b;
5707 let out10im = b1021im_a + b1021im_b;
5708 let out11re = b1120re_a - b1120re_b;
5709 let out11im = b1120im_a + b1120im_b;
5710 let out12re = b1219re_a - b1219re_b;
5711 let out12im = b1219im_a + b1219im_b;
5712 let out13re = b1318re_a - b1318re_b;
5713 let out13im = b1318im_a + b1318im_b;
5714 let out14re = b1417re_a - b1417re_b;
5715 let out14im = b1417im_a + b1417im_b;
5716 let out15re = b1516re_a - b1516re_b;
5717 let out15im = b1516im_a + b1516im_b;
5718 let out16re = b1516re_a + b1516re_b;
5719 let out16im = b1516im_a - b1516im_b;
5720 let out17re = b1417re_a + b1417re_b;
5721 let out17im = b1417im_a - b1417im_b;
5722 let out18re = b1318re_a + b1318re_b;
5723 let out18im = b1318im_a - b1318im_b;
5724 let out19re = b1219re_a + b1219re_b;
5725 let out19im = b1219im_a - b1219im_b;
5726 let out20re = b1120re_a + b1120re_b;
5727 let out20im = b1120im_a - b1120im_b;
5728 let out21re = b1021re_a + b1021re_b;
5729 let out21im = b1021im_a - b1021im_b;
5730 let out22re = b922re_a + b922re_b;
5731 let out22im = b922im_a - b922im_b;
5732 let out23re = b823re_a + b823re_b;
5733 let out23im = b823im_a - b823im_b;
5734 let out24re = b724re_a + b724re_b;
5735 let out24im = b724im_a - b724im_b;
5736 let out25re = b625re_a + b625re_b;
5737 let out25im = b625im_a - b625im_b;
5738 let out26re = b526re_a + b526re_b;
5739 let out26im = b526im_a - b526im_b;
5740 let out27re = b427re_a + b427re_b;
5741 let out27im = b427im_a - b427im_b;
5742 let out28re = b328re_a + b328re_b;
5743 let out28im = b328im_a - b328im_b;
5744 let out29re = b229re_a + b229re_b;
5745 let out29im = b229im_a - b229im_b;
5746 let out30re = b130re_a + b130re_b;
5747 let out30im = b130im_a - b130im_b;
5748 buffer.store(sum, 0);
5749 buffer.store(
5750 Complex {
5751 re: out1re,
5752 im: out1im,
5753 },
5754 1,
5755 );
5756 buffer.store(
5757 Complex {
5758 re: out2re,
5759 im: out2im,
5760 },
5761 2,
5762 );
5763 buffer.store(
5764 Complex {
5765 re: out3re,
5766 im: out3im,
5767 },
5768 3,
5769 );
5770 buffer.store(
5771 Complex {
5772 re: out4re,
5773 im: out4im,
5774 },
5775 4,
5776 );
5777 buffer.store(
5778 Complex {
5779 re: out5re,
5780 im: out5im,
5781 },
5782 5,
5783 );
5784 buffer.store(
5785 Complex {
5786 re: out6re,
5787 im: out6im,
5788 },
5789 6,
5790 );
5791 buffer.store(
5792 Complex {
5793 re: out7re,
5794 im: out7im,
5795 },
5796 7,
5797 );
5798 buffer.store(
5799 Complex {
5800 re: out8re,
5801 im: out8im,
5802 },
5803 8,
5804 );
5805 buffer.store(
5806 Complex {
5807 re: out9re,
5808 im: out9im,
5809 },
5810 9,
5811 );
5812 buffer.store(
5813 Complex {
5814 re: out10re,
5815 im: out10im,
5816 },
5817 10,
5818 );
5819 buffer.store(
5820 Complex {
5821 re: out11re,
5822 im: out11im,
5823 },
5824 11,
5825 );
5826 buffer.store(
5827 Complex {
5828 re: out12re,
5829 im: out12im,
5830 },
5831 12,
5832 );
5833 buffer.store(
5834 Complex {
5835 re: out13re,
5836 im: out13im,
5837 },
5838 13,
5839 );
5840 buffer.store(
5841 Complex {
5842 re: out14re,
5843 im: out14im,
5844 },
5845 14,
5846 );
5847 buffer.store(
5848 Complex {
5849 re: out15re,
5850 im: out15im,
5851 },
5852 15,
5853 );
5854 buffer.store(
5855 Complex {
5856 re: out16re,
5857 im: out16im,
5858 },
5859 16,
5860 );
5861 buffer.store(
5862 Complex {
5863 re: out17re,
5864 im: out17im,
5865 },
5866 17,
5867 );
5868 buffer.store(
5869 Complex {
5870 re: out18re,
5871 im: out18im,
5872 },
5873 18,
5874 );
5875 buffer.store(
5876 Complex {
5877 re: out19re,
5878 im: out19im,
5879 },
5880 19,
5881 );
5882 buffer.store(
5883 Complex {
5884 re: out20re,
5885 im: out20im,
5886 },
5887 20,
5888 );
5889 buffer.store(
5890 Complex {
5891 re: out21re,
5892 im: out21im,
5893 },
5894 21,
5895 );
5896 buffer.store(
5897 Complex {
5898 re: out22re,
5899 im: out22im,
5900 },
5901 22,
5902 );
5903 buffer.store(
5904 Complex {
5905 re: out23re,
5906 im: out23im,
5907 },
5908 23,
5909 );
5910 buffer.store(
5911 Complex {
5912 re: out24re,
5913 im: out24im,
5914 },
5915 24,
5916 );
5917 buffer.store(
5918 Complex {
5919 re: out25re,
5920 im: out25im,
5921 },
5922 25,
5923 );
5924 buffer.store(
5925 Complex {
5926 re: out26re,
5927 im: out26im,
5928 },
5929 26,
5930 );
5931 buffer.store(
5932 Complex {
5933 re: out27re,
5934 im: out27im,
5935 },
5936 27,
5937 );
5938 buffer.store(
5939 Complex {
5940 re: out28re,
5941 im: out28im,
5942 },
5943 28,
5944 );
5945 buffer.store(
5946 Complex {
5947 re: out29re,
5948 im: out29im,
5949 },
5950 29,
5951 );
5952 buffer.store(
5953 Complex {
5954 re: out30re,
5955 im: out30im,
5956 },
5957 30,
5958 );
5959 }
5960 }
5961 pub struct Butterfly32<T> {
5962 butterfly16: Butterfly16<T>,
5963 butterfly8: Butterfly8<T>,
5964 twiddles: [Complex<T>; 7],
5965 }
5966 boilerplate_fft_butterfly!(Butterfly32, 32, |this: &Butterfly32<_>| this
5967 .butterfly8
5968 .fft_direction());
5969 impl<T: FftNum> Butterfly32<T> {
5970 pub fn new(direction: FftDirection) -> Self {
5971 Self {
5972 butterfly16: Butterfly16::new(direction),
5973 butterfly8: Butterfly8::new(direction),
5974 twiddles: [
5975 twiddles::compute_twiddle(1, 32, direction),
5976 twiddles::compute_twiddle(2, 32, direction),
5977 twiddles::compute_twiddle(3, 32, direction),
5978 twiddles::compute_twiddle(4, 32, direction),
5979 twiddles::compute_twiddle(5, 32, direction),
5980 twiddles::compute_twiddle(6, 32, direction),
5981 twiddles::compute_twiddle(7, 32, direction),
5982 ],
5983 }
5984 }
5985
5986 #[inline(never)]
5987 unsafe fn perform_fft_contiguous(&self, mut buffer: impl LoadStore<T>) {
5988 // we're going to hardcode a step of split radix
5989 // step 1: copy and reorder the input into the scratch
5990 let mut scratch_evens = [
5991 buffer.load(0),
5992 buffer.load(2),
5993 buffer.load(4),
5994 buffer.load(6),
5995 buffer.load(8),
5996 buffer.load(10),
5997 buffer.load(12),
5998 buffer.load(14),
5999 buffer.load(16),
6000 buffer.load(18),
6001 buffer.load(20),
6002 buffer.load(22),
6003 buffer.load(24),
6004 buffer.load(26),
6005 buffer.load(28),
6006 buffer.load(30),
6007 ];
6008
6009 let mut scratch_odds_n1 = [
6010 buffer.load(1),
6011 buffer.load(5),
6012 buffer.load(9),
6013 buffer.load(13),
6014 buffer.load(17),
6015 buffer.load(21),
6016 buffer.load(25),
6017 buffer.load(29),
6018 ];
6019 let mut scratch_odds_n3 = [
6020 buffer.load(31),
6021 buffer.load(3),
6022 buffer.load(7),
6023 buffer.load(11),
6024 buffer.load(15),
6025 buffer.load(19),
6026 buffer.load(23),
6027 buffer.load(27),
6028 ];
6029
6030 // step 2: column FFTs
6031 self.butterfly16.perform_fft_contiguous(&mut scratch_evens);
6032 self.butterfly8.perform_fft_contiguous(&mut scratch_odds_n1);
6033 self.butterfly8.perform_fft_contiguous(&mut scratch_odds_n3);
6034
6035 // step 3: apply twiddle factors
6036 scratch_odds_n1[1] = scratch_odds_n1[1] * self.twiddles[0];
6037 scratch_odds_n3[1] = scratch_odds_n3[1] * self.twiddles[0].conj();
6038
6039 scratch_odds_n1[2] = scratch_odds_n1[2] * self.twiddles[1];
6040 scratch_odds_n3[2] = scratch_odds_n3[2] * self.twiddles[1].conj();
6041
6042 scratch_odds_n1[3] = scratch_odds_n1[3] * self.twiddles[2];
6043 scratch_odds_n3[3] = scratch_odds_n3[3] * self.twiddles[2].conj();
6044
6045 scratch_odds_n1[4] = scratch_odds_n1[4] * self.twiddles[3];
6046 scratch_odds_n3[4] = scratch_odds_n3[4] * self.twiddles[3].conj();
6047
6048 scratch_odds_n1[5] = scratch_odds_n1[5] * self.twiddles[4];
6049 scratch_odds_n3[5] = scratch_odds_n3[5] * self.twiddles[4].conj();
6050
6051 scratch_odds_n1[6] = scratch_odds_n1[6] * self.twiddles[5];
6052 scratch_odds_n3[6] = scratch_odds_n3[6] * self.twiddles[5].conj();
6053
6054 scratch_odds_n1[7] = scratch_odds_n1[7] * self.twiddles[6];
6055 scratch_odds_n3[7] = scratch_odds_n3[7] * self.twiddles[6].conj();
6056
6057 // step 4: cross FFTs
6058 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[0], &mut scratch_odds_n3[0]);
6059 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[1], &mut scratch_odds_n3[1]);
6060 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[2], &mut scratch_odds_n3[2]);
6061 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[3], &mut scratch_odds_n3[3]);
6062 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[4], &mut scratch_odds_n3[4]);
6063 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[5], &mut scratch_odds_n3[5]);
6064 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[6], &mut scratch_odds_n3[6]);
6065 Butterfly2::perform_fft_strided(&mut scratch_odds_n1[7], &mut scratch_odds_n3[7]);
6066
6067 // apply the butterfly 4 twiddle factor, which is just a rotation
6068 scratch_odds_n3[0] = twiddles::rotate_90(scratch_odds_n3[0], self.fft_direction());
6069 scratch_odds_n3[1] = twiddles::rotate_90(scratch_odds_n3[1], self.fft_direction());
6070 scratch_odds_n3[2] = twiddles::rotate_90(scratch_odds_n3[2], self.fft_direction());
6071 scratch_odds_n3[3] = twiddles::rotate_90(scratch_odds_n3[3], self.fft_direction());
6072 scratch_odds_n3[4] = twiddles::rotate_90(scratch_odds_n3[4], self.fft_direction());
6073 scratch_odds_n3[5] = twiddles::rotate_90(scratch_odds_n3[5], self.fft_direction());
6074 scratch_odds_n3[6] = twiddles::rotate_90(scratch_odds_n3[6], self.fft_direction());
6075 scratch_odds_n3[7] = twiddles::rotate_90(scratch_odds_n3[7], self.fft_direction());
6076
6077 //step 5: copy/add/subtract data back to buffer
6078 buffer.store(scratch_evens[0] + scratch_odds_n1[0], 0);
6079 buffer.store(scratch_evens[1] + scratch_odds_n1[1], 1);
6080 buffer.store(scratch_evens[2] + scratch_odds_n1[2], 2);
6081 buffer.store(scratch_evens[3] + scratch_odds_n1[3], 3);
6082 buffer.store(scratch_evens[4] + scratch_odds_n1[4], 4);
6083 buffer.store(scratch_evens[5] + scratch_odds_n1[5], 5);
6084 buffer.store(scratch_evens[6] + scratch_odds_n1[6], 6);
6085 buffer.store(scratch_evens[7] + scratch_odds_n1[7], 7);
6086 buffer.store(scratch_evens[8] + scratch_odds_n3[0], 8);
6087 buffer.store(scratch_evens[9] + scratch_odds_n3[1], 9);
6088 buffer.store(scratch_evens[10] + scratch_odds_n3[2], 10);
6089 buffer.store(scratch_evens[11] + scratch_odds_n3[3], 11);
6090 buffer.store(scratch_evens[12] + scratch_odds_n3[4], 12);
6091 buffer.store(scratch_evens[13] + scratch_odds_n3[5], 13);
6092 buffer.store(scratch_evens[14] + scratch_odds_n3[6], 14);
6093 buffer.store(scratch_evens[15] + scratch_odds_n3[7], 15);
6094 buffer.store(scratch_evens[0] - scratch_odds_n1[0], 16);
6095 buffer.store(scratch_evens[1] - scratch_odds_n1[1], 17);
6096 buffer.store(scratch_evens[2] - scratch_odds_n1[2], 18);
6097 buffer.store(scratch_evens[3] - scratch_odds_n1[3], 19);
6098 buffer.store(scratch_evens[4] - scratch_odds_n1[4], 20);
6099 buffer.store(scratch_evens[5] - scratch_odds_n1[5], 21);
6100 buffer.store(scratch_evens[6] - scratch_odds_n1[6], 22);
6101 buffer.store(scratch_evens[7] - scratch_odds_n1[7], 23);
6102 buffer.store(scratch_evens[8] - scratch_odds_n3[0], 24);
6103 buffer.store(scratch_evens[9] - scratch_odds_n3[1], 25);
6104 buffer.store(scratch_evens[10] - scratch_odds_n3[2], 26);
6105 buffer.store(scratch_evens[11] - scratch_odds_n3[3], 27);
6106 buffer.store(scratch_evens[12] - scratch_odds_n3[4], 28);
6107 buffer.store(scratch_evens[13] - scratch_odds_n3[5], 29);
6108 buffer.store(scratch_evens[14] - scratch_odds_n3[6], 30);
6109 buffer.store(scratch_evens[15] - scratch_odds_n3[7], 31);
6110 }
6111 }
6112
6113 #[cfg(test)]
6114 mod unit_tests {
6115 use super::*;
6116 use crate::test_utils::check_fft_algorithm;
6117
6118 //the tests for all butterflies will be identical except for the identifiers used and size
6119 //so it's ideal for a macro
6120 macro_rules! test_butterfly_func {
6121 ($test_name:ident, $struct_name:ident, $size:expr) => {
6122 #[test]
6123 fn $test_name() {
6124 let butterfly = $struct_name::new(FftDirection::Forward);
6125 check_fft_algorithm::<f32>(&butterfly, $size, FftDirection::Forward);
6126
6127 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
6128 check_fft_algorithm::<f32>(&butterfly_direction, $size, FftDirection::Inverse);
6129 }
6130 };
6131 }
6132 test_butterfly_func!(test_butterfly2, Butterfly2, 2);
6133 test_butterfly_func!(test_butterfly3, Butterfly3, 3);
6134 test_butterfly_func!(test_butterfly4, Butterfly4, 4);
6135 test_butterfly_func!(test_butterfly5, Butterfly5, 5);
6136 test_butterfly_func!(test_butterfly6, Butterfly6, 6);
6137 test_butterfly_func!(test_butterfly7, Butterfly7, 7);
6138 test_butterfly_func!(test_butterfly8, Butterfly8, 8);
6139 test_butterfly_func!(test_butterfly9, Butterfly9, 9);
6140 test_butterfly_func!(test_butterfly11, Butterfly11, 11);
6141 test_butterfly_func!(test_butterfly13, Butterfly13, 13);
6142 test_butterfly_func!(test_butterfly16, Butterfly16, 16);
6143 test_butterfly_func!(test_butterfly17, Butterfly17, 17);
6144 test_butterfly_func!(test_butterfly19, Butterfly19, 19);
6145 test_butterfly_func!(test_butterfly23, Butterfly23, 23);
6146 test_butterfly_func!(test_butterfly27, Butterfly27, 27);
6147 test_butterfly_func!(test_butterfly29, Butterfly29, 29);
6148 test_butterfly_func!(test_butterfly31, Butterfly31, 31);
6149 test_butterfly_func!(test_butterfly32, Butterfly32, 32);
6150 }