Debian Codegraph @

.cargo/vendor/rustfft/src/neon/neon_butterflies.rs

raw
1 use core::arch::aarch64::*;
2 use num_complex::Complex;
3
4 use crate::{common::FftNum, FftDirection};
5
6 use crate::array_utils;
7 use crate::array_utils::workaround_transmute_mut;
8 use crate::array_utils::DoubleBuf;
9 use crate::common::{fft_error_inplace, fft_error_outofplace};
10 use crate::twiddles;
11 use crate::{Direction, Fft, Length};
12
13 use super::neon_common::{assert_f32, assert_f64};
14 use super::neon_utils::*;
15 use super::neon_vector::NeonArrayMut;
16
17 #[allow(unused)]
18 macro_rules! boilerplate_fft_neon_f32_butterfly {
19 ($struct_name:ident, $len:expr, $direction_fn:expr) => {
20 impl<T: FftNum> $struct_name<T> {
21 pub(crate) unsafe fn perform_fft_butterfly(&self, buffer: &mut [Complex<T>]) {
22 self.perform_fft_contiguous(workaround_transmute_mut::<_, Complex<f32>>(buffer));
23 }
24
25 pub(crate) unsafe fn perform_parallel_fft_butterfly(&self, buffer: &mut [Complex<T>]) {
26 self.perform_parallel_fft_contiguous(workaround_transmute_mut::<_, Complex<f32>>(
27 buffer,
28 ));
29 }
30
31 // Do multiple ffts over a longer vector inplace, called from "process_with_scratch" of Fft trait
32 pub(crate) unsafe fn perform_fft_butterfly_multi(
33 &self,
34 buffer: &mut [Complex<T>],
35 ) -> Result<(), ()> {
36 let len = buffer.len();
37 let alldone = array_utils::iter_chunks(buffer, 2 * self.len(), |chunk| {
38 self.perform_parallel_fft_butterfly(chunk)
39 });
40 if alldone.is_err() && buffer.len() >= self.len() {
41 self.perform_fft_butterfly(&mut buffer[len - self.len()..]);
42 }
43 Ok(())
44 }
45
46 // Do multiple ffts over a longer vector outofplace, called from "process_outofplace_with_scratch" of Fft trait
47 pub(crate) unsafe fn perform_oop_fft_butterfly_multi(
48 &self,
49 input: &mut [Complex<T>],
50 output: &mut [Complex<T>],
51 ) -> Result<(), ()> {
52 let len = input.len();
53 let alldone = array_utils::iter_chunks_zipped(
54 input,
55 output,
56 2 * self.len(),
57 |in_chunk, out_chunk| {
58 let input_slice = workaround_transmute_mut(in_chunk);
59 let output_slice = workaround_transmute_mut(out_chunk);
60 self.perform_parallel_fft_contiguous(DoubleBuf {
61 input: input_slice,
62 output: output_slice,
63 })
64 },
65 );
66 if alldone.is_err() && input.len() >= self.len() {
67 let input_slice = workaround_transmute_mut(input);
68 let output_slice = workaround_transmute_mut(output);
69 self.perform_fft_contiguous(DoubleBuf {
70 input: &mut input_slice[len - self.len()..],
71 output: &mut output_slice[len - self.len()..],
72 })
73 }
74 Ok(())
75 }
76 }
77 };
78 }
79
80 macro_rules! boilerplate_fft_neon_f64_butterfly {
81 ($struct_name:ident, $len:expr, $direction_fn:expr) => {
82 impl<T: FftNum> $struct_name<T> {
83 // Do a single fft
84 //#[target_feature(enable = "neon")]
85 pub(crate) unsafe fn perform_fft_butterfly(&self, buffer: &mut [Complex<T>]) {
86 self.perform_fft_contiguous(workaround_transmute_mut::<_, Complex<f64>>(buffer));
87 }
88
89 // Do multiple ffts over a longer vector inplace, called from "process_with_scratch" of Fft trait
90 //#[target_feature(enable = "neon")]
91 pub(crate) unsafe fn perform_fft_butterfly_multi(
92 &self,
93 buffer: &mut [Complex<T>],
94 ) -> Result<(), ()> {
95 array_utils::iter_chunks(buffer, self.len(), |chunk| {
96 self.perform_fft_butterfly(chunk)
97 })
98 }
99
100 // Do multiple ffts over a longer vector outofplace, called from "process_outofplace_with_scratch" of Fft trait
101 //#[target_feature(enable = "neon")]
102 pub(crate) unsafe fn perform_oop_fft_butterfly_multi(
103 &self,
104 input: &mut [Complex<T>],
105 output: &mut [Complex<T>],
106 ) -> Result<(), ()> {
107 array_utils::iter_chunks_zipped(input, output, self.len(), |in_chunk, out_chunk| {
108 let input_slice = workaround_transmute_mut(in_chunk);
109 let output_slice = workaround_transmute_mut(out_chunk);
110 self.perform_fft_contiguous(DoubleBuf {
111 input: input_slice,
112 output: output_slice,
113 })
114 })
115 }
116 }
117 };
118 }
119
120 #[allow(unused)]
121 macro_rules! boilerplate_fft_neon_common_butterfly {
122 ($struct_name:ident, $len:expr, $direction_fn:expr) => {
123 impl<T: FftNum> Fft<T> for $struct_name<T> {
124 fn process_outofplace_with_scratch(
125 &self,
126 input: &mut [Complex<T>],
127 output: &mut [Complex<T>],
128 _scratch: &mut [Complex<T>],
129 ) {
130 if input.len() < self.len() || output.len() != input.len() {
131 // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
132 fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
133 return; // Unreachable, because fft_error_outofplace asserts, but it helps codegen to put it here
134 }
135 let result = unsafe { self.perform_oop_fft_butterfly_multi(input, output) };
136
137 if result.is_err() {
138 // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
139 // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
140 fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
141 }
142 }
143 fn process_with_scratch(&self, buffer: &mut [Complex<T>], _scratch: &mut [Complex<T>]) {
144 if buffer.len() < self.len() {
145 // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
146 fft_error_inplace(self.len(), buffer.len(), 0, 0);
147 return; // Unreachable, because fft_error_inplace asserts, but it helps codegen to put it here
148 }
149
150 let result = unsafe { self.perform_fft_butterfly_multi(buffer) };
151
152 if result.is_err() {
153 // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
154 // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
155 fft_error_inplace(self.len(), buffer.len(), 0, 0);
156 }
157 }
158 #[inline(always)]
159 fn get_inplace_scratch_len(&self) -> usize {
160 0
161 }
162 #[inline(always)]
163 fn get_outofplace_scratch_len(&self) -> usize {
164 0
165 }
166 }
167 impl<T> Length for $struct_name<T> {
168 #[inline(always)]
169 fn len(&self) -> usize {
170 $len
171 }
172 }
173 impl<T> Direction for $struct_name<T> {
174 #[inline(always)]
175 fn fft_direction(&self) -> FftDirection {
176 $direction_fn(self)
177 }
178 }
179 };
180 }
181
182 // _ _________ _ _ _
183 // / | |___ /___ \| |__ (_) |_
184 // | | _____ |_ \ __) | '_ \| | __|
185 // | | |_____| ___) / __/| |_) | | |_
186 // |_| |____/_____|_.__/|_|\__|
187 //
188
189 pub struct NeonF32Butterfly1<T> {
190 direction: FftDirection,
191 _phantom: std::marker::PhantomData<T>,
192 }
193
194 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly1, 1, |this: &NeonF32Butterfly1<_>| this
195 .direction);
196 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly1, 1, |this: &NeonF32Butterfly1<_>| this
197 .direction);
198 impl<T: FftNum> NeonF32Butterfly1<T> {
199 #[inline(always)]
200 pub fn new(direction: FftDirection) -> Self {
201 assert_f32::<T>();
202 Self {
203 direction,
204 _phantom: std::marker::PhantomData,
205 }
206 }
207 #[inline(always)]
208 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
209 let value = buffer.load_partial1_complex(0);
210 buffer.store_partial_lo_complex(value, 0);
211 }
212
213 #[inline(always)]
214 pub(crate) unsafe fn perform_parallel_fft_contiguous(
215 &self,
216 mut buffer: impl NeonArrayMut<f32>,
217 ) {
218 let value = buffer.load_complex(0);
219 buffer.store_complex(value, 0);
220 }
221 }
222
223 // _ __ _ _ _ _ _
224 // / | / /_ | || | | |__ (_) |_
225 // | | _____ | '_ \| || |_| '_ \| | __|
226 // | | |_____| | (_) |__ _| |_) | | |_
227 // |_| \___/ |_| |_.__/|_|\__|
228 //
229
230 pub struct NeonF64Butterfly1<T> {
231 direction: FftDirection,
232 _phantom: std::marker::PhantomData<T>,
233 }
234
235 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly1, 1, |this: &NeonF64Butterfly1<_>| this
236 .direction);
237 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly1, 1, |this: &NeonF64Butterfly1<_>| this
238 .direction);
239 impl<T: FftNum> NeonF64Butterfly1<T> {
240 #[inline(always)]
241 pub fn new(direction: FftDirection) -> Self {
242 assert_f64::<T>();
243 Self {
244 direction,
245 _phantom: std::marker::PhantomData,
246 }
247 }
248 #[inline(always)]
249 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
250 let value = buffer.load_complex(0);
251 buffer.store_complex(value, 0);
252 }
253 }
254
255 // ____ _________ _ _ _
256 // |___ \ |___ /___ \| |__ (_) |_
257 // __) | _____ |_ \ __) | '_ \| | __|
258 // / __/ |_____| ___) / __/| |_) | | |_
259 // |_____| |____/_____|_.__/|_|\__|
260 //
261
262 pub struct NeonF32Butterfly2<T> {
263 direction: FftDirection,
264 _phantom: std::marker::PhantomData<T>,
265 }
266
267 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly2, 2, |this: &NeonF32Butterfly2<_>| this
268 .direction);
269 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly2, 2, |this: &NeonF32Butterfly2<_>| this
270 .direction);
271 impl<T: FftNum> NeonF32Butterfly2<T> {
272 #[inline(always)]
273 pub fn new(direction: FftDirection) -> Self {
274 assert_f32::<T>();
275 Self {
276 direction,
277 _phantom: std::marker::PhantomData,
278 }
279 }
280 #[inline(always)]
281 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
282 let values = buffer.load_complex(0);
283
284 let temp = self.perform_fft_direct(values);
285
286 buffer.store_complex(temp, 0);
287 }
288
289 #[inline(always)]
290 pub(crate) unsafe fn perform_parallel_fft_contiguous(
291 &self,
292 mut buffer: impl NeonArrayMut<f32>,
293 ) {
294 let values_a = buffer.load_complex(0);
295 let values_b = buffer.load_complex(2);
296
297 let out = self.perform_parallel_fft_direct(values_a, values_b);
298
299 let [out02, out13] = transpose_complex_2x2_f32(out[0], out[1]);
300
301 buffer.store_complex(out02, 0);
302 buffer.store_complex(out13, 2);
303 }
304
305 // length 2 fft of x, given as [x0, x1]
306 // result is [X0, X1]
307 #[inline(always)]
308 pub(crate) unsafe fn perform_fft_direct(&self, values: float32x4_t) -> float32x4_t {
309 solo_fft2_f32(values)
310 }
311
312 // dual length 2 fft of x and y, given as [x0, x1], [y0, y1]
313 // result is [X0, Y0], [X1, Y1]
314 #[inline(always)]
315 pub(crate) unsafe fn perform_parallel_fft_direct(
316 &self,
317 values_x: float32x4_t,
318 values_y: float32x4_t,
319 ) -> [float32x4_t; 2] {
320 parallel_fft2_contiguous_f32(values_x, values_y)
321 }
322 }
323
324 // double lenth 2 fft of a and b, given as [x0, y0], [x1, y1]
325 // result is [X0, Y0], [X1, Y1]
326 #[inline(always)]
327 pub(crate) unsafe fn parallel_fft2_interleaved_f32(
328 val02: float32x4_t,
329 val13: float32x4_t,
330 ) -> [float32x4_t; 2] {
331 let temp0 = vaddq_f32(val02, val13);
332 let temp1 = vsubq_f32(val02, val13);
333 [temp0, temp1]
334 }
335
336 // double lenth 2 fft of a and b, given as [x0, x1], [y0, y1]
337 // result is [X0, Y0], [X1, Y1]
338 #[inline(always)]
339 unsafe fn parallel_fft2_contiguous_f32(left: float32x4_t, right: float32x4_t) -> [float32x4_t; 2] {
340 let [temp02, temp13] = transpose_complex_2x2_f32(left, right);
341 parallel_fft2_interleaved_f32(temp02, temp13)
342 }
343
344 // length 2 fft of x, given as [x0, x1]
345 // result is [X0, X1]
346 #[inline(always)]
347 unsafe fn solo_fft2_f32(values: float32x4_t) -> float32x4_t {
348 let high = vget_high_f32(values);
349 let low = vget_low_f32(values);
350 vcombine_f32(vadd_f32(low, high), vsub_f32(low, high))
351 }
352
353 // ____ __ _ _ _ _ _
354 // |___ \ / /_ | || | | |__ (_) |_
355 // __) | _____ | '_ \| || |_| '_ \| | __|
356 // / __/ |_____| | (_) |__ _| |_) | | |_
357 // |_____| \___/ |_| |_.__/|_|\__|
358 //
359
360 pub struct NeonF64Butterfly2<T> {
361 direction: FftDirection,
362 _phantom: std::marker::PhantomData<T>,
363 }
364
365 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly2, 2, |this: &NeonF64Butterfly2<_>| this
366 .direction);
367 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly2, 2, |this: &NeonF64Butterfly2<_>| this
368 .direction);
369 impl<T: FftNum> NeonF64Butterfly2<T> {
370 #[inline(always)]
371 pub fn new(direction: FftDirection) -> Self {
372 assert_f64::<T>();
373 Self {
374 direction,
375 _phantom: std::marker::PhantomData,
376 }
377 }
378
379 #[inline(always)]
380 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
381 let value0 = buffer.load_complex(0);
382 let value1 = buffer.load_complex(1);
383
384 let out = self.perform_fft_direct(value0, value1);
385
386 buffer.store_complex(out[0], 0);
387 buffer.store_complex(out[1], 1);
388 }
389
390 #[inline(always)]
391 pub(crate) unsafe fn perform_fft_direct(
392 &self,
393 value0: float64x2_t,
394 value1: float64x2_t,
395 ) -> [float64x2_t; 2] {
396 solo_fft2_f64(value0, value1)
397 }
398 }
399
400 #[inline(always)]
401 pub(crate) unsafe fn solo_fft2_f64(left: float64x2_t, right: float64x2_t) -> [float64x2_t; 2] {
402 let temp0 = vaddq_f64(left, right);
403 let temp1 = vsubq_f64(left, right);
404 [temp0, temp1]
405 }
406
407 // _____ _________ _ _ _
408 // |___ / |___ /___ \| |__ (_) |_
409 // |_ \ _____ |_ \ __) | '_ \| | __|
410 // ___) | |_____| ___) / __/| |_) | | |_
411 // |____/ |____/_____|_.__/|_|\__|
412 //
413
414 pub struct NeonF32Butterfly3<T> {
415 direction: FftDirection,
416 _phantom: std::marker::PhantomData<T>,
417 rotate: Rotate90F32,
418 twiddle: float32x4_t,
419 twiddle1re: float32x4_t,
420 twiddle1im: float32x4_t,
421 }
422
423 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly3, 3, |this: &NeonF32Butterfly3<_>| this
424 .direction);
425 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly3, 3, |this: &NeonF32Butterfly3<_>| this
426 .direction);
427 impl<T: FftNum> NeonF32Butterfly3<T> {
428 #[inline(always)]
429 pub fn new(direction: FftDirection) -> Self {
430 assert_f32::<T>();
431 let rotate = Rotate90F32::new(true);
432 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 3, direction);
433 let twiddle = unsafe { vld1q_f32([tw1.re, tw1.re, -tw1.im, -tw1.im].as_ptr()) };
434 let twiddle1re = unsafe { vmovq_n_f32(tw1.re) };
435 let twiddle1im = unsafe { vmovq_n_f32(tw1.im) };
436 Self {
437 direction,
438 _phantom: std::marker::PhantomData,
439 rotate,
440 twiddle,
441 twiddle1re,
442 twiddle1im,
443 }
444 }
445 #[inline(always)]
446 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
447 let value0x = buffer.load_partial1_complex(0);
448 let value12 = buffer.load_complex(1);
449
450 let out = self.perform_fft_direct(value0x, value12);
451
452 buffer.store_partial_lo_complex(out[0], 0);
453 buffer.store_complex(out[1], 1);
454 }
455
456 #[inline(always)]
457 pub(crate) unsafe fn perform_parallel_fft_contiguous(
458 &self,
459 mut buffer: impl NeonArrayMut<f32>,
460 ) {
461 let valuea0a1 = buffer.load_complex(0);
462 let valuea2b0 = buffer.load_complex(2);
463 let valueb1b2 = buffer.load_complex(4);
464
465 let value0 = extract_lo_hi_f32(valuea0a1, valuea2b0);
466 let value1 = extract_hi_lo_f32(valuea0a1, valueb1b2);
467 let value2 = extract_lo_hi_f32(valuea2b0, valueb1b2);
468
469 let out = self.perform_parallel_fft_direct(value0, value1, value2);
470
471 let out0 = extract_lo_lo_f32(out[0], out[1]);
472 let out1 = extract_lo_hi_f32(out[2], out[0]);
473 let out2 = extract_hi_hi_f32(out[1], out[2]);
474
475 buffer.store_complex(out0, 0);
476 buffer.store_complex(out1, 2);
477 buffer.store_complex(out2, 4);
478 }
479
480 // length 3 fft of a, given as [x0, 0.0], [x1, x2]
481 // result is [X0, Z], [X1, X2]
482 // The value Z should be discarded.
483 #[inline(always)]
484 pub(crate) unsafe fn perform_fft_direct(
485 &self,
486 value0x: float32x4_t,
487 value12: float32x4_t,
488 ) -> [float32x4_t; 2] {
489 // This is a Neon translation of the scalar 3-point butterfly
490 let rev12 = reverse_complex_and_negate_hi_f32(value12);
491 let temp12pn = self.rotate.rotate_hi(vaddq_f32(value12, rev12));
492 let twiddled = vmulq_f32(temp12pn, self.twiddle);
493 let temp = vaddq_f32(value0x, twiddled);
494
495 let out12 = solo_fft2_f32(temp);
496 let out0x = vaddq_f32(value0x, temp12pn);
497 [out0x, out12]
498 }
499
500 // length 3 dual fft of a, given as (x0, y0), (x1, y1), (x2, y2).
501 // result is [(X0, Y0), (X1, Y1), (X2, Y2)]
502 #[inline(always)]
503 pub(crate) unsafe fn perform_parallel_fft_direct(
504 &self,
505 value0: float32x4_t,
506 value1: float32x4_t,
507 value2: float32x4_t,
508 ) -> [float32x4_t; 3] {
509 // This is a Neon translation of the scalar 3-point butterfly
510 let x12p = vaddq_f32(value1, value2);
511 let x12n = vsubq_f32(value1, value2);
512 let sum = vaddq_f32(value0, x12p);
513
514 let temp_a = vmulq_f32(self.twiddle1re, x12p);
515 let temp_a = vaddq_f32(temp_a, value0);
516
517 let n_rot = self.rotate.rotate_both(x12n);
518 let temp_b = vmulq_f32(self.twiddle1im, n_rot);
519
520 let x1 = vaddq_f32(temp_a, temp_b);
521 let x2 = vsubq_f32(temp_a, temp_b);
522 [sum, x1, x2]
523 }
524 }
525
526 // _____ __ _ _ _ _ _
527 // |___ / / /_ | || | | |__ (_) |_
528 // |_ \ _____ | '_ \| || |_| '_ \| | __|
529 // ___) | |_____| | (_) |__ _| |_) | | |_
530 // |____/ \___/ |_| |_.__/|_|\__|
531 //
532
533 pub struct NeonF64Butterfly3<T> {
534 direction: FftDirection,
535 _phantom: std::marker::PhantomData<T>,
536 rotate: Rotate90F64,
537 twiddle1re: float64x2_t,
538 twiddle1im: float64x2_t,
539 }
540
541 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly3, 3, |this: &NeonF64Butterfly3<_>| this
542 .direction);
543 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly3, 3, |this: &NeonF64Butterfly3<_>| this
544 .direction);
545 impl<T: FftNum> NeonF64Butterfly3<T> {
546 #[inline(always)]
547 pub fn new(direction: FftDirection) -> Self {
548 assert_f64::<T>();
549 let rotate = Rotate90F64::new(true);
550 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 3, direction);
551 let twiddle1re = unsafe { vmovq_n_f64(tw1.re) };
552 let twiddle1im = unsafe { vmovq_n_f64(tw1.im) };
553
554 Self {
555 direction,
556 _phantom: std::marker::PhantomData,
557 rotate,
558 twiddle1re,
559 twiddle1im,
560 }
561 }
562
563 #[inline(always)]
564 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
565 let value0 = buffer.load_complex(0);
566 let value1 = buffer.load_complex(1);
567 let value2 = buffer.load_complex(2);
568
569 let out = self.perform_fft_direct(value0, value1, value2);
570
571 buffer.store_complex(out[0], 0);
572 buffer.store_complex(out[1], 1);
573 buffer.store_complex(out[2], 2);
574 }
575
576 // length 3 fft of x, given as x0, x1, x2.
577 // result is [X0, X1, X2]
578 #[inline(always)]
579 pub(crate) unsafe fn perform_fft_direct(
580 &self,
581 value0: float64x2_t,
582 value1: float64x2_t,
583 value2: float64x2_t,
584 ) -> [float64x2_t; 3] {
585 // This is a Neon translation of the scalar 3-point butterfly
586 let x12p = vaddq_f64(value1, value2);
587 let x12n = vsubq_f64(value1, value2);
588 let sum = vaddq_f64(value0, x12p);
589
590 let temp_a = vfmaq_f64(value0, self.twiddle1re, x12p);
591
592 let n_rot = self.rotate.rotate(x12n);
593 let temp_b = vmulq_f64(self.twiddle1im, n_rot);
594
595 let x1 = vaddq_f64(temp_a, temp_b);
596 let x2 = vsubq_f64(temp_a, temp_b);
597 [sum, x1, x2]
598 }
599 }
600
601 // _ _ _________ _ _ _
602 // | || | |___ /___ \| |__ (_) |_
603 // | || |_ _____ |_ \ __) | '_ \| | __|
604 // |__ _| |_____| ___) / __/| |_) | | |_
605 // |_| |____/_____|_.__/|_|\__|
606 //
607
608 pub struct NeonF32Butterfly4<T> {
609 direction: FftDirection,
610 _phantom: std::marker::PhantomData<T>,
611 rotate: Rotate90F32,
612 }
613
614 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly4, 4, |this: &NeonF32Butterfly4<_>| this
615 .direction);
616 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly4, 4, |this: &NeonF32Butterfly4<_>| this
617 .direction);
618 impl<T: FftNum> NeonF32Butterfly4<T> {
619 #[inline(always)]
620 pub fn new(direction: FftDirection) -> Self {
621 assert_f32::<T>();
622 let rotate = if direction == FftDirection::Inverse {
623 Rotate90F32::new(true)
624 } else {
625 Rotate90F32::new(false)
626 };
627 Self {
628 direction,
629 _phantom: std::marker::PhantomData,
630 rotate,
631 }
632 }
633 #[inline(always)]
634 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
635 let value01 = buffer.load_complex(0);
636 let value23 = buffer.load_complex(2);
637
638 let out = self.perform_fft_direct(value01, value23);
639
640 buffer.store_complex(out[0], 0);
641 buffer.store_complex(out[1], 2);
642 }
643
644 #[inline(always)]
645 pub(crate) unsafe fn perform_parallel_fft_contiguous(
646 &self,
647 mut buffer: impl NeonArrayMut<f32>,
648 ) {
649 let value01a = buffer.load_complex(0);
650 let value23a = buffer.load_complex(2);
651 let value01b = buffer.load_complex(4);
652 let value23b = buffer.load_complex(6);
653
654 let [value0ab, value1ab] = transpose_complex_2x2_f32(value01a, value01b);
655 let [value2ab, value3ab] = transpose_complex_2x2_f32(value23a, value23b);
656
657 let out = self.perform_parallel_fft_direct(value0ab, value1ab, value2ab, value3ab);
658
659 let [out0, out1] = transpose_complex_2x2_f32(out[0], out[1]);
660 let [out2, out3] = transpose_complex_2x2_f32(out[2], out[3]);
661
662 buffer.store_complex(out0, 0);
663 buffer.store_complex(out1, 4);
664 buffer.store_complex(out2, 2);
665 buffer.store_complex(out3, 6);
666 }
667
668 // length 4 fft of a, given as [x0, x1], [x2, x3]
669 // result is [[X0, X1], [X2, X3]]
670 #[inline(always)]
671 pub(crate) unsafe fn perform_fft_direct(
672 &self,
673 value01: float32x4_t,
674 value23: float32x4_t,
675 ) -> [float32x4_t; 2] {
676 //we're going to hardcode a step of mixed radix
677 //aka we're going to do the six step algorithm
678
679 // step 1: transpose
680 // and
681 // step 2: column FFTs
682 let mut temp = parallel_fft2_interleaved_f32(value01, value23);
683
684 // step 3: apply twiddle factors (only one in this case, and it's either 0 + i or 0 - i)
685 temp[1] = self.rotate.rotate_hi(temp[1]);
686
687 // step 4: transpose, which we're skipping because we're the previous FFTs were non-contiguous
688
689 // step 5: row FFTs
690 // and
691 // step 6: transpose by swapping index 1 and 2
692 parallel_fft2_contiguous_f32(temp[0], temp[1])
693 }
694
695 #[inline(always)]
696 pub(crate) unsafe fn perform_parallel_fft_direct(
697 &self,
698 values0: float32x4_t,
699 values1: float32x4_t,
700 values2: float32x4_t,
701 values3: float32x4_t,
702 ) -> [float32x4_t; 4] {
703 //we're going to hardcode a step of mixed radix
704 //aka we're going to do the six step algorithm
705
706 // step 1: transpose
707 // and
708 // step 2: column FFTs
709 let temp0 = parallel_fft2_interleaved_f32(values0, values2);
710 let mut temp1 = parallel_fft2_interleaved_f32(values1, values3);
711
712 // step 3: apply twiddle factors (only one in this case, and it's either 0 + i or 0 - i)
713 temp1[1] = self.rotate.rotate_both(temp1[1]);
714
715 // step 4: transpose, which we're skipping because we're the previous FFTs were non-contiguous
716
717 // step 5: row FFTs
718 let out0 = parallel_fft2_interleaved_f32(temp0[0], temp1[0]);
719 let out2 = parallel_fft2_interleaved_f32(temp0[1], temp1[1]);
720
721 // step 6: transpose by swapping index 1 and 2
722 [out0[0], out2[0], out0[1], out2[1]]
723 }
724 }
725
726 // _ _ __ _ _ _ _ _
727 // | || | / /_ | || | | |__ (_) |_
728 // | || |_ _____ | '_ \| || |_| '_ \| | __|
729 // |__ _| |_____| | (_) |__ _| |_) | | |_
730 // |_| \___/ |_| |_.__/|_|\__|
731 //
732
733 pub struct NeonF64Butterfly4<T> {
734 direction: FftDirection,
735 _phantom: std::marker::PhantomData<T>,
736 rotate: Rotate90F64,
737 }
738
739 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly4, 4, |this: &NeonF64Butterfly4<_>| this
740 .direction);
741 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly4, 4, |this: &NeonF64Butterfly4<_>| this
742 .direction);
743 impl<T: FftNum> NeonF64Butterfly4<T> {
744 #[inline(always)]
745 pub fn new(direction: FftDirection) -> Self {
746 assert_f64::<T>();
747 let rotate = if direction == FftDirection::Inverse {
748 Rotate90F64::new(true)
749 } else {
750 Rotate90F64::new(false)
751 };
752
753 Self {
754 direction,
755 _phantom: std::marker::PhantomData,
756 rotate,
757 }
758 }
759
760 #[inline(always)]
761 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
762 let value0 = buffer.load_complex(0);
763 let value1 = buffer.load_complex(1);
764 let value2 = buffer.load_complex(2);
765 let value3 = buffer.load_complex(3);
766
767 let out = self.perform_fft_direct(value0, value1, value2, value3);
768
769 buffer.store_complex(out[0], 0);
770 buffer.store_complex(out[1], 1);
771 buffer.store_complex(out[2], 2);
772 buffer.store_complex(out[3], 3);
773 }
774
775 #[inline(always)]
776 pub(crate) unsafe fn perform_fft_direct(
777 &self,
778 value0: float64x2_t,
779 value1: float64x2_t,
780 value2: float64x2_t,
781 value3: float64x2_t,
782 ) -> [float64x2_t; 4] {
783 //we're going to hardcode a step of mixed radix
784 //aka we're going to do the six step algorithm
785
786 // step 1: transpose
787 // and
788 // step 2: column FFTs
789 let temp0 = solo_fft2_f64(value0, value2);
790 let mut temp1 = solo_fft2_f64(value1, value3);
791
792 // step 3: apply twiddle factors (only one in this case, and it's either 0 + i or 0 - i)
793 temp1[1] = self.rotate.rotate(temp1[1]);
794
795 // step 4: transpose, which we're skipping because we're the previous FFTs were non-contiguous
796
797 // step 5: row FFTs
798 let out0 = solo_fft2_f64(temp0[0], temp1[0]);
799 let out2 = solo_fft2_f64(temp0[1], temp1[1]);
800
801 // step 6: transpose by swapping index 1 and 2
802 [out0[0], out2[0], out0[1], out2[1]]
803 }
804 }
805
806 // ____ _________ _ _ _
807 // | ___| |___ /___ \| |__ (_) |_
808 // |___ \ _____ |_ \ __) | '_ \| | __|
809 // ___) | |_____| ___) / __/| |_) | | |_
810 // |____/ |____/_____|_.__/|_|\__|
811 //
812
813 pub struct NeonF32Butterfly5<T> {
814 direction: FftDirection,
815 _phantom: std::marker::PhantomData<T>,
816 rotate: Rotate90F32,
817 twiddle12re: float32x4_t,
818 twiddle21re: float32x4_t,
819 twiddle12im: float32x4_t,
820 twiddle21im: float32x4_t,
821 twiddle1re: float32x4_t,
822 twiddle1im: float32x4_t,
823 twiddle2re: float32x4_t,
824 twiddle2im: float32x4_t,
825 }
826
827 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly5, 5, |this: &NeonF32Butterfly5<_>| this
828 .direction);
829 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly5, 5, |this: &NeonF32Butterfly5<_>| this
830 .direction);
831 impl<T: FftNum> NeonF32Butterfly5<T> {
832 #[inline(always)]
833 pub fn new(direction: FftDirection) -> Self {
834 assert_f32::<T>();
835 let rotate = Rotate90F32::new(true);
836 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 5, direction);
837 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 5, direction);
838 let twiddle12re = unsafe { vld1q_f32([tw1.re, tw1.re, tw2.re, tw2.re].as_ptr()) };
839 let twiddle21re = unsafe { vld1q_f32([tw2.re, tw2.re, tw1.re, tw1.re].as_ptr()) };
840 let twiddle12im = unsafe { vld1q_f32([tw1.im, tw1.im, tw2.im, tw2.im].as_ptr()) };
841 let twiddle21im = unsafe { vld1q_f32([tw2.im, tw2.im, -tw1.im, -tw1.im].as_ptr()) };
842 let twiddle1re = unsafe { vmovq_n_f32(tw1.re) };
843 let twiddle1im = unsafe { vmovq_n_f32(tw1.im) };
844 let twiddle2re = unsafe { vmovq_n_f32(tw2.re) };
845 let twiddle2im = unsafe { vmovq_n_f32(tw2.im) };
846
847 Self {
848 direction,
849 _phantom: std::marker::PhantomData,
850 rotate,
851 twiddle12re,
852 twiddle21re,
853 twiddle12im,
854 twiddle21im,
855 twiddle1re,
856 twiddle1im,
857 twiddle2re,
858 twiddle2im,
859 }
860 }
861 #[inline(always)]
862 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
863 let value00 = buffer.load1_complex(0);
864 let value12 = buffer.load_complex(1);
865 let value34 = buffer.load_complex(3);
866
867 let out = self.perform_fft_direct(value00, value12, value34);
868
869 buffer.store_partial_lo_complex(out[0], 0);
870 buffer.store_complex(out[1], 1);
871 buffer.store_complex(out[2], 3);
872 }
873
874 #[inline(always)]
875 pub(crate) unsafe fn perform_parallel_fft_contiguous(
876 &self,
877 mut buffer: impl NeonArrayMut<f32>,
878 ) {
879 let input_packed = read_complex_to_array!(buffer, {0, 2, 4 ,6, 8});
880
881 let value0 = extract_lo_hi_f32(input_packed[0], input_packed[2]);
882 let value1 = extract_hi_lo_f32(input_packed[0], input_packed[3]);
883 let value2 = extract_lo_hi_f32(input_packed[1], input_packed[3]);
884 let value3 = extract_hi_lo_f32(input_packed[1], input_packed[4]);
885 let value4 = extract_lo_hi_f32(input_packed[2], input_packed[4]);
886
887 let out = self.perform_parallel_fft_direct(value0, value1, value2, value3, value4);
888
889 let out_packed = [
890 extract_lo_lo_f32(out[0], out[1]),
891 extract_lo_lo_f32(out[2], out[3]),
892 extract_lo_hi_f32(out[4], out[0]),
893 extract_hi_hi_f32(out[1], out[2]),
894 extract_hi_hi_f32(out[3], out[4]),
895 ];
896
897 write_complex_to_array_strided!(out_packed, buffer, 2, {0, 1, 2, 3, 4});
898 }
899
900 // length 5 fft of a, given as [x0, x0], [x1, x2], [x3, x4].
901 // result is [[X0, Z], [X1, X2], [X3, X4]]
902 // Note that Z should not be used.
903 #[inline(always)]
904 pub(crate) unsafe fn perform_fft_direct(
905 &self,
906 value00: float32x4_t,
907 value12: float32x4_t,
908 value34: float32x4_t,
909 ) -> [float32x4_t; 3] {
910 // This is a Neon translation of the scalar 5-point butterfly
911 let temp43 = reverse_complex_elements_f32(value34);
912 let x1423p = vaddq_f32(value12, temp43);
913 let x1423n = vsubq_f32(value12, temp43);
914
915 let x1414p = duplicate_lo_f32(x1423p);
916 let x2323p = duplicate_hi_f32(x1423p);
917 let x1414n = duplicate_lo_f32(x1423n);
918 let x2323n = duplicate_hi_f32(x1423n);
919
920 let temp_a1 = vmulq_f32(self.twiddle12re, x1414p);
921 let temp_b1 = vmulq_f32(self.twiddle12im, x1414n);
922
923 let temp_a = vfmaq_f32(temp_a1, self.twiddle21re, x2323p);
924 let temp_a = vaddq_f32(value00, temp_a);
925 let temp_b = vfmaq_f32(temp_b1, self.twiddle21im, x2323n);
926
927 let b_rot = self.rotate.rotate_both(temp_b);
928
929 let x00 = vaddq_f32(value00, vaddq_f32(x1414p, x2323p));
930
931 let x12 = vaddq_f32(temp_a, b_rot);
932 let x34 = reverse_complex_elements_f32(vsubq_f32(temp_a, b_rot));
933 [x00, x12, x34]
934 }
935
936 // length 5 dual fft of x and y, given as (x0, y0), (x1, y1) ... (x4, y4).
937 // result is [(X0, Y0), (X1, Y1) ... (X2, Y2)]
938 #[inline(always)]
939 pub(crate) unsafe fn perform_parallel_fft_direct(
940 &self,
941 value0: float32x4_t,
942 value1: float32x4_t,
943 value2: float32x4_t,
944 value3: float32x4_t,
945 value4: float32x4_t,
946 ) -> [float32x4_t; 5] {
947 // This is a Neon translation of the scalar 3-point butterfly
948 let x14p = vaddq_f32(value1, value4);
949 let x14n = vsubq_f32(value1, value4);
950 let x23p = vaddq_f32(value2, value3);
951 let x23n = vsubq_f32(value2, value3);
952
953 let temp_a1_1 = vmulq_f32(self.twiddle1re, x14p);
954 let temp_a1_2 = vmulq_f32(self.twiddle2re, x23p);
955 let temp_b1_1 = vmulq_f32(self.twiddle1im, x14n);
956 let temp_b1_2 = vmulq_f32(self.twiddle2im, x23n);
957 let temp_a2_1 = vmulq_f32(self.twiddle1re, x23p);
958 let temp_a2_2 = vmulq_f32(self.twiddle2re, x14p);
959 let temp_b2_1 = vmulq_f32(self.twiddle2im, x14n);
960 let temp_b2_2 = vmulq_f32(self.twiddle1im, x23n);
961
962 let temp_a1 = vaddq_f32(value0, vaddq_f32(temp_a1_1, temp_a1_2));
963 let temp_b1 = vaddq_f32(temp_b1_1, temp_b1_2);
964 let temp_a2 = vaddq_f32(value0, vaddq_f32(temp_a2_1, temp_a2_2));
965 let temp_b2 = vsubq_f32(temp_b2_1, temp_b2_2);
966
967 [
968 vaddq_f32(value0, vaddq_f32(x14p, x23p)),
969 vaddq_f32(temp_a1, self.rotate.rotate_both(temp_b1)),
970 vaddq_f32(temp_a2, self.rotate.rotate_both(temp_b2)),
971 vsubq_f32(temp_a2, self.rotate.rotate_both(temp_b2)),
972 vsubq_f32(temp_a1, self.rotate.rotate_both(temp_b1)),
973 ]
974 }
975 }
976
977 // ____ __ _ _ _ _ _
978 // | ___| / /_ | || | | |__ (_) |_
979 // |___ \ _____ | '_ \| || |_| '_ \| | __|
980 // ___) | |_____| | (_) |__ _| |_) | | |_
981 // |____/ \___/ |_| |_.__/|_|\__|
982 //
983
984 pub struct NeonF64Butterfly5<T> {
985 direction: FftDirection,
986 _phantom: std::marker::PhantomData<T>,
987 rotate: Rotate90F64,
988 twiddle1re: float64x2_t,
989 twiddle1im: float64x2_t,
990 twiddle2re: float64x2_t,
991 twiddle2im: float64x2_t,
992 }
993
994 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly5, 5, |this: &NeonF64Butterfly5<_>| this
995 .direction);
996 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly5, 5, |this: &NeonF64Butterfly5<_>| this
997 .direction);
998 impl<T: FftNum> NeonF64Butterfly5<T> {
999 #[inline(always)]
1000 pub fn new(direction: FftDirection) -> Self {
1001 assert_f64::<T>();
1002 let rotate = Rotate90F64::new(true);
1003 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 5, direction);
1004 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 5, direction);
1005 let twiddle1re = unsafe { vmovq_n_f64(tw1.re) };
1006 let twiddle1im = unsafe { vmovq_n_f64(tw1.im) };
1007 let twiddle2re = unsafe { vmovq_n_f64(tw2.re) };
1008 let twiddle2im = unsafe { vmovq_n_f64(tw2.im) };
1009
1010 Self {
1011 direction,
1012 _phantom: std::marker::PhantomData,
1013 rotate,
1014 twiddle1re,
1015 twiddle1im,
1016 twiddle2re,
1017 twiddle2im,
1018 }
1019 }
1020
1021 #[inline(always)]
1022 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
1023 let value0 = buffer.load_complex(0);
1024 let value1 = buffer.load_complex(1);
1025 let value2 = buffer.load_complex(2);
1026 let value3 = buffer.load_complex(3);
1027 let value4 = buffer.load_complex(4);
1028
1029 let out = self.perform_fft_direct(value0, value1, value2, value3, value4);
1030
1031 buffer.store_complex(out[0], 0);
1032 buffer.store_complex(out[1], 1);
1033 buffer.store_complex(out[2], 2);
1034 buffer.store_complex(out[3], 3);
1035 buffer.store_complex(out[4], 4);
1036 }
1037
1038 // length 5 fft of x, given as x0, x1, x2, x3, x4.
1039 // result is [X0, X1, X2, X3, X4]
1040 #[inline(always)]
1041 pub(crate) unsafe fn perform_fft_direct(
1042 &self,
1043 value0: float64x2_t,
1044 value1: float64x2_t,
1045 value2: float64x2_t,
1046 value3: float64x2_t,
1047 value4: float64x2_t,
1048 ) -> [float64x2_t; 5] {
1049 // This is a Neon translation of the scalar 5-point butterfly
1050 let x14p = vaddq_f64(value1, value4);
1051 let x14n = vsubq_f64(value1, value4);
1052 let x23p = vaddq_f64(value2, value3);
1053 let x23n = vsubq_f64(value2, value3);
1054
1055 let temp_a1_1 = vmulq_f64(self.twiddle1re, x14p);
1056 let temp_a1_2 = vmulq_f64(self.twiddle2re, x23p);
1057 let temp_a2_1 = vmulq_f64(self.twiddle2re, x14p);
1058 let temp_a2_2 = vmulq_f64(self.twiddle1re, x23p);
1059
1060 let temp_b1_1 = vmulq_f64(self.twiddle1im, x14n);
1061 let temp_b1_2 = vmulq_f64(self.twiddle2im, x23n);
1062 let temp_b2_1 = vmulq_f64(self.twiddle2im, x14n);
1063 let temp_b2_2 = vmulq_f64(self.twiddle1im, x23n);
1064
1065 let temp_a1 = vaddq_f64(value0, vaddq_f64(temp_a1_1, temp_a1_2));
1066 let temp_a2 = vaddq_f64(value0, vaddq_f64(temp_a2_1, temp_a2_2));
1067
1068 let temp_b1 = vaddq_f64(temp_b1_1, temp_b1_2);
1069 let temp_b2 = vsubq_f64(temp_b2_1, temp_b2_2);
1070
1071 let temp_b1_rot = self.rotate.rotate(temp_b1);
1072 let temp_b2_rot = self.rotate.rotate(temp_b2);
1073 [
1074 vaddq_f64(value0, vaddq_f64(x14p, x23p)),
1075 vaddq_f64(temp_a1, temp_b1_rot),
1076 vaddq_f64(temp_a2, temp_b2_rot),
1077 vsubq_f64(temp_a2, temp_b2_rot),
1078 vsubq_f64(temp_a1, temp_b1_rot),
1079 ]
1080 }
1081 }
1082
1083 // __ _________ _ _ _
1084 // / /_ |___ /___ \| |__ (_) |_
1085 // | '_ \ _____ |_ \ __) | '_ \| | __|
1086 // | (_) | |_____| ___) / __/| |_) | | |_
1087 // \___/ |____/_____|_.__/|_|\__|
1088 //
1089
1090 pub struct NeonF32Butterfly6<T> {
1091 direction: FftDirection,
1092 _phantom: std::marker::PhantomData<T>,
1093 bf3: NeonF32Butterfly3<T>,
1094 }
1095
1096 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly6, 6, |this: &NeonF32Butterfly6<_>| this
1097 .direction);
1098 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly6, 6, |this: &NeonF32Butterfly6<_>| this
1099 .direction);
1100 impl<T: FftNum> NeonF32Butterfly6<T> {
1101 #[inline(always)]
1102 pub fn new(direction: FftDirection) -> Self {
1103 assert_f32::<T>();
1104 let bf3 = NeonF32Butterfly3::new(direction);
1105
1106 Self {
1107 direction,
1108 _phantom: std::marker::PhantomData,
1109 bf3,
1110 }
1111 }
1112
1113 #[inline(always)]
1114 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
1115 let value01 = buffer.load_complex(0);
1116 let value23 = buffer.load_complex(2);
1117 let value45 = buffer.load_complex(4);
1118
1119 let out = self.perform_fft_direct(value01, value23, value45);
1120
1121 buffer.store_complex(out[0], 0);
1122 buffer.store_complex(out[1], 2);
1123 buffer.store_complex(out[2], 4);
1124 }
1125
1126 #[inline(always)]
1127 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1128 &self,
1129 mut buffer: impl NeonArrayMut<f32>,
1130 ) {
1131 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10});
1132
1133 let values = interleave_complex_f32!(input_packed, 3, {0, 1, 2});
1134
1135 let out = self.perform_parallel_fft_direct(
1136 values[0], values[1], values[2], values[3], values[4], values[5],
1137 );
1138
1139 let out_sorted = separate_interleaved_complex_f32!(out, {0, 2, 4});
1140 write_complex_to_array_strided!(out_sorted, buffer, 2, {0, 1, 2, 3, 4, 5});
1141 }
1142
1143 #[inline(always)]
1144 pub(crate) unsafe fn perform_fft_direct(
1145 &self,
1146 value01: float32x4_t,
1147 value23: float32x4_t,
1148 value45: float32x4_t,
1149 ) -> [float32x4_t; 3] {
1150 // Algorithm: 3x2 good-thomas
1151
1152 // Size-3 FFTs down the columns of our reordered array
1153 let reord0 = extract_lo_hi_f32(value01, value23);
1154 let reord1 = extract_lo_hi_f32(value23, value45);
1155 let reord2 = extract_lo_hi_f32(value45, value01);
1156
1157 let mid = self.bf3.perform_parallel_fft_direct(reord0, reord1, reord2);
1158
1159 // We normally would put twiddle factors right here, but since this is good-thomas algorithm, we don't need twiddle factors
1160
1161 // Transpose the data and do size-2 FFTs down the columns
1162 let [output0, output1] = parallel_fft2_contiguous_f32(mid[0], mid[1]);
1163 let output2 = solo_fft2_f32(mid[2]);
1164
1165 // Reorder into output
1166 [
1167 extract_lo_hi_f32(output0, output1),
1168 extract_lo_lo_f32(output2, output1),
1169 extract_hi_hi_f32(output0, output2),
1170 ]
1171 }
1172
1173 #[inline(always)]
1174 pub(crate) unsafe fn perform_parallel_fft_direct(
1175 &self,
1176 value0: float32x4_t,
1177 value1: float32x4_t,
1178 value2: float32x4_t,
1179 value3: float32x4_t,
1180 value4: float32x4_t,
1181 value5: float32x4_t,
1182 ) -> [float32x4_t; 6] {
1183 // Algorithm: 3x2 good-thomas
1184
1185 // Size-3 FFTs down the columns of our reordered array
1186 let mid0 = self.bf3.perform_parallel_fft_direct(value0, value2, value4);
1187 let mid1 = self.bf3.perform_parallel_fft_direct(value3, value5, value1);
1188
1189 // We normally would put twiddle factors right here, but since this is good-thomas algorithm, we don't need twiddle factors
1190
1191 // Transpose the data and do size-2 FFTs down the columns
1192 let [output0, output1] = parallel_fft2_interleaved_f32(mid0[0], mid1[0]);
1193 let [output2, output3] = parallel_fft2_interleaved_f32(mid0[1], mid1[1]);
1194 let [output4, output5] = parallel_fft2_interleaved_f32(mid0[2], mid1[2]);
1195
1196 // Reorder into output
1197 [output0, output3, output4, output1, output2, output5]
1198 }
1199 }
1200
1201 // __ __ _ _ _ _ _
1202 // / /_ / /_ | || | | |__ (_) |_
1203 // | '_ \ _____ | '_ \| || |_| '_ \| | __|
1204 // | (_) | |_____| | (_) |__ _| |_) | | |_
1205 // \___/ \___/ |_| |_.__/|_|\__|
1206 //
1207
1208 pub struct NeonF64Butterfly6<T> {
1209 direction: FftDirection,
1210 _phantom: std::marker::PhantomData<T>,
1211 bf3: NeonF64Butterfly3<T>,
1212 }
1213
1214 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly6, 6, |this: &NeonF64Butterfly6<_>| this
1215 .direction);
1216 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly6, 6, |this: &NeonF64Butterfly6<_>| this
1217 .direction);
1218 impl<T: FftNum> NeonF64Butterfly6<T> {
1219 #[inline(always)]
1220 pub fn new(direction: FftDirection) -> Self {
1221 assert_f64::<T>();
1222 let bf3 = NeonF64Butterfly3::new(direction);
1223
1224 Self {
1225 direction,
1226 _phantom: std::marker::PhantomData,
1227 bf3,
1228 }
1229 }
1230
1231 #[inline(always)]
1232 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
1233 let value0 = buffer.load_complex(0);
1234 let value1 = buffer.load_complex(1);
1235 let value2 = buffer.load_complex(2);
1236 let value3 = buffer.load_complex(3);
1237 let value4 = buffer.load_complex(4);
1238 let value5 = buffer.load_complex(5);
1239
1240 let out = self.perform_fft_direct(value0, value1, value2, value3, value4, value5);
1241
1242 buffer.store_complex(out[0], 0);
1243 buffer.store_complex(out[1], 1);
1244 buffer.store_complex(out[2], 2);
1245 buffer.store_complex(out[3], 3);
1246 buffer.store_complex(out[4], 4);
1247 buffer.store_complex(out[5], 5);
1248 }
1249
1250 #[inline(always)]
1251 pub(crate) unsafe fn perform_fft_direct(
1252 &self,
1253 value0: float64x2_t,
1254 value1: float64x2_t,
1255 value2: float64x2_t,
1256 value3: float64x2_t,
1257 value4: float64x2_t,
1258 value5: float64x2_t,
1259 ) -> [float64x2_t; 6] {
1260 // Algorithm: 3x2 good-thomas
1261
1262 // Size-3 FFTs down the columns of our reordered array
1263 let mid0 = self.bf3.perform_fft_direct(value0, value2, value4);
1264 let mid1 = self.bf3.perform_fft_direct(value3, value5, value1);
1265
1266 // We normally would put twiddle factors right here, but since this is good-thomas algorithm, we don't need twiddle factors
1267
1268 // Transpose the data and do size-2 FFTs down the columns
1269 let [output0, output1] = solo_fft2_f64(mid0[0], mid1[0]);
1270 let [output2, output3] = solo_fft2_f64(mid0[1], mid1[1]);
1271 let [output4, output5] = solo_fft2_f64(mid0[2], mid1[2]);
1272
1273 // Reorder into output
1274 [output0, output3, output4, output1, output2, output5]
1275 }
1276 }
1277
1278 // ___ _________ _ _ _
1279 // ( _ ) |___ /___ \| |__ (_) |_
1280 // / _ \ _____ |_ \ __) | '_ \| | __|
1281 // | (_) | |_____| ___) / __/| |_) | | |_
1282 // \___/ |____/_____|_.__/|_|\__|
1283 //
1284
1285 pub struct NeonF32Butterfly8<T> {
1286 root2: float32x4_t,
1287 root2_dual: float32x4_t,
1288 direction: FftDirection,
1289 bf4: NeonF32Butterfly4<T>,
1290 rotate90: Rotate90F32,
1291 }
1292
1293 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly8, 8, |this: &NeonF32Butterfly8<_>| this
1294 .direction);
1295 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly8, 8, |this: &NeonF32Butterfly8<_>| this
1296 .direction);
1297 impl<T: FftNum> NeonF32Butterfly8<T> {
1298 #[inline(always)]
1299 pub fn new(direction: FftDirection) -> Self {
1300 assert_f32::<T>();
1301 let bf4 = NeonF32Butterfly4::new(direction);
1302 let root2 =
1303 unsafe { vld1q_f32([1.0, 1.0, 0.5f32.sqrt(), 0.5f32.sqrt(), 1.0, 1.0].as_ptr()) };
1304 let root2_dual = unsafe { vmovq_n_f32(0.5f32.sqrt()) };
1305 let rotate90 = if direction == FftDirection::Inverse {
1306 Rotate90F32::new(true)
1307 } else {
1308 Rotate90F32::new(false)
1309 };
1310 Self {
1311 root2,
1312 root2_dual,
1313 direction,
1314 bf4,
1315 rotate90,
1316 }
1317 }
1318
1319 #[inline(always)]
1320 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
1321 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6});
1322
1323 let out = self.perform_fft_direct(input_packed);
1324
1325 write_complex_to_array_strided!(out, buffer, 2, {0,1,2,3});
1326 }
1327
1328 #[inline(always)]
1329 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1330 &self,
1331 mut buffer: impl NeonArrayMut<f32>,
1332 ) {
1333 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14});
1334
1335 let values = interleave_complex_f32!(input_packed, 4, {0, 1, 2, 3});
1336
1337 let out = self.perform_parallel_fft_direct(values);
1338
1339 let out_sorted = separate_interleaved_complex_f32!(out, {0, 2, 4, 6});
1340
1341 write_complex_to_array_strided!(out_sorted, buffer, 2, {0,1,2,3,4,5,6,7});
1342 }
1343
1344 #[inline(always)]
1345 unsafe fn perform_fft_direct(&self, values: [float32x4_t; 4]) -> [float32x4_t; 4] {
1346 // we're going to hardcode a step of mixed radix
1347 // step 1: copy and reorder the input into the scratch
1348 let [in02, in13] = transpose_complex_2x2_f32(values[0], values[1]);
1349 let [in46, in57] = transpose_complex_2x2_f32(values[2], values[3]);
1350
1351 // step 2: column FFTs
1352 let val0 = self.bf4.perform_fft_direct(in02, in46);
1353 let mut val2 = self.bf4.perform_fft_direct(in13, in57);
1354
1355 // step 3: apply twiddle factors
1356 let val2b = self.rotate90.rotate_hi(val2[0]);
1357 let val2c = vaddq_f32(val2b, val2[0]);
1358 let val2d = vmulq_f32(val2c, self.root2);
1359 val2[0] = extract_lo_hi_f32(val2[0], val2d);
1360
1361 let val3b = self.rotate90.rotate_both(val2[1]);
1362 let val3c = vsubq_f32(val3b, val2[1]);
1363 let val3d = vmulq_f32(val3c, self.root2);
1364 val2[1] = extract_lo_hi_f32(val3b, val3d);
1365
1366 // step 4: transpose -- skipped because we're going to do the next FFTs non-contiguously
1367
1368 // step 5: row FFTs
1369 let out0 = parallel_fft2_interleaved_f32(val0[0], val2[0]);
1370 let out1 = parallel_fft2_interleaved_f32(val0[1], val2[1]);
1371
1372 // step 6: rearrange and copy to buffer
1373 [out0[0], out1[0], out0[1], out1[1]]
1374 }
1375
1376 #[inline(always)]
1377 unsafe fn perform_parallel_fft_direct(&self, values: [float32x4_t; 8]) -> [float32x4_t; 8] {
1378 // we're going to hardcode a step of mixed radix
1379 // step 1: copy and reorder the input into the scratch
1380 // and
1381 // step 2: column FFTs
1382 let val03 = self
1383 .bf4
1384 .perform_parallel_fft_direct(values[0], values[2], values[4], values[6]);
1385 let mut val47 = self
1386 .bf4
1387 .perform_parallel_fft_direct(values[1], values[3], values[5], values[7]);
1388
1389 // step 3: apply twiddle factors
1390 let val5b = self.rotate90.rotate_both(val47[1]);
1391 let val5c = vaddq_f32(val5b, val47[1]);
1392 val47[1] = vmulq_f32(val5c, self.root2_dual);
1393 val47[2] = self.rotate90.rotate_both(val47[2]);
1394 let val7b = self.rotate90.rotate_both(val47[3]);
1395 let val7c = vsubq_f32(val7b, val47[3]);
1396 val47[3] = vmulq_f32(val7c, self.root2_dual);
1397
1398 // step 4: transpose -- skipped because we're going to do the next FFTs non-contiguously
1399
1400 // step 5: row FFTs
1401 let out0 = parallel_fft2_interleaved_f32(val03[0], val47[0]);
1402 let out1 = parallel_fft2_interleaved_f32(val03[1], val47[1]);
1403 let out2 = parallel_fft2_interleaved_f32(val03[2], val47[2]);
1404 let out3 = parallel_fft2_interleaved_f32(val03[3], val47[3]);
1405
1406 // step 6: rearrange and copy to buffer
1407 [
1408 out0[0], out1[0], out2[0], out3[0], out0[1], out1[1], out2[1], out3[1],
1409 ]
1410 }
1411 }
1412
1413 // ___ __ _ _ _ _ _
1414 // ( _ ) / /_ | || | | |__ (_) |_
1415 // / _ \ _____ | '_ \| || |_| '_ \| | __|
1416 // | (_) | |_____| | (_) |__ _| |_) | | |_
1417 // \___/ \___/ |_| |_.__/|_|\__|
1418 //
1419
1420 pub struct NeonF64Butterfly8<T> {
1421 root2: float64x2_t,
1422 direction: FftDirection,
1423 bf4: NeonF64Butterfly4<T>,
1424 rotate90: Rotate90F64,
1425 }
1426
1427 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly8, 8, |this: &NeonF64Butterfly8<_>| this
1428 .direction);
1429 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly8, 8, |this: &NeonF64Butterfly8<_>| this
1430 .direction);
1431 impl<T: FftNum> NeonF64Butterfly8<T> {
1432 #[inline(always)]
1433 pub fn new(direction: FftDirection) -> Self {
1434 assert_f64::<T>();
1435 let bf4 = NeonF64Butterfly4::new(direction);
1436 let root2 = unsafe { vmovq_n_f64(0.5f64.sqrt()) };
1437 let rotate90 = if direction == FftDirection::Inverse {
1438 Rotate90F64::new(true)
1439 } else {
1440 Rotate90F64::new(false)
1441 };
1442 Self {
1443 root2,
1444 direction,
1445 bf4,
1446 rotate90,
1447 }
1448 }
1449
1450 #[inline(always)]
1451 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
1452 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7});
1453
1454 let out = self.perform_fft_direct(values);
1455
1456 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7});
1457 }
1458
1459 #[inline(always)]
1460 unsafe fn perform_fft_direct(&self, values: [float64x2_t; 8]) -> [float64x2_t; 8] {
1461 // we're going to hardcode a step of mixed radix
1462 // step 1: copy and reorder the input into the scratch
1463 // and
1464 // step 2: column FFTs
1465 let val03 = self
1466 .bf4
1467 .perform_fft_direct(values[0], values[2], values[4], values[6]);
1468 let mut val47 = self
1469 .bf4
1470 .perform_fft_direct(values[1], values[3], values[5], values[7]);
1471
1472 // step 3: apply twiddle factors
1473 let val5b = self.rotate90.rotate(val47[1]);
1474 let val5c = vaddq_f64(val5b, val47[1]);
1475 val47[1] = vmulq_f64(val5c, self.root2);
1476 val47[2] = self.rotate90.rotate(val47[2]);
1477 let val7b = self.rotate90.rotate(val47[3]);
1478 let val7c = vsubq_f64(val7b, val47[3]);
1479 val47[3] = vmulq_f64(val7c, self.root2);
1480
1481 // step 4: transpose -- skipped because we're going to do the next FFTs non-contiguously
1482
1483 // step 5: row FFTs
1484 let out0 = solo_fft2_f64(val03[0], val47[0]);
1485 let out1 = solo_fft2_f64(val03[1], val47[1]);
1486 let out2 = solo_fft2_f64(val03[2], val47[2]);
1487 let out3 = solo_fft2_f64(val03[3], val47[3]);
1488
1489 // step 6: rearrange and copy to buffer
1490 [
1491 out0[0], out1[0], out2[0], out3[0], out0[1], out1[1], out2[1], out3[1],
1492 ]
1493 }
1494 }
1495
1496 // ___ _________ _ _ _
1497 // / _ \ |___ /___ \| |__ (_) |_
1498 // | (_) | _____ |_ \ __) | '_ \| | __|
1499 // \__, | |_____| ___) / __/| |_) | | |_
1500 // /_/ |____/_____|_.__/|_|\__|
1501 //
1502 pub struct NeonF32Butterfly9<T> {
1503 direction: FftDirection,
1504 _phantom: std::marker::PhantomData<T>,
1505 bf3: NeonF32Butterfly3<T>,
1506 twiddle1: float32x4_t,
1507 twiddle2: float32x4_t,
1508 twiddle4: float32x4_t,
1509 }
1510
1511 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly9, 9, |this: &NeonF32Butterfly9<_>| this
1512 .direction);
1513 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly9, 9, |this: &NeonF32Butterfly9<_>| this
1514 .direction);
1515 impl<T: FftNum> NeonF32Butterfly9<T> {
1516 #[inline(always)]
1517 pub fn new(direction: FftDirection) -> Self {
1518 assert_f32::<T>();
1519 let bf3 = NeonF32Butterfly3::new(direction);
1520 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 9, direction);
1521 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 9, direction);
1522 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 9, direction);
1523 let twiddle1 = unsafe { vld1q_f32([tw1.re, tw1.im, tw1.re, tw1.im].as_ptr()) };
1524 let twiddle2 = unsafe { vld1q_f32([tw2.re, tw2.im, tw2.re, tw2.im].as_ptr()) };
1525 let twiddle4 = unsafe { vld1q_f32([tw4.re, tw4.im, tw4.re, tw4.im].as_ptr()) };
1526
1527 Self {
1528 direction,
1529 _phantom: std::marker::PhantomData,
1530 bf3,
1531 twiddle1,
1532 twiddle2,
1533 twiddle4,
1534 }
1535 }
1536
1537 #[inline(always)]
1538 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
1539 // A single Neon 9-point will need a lot of shuffling, let's just reuse the dual one
1540 let values = read_partial1_complex_to_array!(buffer, {0,1,2,3,4,5,6,7,8});
1541
1542 let out = self.perform_parallel_fft_direct(values);
1543
1544 for n in 0..9 {
1545 buffer.store_partial_lo_complex(out[n], n);
1546 }
1547 }
1548
1549 #[inline(always)]
1550 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1551 &self,
1552 mut buffer: impl NeonArrayMut<f32>,
1553 ) {
1554 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16});
1555
1556 let values = [
1557 extract_lo_hi_f32(input_packed[0], input_packed[4]),
1558 extract_hi_lo_f32(input_packed[0], input_packed[5]),
1559 extract_lo_hi_f32(input_packed[1], input_packed[5]),
1560 extract_hi_lo_f32(input_packed[1], input_packed[6]),
1561 extract_lo_hi_f32(input_packed[2], input_packed[6]),
1562 extract_hi_lo_f32(input_packed[2], input_packed[7]),
1563 extract_lo_hi_f32(input_packed[3], input_packed[7]),
1564 extract_hi_lo_f32(input_packed[3], input_packed[8]),
1565 extract_lo_hi_f32(input_packed[4], input_packed[8]),
1566 ];
1567
1568 let out = self.perform_parallel_fft_direct(values);
1569
1570 let out_packed = [
1571 extract_lo_lo_f32(out[0], out[1]),
1572 extract_lo_lo_f32(out[2], out[3]),
1573 extract_lo_lo_f32(out[4], out[5]),
1574 extract_lo_lo_f32(out[6], out[7]),
1575 extract_lo_hi_f32(out[8], out[0]),
1576 extract_hi_hi_f32(out[1], out[2]),
1577 extract_hi_hi_f32(out[3], out[4]),
1578 extract_hi_hi_f32(out[5], out[6]),
1579 extract_hi_hi_f32(out[7], out[8]),
1580 ];
1581
1582 write_complex_to_array_strided!(out_packed, buffer, 2, {0,1,2,3,4,5,6,7,8});
1583 }
1584
1585 #[inline(always)]
1586 pub(crate) unsafe fn perform_parallel_fft_direct(
1587 &self,
1588 values: [float32x4_t; 9],
1589 ) -> [float32x4_t; 9] {
1590 // Algorithm: 3x3 mixed radix
1591
1592 // Size-3 FFTs down the columns
1593 let mid0 = self
1594 .bf3
1595 .perform_parallel_fft_direct(values[0], values[3], values[6]);
1596 let mut mid1 = self
1597 .bf3
1598 .perform_parallel_fft_direct(values[1], values[4], values[7]);
1599 let mut mid2 = self
1600 .bf3
1601 .perform_parallel_fft_direct(values[2], values[5], values[8]);
1602
1603 // Apply twiddle factors. Note that we're re-using twiddle2
1604 mid1[1] = mul_complex_f32(self.twiddle1, mid1[1]);
1605 mid1[2] = mul_complex_f32(self.twiddle2, mid1[2]);
1606 mid2[1] = mul_complex_f32(self.twiddle2, mid2[1]);
1607 mid2[2] = mul_complex_f32(self.twiddle4, mid2[2]);
1608
1609 let [output0, output1, output2] = self
1610 .bf3
1611 .perform_parallel_fft_direct(mid0[0], mid1[0], mid2[0]);
1612 let [output3, output4, output5] = self
1613 .bf3
1614 .perform_parallel_fft_direct(mid0[1], mid1[1], mid2[1]);
1615 let [output6, output7, output8] = self
1616 .bf3
1617 .perform_parallel_fft_direct(mid0[2], mid1[2], mid2[2]);
1618
1619 [
1620 output0, output3, output6, output1, output4, output7, output2, output5, output8,
1621 ]
1622 }
1623 }
1624
1625 // ___ __ _ _ _ _ _
1626 // / _ \ / /_ | || | | |__ (_) |_
1627 // | (_) | _____ | '_ \| || |_| '_ \| | __|
1628 // \__, | |_____| | (_) |__ _| |_) | | |_
1629 // /_/ \___/ |_| |_.__/|_|\__|
1630 //
1631
1632 pub struct NeonF64Butterfly9<T> {
1633 direction: FftDirection,
1634 _phantom: std::marker::PhantomData<T>,
1635 bf3: NeonF64Butterfly3<T>,
1636 twiddle1: float64x2_t,
1637 twiddle2: float64x2_t,
1638 twiddle4: float64x2_t,
1639 }
1640
1641 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly9, 9, |this: &NeonF64Butterfly9<_>| this
1642 .direction);
1643 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly9, 9, |this: &NeonF64Butterfly9<_>| this
1644 .direction);
1645 impl<T: FftNum> NeonF64Butterfly9<T> {
1646 #[inline(always)]
1647 pub fn new(direction: FftDirection) -> Self {
1648 assert_f64::<T>();
1649 let bf3 = NeonF64Butterfly3::new(direction);
1650 let tw1: Complex<f64> = twiddles::compute_twiddle(1, 9, direction);
1651 let tw2: Complex<f64> = twiddles::compute_twiddle(2, 9, direction);
1652 let tw4: Complex<f64> = twiddles::compute_twiddle(4, 9, direction);
1653 let twiddle1 = unsafe { vld1q_f64([tw1.re, tw1.im].as_ptr()) };
1654 let twiddle2 = unsafe { vld1q_f64([tw2.re, tw2.im].as_ptr()) };
1655 let twiddle4 = unsafe { vld1q_f64([tw4.re, tw4.im].as_ptr()) };
1656
1657 Self {
1658 direction,
1659 _phantom: std::marker::PhantomData,
1660 bf3,
1661 twiddle1,
1662 twiddle2,
1663 twiddle4,
1664 }
1665 }
1666
1667 #[inline(always)]
1668 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
1669 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8});
1670
1671 let out = self.perform_fft_direct(values);
1672
1673 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8});
1674 }
1675
1676 #[inline(always)]
1677 pub(crate) unsafe fn perform_fft_direct(&self, values: [float64x2_t; 9]) -> [float64x2_t; 9] {
1678 // Algorithm: 3x3 mixed radix
1679
1680 // Size-3 FFTs down the columns
1681 let mid0 = self.bf3.perform_fft_direct(values[0], values[3], values[6]);
1682 let mut mid1 = self.bf3.perform_fft_direct(values[1], values[4], values[7]);
1683 let mut mid2 = self.bf3.perform_fft_direct(values[2], values[5], values[8]);
1684
1685 // Apply twiddle factors. Note that we're re-using twiddle2
1686 mid1[1] = mul_complex_f64(self.twiddle1, mid1[1]);
1687 mid1[2] = mul_complex_f64(self.twiddle2, mid1[2]);
1688 mid2[1] = mul_complex_f64(self.twiddle2, mid2[1]);
1689 mid2[2] = mul_complex_f64(self.twiddle4, mid2[2]);
1690
1691 let [output0, output1, output2] = self.bf3.perform_fft_direct(mid0[0], mid1[0], mid2[0]);
1692 let [output3, output4, output5] = self.bf3.perform_fft_direct(mid0[1], mid1[1], mid2[1]);
1693 let [output6, output7, output8] = self.bf3.perform_fft_direct(mid0[2], mid1[2], mid2[2]);
1694
1695 [
1696 output0, output3, output6, output1, output4, output7, output2, output5, output8,
1697 ]
1698 }
1699 }
1700
1701 // _ ___ _________ _ _ _
1702 // / |/ _ \ |___ /___ \| |__ (_) |_
1703 // | | | | | _____ |_ \ __) | '_ \| | __|
1704 // | | |_| | |_____| ___) / __/| |_) | | |_
1705 // |_|\___/ |____/_____|_.__/|_|\__|
1706 //
1707
1708 pub struct NeonF32Butterfly10<T> {
1709 direction: FftDirection,
1710 _phantom: std::marker::PhantomData<T>,
1711 bf5: NeonF32Butterfly5<T>,
1712 }
1713
1714 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly10, 10, |this: &NeonF32Butterfly10<_>| this
1715 .direction);
1716 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly10, 10, |this: &NeonF32Butterfly10<_>| this
1717 .direction);
1718 impl<T: FftNum> NeonF32Butterfly10<T> {
1719 #[inline(always)]
1720 pub fn new(direction: FftDirection) -> Self {
1721 assert_f32::<T>();
1722 let bf5 = NeonF32Butterfly5::new(direction);
1723 Self {
1724 direction,
1725 _phantom: std::marker::PhantomData,
1726 bf5,
1727 }
1728 }
1729
1730 #[inline(always)]
1731 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
1732 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8});
1733
1734 let out = self.perform_fft_direct(input_packed);
1735
1736 write_complex_to_array_strided!(out, buffer, 2, {0,1,2,3,4});
1737 }
1738
1739 #[inline(always)]
1740 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1741 &self,
1742 mut buffer: impl NeonArrayMut<f32>,
1743 ) {
1744 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18});
1745
1746 let values = interleave_complex_f32!(input_packed, 5, {0, 1, 2, 3, 4});
1747
1748 let out = self.perform_parallel_fft_direct(values);
1749
1750 let out_sorted = separate_interleaved_complex_f32!(out, {0, 2, 4, 6, 8});
1751
1752 write_complex_to_array_strided!(out_sorted, buffer, 2, {0,1,2,3,4,5,6,7,8,9});
1753 }
1754
1755 #[inline(always)]
1756 pub(crate) unsafe fn perform_fft_direct(&self, values: [float32x4_t; 5]) -> [float32x4_t; 5] {
1757 // Algorithm: 5x2 good-thomas
1758 // Reorder and pack
1759 let reord0 = extract_lo_hi_f32(values[0], values[2]);
1760 let reord1 = extract_lo_hi_f32(values[1], values[3]);
1761 let reord2 = extract_lo_hi_f32(values[2], values[4]);
1762 let reord3 = extract_lo_hi_f32(values[3], values[0]);
1763 let reord4 = extract_lo_hi_f32(values[4], values[1]);
1764
1765 // Size-5 FFTs down the columns of our reordered array
1766 let mids = self
1767 .bf5
1768 .perform_parallel_fft_direct(reord0, reord1, reord2, reord3, reord4);
1769
1770 // Since this is good-thomas algorithm, we don't need twiddle factors
1771
1772 // Transpose the data and do size-2 FFTs down the columns
1773 let [temp01, temp23] = parallel_fft2_contiguous_f32(mids[0], mids[1]);
1774 let [temp45, temp67] = parallel_fft2_contiguous_f32(mids[2], mids[3]);
1775 let temp89 = solo_fft2_f32(mids[4]);
1776
1777 // Reorder
1778 let out01 = extract_lo_hi_f32(temp01, temp23);
1779 let out23 = extract_lo_hi_f32(temp45, temp67);
1780 let out45 = extract_lo_lo_f32(temp89, temp23);
1781 let out67 = extract_hi_lo_f32(temp01, temp67);
1782 let out89 = extract_hi_hi_f32(temp45, temp89);
1783
1784 [out01, out23, out45, out67, out89]
1785 }
1786
1787 #[inline(always)]
1788 pub(crate) unsafe fn perform_parallel_fft_direct(
1789 &self,
1790 values: [float32x4_t; 10],
1791 ) -> [float32x4_t; 10] {
1792 // Algorithm: 5x2 good-thomas
1793
1794 // Size-5 FFTs down the columns of our reordered array
1795 let mid0 = self
1796 .bf5
1797 .perform_parallel_fft_direct(values[0], values[2], values[4], values[6], values[8]);
1798 let mid1 = self
1799 .bf5
1800 .perform_parallel_fft_direct(values[5], values[7], values[9], values[1], values[3]);
1801
1802 // Since this is good-thomas algorithm, we don't need twiddle factors
1803
1804 // Transpose the data and do size-2 FFTs down the columns
1805 let [output0, output1] = parallel_fft2_interleaved_f32(mid0[0], mid1[0]);
1806 let [output2, output3] = parallel_fft2_interleaved_f32(mid0[1], mid1[1]);
1807 let [output4, output5] = parallel_fft2_interleaved_f32(mid0[2], mid1[2]);
1808 let [output6, output7] = parallel_fft2_interleaved_f32(mid0[3], mid1[3]);
1809 let [output8, output9] = parallel_fft2_interleaved_f32(mid0[4], mid1[4]);
1810
1811 // Reorder and return
1812 [
1813 output0, output3, output4, output7, output8, output1, output2, output5, output6,
1814 output9,
1815 ]
1816 }
1817 }
1818
1819 // _ ___ __ _ _ _ _ _
1820 // / |/ _ \ / /_ | || | | |__ (_) |_
1821 // | | | | | _____ | '_ \| || |_| '_ \| | __|
1822 // | | |_| | |_____| | (_) |__ _| |_) | | |_
1823 // |_|\___/ \___/ |_| |_.__/|_|\__|
1824 //
1825
1826 pub struct NeonF64Butterfly10<T> {
1827 direction: FftDirection,
1828 _phantom: std::marker::PhantomData<T>,
1829 bf2: NeonF64Butterfly2<T>,
1830 bf5: NeonF64Butterfly5<T>,
1831 }
1832
1833 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly10, 10, |this: &NeonF64Butterfly10<_>| this
1834 .direction);
1835 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly10, 10, |this: &NeonF64Butterfly10<_>| this
1836 .direction);
1837 impl<T: FftNum> NeonF64Butterfly10<T> {
1838 #[inline(always)]
1839 pub fn new(direction: FftDirection) -> Self {
1840 assert_f64::<T>();
1841 let bf2 = NeonF64Butterfly2::new(direction);
1842 let bf5 = NeonF64Butterfly5::new(direction);
1843 Self {
1844 direction,
1845 _phantom: std::marker::PhantomData,
1846 bf2,
1847 bf5,
1848 }
1849 }
1850
1851 #[inline(always)]
1852 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
1853 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
1854
1855 let out = self.perform_fft_direct(values);
1856
1857 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
1858 }
1859
1860 #[inline(always)]
1861 pub(crate) unsafe fn perform_fft_direct(&self, values: [float64x2_t; 10]) -> [float64x2_t; 10] {
1862 // Algorithm: 5x2 good-thomas
1863
1864 // Size-5 FFTs down the columns of our reordered array
1865 let mid0 = self
1866 .bf5
1867 .perform_fft_direct(values[0], values[2], values[4], values[6], values[8]);
1868 let mid1 = self
1869 .bf5
1870 .perform_fft_direct(values[5], values[7], values[9], values[1], values[3]);
1871
1872 // Since this is good-thomas algorithm, we don't need twiddle factors
1873
1874 // Transpose the data and do size-2 FFTs down the columns
1875 let [output0, output1] = self.bf2.perform_fft_direct(mid0[0], mid1[0]);
1876 let [output2, output3] = self.bf2.perform_fft_direct(mid0[1], mid1[1]);
1877 let [output4, output5] = self.bf2.perform_fft_direct(mid0[2], mid1[2]);
1878 let [output6, output7] = self.bf2.perform_fft_direct(mid0[3], mid1[3]);
1879 let [output8, output9] = self.bf2.perform_fft_direct(mid0[4], mid1[4]);
1880
1881 // Reorder and return
1882 [
1883 output0, output3, output4, output7, output8, output1, output2, output5, output6,
1884 output9,
1885 ]
1886 }
1887 }
1888
1889 // _ ____ _________ _ _ _
1890 // / |___ \ |___ /___ \| |__ (_) |_
1891 // | | __) | _____ |_ \ __) | '_ \| | __|
1892 // | |/ __/ |_____| ___) / __/| |_) | | |_
1893 // |_|_____| |____/_____|_.__/|_|\__|
1894 //
1895
1896 pub struct NeonF32Butterfly12<T> {
1897 direction: FftDirection,
1898 _phantom: std::marker::PhantomData<T>,
1899 bf3: NeonF32Butterfly3<T>,
1900 bf4: NeonF32Butterfly4<T>,
1901 }
1902
1903 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly12, 12, |this: &NeonF32Butterfly12<_>| this
1904 .direction);
1905 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly12, 12, |this: &NeonF32Butterfly12<_>| this
1906 .direction);
1907 impl<T: FftNum> NeonF32Butterfly12<T> {
1908 #[inline(always)]
1909 pub fn new(direction: FftDirection) -> Self {
1910 assert_f32::<T>();
1911 let bf3 = NeonF32Butterfly3::new(direction);
1912 let bf4 = NeonF32Butterfly4::new(direction);
1913 Self {
1914 direction,
1915 _phantom: std::marker::PhantomData,
1916 bf3,
1917 bf4,
1918 }
1919 }
1920
1921 #[inline(always)]
1922 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
1923 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10 });
1924
1925 let out = self.perform_fft_direct(input_packed);
1926
1927 write_complex_to_array_strided!(out, buffer, 2, {0,1,2,3,4,5});
1928 }
1929
1930 #[inline(always)]
1931 pub(crate) unsafe fn perform_parallel_fft_contiguous(
1932 &self,
1933 mut buffer: impl NeonArrayMut<f32>,
1934 ) {
1935 let input_packed =
1936 read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22});
1937
1938 let values = interleave_complex_f32!(input_packed, 6, {0, 1, 2, 3, 4, 5});
1939
1940 let out = self.perform_parallel_fft_direct(values);
1941
1942 let out_sorted = separate_interleaved_complex_f32!(out, {0, 2, 4, 6, 8, 10});
1943
1944 write_complex_to_array_strided!(out_sorted, buffer, 2, {0,1,2,3,4,5,6,7,8,9, 10, 11});
1945 }
1946
1947 #[inline(always)]
1948 pub(crate) unsafe fn perform_fft_direct(&self, values: [float32x4_t; 6]) -> [float32x4_t; 6] {
1949 // Algorithm: 4x3 good-thomas
1950
1951 // Reorder and pack
1952 let packed03 = extract_lo_hi_f32(values[0], values[1]);
1953 let packed47 = extract_lo_hi_f32(values[2], values[3]);
1954 let packed69 = extract_lo_hi_f32(values[3], values[4]);
1955 let packed101 = extract_lo_hi_f32(values[5], values[0]);
1956 let packed811 = extract_lo_hi_f32(values[4], values[5]);
1957 let packed25 = extract_lo_hi_f32(values[1], values[2]);
1958
1959 // Size-4 FFTs down the columns of our reordered array
1960 let mid0 = self.bf4.perform_fft_direct(packed03, packed69);
1961 let mid1 = self.bf4.perform_fft_direct(packed47, packed101);
1962 let mid2 = self.bf4.perform_fft_direct(packed811, packed25);
1963
1964 // Since this is good-thomas algorithm, we don't need twiddle factors
1965
1966 // Transpose the data and do size-3 FFTs down the columns
1967 let [temp03, temp14, temp25] = self
1968 .bf3
1969 .perform_parallel_fft_direct(mid0[0], mid1[0], mid2[0]);
1970 let [temp69, temp710, temp811] = self
1971 .bf3
1972 .perform_parallel_fft_direct(mid0[1], mid1[1], mid2[1]);
1973
1974 // Reorder and return
1975 [
1976 extract_lo_hi_f32(temp03, temp14),
1977 extract_lo_hi_f32(temp811, temp69),
1978 extract_lo_hi_f32(temp14, temp25),
1979 extract_lo_hi_f32(temp69, temp710),
1980 extract_lo_hi_f32(temp25, temp03),
1981 extract_lo_hi_f32(temp710, temp811),
1982 ]
1983 }
1984
1985 #[inline(always)]
1986 pub(crate) unsafe fn perform_parallel_fft_direct(
1987 &self,
1988 values: [float32x4_t; 12],
1989 ) -> [float32x4_t; 12] {
1990 // Algorithm: 4x3 good-thomas
1991
1992 // Size-4 FFTs down the columns of our reordered array
1993 let mid0 = self
1994 .bf4
1995 .perform_parallel_fft_direct(values[0], values[3], values[6], values[9]);
1996 let mid1 = self
1997 .bf4
1998 .perform_parallel_fft_direct(values[4], values[7], values[10], values[1]);
1999 let mid2 = self
2000 .bf4
2001 .perform_parallel_fft_direct(values[8], values[11], values[2], values[5]);
2002
2003 // Since this is good-thomas algorithm, we don't need twiddle factors
2004
2005 // Transpose the data and do size-3 FFTs down the columns
2006 let [output0, output1, output2] = self
2007 .bf3
2008 .perform_parallel_fft_direct(mid0[0], mid1[0], mid2[0]);
2009 let [output3, output4, output5] = self
2010 .bf3
2011 .perform_parallel_fft_direct(mid0[1], mid1[1], mid2[1]);
2012 let [output6, output7, output8] = self
2013 .bf3
2014 .perform_parallel_fft_direct(mid0[2], mid1[2], mid2[2]);
2015 let [output9, output10, output11] = self
2016 .bf3
2017 .perform_parallel_fft_direct(mid0[3], mid1[3], mid2[3]);
2018
2019 // Reorder and return
2020 [
2021 output0, output4, output8, output9, output1, output5, output6, output10, output2,
2022 output3, output7, output11,
2023 ]
2024 }
2025 }
2026
2027 // _ ____ __ _ _ _ _ _
2028 // / |___ \ / /_ | || | | |__ (_) |_
2029 // | | __) | _____ | '_ \| || |_| '_ \| | __|
2030 // | |/ __/ |_____| | (_) |__ _| |_) | | |_
2031 // |_|_____| \___/ |_| |_.__/|_|\__|
2032 //
2033
2034 pub struct NeonF64Butterfly12<T> {
2035 direction: FftDirection,
2036 _phantom: std::marker::PhantomData<T>,
2037 bf3: NeonF64Butterfly3<T>,
2038 bf4: NeonF64Butterfly4<T>,
2039 }
2040
2041 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly12, 12, |this: &NeonF64Butterfly12<_>| this
2042 .direction);
2043 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly12, 12, |this: &NeonF64Butterfly12<_>| this
2044 .direction);
2045 impl<T: FftNum> NeonF64Butterfly12<T> {
2046 #[inline(always)]
2047 pub fn new(direction: FftDirection) -> Self {
2048 assert_f64::<T>();
2049 let bf3 = NeonF64Butterfly3::new(direction);
2050 let bf4 = NeonF64Butterfly4::new(direction);
2051 Self {
2052 direction,
2053 _phantom: std::marker::PhantomData,
2054 bf3,
2055 bf4,
2056 }
2057 }
2058
2059 #[inline(always)]
2060 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
2061 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11});
2062
2063 let out = self.perform_fft_direct(values);
2064
2065 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11});
2066 }
2067
2068 #[inline(always)]
2069 pub(crate) unsafe fn perform_fft_direct(&self, values: [float64x2_t; 12]) -> [float64x2_t; 12] {
2070 // Algorithm: 4x3 good-thomas
2071
2072 // Size-4 FFTs down the columns of our reordered array
2073 let mid0 = self
2074 .bf4
2075 .perform_fft_direct(values[0], values[3], values[6], values[9]);
2076 let mid1 = self
2077 .bf4
2078 .perform_fft_direct(values[4], values[7], values[10], values[1]);
2079 let mid2 = self
2080 .bf4
2081 .perform_fft_direct(values[8], values[11], values[2], values[5]);
2082
2083 // Since this is good-thomas algorithm, we don't need twiddle factors
2084
2085 // Transpose the data and do size-3 FFTs down the columns
2086 let [output0, output1, output2] = self.bf3.perform_fft_direct(mid0[0], mid1[0], mid2[0]);
2087 let [output3, output4, output5] = self.bf3.perform_fft_direct(mid0[1], mid1[1], mid2[1]);
2088 let [output6, output7, output8] = self.bf3.perform_fft_direct(mid0[2], mid1[2], mid2[2]);
2089 let [output9, output10, output11] = self.bf3.perform_fft_direct(mid0[3], mid1[3], mid2[3]);
2090
2091 [
2092 output0, output4, output8, output9, output1, output5, output6, output10, output2,
2093 output3, output7, output11,
2094 ]
2095 }
2096 }
2097
2098 // _ ____ _________ _ _ _
2099 // / | ___| |___ /___ \| |__ (_) |_
2100 // | |___ \ _____ |_ \ __) | '_ \| | __|
2101 // | |___) | |_____| ___) / __/| |_) | | |_
2102 // |_|____/ |____/_____|_.__/|_|\__|
2103 //
2104 pub struct NeonF32Butterfly15<T> {
2105 direction: FftDirection,
2106 _phantom: std::marker::PhantomData<T>,
2107 bf3: NeonF32Butterfly3<T>,
2108 bf5: NeonF32Butterfly5<T>,
2109 }
2110
2111 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly15, 15, |this: &NeonF32Butterfly15<_>| this
2112 .direction);
2113 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly15, 15, |this: &NeonF32Butterfly15<_>| this
2114 .direction);
2115 impl<T: FftNum> NeonF32Butterfly15<T> {
2116 #[inline(always)]
2117 pub fn new(direction: FftDirection) -> Self {
2118 assert_f32::<T>();
2119 let bf3 = NeonF32Butterfly3::new(direction);
2120 let bf5 = NeonF32Butterfly5::new(direction);
2121 Self {
2122 direction,
2123 _phantom: std::marker::PhantomData,
2124 bf3,
2125 bf5,
2126 }
2127 }
2128
2129 #[inline(always)]
2130 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
2131 // A single Neon 15-point will need a lot of shuffling, let's just reuse the dual one
2132 let values = read_partial1_complex_to_array!(buffer, {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14});
2133
2134 let out = self.perform_parallel_fft_direct(values);
2135
2136 for n in 0..15 {
2137 buffer.store_partial_lo_complex(out[n], n);
2138 }
2139 }
2140
2141 #[inline(always)]
2142 pub(crate) unsafe fn perform_parallel_fft_contiguous(
2143 &self,
2144 mut buffer: impl NeonArrayMut<f32>,
2145 ) {
2146 let input_packed =
2147 read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28});
2148
2149 let values = [
2150 extract_lo_hi_f32(input_packed[0], input_packed[7]),
2151 extract_hi_lo_f32(input_packed[0], input_packed[8]),
2152 extract_lo_hi_f32(input_packed[1], input_packed[8]),
2153 extract_hi_lo_f32(input_packed[1], input_packed[9]),
2154 extract_lo_hi_f32(input_packed[2], input_packed[9]),
2155 extract_hi_lo_f32(input_packed[2], input_packed[10]),
2156 extract_lo_hi_f32(input_packed[3], input_packed[10]),
2157 extract_hi_lo_f32(input_packed[3], input_packed[11]),
2158 extract_lo_hi_f32(input_packed[4], input_packed[11]),
2159 extract_hi_lo_f32(input_packed[4], input_packed[12]),
2160 extract_lo_hi_f32(input_packed[5], input_packed[12]),
2161 extract_hi_lo_f32(input_packed[5], input_packed[13]),
2162 extract_lo_hi_f32(input_packed[6], input_packed[13]),
2163 extract_hi_lo_f32(input_packed[6], input_packed[14]),
2164 extract_lo_hi_f32(input_packed[7], input_packed[14]),
2165 ];
2166
2167 let out = self.perform_parallel_fft_direct(values);
2168
2169 let out_packed = [
2170 extract_lo_lo_f32(out[0], out[1]),
2171 extract_lo_lo_f32(out[2], out[3]),
2172 extract_lo_lo_f32(out[4], out[5]),
2173 extract_lo_lo_f32(out[6], out[7]),
2174 extract_lo_lo_f32(out[8], out[9]),
2175 extract_lo_lo_f32(out[10], out[11]),
2176 extract_lo_lo_f32(out[12], out[13]),
2177 extract_lo_hi_f32(out[14], out[0]),
2178 extract_hi_hi_f32(out[1], out[2]),
2179 extract_hi_hi_f32(out[3], out[4]),
2180 extract_hi_hi_f32(out[5], out[6]),
2181 extract_hi_hi_f32(out[7], out[8]),
2182 extract_hi_hi_f32(out[9], out[10]),
2183 extract_hi_hi_f32(out[11], out[12]),
2184 extract_hi_hi_f32(out[13], out[14]),
2185 ];
2186
2187 write_complex_to_array_strided!(out_packed, buffer, 2, {0,1,2,3,4,5,6,7,8,9, 10, 11, 12, 13, 14});
2188 }
2189
2190 #[inline(always)]
2191 pub(crate) unsafe fn perform_parallel_fft_direct(
2192 &self,
2193 values: [float32x4_t; 15],
2194 ) -> [float32x4_t; 15] {
2195 // Algorithm: 5x3 good-thomas
2196
2197 // Size-5 FFTs down the columns of our reordered array
2198 let mid0 = self
2199 .bf5
2200 .perform_parallel_fft_direct(values[0], values[3], values[6], values[9], values[12]);
2201 let mid1 = self
2202 .bf5
2203 .perform_parallel_fft_direct(values[5], values[8], values[11], values[14], values[2]);
2204 let mid2 = self
2205 .bf5
2206 .perform_parallel_fft_direct(values[10], values[13], values[1], values[4], values[7]);
2207
2208 // Since this is good-thomas algorithm, we don't need twiddle factors
2209
2210 // Transpose the data and do size-3 FFTs down the columns
2211 let [output0, output1, output2] = self
2212 .bf3
2213 .perform_parallel_fft_direct(mid0[0], mid1[0], mid2[0]);
2214 let [output3, output4, output5] = self
2215 .bf3
2216 .perform_parallel_fft_direct(mid0[1], mid1[1], mid2[1]);
2217 let [output6, output7, output8] = self
2218 .bf3
2219 .perform_parallel_fft_direct(mid0[2], mid1[2], mid2[2]);
2220 let [output9, output10, output11] = self
2221 .bf3
2222 .perform_parallel_fft_direct(mid0[3], mid1[3], mid2[3]);
2223 let [output12, output13, output14] = self
2224 .bf3
2225 .perform_parallel_fft_direct(mid0[4], mid1[4], mid2[4]);
2226
2227 [
2228 output0, output4, output8, output9, output13, output2, output3, output7, output11,
2229 output12, output1, output5, output6, output10, output14,
2230 ]
2231 }
2232 }
2233
2234 // _ ____ __ _ _ _ _ _
2235 // / | ___| / /_ | || | | |__ (_) |_
2236 // | |___ \ _____ | '_ \| || |_| '_ \| | __|
2237 // | |___) | |_____| | (_) |__ _| |_) | | |_
2238 // |_|____/ \___/ |_| |_.__/|_|\__|
2239 //
2240
2241 pub struct NeonF64Butterfly15<T> {
2242 direction: FftDirection,
2243 _phantom: std::marker::PhantomData<T>,
2244 bf3: NeonF64Butterfly3<T>,
2245 bf5: NeonF64Butterfly5<T>,
2246 }
2247
2248 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly15, 15, |this: &NeonF64Butterfly15<_>| this
2249 .direction);
2250 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly15, 15, |this: &NeonF64Butterfly15<_>| this
2251 .direction);
2252 impl<T: FftNum> NeonF64Butterfly15<T> {
2253 #[inline(always)]
2254 pub fn new(direction: FftDirection) -> Self {
2255 assert_f64::<T>();
2256 let bf3 = NeonF64Butterfly3::new(direction);
2257 let bf5 = NeonF64Butterfly5::new(direction);
2258 Self {
2259 direction,
2260 _phantom: std::marker::PhantomData,
2261 bf3,
2262 bf5,
2263 }
2264 }
2265
2266 #[inline(always)]
2267 pub(crate) unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
2268 let values =
2269 read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14});
2270
2271 let out = self.perform_fft_direct(values);
2272
2273 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14});
2274 }
2275
2276 #[inline(always)]
2277 pub(crate) unsafe fn perform_fft_direct(&self, values: [float64x2_t; 15]) -> [float64x2_t; 15] {
2278 // Algorithm: 5x3 good-thomas
2279
2280 // Size-5 FFTs down the columns of our reordered array
2281 let mid0 = self
2282 .bf5
2283 .perform_fft_direct(values[0], values[3], values[6], values[9], values[12]);
2284 let mid1 = self
2285 .bf5
2286 .perform_fft_direct(values[5], values[8], values[11], values[14], values[2]);
2287 let mid2 = self
2288 .bf5
2289 .perform_fft_direct(values[10], values[13], values[1], values[4], values[7]);
2290
2291 // Since this is good-thomas algorithm, we don't need twiddle factors
2292
2293 // Transpose the data and do size-3 FFTs down the columns
2294 let [output0, output1, output2] = self.bf3.perform_fft_direct(mid0[0], mid1[0], mid2[0]);
2295 let [output3, output4, output5] = self.bf3.perform_fft_direct(mid0[1], mid1[1], mid2[1]);
2296 let [output6, output7, output8] = self.bf3.perform_fft_direct(mid0[2], mid1[2], mid2[2]);
2297 let [output9, output10, output11] = self.bf3.perform_fft_direct(mid0[3], mid1[3], mid2[3]);
2298 let [output12, output13, output14] = self.bf3.perform_fft_direct(mid0[4], mid1[4], mid2[4]);
2299
2300 [
2301 output0, output4, output8, output9, output13, output2, output3, output7, output11,
2302 output12, output1, output5, output6, output10, output14,
2303 ]
2304 }
2305 }
2306
2307 // _ __ _________ _ _ _
2308 // / |/ /_ |___ /___ \| |__ (_) |_
2309 // | | '_ \ _____ |_ \ __) | '_ \| | __|
2310 // | | (_) | |_____| ___) / __/| |_) | | |_
2311 // |_|\___/ |____/_____|_.__/|_|\__|
2312 //
2313
2314 pub struct NeonF32Butterfly16<T> {
2315 direction: FftDirection,
2316 bf4: NeonF32Butterfly4<T>,
2317 bf8: NeonF32Butterfly8<T>,
2318 rotate90: Rotate90F32,
2319 twiddle01: float32x4_t,
2320 twiddle23: float32x4_t,
2321 twiddle01conj: float32x4_t,
2322 twiddle23conj: float32x4_t,
2323 twiddle1: float32x4_t,
2324 twiddle2: float32x4_t,
2325 twiddle3: float32x4_t,
2326 twiddle1c: float32x4_t,
2327 twiddle2c: float32x4_t,
2328 twiddle3c: float32x4_t,
2329 }
2330
2331 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly16, 16, |this: &NeonF32Butterfly16<_>| this
2332 .direction);
2333 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly16, 16, |this: &NeonF32Butterfly16<_>| this
2334 .direction);
2335 impl<T: FftNum> NeonF32Butterfly16<T> {
2336 #[inline(always)]
2337 pub fn new(direction: FftDirection) -> Self {
2338 assert_f32::<T>();
2339 let bf8 = NeonF32Butterfly8::new(direction);
2340 let bf4 = NeonF32Butterfly4::new(direction);
2341 let rotate90 = if direction == FftDirection::Inverse {
2342 Rotate90F32::new(true)
2343 } else {
2344 Rotate90F32::new(false)
2345 };
2346 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 16, direction);
2347 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 16, direction);
2348 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 16, direction);
2349 let twiddle01 = unsafe { vld1q_f32([1.0, 0.0, tw1.re, tw1.im].as_ptr()) };
2350 let twiddle23 = unsafe { vld1q_f32([tw2.re, tw2.im, tw3.re, tw3.im].as_ptr()) };
2351 let twiddle01conj = unsafe { vld1q_f32([1.0, 0.0, tw1.re, -tw1.im].as_ptr()) };
2352 let twiddle23conj = unsafe { vld1q_f32([tw2.re, -tw2.im, tw3.re, -tw3.im].as_ptr()) };
2353 let twiddle1 = unsafe { vld1q_f32([tw1.re, tw1.im, tw1.re, tw1.im].as_ptr()) };
2354 let twiddle2 = unsafe { vld1q_f32([tw2.re, tw2.im, tw2.re, tw2.im].as_ptr()) };
2355 let twiddle3 = unsafe { vld1q_f32([tw3.re, tw3.im, tw3.re, tw3.im].as_ptr()) };
2356 let twiddle1c = unsafe { vld1q_f32([tw1.re, -tw1.im, tw1.re, -tw1.im].as_ptr()) };
2357 let twiddle2c = unsafe { vld1q_f32([tw2.re, -tw2.im, tw2.re, -tw2.im].as_ptr()) };
2358 let twiddle3c = unsafe { vld1q_f32([tw3.re, -tw3.im, tw3.re, -tw3.im].as_ptr()) };
2359 Self {
2360 direction,
2361 bf4,
2362 bf8,
2363 rotate90,
2364 twiddle01,
2365 twiddle23,
2366 twiddle01conj,
2367 twiddle23conj,
2368 twiddle1,
2369 twiddle2,
2370 twiddle3,
2371 twiddle1c,
2372 twiddle2c,
2373 twiddle3c,
2374 }
2375 }
2376
2377 #[inline(always)]
2378 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
2379 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14 });
2380
2381 let out = self.perform_fft_direct(input_packed);
2382
2383 write_complex_to_array_strided!(out, buffer, 2, {0,1,2,3,4,5,6,7});
2384 }
2385
2386 #[inline(always)]
2387 pub(crate) unsafe fn perform_parallel_fft_contiguous(
2388 &self,
2389 mut buffer: impl NeonArrayMut<f32>,
2390 ) {
2391 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
2392
2393 let values = interleave_complex_f32!(input_packed, 8, {0, 1, 2, 3 ,4 ,5 ,6 ,7});
2394
2395 let out = self.perform_parallel_fft_direct(values);
2396
2397 let out_sorted = separate_interleaved_complex_f32!(out, {0, 2, 4, 6, 8, 10, 12, 14});
2398
2399 write_complex_to_array_strided!(out_sorted, buffer, 2, {0,1,2,3,4,5,6,7,8,9, 10, 11,12,13,14, 15});
2400 }
2401
2402 #[inline(always)]
2403 unsafe fn perform_fft_direct(&self, input: [float32x4_t; 8]) -> [float32x4_t; 8] {
2404 // we're going to hardcode a step of split radix
2405 // step 1: copy and reorder the input into the scratch
2406 let in0002 = extract_lo_lo_f32(input[0], input[1]);
2407 let in0406 = extract_lo_lo_f32(input[2], input[3]);
2408 let in0810 = extract_lo_lo_f32(input[4], input[5]);
2409 let in1214 = extract_lo_lo_f32(input[6], input[7]);
2410
2411 let in0105 = extract_hi_hi_f32(input[0], input[2]);
2412 let in0913 = extract_hi_hi_f32(input[4], input[6]);
2413 let in1503 = extract_hi_hi_f32(input[7], input[1]);
2414 let in0711 = extract_hi_hi_f32(input[3], input[5]);
2415
2416 let in_evens = [in0002, in0406, in0810, in1214];
2417
2418 // step 2: column FFTs
2419 let evens = self.bf8.perform_fft_direct(in_evens);
2420 let mut odds1 = self.bf4.perform_fft_direct(in0105, in0913);
2421 let mut odds3 = self.bf4.perform_fft_direct(in1503, in0711);
2422
2423 // step 3: apply twiddle factors
2424 odds1[0] = mul_complex_f32(odds1[0], self.twiddle01);
2425 odds3[0] = mul_complex_f32(odds3[0], self.twiddle01conj);
2426
2427 odds1[1] = mul_complex_f32(odds1[1], self.twiddle23);
2428 odds3[1] = mul_complex_f32(odds3[1], self.twiddle23conj);
2429
2430 // step 4: cross FFTs
2431 let mut temp0 = parallel_fft2_interleaved_f32(odds1[0], odds3[0]);
2432 let mut temp1 = parallel_fft2_interleaved_f32(odds1[1], odds3[1]);
2433
2434 // apply the butterfly 4 twiddle factor, which is just a rotation
2435 temp0[1] = self.rotate90.rotate_both(temp0[1]);
2436 temp1[1] = self.rotate90.rotate_both(temp1[1]);
2437
2438 //step 5: copy/add/subtract data back to buffer
2439 [
2440 vaddq_f32(evens[0], temp0[0]),
2441 vaddq_f32(evens[1], temp1[0]),
2442 vaddq_f32(evens[2], temp0[1]),
2443 vaddq_f32(evens[3], temp1[1]),
2444 vsubq_f32(evens[0], temp0[0]),
2445 vsubq_f32(evens[1], temp1[0]),
2446 vsubq_f32(evens[2], temp0[1]),
2447 vsubq_f32(evens[3], temp1[1]),
2448 ]
2449 }
2450
2451 #[inline(always)]
2452 unsafe fn perform_parallel_fft_direct(&self, input: [float32x4_t; 16]) -> [float32x4_t; 16] {
2453 // we're going to hardcode a step of split radix
2454 // step 1: copy and reorder the input into the scratch
2455 // and
2456 // step 2: column FFTs
2457 let evens = self.bf8.perform_parallel_fft_direct([
2458 input[0], input[2], input[4], input[6], input[8], input[10], input[12], input[14],
2459 ]);
2460 let mut odds1 = self
2461 .bf4
2462 .perform_parallel_fft_direct(input[1], input[5], input[9], input[13]);
2463 let mut odds3 = self
2464 .bf4
2465 .perform_parallel_fft_direct(input[15], input[3], input[7], input[11]);
2466
2467 // step 3: apply twiddle factors
2468 odds1[1] = mul_complex_f32(odds1[1], self.twiddle1);
2469 odds3[1] = mul_complex_f32(odds3[1], self.twiddle1c);
2470
2471 odds1[2] = mul_complex_f32(odds1[2], self.twiddle2);
2472 odds3[2] = mul_complex_f32(odds3[2], self.twiddle2c);
2473
2474 odds1[3] = mul_complex_f32(odds1[3], self.twiddle3);
2475 odds3[3] = mul_complex_f32(odds3[3], self.twiddle3c);
2476
2477 // step 4: cross FFTs
2478 let mut temp0 = parallel_fft2_interleaved_f32(odds1[0], odds3[0]);
2479 let mut temp1 = parallel_fft2_interleaved_f32(odds1[1], odds3[1]);
2480 let mut temp2 = parallel_fft2_interleaved_f32(odds1[2], odds3[2]);
2481 let mut temp3 = parallel_fft2_interleaved_f32(odds1[3], odds3[3]);
2482
2483 // apply the butterfly 4 twiddle factor, which is just a rotation
2484 temp0[1] = self.rotate90.rotate_both(temp0[1]);
2485 temp1[1] = self.rotate90.rotate_both(temp1[1]);
2486 temp2[1] = self.rotate90.rotate_both(temp2[1]);
2487 temp3[1] = self.rotate90.rotate_both(temp3[1]);
2488
2489 //step 5: copy/add/subtract data back to buffer
2490 [
2491 vaddq_f32(evens[0], temp0[0]),
2492 vaddq_f32(evens[1], temp1[0]),
2493 vaddq_f32(evens[2], temp2[0]),
2494 vaddq_f32(evens[3], temp3[0]),
2495 vaddq_f32(evens[4], temp0[1]),
2496 vaddq_f32(evens[5], temp1[1]),
2497 vaddq_f32(evens[6], temp2[1]),
2498 vaddq_f32(evens[7], temp3[1]),
2499 vsubq_f32(evens[0], temp0[0]),
2500 vsubq_f32(evens[1], temp1[0]),
2501 vsubq_f32(evens[2], temp2[0]),
2502 vsubq_f32(evens[3], temp3[0]),
2503 vsubq_f32(evens[4], temp0[1]),
2504 vsubq_f32(evens[5], temp1[1]),
2505 vsubq_f32(evens[6], temp2[1]),
2506 vsubq_f32(evens[7], temp3[1]),
2507 ]
2508 }
2509 }
2510
2511 // _ __ __ _ _ _ _ _
2512 // / |/ /_ / /_ | || | | |__ (_) |_
2513 // | | '_ \ _____ | '_ \| || |_| '_ \| | __|
2514 // | | (_) | |_____| | (_) |__ _| |_) | | |_
2515 // |_|\___/ \___/ |_| |_.__/|_|\__|
2516 //
2517
2518 pub struct NeonF64Butterfly16<T> {
2519 direction: FftDirection,
2520 bf4: NeonF64Butterfly4<T>,
2521 bf8: NeonF64Butterfly8<T>,
2522 rotate90: Rotate90F64,
2523 twiddle1: float64x2_t,
2524 twiddle2: float64x2_t,
2525 twiddle3: float64x2_t,
2526 twiddle1c: float64x2_t,
2527 twiddle2c: float64x2_t,
2528 twiddle3c: float64x2_t,
2529 }
2530
2531 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly16, 16, |this: &NeonF64Butterfly16<_>| this
2532 .direction);
2533 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly16, 16, |this: &NeonF64Butterfly16<_>| this
2534 .direction);
2535 impl<T: FftNum> NeonF64Butterfly16<T> {
2536 #[inline(always)]
2537 pub fn new(direction: FftDirection) -> Self {
2538 assert_f64::<T>();
2539 let bf8 = NeonF64Butterfly8::new(direction);
2540 let bf4 = NeonF64Butterfly4::new(direction);
2541 let rotate90 = if direction == FftDirection::Inverse {
2542 Rotate90F64::new(true)
2543 } else {
2544 Rotate90F64::new(false)
2545 };
2546 let twiddle1 = unsafe {
2547 vld1q_f64(&twiddles::compute_twiddle::<f64>(1, 16, direction) as *const _ as *const f64)
2548 };
2549 let twiddle2 = unsafe {
2550 vld1q_f64(&twiddles::compute_twiddle::<f64>(2, 16, direction) as *const _ as *const f64)
2551 };
2552 let twiddle3 = unsafe {
2553 vld1q_f64(&twiddles::compute_twiddle::<f64>(3, 16, direction) as *const _ as *const f64)
2554 };
2555 let twiddle1c = unsafe {
2556 vld1q_f64(
2557 &twiddles::compute_twiddle::<f64>(1, 16, direction).conj() as *const _
2558 as *const f64,
2559 )
2560 };
2561 let twiddle2c = unsafe {
2562 vld1q_f64(
2563 &twiddles::compute_twiddle::<f64>(2, 16, direction).conj() as *const _
2564 as *const f64,
2565 )
2566 };
2567 let twiddle3c = unsafe {
2568 vld1q_f64(
2569 &twiddles::compute_twiddle::<f64>(3, 16, direction).conj() as *const _
2570 as *const f64,
2571 )
2572 };
2573
2574 Self {
2575 direction,
2576 bf4,
2577 bf8,
2578 rotate90,
2579 twiddle1,
2580 twiddle2,
2581 twiddle3,
2582 twiddle1c,
2583 twiddle2c,
2584 twiddle3c,
2585 }
2586 }
2587
2588 #[inline(always)]
2589 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
2590 let values =
2591 read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15});
2592
2593 let out = self.perform_fft_direct(values);
2594
2595 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15});
2596 }
2597
2598 #[inline(always)]
2599 unsafe fn perform_fft_direct(&self, input: [float64x2_t; 16]) -> [float64x2_t; 16] {
2600 // we're going to hardcode a step of split radix
2601
2602 // step 1: copy and reorder the input into the scratch
2603 // and
2604 // step 2: column FFTs
2605 let evens = self.bf8.perform_fft_direct([
2606 input[0], input[2], input[4], input[6], input[8], input[10], input[12], input[14],
2607 ]);
2608 let mut odds1 = self
2609 .bf4
2610 .perform_fft_direct(input[1], input[5], input[9], input[13]);
2611 let mut odds3 = self
2612 .bf4
2613 .perform_fft_direct(input[15], input[3], input[7], input[11]);
2614
2615 // step 3: apply twiddle factors
2616 odds1[1] = mul_complex_f64(odds1[1], self.twiddle1);
2617 odds3[1] = mul_complex_f64(odds3[1], self.twiddle1c);
2618
2619 odds1[2] = mul_complex_f64(odds1[2], self.twiddle2);
2620 odds3[2] = mul_complex_f64(odds3[2], self.twiddle2c);
2621
2622 odds1[3] = mul_complex_f64(odds1[3], self.twiddle3);
2623 odds3[3] = mul_complex_f64(odds3[3], self.twiddle3c);
2624
2625 // step 4: cross FFTs
2626 let mut temp0 = solo_fft2_f64(odds1[0], odds3[0]);
2627 let mut temp1 = solo_fft2_f64(odds1[1], odds3[1]);
2628 let mut temp2 = solo_fft2_f64(odds1[2], odds3[2]);
2629 let mut temp3 = solo_fft2_f64(odds1[3], odds3[3]);
2630
2631 // apply the butterfly 4 twiddle factor, which is just a rotation
2632 temp0[1] = self.rotate90.rotate(temp0[1]);
2633 temp1[1] = self.rotate90.rotate(temp1[1]);
2634 temp2[1] = self.rotate90.rotate(temp2[1]);
2635 temp3[1] = self.rotate90.rotate(temp3[1]);
2636
2637 //step 5: copy/add/subtract data back to buffer
2638 [
2639 vaddq_f64(evens[0], temp0[0]),
2640 vaddq_f64(evens[1], temp1[0]),
2641 vaddq_f64(evens[2], temp2[0]),
2642 vaddq_f64(evens[3], temp3[0]),
2643 vaddq_f64(evens[4], temp0[1]),
2644 vaddq_f64(evens[5], temp1[1]),
2645 vaddq_f64(evens[6], temp2[1]),
2646 vaddq_f64(evens[7], temp3[1]),
2647 vsubq_f64(evens[0], temp0[0]),
2648 vsubq_f64(evens[1], temp1[0]),
2649 vsubq_f64(evens[2], temp2[0]),
2650 vsubq_f64(evens[3], temp3[0]),
2651 vsubq_f64(evens[4], temp0[1]),
2652 vsubq_f64(evens[5], temp1[1]),
2653 vsubq_f64(evens[6], temp2[1]),
2654 vsubq_f64(evens[7], temp3[1]),
2655 ]
2656 }
2657 }
2658
2659 // _________ _________ _ _ _
2660 // |___ /___ \ |___ /___ \| |__ (_) |_
2661 // |_ \ __) | _____ |_ \ __) | '_ \| | __|
2662 // ___) / __/ |_____| ___) / __/| |_) | | |_
2663 // |____/_____| |____/_____|_.__/|_|\__|
2664 //
2665
2666 pub struct NeonF32Butterfly32<T> {
2667 direction: FftDirection,
2668 bf8: NeonF32Butterfly8<T>,
2669 bf16: NeonF32Butterfly16<T>,
2670 rotate90: Rotate90F32,
2671 twiddle01: float32x4_t,
2672 twiddle23: float32x4_t,
2673 twiddle45: float32x4_t,
2674 twiddle67: float32x4_t,
2675 twiddle01conj: float32x4_t,
2676 twiddle23conj: float32x4_t,
2677 twiddle45conj: float32x4_t,
2678 twiddle67conj: float32x4_t,
2679 twiddle1: float32x4_t,
2680 twiddle2: float32x4_t,
2681 twiddle3: float32x4_t,
2682 twiddle4: float32x4_t,
2683 twiddle5: float32x4_t,
2684 twiddle6: float32x4_t,
2685 twiddle7: float32x4_t,
2686 twiddle1c: float32x4_t,
2687 twiddle2c: float32x4_t,
2688 twiddle3c: float32x4_t,
2689 twiddle4c: float32x4_t,
2690 twiddle5c: float32x4_t,
2691 twiddle6c: float32x4_t,
2692 twiddle7c: float32x4_t,
2693 }
2694
2695 boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly32, 32, |this: &NeonF32Butterfly32<_>| this
2696 .direction);
2697 boilerplate_fft_neon_common_butterfly!(NeonF32Butterfly32, 32, |this: &NeonF32Butterfly32<_>| this
2698 .direction);
2699 impl<T: FftNum> NeonF32Butterfly32<T> {
2700 #[inline(always)]
2701 pub fn new(direction: FftDirection) -> Self {
2702 assert_f32::<T>();
2703 let bf8 = NeonF32Butterfly8::new(direction);
2704 let bf16 = NeonF32Butterfly16::new(direction);
2705 let rotate90 = if direction == FftDirection::Inverse {
2706 Rotate90F32::new(true)
2707 } else {
2708 Rotate90F32::new(false)
2709 };
2710 let tw1: Complex<f32> = twiddles::compute_twiddle(1, 32, direction);
2711 let tw2: Complex<f32> = twiddles::compute_twiddle(2, 32, direction);
2712 let tw3: Complex<f32> = twiddles::compute_twiddle(3, 32, direction);
2713 let tw4: Complex<f32> = twiddles::compute_twiddle(4, 32, direction);
2714 let tw5: Complex<f32> = twiddles::compute_twiddle(5, 32, direction);
2715 let tw6: Complex<f32> = twiddles::compute_twiddle(6, 32, direction);
2716 let tw7: Complex<f32> = twiddles::compute_twiddle(7, 32, direction);
2717 let twiddle01 = unsafe { vld1q_f32([1.0, 0.0, tw1.re, tw1.im].as_ptr()) };
2718 let twiddle23 = unsafe { vld1q_f32([tw2.re, tw2.im, tw3.re, tw3.im].as_ptr()) };
2719 let twiddle45 = unsafe { vld1q_f32([tw4.re, tw4.im, tw5.re, tw5.im].as_ptr()) };
2720 let twiddle67 = unsafe { vld1q_f32([tw6.re, tw6.im, tw7.re, tw7.im].as_ptr()) };
2721 let twiddle01conj = unsafe { vld1q_f32([1.0, 0.0, tw1.re, -tw1.im].as_ptr()) };
2722 let twiddle23conj = unsafe { vld1q_f32([tw2.re, -tw2.im, tw3.re, -tw3.im].as_ptr()) };
2723 let twiddle45conj = unsafe { vld1q_f32([tw4.re, -tw4.im, tw5.re, -tw5.im].as_ptr()) };
2724 let twiddle67conj = unsafe { vld1q_f32([tw6.re, -tw6.im, tw7.re, -tw7.im].as_ptr()) };
2725 let twiddle1 = unsafe { vld1q_f32([tw1.re, tw1.im, tw1.re, tw1.im].as_ptr()) };
2726 let twiddle2 = unsafe { vld1q_f32([tw2.re, tw2.im, tw2.re, tw2.im].as_ptr()) };
2727 let twiddle3 = unsafe { vld1q_f32([tw3.re, tw3.im, tw3.re, tw3.im].as_ptr()) };
2728 let twiddle4 = unsafe { vld1q_f32([tw4.re, tw4.im, tw4.re, tw4.im].as_ptr()) };
2729 let twiddle5 = unsafe { vld1q_f32([tw5.re, tw5.im, tw5.re, tw5.im].as_ptr()) };
2730 let twiddle6 = unsafe { vld1q_f32([tw6.re, tw6.im, tw6.re, tw6.im].as_ptr()) };
2731 let twiddle7 = unsafe { vld1q_f32([tw7.re, tw7.im, tw7.re, tw7.im].as_ptr()) };
2732 let twiddle1c = unsafe { vld1q_f32([tw1.re, -tw1.im, tw1.re, -tw1.im].as_ptr()) };
2733 let twiddle2c = unsafe { vld1q_f32([tw2.re, -tw2.im, tw2.re, -tw2.im].as_ptr()) };
2734 let twiddle3c = unsafe { vld1q_f32([tw3.re, -tw3.im, tw3.re, -tw3.im].as_ptr()) };
2735 let twiddle4c = unsafe { vld1q_f32([tw4.re, -tw4.im, tw4.re, -tw4.im].as_ptr()) };
2736 let twiddle5c = unsafe { vld1q_f32([tw5.re, -tw5.im, tw5.re, -tw5.im].as_ptr()) };
2737 let twiddle6c = unsafe { vld1q_f32([tw6.re, -tw6.im, tw6.re, -tw6.im].as_ptr()) };
2738 let twiddle7c = unsafe { vld1q_f32([tw7.re, -tw7.im, tw7.re, -tw7.im].as_ptr()) };
2739 Self {
2740 direction,
2741 bf8,
2742 bf16,
2743 rotate90,
2744 twiddle01,
2745 twiddle23,
2746 twiddle45,
2747 twiddle67,
2748 twiddle01conj,
2749 twiddle23conj,
2750 twiddle45conj,
2751 twiddle67conj,
2752 twiddle1,
2753 twiddle2,
2754 twiddle3,
2755 twiddle4,
2756 twiddle5,
2757 twiddle6,
2758 twiddle7,
2759 twiddle1c,
2760 twiddle2c,
2761 twiddle3c,
2762 twiddle4c,
2763 twiddle5c,
2764 twiddle6c,
2765 twiddle7c,
2766 }
2767 }
2768
2769 #[inline(always)]
2770 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f32>) {
2771 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 });
2772
2773 let out = self.perform_fft_direct(input_packed);
2774
2775 write_complex_to_array_strided!(out, buffer, 2, {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15});
2776 }
2777
2778 #[inline(always)]
2779 pub(crate) unsafe fn perform_parallel_fft_contiguous(
2780 &self,
2781 mut buffer: impl NeonArrayMut<f32>,
2782 ) {
2783 let input_packed = read_complex_to_array!(buffer, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62});
2784
2785 let values = interleave_complex_f32!(input_packed, 16, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15});
2786
2787 let out = self.perform_parallel_fft_direct(values);
2788
2789 let out_sorted = separate_interleaved_complex_f32!(out, {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
2790
2791 write_complex_to_array_strided!(out_sorted, buffer, 2, {0,1,2,3,4,5,6,7,8,9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 });
2792 }
2793
2794 #[inline(always)]
2795 unsafe fn perform_fft_direct(&self, input: [float32x4_t; 16]) -> [float32x4_t; 16] {
2796 // we're going to hardcode a step of split radix
2797
2798 // step 1: copy and reorder the input into the scratch
2799 let in0002 = extract_lo_lo_f32(input[0], input[1]);
2800 let in0406 = extract_lo_lo_f32(input[2], input[3]);
2801 let in0810 = extract_lo_lo_f32(input[4], input[5]);
2802 let in1214 = extract_lo_lo_f32(input[6], input[7]);
2803 let in1618 = extract_lo_lo_f32(input[8], input[9]);
2804 let in2022 = extract_lo_lo_f32(input[10], input[11]);
2805 let in2426 = extract_lo_lo_f32(input[12], input[13]);
2806 let in2830 = extract_lo_lo_f32(input[14], input[15]);
2807
2808 let in0105 = extract_hi_hi_f32(input[0], input[2]);
2809 let in0913 = extract_hi_hi_f32(input[4], input[6]);
2810 let in1721 = extract_hi_hi_f32(input[8], input[10]);
2811 let in2529 = extract_hi_hi_f32(input[12], input[14]);
2812
2813 let in3103 = extract_hi_hi_f32(input[15], input[1]);
2814 let in0711 = extract_hi_hi_f32(input[3], input[5]);
2815 let in1519 = extract_hi_hi_f32(input[7], input[9]);
2816 let in2327 = extract_hi_hi_f32(input[11], input[13]);
2817
2818 let in_evens = [
2819 in0002, in0406, in0810, in1214, in1618, in2022, in2426, in2830,
2820 ];
2821
2822 // step 2: column FFTs
2823 let evens = self.bf16.perform_fft_direct(in_evens);
2824 let mut odds1 = self
2825 .bf8
2826 .perform_fft_direct([in0105, in0913, in1721, in2529]);
2827 let mut odds3 = self
2828 .bf8
2829 .perform_fft_direct([in3103, in0711, in1519, in2327]);
2830
2831 // step 3: apply twiddle factors
2832 odds1[0] = mul_complex_f32(odds1[0], self.twiddle01);
2833 odds3[0] = mul_complex_f32(odds3[0], self.twiddle01conj);
2834
2835 odds1[1] = mul_complex_f32(odds1[1], self.twiddle23);
2836 odds3[1] = mul_complex_f32(odds3[1], self.twiddle23conj);
2837
2838 odds1[2] = mul_complex_f32(odds1[2], self.twiddle45);
2839 odds3[2] = mul_complex_f32(odds3[2], self.twiddle45conj);
2840
2841 odds1[3] = mul_complex_f32(odds1[3], self.twiddle67);
2842 odds3[3] = mul_complex_f32(odds3[3], self.twiddle67conj);
2843
2844 // step 4: cross FFTs
2845 let mut temp0 = parallel_fft2_interleaved_f32(odds1[0], odds3[0]);
2846 let mut temp1 = parallel_fft2_interleaved_f32(odds1[1], odds3[1]);
2847 let mut temp2 = parallel_fft2_interleaved_f32(odds1[2], odds3[2]);
2848 let mut temp3 = parallel_fft2_interleaved_f32(odds1[3], odds3[3]);
2849
2850 // apply the butterfly 4 twiddle factor, which is just a rotation
2851 temp0[1] = self.rotate90.rotate_both(temp0[1]);
2852 temp1[1] = self.rotate90.rotate_both(temp1[1]);
2853 temp2[1] = self.rotate90.rotate_both(temp2[1]);
2854 temp3[1] = self.rotate90.rotate_both(temp3[1]);
2855
2856 //step 5: copy/add/subtract data back to buffer
2857 [
2858 vaddq_f32(evens[0], temp0[0]),
2859 vaddq_f32(evens[1], temp1[0]),
2860 vaddq_f32(evens[2], temp2[0]),
2861 vaddq_f32(evens[3], temp3[0]),
2862 vaddq_f32(evens[4], temp0[1]),
2863 vaddq_f32(evens[5], temp1[1]),
2864 vaddq_f32(evens[6], temp2[1]),
2865 vaddq_f32(evens[7], temp3[1]),
2866 vsubq_f32(evens[0], temp0[0]),
2867 vsubq_f32(evens[1], temp1[0]),
2868 vsubq_f32(evens[2], temp2[0]),
2869 vsubq_f32(evens[3], temp3[0]),
2870 vsubq_f32(evens[4], temp0[1]),
2871 vsubq_f32(evens[5], temp1[1]),
2872 vsubq_f32(evens[6], temp2[1]),
2873 vsubq_f32(evens[7], temp3[1]),
2874 ]
2875 }
2876
2877 #[inline(always)]
2878 pub(crate) unsafe fn perform_parallel_fft_direct(
2879 &self,
2880 input: [float32x4_t; 32],
2881 ) -> [float32x4_t; 32] {
2882 // we're going to hardcode a step of split radix
2883
2884 // step 1: copy and reorder the input into the scratch
2885 // and
2886 // step 2: column FFTs
2887 let evens = self.bf16.perform_parallel_fft_direct([
2888 input[0], input[2], input[4], input[6], input[8], input[10], input[12], input[14],
2889 input[16], input[18], input[20], input[22], input[24], input[26], input[28], input[30],
2890 ]);
2891 let mut odds1 = self.bf8.perform_parallel_fft_direct([
2892 input[1], input[5], input[9], input[13], input[17], input[21], input[25], input[29],
2893 ]);
2894 let mut odds3 = self.bf8.perform_parallel_fft_direct([
2895 input[31], input[3], input[7], input[11], input[15], input[19], input[23], input[27],
2896 ]);
2897
2898 // step 3: apply twiddle factors
2899 odds1[1] = mul_complex_f32(odds1[1], self.twiddle1);
2900 odds3[1] = mul_complex_f32(odds3[1], self.twiddle1c);
2901
2902 odds1[2] = mul_complex_f32(odds1[2], self.twiddle2);
2903 odds3[2] = mul_complex_f32(odds3[2], self.twiddle2c);
2904
2905 odds1[3] = mul_complex_f32(odds1[3], self.twiddle3);
2906 odds3[3] = mul_complex_f32(odds3[3], self.twiddle3c);
2907
2908 odds1[4] = mul_complex_f32(odds1[4], self.twiddle4);
2909 odds3[4] = mul_complex_f32(odds3[4], self.twiddle4c);
2910
2911 odds1[5] = mul_complex_f32(odds1[5], self.twiddle5);
2912 odds3[5] = mul_complex_f32(odds3[5], self.twiddle5c);
2913
2914 odds1[6] = mul_complex_f32(odds1[6], self.twiddle6);
2915 odds3[6] = mul_complex_f32(odds3[6], self.twiddle6c);
2916
2917 odds1[7] = mul_complex_f32(odds1[7], self.twiddle7);
2918 odds3[7] = mul_complex_f32(odds3[7], self.twiddle7c);
2919
2920 // step 4: cross FFTs
2921 let mut temp0 = parallel_fft2_interleaved_f32(odds1[0], odds3[0]);
2922 let mut temp1 = parallel_fft2_interleaved_f32(odds1[1], odds3[1]);
2923 let mut temp2 = parallel_fft2_interleaved_f32(odds1[2], odds3[2]);
2924 let mut temp3 = parallel_fft2_interleaved_f32(odds1[3], odds3[3]);
2925 let mut temp4 = parallel_fft2_interleaved_f32(odds1[4], odds3[4]);
2926 let mut temp5 = parallel_fft2_interleaved_f32(odds1[5], odds3[5]);
2927 let mut temp6 = parallel_fft2_interleaved_f32(odds1[6], odds3[6]);
2928 let mut temp7 = parallel_fft2_interleaved_f32(odds1[7], odds3[7]);
2929
2930 // apply the butterfly 4 twiddle factor, which is just a rotation
2931 temp0[1] = self.rotate90.rotate_both(temp0[1]);
2932 temp1[1] = self.rotate90.rotate_both(temp1[1]);
2933 temp2[1] = self.rotate90.rotate_both(temp2[1]);
2934 temp3[1] = self.rotate90.rotate_both(temp3[1]);
2935 temp4[1] = self.rotate90.rotate_both(temp4[1]);
2936 temp5[1] = self.rotate90.rotate_both(temp5[1]);
2937 temp6[1] = self.rotate90.rotate_both(temp6[1]);
2938 temp7[1] = self.rotate90.rotate_both(temp7[1]);
2939
2940 //step 5: copy/add/subtract data back to buffer
2941 [
2942 vaddq_f32(evens[0], temp0[0]),
2943 vaddq_f32(evens[1], temp1[0]),
2944 vaddq_f32(evens[2], temp2[0]),
2945 vaddq_f32(evens[3], temp3[0]),
2946 vaddq_f32(evens[4], temp4[0]),
2947 vaddq_f32(evens[5], temp5[0]),
2948 vaddq_f32(evens[6], temp6[0]),
2949 vaddq_f32(evens[7], temp7[0]),
2950 vaddq_f32(evens[8], temp0[1]),
2951 vaddq_f32(evens[9], temp1[1]),
2952 vaddq_f32(evens[10], temp2[1]),
2953 vaddq_f32(evens[11], temp3[1]),
2954 vaddq_f32(evens[12], temp4[1]),
2955 vaddq_f32(evens[13], temp5[1]),
2956 vaddq_f32(evens[14], temp6[1]),
2957 vaddq_f32(evens[15], temp7[1]),
2958 vsubq_f32(evens[0], temp0[0]),
2959 vsubq_f32(evens[1], temp1[0]),
2960 vsubq_f32(evens[2], temp2[0]),
2961 vsubq_f32(evens[3], temp3[0]),
2962 vsubq_f32(evens[4], temp4[0]),
2963 vsubq_f32(evens[5], temp5[0]),
2964 vsubq_f32(evens[6], temp6[0]),
2965 vsubq_f32(evens[7], temp7[0]),
2966 vsubq_f32(evens[8], temp0[1]),
2967 vsubq_f32(evens[9], temp1[1]),
2968 vsubq_f32(evens[10], temp2[1]),
2969 vsubq_f32(evens[11], temp3[1]),
2970 vsubq_f32(evens[12], temp4[1]),
2971 vsubq_f32(evens[13], temp5[1]),
2972 vsubq_f32(evens[14], temp6[1]),
2973 vsubq_f32(evens[15], temp7[1]),
2974 ]
2975 }
2976 }
2977
2978 // _________ __ _ _ _ _ _
2979 // |___ /___ \ / /_ | || | | |__ (_) |_
2980 // |_ \ __) | _____ | '_ \| || |_| '_ \| | __|
2981 // ___) / __/ |_____| | (_) |__ _| |_) | | |_
2982 // |____/_____| \___/ |_| |_.__/|_|\__|
2983 //
2984
2985 pub struct NeonF64Butterfly32<T> {
2986 direction: FftDirection,
2987 bf8: NeonF64Butterfly8<T>,
2988 bf16: NeonF64Butterfly16<T>,
2989 rotate90: Rotate90F64,
2990 twiddle1: float64x2_t,
2991 twiddle2: float64x2_t,
2992 twiddle3: float64x2_t,
2993 twiddle4: float64x2_t,
2994 twiddle5: float64x2_t,
2995 twiddle6: float64x2_t,
2996 twiddle7: float64x2_t,
2997 twiddle1c: float64x2_t,
2998 twiddle2c: float64x2_t,
2999 twiddle3c: float64x2_t,
3000 twiddle4c: float64x2_t,
3001 twiddle5c: float64x2_t,
3002 twiddle6c: float64x2_t,
3003 twiddle7c: float64x2_t,
3004 }
3005
3006 boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly32, 32, |this: &NeonF64Butterfly32<_>| this
3007 .direction);
3008 boilerplate_fft_neon_common_butterfly!(NeonF64Butterfly32, 32, |this: &NeonF64Butterfly32<_>| this
3009 .direction);
3010 impl<T: FftNum> NeonF64Butterfly32<T> {
3011 #[inline(always)]
3012 pub fn new(direction: FftDirection) -> Self {
3013 assert_f64::<T>();
3014 let bf8 = NeonF64Butterfly8::new(direction);
3015 let bf16 = NeonF64Butterfly16::new(direction);
3016 let rotate90 = if direction == FftDirection::Inverse {
3017 Rotate90F64::new(true)
3018 } else {
3019 Rotate90F64::new(false)
3020 };
3021 let twiddle1 = unsafe {
3022 vld1q_f64(&twiddles::compute_twiddle::<f64>(1, 32, direction) as *const _ as *const f64)
3023 };
3024 let twiddle2 = unsafe {
3025 vld1q_f64(&twiddles::compute_twiddle::<f64>(2, 32, direction) as *const _ as *const f64)
3026 };
3027 let twiddle3 = unsafe {
3028 vld1q_f64(&twiddles::compute_twiddle::<f64>(3, 32, direction) as *const _ as *const f64)
3029 };
3030 let twiddle4 = unsafe {
3031 vld1q_f64(&twiddles::compute_twiddle::<f64>(4, 32, direction) as *const _ as *const f64)
3032 };
3033 let twiddle5 = unsafe {
3034 vld1q_f64(&twiddles::compute_twiddle::<f64>(5, 32, direction) as *const _ as *const f64)
3035 };
3036 let twiddle6 = unsafe {
3037 vld1q_f64(&twiddles::compute_twiddle::<f64>(6, 32, direction) as *const _ as *const f64)
3038 };
3039 let twiddle7 = unsafe {
3040 vld1q_f64(&twiddles::compute_twiddle::<f64>(7, 32, direction) as *const _ as *const f64)
3041 };
3042 let twiddle1c = unsafe {
3043 vld1q_f64(
3044 &twiddles::compute_twiddle::<f64>(1, 32, direction).conj() as *const _
3045 as *const f64,
3046 )
3047 };
3048 let twiddle2c = unsafe {
3049 vld1q_f64(
3050 &twiddles::compute_twiddle::<f64>(2, 32, direction).conj() as *const _
3051 as *const f64,
3052 )
3053 };
3054 let twiddle3c = unsafe {
3055 vld1q_f64(
3056 &twiddles::compute_twiddle::<f64>(3, 32, direction).conj() as *const _
3057 as *const f64,
3058 )
3059 };
3060 let twiddle4c = unsafe {
3061 vld1q_f64(
3062 &twiddles::compute_twiddle::<f64>(4, 32, direction).conj() as *const _
3063 as *const f64,
3064 )
3065 };
3066 let twiddle5c = unsafe {
3067 vld1q_f64(
3068 &twiddles::compute_twiddle::<f64>(5, 32, direction).conj() as *const _
3069 as *const f64,
3070 )
3071 };
3072 let twiddle6c = unsafe {
3073 vld1q_f64(
3074 &twiddles::compute_twiddle::<f64>(6, 32, direction).conj() as *const _
3075 as *const f64,
3076 )
3077 };
3078 let twiddle7c = unsafe {
3079 vld1q_f64(
3080 &twiddles::compute_twiddle::<f64>(7, 32, direction).conj() as *const _
3081 as *const f64,
3082 )
3083 };
3084
3085 Self {
3086 direction,
3087 bf8,
3088 bf16,
3089 rotate90,
3090 twiddle1,
3091 twiddle2,
3092 twiddle3,
3093 twiddle4,
3094 twiddle5,
3095 twiddle6,
3096 twiddle7,
3097 twiddle1c,
3098 twiddle2c,
3099 twiddle3c,
3100 twiddle4c,
3101 twiddle5c,
3102 twiddle6c,
3103 twiddle7c,
3104 }
3105 }
3106
3107 #[inline(always)]
3108 unsafe fn perform_fft_contiguous(&self, mut buffer: impl NeonArrayMut<f64>) {
3109 let values = read_complex_to_array!(buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31});
3110
3111 let out = self.perform_fft_direct(values);
3112
3113 write_complex_to_array!(out, buffer, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31});
3114 }
3115
3116 #[inline(always)]
3117 unsafe fn perform_fft_direct(&self, input: [float64x2_t; 32]) -> [float64x2_t; 32] {
3118 // we're going to hardcode a step of split radix
3119
3120 // step 1: copy and reorder the input into the scratch
3121 // and
3122 // step 2: column FFTs
3123 let evens = self.bf16.perform_fft_direct([
3124 input[0], input[2], input[4], input[6], input[8], input[10], input[12], input[14],
3125 input[16], input[18], input[20], input[22], input[24], input[26], input[28], input[30],
3126 ]);
3127 let mut odds1 = self.bf8.perform_fft_direct([
3128 input[1], input[5], input[9], input[13], input[17], input[21], input[25], input[29],
3129 ]);
3130 let mut odds3 = self.bf8.perform_fft_direct([
3131 input[31], input[3], input[7], input[11], input[15], input[19], input[23], input[27],
3132 ]);
3133
3134 // step 3: apply twiddle factors
3135 odds1[1] = mul_complex_f64(odds1[1], self.twiddle1);
3136 odds3[1] = mul_complex_f64(odds3[1], self.twiddle1c);
3137
3138 odds1[2] = mul_complex_f64(odds1[2], self.twiddle2);
3139 odds3[2] = mul_complex_f64(odds3[2], self.twiddle2c);
3140
3141 odds1[3] = mul_complex_f64(odds1[3], self.twiddle3);
3142 odds3[3] = mul_complex_f64(odds3[3], self.twiddle3c);
3143
3144 odds1[4] = mul_complex_f64(odds1[4], self.twiddle4);
3145 odds3[4] = mul_complex_f64(odds3[4], self.twiddle4c);
3146
3147 odds1[5] = mul_complex_f64(odds1[5], self.twiddle5);
3148 odds3[5] = mul_complex_f64(odds3[5], self.twiddle5c);
3149
3150 odds1[6] = mul_complex_f64(odds1[6], self.twiddle6);
3151 odds3[6] = mul_complex_f64(odds3[6], self.twiddle6c);
3152
3153 odds1[7] = mul_complex_f64(odds1[7], self.twiddle7);
3154 odds3[7] = mul_complex_f64(odds3[7], self.twiddle7c);
3155
3156 // step 4: cross FFTs
3157 let mut temp0 = solo_fft2_f64(odds1[0], odds3[0]);
3158 let mut temp1 = solo_fft2_f64(odds1[1], odds3[1]);
3159 let mut temp2 = solo_fft2_f64(odds1[2], odds3[2]);
3160 let mut temp3 = solo_fft2_f64(odds1[3], odds3[3]);
3161 let mut temp4 = solo_fft2_f64(odds1[4], odds3[4]);
3162 let mut temp5 = solo_fft2_f64(odds1[5], odds3[5]);
3163 let mut temp6 = solo_fft2_f64(odds1[6], odds3[6]);
3164 let mut temp7 = solo_fft2_f64(odds1[7], odds3[7]);
3165
3166 // apply the butterfly 4 twiddle factor, which is just a rotation
3167 temp0[1] = self.rotate90.rotate(temp0[1]);
3168 temp1[1] = self.rotate90.rotate(temp1[1]);
3169 temp2[1] = self.rotate90.rotate(temp2[1]);
3170 temp3[1] = self.rotate90.rotate(temp3[1]);
3171 temp4[1] = self.rotate90.rotate(temp4[1]);
3172 temp5[1] = self.rotate90.rotate(temp5[1]);
3173 temp6[1] = self.rotate90.rotate(temp6[1]);
3174 temp7[1] = self.rotate90.rotate(temp7[1]);
3175
3176 //step 5: copy/add/subtract data back to buffer
3177 [
3178 vaddq_f64(evens[0], temp0[0]),
3179 vaddq_f64(evens[1], temp1[0]),
3180 vaddq_f64(evens[2], temp2[0]),
3181 vaddq_f64(evens[3], temp3[0]),
3182 vaddq_f64(evens[4], temp4[0]),
3183 vaddq_f64(evens[5], temp5[0]),
3184 vaddq_f64(evens[6], temp6[0]),
3185 vaddq_f64(evens[7], temp7[0]),
3186 vaddq_f64(evens[8], temp0[1]),
3187 vaddq_f64(evens[9], temp1[1]),
3188 vaddq_f64(evens[10], temp2[1]),
3189 vaddq_f64(evens[11], temp3[1]),
3190 vaddq_f64(evens[12], temp4[1]),
3191 vaddq_f64(evens[13], temp5[1]),
3192 vaddq_f64(evens[14], temp6[1]),
3193 vaddq_f64(evens[15], temp7[1]),
3194 vsubq_f64(evens[0], temp0[0]),
3195 vsubq_f64(evens[1], temp1[0]),
3196 vsubq_f64(evens[2], temp2[0]),
3197 vsubq_f64(evens[3], temp3[0]),
3198 vsubq_f64(evens[4], temp4[0]),
3199 vsubq_f64(evens[5], temp5[0]),
3200 vsubq_f64(evens[6], temp6[0]),
3201 vsubq_f64(evens[7], temp7[0]),
3202 vsubq_f64(evens[8], temp0[1]),
3203 vsubq_f64(evens[9], temp1[1]),
3204 vsubq_f64(evens[10], temp2[1]),
3205 vsubq_f64(evens[11], temp3[1]),
3206 vsubq_f64(evens[12], temp4[1]),
3207 vsubq_f64(evens[13], temp5[1]),
3208 vsubq_f64(evens[14], temp6[1]),
3209 vsubq_f64(evens[15], temp7[1]),
3210 ]
3211 }
3212 }
3213
3214 #[cfg(test)]
3215 mod unit_tests {
3216 use super::*;
3217 use crate::algorithm::Dft;
3218 use crate::test_utils::{check_fft_algorithm, compare_vectors};
3219
3220 //the tests for all butterflies will be identical except for the identifiers used and size
3221 //so it's ideal for a macro
3222 macro_rules! test_butterfly_32_func {
3223 ($test_name:ident, $struct_name:ident, $size:expr) => {
3224 #[test]
3225 fn $test_name() {
3226 let butterfly = $struct_name::new(FftDirection::Forward);
3227 check_fft_algorithm::<f32>(&butterfly, $size, FftDirection::Forward);
3228
3229 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
3230 check_fft_algorithm::<f32>(&butterfly_direction, $size, FftDirection::Inverse);
3231 }
3232 };
3233 }
3234 test_butterfly_32_func!(test_neonf32_butterfly1, NeonF32Butterfly1, 1);
3235 test_butterfly_32_func!(test_neonf32_butterfly2, NeonF32Butterfly2, 2);
3236 test_butterfly_32_func!(test_neonf32_butterfly3, NeonF32Butterfly3, 3);
3237 test_butterfly_32_func!(test_neonf32_butterfly4, NeonF32Butterfly4, 4);
3238 test_butterfly_32_func!(test_neonf32_butterfly5, NeonF32Butterfly5, 5);
3239 test_butterfly_32_func!(test_neonf32_butterfly6, NeonF32Butterfly6, 6);
3240 test_butterfly_32_func!(test_neonf32_butterfly8, NeonF32Butterfly8, 8);
3241 test_butterfly_32_func!(test_neonf32_butterfly9, NeonF32Butterfly9, 9);
3242 test_butterfly_32_func!(test_neonf32_butterfly10, NeonF32Butterfly10, 10);
3243 test_butterfly_32_func!(test_neonf32_butterfly12, NeonF32Butterfly12, 12);
3244 test_butterfly_32_func!(test_neonf32_butterfly15, NeonF32Butterfly15, 15);
3245 test_butterfly_32_func!(test_neonf32_butterfly16, NeonF32Butterfly16, 16);
3246 test_butterfly_32_func!(test_neonf32_butterfly32, NeonF32Butterfly32, 32);
3247
3248 //the tests for all butterflies will be identical except for the identifiers used and size
3249 //so it's ideal for a macro
3250 macro_rules! test_butterfly_64_func {
3251 ($test_name:ident, $struct_name:ident, $size:expr) => {
3252 #[test]
3253 fn $test_name() {
3254 let butterfly = $struct_name::new(FftDirection::Forward);
3255 check_fft_algorithm::<f64>(&butterfly, $size, FftDirection::Forward);
3256
3257 let butterfly_direction = $struct_name::new(FftDirection::Inverse);
3258 check_fft_algorithm::<f64>(&butterfly_direction, $size, FftDirection::Inverse);
3259 }
3260 };
3261 }
3262 test_butterfly_64_func!(test_neonf64_butterfly1, NeonF64Butterfly1, 1);
3263 test_butterfly_64_func!(test_neonf64_butterfly2, NeonF64Butterfly2, 2);
3264 test_butterfly_64_func!(test_neonf64_butterfly3, NeonF64Butterfly3, 3);
3265 test_butterfly_64_func!(test_neonf64_butterfly4, NeonF64Butterfly4, 4);
3266 test_butterfly_64_func!(test_neonf64_butterfly5, NeonF64Butterfly5, 5);
3267 test_butterfly_64_func!(test_neonf64_butterfly6, NeonF64Butterfly6, 6);
3268 test_butterfly_64_func!(test_neonf64_butterfly8, NeonF64Butterfly8, 8);
3269 test_butterfly_64_func!(test_neonf64_butterfly9, NeonF64Butterfly9, 9);
3270 test_butterfly_64_func!(test_neonf64_butterfly10, NeonF64Butterfly10, 10);
3271 test_butterfly_64_func!(test_neonf64_butterfly12, NeonF64Butterfly12, 12);
3272 test_butterfly_64_func!(test_neonf64_butterfly15, NeonF64Butterfly15, 15);
3273 test_butterfly_64_func!(test_neonf64_butterfly16, NeonF64Butterfly16, 16);
3274 test_butterfly_64_func!(test_neonf64_butterfly32, NeonF64Butterfly32, 32);
3275
3276 #[test]
3277 fn test_solo_fft2_32() {
3278 unsafe {
3279 let val1 = Complex::<f32>::new(1.0, 2.5);
3280 let val2 = Complex::<f32>::new(3.2, 4.2);
3281
3282 let mut val = vec![val1, val2];
3283
3284 let in_packed = vld1q_f32(val.as_ptr() as *const f32);
3285
3286 let dft = Dft::new(2, FftDirection::Forward);
3287
3288 let bf2 = NeonF32Butterfly2::<f32>::new(FftDirection::Forward);
3289
3290 dft.process(&mut val);
3291 let res_packed = bf2.perform_fft_direct(in_packed);
3292
3293 let res = std::mem::transmute::<float32x4_t, [Complex<f32>; 2]>(res_packed);
3294 assert_eq!(val[0], res[0]);
3295 assert_eq!(val[1], res[1]);
3296 }
3297 }
3298
3299 #[test]
3300 fn test_parallel_fft2_32() {
3301 unsafe {
3302 let val_a1 = Complex::<f32>::new(1.0, 2.5);
3303 let val_a2 = Complex::<f32>::new(3.2, 4.2);
3304
3305 let val_b1 = Complex::<f32>::new(6.0, 24.5);
3306 let val_b2 = Complex::<f32>::new(4.3, 34.2);
3307
3308 let mut val_a = vec![val_a1, val_a2];
3309 let mut val_b = vec![val_b1, val_b2];
3310
3311 let p1 = vld1q_f32(val_a.as_ptr() as *const f32);
3312 let p2 = vld1q_f32(val_b.as_ptr() as *const f32);
3313
3314 let dft = Dft::new(2, FftDirection::Forward);
3315
3316 let bf2 = NeonF32Butterfly2::<f32>::new(FftDirection::Forward);
3317
3318 dft.process(&mut val_a);
3319 dft.process(&mut val_b);
3320 let res_both = bf2.perform_parallel_fft_direct(p1, p2);
3321
3322 let res = std::mem::transmute::<[float32x4_t; 2], [Complex<f32>; 4]>(res_both);
3323 let neon_res_a = [res[0], res[2]];
3324 let neon_res_b = [res[1], res[3]];
3325 assert!(compare_vectors(&val_a, &neon_res_a));
3326 assert!(compare_vectors(&val_b, &neon_res_b));
3327 }
3328 }
3329 }