Debian Codegraph @

.cargo/vendor/rustfft/src/neon/neon_vector.rs

raw
1 use core::arch::aarch64::*;
2 use num_complex::Complex;
3 use std::ops::{Deref, DerefMut};
4
5 use crate::array_utils::DoubleBuf;
6
7 // Read these indexes from an NeonArray and build an array of simd vectors.
8 // Takes a name of a vector to read from, and a list of indexes to read.
9 // This statement:
10 // ```
11 // let values = read_complex_to_array!(input, {0, 1, 2, 3});
12 // ```
13 // is equivalent to:
14 // ```
15 // let values = [
16 // input.load_complex(0),
17 // input.load_complex(1),
18 // input.load_complex(2),
19 // input.load_complex(3),
20 // ];
21 // ```
22 macro_rules! read_complex_to_array {
23 ($input:ident, { $($idx:literal),* }) => {
24 [
25 $(
26 $input.load_complex($idx),
27 )*
28 ]
29 }
30 }
31
32 // Read these indexes from an NeonArray and build an array or partially filled simd vectors.
33 // Takes a name of a vector to read from, and a list of indexes to read.
34 // This statement:
35 // ```
36 // let values = read_partial1_complex_to_array!(input, {0, 1, 2, 3});
37 // ```
38 // is equivalent to:
39 // ```
40 // let values = [
41 // input.load1_complex(0),
42 // input.load1_complex(1),
43 // input.load1_complex(2),
44 // input.load1_complex(3),
45 // ];
46 // ```
47 macro_rules! read_partial1_complex_to_array {
48 ($input:ident, { $($idx:literal),* }) => {
49 [
50 $(
51 $input.load1_complex($idx),
52 )*
53 ]
54 }
55 }
56
57 // Write these indexes of an array of simd vectors to the same indexes of an NeonArray.
58 // Takes a name of a vector to read from, one to write to, and a list of indexes.
59 // This statement:
60 // ```
61 // let values = write_complex_to_array!(input, output, {0, 1, 2, 3});
62 // ```
63 // is equivalent to:
64 // ```
65 // let values = [
66 // output.store_complex(input[0], 0),
67 // output.store_complex(input[1], 1),
68 // output.store_complex(input[2], 2),
69 // output.store_complex(input[3], 3),
70 // ];
71 // ```
72 macro_rules! write_complex_to_array {
73 ($input:ident, $output:ident, { $($idx:literal),* }) => {
74 $(
75 $output.store_complex($input[$idx], $idx);
76 )*
77 }
78 }
79
80 // Write the low half of these indexes of an array of simd vectors to the same indexes of an NeonArray.
81 // Takes a name of a vector to read from, one to write to, and a list of indexes.
82 // This statement:
83 // ```
84 // let values = write_partial_lo_complex_to_array!(input, output, {0, 1, 2, 3});
85 // ```
86 // is equivalent to:
87 // ```
88 // let values = [
89 // output.store_partial_lo_complex(input[0], 0),
90 // output.store_partial_lo_complex(input[1], 1),
91 // output.store_partial_lo_complex(input[2], 2),
92 // output.store_partial_lo_complex(input[3], 3),
93 // ];
94 // ```
95 macro_rules! write_partial_lo_complex_to_array {
96 ($input:ident, $output:ident, { $($idx:literal),* }) => {
97 $(
98 $output.store_partial_lo_complex($input[$idx], $idx);
99 )*
100 }
101 }
102
103 // Write these indexes of an array of simd vectors to the same indexes, multiplied by a stride, of an NeonArray.
104 // Takes a name of a vector to read from, one to write to, an integer stride, and a list of indexes.
105 // This statement:
106 // ```
107 // let values = write_complex_to_array_separate!(input, output, {0, 1, 2, 3});
108 // ```
109 // is equivalent to:
110 // ```
111 // let values = [
112 // output.store_complex(input[0], 0),
113 // output.store_complex(input[1], 2),
114 // output.store_complex(input[2], 4),
115 // output.store_complex(input[3], 6),
116 // ];
117 // ```
118 macro_rules! write_complex_to_array_strided {
119 ($input:ident, $output:ident, $stride:literal, { $($idx:literal),* }) => {
120 $(
121 $output.store_complex($input[$idx], $idx*$stride);
122 )*
123 }
124 }
125
126 pub trait NeonNum {
127 type VectorType;
128 const COMPLEX_PER_VECTOR: usize;
129 }
130 impl NeonNum for f32 {
131 type VectorType = float32x4_t;
132 const COMPLEX_PER_VECTOR: usize = 2;
133 }
134 impl NeonNum for f64 {
135 type VectorType = float64x2_t;
136 const COMPLEX_PER_VECTOR: usize = 1;
137 }
138
139 // A trait to handle reading from an array of complex floats into Neon vectors.
140 // Neon works with 128-bit vectors, meaning a vector can hold two complex f32,
141 // or a single complex f64.
142 pub trait NeonArray<T: NeonNum>: Deref {
143 // Load complex numbers from the array to fill a Neon vector.
144 unsafe fn load_complex(&self, index: usize) -> T::VectorType;
145 // Load a single complex number from the array into a Neon vector, setting the unused elements to zero.
146 unsafe fn load_partial1_complex(&self, index: usize) -> T::VectorType;
147 // Load a single complex number from the array, and copy it to all elements of a Neon vector.
148 unsafe fn load1_complex(&self, index: usize) -> T::VectorType;
149 }
150
151 impl NeonArray<f32> for &[Complex<f32>] {
152 #[inline(always)]
153 unsafe fn load_complex(&self, index: usize) -> <f32 as NeonNum>::VectorType {
154 debug_assert!(self.len() >= index + <f32 as NeonNum>::COMPLEX_PER_VECTOR);
155 vld1q_f32(self.as_ptr().add(index) as *const f32)
156 }
157
158 #[inline(always)]
159 unsafe fn load_partial1_complex(&self, index: usize) -> <f32 as NeonNum>::VectorType {
160 debug_assert!(self.len() >= index + 1);
161 let temp = vmovq_n_f32(0.0);
162 vreinterpretq_f32_u64(vld1q_lane_u64::<0>(
163 self.as_ptr().add(index) as *const u64,
164 vreinterpretq_u64_f32(temp),
165 ))
166 }
167
168 #[inline(always)]
169 unsafe fn load1_complex(&self, index: usize) -> <f32 as NeonNum>::VectorType {
170 debug_assert!(self.len() >= index + 1);
171 vreinterpretq_f32_u64(vld1q_dup_u64(self.as_ptr().add(index) as *const u64))
172 }
173 }
174
175 impl NeonArray<f32> for &mut [Complex<f32>] {
176 #[inline(always)]
177 unsafe fn load_complex(&self, index: usize) -> <f32 as NeonNum>::VectorType {
178 debug_assert!(self.len() >= index + <f32 as NeonNum>::COMPLEX_PER_VECTOR);
179 vld1q_f32(self.as_ptr().add(index) as *const f32)
180 }
181
182 #[inline(always)]
183 unsafe fn load_partial1_complex(&self, index: usize) -> <f32 as NeonNum>::VectorType {
184 debug_assert!(self.len() >= index + 1);
185 let temp = vmovq_n_f32(0.0);
186 vreinterpretq_f32_u64(vld1q_lane_u64::<0>(
187 self.as_ptr().add(index) as *const u64,
188 vreinterpretq_u64_f32(temp),
189 ))
190 }
191
192 #[inline(always)]
193 unsafe fn load1_complex(&self, index: usize) -> <f32 as NeonNum>::VectorType {
194 debug_assert!(self.len() >= index + 1);
195 vreinterpretq_f32_u64(vld1q_dup_u64(self.as_ptr().add(index) as *const u64))
196 }
197 }
198
199 impl NeonArray<f64> for &[Complex<f64>] {
200 #[inline(always)]
201 unsafe fn load_complex(&self, index: usize) -> <f64 as NeonNum>::VectorType {
202 debug_assert!(self.len() >= index + <f64 as NeonNum>::COMPLEX_PER_VECTOR);
203 vld1q_f64(self.as_ptr().add(index) as *const f64)
204 }
205
206 #[inline(always)]
207 unsafe fn load_partial1_complex(&self, _index: usize) -> <f64 as NeonNum>::VectorType {
208 unimplemented!("Impossible to do a partial load of complex f64's");
209 }
210
211 #[inline(always)]
212 unsafe fn load1_complex(&self, _index: usize) -> <f64 as NeonNum>::VectorType {
213 unimplemented!("Impossible to do a partial load of complex f64's");
214 }
215 }
216
217 impl NeonArray<f64> for &mut [Complex<f64>] {
218 #[inline(always)]
219 unsafe fn load_complex(&self, index: usize) -> <f64 as NeonNum>::VectorType {
220 debug_assert!(self.len() >= index + <f64 as NeonNum>::COMPLEX_PER_VECTOR);
221 vld1q_f64(self.as_ptr().add(index) as *const f64)
222 }
223
224 #[inline(always)]
225 unsafe fn load_partial1_complex(&self, _index: usize) -> <f64 as NeonNum>::VectorType {
226 unimplemented!("Impossible to do a partial load of complex f64's");
227 }
228
229 #[inline(always)]
230 unsafe fn load1_complex(&self, _index: usize) -> <f64 as NeonNum>::VectorType {
231 unimplemented!("Impossible to do a partial load of complex f64's");
232 }
233 }
234
235 impl<'a, T: NeonNum> NeonArray<T> for DoubleBuf<'a, T>
236 where
237 &'a [Complex<T>]: NeonArray<T>,
238 {
239 #[inline(always)]
240 unsafe fn load_complex(&self, index: usize) -> T::VectorType {
241 self.input.load_complex(index)
242 }
243 #[inline(always)]
244 unsafe fn load_partial1_complex(&self, index: usize) -> T::VectorType {
245 self.input.load_partial1_complex(index)
246 }
247 #[inline(always)]
248 unsafe fn load1_complex(&self, index: usize) -> T::VectorType {
249 self.input.load1_complex(index)
250 }
251 }
252
253 // A trait to handle writing to an array of complex floats from Neon vectors.
254 // Neon works with 128-bit vectors, meaning a vector can hold two complex f32,
255 // or a single complex f64.
256 pub trait NeonArrayMut<T: NeonNum>: NeonArray<T> + DerefMut {
257 // Store all complex numbers from a Neon vector to the array.
258 unsafe fn store_complex(&mut self, vector: T::VectorType, index: usize);
259 // Store the low complex number from a Neon vector to the array.
260 unsafe fn store_partial_lo_complex(&mut self, vector: T::VectorType, index: usize);
261 // Store the high complex number from a Neon vector to the array.
262 unsafe fn store_partial_hi_complex(&mut self, vector: T::VectorType, index: usize);
263 }
264
265 impl NeonArrayMut<f32> for &mut [Complex<f32>] {
266 #[inline(always)]
267 unsafe fn store_complex(&mut self, vector: <f32 as NeonNum>::VectorType, index: usize) {
268 debug_assert!(self.len() >= index + <f32 as NeonNum>::COMPLEX_PER_VECTOR);
269 vst1q_f32(self.as_mut_ptr().add(index) as *mut f32, vector);
270 }
271
272 #[inline(always)]
273 unsafe fn store_partial_hi_complex(
274 &mut self,
275 vector: <f32 as NeonNum>::VectorType,
276 index: usize,
277 ) {
278 debug_assert!(self.len() >= index + 1);
279 let high = vget_high_f32(vector);
280 vst1_f32(self.as_mut_ptr().add(index) as *mut f32, high);
281 }
282
283 #[inline(always)]
284 unsafe fn store_partial_lo_complex(
285 &mut self,
286 vector: <f32 as NeonNum>::VectorType,
287 index: usize,
288 ) {
289 debug_assert!(self.len() >= index + 1);
290 let low = vget_low_f32(vector);
291 vst1_f32(self.as_mut_ptr().add(index) as *mut f32, low);
292 }
293 }
294
295 impl NeonArrayMut<f64> for &mut [Complex<f64>] {
296 #[inline(always)]
297 unsafe fn store_complex(&mut self, vector: <f64 as NeonNum>::VectorType, index: usize) {
298 debug_assert!(self.len() >= index + <f64 as NeonNum>::COMPLEX_PER_VECTOR);
299 vst1q_f64(self.as_mut_ptr().add(index) as *mut f64, vector);
300 }
301
302 #[inline(always)]
303 unsafe fn store_partial_hi_complex(
304 &mut self,
305 _vector: <f64 as NeonNum>::VectorType,
306 _index: usize,
307 ) {
308 unimplemented!("Impossible to do a partial store of complex f64's");
309 }
310 #[inline(always)]
311 unsafe fn store_partial_lo_complex(
312 &mut self,
313 _vector: <f64 as NeonNum>::VectorType,
314 _index: usize,
315 ) {
316 unimplemented!("Impossible to do a partial store of complex f64's");
317 }
318 }
319
320 impl<'a, T: NeonNum> NeonArrayMut<T> for DoubleBuf<'a, T>
321 where
322 Self: NeonArray<T>,
323 &'a mut [Complex<T>]: NeonArrayMut<T>,
324 {
325 #[inline(always)]
326 unsafe fn store_complex(&mut self, vector: T::VectorType, index: usize) {
327 self.output.store_complex(vector, index);
328 }
329 #[inline(always)]
330 unsafe fn store_partial_hi_complex(&mut self, vector: T::VectorType, index: usize) {
331 self.output.store_partial_hi_complex(vector, index);
332 }
333 #[inline(always)]
334 unsafe fn store_partial_lo_complex(&mut self, vector: T::VectorType, index: usize) {
335 self.output.store_partial_lo_complex(vector, index);
336 }
337 }
338
339 #[cfg(test)]
340 mod unit_tests {
341 use super::*;
342
343 use num_complex::Complex;
344
345 #[test]
346 fn test_load_f64() {
347 unsafe {
348 let val1: Complex<f64> = Complex::new(1.0, 2.0);
349 let val2: Complex<f64> = Complex::new(3.0, 4.0);
350 let val3: Complex<f64> = Complex::new(5.0, 6.0);
351 let val4: Complex<f64> = Complex::new(7.0, 8.0);
352 let values = vec![val1, val2, val3, val4];
353 let slice = values.as_slice();
354 let load1 = slice.load_complex(0);
355 let load2 = slice.load_complex(1);
356 let load3 = slice.load_complex(2);
357 let load4 = slice.load_complex(3);
358 assert_eq!(
359 val1,
360 std::mem::transmute::<float64x2_t, Complex<f64>>(load1)
361 );
362 assert_eq!(
363 val2,
364 std::mem::transmute::<float64x2_t, Complex<f64>>(load2)
365 );
366 assert_eq!(
367 val3,
368 std::mem::transmute::<float64x2_t, Complex<f64>>(load3)
369 );
370 assert_eq!(
371 val4,
372 std::mem::transmute::<float64x2_t, Complex<f64>>(load4)
373 );
374 }
375 }
376
377 #[test]
378 fn test_store_f64() {
379 unsafe {
380 let val1: Complex<f64> = Complex::new(1.0, 2.0);
381 let val2: Complex<f64> = Complex::new(3.0, 4.0);
382 let val3: Complex<f64> = Complex::new(5.0, 6.0);
383 let val4: Complex<f64> = Complex::new(7.0, 8.0);
384
385 let nbr1 = vld1q_f64(&val1 as *const _ as *const f64);
386 let nbr2 = vld1q_f64(&val2 as *const _ as *const f64);
387 let nbr3 = vld1q_f64(&val3 as *const _ as *const f64);
388 let nbr4 = vld1q_f64(&val4 as *const _ as *const f64);
389
390 let mut values: Vec<Complex<f64>> = vec![Complex::new(0.0, 0.0); 4];
391 let mut slice = values.as_mut_slice();
392 slice.store_complex(nbr1, 0);
393 slice.store_complex(nbr2, 1);
394 slice.store_complex(nbr3, 2);
395 slice.store_complex(nbr4, 3);
396 assert_eq!(val1, values[0]);
397 assert_eq!(val2, values[1]);
398 assert_eq!(val3, values[2]);
399 assert_eq!(val4, values[3]);
400 }
401 }
402 }