Debian Codegraph @

.cargo/vendor/rustfft/src/sse/sse_vector.rs

raw
1 use core::arch::x86_64::*;
2 use num_complex::Complex;
3 use std::ops::{Deref, DerefMut};
4
5 use crate::array_utils::DoubleBuf;
6
7 // Read these indexes from an SseArray and build an array of simd vectors.
8 // Takes a name of a vector to read from, and a list of indexes to read.
9 // This statement:
10 // ```
11 // let values = read_complex_to_array!(input, {0, 1, 2, 3});
12 // ```
13 // is equivalent to:
14 // ```
15 // let values = [
16 // input.load_complex(0),
17 // input.load_complex(1),
18 // input.load_complex(2),
19 // input.load_complex(3),
20 // ];
21 // ```
22 macro_rules! read_complex_to_array {
23 ($input:ident, { $($idx:literal),* }) => {
24 [
25 $(
26 $input.load_complex($idx),
27 )*
28 ]
29 }
30 }
31
32 // Read these indexes from an SseArray and build an array or partially filled simd vectors.
33 // Takes a name of a vector to read from, and a list of indexes to read.
34 // This statement:
35 // ```
36 // let values = read_partial1_complex_to_array!(input, {0, 1, 2, 3});
37 // ```
38 // is equivalent to:
39 // ```
40 // let values = [
41 // input.load1_complex(0),
42 // input.load1_complex(1),
43 // input.load1_complex(2),
44 // input.load1_complex(3),
45 // ];
46 // ```
47 macro_rules! read_partial1_complex_to_array {
48 ($input:ident, { $($idx:literal),* }) => {
49 [
50 $(
51 $input.load1_complex($idx),
52 )*
53 ]
54 }
55 }
56
57 // Write these indexes of an array of simd vectors to the same indexes of an SseArray.
58 // Takes a name of a vector to read from, one to write to, and a list of indexes.
59 // This statement:
60 // ```
61 // let values = write_complex_to_array!(input, output, {0, 1, 2, 3});
62 // ```
63 // is equivalent to:
64 // ```
65 // let values = [
66 // output.store_complex(input[0], 0),
67 // output.store_complex(input[1], 1),
68 // output.store_complex(input[2], 2),
69 // output.store_complex(input[3], 3),
70 // ];
71 // ```
72 macro_rules! write_complex_to_array {
73 ($input:ident, $output:ident, { $($idx:literal),* }) => {
74 $(
75 $output.store_complex($input[$idx], $idx);
76 )*
77 }
78 }
79
80 // Write the low half of these indexes of an array of simd vectors to the same indexes of an SseArray.
81 // Takes a name of a vector to read from, one to write to, and a list of indexes.
82 // This statement:
83 // ```
84 // let values = write_partial_lo_complex_to_array!(input, output, {0, 1, 2, 3});
85 // ```
86 // is equivalent to:
87 // ```
88 // let values = [
89 // output.store_partial_lo_complex(input[0], 0),
90 // output.store_partial_lo_complex(input[1], 1),
91 // output.store_partial_lo_complex(input[2], 2),
92 // output.store_partial_lo_complex(input[3], 3),
93 // ];
94 // ```
95 macro_rules! write_partial_lo_complex_to_array {
96 ($input:ident, $output:ident, { $($idx:literal),* }) => {
97 $(
98 $output.store_partial_lo_complex($input[$idx], $idx);
99 )*
100 }
101 }
102
103 // Write these indexes of an array of simd vectors to the same indexes, multiplied by a stride, of an SseArray.
104 // Takes a name of a vector to read from, one to write to, an integer stride, and a list of indexes.
105 // This statement:
106 // ```
107 // let values = write_complex_to_array_separate!(input, output, {0, 1, 2, 3});
108 // ```
109 // is equivalent to:
110 // ```
111 // let values = [
112 // output.store_complex(input[0], 0),
113 // output.store_complex(input[1], 2),
114 // output.store_complex(input[2], 4),
115 // output.store_complex(input[3], 6),
116 // ];
117 // ```
118 macro_rules! write_complex_to_array_strided {
119 ($input:ident, $output:ident, $stride:literal, { $($idx:literal),* }) => {
120 $(
121 $output.store_complex($input[$idx], $idx*$stride);
122 )*
123 }
124 }
125
126 // A trait to hold the BVectorType and COMPLEX_PER_VECTOR associated data
127
128 pub trait SseNum {
129 type VectorType;
130 const COMPLEX_PER_VECTOR: usize;
131 }
132 impl SseNum for f32 {
133 type VectorType = __m128;
134 const COMPLEX_PER_VECTOR: usize = 2;
135 }
136 impl SseNum for f64 {
137 type VectorType = __m128d;
138 const COMPLEX_PER_VECTOR: usize = 1;
139 }
140
141 // A trait to handle reading from an array of complex floats into SSE vectors.
142 // SSE works with 128-bit vectors, meaning a vector can hold two complex f32,
143 // or a single complex f64.
144 pub trait SseArray<T: SseNum>: Deref {
145 // Load complex numbers from the array to fill a SSE vector.
146 unsafe fn load_complex(&self, index: usize) -> T::VectorType;
147 // Load a single complex number from the array into a SSE vector, setting the unused elements to zero.
148 unsafe fn load_partial1_complex(&self, index: usize) -> T::VectorType;
149 // Load a single complex number from the array, and copy it to all elements of a SSE vector.
150 unsafe fn load1_complex(&self, index: usize) -> T::VectorType;
151 }
152
153 impl SseArray<f32> for &[Complex<f32>] {
154 #[inline(always)]
155 unsafe fn load_complex(&self, index: usize) -> <f32 as SseNum>::VectorType {
156 debug_assert!(self.len() >= index + <f32 as SseNum>::COMPLEX_PER_VECTOR);
157 _mm_loadu_ps(self.as_ptr().add(index) as *const f32)
158 }
159
160 #[inline(always)]
161 unsafe fn load_partial1_complex(&self, index: usize) -> <f32 as SseNum>::VectorType {
162 debug_assert!(self.len() >= index + 1);
163 _mm_castpd_ps(_mm_load_sd(self.as_ptr().add(index) as *const f64))
164 }
165
166 #[inline(always)]
167 unsafe fn load1_complex(&self, index: usize) -> <f32 as SseNum>::VectorType {
168 debug_assert!(self.len() >= index + 1);
169 _mm_castpd_ps(_mm_load1_pd(self.as_ptr().add(index) as *const f64))
170 }
171 }
172 impl SseArray<f32> for &mut [Complex<f32>] {
173 #[inline(always)]
174 unsafe fn load_complex(&self, index: usize) -> <f32 as SseNum>::VectorType {
175 debug_assert!(self.len() >= index + <f32 as SseNum>::COMPLEX_PER_VECTOR);
176 _mm_loadu_ps(self.as_ptr().add(index) as *const f32)
177 }
178
179 #[inline(always)]
180 unsafe fn load_partial1_complex(&self, index: usize) -> <f32 as SseNum>::VectorType {
181 debug_assert!(self.len() >= index + 1);
182 _mm_castpd_ps(_mm_load_sd(self.as_ptr().add(index) as *const f64))
183 }
184
185 #[inline(always)]
186 unsafe fn load1_complex(&self, index: usize) -> <f32 as SseNum>::VectorType {
187 debug_assert!(self.len() >= index + 1);
188 _mm_castpd_ps(_mm_load1_pd(self.as_ptr().add(index) as *const f64))
189 }
190 }
191
192 impl SseArray<f64> for &[Complex<f64>] {
193 #[inline(always)]
194 unsafe fn load_complex(&self, index: usize) -> <f64 as SseNum>::VectorType {
195 debug_assert!(self.len() >= index + <f64 as SseNum>::COMPLEX_PER_VECTOR);
196 _mm_loadu_pd(self.as_ptr().add(index) as *const f64)
197 }
198
199 #[inline(always)]
200 unsafe fn load_partial1_complex(&self, _index: usize) -> <f64 as SseNum>::VectorType {
201 unimplemented!("Impossible to do a partial load of complex f64's");
202 }
203
204 #[inline(always)]
205 unsafe fn load1_complex(&self, _index: usize) -> <f64 as SseNum>::VectorType {
206 unimplemented!("Impossible to do a partial load of complex f64's");
207 }
208 }
209 impl SseArray<f64> for &mut [Complex<f64>] {
210 #[inline(always)]
211 unsafe fn load_complex(&self, index: usize) -> <f64 as SseNum>::VectorType {
212 debug_assert!(self.len() >= index + <f64 as SseNum>::COMPLEX_PER_VECTOR);
213 _mm_loadu_pd(self.as_ptr().add(index) as *const f64)
214 }
215
216 #[inline(always)]
217 unsafe fn load_partial1_complex(&self, _index: usize) -> <f64 as SseNum>::VectorType {
218 unimplemented!("Impossible to do a partial load of complex f64's");
219 }
220
221 #[inline(always)]
222 unsafe fn load1_complex(&self, _index: usize) -> <f64 as SseNum>::VectorType {
223 unimplemented!("Impossible to do a partial load of complex f64's");
224 }
225 }
226
227 impl<'a, T: SseNum> SseArray<T> for DoubleBuf<'a, T>
228 where
229 &'a [Complex<T>]: SseArray<T>,
230 {
231 #[inline(always)]
232 unsafe fn load_complex(&self, index: usize) -> T::VectorType {
233 self.input.load_complex(index)
234 }
235 #[inline(always)]
236 unsafe fn load_partial1_complex(&self, index: usize) -> T::VectorType {
237 self.input.load_partial1_complex(index)
238 }
239 #[inline(always)]
240 unsafe fn load1_complex(&self, index: usize) -> T::VectorType {
241 self.input.load1_complex(index)
242 }
243 }
244
245 // A trait to handle writing to an array of complex floats from SSE vectors.
246 // SSE works with 128-bit vectors, meaning a vector can hold two complex f32,
247 // or a single complex f64.
248 pub trait SseArrayMut<T: SseNum>: SseArray<T> + DerefMut {
249 // Store all complex numbers from a SSE vector to the array.
250 unsafe fn store_complex(&mut self, vector: T::VectorType, index: usize);
251 // Store the low complex number from a SSE vector to the array.
252 unsafe fn store_partial_lo_complex(&mut self, vector: T::VectorType, index: usize);
253 // Store the high complex number from a SSE vector to the array.
254 unsafe fn store_partial_hi_complex(&mut self, vector: T::VectorType, index: usize);
255 }
256
257 impl SseArrayMut<f32> for &mut [Complex<f32>] {
258 #[inline(always)]
259 unsafe fn store_complex(&mut self, vector: <f32 as SseNum>::VectorType, index: usize) {
260 debug_assert!(self.len() >= index + <f32 as SseNum>::COMPLEX_PER_VECTOR);
261 _mm_storeu_ps(self.as_mut_ptr().add(index) as *mut f32, vector);
262 }
263
264 #[inline(always)]
265 unsafe fn store_partial_hi_complex(
266 &mut self,
267 vector: <f32 as SseNum>::VectorType,
268 index: usize,
269 ) {
270 debug_assert!(self.len() >= index + 1);
271 _mm_storeh_pd(
272 self.as_mut_ptr().add(index) as *mut f64,
273 _mm_castps_pd(vector),
274 );
275 }
276 #[inline(always)]
277 unsafe fn store_partial_lo_complex(
278 &mut self,
279 vector: <f32 as SseNum>::VectorType,
280 index: usize,
281 ) {
282 debug_assert!(self.len() >= index + 1);
283 _mm_storel_pd(
284 self.as_mut_ptr().add(index) as *mut f64,
285 _mm_castps_pd(vector),
286 );
287 }
288 }
289
290 impl SseArrayMut<f64> for &mut [Complex<f64>] {
291 #[inline(always)]
292 unsafe fn store_complex(&mut self, vector: <f64 as SseNum>::VectorType, index: usize) {
293 debug_assert!(self.len() >= index + <f64 as SseNum>::COMPLEX_PER_VECTOR);
294 _mm_storeu_pd(self.as_mut_ptr().add(index) as *mut f64, vector);
295 }
296
297 #[inline(always)]
298 unsafe fn store_partial_hi_complex(
299 &mut self,
300 _vector: <f64 as SseNum>::VectorType,
301 _index: usize,
302 ) {
303 unimplemented!("Impossible to do a partial store of complex f64's");
304 }
305 #[inline(always)]
306 unsafe fn store_partial_lo_complex(
307 &mut self,
308 _vector: <f64 as SseNum>::VectorType,
309 _index: usize,
310 ) {
311 unimplemented!("Impossible to do a partial store of complex f64's");
312 }
313 }
314
315 impl<'a, T: SseNum> SseArrayMut<T> for DoubleBuf<'a, T>
316 where
317 Self: SseArray<T>,
318 &'a mut [Complex<T>]: SseArrayMut<T>,
319 {
320 #[inline(always)]
321 unsafe fn store_complex(&mut self, vector: T::VectorType, index: usize) {
322 self.output.store_complex(vector, index);
323 }
324 #[inline(always)]
325 unsafe fn store_partial_lo_complex(&mut self, vector: T::VectorType, index: usize) {
326 self.output.store_partial_lo_complex(vector, index);
327 }
328 #[inline(always)]
329 unsafe fn store_partial_hi_complex(&mut self, vector: T::VectorType, index: usize) {
330 self.output.store_partial_hi_complex(vector, index);
331 }
332 }