Debian Codegraph @

.cargo/vendor/rustfft/src/sse/sse_utils.rs

raw
1 use core::arch::x86_64::*;
2
3 // __ __ _ _ _________ _ _ _
4 // | \/ | __ _| |_| |__ |___ /___ \| |__ (_) |_
5 // | |\/| |/ _` | __| '_ \ _____ |_ \ __) | '_ \| | __|
6 // | | | | (_| | |_| | | | |_____| ___) / __/| |_) | | |_
7 // |_| |_|\__,_|\__|_| |_| |____/_____|_.__/|_|\__|
8 //
9
10 pub struct Rotate90F32 {
11 //sign_lo: __m128,
12 sign_hi: __m128,
13 sign_both: __m128,
14 }
15
16 impl Rotate90F32 {
17 pub fn new(positive: bool) -> Self {
18 // There doesn't seem to be any need for rotating just the first element, but let's keep the code just in case
19 //let sign_lo = unsafe {
20 // if positive {
21 // _mm_set_ps(0.0, 0.0, 0.0, -0.0)
22 // }
23 // else {
24 // _mm_set_ps(0.0, 0.0, -0.0, 0.0)
25 // }
26 //};
27 let sign_hi = unsafe {
28 if positive {
29 _mm_set_ps(0.0, -0.0, 0.0, 0.0)
30 } else {
31 _mm_set_ps(-0.0, 0.0, 0.0, 0.0)
32 }
33 };
34 let sign_both = unsafe {
35 if positive {
36 _mm_set_ps(0.0, -0.0, 0.0, -0.0)
37 } else {
38 _mm_set_ps(-0.0, 0.0, -0.0, 0.0)
39 }
40 };
41 Self {
42 //sign_lo,
43 sign_hi,
44 sign_both,
45 }
46 }
47
48 #[inline(always)]
49 pub unsafe fn rotate_hi(&self, values: __m128) -> __m128 {
50 let temp = _mm_shuffle_ps(values, values, 0xB4);
51 _mm_xor_ps(temp, self.sign_hi)
52 }
53
54 // There doesn't seem to be any need for rotating just the first element, but let's keep the code just in case
55 //#[inline(always)]
56 //pub unsafe fn rotate_lo(&self, values: __m128) -> __m128 {
57 // let temp = _mm_shuffle_ps(values, values, 0xE1);
58 // _mm_xor_ps(temp, self.sign_lo)
59 //}
60
61 #[inline(always)]
62 pub unsafe fn rotate_both(&self, values: __m128) -> __m128 {
63 let temp = _mm_shuffle_ps(values, values, 0xB1);
64 _mm_xor_ps(temp, self.sign_both)
65 }
66 }
67
68 // Pack low (1st) complex
69 // left: r1.re, r1.im, r2.re, r2.im
70 // right: l1.re, l1.im, l2.re, l2.im
71 // --> r1.re, r1.im, l1.re, l1.im
72 #[inline(always)]
73 pub unsafe fn extract_lo_lo_f32(left: __m128, right: __m128) -> __m128 {
74 //_mm_shuffle_ps(left, right, 0x44)
75 _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(left), _mm_castps_pd(right)))
76 }
77
78 // Pack high (2nd) complex
79 // left: r1.re, r1.im, r2.re, r2.im
80 // right: l1.re, l1.im, l2.re, l2.im
81 // --> r2.re, r2.im, l2.re, l2.im
82 #[inline(always)]
83 pub unsafe fn extract_hi_hi_f32(left: __m128, right: __m128) -> __m128 {
84 _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(left), _mm_castps_pd(right)))
85 }
86
87 // Pack low (1st) and high (2nd) complex
88 // left: r1.re, r1.im, r2.re, r2.im
89 // right: l1.re, l1.im, l2.re, l2.im
90 // --> r1.re, r1.im, l2.re, l2.im
91 #[inline(always)]
92 pub unsafe fn extract_lo_hi_f32(left: __m128, right: __m128) -> __m128 {
93 _mm_blend_ps(left, right, 0x0C)
94 }
95
96 // Pack high (2nd) and low (1st) complex
97 // left: r1.re, r1.im, r2.re, r2.im
98 // right: l1.re, l1.im, l2.re, l2.im
99 // --> r2.re, r2.im, l1.re, l1.im
100 #[inline(always)]
101 pub unsafe fn extract_hi_lo_f32(left: __m128, right: __m128) -> __m128 {
102 _mm_shuffle_ps(left, right, 0x4E)
103 }
104
105 // Reverse complex
106 // values: a.re, a.im, b.re, b.im
107 // --> b.re, b.im, a.re, a.im
108 #[inline(always)]
109 pub unsafe fn reverse_complex_elements_f32(values: __m128) -> __m128 {
110 _mm_shuffle_ps(values, values, 0x4E)
111 }
112
113 // Invert sign of high (2nd) complex
114 // values: a.re, a.im, b.re, b.im
115 // --> a.re, a.im, -b.re, -b.im
116 #[inline(always)]
117 pub unsafe fn negate_hi_f32(values: __m128) -> __m128 {
118 _mm_xor_ps(values, _mm_set_ps(-0.0, -0.0, 0.0, 0.0))
119 }
120
121 // Duplicate low (1st) complex
122 // values: a.re, a.im, b.re, b.im
123 // --> a.re, a.im, a.re, a.im
124 #[inline(always)]
125 pub unsafe fn duplicate_lo_f32(values: __m128) -> __m128 {
126 _mm_shuffle_ps(values, values, 0x44)
127 }
128
129 // Duplicate high (2nd) complex
130 // values: a.re, a.im, b.re, b.im
131 // --> b.re, b.im, b.re, b.im
132 #[inline(always)]
133 pub unsafe fn duplicate_hi_f32(values: __m128) -> __m128 {
134 _mm_shuffle_ps(values, values, 0xEE)
135 }
136
137 // transpose a 2x2 complex matrix given as [x0, x1], [x2, x3]
138 // result is [x0, x2], [x1, x3]
139 #[inline(always)]
140 pub unsafe fn transpose_complex_2x2_f32(left: __m128, right: __m128) -> [__m128; 2] {
141 let temp02 = extract_lo_lo_f32(left, right);
142 let temp13 = extract_hi_hi_f32(left, right);
143 [temp02, temp13]
144 }
145
146 // Complex multiplication.
147 // Each input contains two complex values, which are multiplied in parallel.
148 #[inline(always)]
149 pub unsafe fn mul_complex_f32(left: __m128, right: __m128) -> __m128 {
150 //SSE3, taken from Intel performance manual
151 let mut temp1 = _mm_shuffle_ps(right, right, 0xA0);
152 let mut temp2 = _mm_shuffle_ps(right, right, 0xF5);
153 temp1 = _mm_mul_ps(temp1, left);
154 temp2 = _mm_mul_ps(temp2, left);
155 temp2 = _mm_shuffle_ps(temp2, temp2, 0xB1);
156 _mm_addsub_ps(temp1, temp2)
157 }
158
159 // __ __ _ _ __ _ _ _ _ _
160 // | \/ | __ _| |_| |__ / /_ | || | | |__ (_) |_
161 // | |\/| |/ _` | __| '_ \ _____ | '_ \| || |_| '_ \| | __|
162 // | | | | (_| | |_| | | | |_____| | (_) |__ _| |_) | | |_
163 // |_| |_|\__,_|\__|_| |_| \___/ |_| |_.__/|_|\__|
164 //
165
166 pub(crate) struct Rotate90F64 {
167 sign: __m128d,
168 }
169
170 impl Rotate90F64 {
171 pub fn new(positive: bool) -> Self {
172 let sign = unsafe {
173 if positive {
174 _mm_set_pd(0.0, -0.0)
175 } else {
176 _mm_set_pd(-0.0, 0.0)
177 }
178 };
179 Self { sign }
180 }
181
182 #[inline(always)]
183 pub unsafe fn rotate(&self, values: __m128d) -> __m128d {
184 let temp = _mm_shuffle_pd(values, values, 0x01);
185 _mm_xor_pd(temp, self.sign)
186 }
187 }
188
189 #[inline(always)]
190 pub unsafe fn mul_complex_f64(left: __m128d, right: __m128d) -> __m128d {
191 // SSE3, taken from Intel performance manual
192 let mut temp1 = _mm_unpacklo_pd(right, right);
193 let mut temp2 = _mm_unpackhi_pd(right, right);
194 temp1 = _mm_mul_pd(temp1, left);
195 temp2 = _mm_mul_pd(temp2, left);
196 temp2 = _mm_shuffle_pd(temp2, temp2, 0x01);
197 _mm_addsub_pd(temp1, temp2)
198 }
199
200 #[cfg(test)]
201 mod unit_tests {
202 use super::*;
203 use num_complex::Complex;
204
205 #[test]
206 fn test_mul_complex_f64() {
207 unsafe {
208 let right = _mm_set_pd(1.0, 2.0);
209 let left = _mm_set_pd(5.0, 7.0);
210 let res = mul_complex_f64(left, right);
211 let expected = _mm_set_pd(2.0 * 5.0 + 1.0 * 7.0, 2.0 * 7.0 - 1.0 * 5.0);
212 assert_eq!(
213 std::mem::transmute::<__m128d, Complex<f64>>(res),
214 std::mem::transmute::<__m128d, Complex<f64>>(expected)
215 );
216 }
217 }
218
219 #[test]
220 fn test_mul_complex_f32() {
221 unsafe {
222 let val1 = Complex::<f32>::new(1.0, 2.5);
223 let val2 = Complex::<f32>::new(3.2, 4.2);
224 let val3 = Complex::<f32>::new(5.6, 6.2);
225 let val4 = Complex::<f32>::new(7.4, 8.3);
226
227 let nbr2 = _mm_set_ps(val4.im, val4.re, val3.im, val3.re);
228 let nbr1 = _mm_set_ps(val2.im, val2.re, val1.im, val1.re);
229 let res = mul_complex_f32(nbr1, nbr2);
230 let res = std::mem::transmute::<__m128, [Complex<f32>; 2]>(res);
231 let expected = [val1 * val3, val2 * val4];
232 assert_eq!(res, expected);
233 }
234 }
235
236 #[test]
237 fn test_pack() {
238 unsafe {
239 let nbr2 = _mm_set_ps(8.0, 7.0, 6.0, 5.0);
240 let nbr1 = _mm_set_ps(4.0, 3.0, 2.0, 1.0);
241 let first = extract_lo_lo_f32(nbr1, nbr2);
242 let second = extract_hi_hi_f32(nbr1, nbr2);
243 let first = std::mem::transmute::<__m128, [Complex<f32>; 2]>(first);
244 let second = std::mem::transmute::<__m128, [Complex<f32>; 2]>(second);
245 let first_expected = [Complex::new(1.0, 2.0), Complex::new(5.0, 6.0)];
246 let second_expected = [Complex::new(3.0, 4.0), Complex::new(7.0, 8.0)];
247 assert_eq!(first, first_expected);
248 assert_eq!(second, second_expected);
249 }
250 }
251 }