Debian Codegraph @

.cargo/vendor/rustfft/src/neon/neon_utils.rs

raw
1 use core::arch::aarch64::*;
2
3 // __ __ _ _ _________ _ _ _
4 // | \/ | __ _| |_| |__ |___ /___ \| |__ (_) |_
5 // | |\/| |/ _` | __| '_ \ _____ |_ \ __) | '_ \| | __|
6 // | | | | (_| | |_| | | | |_____| ___) / __/| |_) | | |_
7 // |_| |_|\__,_|\__|_| |_| |____/_____|_.__/|_|\__|
8 //
9
10 pub struct Rotate90F32 {
11 //sign_lo: float32x4_t,
12 sign_hi: float32x2_t,
13 sign_both: float32x4_t,
14 }
15
16 impl Rotate90F32 {
17 pub fn new(positive: bool) -> Self {
18 // There doesn't seem to be any need for rotating just the first element, but let's keep the code just in case
19 //let sign_lo = unsafe {
20 // if positive {
21 // _mm_set_ps(0.0, 0.0, 0.0, -0.0)
22 // }
23 // else {
24 // _mm_set_ps(0.0, 0.0, -0.0, 0.0)
25 // }
26 //};
27 let sign_hi = unsafe {
28 if positive {
29 vld1_f32([-0.0, 0.0].as_ptr())
30 } else {
31 vld1_f32([0.0, -0.0].as_ptr())
32 }
33 };
34 let sign_both = unsafe {
35 if positive {
36 vld1q_f32([-0.0, 0.0, -0.0, 0.0].as_ptr())
37 } else {
38 vld1q_f32([0.0, -0.0, 0.0, -0.0].as_ptr())
39 }
40 };
41 Self {
42 //sign_lo,
43 sign_hi,
44 sign_both,
45 }
46 }
47
48 #[inline(always)]
49 pub unsafe fn rotate_hi(&self, values: float32x4_t) -> float32x4_t {
50 vcombine_f32(
51 vget_low_f32(values),
52 vreinterpret_f32_u32(veor_u32(
53 vrev64_u32(vreinterpret_u32_f32(vget_high_f32(values))),
54 vreinterpret_u32_f32(self.sign_hi),
55 )),
56 )
57 }
58
59 // There doesn't seem to be any need for rotating just the first element, but let's keep the code just in case
60 //#[inline(always)]
61 //pub unsafe fn rotate_lo(&self, values: __m128) -> __m128 {
62 // let temp = _mm_shuffle_ps(values, values, 0xE1);
63 // _mm_xor_ps(temp, self.sign_lo)
64 //}
65
66 #[inline(always)]
67 pub unsafe fn rotate_both(&self, values: float32x4_t) -> float32x4_t {
68 let temp = vrev64q_f32(values);
69 vreinterpretq_f32_u32(veorq_u32(
70 vreinterpretq_u32_f32(temp),
71 vreinterpretq_u32_f32(self.sign_both),
72 ))
73 }
74 }
75
76 // Pack low (1st) complex
77 // left: r1.re, r1.im, r2.re, r2.im
78 // right: l1.re, l1.im, l2.re, l2.im
79 // --> r1.re, r1.im, l1.re, l1.im
80 #[inline(always)]
81 pub unsafe fn extract_lo_lo_f32(left: float32x4_t, right: float32x4_t) -> float32x4_t {
82 //_mm_shuffle_ps(left, right, 0x44)
83 vreinterpretq_f32_f64(vtrn1q_f64(
84 vreinterpretq_f64_f32(left),
85 vreinterpretq_f64_f32(right),
86 ))
87 }
88
89 // Pack high (2nd) complex
90 // left: r1.re, r1.im, r2.re, r2.im
91 // right: l1.re, l1.im, l2.re, l2.im
92 // --> r2.re, r2.im, l2.re, l2.im
93 #[inline(always)]
94 pub unsafe fn extract_hi_hi_f32(left: float32x4_t, right: float32x4_t) -> float32x4_t {
95 vreinterpretq_f32_f64(vtrn2q_f64(
96 vreinterpretq_f64_f32(left),
97 vreinterpretq_f64_f32(right),
98 ))
99 }
100
101 // Pack low (1st) and high (2nd) complex
102 // left: r1.re, r1.im, r2.re, r2.im
103 // right: l1.re, l1.im, l2.re, l2.im
104 // --> r1.re, r1.im, l2.re, l2.im
105 #[inline(always)]
106 pub unsafe fn extract_lo_hi_f32(left: float32x4_t, right: float32x4_t) -> float32x4_t {
107 vcombine_f32(vget_low_f32(left), vget_high_f32(right))
108 }
109
110 // Pack high (2nd) and low (1st) complex
111 // left: r1.re, r1.im, r2.re, r2.im
112 // right: l1.re, l1.im, l2.re, l2.im
113 // --> r2.re, r2.im, l1.re, l1.im
114 #[inline(always)]
115 pub unsafe fn extract_hi_lo_f32(left: float32x4_t, right: float32x4_t) -> float32x4_t {
116 vcombine_f32(vget_high_f32(left), vget_low_f32(right))
117 }
118
119 // Reverse complex
120 // values: a.re, a.im, b.re, b.im
121 // --> b.re, b.im, a.re, a.im
122 #[inline(always)]
123 pub unsafe fn reverse_complex_elements_f32(values: float32x4_t) -> float32x4_t {
124 vcombine_f32(vget_high_f32(values), vget_low_f32(values))
125 }
126
127 // Reverse complex and then negate hi complex
128 // values: a.re, a.im, b.re, b.im
129 // --> b.re, b.im, -a.re, -a.im
130 #[inline(always)]
131 pub unsafe fn reverse_complex_and_negate_hi_f32(values: float32x4_t) -> float32x4_t {
132 vcombine_f32(vget_high_f32(values), vneg_f32(vget_low_f32(values)))
133 }
134
135 // Invert sign of high (2nd) complex
136 // values: a.re, a.im, b.re, b.im
137 // --> a.re, a.im, -b.re, -b.im
138 //#[inline(always)]
139 //pub unsafe fn negate_hi_f32(values: float32x4_t) -> float32x4_t {
140 // vcombine_f32(vget_low_f32(values), vneg_f32(vget_high_f32(values)))
141 //}
142
143 // Duplicate low (1st) complex
144 // values: a.re, a.im, b.re, b.im
145 // --> a.re, a.im, a.re, a.im
146 #[inline(always)]
147 pub unsafe fn duplicate_lo_f32(values: float32x4_t) -> float32x4_t {
148 vreinterpretq_f32_f64(vtrn1q_f64(
149 vreinterpretq_f64_f32(values),
150 vreinterpretq_f64_f32(values),
151 ))
152 }
153
154 // Duplicate high (2nd) complex
155 // values: a.re, a.im, b.re, b.im
156 // --> b.re, b.im, b.re, b.im
157 #[inline(always)]
158 pub unsafe fn duplicate_hi_f32(values: float32x4_t) -> float32x4_t {
159 vreinterpretq_f32_f64(vtrn2q_f64(
160 vreinterpretq_f64_f32(values),
161 vreinterpretq_f64_f32(values),
162 ))
163 }
164
165 // transpose a 2x2 complex matrix given as [x0, x1], [x2, x3]
166 // result is [x0, x2], [x1, x3]
167 #[inline(always)]
168 pub unsafe fn transpose_complex_2x2_f32(left: float32x4_t, right: float32x4_t) -> [float32x4_t; 2] {
169 let temp02 = extract_lo_lo_f32(left, right);
170 let temp13 = extract_hi_hi_f32(left, right);
171 [temp02, temp13]
172 }
173
174 // Complex multiplication.
175 // Each input contains two complex values, which are multiplied in parallel.
176 #[inline(always)]
177 pub unsafe fn mul_complex_f32(left: float32x4_t, right: float32x4_t) -> float32x4_t {
178 // ARMv8.2-A introduced vcmulq_f32 and vcmlaq_f32 for complex multiplication, these intrinsics are not yet available.
179 let temp1 = vtrn1q_f32(right, right);
180 let temp2 = vtrn2q_f32(right, vnegq_f32(right));
181 let temp3 = vmulq_f32(temp2, left);
182 let temp4 = vrev64q_f32(temp3);
183 vfmaq_f32(temp4, temp1, left)
184 }
185
186 // __ __ _ _ __ _ _ _ _ _
187 // | \/ | __ _| |_| |__ / /_ | || | | |__ (_) |_
188 // | |\/| |/ _` | __| '_ \ _____ | '_ \| || |_| '_ \| | __|
189 // | | | | (_| | |_| | | | |_____| | (_) |__ _| |_) | | |_
190 // |_| |_|\__,_|\__|_| |_| \___/ |_| |_.__/|_|\__|
191 //
192
193 pub(crate) struct Rotate90F64 {
194 sign: float64x2_t,
195 }
196
197 impl Rotate90F64 {
198 pub fn new(positive: bool) -> Self {
199 let sign = unsafe {
200 if positive {
201 vld1q_f64([-0.0, 0.0].as_ptr())
202 } else {
203 vld1q_f64([0.0, -0.0].as_ptr())
204 }
205 };
206 Self { sign }
207 }
208
209 #[inline(always)]
210 pub unsafe fn rotate(&self, values: float64x2_t) -> float64x2_t {
211 let temp = vcombine_f64(vget_high_f64(values), vget_low_f64(values));
212 vreinterpretq_f64_u64(veorq_u64(
213 vreinterpretq_u64_f64(temp),
214 vreinterpretq_u64_f64(self.sign),
215 ))
216 }
217 }
218
219 #[inline(always)]
220 pub unsafe fn mul_complex_f64(left: float64x2_t, right: float64x2_t) -> float64x2_t {
221 // ARMv8.2-A introduced vcmulq_f64 and vcmlaq_f64 for complex multiplication, these intrinsics are not yet available.
222 let temp = vcombine_f64(vneg_f64(vget_high_f64(left)), vget_low_f64(left));
223 let sum = vmulq_laneq_f64::<0>(left, right);
224 vfmaq_laneq_f64::<1>(sum, temp, right)
225 }
226
227 #[cfg(test)]
228 mod unit_tests {
229 use super::*;
230 use num_complex::Complex;
231
232 #[test]
233 fn test_mul_complex_f64() {
234 unsafe {
235 let right = vld1q_f64([1.0, 2.0].as_ptr());
236 let left = vld1q_f64([5.0, 7.0].as_ptr());
237 let res = mul_complex_f64(left, right);
238 let expected = vld1q_f64([1.0 * 5.0 - 2.0 * 7.0, 1.0 * 7.0 + 2.0 * 5.0].as_ptr());
239 assert_eq!(
240 std::mem::transmute::<float64x2_t, Complex<f64>>(res),
241 std::mem::transmute::<float64x2_t, Complex<f64>>(expected)
242 );
243 }
244 }
245
246 #[test]
247 fn test_mul_complex_f32() {
248 unsafe {
249 let val1 = Complex::<f32>::new(1.0, 2.5);
250 let val2 = Complex::<f32>::new(3.2, 4.75);
251 let val3 = Complex::<f32>::new(5.75, 6.25);
252 let val4 = Complex::<f32>::new(7.4, 8.5);
253
254 let nbr2 = vld1q_f32([val3, val4].as_ptr() as *const f32);
255 let nbr1 = vld1q_f32([val1, val2].as_ptr() as *const f32);
256 let res = mul_complex_f32(nbr1, nbr2);
257 let res = std::mem::transmute::<float32x4_t, [Complex<f32>; 2]>(res);
258 let expected = [val1 * val3, val2 * val4];
259 assert_eq!(res, expected);
260 }
261 }
262
263 #[test]
264 fn test_pack() {
265 unsafe {
266 let nbr2 = vld1q_f32([5.0, 6.0, 7.0, 8.0].as_ptr());
267 let nbr1 = vld1q_f32([1.0, 2.0, 3.0, 4.0].as_ptr());
268 let first = extract_lo_lo_f32(nbr1, nbr2);
269 let second = extract_hi_hi_f32(nbr1, nbr2);
270 let first = std::mem::transmute::<float32x4_t, [Complex<f32>; 2]>(first);
271 let second = std::mem::transmute::<float32x4_t, [Complex<f32>; 2]>(second);
272 let first_expected = [Complex::new(1.0, 2.0), Complex::new(5.0, 6.0)];
273 let second_expected = [Complex::new(3.0, 4.0), Complex::new(7.0, 8.0)];
274 assert_eq!(first, first_expected);
275 assert_eq!(second, second_expected);
276 }
277 }
278 }