Debian Codegraph @

.cargo/vendor/zune-jpeg/src/unsafe_utils_neon.rs

raw
1 /*
2 * Copyright (c) 2023.
3 *
4 * This software is free software;
5 *
6 * You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license
7 */
8
9 #![cfg(all(feature = "neon", target_arch = "aarch64"))]
10 // TODO can this be extended to armv7
11
12 //! This module provides unsafe ways to do some things
13 #![allow(clippy::wildcard_imports)]
14
15 use core::arch::aarch64::*;
16 use core::ops::{Add, AddAssign, BitOr, BitOrAssign, Mul, MulAssign, Sub};
17
18 pub type VecType = int32x4x2_t;
19
20 pub unsafe fn loadu(src: *const i32) -> VecType {
21 vld1q_s32_x2(src as *const _)
22 }
23
24 /// An abstraction of an AVX ymm register that
25 ///allows some things to not look ugly
26 #[derive(Clone, Copy)]
27 pub struct YmmRegister {
28 /// An AVX register
29 pub(crate) mm256: VecType
30 }
31
32 impl YmmRegister {
33 #[inline]
34 pub unsafe fn load(src: *const i32) -> Self {
35 loadu(src).into()
36 }
37
38 #[inline]
39 pub fn map2(self, other: Self, f: impl Fn(int32x4_t, int32x4_t) -> int32x4_t) -> Self {
40 let m0 = f(self.mm256.0, other.mm256.0);
41 let m1 = f(self.mm256.1, other.mm256.1);
42
43 YmmRegister {
44 mm256: int32x4x2_t(m0, m1)
45 }
46 }
47
48 #[inline]
49 pub fn all_zero(self) -> bool {
50 unsafe {
51 let both = vorrq_s32(self.mm256.0, self.mm256.1);
52 let both_unsigned = vreinterpretq_u32_s32(both);
53 0 == vmaxvq_u32(both_unsigned)
54 }
55 }
56
57 #[inline]
58 pub fn const_shl<const N: i32>(self) -> Self {
59 // Ensure that we logically shift left
60 unsafe {
61 let m0 = vreinterpretq_s32_u32(vshlq_n_u32::<N>(vreinterpretq_u32_s32(self.mm256.0)));
62 let m1 = vreinterpretq_s32_u32(vshlq_n_u32::<N>(vreinterpretq_u32_s32(self.mm256.1)));
63
64 YmmRegister {
65 mm256: int32x4x2_t(m0, m1)
66 }
67 }
68 }
69
70 #[inline]
71 pub fn const_shra<const N: i32>(self) -> Self {
72 unsafe {
73 let i0 = vshrq_n_s32::<N>(self.mm256.0);
74 let i1 = vshrq_n_s32::<N>(self.mm256.1);
75
76 YmmRegister {
77 mm256: int32x4x2_t(i0, i1)
78 }
79 }
80 }
81 }
82
83 impl<T> Add<T> for YmmRegister
84 where
85 T: Into<Self>
86 {
87 type Output = YmmRegister;
88
89 #[inline]
90 fn add(self, rhs: T) -> Self::Output {
91 let rhs = rhs.into();
92 unsafe { self.map2(rhs, |a, b| vaddq_s32(a, b)) }
93 }
94 }
95
96 impl<T> Sub<T> for YmmRegister
97 where
98 T: Into<Self>
99 {
100 type Output = YmmRegister;
101
102 #[inline]
103 fn sub(self, rhs: T) -> Self::Output {
104 let rhs = rhs.into();
105 unsafe { self.map2(rhs, |a, b| vsubq_s32(a, b)) }
106 }
107 }
108
109 impl<T> AddAssign<T> for YmmRegister
110 where
111 T: Into<Self>
112 {
113 #[inline]
114 fn add_assign(&mut self, rhs: T) {
115 let rhs: Self = rhs.into();
116 *self = *self + rhs;
117 }
118 }
119
120 impl<T> Mul<T> for YmmRegister
121 where
122 T: Into<Self>
123 {
124 type Output = YmmRegister;
125
126 #[inline]
127 fn mul(self, rhs: T) -> Self::Output {
128 let rhs = rhs.into();
129 unsafe { self.map2(rhs, |a, b| vmulq_s32(a, b)) }
130 }
131 }
132
133 impl<T> MulAssign<T> for YmmRegister
134 where
135 T: Into<Self>
136 {
137 #[inline]
138 fn mul_assign(&mut self, rhs: T) {
139 let rhs: Self = rhs.into();
140 *self = *self * rhs;
141 }
142 }
143
144 impl<T> BitOr<T> for YmmRegister
145 where
146 T: Into<Self>
147 {
148 type Output = YmmRegister;
149
150 #[inline]
151 fn bitor(self, rhs: T) -> Self::Output {
152 let rhs = rhs.into();
153 unsafe { self.map2(rhs, |a, b| vorrq_s32(a, b)) }
154 }
155 }
156
157 impl<T> BitOrAssign<T> for YmmRegister
158 where
159 T: Into<Self>
160 {
161 #[inline]
162 fn bitor_assign(&mut self, rhs: T) {
163 let rhs: Self = rhs.into();
164 *self = *self | rhs;
165 }
166 }
167
168 impl From<i32> for YmmRegister {
169 #[inline]
170 fn from(val: i32) -> Self {
171 unsafe {
172 let dup = vdupq_n_s32(val);
173
174 YmmRegister {
175 mm256: int32x4x2_t(dup, dup)
176 }
177 }
178 }
179 }
180
181 impl From<VecType> for YmmRegister {
182 #[inline]
183 fn from(mm256: VecType) -> Self {
184 YmmRegister { mm256 }
185 }
186 }
187
188 #[allow(clippy::too_many_arguments)]
189 #[inline]
190 unsafe fn transpose4(
191 v0: &mut int32x4_t, v1: &mut int32x4_t, v2: &mut int32x4_t, v3: &mut int32x4_t
192 ) {
193 let w0 = vtrnq_s32(
194 vreinterpretq_s32_s64(vtrn1q_s64(
195 vreinterpretq_s64_s32(*v0),
196 vreinterpretq_s64_s32(*v2)
197 )),
198 vreinterpretq_s32_s64(vtrn1q_s64(
199 vreinterpretq_s64_s32(*v1),
200 vreinterpretq_s64_s32(*v3)
201 ))
202 );
203 let w1 = vtrnq_s32(
204 vreinterpretq_s32_s64(vtrn2q_s64(
205 vreinterpretq_s64_s32(*v0),
206 vreinterpretq_s64_s32(*v2)
207 )),
208 vreinterpretq_s32_s64(vtrn2q_s64(
209 vreinterpretq_s64_s32(*v1),
210 vreinterpretq_s64_s32(*v3)
211 ))
212 );
213
214 *v0 = w0.0;
215 *v1 = w0.1;
216 *v2 = w1.0;
217 *v3 = w1.1;
218 }
219
220 /// Transpose an array of 8 by 8 i32
221 /// Arm has dedicated interleave/transpose instructions
222 /// we:
223 /// 1. Transpose the upper left and lower right quadrants
224 /// 2. Swap and transpose the upper right and lower left quadrants
225 #[allow(clippy::too_many_arguments)]
226 #[inline]
227 pub unsafe fn transpose(
228 v0: &mut YmmRegister, v1: &mut YmmRegister, v2: &mut YmmRegister, v3: &mut YmmRegister,
229 v4: &mut YmmRegister, v5: &mut YmmRegister, v6: &mut YmmRegister, v7: &mut YmmRegister
230 ) {
231 use core::mem::swap;
232
233 let ul0 = &mut v0.mm256.0;
234 let ul1 = &mut v1.mm256.0;
235 let ul2 = &mut v2.mm256.0;
236 let ul3 = &mut v3.mm256.0;
237
238 let ur0 = &mut v0.mm256.1;
239 let ur1 = &mut v1.mm256.1;
240 let ur2 = &mut v2.mm256.1;
241 let ur3 = &mut v3.mm256.1;
242
243 let ll0 = &mut v4.mm256.0;
244 let ll1 = &mut v5.mm256.0;
245 let ll2 = &mut v6.mm256.0;
246 let ll3 = &mut v7.mm256.0;
247
248 let lr0 = &mut v4.mm256.1;
249 let lr1 = &mut v5.mm256.1;
250 let lr2 = &mut v6.mm256.1;
251 let lr3 = &mut v7.mm256.1;
252
253 swap(ur0, ll0);
254 swap(ur1, ll1);
255 swap(ur2, ll2);
256 swap(ur3, ll3);
257
258 transpose4(ul0, ul1, ul2, ul3);
259
260 transpose4(ur0, ur1, ur2, ur3);
261
262 transpose4(ll0, ll1, ll2, ll3);
263
264 transpose4(lr0, lr1, lr2, lr3);
265 }
266
267 #[cfg(test)]
268 mod tests {
269 use super::*;
270
271 #[test]
272 fn test_transpose() {
273 fn get_val(i: usize, j: usize) -> i32 {
274 ((i * 8) / (j + 1)) as i32
275 }
276 unsafe {
277 let mut vals: [i32; 8 * 8] = [0; 8 * 8];
278
279 for i in 0..8 {
280 for j in 0..8 {
281 // some order-dependent value of i and j
282 let value = get_val(i, j);
283 vals[i * 8 + j] = value;
284 }
285 }
286
287 let mut regs: [YmmRegister; 8] = core::mem::transmute(vals);
288 let mut reg0 = regs[0];
289 let mut reg1 = regs[1];
290 let mut reg2 = regs[2];
291 let mut reg3 = regs[3];
292 let mut reg4 = regs[4];
293 let mut reg5 = regs[5];
294 let mut reg6 = regs[6];
295 let mut reg7 = regs[7];
296
297 transpose(
298 &mut reg0, &mut reg1, &mut reg2, &mut reg3, &mut reg4, &mut reg5, &mut reg6,
299 &mut reg7
300 );
301
302 regs[0] = reg0;
303 regs[1] = reg1;
304 regs[2] = reg2;
305 regs[3] = reg3;
306 regs[4] = reg4;
307 regs[5] = reg5;
308 regs[6] = reg6;
309 regs[7] = reg7;
310
311 let vals_from_reg: [i32; 8 * 8] = core::mem::transmute(regs);
312
313 for i in 0..8 {
314 for j in 0..i {
315 let orig = vals[i * 8 + j];
316 vals[i * 8 + j] = vals[j * 8 + i];
317 vals[j * 8 + i] = orig;
318 }
319 }
320
321 for i in 0..8 {
322 for j in 0..8 {
323 assert_eq!(vals[j * 8 + i], get_val(i, j));
324 assert_eq!(vals_from_reg[j * 8 + i], get_val(i, j));
325 }
326 }
327
328 assert_eq!(vals, vals_from_reg);
329 }
330 }
331 }