Debian Codegraph @

.cargo/vendor/zune-jpeg-0.4.21/src/unsafe_utils_avx2.rs

raw
1 /*
2 * Copyright (c) 2023.
3 *
4 * This software is free software;
5 *
6 * You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license
7 */
8
9 #![cfg(all(feature = "x86", any(target_arch = "x86", target_arch = "x86_64")))]
10 //! This module provides unsafe ways to do some things
11 #![allow(clippy::wildcard_imports)]
12
13 #[cfg(target_arch = "x86")]
14 use core::arch::x86::*;
15 #[cfg(target_arch = "x86_64")]
16 use core::arch::x86_64::*;
17 use core::ops::{Add, AddAssign, Mul, MulAssign, Sub};
18
19 /// A copy of `_MM_SHUFFLE()` that doesn't require
20 /// a nightly compiler
21 #[inline]
22 const fn shuffle(z: i32, y: i32, x: i32, w: i32) -> i32 {
23 (z << 6) | (y << 4) | (x << 2) | w
24 }
25
26 /// An abstraction of an AVX ymm register that
27 ///allows some things to not look ugly
28 #[derive(Clone, Copy)]
29 pub struct YmmRegister {
30 /// An AVX register
31 pub(crate) mm256: __m256i
32 }
33
34 impl Add for YmmRegister {
35 type Output = YmmRegister;
36
37 #[inline]
38 fn add(self, rhs: Self) -> Self::Output {
39 unsafe {
40 return YmmRegister {
41 mm256: _mm256_add_epi32(self.mm256, rhs.mm256)
42 };
43 }
44 }
45 }
46
47 impl Add<i32> for YmmRegister {
48 type Output = YmmRegister;
49
50 #[inline]
51 fn add(self, rhs: i32) -> Self::Output {
52 unsafe {
53 let tmp = _mm256_set1_epi32(rhs);
54
55 return YmmRegister {
56 mm256: _mm256_add_epi32(self.mm256, tmp)
57 };
58 }
59 }
60 }
61
62 impl Sub for YmmRegister {
63 type Output = YmmRegister;
64
65 #[inline]
66 fn sub(self, rhs: Self) -> Self::Output {
67 unsafe {
68 return YmmRegister {
69 mm256: _mm256_sub_epi32(self.mm256, rhs.mm256)
70 };
71 }
72 }
73 }
74
75 impl AddAssign for YmmRegister {
76 #[inline]
77 fn add_assign(&mut self, rhs: Self) {
78 unsafe {
79 self.mm256 = _mm256_add_epi32(self.mm256, rhs.mm256);
80 }
81 }
82 }
83
84 impl AddAssign<i32> for YmmRegister {
85 #[inline]
86 fn add_assign(&mut self, rhs: i32) {
87 unsafe {
88 let tmp = _mm256_set1_epi32(rhs);
89
90 self.mm256 = _mm256_add_epi32(self.mm256, tmp);
91 }
92 }
93 }
94
95 impl Mul for YmmRegister {
96 type Output = YmmRegister;
97
98 #[inline]
99 fn mul(self, rhs: Self) -> Self::Output {
100 unsafe {
101 YmmRegister {
102 mm256: _mm256_mullo_epi32(self.mm256, rhs.mm256)
103 }
104 }
105 }
106 }
107
108 impl Mul<i32> for YmmRegister {
109 type Output = YmmRegister;
110
111 #[inline]
112 fn mul(self, rhs: i32) -> Self::Output {
113 unsafe {
114 let tmp = _mm256_set1_epi32(rhs);
115
116 YmmRegister {
117 mm256: _mm256_mullo_epi32(self.mm256, tmp)
118 }
119 }
120 }
121 }
122
123 impl MulAssign for YmmRegister {
124 #[inline]
125 fn mul_assign(&mut self, rhs: Self) {
126 unsafe {
127 self.mm256 = _mm256_mullo_epi32(self.mm256, rhs.mm256);
128 }
129 }
130 }
131
132 impl MulAssign<i32> for YmmRegister {
133 #[inline]
134 fn mul_assign(&mut self, rhs: i32) {
135 unsafe {
136 let tmp = _mm256_set1_epi32(rhs);
137
138 self.mm256 = _mm256_mullo_epi32(self.mm256, tmp);
139 }
140 }
141 }
142
143 impl MulAssign<__m256i> for YmmRegister {
144 #[inline]
145 fn mul_assign(&mut self, rhs: __m256i) {
146 unsafe {
147 self.mm256 = _mm256_mullo_epi32(self.mm256, rhs);
148 }
149 }
150 }
151
152 type Reg = YmmRegister;
153
154 /// Transpose an array of 8 by 8 i32's using avx intrinsics
155 ///
156 /// This was translated from [here](https://newbedev.com/transpose-an-8x8-float-using-avx-avx2)
157 #[allow(unused_parens, clippy::too_many_arguments)]
158 #[target_feature(enable = "avx2")]
159 #[inline]
160 pub unsafe fn transpose(
161 v0: &mut Reg, v1: &mut Reg, v2: &mut Reg, v3: &mut Reg, v4: &mut Reg, v5: &mut Reg,
162 v6: &mut Reg, v7: &mut Reg
163 ) {
164 macro_rules! merge_epi32 {
165 ($v0:tt,$v1:tt,$v2:tt,$v3:tt) => {
166 let va = _mm256_permute4x64_epi64($v0, shuffle(3, 1, 2, 0));
167
168 let vb = _mm256_permute4x64_epi64($v1, shuffle(3, 1, 2, 0));
169
170 $v2 = _mm256_unpacklo_epi32(va, vb);
171
172 $v3 = _mm256_unpackhi_epi32(va, vb);
173 };
174 }
175
176 macro_rules! merge_epi64 {
177 ($v0:tt,$v1:tt,$v2:tt,$v3:tt) => {
178 let va = _mm256_permute4x64_epi64($v0, shuffle(3, 1, 2, 0));
179
180 let vb = _mm256_permute4x64_epi64($v1, shuffle(3, 1, 2, 0));
181
182 $v2 = _mm256_unpacklo_epi64(va, vb);
183
184 $v3 = _mm256_unpackhi_epi64(va, vb);
185 };
186 }
187
188 macro_rules! merge_si128 {
189 ($v0:tt,$v1:tt,$v2:tt,$v3:tt) => {
190 $v2 = _mm256_permute2x128_si256($v0, $v1, shuffle(0, 2, 0, 0));
191
192 $v3 = _mm256_permute2x128_si256($v0, $v1, shuffle(0, 3, 0, 1));
193 };
194 }
195
196 let (w0, w1, w2, w3, w4, w5, w6, w7);
197
198 merge_epi32!((v0.mm256), (v1.mm256), w0, w1);
199
200 merge_epi32!((v2.mm256), (v3.mm256), w2, w3);
201
202 merge_epi32!((v4.mm256), (v5.mm256), w4, w5);
203
204 merge_epi32!((v6.mm256), (v7.mm256), w6, w7);
205
206 let (x0, x1, x2, x3, x4, x5, x6, x7);
207
208 merge_epi64!(w0, w2, x0, x1);
209
210 merge_epi64!(w1, w3, x2, x3);
211
212 merge_epi64!(w4, w6, x4, x5);
213
214 merge_epi64!(w5, w7, x6, x7);
215
216 merge_si128!(x0, x4, (v0.mm256), (v1.mm256));
217
218 merge_si128!(x1, x5, (v2.mm256), (v3.mm256));
219
220 merge_si128!(x2, x6, (v4.mm256), (v5.mm256));
221
222 merge_si128!(x3, x7, (v6.mm256), (v7.mm256));
223 }