Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/transform/forward.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::asm::shared::transform::forward::*;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::transform::forward::rust;
13 use crate::transform::forward_shared::*;
14 use crate::transform::*;
15 use crate::util::*;
16 use std::mem::MaybeUninit;
17
18 use debug_unreachable::debug_unreachable;
19
20 #[cfg(target_arch = "x86")]
21 use std::arch::x86::*;
22 #[cfg(target_arch = "x86_64")]
23 use std::arch::x86_64::*;
24
25 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
26 #[derive(Copy, Clone)]
27 struct I32X8 {
28 data: __m256i,
29 }
30
31 impl I32X8 {
32 #[target_feature(enable = "avx2")]
33 #[inline]
34 const unsafe fn vec(self) -> __m256i {
35 self.data
36 }
37
38 #[target_feature(enable = "avx2")]
39 #[inline]
40 const unsafe fn new(a: __m256i) -> I32X8 {
41 I32X8 { data: a }
42 }
43 }
44
45 type TxfmFunc = unsafe fn(&mut [I32X8]);
46
47 impl_1d_tx!(target_feature(enable = "avx2"), unsafe);
48
49 impl TxOperations for I32X8 {
50 #[target_feature(enable = "avx2")]
51 #[inline]
52 unsafe fn zero() -> Self {
53 I32X8::new(_mm256_setzero_si256())
54 }
55
56 #[target_feature(enable = "avx2")]
57 #[inline]
58 unsafe fn tx_mul<const SHIFT: i32>(self, mul: i32) -> Self {
59 I32X8::new(_mm256_srav_epi32(
60 _mm256_add_epi32(
61 _mm256_mullo_epi32(self.vec(), _mm256_set1_epi32(mul)),
62 _mm256_set1_epi32(1 << SHIFT >> 1),
63 ),
64 _mm256_set1_epi32(SHIFT),
65 ))
66 }
67
68 #[target_feature(enable = "avx2")]
69 #[inline]
70 unsafe fn rshift1(self) -> Self {
71 I32X8::new(_mm256_srai_epi32(
72 _mm256_sub_epi32(
73 self.vec(),
74 _mm256_cmpgt_epi32(_mm256_setzero_si256(), self.vec()),
75 ),
76 1,
77 ))
78 }
79
80 #[target_feature(enable = "avx2")]
81 #[inline]
82 unsafe fn add(self, b: Self) -> Self {
83 I32X8::new(_mm256_add_epi32(self.vec(), b.vec()))
84 }
85
86 #[target_feature(enable = "avx2")]
87 #[inline]
88 unsafe fn sub(self, b: Self) -> Self {
89 I32X8::new(_mm256_sub_epi32(self.vec(), b.vec()))
90 }
91
92 #[target_feature(enable = "avx2")]
93 #[inline]
94 unsafe fn add_avg(self, b: Self) -> Self {
95 I32X8::new(_mm256_srai_epi32(_mm256_add_epi32(self.vec(), b.vec()), 1))
96 }
97
98 #[target_feature(enable = "avx2")]
99 #[inline]
100 unsafe fn sub_avg(self, b: Self) -> Self {
101 I32X8::new(_mm256_srai_epi32(_mm256_sub_epi32(self.vec(), b.vec()), 1))
102 }
103 }
104
105 #[target_feature(enable = "avx2")]
106 unsafe fn transpose_8x8_avx2(input: &[I32X8; 8], into: &mut [I32X8; 8]) {
107 let stage1 = (
108 _mm256_unpacklo_epi32(input[0].vec(), input[1].vec()),
109 _mm256_unpackhi_epi32(input[0].vec(), input[1].vec()),
110 _mm256_unpacklo_epi32(input[2].vec(), input[3].vec()),
111 _mm256_unpackhi_epi32(input[2].vec(), input[3].vec()),
112 _mm256_unpacklo_epi32(input[4].vec(), input[5].vec()),
113 _mm256_unpackhi_epi32(input[4].vec(), input[5].vec()),
114 _mm256_unpacklo_epi32(input[6].vec(), input[7].vec()),
115 _mm256_unpackhi_epi32(input[6].vec(), input[7].vec()),
116 );
117
118 let stage2 = (
119 _mm256_unpacklo_epi64(stage1.0, stage1.2),
120 _mm256_unpackhi_epi64(stage1.0, stage1.2),
121 _mm256_unpacklo_epi64(stage1.1, stage1.3),
122 _mm256_unpackhi_epi64(stage1.1, stage1.3),
123 _mm256_unpacklo_epi64(stage1.4, stage1.6),
124 _mm256_unpackhi_epi64(stage1.4, stage1.6),
125 _mm256_unpacklo_epi64(stage1.5, stage1.7),
126 _mm256_unpackhi_epi64(stage1.5, stage1.7),
127 );
128
129 #[allow(clippy::identity_op)]
130 const LO: i32 = (2 << 4) | 0;
131 const HI: i32 = (3 << 4) | 1;
132 into[0] = I32X8::new(_mm256_permute2x128_si256(stage2.0, stage2.4, LO));
133 into[1] = I32X8::new(_mm256_permute2x128_si256(stage2.1, stage2.5, LO));
134 into[2] = I32X8::new(_mm256_permute2x128_si256(stage2.2, stage2.6, LO));
135 into[3] = I32X8::new(_mm256_permute2x128_si256(stage2.3, stage2.7, LO));
136 into[4] = I32X8::new(_mm256_permute2x128_si256(stage2.0, stage2.4, HI));
137 into[5] = I32X8::new(_mm256_permute2x128_si256(stage2.1, stage2.5, HI));
138 into[6] = I32X8::new(_mm256_permute2x128_si256(stage2.2, stage2.6, HI));
139 into[7] = I32X8::new(_mm256_permute2x128_si256(stage2.3, stage2.7, HI));
140 }
141
142 #[target_feature(enable = "avx2")]
143 unsafe fn transpose_8x4_avx2(input: &[I32X8; 8], into: &mut [I32X8; 4]) {
144 // Last 8 are empty
145 let stage1 = (
146 //0101
147 _mm256_unpacklo_epi32(input[0].vec(), input[1].vec()),
148 _mm256_unpackhi_epi32(input[0].vec(), input[1].vec()),
149 _mm256_unpacklo_epi32(input[2].vec(), input[3].vec()),
150 _mm256_unpackhi_epi32(input[2].vec(), input[3].vec()),
151 _mm256_unpacklo_epi32(input[4].vec(), input[5].vec()),
152 _mm256_unpackhi_epi32(input[4].vec(), input[5].vec()),
153 _mm256_unpacklo_epi32(input[6].vec(), input[7].vec()),
154 _mm256_unpackhi_epi32(input[6].vec(), input[7].vec()),
155 );
156
157 let stage2 = (
158 _mm256_unpacklo_epi64(stage1.0, stage1.2),
159 _mm256_unpackhi_epi64(stage1.0, stage1.2),
160 _mm256_unpacklo_epi64(stage1.1, stage1.3),
161 _mm256_unpackhi_epi64(stage1.1, stage1.3),
162 _mm256_unpacklo_epi64(stage1.4, stage1.6),
163 _mm256_unpackhi_epi64(stage1.4, stage1.6),
164 _mm256_unpacklo_epi64(stage1.5, stage1.7),
165 _mm256_unpackhi_epi64(stage1.5, stage1.7),
166 );
167
168 #[allow(clippy::identity_op)]
169 const LO: i32 = (2 << 4) | 0;
170 into[0] = I32X8::new(_mm256_permute2x128_si256(stage2.0, stage2.4, LO));
171 into[1] = I32X8::new(_mm256_permute2x128_si256(stage2.1, stage2.5, LO));
172 into[2] = I32X8::new(_mm256_permute2x128_si256(stage2.2, stage2.6, LO));
173 into[3] = I32X8::new(_mm256_permute2x128_si256(stage2.3, stage2.7, LO));
174 }
175
176 #[target_feature(enable = "avx2")]
177 unsafe fn transpose_4x8_avx2(input: &[I32X8; 4], into: &mut [I32X8; 8]) {
178 let stage1 = (
179 // 0101
180 _mm256_unpacklo_epi32(input[0].vec(), input[1].vec()),
181 _mm256_unpackhi_epi32(input[0].vec(), input[1].vec()),
182 // 2323
183 _mm256_unpacklo_epi32(input[2].vec(), input[3].vec()),
184 _mm256_unpackhi_epi32(input[2].vec(), input[3].vec()),
185 );
186
187 let stage2 = (
188 // 01234567
189 _mm256_unpacklo_epi64(stage1.0, stage1.2),
190 _mm256_unpackhi_epi64(stage1.0, stage1.2),
191 _mm256_unpacklo_epi64(stage1.1, stage1.3),
192 _mm256_unpackhi_epi64(stage1.1, stage1.3),
193 );
194
195 into[0] = I32X8::new(stage2.0);
196 into[1] = I32X8::new(stage2.1);
197 into[2] = I32X8::new(stage2.2);
198 into[3] = I32X8::new(stage2.3);
199 into[4] =
200 I32X8::new(_mm256_castsi128_si256(_mm256_extractf128_si256(stage2.0, 1)));
201 into[5] =
202 I32X8::new(_mm256_castsi128_si256(_mm256_extractf128_si256(stage2.1, 1)));
203 into[6] =
204 I32X8::new(_mm256_castsi128_si256(_mm256_extractf128_si256(stage2.2, 1)));
205 into[7] =
206 I32X8::new(_mm256_castsi128_si256(_mm256_extractf128_si256(stage2.3, 1)));
207 }
208
209 #[target_feature(enable = "avx2")]
210 unsafe fn transpose_4x4_avx2(input: &[I32X8; 4], into: &mut [I32X8; 4]) {
211 let stage1 = (
212 _mm256_unpacklo_epi32(input[0].vec(), input[1].vec()),
213 _mm256_unpackhi_epi32(input[0].vec(), input[1].vec()),
214 _mm256_unpacklo_epi32(input[2].vec(), input[3].vec()),
215 _mm256_unpackhi_epi32(input[2].vec(), input[3].vec()),
216 );
217
218 into[0] = I32X8::new(_mm256_unpacklo_epi64(stage1.0, stage1.2));
219 into[1] = I32X8::new(_mm256_unpackhi_epi64(stage1.0, stage1.2));
220 into[2] = I32X8::new(_mm256_unpacklo_epi64(stage1.1, stage1.3));
221 into[3] = I32X8::new(_mm256_unpackhi_epi64(stage1.1, stage1.3));
222 }
223
224 #[target_feature(enable = "avx2")]
225 #[inline]
226 unsafe fn shift_left(a: I32X8, shift: u8) -> I32X8 {
227 I32X8::new(_mm256_sllv_epi32(a.vec(), _mm256_set1_epi32(shift as i32)))
228 }
229
230 #[target_feature(enable = "avx2")]
231 #[inline]
232 unsafe fn shift_right(a: I32X8, shift: u8) -> I32X8 {
233 I32X8::new(_mm256_srav_epi32(
234 _mm256_add_epi32(a.vec(), _mm256_set1_epi32(1 << (shift as i32) >> 1)),
235 _mm256_set1_epi32(shift as i32),
236 ))
237 }
238
239 #[target_feature(enable = "avx2")]
240 #[inline]
241 unsafe fn round_shift_array_avx2(arr: &mut [I32X8], bit: i8) {
242 if arr.len() % 4 != 0 {
243 debug_unreachable!();
244 }
245
246 if bit == 0 {
247 return;
248 }
249 if bit > 0 {
250 let shift = bit as u8;
251 for s in arr.chunks_exact_mut(4) {
252 for chunk in s {
253 *chunk = shift_right(*chunk, shift);
254 }
255 }
256 } else {
257 let shift = (-bit) as u8;
258 for s in arr.chunks_exact_mut(4) {
259 for chunk in s {
260 *chunk = shift_left(*chunk, shift);
261 }
262 }
263 }
264 }
265
266 #[allow(clippy::identity_op, clippy::erasing_op)]
267 #[target_feature(enable = "avx2")]
268 unsafe fn forward_transform_avx2<T: Coefficient>(
269 input: &[i16], output: &mut [MaybeUninit<T>], stride: usize,
270 tx_size: TxSize, tx_type: TxType, bd: usize,
271 ) {
272 // Note when assigning txfm_size_col, we use the txfm_size from the
273 // row configuration and vice versa. This is intentionally done to
274 // accurately perform rectangular transforms. When the transform is
275 // rectangular, the number of columns will be the same as the
276 // txfm_size stored in the row cfg struct. It will make no difference
277 // for square transforms.
278 let txfm_size_col = tx_size.width();
279 let txfm_size_row = tx_size.height();
280
281 let col_class = SizeClass1D::from_length(txfm_size_col);
282 let row_class = SizeClass1D::from_length(txfm_size_row);
283
284 let mut tmp: Aligned<[I32X8; 64 * 64 / 8]> = Aligned::uninitialized();
285 let buf = &mut tmp.data[..txfm_size_col * (txfm_size_row / 8).max(1)];
286 let cfg = Txfm2DFlipCfg::fwd(tx_type, tx_size, bd);
287
288 let txfm_func_col = get_func(cfg.txfm_type_col);
289 let txfm_func_row = get_func(cfg.txfm_type_row);
290
291 // Columns
292 for cg in (0..txfm_size_col).step_by(8) {
293 let shift = cfg.shift[0] as u8;
294 #[target_feature(enable = "avx2")]
295 #[inline]
296 unsafe fn load_columns(input_ptr: *const i16, shift: u8) -> I32X8 {
297 // TODO: load 64-bits for x4 wide columns
298 shift_left(
299 I32X8::new(_mm256_cvtepi16_epi32(_mm_loadu_si128(
300 input_ptr as *const _,
301 ))),
302 shift,
303 )
304 }
305
306 // Avoid zero initialization
307 let tx_in = &mut [MaybeUninit::<I32X8>::uninit(); 64][..txfm_size_row];
308
309 if cfg.ud_flip {
310 // flip upside down
311 for (in_slice, out_reg) in
312 input[cg..].chunks(stride).zip(tx_in.iter_mut().rev())
313 {
314 *out_reg = MaybeUninit::new(load_columns(in_slice.as_ptr(), shift));
315 }
316 } else {
317 for (in_slice, out_reg) in
318 input[cg..].chunks(stride).zip(tx_in.iter_mut())
319 {
320 *out_reg = MaybeUninit::new(load_columns(in_slice.as_ptr(), shift));
321 }
322 }
323
324 let col_coeffs = slice_assume_init_mut(tx_in);
325
326 txfm_func_col(col_coeffs);
327 round_shift_array_avx2(col_coeffs, -cfg.shift[1]);
328
329 // Transpose the array. Select the appropriate method to do so.
330 match (row_class, col_class) {
331 (SizeClass1D::X8UP, SizeClass1D::X8UP) => {
332 for rg in (0..txfm_size_row).step_by(8) {
333 let buf = &mut buf[(rg / 8 * txfm_size_col) + cg..];
334 let buf = cast_mut::<8, _>(buf);
335 let input = &col_coeffs[rg..];
336 let input = cast::<8, _>(input);
337 transpose_8x8_avx2(input, buf);
338 }
339 }
340 (SizeClass1D::X8UP, SizeClass1D::X4) => {
341 for rg in (0..txfm_size_row).step_by(8) {
342 let buf = &mut buf[(rg / 8 * txfm_size_col) + cg..];
343 let buf = cast_mut::<4, _>(buf);
344 let input = &col_coeffs[rg..];
345 let input = cast::<8, _>(input);
346 transpose_8x4_avx2(input, buf);
347 }
348 }
349 (SizeClass1D::X4, SizeClass1D::X8UP) => {
350 // Don't need to loop over rows
351 let buf = &mut buf[cg..];
352 let buf = cast_mut::<8, _>(buf);
353 let input = cast::<4, _>(col_coeffs);
354 transpose_4x8_avx2(input, buf);
355 }
356 (SizeClass1D::X4, SizeClass1D::X4) => {
357 // Don't need to loop over rows
358 let buf = cast_mut::<4, _>(buf);
359 let input = cast::<4, _>(col_coeffs);
360 transpose_4x4_avx2(input, buf);
361 }
362 }
363 }
364
365 // Rows
366 for rg in (0..txfm_size_row).step_by(8) {
367 let row_coeffs = &mut buf[rg / 8 * txfm_size_col..][..txfm_size_col];
368
369 if cfg.lr_flip {
370 row_coeffs.reverse();
371 }
372
373 txfm_func_row(row_coeffs);
374 round_shift_array_avx2(row_coeffs, -cfg.shift[2]);
375
376 // Write out the coefficients using the correct method for transforms of
377 // this size.
378 match row_class {
379 SizeClass1D::X8UP => {
380 // Store output in at most 32x32 chunks. See rust code for details.
381
382 // Output is grouped into 32x32 chunks so a stride of at most 32 is
383 // used for each chunk
384 let output_stride = txfm_size_row.min(32);
385
386 // Split the first 32 rows from the last 32 rows and offset by rg % 32
387 let output = &mut output[(rg & 31)
388 + (rg >= 32) as usize * output_stride * txfm_size_col.min(32)..];
389
390 for cg in (0..txfm_size_col).step_by(32) {
391 // Offset by zero or half of output
392 let output = &mut output[txfm_size_row * cg..];
393
394 for c in 0..txfm_size_col.min(32) {
395 match T::Pixel::type_enum() {
396 PixelType::U8 => {
397 let vec = row_coeffs[c + cg].vec();
398 let lo = _mm256_castsi256_si128(vec);
399 let hi = _mm256_extracti128_si256(vec, 1);
400 _mm_storeu_si128(
401 output[c * output_stride..].as_mut_ptr() as *mut _,
402 _mm_packs_epi32(lo, hi),
403 );
404 }
405 PixelType::U16 => {
406 _mm256_storeu_si256(
407 output[c * output_stride..].as_mut_ptr() as *mut _,
408 row_coeffs[c + cg].vec(),
409 );
410 }
411 }
412 }
413 }
414 }
415 SizeClass1D::X4 => {
416 // Write out coefficients in normal order - it isn't possible to have
417 // more than 32 rows.
418 for c in 0..txfm_size_col {
419 match T::Pixel::type_enum() {
420 PixelType::U8 => {
421 let lo = _mm256_castsi256_si128(row_coeffs[c].vec());
422 _mm_storel_epi64(
423 output[c * txfm_size_row + rg..].as_mut_ptr() as *mut _,
424 _mm_packs_epi32(lo, lo),
425 );
426 }
427 PixelType::U16 => {
428 _mm256_storeu_si256(
429 output[c * txfm_size_row + rg..].as_mut_ptr() as *mut _,
430 row_coeffs[c].vec(),
431 );
432 }
433 }
434 }
435 }
436 }
437 }
438 }
439
440 /// # Panics
441 ///
442 /// - If called with an invalid combination of `tx_size` and `tx_type`
443 #[inline]
444 pub fn forward_transform<T: Coefficient>(
445 input: &[i16], output: &mut [MaybeUninit<T>], stride: usize,
446 tx_size: TxSize, tx_type: TxType, bd: usize, cpu: CpuFeatureLevel,
447 ) {
448 assert!(valid_av1_transform(tx_size, tx_type));
449 if cpu >= CpuFeatureLevel::AVX2 {
450 // SAFETY: Calls Assembly code.
451 unsafe {
452 forward_transform_avx2(input, output, stride, tx_size, tx_type, bd);
453 }
454 } else {
455 rust::forward_transform(input, output, stride, tx_size, tx_type, bd, cpu);
456 }
457 }