Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/transform/forward.rs

raw
1 // Copyright (c) 2019-2023, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::asm::shared::transform::forward::*;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::transform::forward::rust;
13 use crate::transform::forward_shared::*;
14 use crate::transform::*;
15 use crate::util::*;
16 use std::mem::MaybeUninit;
17
18 use debug_unreachable::debug_unreachable;
19
20 use core::arch::aarch64::*;
21
22 #[derive(Clone, Copy)]
23 #[repr(transparent)]
24 struct I32X8(int32x4x2_t);
25
26 impl I32X8 {
27 #[inline]
28 const unsafe fn vec(self) -> int32x4x2_t {
29 self.0
30 }
31
32 #[inline]
33 const unsafe fn raw(a: int32x4x2_t) -> I32X8 {
34 Self(a)
35 }
36
37 #[inline]
38 const unsafe fn new(a: int32x4_t, b: int32x4_t) -> I32X8 {
39 Self(int32x4x2_t(a, b))
40 }
41 }
42
43 type TxfmFunc = unsafe fn(&mut [I32X8]);
44
45 impl_1d_tx!(allow(unused_attributes), unsafe);
46
47 impl TxOperations for I32X8 {
48 #[inline]
49 unsafe fn zero() -> Self {
50 let zero = vdupq_n_s32(0);
51 I32X8::new(zero, zero)
52 }
53
54 #[inline]
55 unsafe fn tx_mul<const SHIFT: i32>(self, mul: i32) -> Self {
56 I32X8::new(
57 vrshrq_n_s32(vmulq_n_s32(self.vec().0, mul), SHIFT),
58 vrshrq_n_s32(vmulq_n_s32(self.vec().1, mul), SHIFT),
59 )
60 }
61
62 #[inline]
63 unsafe fn rshift1(self) -> Self {
64 I32X8::new(
65 vhsubq_s32(
66 self.vec().0,
67 vreinterpretq_s32_u32(vcltzq_s32(self.vec().0)),
68 ),
69 vhsubq_s32(
70 self.vec().1,
71 vreinterpretq_s32_u32(vcltzq_s32(self.vec().1)),
72 ),
73 )
74 }
75
76 #[inline]
77 unsafe fn add(self, b: Self) -> Self {
78 I32X8::new(
79 vaddq_s32(self.vec().0, b.vec().0),
80 vaddq_s32(self.vec().1, b.vec().1),
81 )
82 }
83
84 #[inline]
85 unsafe fn sub(self, b: Self) -> Self {
86 I32X8::new(
87 vsubq_s32(self.vec().0, b.vec().0),
88 vsubq_s32(self.vec().1, b.vec().1),
89 )
90 }
91
92 #[inline]
93 unsafe fn add_avg(self, b: Self) -> Self {
94 I32X8::new(
95 vhaddq_s32(self.vec().0, b.vec().0),
96 vhaddq_s32(self.vec().1, b.vec().1),
97 )
98 }
99
100 #[inline]
101 unsafe fn sub_avg(self, b: Self) -> Self {
102 I32X8::new(
103 vhsubq_s32(self.vec().0, b.vec().0),
104 vhsubq_s32(self.vec().1, b.vec().1),
105 )
106 }
107 }
108
109 #[inline]
110 unsafe fn vreinterpretq_u32_s32_x2(a: int32x4x2_t) -> uint32x4x2_t {
111 uint32x4x2_t(vreinterpretq_u32_s32(a.0), vreinterpretq_u32_s32(a.1))
112 }
113
114 #[inline]
115 unsafe fn vtrnq_s64_to_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t {
116 let a = vreinterpretq_s64_s32(a);
117 let b = vreinterpretq_s64_s32(b);
118 int32x4x2_t(
119 vreinterpretq_s32_s64(vtrn1q_s64(a, b)),
120 vreinterpretq_s32_s64(vtrn2q_s64(a, b)),
121 )
122 }
123
124 #[inline]
125 unsafe fn transpose_8x8_neon(
126 input: &[I32X8; 8], into: &mut [MaybeUninit<I32X8>; 8],
127 ) {
128 let stage1 = (
129 vtrnq_s32(input[0].vec().0, input[1].vec().0),
130 vtrnq_s32(input[2].vec().0, input[3].vec().0),
131 vtrnq_s32(input[4].vec().0, input[5].vec().0),
132 vtrnq_s32(input[6].vec().0, input[7].vec().0),
133 vtrnq_s32(input[0].vec().1, input[1].vec().1),
134 vtrnq_s32(input[2].vec().1, input[3].vec().1),
135 vtrnq_s32(input[4].vec().1, input[5].vec().1),
136 vtrnq_s32(input[6].vec().1, input[7].vec().1),
137 );
138 let stage2 = (
139 vtrnq_s64_to_s32(stage1.0 .0, stage1.1 .0),
140 vtrnq_s64_to_s32(stage1.0 .1, stage1.1 .1),
141 vtrnq_s64_to_s32(stage1.2 .0, stage1.3 .0),
142 vtrnq_s64_to_s32(stage1.2 .1, stage1.3 .1),
143 vtrnq_s64_to_s32(stage1.4 .0, stage1.5 .0),
144 vtrnq_s64_to_s32(stage1.4 .1, stage1.5 .1),
145 vtrnq_s64_to_s32(stage1.6 .0, stage1.7 .0),
146 vtrnq_s64_to_s32(stage1.6 .1, stage1.7 .1),
147 );
148 into[0].write(I32X8::new(stage2.0 .0, stage2.2 .0));
149 into[1].write(I32X8::new(stage2.1 .0, stage2.3 .0));
150 into[2].write(I32X8::new(stage2.0 .1, stage2.2 .1));
151 into[3].write(I32X8::new(stage2.1 .1, stage2.3 .1));
152 into[4].write(I32X8::new(stage2.4 .0, stage2.6 .0));
153 into[5].write(I32X8::new(stage2.5 .0, stage2.7 .0));
154 into[6].write(I32X8::new(stage2.4 .1, stage2.6 .1));
155 into[7].write(I32X8::new(stage2.5 .1, stage2.7 .1));
156 }
157
158 #[inline]
159 unsafe fn transpose_8x4_neon(
160 input: &[I32X8; 8], into: &mut [MaybeUninit<I32X8>; 4],
161 ) {
162 let stage1 = (
163 vtrnq_s32(input[0].vec().0, input[1].vec().0),
164 vtrnq_s32(input[2].vec().0, input[3].vec().0),
165 vtrnq_s32(input[4].vec().0, input[5].vec().0),
166 vtrnq_s32(input[6].vec().0, input[7].vec().0),
167 );
168 let stage2 = (
169 vtrnq_s64_to_s32(stage1.0 .0, stage1.1 .0),
170 vtrnq_s64_to_s32(stage1.0 .1, stage1.1 .1),
171 vtrnq_s64_to_s32(stage1.2 .0, stage1.3 .0),
172 vtrnq_s64_to_s32(stage1.2 .1, stage1.3 .1),
173 );
174 into[0].write(I32X8::new(stage2.0 .0, stage2.2 .0));
175 into[1].write(I32X8::new(stage2.1 .0, stage2.3 .0));
176 into[2].write(I32X8::new(stage2.0 .1, stage2.2 .1));
177 into[3].write(I32X8::new(stage2.1 .1, stage2.3 .1));
178 }
179
180 #[inline]
181 unsafe fn transpose_4x8_neon(
182 input: &[I32X8; 4], into: &mut [MaybeUninit<I32X8>; 8],
183 ) {
184 let stage1 = (
185 vtrnq_s32(input[0].vec().0, input[1].vec().0),
186 vtrnq_s32(input[2].vec().0, input[3].vec().0),
187 vtrnq_s32(input[0].vec().1, input[1].vec().1),
188 vtrnq_s32(input[2].vec().1, input[3].vec().1),
189 );
190 let stage2 = (
191 vtrnq_s64_to_s32(stage1.0 .0, stage1.1 .0),
192 vtrnq_s64_to_s32(stage1.0 .1, stage1.1 .1),
193 vtrnq_s64_to_s32(stage1.2 .0, stage1.3 .0),
194 vtrnq_s64_to_s32(stage1.2 .1, stage1.3 .1),
195 );
196 into[0].write(I32X8::raw(stage2.0));
197 into[1].write(I32X8::raw(stage2.1));
198 into[2].write(I32X8::new(stage2.0 .1, stage2.0 .0));
199 into[3].write(I32X8::new(stage2.1 .1, stage2.1 .0));
200 into[4].write(I32X8::raw(stage2.2));
201 into[5].write(I32X8::raw(stage2.3));
202 into[6].write(I32X8::new(stage2.2 .1, stage2.2 .0));
203 into[7].write(I32X8::new(stage2.3 .1, stage2.3 .0));
204 }
205
206 #[inline]
207 unsafe fn transpose_4x4_neon(
208 input: &[I32X8; 4], into: &mut [MaybeUninit<I32X8>; 4],
209 ) {
210 let stage1 = (
211 vtrnq_s32(input[0].vec().0, input[1].vec().0),
212 vtrnq_s32(input[2].vec().0, input[3].vec().0),
213 );
214 let stage2 = (
215 vtrnq_s64_to_s32(stage1.0 .0, stage1.1 .0),
216 vtrnq_s64_to_s32(stage1.0 .1, stage1.1 .1),
217 );
218 into[0].write(I32X8::raw(stage2.0));
219 into[1].write(I32X8::raw(stage2.1));
220 into[2].write(I32X8::new(stage2.0 .1, stage2.0 .0));
221 into[3].write(I32X8::new(stage2.1 .1, stage2.1 .0));
222 }
223
224 #[inline]
225 unsafe fn shift_left_neon(a: I32X8, shift: u8) -> I32X8 {
226 let shift = vdupq_n_s32(shift.into());
227 I32X8::new(vrshlq_s32(a.vec().0, shift), vrshlq_s32(a.vec().1, shift))
228 }
229
230 #[inline]
231 unsafe fn shift_right_neon<const SHIFT: i32>(a: I32X8) -> I32X8 {
232 I32X8::new(vrshrq_n_s32(a.vec().0, SHIFT), vrshrq_n_s32(a.vec().1, SHIFT))
233 }
234
235 #[inline]
236 unsafe fn round_shift_array_neon(arr: &mut [I32X8], bit: i8) {
237 if arr.len() % 4 != 0 {
238 debug_unreachable!();
239 }
240
241 if bit == 0 {
242 return;
243 }
244 if bit > 0 {
245 if bit == 1 {
246 for s in arr.chunks_exact_mut(4) {
247 for chunk in s {
248 *chunk = shift_right_neon::<1>(*chunk)
249 }
250 }
251 } else if bit == 2 {
252 for s in arr.chunks_exact_mut(4) {
253 for chunk in s {
254 *chunk = shift_right_neon::<2>(*chunk)
255 }
256 }
257 } else {
258 debug_unreachable!();
259 }
260 } else {
261 let shift = (-bit) as u8;
262 for s in arr.chunks_exact_mut(4) {
263 for chunk in s {
264 *chunk = shift_left_neon(*chunk, shift);
265 }
266 }
267 }
268 }
269
270 #[allow(clippy::identity_op, clippy::erasing_op)]
271 unsafe fn forward_transform_neon<T: Coefficient>(
272 input: &[i16], output: &mut [MaybeUninit<T>], stride: usize,
273 tx_size: TxSize, tx_type: TxType, bd: usize,
274 ) {
275 // Note when assigning txfm_size_col, we use the txfm_size from the
276 // row configuration and vice versa. This is intentionally done to
277 // accurately perform rectangular transforms. When the transform is
278 // rectangular, the number of columns will be the same as the
279 // txfm_size stored in the row cfg struct. It will make no difference
280 // for square transforms.
281 let txfm_size_col = tx_size.width();
282 let txfm_size_row = tx_size.height();
283
284 let col_class = SizeClass1D::from_length(txfm_size_col);
285 let row_class = SizeClass1D::from_length(txfm_size_row);
286
287 let mut buf = Aligned::new([MaybeUninit::<I32X8>::uninit(); 64 * 64 / 8]);
288 let buf = &mut buf.data[..txfm_size_col * (txfm_size_row / 8).max(1)];
289 let cfg = Txfm2DFlipCfg::fwd(tx_type, tx_size, bd);
290
291 let txfm_func_col = get_func(cfg.txfm_type_col);
292 let txfm_func_row = get_func(cfg.txfm_type_row);
293
294 // Columns
295 for cg in (0..txfm_size_col).step_by(8) {
296 let shift = cfg.shift[0] as u8;
297 #[inline]
298 unsafe fn load_columns(input_ptr: *const i16, shift: u8) -> I32X8 {
299 // TODO: load 64-bits for x4 wide columns
300 let a = vld1q_s16(input_ptr);
301 shift_left_neon(
302 I32X8::new(vmovl_s16(vget_low_s16(a)), vmovl_high_s16(a)),
303 shift,
304 )
305 }
306
307 // Avoid zero initialization
308 let col_coeffs =
309 &mut [MaybeUninit::<I32X8>::uninit(); 64][..txfm_size_row];
310
311 if cfg.ud_flip {
312 // flip upside down
313 for (in_slice, out_reg) in
314 input[cg..].chunks(stride).zip(col_coeffs.iter_mut().rev())
315 {
316 *out_reg = MaybeUninit::new(load_columns(in_slice.as_ptr(), shift));
317 }
318 } else {
319 for (in_slice, out_reg) in
320 input[cg..].chunks(stride).zip(col_coeffs.iter_mut())
321 {
322 *out_reg = MaybeUninit::new(load_columns(in_slice.as_ptr(), shift));
323 }
324 }
325
326 let col_coeffs = slice_assume_init_mut(col_coeffs);
327
328 txfm_func_col(col_coeffs);
329 round_shift_array_neon(col_coeffs, -cfg.shift[1]);
330
331 // Transpose the array. Select the appropriate method to do so.
332 match (row_class, col_class) {
333 (SizeClass1D::X8UP, SizeClass1D::X8UP) => {
334 for rg in (0..txfm_size_row).step_by(8) {
335 let buf = &mut buf[(rg / 8 * txfm_size_col) + cg..];
336 let buf = cast_mut::<8, _>(buf);
337 let input = &col_coeffs[rg..];
338 let input = cast::<8, _>(input);
339 transpose_8x8_neon(input, buf);
340 }
341 }
342 (SizeClass1D::X8UP, SizeClass1D::X4) => {
343 for rg in (0..txfm_size_row).step_by(8) {
344 let buf = &mut buf[(rg / 8 * txfm_size_col) + cg..];
345 let buf = cast_mut::<4, _>(buf);
346 let input = &col_coeffs[rg..];
347 let input = cast::<8, _>(input);
348 transpose_8x4_neon(input, buf);
349 }
350 }
351 (SizeClass1D::X4, SizeClass1D::X8UP) => {
352 // Don't need to loop over rows
353 let buf = &mut buf[cg..];
354 let buf = cast_mut::<8, _>(buf);
355 let input = cast::<4, _>(col_coeffs);
356 transpose_4x8_neon(input, buf);
357 }
358 (SizeClass1D::X4, SizeClass1D::X4) => {
359 // Don't need to loop over rows
360 let buf = cast_mut::<4, _>(buf);
361 let input = cast::<4, _>(col_coeffs);
362 transpose_4x4_neon(input, buf);
363 }
364 }
365 }
366
367 let buf = slice_assume_init_mut(buf);
368
369 // Rows
370 for rg in (0..txfm_size_row).step_by(8) {
371 let row_coeffs = &mut buf[rg / 8 * txfm_size_col..][..txfm_size_col];
372
373 if cfg.lr_flip {
374 row_coeffs.reverse();
375 }
376
377 txfm_func_row(row_coeffs);
378 round_shift_array_neon(row_coeffs, -cfg.shift[2]);
379
380 // Write out the coefficients using the correct method for transforms of
381 // this size.
382 match row_class {
383 SizeClass1D::X8UP => {
384 // Store output in at most 32x32 chunks. See rust code for details.
385
386 // Output is grouped into 32x32 chunks so a stride of at most 32 is
387 // used for each chunk
388 let output_stride = txfm_size_row.min(32);
389
390 // Split the first 32 rows from the last 32 rows and offset by rg % 32
391 let output = &mut output[(rg & 31)
392 + (rg >= 32) as usize * output_stride * txfm_size_col.min(32)..];
393
394 for cg in (0..txfm_size_col).step_by(32) {
395 // Offset by zero or half of output
396 let output = &mut output[txfm_size_row * cg..];
397
398 for c in 0..txfm_size_col.min(32) {
399 match T::Pixel::type_enum() {
400 PixelType::U8 => {
401 let vec = row_coeffs[c + cg].vec();
402 vst1_u16_x2(
403 output[c * output_stride..].as_mut_ptr() as *mut _,
404 uint16x4x2_t(
405 vreinterpret_u16_s16(vmovn_s32(vec.0)),
406 vreinterpret_u16_s16(vmovn_s32(vec.1)),
407 ),
408 );
409 }
410 PixelType::U16 => {
411 vst1q_u32_x2(
412 output[c * output_stride..].as_mut_ptr() as *mut _,
413 vreinterpretq_u32_s32_x2(row_coeffs[c + cg].vec()),
414 );
415 }
416 }
417 }
418 }
419 }
420 SizeClass1D::X4 => {
421 // Write out coefficients in normal order - it isn't possible to have
422 // more than 32 rows.
423 for c in 0..txfm_size_col {
424 match T::Pixel::type_enum() {
425 PixelType::U8 => {
426 vst1_s16(
427 output[c * txfm_size_row + rg..].as_mut_ptr() as *mut _,
428 vmovn_s32(row_coeffs[c].vec().0),
429 );
430 }
431 PixelType::U16 => {
432 vst1q_s32(
433 output[c * txfm_size_row + rg..].as_mut_ptr() as *mut _,
434 row_coeffs[c].vec().0,
435 );
436 }
437 }
438 }
439 }
440 }
441 }
442 }
443
444 /// # Panics
445 ///
446 /// - If called with an invalid combination of `tx_size` and `tx_type`
447 #[inline]
448 pub fn forward_transform<T: Coefficient>(
449 input: &[i16], output: &mut [MaybeUninit<T>], stride: usize,
450 tx_size: TxSize, tx_type: TxType, bd: usize, cpu: CpuFeatureLevel,
451 ) {
452 assert!(valid_av1_transform(tx_size, tx_type));
453 if cpu >= CpuFeatureLevel::NEON {
454 // SAFETY: Calls Assembly code.
455 unsafe {
456 forward_transform_neon(input, output, stride, tx_size, tx_type, bd);
457 }
458 } else {
459 rust::forward_transform(input, output, stride, tx_size, tx_type, bd, cpu);
460 }
461 }