Debian Codegraph @

.cargo/vendor/rav1e/src/asm/shared/transform/inverse.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::tiling::PlaneRegionMut;
11 use crate::util::*;
12 use std::mem::MaybeUninit;
13
14 // Note: Input coeffs are mutable since the assembly uses them as a scratchpad
15 pub type InvTxfmFunc =
16 unsafe extern fn(*mut u8, libc::ptrdiff_t, *mut i16, i32);
17
18 pub type InvTxfmHBDFunc =
19 unsafe extern fn(*mut u16, libc::ptrdiff_t, *mut i16, i32, i32);
20
21 pub fn call_inverse_func<T: Pixel>(
22 func: InvTxfmFunc, input: &[T::Coeff], output: &mut PlaneRegionMut<'_, T>,
23 eob: u16, width: usize, height: usize, bd: usize,
24 ) {
25 debug_assert!(bd == 8);
26
27 // Only use at most 32 columns and 32 rows of input coefficients.
28 let input: &[T::Coeff] = &input[..width.min(32) * height.min(32)];
29
30 let mut copied = Aligned::<[MaybeUninit<T::Coeff>; 32 * 32]>::uninit_array();
31
32 // Convert input to 16-bits.
33 // TODO: Remove by changing inverse assembly to not overwrite its input
34 for (a, b) in copied.data.iter_mut().zip(input) {
35 a.write(*b);
36 }
37
38 // perform the inverse transform
39 // SAFETY: Calls Assembly code.
40 unsafe {
41 func(
42 output.data_ptr_mut() as *mut _,
43 output.plane_cfg.stride as isize,
44 copied.data.as_mut_ptr() as *mut _,
45 eob as i32 - 1,
46 );
47 }
48 }
49
50 pub fn call_inverse_hbd_func<T: Pixel>(
51 func: InvTxfmHBDFunc, input: &[T::Coeff],
52 output: &mut PlaneRegionMut<'_, T>, eob: u16, width: usize, height: usize,
53 bd: usize,
54 ) {
55 // Only use at most 32 columns and 32 rows of input coefficients.
56 let input: &[T::Coeff] = &input[..width.min(32) * height.min(32)];
57
58 let mut copied = Aligned::<[MaybeUninit<T::Coeff>; 32 * 32]>::uninit_array();
59
60 // Convert input to 16-bits.
61 // TODO: Remove by changing inverse assembly to not overwrite its input
62 for (a, b) in copied.data.iter_mut().zip(input) {
63 a.write(*b);
64 }
65
66 // perform the inverse transform
67 // SAFETY: Calls Assembly code.
68 unsafe {
69 func(
70 output.data_ptr_mut() as *mut _,
71 T::to_asm_stride(output.plane_cfg.stride),
72 copied.data.as_mut_ptr() as *mut _,
73 eob as i32 - 1,
74 (1 << bd) - 1,
75 );
76 }
77 }
78
79 #[cfg(test)]
80 pub mod test {
81 use super::*;
82 use crate::context::av1_get_coded_tx_size;
83 use crate::cpu_features::CpuFeatureLevel;
84 use crate::frame::{AsRegion, Plane};
85 use crate::scan_order::av1_scan_orders;
86 use crate::transform::TxSize::*;
87 use crate::transform::*;
88 use rand::{random, rng, Rng};
89
90 pub fn pick_eob<T: Coefficient>(
91 coeffs: &mut [T], tx_size: TxSize, tx_type: TxType, sub_h: usize,
92 ) -> u16 {
93 /* From dav1d
94 * copy the topleft coefficients such that the return value (being the
95 * coefficient scantable index for the eob token) guarantees that only
96 * the topleft $sub out of $sz (where $sz >= $sub) coefficients in both
97 * dimensions are non-zero. This leads to braching to specific optimized
98 * simd versions (e.g. dc-only) so that we get full asm coverage in this
99 * test */
100 let coeff_h = av1_get_coded_tx_size(tx_size).height();
101 let sub_high: usize = if sub_h > 0 { sub_h * 8 - 1 } else { 0 };
102 let sub_low: usize = if sub_h > 1 { sub_high - 8 } else { 0 };
103 let mut eob = 0u16;
104 let mut exit = 0;
105
106 let scan = av1_scan_orders[tx_size][tx_type].scan;
107
108 for (i, &pos) in scan.iter().enumerate() {
109 exit = i as u16;
110
111 let rc = pos as usize;
112 let rcx = rc % coeff_h;
113 let rcy = rc / coeff_h;
114
115 if rcx > sub_high || rcy > sub_high {
116 break;
117 } else if eob == 0 && (rcx > sub_low || rcy > sub_low) {
118 eob = i as u16;
119 }
120 }
121
122 if eob != 0 {
123 eob += rng().random_range(0..(exit - eob).min(1));
124 }
125 for &pos in scan.iter().skip(usize::from(eob)) {
126 coeffs[pos as usize] = T::cast_from(0);
127 }
128
129 eob + 1
130 }
131
132 pub fn test_transform<T: Pixel>(
133 tx_size: TxSize, tx_type: TxType, bit_depth: usize, cpu: CpuFeatureLevel,
134 ) {
135 let sub_h_iterations: usize = match tx_size.height().max(tx_size.width()) {
136 4 => 2,
137 8 => 2,
138 16 => 3,
139 32 | 64 => 4,
140 _ => unreachable!(),
141 };
142
143 for sub_h in 0..sub_h_iterations {
144 let mut src_storage = [T::zero(); 64 * 64];
145 let src = &mut src_storage[..tx_size.area()];
146 let mut dst = Plane::from_slice(
147 &[T::zero(); 64 * 64][..tx_size.area()],
148 tx_size.width(),
149 );
150 let mut res = Aligned::<[MaybeUninit<i16>; 64 * 64]>::uninit_array();
151 let res = &mut res.data[..tx_size.area()];
152 let mut freq =
153 Aligned::<[MaybeUninit<T::Coeff>; 64 * 64]>::uninit_array();
154 let freq = &mut freq.data[..tx_size.area()];
155 for ((r, s), d) in
156 res.iter_mut().zip(src.iter_mut()).zip(dst.data.iter_mut())
157 {
158 *s = T::cast_from(random::<u16>() >> (16 - bit_depth));
159 *d = T::cast_from(random::<u16>() >> (16 - bit_depth));
160 r.write(i16::cast_from(*s) - i16::cast_from(*d));
161 }
162 // SAFETY: The loop just initialized res, and all three slices have the same length
163 let res = unsafe { slice_assume_init_mut(res) };
164 forward_transform(
165 res,
166 freq,
167 tx_size.width(),
168 tx_size,
169 tx_type,
170 bit_depth,
171 CpuFeatureLevel::RUST,
172 );
173 // SAFETY: forward_transform initialized freq
174 let freq = unsafe { slice_assume_init_mut(freq) };
175
176 let eob: u16 = pick_eob(freq, tx_size, tx_type, sub_h);
177 let mut rust_dst = dst.clone();
178
179 inverse_transform_add(
180 freq,
181 &mut dst.as_region_mut(),
182 eob,
183 tx_size,
184 tx_type,
185 bit_depth,
186 cpu,
187 );
188 inverse_transform_add(
189 freq,
190 &mut rust_dst.as_region_mut(),
191 eob,
192 tx_size,
193 tx_type,
194 bit_depth,
195 CpuFeatureLevel::RUST,
196 );
197 assert_eq!(rust_dst.data_origin(), dst.data_origin());
198 }
199 }
200
201 macro_rules! test_itx_fns {
202 ([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr) => {
203 paste::item! {
204 $(
205 $(
206 #[test]
207 fn [<inv_txfm2d_add_ $TYPE2 _$TYPE1 _$W x $H>]() {
208 for &cpu in
209 &CpuFeatureLevel::all()[..=CpuFeatureLevel::default().as_index()]
210 {
211 test_transform::<u8>([<TX_ $W X $H>], $ENUM, 8, cpu);
212 test_transform::<u16>([<TX_ $W X $H>], $ENUM, 10, cpu);
213 test_transform::<u16>([<TX_ $W X $H>], $ENUM, 12, cpu);
214 }
215 }
216 )*
217 )*
218 }
219 };
220
221 ($TYPES_VALID:tt, [$(($W:expr, $H:expr)),*]) => {
222 $(
223 test_itx_fns!($TYPES_VALID, $W, $H);
224 )*
225 };
226
227 ($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
228 $TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt) => {
229 test_itx_fns!([$TYPES64], $DIMS64);
230 test_itx_fns!([$TYPES64, $TYPES32], $DIMS32);
231 test_itx_fns!([$TYPES64, $TYPES32, $TYPES16], $DIMS16);
232 test_itx_fns!(
233 [$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84
234 );
235 test_itx_fns!(
236 [$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4
237 );
238 };
239 }
240
241 test_itx_fns!(
242 // 64x
243 [(TxType::DCT_DCT, dct, dct)],
244 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
245 // 32x
246 [(TxType::IDTX, identity, identity)],
247 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
248 // 16x16
249 [
250 (TxType::DCT_ADST, dct, adst),
251 (TxType::ADST_DCT, adst, dct),
252 (TxType::DCT_FLIPADST, dct, flipadst),
253 (TxType::FLIPADST_DCT, flipadst, dct),
254 (TxType::V_DCT, dct, identity),
255 (TxType::H_DCT, identity, dct),
256 (TxType::ADST_ADST, adst, adst),
257 (TxType::ADST_FLIPADST, adst, flipadst),
258 (TxType::FLIPADST_ADST, flipadst, adst),
259 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
260 ],
261 [(16, 16)],
262 // 8x, 4x and 16x (minus 16x16 and 4x4)
263 [
264 (TxType::V_ADST, adst, identity),
265 (TxType::H_ADST, identity, adst),
266 (TxType::V_FLIPADST, flipadst, identity),
267 (TxType::H_FLIPADST, identity, flipadst)
268 ],
269 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8)],
270 // 4x4
271 [(TxType::WHT_WHT, wht, wht)],
272 [(4, 4)]
273 );
274 }