| 1 |
// Copyright (c) 2019-2022, The rav1e contributors. All rights reserved |
| 2 |
// |
| 3 |
// This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
// was not distributed with this source code in the LICENSE file, you can |
| 6 |
// obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
// Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
// PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
use crate::tiling::PlaneRegionMut; |
| 11 |
use crate::util::*; |
| 12 |
use std::mem::MaybeUninit; |
| 13 |
|
| 14 |
// Note: Input coeffs are mutable since the assembly uses them as a scratchpad |
| 15 |
pub type InvTxfmFunc = |
| 16 |
unsafe extern fn(*mut u8, libc::ptrdiff_t, *mut i16, i32); |
| 17 |
|
| 18 |
pub type InvTxfmHBDFunc = |
| 19 |
unsafe extern fn(*mut u16, libc::ptrdiff_t, *mut i16, i32, i32); |
| 20 |
|
| 21 |
pub fn call_inverse_func<T: Pixel>( |
| 22 |
func: InvTxfmFunc, input: &[T::Coeff], output: &mut PlaneRegionMut<'_, T>, |
| 23 |
eob: u16, width: usize, height: usize, bd: usize, |
| 24 |
) { |
| 25 |
debug_assert!(bd == 8); |
| 26 |
|
| 27 |
// Only use at most 32 columns and 32 rows of input coefficients. |
| 28 |
let input: &[T::Coeff] = &input[..width.min(32) * height.min(32)]; |
| 29 |
|
| 30 |
let mut copied = Aligned::<[MaybeUninit<T::Coeff>; 32 * 32]>::uninit_array(); |
| 31 |
|
| 32 |
// Convert input to 16-bits. |
| 33 |
// TODO: Remove by changing inverse assembly to not overwrite its input |
| 34 |
for (a, b) in copied.data.iter_mut().zip(input) { |
| 35 |
a.write(*b); |
| 36 |
} |
| 37 |
|
| 38 |
// perform the inverse transform |
| 39 |
// SAFETY: Calls Assembly code. |
| 40 |
unsafe { |
| 41 |
func( |
| 42 |
output.data_ptr_mut() as *mut _, |
| 43 |
output.plane_cfg.stride as isize, |
| 44 |
copied.data.as_mut_ptr() as *mut _, |
| 45 |
eob as i32 - 1, |
| 46 |
); |
| 47 |
} |
| 48 |
} |
| 49 |
|
| 50 |
pub fn call_inverse_hbd_func<T: Pixel>( |
| 51 |
func: InvTxfmHBDFunc, input: &[T::Coeff], |
| 52 |
output: &mut PlaneRegionMut<'_, T>, eob: u16, width: usize, height: usize, |
| 53 |
bd: usize, |
| 54 |
) { |
| 55 |
// Only use at most 32 columns and 32 rows of input coefficients. |
| 56 |
let input: &[T::Coeff] = &input[..width.min(32) * height.min(32)]; |
| 57 |
|
| 58 |
let mut copied = Aligned::<[MaybeUninit<T::Coeff>; 32 * 32]>::uninit_array(); |
| 59 |
|
| 60 |
// Convert input to 16-bits. |
| 61 |
// TODO: Remove by changing inverse assembly to not overwrite its input |
| 62 |
for (a, b) in copied.data.iter_mut().zip(input) { |
| 63 |
a.write(*b); |
| 64 |
} |
| 65 |
|
| 66 |
// perform the inverse transform |
| 67 |
// SAFETY: Calls Assembly code. |
| 68 |
unsafe { |
| 69 |
func( |
| 70 |
output.data_ptr_mut() as *mut _, |
| 71 |
T::to_asm_stride(output.plane_cfg.stride), |
| 72 |
copied.data.as_mut_ptr() as *mut _, |
| 73 |
eob as i32 - 1, |
| 74 |
(1 << bd) - 1, |
| 75 |
); |
| 76 |
} |
| 77 |
} |
| 78 |
|
| 79 |
#[cfg(test)] |
| 80 |
pub mod test { |
| 81 |
use super::*; |
| 82 |
use crate::context::av1_get_coded_tx_size; |
| 83 |
use crate::cpu_features::CpuFeatureLevel; |
| 84 |
use crate::frame::{AsRegion, Plane}; |
| 85 |
use crate::scan_order::av1_scan_orders; |
| 86 |
use crate::transform::TxSize::*; |
| 87 |
use crate::transform::*; |
| 88 |
use rand::{random, rng, Rng}; |
| 89 |
|
| 90 |
pub fn pick_eob<T: Coefficient>( |
| 91 |
coeffs: &mut [T], tx_size: TxSize, tx_type: TxType, sub_h: usize, |
| 92 |
) -> u16 { |
| 93 |
/* From dav1d |
| 94 |
* copy the topleft coefficients such that the return value (being the |
| 95 |
* coefficient scantable index for the eob token) guarantees that only |
| 96 |
* the topleft $sub out of $sz (where $sz >= $sub) coefficients in both |
| 97 |
* dimensions are non-zero. This leads to braching to specific optimized |
| 98 |
* simd versions (e.g. dc-only) so that we get full asm coverage in this |
| 99 |
* test */ |
| 100 |
let coeff_h = av1_get_coded_tx_size(tx_size).height(); |
| 101 |
let sub_high: usize = if sub_h > 0 { sub_h * 8 - 1 } else { 0 }; |
| 102 |
let sub_low: usize = if sub_h > 1 { sub_high - 8 } else { 0 }; |
| 103 |
let mut eob = 0u16; |
| 104 |
let mut exit = 0; |
| 105 |
|
| 106 |
let scan = av1_scan_orders[tx_size][tx_type].scan; |
| 107 |
|
| 108 |
for (i, &pos) in scan.iter().enumerate() { |
| 109 |
exit = i as u16; |
| 110 |
|
| 111 |
let rc = pos as usize; |
| 112 |
let rcx = rc % coeff_h; |
| 113 |
let rcy = rc / coeff_h; |
| 114 |
|
| 115 |
if rcx > sub_high || rcy > sub_high { |
| 116 |
break; |
| 117 |
} else if eob == 0 && (rcx > sub_low || rcy > sub_low) { |
| 118 |
eob = i as u16; |
| 119 |
} |
| 120 |
} |
| 121 |
|
| 122 |
if eob != 0 { |
| 123 |
eob += rng().random_range(0..(exit - eob).min(1)); |
| 124 |
} |
| 125 |
for &pos in scan.iter().skip(usize::from(eob)) { |
| 126 |
coeffs[pos as usize] = T::cast_from(0); |
| 127 |
} |
| 128 |
|
| 129 |
eob + 1 |
| 130 |
} |
| 131 |
|
| 132 |
pub fn test_transform<T: Pixel>( |
| 133 |
tx_size: TxSize, tx_type: TxType, bit_depth: usize, cpu: CpuFeatureLevel, |
| 134 |
) { |
| 135 |
let sub_h_iterations: usize = match tx_size.height().max(tx_size.width()) { |
| 136 |
4 => 2, |
| 137 |
8 => 2, |
| 138 |
16 => 3, |
| 139 |
32 | 64 => 4, |
| 140 |
_ => unreachable!(), |
| 141 |
}; |
| 142 |
|
| 143 |
for sub_h in 0..sub_h_iterations { |
| 144 |
let mut src_storage = [T::zero(); 64 * 64]; |
| 145 |
let src = &mut src_storage[..tx_size.area()]; |
| 146 |
let mut dst = Plane::from_slice( |
| 147 |
&[T::zero(); 64 * 64][..tx_size.area()], |
| 148 |
tx_size.width(), |
| 149 |
); |
| 150 |
let mut res = Aligned::<[MaybeUninit<i16>; 64 * 64]>::uninit_array(); |
| 151 |
let res = &mut res.data[..tx_size.area()]; |
| 152 |
let mut freq = |
| 153 |
Aligned::<[MaybeUninit<T::Coeff>; 64 * 64]>::uninit_array(); |
| 154 |
let freq = &mut freq.data[..tx_size.area()]; |
| 155 |
for ((r, s), d) in |
| 156 |
res.iter_mut().zip(src.iter_mut()).zip(dst.data.iter_mut()) |
| 157 |
{ |
| 158 |
*s = T::cast_from(random::<u16>() >> (16 - bit_depth)); |
| 159 |
*d = T::cast_from(random::<u16>() >> (16 - bit_depth)); |
| 160 |
r.write(i16::cast_from(*s) - i16::cast_from(*d)); |
| 161 |
} |
| 162 |
// SAFETY: The loop just initialized res, and all three slices have the same length |
| 163 |
let res = unsafe { slice_assume_init_mut(res) }; |
| 164 |
forward_transform( |
| 165 |
res, |
| 166 |
freq, |
| 167 |
tx_size.width(), |
| 168 |
tx_size, |
| 169 |
tx_type, |
| 170 |
bit_depth, |
| 171 |
CpuFeatureLevel::RUST, |
| 172 |
); |
| 173 |
// SAFETY: forward_transform initialized freq |
| 174 |
let freq = unsafe { slice_assume_init_mut(freq) }; |
| 175 |
|
| 176 |
let eob: u16 = pick_eob(freq, tx_size, tx_type, sub_h); |
| 177 |
let mut rust_dst = dst.clone(); |
| 178 |
|
| 179 |
inverse_transform_add( |
| 180 |
freq, |
| 181 |
&mut dst.as_region_mut(), |
| 182 |
eob, |
| 183 |
tx_size, |
| 184 |
tx_type, |
| 185 |
bit_depth, |
| 186 |
cpu, |
| 187 |
); |
| 188 |
inverse_transform_add( |
| 189 |
freq, |
| 190 |
&mut rust_dst.as_region_mut(), |
| 191 |
eob, |
| 192 |
tx_size, |
| 193 |
tx_type, |
| 194 |
bit_depth, |
| 195 |
CpuFeatureLevel::RUST, |
| 196 |
); |
| 197 |
assert_eq!(rust_dst.data_origin(), dst.data_origin()); |
| 198 |
} |
| 199 |
} |
| 200 |
|
| 201 |
macro_rules! test_itx_fns { |
| 202 |
([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr) => { |
| 203 |
paste::item! { |
| 204 |
$( |
| 205 |
$( |
| 206 |
#[test] |
| 207 |
fn [<inv_txfm2d_add_ $TYPE2 _$TYPE1 _$W x $H>]() { |
| 208 |
for &cpu in |
| 209 |
&CpuFeatureLevel::all()[..=CpuFeatureLevel::default().as_index()] |
| 210 |
{ |
| 211 |
test_transform::<u8>([<TX_ $W X $H>], $ENUM, 8, cpu); |
| 212 |
test_transform::<u16>([<TX_ $W X $H>], $ENUM, 10, cpu); |
| 213 |
test_transform::<u16>([<TX_ $W X $H>], $ENUM, 12, cpu); |
| 214 |
} |
| 215 |
} |
| 216 |
)* |
| 217 |
)* |
| 218 |
} |
| 219 |
}; |
| 220 |
|
| 221 |
($TYPES_VALID:tt, [$(($W:expr, $H:expr)),*]) => { |
| 222 |
$( |
| 223 |
test_itx_fns!($TYPES_VALID, $W, $H); |
| 224 |
)* |
| 225 |
}; |
| 226 |
|
| 227 |
($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt, |
| 228 |
$TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt) => { |
| 229 |
test_itx_fns!([$TYPES64], $DIMS64); |
| 230 |
test_itx_fns!([$TYPES64, $TYPES32], $DIMS32); |
| 231 |
test_itx_fns!([$TYPES64, $TYPES32, $TYPES16], $DIMS16); |
| 232 |
test_itx_fns!( |
| 233 |
[$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84 |
| 234 |
); |
| 235 |
test_itx_fns!( |
| 236 |
[$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4 |
| 237 |
); |
| 238 |
}; |
| 239 |
} |
| 240 |
|
| 241 |
test_itx_fns!( |
| 242 |
// 64x |
| 243 |
[(TxType::DCT_DCT, dct, dct)], |
| 244 |
[(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)], |
| 245 |
// 32x |
| 246 |
[(TxType::IDTX, identity, identity)], |
| 247 |
[(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)], |
| 248 |
// 16x16 |
| 249 |
[ |
| 250 |
(TxType::DCT_ADST, dct, adst), |
| 251 |
(TxType::ADST_DCT, adst, dct), |
| 252 |
(TxType::DCT_FLIPADST, dct, flipadst), |
| 253 |
(TxType::FLIPADST_DCT, flipadst, dct), |
| 254 |
(TxType::V_DCT, dct, identity), |
| 255 |
(TxType::H_DCT, identity, dct), |
| 256 |
(TxType::ADST_ADST, adst, adst), |
| 257 |
(TxType::ADST_FLIPADST, adst, flipadst), |
| 258 |
(TxType::FLIPADST_ADST, flipadst, adst), |
| 259 |
(TxType::FLIPADST_FLIPADST, flipadst, flipadst) |
| 260 |
], |
| 261 |
[(16, 16)], |
| 262 |
// 8x, 4x and 16x (minus 16x16 and 4x4) |
| 263 |
[ |
| 264 |
(TxType::V_ADST, adst, identity), |
| 265 |
(TxType::H_ADST, identity, adst), |
| 266 |
(TxType::V_FLIPADST, flipadst, identity), |
| 267 |
(TxType::H_FLIPADST, identity, flipadst) |
| 268 |
], |
| 269 |
[(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8)], |
| 270 |
// 4x4 |
| 271 |
[(TxType::WHT_WHT, wht, wht)], |
| 272 |
[(4, 4)] |
| 273 |
); |
| 274 |
} |