| 1 |
// Copyright (c) 2019-2022, The rav1e contributors. All rights reserved |
| 2 |
// |
| 3 |
// This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
// was not distributed with this source code in the LICENSE file, you can |
| 6 |
// obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
// Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
// PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
#[cfg(target_arch = "x86")] |
| 11 |
use std::arch::x86::*; |
| 12 |
#[cfg(target_arch = "x86_64")] |
| 13 |
use std::arch::x86_64::*; |
| 14 |
|
| 15 |
use crate::context::av1_get_coded_tx_size; |
| 16 |
use crate::cpu_features::CpuFeatureLevel; |
| 17 |
use crate::quantize::*; |
| 18 |
use crate::transform::TxSize; |
| 19 |
use crate::util::*; |
| 20 |
use std::mem::MaybeUninit; |
| 21 |
|
| 22 |
type DequantizeFn = unsafe fn( |
| 23 |
qindex: u8, |
| 24 |
coeffs_ptr: *const i16, |
| 25 |
_eob: u16, |
| 26 |
rcoeffs_ptr: *mut i16, |
| 27 |
tx_size: TxSize, |
| 28 |
bit_depth: usize, |
| 29 |
dc_delta_q: i8, |
| 30 |
ac_delta_q: i8, |
| 31 |
); |
| 32 |
|
| 33 |
cpu_function_lookup_table!( |
| 34 |
DEQUANTIZE_FNS: [Option<DequantizeFn>], |
| 35 |
default: None, |
| 36 |
[(AVX2, Some(dequantize_avx2))] |
| 37 |
); |
| 38 |
|
| 39 |
#[inline(always)] |
| 40 |
pub fn dequantize<T: Coefficient>( |
| 41 |
qindex: u8, coeffs: &[T], eob: u16, rcoeffs: &mut [MaybeUninit<T>], |
| 42 |
tx_size: TxSize, bit_depth: usize, dc_delta_q: i8, ac_delta_q: i8, |
| 43 |
cpu: CpuFeatureLevel, |
| 44 |
) { |
| 45 |
let call_rust = |rcoeffs: &mut [MaybeUninit<T>]| { |
| 46 |
crate::quantize::rust::dequantize( |
| 47 |
qindex, coeffs, eob, rcoeffs, tx_size, bit_depth, dc_delta_q, |
| 48 |
ac_delta_q, cpu, |
| 49 |
); |
| 50 |
}; |
| 51 |
|
| 52 |
#[cfg(any(feature = "check_asm", test))] |
| 53 |
let mut ref_rcoeffs = { |
| 54 |
let area = av1_get_coded_tx_size(tx_size).area(); |
| 55 |
let mut copy = vec![MaybeUninit::new(T::cast_from(0)); area]; |
| 56 |
call_rust(&mut copy); |
| 57 |
copy |
| 58 |
}; |
| 59 |
|
| 60 |
match T::Pixel::type_enum() { |
| 61 |
PixelType::U8 => { |
| 62 |
if let Some(func) = DEQUANTIZE_FNS[cpu.as_index()] { |
| 63 |
// SAFETY: Calls Assembly code. |
| 64 |
unsafe { |
| 65 |
(func)( |
| 66 |
qindex, |
| 67 |
coeffs.as_ptr() as *const _, |
| 68 |
eob, |
| 69 |
rcoeffs.as_mut_ptr() as *mut _, |
| 70 |
tx_size, |
| 71 |
bit_depth, |
| 72 |
dc_delta_q, |
| 73 |
ac_delta_q, |
| 74 |
) |
| 75 |
} |
| 76 |
} else { |
| 77 |
call_rust(rcoeffs) |
| 78 |
} |
| 79 |
} |
| 80 |
PixelType::U16 => call_rust(rcoeffs), |
| 81 |
} |
| 82 |
|
| 83 |
#[cfg(any(feature = "check_asm", test))] |
| 84 |
{ |
| 85 |
let area = av1_get_coded_tx_size(tx_size).area(); |
| 86 |
let rcoeffs = unsafe { slice_assume_init_mut(&mut rcoeffs[..area]) }; |
| 87 |
let ref_rcoeffs = unsafe { slice_assume_init_mut(&mut ref_rcoeffs[..]) }; |
| 88 |
assert_eq!(rcoeffs, ref_rcoeffs); |
| 89 |
} |
| 90 |
} |
| 91 |
|
| 92 |
#[target_feature(enable = "avx2")] |
| 93 |
unsafe fn dequantize_avx2( |
| 94 |
qindex: u8, coeffs_ptr: *const i16, _eob: u16, rcoeffs_ptr: *mut i16, |
| 95 |
tx_size: TxSize, bit_depth: usize, dc_delta_q: i8, ac_delta_q: i8, |
| 96 |
) { |
| 97 |
let log_tx_scale = _mm256_set1_epi32(get_log_tx_scale(tx_size) as i32); |
| 98 |
|
| 99 |
let quants_ac = |
| 100 |
_mm256_set1_epi32(ac_q(qindex, ac_delta_q, bit_depth).get() as i32); |
| 101 |
// Use the dc quantize as first vector element for the first iteration |
| 102 |
let mut quants = _mm256_insert_epi32( |
| 103 |
quants_ac, |
| 104 |
dc_q(qindex, dc_delta_q, bit_depth).get() as i32, |
| 105 |
0, |
| 106 |
); |
| 107 |
|
| 108 |
let area: usize = av1_get_coded_tx_size(tx_size).area(); |
| 109 |
// Step by 16 (256/16) coefficients for each iteration |
| 110 |
let step: usize = 16; |
| 111 |
assert!(area >= step); |
| 112 |
|
| 113 |
// Increase the pointers as we iterate |
| 114 |
let mut coeffs_ptr: *const i16 = coeffs_ptr; |
| 115 |
let mut rcoeffs_ptr: *mut i16 = rcoeffs_ptr; |
| 116 |
for _i in (0..area).step_by(step) { |
| 117 |
let coeffs = _mm256_load_si256(coeffs_ptr as *const _); |
| 118 |
let coeffs_abs = _mm256_abs_epi16(coeffs); |
| 119 |
|
| 120 |
// TODO: Since log_tx_scale is at most 2 and we gain an extra bit by taking |
| 121 |
// the abs value (unless the range is [-(2^15), 2^15 + 1]), it might be |
| 122 |
// possible to perform a 16-bit multiply and get the highest bit by |
| 123 |
// comparing coeffs to (1<<16) / quant. The would cost 1 compare, 1 blend, |
| 124 |
// and 1 add, but would get rid of 1 pack, 2 unpacks, 1 shift, and 1 |
| 125 |
// multiply. |
| 126 |
|
| 127 |
let rcoeffs = _mm256_sign_epi16( |
| 128 |
_mm256_packs_epi32( |
| 129 |
// (abs_coeff * quant) >> log_tx_scale |
| 130 |
_mm256_srlv_epi32( |
| 131 |
_mm256_madd_epi16( |
| 132 |
quants, |
| 133 |
// Convert the first half of each lane to 32-bits |
| 134 |
_mm256_unpacklo_epi16(coeffs_abs, _mm256_setzero_si256()), |
| 135 |
), |
| 136 |
log_tx_scale, |
| 137 |
), |
| 138 |
// Second half |
| 139 |
_mm256_srlv_epi32( |
| 140 |
_mm256_madd_epi16( |
| 141 |
quants_ac, |
| 142 |
_mm256_unpackhi_epi16(coeffs_abs, _mm256_setzero_si256()), |
| 143 |
), |
| 144 |
log_tx_scale, |
| 145 |
), |
| 146 |
), |
| 147 |
coeffs, |
| 148 |
); |
| 149 |
_mm256_store_si256(rcoeffs_ptr as *mut _, rcoeffs); |
| 150 |
|
| 151 |
// Only use a dc quantizer for the first iteration |
| 152 |
quants = quants_ac; |
| 153 |
coeffs_ptr = coeffs_ptr.add(step); |
| 154 |
rcoeffs_ptr = rcoeffs_ptr.add(step); |
| 155 |
} |
| 156 |
} |
| 157 |
|
| 158 |
#[cfg(test)] |
| 159 |
mod test { |
| 160 |
use super::*; |
| 161 |
use rand::distr::{Distribution, Uniform}; |
| 162 |
use rand::{rng, Rng}; |
| 163 |
|
| 164 |
#[test] |
| 165 |
fn dequantize_test() { |
| 166 |
let mut rng = rng(); |
| 167 |
|
| 168 |
use TxSize::*; |
| 169 |
let tx_sizes = [ |
| 170 |
TX_4X4, TX_8X8, TX_16X16, TX_32X32, TX_64X64, TX_4X8, TX_8X4, TX_8X16, |
| 171 |
TX_16X8, TX_16X32, TX_32X16, TX_32X64, TX_64X32, TX_4X16, TX_16X4, |
| 172 |
TX_8X32, TX_32X8, TX_16X64, TX_64X16, |
| 173 |
]; |
| 174 |
|
| 175 |
let bd: usize = 8; |
| 176 |
|
| 177 |
for &tx_size in &tx_sizes { |
| 178 |
let qindex: u8 = rng.random_range((MINQ as u8)..(MAXQ as u8)); |
| 179 |
let dc_quant = dc_q(qindex, 0, bd).get() as i16; |
| 180 |
let ac_quant = ac_q(qindex, 0, bd).get() as i16; |
| 181 |
|
| 182 |
// Test the min, max, and random eobs |
| 183 |
let eobs = { |
| 184 |
let mut out = [0u16; 16]; |
| 185 |
let area: usize = av1_get_coded_tx_size(tx_size).area(); |
| 186 |
out[0] = 0; |
| 187 |
out[1] = area as u16; |
| 188 |
for eob in out.iter_mut().skip(2) { |
| 189 |
*eob = rng.random_range(0..area as u16); |
| 190 |
} |
| 191 |
out |
| 192 |
}; |
| 193 |
|
| 194 |
for &eob in &eobs { |
| 195 |
let mut qcoeffs = Aligned::new([0i16; 32 * 32]); |
| 196 |
let mut rcoeffs = Aligned::new([MaybeUninit::new(0i16); 32 * 32]); |
| 197 |
|
| 198 |
// Generate quantized coefficients up to the eob |
| 199 |
// SAFETY: always correct |
| 200 |
let between = Uniform::new_inclusive(-i16::MAX, i16::MAX).unwrap(); |
| 201 |
for (i, qcoeff) in |
| 202 |
qcoeffs.data.iter_mut().enumerate().take(eob as usize) |
| 203 |
{ |
| 204 |
*qcoeff = between.sample(&mut rng) |
| 205 |
/ if i == 0 { dc_quant } else { ac_quant }; |
| 206 |
} |
| 207 |
|
| 208 |
// Rely on quantize's internal tests |
| 209 |
dequantize( |
| 210 |
qindex, |
| 211 |
&qcoeffs.data, |
| 212 |
eob, |
| 213 |
&mut rcoeffs.data, |
| 214 |
tx_size, |
| 215 |
bd, |
| 216 |
0, |
| 217 |
0, |
| 218 |
CpuFeatureLevel::default(), |
| 219 |
); |
| 220 |
} |
| 221 |
} |
| 222 |
} |
| 223 |
} |