Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/quantize.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 #[cfg(target_arch = "x86")]
11 use std::arch::x86::*;
12 #[cfg(target_arch = "x86_64")]
13 use std::arch::x86_64::*;
14
15 use crate::context::av1_get_coded_tx_size;
16 use crate::cpu_features::CpuFeatureLevel;
17 use crate::quantize::*;
18 use crate::transform::TxSize;
19 use crate::util::*;
20 use std::mem::MaybeUninit;
21
22 type DequantizeFn = unsafe fn(
23 qindex: u8,
24 coeffs_ptr: *const i16,
25 _eob: u16,
26 rcoeffs_ptr: *mut i16,
27 tx_size: TxSize,
28 bit_depth: usize,
29 dc_delta_q: i8,
30 ac_delta_q: i8,
31 );
32
33 cpu_function_lookup_table!(
34 DEQUANTIZE_FNS: [Option<DequantizeFn>],
35 default: None,
36 [(AVX2, Some(dequantize_avx2))]
37 );
38
39 #[inline(always)]
40 pub fn dequantize<T: Coefficient>(
41 qindex: u8, coeffs: &[T], eob: u16, rcoeffs: &mut [MaybeUninit<T>],
42 tx_size: TxSize, bit_depth: usize, dc_delta_q: i8, ac_delta_q: i8,
43 cpu: CpuFeatureLevel,
44 ) {
45 let call_rust = |rcoeffs: &mut [MaybeUninit<T>]| {
46 crate::quantize::rust::dequantize(
47 qindex, coeffs, eob, rcoeffs, tx_size, bit_depth, dc_delta_q,
48 ac_delta_q, cpu,
49 );
50 };
51
52 #[cfg(any(feature = "check_asm", test))]
53 let mut ref_rcoeffs = {
54 let area = av1_get_coded_tx_size(tx_size).area();
55 let mut copy = vec![MaybeUninit::new(T::cast_from(0)); area];
56 call_rust(&mut copy);
57 copy
58 };
59
60 match T::Pixel::type_enum() {
61 PixelType::U8 => {
62 if let Some(func) = DEQUANTIZE_FNS[cpu.as_index()] {
63 // SAFETY: Calls Assembly code.
64 unsafe {
65 (func)(
66 qindex,
67 coeffs.as_ptr() as *const _,
68 eob,
69 rcoeffs.as_mut_ptr() as *mut _,
70 tx_size,
71 bit_depth,
72 dc_delta_q,
73 ac_delta_q,
74 )
75 }
76 } else {
77 call_rust(rcoeffs)
78 }
79 }
80 PixelType::U16 => call_rust(rcoeffs),
81 }
82
83 #[cfg(any(feature = "check_asm", test))]
84 {
85 let area = av1_get_coded_tx_size(tx_size).area();
86 let rcoeffs = unsafe { slice_assume_init_mut(&mut rcoeffs[..area]) };
87 let ref_rcoeffs = unsafe { slice_assume_init_mut(&mut ref_rcoeffs[..]) };
88 assert_eq!(rcoeffs, ref_rcoeffs);
89 }
90 }
91
92 #[target_feature(enable = "avx2")]
93 unsafe fn dequantize_avx2(
94 qindex: u8, coeffs_ptr: *const i16, _eob: u16, rcoeffs_ptr: *mut i16,
95 tx_size: TxSize, bit_depth: usize, dc_delta_q: i8, ac_delta_q: i8,
96 ) {
97 let log_tx_scale = _mm256_set1_epi32(get_log_tx_scale(tx_size) as i32);
98
99 let quants_ac =
100 _mm256_set1_epi32(ac_q(qindex, ac_delta_q, bit_depth).get() as i32);
101 // Use the dc quantize as first vector element for the first iteration
102 let mut quants = _mm256_insert_epi32(
103 quants_ac,
104 dc_q(qindex, dc_delta_q, bit_depth).get() as i32,
105 0,
106 );
107
108 let area: usize = av1_get_coded_tx_size(tx_size).area();
109 // Step by 16 (256/16) coefficients for each iteration
110 let step: usize = 16;
111 assert!(area >= step);
112
113 // Increase the pointers as we iterate
114 let mut coeffs_ptr: *const i16 = coeffs_ptr;
115 let mut rcoeffs_ptr: *mut i16 = rcoeffs_ptr;
116 for _i in (0..area).step_by(step) {
117 let coeffs = _mm256_load_si256(coeffs_ptr as *const _);
118 let coeffs_abs = _mm256_abs_epi16(coeffs);
119
120 // TODO: Since log_tx_scale is at most 2 and we gain an extra bit by taking
121 // the abs value (unless the range is [-(2^15), 2^15 + 1]), it might be
122 // possible to perform a 16-bit multiply and get the highest bit by
123 // comparing coeffs to (1<<16) / quant. The would cost 1 compare, 1 blend,
124 // and 1 add, but would get rid of 1 pack, 2 unpacks, 1 shift, and 1
125 // multiply.
126
127 let rcoeffs = _mm256_sign_epi16(
128 _mm256_packs_epi32(
129 // (abs_coeff * quant) >> log_tx_scale
130 _mm256_srlv_epi32(
131 _mm256_madd_epi16(
132 quants,
133 // Convert the first half of each lane to 32-bits
134 _mm256_unpacklo_epi16(coeffs_abs, _mm256_setzero_si256()),
135 ),
136 log_tx_scale,
137 ),
138 // Second half
139 _mm256_srlv_epi32(
140 _mm256_madd_epi16(
141 quants_ac,
142 _mm256_unpackhi_epi16(coeffs_abs, _mm256_setzero_si256()),
143 ),
144 log_tx_scale,
145 ),
146 ),
147 coeffs,
148 );
149 _mm256_store_si256(rcoeffs_ptr as *mut _, rcoeffs);
150
151 // Only use a dc quantizer for the first iteration
152 quants = quants_ac;
153 coeffs_ptr = coeffs_ptr.add(step);
154 rcoeffs_ptr = rcoeffs_ptr.add(step);
155 }
156 }
157
158 #[cfg(test)]
159 mod test {
160 use super::*;
161 use rand::distr::{Distribution, Uniform};
162 use rand::{rng, Rng};
163
164 #[test]
165 fn dequantize_test() {
166 let mut rng = rng();
167
168 use TxSize::*;
169 let tx_sizes = [
170 TX_4X4, TX_8X8, TX_16X16, TX_32X32, TX_64X64, TX_4X8, TX_8X4, TX_8X16,
171 TX_16X8, TX_16X32, TX_32X16, TX_32X64, TX_64X32, TX_4X16, TX_16X4,
172 TX_8X32, TX_32X8, TX_16X64, TX_64X16,
173 ];
174
175 let bd: usize = 8;
176
177 for &tx_size in &tx_sizes {
178 let qindex: u8 = rng.random_range((MINQ as u8)..(MAXQ as u8));
179 let dc_quant = dc_q(qindex, 0, bd).get() as i16;
180 let ac_quant = ac_q(qindex, 0, bd).get() as i16;
181
182 // Test the min, max, and random eobs
183 let eobs = {
184 let mut out = [0u16; 16];
185 let area: usize = av1_get_coded_tx_size(tx_size).area();
186 out[0] = 0;
187 out[1] = area as u16;
188 for eob in out.iter_mut().skip(2) {
189 *eob = rng.random_range(0..area as u16);
190 }
191 out
192 };
193
194 for &eob in &eobs {
195 let mut qcoeffs = Aligned::new([0i16; 32 * 32]);
196 let mut rcoeffs = Aligned::new([MaybeUninit::new(0i16); 32 * 32]);
197
198 // Generate quantized coefficients up to the eob
199 // SAFETY: always correct
200 let between = Uniform::new_inclusive(-i16::MAX, i16::MAX).unwrap();
201 for (i, qcoeff) in
202 qcoeffs.data.iter_mut().enumerate().take(eob as usize)
203 {
204 *qcoeff = between.sample(&mut rng)
205 / if i == 0 { dc_quant } else { ac_quant };
206 }
207
208 // Rely on quantize's internal tests
209 dequantize(
210 qindex,
211 &qcoeffs.data,
212 eob,
213 &mut rcoeffs.data,
214 tx_size,
215 bd,
216 0,
217 0,
218 CpuFeatureLevel::default(),
219 );
220 }
221 }
222 }
223 }