Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/cdef.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cdef::*;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::frame::*;
13 use crate::tiling::PlaneRegionMut;
14 use crate::util::*;
15
16 type CdefFilterFn = unsafe extern fn(
17 dst: *mut u8,
18 dst_stride: isize,
19 tmp: *const u16,
20 tmp_stride: isize,
21 pri_strength: i32,
22 sec_strength: i32,
23 dir: i32,
24 damping: i32,
25 );
26
27 type CdefFilterHBDFn = unsafe extern fn(
28 dst: *mut u16,
29 dst_stride: isize,
30 tmp: *const u16,
31 tmp_stride: isize,
32 pri_strength: i32,
33 sec_strength: i32,
34 dir: i32,
35 damping: i32,
36 bitdepth_max: i32,
37 );
38
39 #[inline(always)]
40 const fn decimate_index(xdec: usize, ydec: usize) -> usize {
41 ((ydec << 1) | xdec) & 3
42 }
43
44 pub(crate) unsafe fn cdef_filter_block<T: Pixel>(
45 dst: &mut PlaneRegionMut<'_, T>, src: *const T, src_stride: isize,
46 pri_strength: i32, sec_strength: i32, dir: usize, damping: i32,
47 bit_depth: usize, xdec: usize, ydec: usize, edges: u8, cpu: CpuFeatureLevel,
48 ) {
49 let call_rust = |dst: &mut PlaneRegionMut<T>| {
50 rust::cdef_filter_block(
51 dst,
52 src,
53 src_stride,
54 pri_strength,
55 sec_strength,
56 dir,
57 damping,
58 bit_depth,
59 xdec,
60 ydec,
61 edges,
62 cpu,
63 );
64 };
65
66 // TODO: handle padding in the fast path
67 if edges != CDEF_HAVE_ALL && matches!(T::type_enum(), PixelType::U16) {
68 call_rust(dst);
69 } else {
70 #[cfg(feature = "check_asm")]
71 let ref_dst = {
72 let mut copy = dst.scratch_copy();
73 call_rust(&mut copy.as_region_mut());
74 copy
75 };
76 match T::type_enum() {
77 PixelType::U8 => {
78 match CDEF_FILTER_FNS[cpu.as_index()][decimate_index(xdec, ydec)] {
79 Some(func) => {
80 // current cdef_filter_block asm does 16->8 for historical
81 // reasons. Copy into tmp space for now (also handling
82 // padding) until asm is updated
83 const TMPSTRIDE: isize =
84 std::mem::align_of::<Aligned<u16>>() as isize;
85 /* 256 or 512-bit alignment, greater than 2 * (8>>xdec) + 2 */
86 const TMPSIZE: usize =
87 ((2 + 8 + 2) * TMPSTRIDE + TMPSTRIDE) as usize;
88 let mut tmp: Aligned<[u16; TMPSIZE]> =
89 Aligned::new([CDEF_VERY_LARGE; TMPSIZE]);
90 rust::pad_into_tmp16(
91 tmp.data.as_mut_ptr().offset(TMPSTRIDE - 2), // points to
92 // *padding* upper left; the -2 is to make sure the
93 // block area is SIMD-aligned, not the padding
94 TMPSTRIDE,
95 src, // points to *block* upper left
96 src_stride,
97 8 >> xdec,
98 8 >> ydec,
99 edges,
100 );
101 (func)(
102 dst.data_ptr_mut() as *mut _,
103 T::to_asm_stride(dst.plane_cfg.stride),
104 tmp.data.as_ptr().offset(3 * TMPSTRIDE),
105 TMPSTRIDE,
106 pri_strength,
107 sec_strength,
108 dir as i32,
109 damping,
110 );
111 }
112 None => call_rust(dst),
113 }
114 }
115 PixelType::U16 => {
116 match CDEF_FILTER_HBD_FNS[cpu.as_index()][decimate_index(xdec, ydec)] {
117 Some(func) => {
118 (func)(
119 dst.data_ptr_mut() as *mut _,
120 T::to_asm_stride(dst.plane_cfg.stride),
121 src as *const _,
122 src_stride,
123 pri_strength,
124 sec_strength,
125 dir as i32,
126 damping,
127 (1 << bit_depth) - 1,
128 );
129 }
130 None => call_rust(dst),
131 }
132 }
133 }
134 #[cfg(feature = "check_asm")]
135 {
136 for (dst_row, ref_row) in
137 dst.rows_iter().zip(ref_dst.as_region().rows_iter())
138 {
139 for (dst, reference) in dst_row.iter().zip(ref_row) {
140 assert_eq!(*dst, *reference);
141 }
142 }
143 }
144 }
145 }
146
147 extern {
148 fn rav1e_cdef_filter_4x4_avx2(
149 dst: *mut u8, dst_stride: isize, tmp: *const u16, tmp_stride: isize,
150 pri_strength: i32, sec_strength: i32, dir: i32, damping: i32,
151 );
152
153 fn rav1e_cdef_filter_4x8_avx2(
154 dst: *mut u8, dst_stride: isize, tmp: *const u16, tmp_stride: isize,
155 pri_strength: i32, sec_strength: i32, dir: i32, damping: i32,
156 );
157
158 fn rav1e_cdef_filter_8x8_avx2(
159 dst: *mut u8, dst_stride: isize, tmp: *const u16, tmp_stride: isize,
160 pri_strength: i32, sec_strength: i32, dir: i32, damping: i32,
161 );
162 }
163
164 static CDEF_FILTER_FNS_AVX2: [Option<CdefFilterFn>; 4] = {
165 let mut out: [Option<CdefFilterFn>; 4] = [None; 4];
166 out[decimate_index(1, 1)] = Some(rav1e_cdef_filter_4x4_avx2);
167 out[decimate_index(1, 0)] = Some(rav1e_cdef_filter_4x8_avx2);
168 out[decimate_index(0, 0)] = Some(rav1e_cdef_filter_8x8_avx2);
169 out
170 };
171
172 cpu_function_lookup_table!(
173 CDEF_FILTER_FNS: [[Option<CdefFilterFn>; 4]],
174 default: [None; 4],
175 [AVX2]
176 );
177
178 cpu_function_lookup_table!(
179 CDEF_FILTER_HBD_FNS: [[Option<CdefFilterHBDFn>; 4]],
180 default: [None; 4],
181 []
182 );
183
184 type CdefDirLBDFn =
185 unsafe extern fn(tmp: *const u8, tmp_stride: isize, var: *mut u32) -> i32;
186 type CdefDirHBDFn = unsafe extern fn(
187 tmp: *const u16,
188 tmp_stride: isize,
189 var: *mut u32,
190 bitdepth_max: i32,
191 ) -> i32;
192
193 #[inline(always)]
194 #[allow(clippy::let_and_return)]
195 pub(crate) fn cdef_find_dir<T: Pixel>(
196 img: &PlaneSlice<'_, T>, var: &mut u32, coeff_shift: usize,
197 cpu: CpuFeatureLevel,
198 ) -> i32 {
199 let call_rust =
200 |var: &mut u32| rust::cdef_find_dir::<T>(img, var, coeff_shift, cpu);
201
202 #[cfg(feature = "check_asm")]
203 let (ref_dir, ref_var) = {
204 let mut var: u32 = 0;
205 let dir = call_rust(&mut var);
206 (dir, var)
207 };
208
209 let dir = match T::type_enum() {
210 PixelType::U8 => {
211 if let Some(func) = CDEF_DIR_LBD_FNS[cpu.as_index()] {
212 // SAFETY: Calls Assembly code.
213 unsafe {
214 (func)(
215 img.as_ptr() as *const _,
216 T::to_asm_stride(img.plane.cfg.stride),
217 var as *mut u32,
218 )
219 }
220 } else {
221 call_rust(var)
222 }
223 }
224 PixelType::U16 if coeff_shift > 0 => {
225 if let Some(func) = CDEF_DIR_HBD_FNS[cpu.as_index()] {
226 // SAFETY: Calls Assembly code.
227 unsafe {
228 (func)(
229 img.as_ptr() as *const _,
230 T::to_asm_stride(img.plane.cfg.stride),
231 var as *mut u32,
232 (1 << (coeff_shift + 8)) - 1,
233 )
234 }
235 } else {
236 call_rust(var)
237 }
238 }
239 _ => call_rust(var),
240 };
241
242 #[cfg(feature = "check_asm")]
243 {
244 assert_eq!(dir, ref_dir);
245 assert_eq!(*var, ref_var);
246 }
247
248 dir
249 }
250
251 extern {
252 fn rav1e_cdef_dir_8bpc_ssse3(
253 tmp: *const u8, tmp_stride: isize, var: *mut u32,
254 ) -> i32;
255
256 fn rav1e_cdef_dir_8bpc_avx2(
257 tmp: *const u8, tmp_stride: isize, var: *mut u32,
258 ) -> i32;
259
260 fn rav1e_cdef_dir_16bpc_ssse3(
261 tmp: *const u16, tmp_stride: isize, var: *mut u32, bitdepth_max: i32,
262 ) -> i32;
263
264 fn rav1e_cdef_dir_16bpc_sse4(
265 tmp: *const u16, tmp_stride: isize, var: *mut u32, bitdepth_max: i32,
266 ) -> i32;
267
268 fn rav1e_cdef_dir_16bpc_avx2(
269 tmp: *const u16, tmp_stride: isize, var: *mut u32, bitdepth_max: i32,
270 ) -> i32;
271 }
272
273 cpu_function_lookup_table!(
274 CDEF_DIR_LBD_FNS: [Option<CdefDirLBDFn>],
275 default: None,
276 [
277 (SSSE3, Some(rav1e_cdef_dir_8bpc_ssse3)),
278 (AVX2, Some(rav1e_cdef_dir_8bpc_avx2))
279 ]
280 );
281
282 cpu_function_lookup_table!(
283 CDEF_DIR_HBD_FNS: [Option<CdefDirHBDFn>],
284 default: None,
285 [
286 (SSSE3, Some(rav1e_cdef_dir_16bpc_ssse3)),
287 (SSE4_1, Some(rav1e_cdef_dir_16bpc_sse4)),
288 (AVX2, Some(rav1e_cdef_dir_16bpc_avx2))
289 ]
290 );
291
292 #[cfg(test)]
293 mod test {
294 pub const CDEF_HAVE_NONE: u8 = 0;
295 use super::*;
296 use interpolate_name::interpolate_test;
297 use rand::random;
298 use std::str::FromStr;
299
300 macro_rules! test_cdef_filter_block {
301 ($(($XDEC:expr, $YDEC:expr)),*, $OPT:ident, $OPTLIT:literal) => {
302 $(
303 paste::item! {
304 #[interpolate_test(dir_0, 0)]
305 #[interpolate_test(dir_1, 1)]
306 #[interpolate_test(dir_2, 2)]
307 #[interpolate_test(dir_3, 3)]
308 #[interpolate_test(dir_4, 4)]
309 #[interpolate_test(dir_5, 5)]
310 #[interpolate_test(dir_6, 6)]
311 #[interpolate_test(dir_7, 7)]
312 fn [<cdef_filter_block_dec_ $XDEC _ $YDEC _ $OPT>](dir: usize) {
313 if CpuFeatureLevel::default() < CpuFeatureLevel::from_str($OPTLIT).unwrap() {
314 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
315 return;
316 }
317
318 let width = 8 >> $XDEC;
319 let height = 8 >> $YDEC;
320 let area = width * height;
321 // dynamic allocation: test
322 let mut src = vec![0u8; area];
323 // dynamic allocation: test
324 let mut dst = Plane::from_slice(&vec![0u8; area], width);
325 for (s, d) in src.iter_mut().zip(dst.data.iter_mut()) {
326 *s = random::<u8>();
327 *d = random::<u8>();
328 }
329 let mut rust_dst = dst.clone();
330
331 let src_stride = width as isize;
332 let pri_strength = 1;
333 let sec_strength = 0;
334 let damping = 2;
335 let bit_depth = 8;
336
337 // SAFETY: Calling functions with raw pointers--we created the
338 // planes above and only read from the start.
339 unsafe {
340 cdef_filter_block(&mut dst.as_region_mut(), src.as_ptr(), src_stride, pri_strength, sec_strength, dir, damping, bit_depth, $XDEC, $YDEC, CDEF_HAVE_NONE, CpuFeatureLevel::from_str($OPTLIT).unwrap());
341 cdef_filter_block(&mut rust_dst.as_region_mut(), src.as_ptr(), src_stride, pri_strength, sec_strength, dir, damping, bit_depth, $XDEC, $YDEC, CDEF_HAVE_NONE, CpuFeatureLevel::RUST);
342 assert_eq!(rust_dst.data_origin(), dst.data_origin());
343 }
344 }
345 }
346 )*
347 }
348 }
349
350 macro_rules! test_cdef_filter_block_hbd {
351 ($(($XDEC:expr, $YDEC:expr)),*, $OPT:ident, $OPTLIT:literal) => {
352 $(
353 paste::item! {
354 #[interpolate_test(dir_0, 0)]
355 #[interpolate_test(dir_1, 1)]
356 #[interpolate_test(dir_2, 2)]
357 #[interpolate_test(dir_3, 3)]
358 #[interpolate_test(dir_4, 4)]
359 #[interpolate_test(dir_5, 5)]
360 #[interpolate_test(dir_6, 6)]
361 #[interpolate_test(dir_7, 7)]
362 fn [<cdef_filter_block_hbd_dec_ $XDEC _ $YDEC _ $OPT>](dir: usize) {
363 if !is_x86_feature_detected!($OPTLIT) {
364 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
365 return;
366 }
367
368 let width = 8 >> $XDEC;
369 let height = 8 >> $YDEC;
370 let area = width * height;
371 // dynamic allocation: test
372 let mut src = vec![0u16; area];
373 // dynamic allocation: test
374 let mut dst = Plane::from_slice(&vec![0u16; area], width);
375 for (s, d) in src.iter_mut().zip(dst.data.iter_mut()) {
376 *s = (random::<u8>() as u16) << 2;
377 *d = (random::<u8>() as u16) << 2;
378 }
379 let mut rust_dst = dst.clone();
380
381 let src_stride = width as isize;
382 let pri_strength = 1;
383 let sec_strength = 0;
384 let damping = 2;
385 let bit_depth = 10;
386
387 // SAFETY: Calling functions with raw pointers--we created the
388 // planes above and only read from the start.
389 unsafe {
390 cdef_filter_block(&mut dst.as_region_mut(), src.as_ptr(), src_stride, pri_strength, sec_strength, dir, damping, bit_depth, $XDEC, $YDEC, CDEF_HAVE_NONE, CpuFeatureLevel::from_str($OPTLIT).unwrap());
391 cdef_filter_block(&mut rust_dst.as_region_mut(), src.as_ptr(), src_stride, pri_strength, sec_strength, dir, damping, bit_depth, $XDEC, $YDEC, CDEF_HAVE_NONE, CpuFeatureLevel::RUST);
392 assert_eq!(rust_dst.data_origin(), dst.data_origin());
393 }
394 }
395 }
396 )*
397 }
398 }
399
400 macro_rules! test_cdef_dir {
401 ($OPT:ident, $OPTLIT:literal) => {
402 paste::item! {
403 #[test]
404 fn [<cdef_dir_ $OPT>]() {
405 use crate::context::{TileSuperBlockOffset, SuperBlockOffset};
406
407 if !is_x86_feature_detected!($OPTLIT) {
408 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
409 return;
410 }
411
412 let width = 8;
413 let height = 8;
414 let area = width * height;
415 // dynamic allocation: test
416 let mut src = vec![0u8; area];
417 for s in src.iter_mut() {
418 *s = random::<u8>();
419 }
420 let src = Plane::from_slice(&src, width);
421 let bit_depth = 8;
422 let coeff_shift = bit_depth - 8;
423 let sbo = TileSuperBlockOffset(SuperBlockOffset{ x:0, y:0 });
424
425 let mut var_asm: u32 = 0;
426 let mut var_rust: u32 = 0;
427 let in_po = sbo.plane_offset(&src.cfg);
428 let dir_asm = cdef_find_dir::<u8>(
429 &src.slice(in_po),
430 &mut var_asm,
431 coeff_shift,
432 CpuFeatureLevel::from_str($OPTLIT).unwrap(),
433 );
434 let dir_rust = cdef_find_dir::<u8>(
435 &src.slice(in_po),
436 &mut var_rust,
437 coeff_shift,
438 CpuFeatureLevel::RUST,
439 );
440 assert_eq!(var_asm, var_rust);
441 assert_eq!(dir_asm, dir_rust);
442 }
443 }
444 }
445 }
446
447 macro_rules! test_cdef_dir_hbd {
448 ($OPT:ident, $OPTLIT:literal) => {
449 paste::item! {
450 #[test]
451 fn [<cdef_dir_ $OPT _hbd>]() {
452 use crate::context::{TileSuperBlockOffset, SuperBlockOffset};
453
454 if !is_x86_feature_detected!($OPTLIT) {
455 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
456 return;
457 }
458
459 let width = 8;
460 let height = 8;
461 let area = width * height;
462 // dynamic allocation: test
463 let mut src = vec![0u16; area];
464 for s in src.iter_mut() {
465 *s = (random::<u8>() as u16) << 2;
466 }
467 let src = Plane::from_slice(&src, width);
468 let bit_depth = 10;
469 let coeff_shift = bit_depth - 8;
470 let sbo = TileSuperBlockOffset(SuperBlockOffset{ x:0, y:0 });
471
472 let mut var_asm: u32 = 0;
473 let mut var_rust: u32 = 0;
474 let in_po = sbo.plane_offset(&src.cfg);
475 let dir_asm = cdef_find_dir::<u16>(
476 &src.slice(in_po),
477 &mut var_asm,
478 coeff_shift,
479 CpuFeatureLevel::from_str($OPTLIT).unwrap(),
480 );
481 let dir_rust = cdef_find_dir::<u16>(
482 &src.slice(in_po),
483 &mut var_rust,
484 coeff_shift,
485 CpuFeatureLevel::RUST,
486 );
487 assert_eq!(var_asm, var_rust);
488 assert_eq!(dir_asm, dir_rust);
489 }
490 }
491 }
492 }
493
494 test_cdef_filter_block!((1, 1), (1, 0), (0, 0), avx2, "avx2");
495 test_cdef_filter_block_hbd!((1, 1), (1, 0), (0, 0), avx2, "avx2");
496 test_cdef_dir!(ssse3, "ssse3");
497 test_cdef_dir!(avx2, "avx2");
498 test_cdef_dir_hbd!(ssse3, "ssse3");
499 test_cdef_dir_hbd!(sse4, "sse4.1");
500 test_cdef_dir_hbd!(avx2, "avx2");
501 }