Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/dist/cdef_dist.rs

raw
1 // Copyright (c) 2023, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::activity::apply_ssim_boost;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::dist::*;
13 use crate::tiling::PlaneRegion;
14 use crate::util::Pixel;
15 use crate::util::PixelType;
16
17 type CdefDistKernelFn = unsafe extern fn(
18 src: *const u8,
19 src_stride: isize,
20 dst: *const u8,
21 dst_stride: isize,
22 ret_ptr: *mut u32,
23 );
24
25 type CdefDistKernelHBDFn = unsafe extern fn(
26 src: *const u16,
27 src_stride: isize,
28 dst: *const u16,
29 dst_stride: isize,
30 ret_ptr: *mut u32,
31 );
32
33 extern {
34 fn rav1e_cdef_dist_kernel_4x4_neon(
35 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
36 ret_ptr: *mut u32,
37 );
38 fn rav1e_cdef_dist_kernel_4x8_neon(
39 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
40 ret_ptr: *mut u32,
41 );
42 fn rav1e_cdef_dist_kernel_8x4_neon(
43 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
44 ret_ptr: *mut u32,
45 );
46 fn rav1e_cdef_dist_kernel_8x8_neon(
47 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
48 ret_ptr: *mut u32,
49 );
50 fn rav1e_cdef_dist_kernel_4x4_hbd_neon(
51 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
52 ret_ptr: *mut u32,
53 );
54 fn rav1e_cdef_dist_kernel_4x8_hbd_neon(
55 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
56 ret_ptr: *mut u32,
57 );
58 fn rav1e_cdef_dist_kernel_8x4_hbd_neon(
59 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
60 ret_ptr: *mut u32,
61 );
62 fn rav1e_cdef_dist_kernel_8x8_hbd_neon(
63 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
64 ret_ptr: *mut u32,
65 );
66 }
67
68 /// # Panics
69 ///
70 /// - If in `check_asm` mode, panics on mismatch between native and ASM results.
71 #[allow(clippy::let_and_return)]
72 pub fn cdef_dist_kernel<T: Pixel>(
73 src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, w: usize, h: usize,
74 bit_depth: usize, cpu: CpuFeatureLevel,
75 ) -> u32 {
76 debug_assert!(src.plane_cfg.xdec == 0);
77 debug_assert!(src.plane_cfg.ydec == 0);
78 debug_assert!(dst.plane_cfg.xdec == 0);
79 debug_assert!(dst.plane_cfg.ydec == 0);
80
81 // Limit kernel to 8x8
82 debug_assert!(w <= 8);
83 debug_assert!(h <= 8);
84
85 let call_rust =
86 || -> u32 { rust::cdef_dist_kernel(dst, src, w, h, bit_depth, cpu) };
87 #[cfg(feature = "check_asm")]
88 let ref_dist = call_rust();
89
90 let (svar, dvar, sse) = match T::type_enum() {
91 PixelType::U8 => {
92 if let Some(func) =
93 CDEF_DIST_KERNEL_FNS[cpu.as_index()][kernel_fn_index(w, h)]
94 {
95 let mut ret_buf = [0u32; 3];
96 // SAFETY: Calls Assembly code.
97 unsafe {
98 func(
99 src.data_ptr() as *const _,
100 T::to_asm_stride(src.plane_cfg.stride),
101 dst.data_ptr() as *const _,
102 T::to_asm_stride(dst.plane_cfg.stride),
103 ret_buf.as_mut_ptr(),
104 )
105 }
106
107 (ret_buf[0], ret_buf[1], ret_buf[2])
108 } else {
109 return call_rust();
110 }
111 }
112 PixelType::U16 => {
113 if let Some(func) =
114 CDEF_DIST_KERNEL_HBD_FNS[cpu.as_index()][kernel_fn_index(w, h)]
115 {
116 let mut ret_buf = [0u32; 3];
117 // SAFETY: Calls Assembly code.
118 unsafe {
119 func(
120 src.data_ptr() as *const _,
121 T::to_asm_stride(src.plane_cfg.stride),
122 dst.data_ptr() as *const _,
123 T::to_asm_stride(dst.plane_cfg.stride),
124 ret_buf.as_mut_ptr(),
125 )
126 }
127
128 (ret_buf[0], ret_buf[1], ret_buf[2])
129 } else {
130 return call_rust();
131 }
132 }
133 };
134
135 let dist = apply_ssim_boost(sse, svar, dvar, bit_depth);
136 #[cfg(feature = "check_asm")]
137 assert_eq!(
138 dist, ref_dist,
139 "CDEF Distortion {}x{}: Assembly doesn't match reference code.",
140 w, h
141 );
142
143 dist
144 }
145
146 /// Store functions in a 8x8 grid. Most will be empty.
147 const CDEF_DIST_KERNEL_FNS_LENGTH: usize = 8 * 8;
148
149 const fn kernel_fn_index(w: usize, h: usize) -> usize {
150 ((w - 1) << 3) | (h - 1)
151 }
152
153 static CDEF_DIST_KERNEL_FNS_NEON: [Option<CdefDistKernelFn>;
154 CDEF_DIST_KERNEL_FNS_LENGTH] = {
155 let mut out: [Option<CdefDistKernelFn>; CDEF_DIST_KERNEL_FNS_LENGTH] =
156 [None; CDEF_DIST_KERNEL_FNS_LENGTH];
157
158 out[kernel_fn_index(4, 4)] = Some(rav1e_cdef_dist_kernel_4x4_neon);
159 out[kernel_fn_index(4, 8)] = Some(rav1e_cdef_dist_kernel_4x8_neon);
160 out[kernel_fn_index(8, 4)] = Some(rav1e_cdef_dist_kernel_8x4_neon);
161 out[kernel_fn_index(8, 8)] = Some(rav1e_cdef_dist_kernel_8x8_neon);
162
163 out
164 };
165
166 cpu_function_lookup_table!(
167 CDEF_DIST_KERNEL_FNS:
168 [[Option<CdefDistKernelFn>; CDEF_DIST_KERNEL_FNS_LENGTH]],
169 default: [None; CDEF_DIST_KERNEL_FNS_LENGTH],
170 [NEON]
171 );
172
173 static CDEF_DIST_KERNEL_HBD_FNS_NEON: [Option<CdefDistKernelHBDFn>;
174 CDEF_DIST_KERNEL_FNS_LENGTH] = {
175 let mut out: [Option<CdefDistKernelHBDFn>; CDEF_DIST_KERNEL_FNS_LENGTH] =
176 [None; CDEF_DIST_KERNEL_FNS_LENGTH];
177
178 out[kernel_fn_index(4, 4)] = Some(rav1e_cdef_dist_kernel_4x4_hbd_neon);
179 out[kernel_fn_index(4, 8)] = Some(rav1e_cdef_dist_kernel_4x8_hbd_neon);
180 out[kernel_fn_index(8, 4)] = Some(rav1e_cdef_dist_kernel_8x4_hbd_neon);
181 out[kernel_fn_index(8, 8)] = Some(rav1e_cdef_dist_kernel_8x8_hbd_neon);
182
183 out
184 };
185
186 cpu_function_lookup_table!(
187 CDEF_DIST_KERNEL_HBD_FNS:
188 [[Option<CdefDistKernelHBDFn>; CDEF_DIST_KERNEL_FNS_LENGTH]],
189 default: [None; CDEF_DIST_KERNEL_FNS_LENGTH],
190 [NEON]
191 );