Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/dist/cdef_dist.rs

raw
1 // Copyright (c) 2022-2023, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::activity::apply_ssim_boost;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::dist::*;
13 use crate::tiling::PlaneRegion;
14 use crate::util::Pixel;
15 use crate::util::PixelType;
16 use std::arch::x86_64::*;
17
18 type CdefDistKernelFn = unsafe extern fn(
19 src: *const u8,
20 src_stride: isize,
21 dst: *const u8,
22 dst_stride: isize,
23 ret_ptr: *mut u32,
24 );
25
26 type CdefDistKernelHBDFn = unsafe fn(
27 src: *const u16,
28 src_stride: isize,
29 dst: *const u16,
30 dst_stride: isize,
31 ) -> (u32, u32, u32);
32
33 extern {
34 fn rav1e_cdef_dist_kernel_4x4_sse2(
35 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
36 ret_ptr: *mut u32,
37 );
38 fn rav1e_cdef_dist_kernel_4x8_sse2(
39 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
40 ret_ptr: *mut u32,
41 );
42 fn rav1e_cdef_dist_kernel_8x4_sse2(
43 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
44 ret_ptr: *mut u32,
45 );
46 fn rav1e_cdef_dist_kernel_8x8_sse2(
47 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
48 ret_ptr: *mut u32,
49 );
50 }
51
52 /// # Panics
53 ///
54 /// - If in `check_asm` mode, panics on mismatch between native and ASM results.
55 #[allow(clippy::let_and_return)]
56 pub fn cdef_dist_kernel<T: Pixel>(
57 src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, w: usize, h: usize,
58 bit_depth: usize, cpu: CpuFeatureLevel,
59 ) -> u32 {
60 debug_assert!(src.plane_cfg.xdec == 0);
61 debug_assert!(src.plane_cfg.ydec == 0);
62 debug_assert!(dst.plane_cfg.xdec == 0);
63 debug_assert!(dst.plane_cfg.ydec == 0);
64
65 // Limit kernel to 8x8
66 debug_assert!(w <= 8);
67 debug_assert!(h <= 8);
68
69 let call_rust =
70 || -> u32 { rust::cdef_dist_kernel(dst, src, w, h, bit_depth, cpu) };
71 #[cfg(feature = "check_asm")]
72 let ref_dist = call_rust();
73
74 let (svar, dvar, sse) = match T::type_enum() {
75 PixelType::U8 => {
76 if let Some(func) =
77 CDEF_DIST_KERNEL_FNS[cpu.as_index()][kernel_fn_index(w, h)]
78 {
79 let mut ret_buf = [0u32; 3];
80 // SAFETY: Calls Assembly code.
81 unsafe {
82 func(
83 src.data_ptr() as *const _,
84 T::to_asm_stride(src.plane_cfg.stride),
85 dst.data_ptr() as *const _,
86 T::to_asm_stride(dst.plane_cfg.stride),
87 ret_buf.as_mut_ptr(),
88 )
89 }
90
91 (ret_buf[0], ret_buf[1], ret_buf[2])
92 } else {
93 return call_rust();
94 }
95 }
96 PixelType::U16 => {
97 if let Some(func) =
98 CDEF_DIST_KERNEL_HBD_FNS[cpu.as_index()][kernel_fn_index(w, h)]
99 {
100 // SAFETY: Calls Assembly code.
101 unsafe {
102 func(
103 src.data_ptr() as *const _,
104 T::to_asm_stride(src.plane_cfg.stride),
105 dst.data_ptr() as *const _,
106 T::to_asm_stride(dst.plane_cfg.stride),
107 )
108 }
109 } else {
110 return call_rust();
111 }
112 }
113 };
114
115 let dist = apply_ssim_boost(sse, svar, dvar, bit_depth);
116 #[cfg(feature = "check_asm")]
117 assert_eq!(
118 dist, ref_dist,
119 "CDEF Distortion {}x{}: Assembly doesn't match reference code.",
120 w, h
121 );
122
123 dist
124 }
125
126 /// Store functions in a 8x8 grid. Most will be empty.
127 const CDEF_DIST_KERNEL_FNS_LENGTH: usize = 8 * 8;
128
129 const fn kernel_fn_index(w: usize, h: usize) -> usize {
130 ((w - 1) << 3) | (h - 1)
131 }
132
133 static CDEF_DIST_KERNEL_FNS_SSE2: [Option<CdefDistKernelFn>;
134 CDEF_DIST_KERNEL_FNS_LENGTH] = {
135 let mut out: [Option<CdefDistKernelFn>; CDEF_DIST_KERNEL_FNS_LENGTH] =
136 [None; CDEF_DIST_KERNEL_FNS_LENGTH];
137
138 out[kernel_fn_index(4, 4)] = Some(rav1e_cdef_dist_kernel_4x4_sse2);
139 out[kernel_fn_index(4, 8)] = Some(rav1e_cdef_dist_kernel_4x8_sse2);
140 out[kernel_fn_index(8, 4)] = Some(rav1e_cdef_dist_kernel_8x4_sse2);
141 out[kernel_fn_index(8, 8)] = Some(rav1e_cdef_dist_kernel_8x8_sse2);
142
143 out
144 };
145
146 cpu_function_lookup_table!(
147 CDEF_DIST_KERNEL_FNS:
148 [[Option<CdefDistKernelFn>; CDEF_DIST_KERNEL_FNS_LENGTH]],
149 default: [None; CDEF_DIST_KERNEL_FNS_LENGTH],
150 [SSE2]
151 );
152
153 #[target_feature(enable = "avx2")]
154 #[inline]
155 unsafe fn mm256_sum_i32(ymm: __m256i) -> i32 {
156 // We split the vector in half and then add the two halves and sum.
157 let m1 = _mm256_extracti128_si256(ymm, 1);
158 let m2 = _mm256_castsi256_si128(ymm);
159 let m2 = _mm_add_epi32(m2, m1);
160 let m1 = _mm_shuffle_epi32(m2, 0b11_10_11_10);
161 let m2 = _mm_add_epi32(m2, m1);
162 let m1 = _mm_shuffle_epi32(m2, 0b01_01_01_01);
163 let m2 = _mm_add_epi32(m2, m1);
164 _mm_cvtsi128_si32(m2)
165 }
166
167 #[target_feature(enable = "avx2")]
168 #[inline]
169 unsafe fn rav1e_cdef_dist_kernel_8x8_hbd_avx2(
170 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
171 ) -> (u32, u32, u32) {
172 let src = src as *const u8;
173 let dst = dst as *const u8;
174
175 unsafe fn sum16(src: *const u8, src_stride: isize) -> u32 {
176 let h = 8;
177 let res = (0..h)
178 .map(|row| _mm_load_si128(src.offset(row * src_stride) as *const _))
179 .reduce(|a, b| _mm_add_epi16(a, b))
180 .unwrap();
181
182 let m32 = _mm256_cvtepi16_epi32(res);
183 mm256_sum_i32(m32) as u32
184 }
185 unsafe fn mpadd32(
186 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
187 ) -> u32 {
188 let h = 8;
189 let res = (0..h / 2)
190 .map(|row| {
191 let s = _mm256_loadu2_m128i(
192 src.offset(2 * row * src_stride) as *const _,
193 src.offset((2 * row + 1) * src_stride) as *const _,
194 );
195
196 let d = _mm256_loadu2_m128i(
197 dst.offset(2 * row * dst_stride) as *const _,
198 dst.offset((2 * row + 1) * dst_stride) as *const _,
199 );
200
201 _mm256_madd_epi16(s, d)
202 })
203 .reduce(|a, b| _mm256_add_epi32(a, b))
204 .unwrap();
205 mm256_sum_i32(res) as u32
206 }
207
208 let sum_s = sum16(src, src_stride);
209 let sum_d = sum16(dst, dst_stride);
210 let sum_s2 = mpadd32(src, src_stride, src, src_stride);
211 let sum_d2 = mpadd32(dst, dst_stride, dst, dst_stride);
212 let sum_sd = mpadd32(src, src_stride, dst, dst_stride);
213
214 // To get the distortion, compute sum of squared error and apply a weight
215 // based on the variance of the two planes.
216 let sse = sum_d2 + sum_s2 - 2 * sum_sd;
217
218 // Convert to 64-bits to avoid overflow when squaring
219 let sum_s = sum_s as u64;
220 let sum_d = sum_d as u64;
221
222 let svar = (sum_s2 as u64 - (sum_s * sum_s + 32) / 64) as u32;
223 let dvar = (sum_d2 as u64 - (sum_d * sum_d + 32) / 64) as u32;
224
225 (svar, dvar, sse)
226 }
227
228 static CDEF_DIST_KERNEL_HBD_FNS_AVX2: [Option<CdefDistKernelHBDFn>;
229 CDEF_DIST_KERNEL_FNS_LENGTH] = {
230 let mut out: [Option<CdefDistKernelHBDFn>; CDEF_DIST_KERNEL_FNS_LENGTH] =
231 [None; CDEF_DIST_KERNEL_FNS_LENGTH];
232
233 out[kernel_fn_index(8, 8)] = Some(rav1e_cdef_dist_kernel_8x8_hbd_avx2);
234
235 out
236 };
237
238 cpu_function_lookup_table!(
239 CDEF_DIST_KERNEL_HBD_FNS:
240 [[Option<CdefDistKernelHBDFn>; CDEF_DIST_KERNEL_FNS_LENGTH]],
241 default: [None; CDEF_DIST_KERNEL_FNS_LENGTH],
242 [AVX2]
243 );