Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/dist/sse.rs

raw
1 // Copyright (c) 2020-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::dist::*;
12 use crate::encoder::IMPORTANCE_BLOCK_SIZE;
13 use crate::partition::BlockSize;
14 use crate::rdo::DistortionScale;
15 use crate::tiling::PlaneRegion;
16 use crate::util::*;
17
18 type WeightedSseFn = unsafe extern fn(
19 src: *const u8,
20 src_stride: isize,
21 dst: *const u8,
22 dst_stride: isize,
23 scale: *const u32,
24 scale_stride: isize,
25 ) -> u64;
26
27 type WeightedSseHBDFn = unsafe extern fn(
28 src: *const u16,
29 src_stride: isize,
30 dst: *const u16,
31 dst_stride: isize,
32 scale: *const u32,
33 scale_stride: isize,
34 ) -> u64;
35
36 macro_rules! declare_asm_sse_fn {
37 ($($name: ident),+) => (
38 $(
39 extern { fn $name (
40 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize, scale: *const u32, scale_stride: isize
41 ) -> u64; }
42 )+
43 )
44 }
45
46 macro_rules! declare_asm_hbd_sse_fn {
47 ($($name: ident),+) => (
48 $(
49 extern { fn $name (
50 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, scale: *const u32, scale_stride: isize
51 ) -> u64; }
52 )+
53 )
54 }
55
56 declare_asm_sse_fn![
57 // SSSE3
58 rav1e_weighted_sse_4x4_ssse3,
59 rav1e_weighted_sse_4x8_ssse3,
60 rav1e_weighted_sse_4x16_ssse3,
61 rav1e_weighted_sse_8x4_ssse3,
62 rav1e_weighted_sse_8x8_ssse3,
63 rav1e_weighted_sse_8x16_ssse3,
64 rav1e_weighted_sse_8x32_ssse3,
65 // AVX2
66 rav1e_weighted_sse_16x4_avx2,
67 rav1e_weighted_sse_16x8_avx2,
68 rav1e_weighted_sse_16x16_avx2,
69 rav1e_weighted_sse_16x32_avx2,
70 rav1e_weighted_sse_16x64_avx2,
71 rav1e_weighted_sse_32x8_avx2,
72 rav1e_weighted_sse_32x16_avx2,
73 rav1e_weighted_sse_32x32_avx2,
74 rav1e_weighted_sse_32x64_avx2,
75 rav1e_weighted_sse_64x16_avx2,
76 rav1e_weighted_sse_64x32_avx2,
77 rav1e_weighted_sse_64x64_avx2,
78 rav1e_weighted_sse_64x128_avx2,
79 rav1e_weighted_sse_128x64_avx2,
80 rav1e_weighted_sse_128x128_avx2
81 ];
82
83 declare_asm_hbd_sse_fn![
84 // SSE2
85 rav1e_weighted_sse_4x4_hbd_sse2
86 ];
87
88 /// # Panics
89 ///
90 /// - If in `check_asm` mode, panics on mismatch between native and ASM results.
91 #[inline(always)]
92 #[allow(clippy::let_and_return)]
93 pub fn get_weighted_sse<T: Pixel>(
94 src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, scale: &[u32],
95 scale_stride: usize, w: usize, h: usize, bit_depth: usize,
96 cpu: CpuFeatureLevel,
97 ) -> u64 {
98 // Assembly breaks if imp block size changes.
99 assert_eq!(IMPORTANCE_BLOCK_SIZE >> 1, 4);
100
101 let bsize_opt = BlockSize::from_width_and_height_opt(w, h);
102
103 let call_rust = || -> u64 {
104 rust::get_weighted_sse(dst, src, scale, scale_stride, w, h, bit_depth, cpu)
105 };
106
107 #[cfg(feature = "check_asm")]
108 let ref_dist = call_rust();
109
110 #[inline]
111 const fn size_of_element<T: Sized>(_: &[T]) -> usize {
112 std::mem::size_of::<T>()
113 }
114
115 let den =
116 DistortionScale::new(1, 1 << rust::GET_WEIGHTED_SSE_SHIFT).0 as u64;
117 let dist = match (bsize_opt, T::type_enum()) {
118 (Err(_), _) => call_rust(),
119 (Ok(bsize), PixelType::U8) => {
120 match SSE_FNS[cpu.as_index()][to_index(bsize)] {
121 // SAFETY: Calls Assembly code.
122 Some(func) => unsafe {
123 ((func)(
124 src.data_ptr() as *const _,
125 T::to_asm_stride(src.plane_cfg.stride),
126 dst.data_ptr() as *const _,
127 T::to_asm_stride(dst.plane_cfg.stride),
128 scale.as_ptr(),
129 (scale_stride * size_of_element(scale)) as isize,
130 ) + (den >> 1))
131 / den
132 },
133 None => call_rust(),
134 }
135 }
136 (Ok(bsize), PixelType::U16) => {
137 match SSE_HBD_FNS[cpu.as_index()][to_index(bsize)] {
138 // SAFETY: Calls Assembly code.
139 Some(func) => unsafe {
140 ((func)(
141 src.data_ptr() as *const _,
142 T::to_asm_stride(src.plane_cfg.stride),
143 dst.data_ptr() as *const _,
144 T::to_asm_stride(dst.plane_cfg.stride),
145 scale.as_ptr(),
146 (scale_stride * size_of_element(scale)) as isize,
147 ) + (den >> 1))
148 / den
149 },
150 None => call_rust(),
151 }
152 }
153 };
154
155 #[cfg(feature = "check_asm")]
156 assert_eq!(
157 dist, ref_dist,
158 "Weighted SSE {:?}: Assembly doesn't match reference code.",
159 bsize_opt
160 );
161
162 dist
163 }
164
165 static SSE_FNS_SSSE3: [Option<WeightedSseFn>; DIST_FNS_LENGTH] = {
166 let mut out: [Option<WeightedSseFn>; DIST_FNS_LENGTH] =
167 [None; DIST_FNS_LENGTH];
168
169 use BlockSize::*;
170 out[BLOCK_4X4 as usize] = Some(rav1e_weighted_sse_4x4_ssse3);
171 out[BLOCK_4X8 as usize] = Some(rav1e_weighted_sse_4x8_ssse3);
172 out[BLOCK_4X16 as usize] = Some(rav1e_weighted_sse_4x16_ssse3);
173 out[BLOCK_8X4 as usize] = Some(rav1e_weighted_sse_8x4_ssse3);
174 out[BLOCK_8X8 as usize] = Some(rav1e_weighted_sse_8x8_ssse3);
175 out[BLOCK_8X16 as usize] = Some(rav1e_weighted_sse_8x16_ssse3);
176 out[BLOCK_8X32 as usize] = Some(rav1e_weighted_sse_8x32_ssse3);
177
178 out
179 };
180
181 static SSE_FNS_AVX2: [Option<WeightedSseFn>; DIST_FNS_LENGTH] = {
182 let mut out: [Option<WeightedSseFn>; DIST_FNS_LENGTH] = SSE_FNS_SSSE3;
183
184 use BlockSize::*;
185 out[BLOCK_16X4 as usize] = Some(rav1e_weighted_sse_16x4_avx2);
186 out[BLOCK_16X8 as usize] = Some(rav1e_weighted_sse_16x8_avx2);
187 out[BLOCK_16X16 as usize] = Some(rav1e_weighted_sse_16x16_avx2);
188 out[BLOCK_16X32 as usize] = Some(rav1e_weighted_sse_16x32_avx2);
189 out[BLOCK_16X64 as usize] = Some(rav1e_weighted_sse_16x64_avx2);
190 out[BLOCK_32X8 as usize] = Some(rav1e_weighted_sse_32x8_avx2);
191 out[BLOCK_32X16 as usize] = Some(rav1e_weighted_sse_32x16_avx2);
192 out[BLOCK_32X32 as usize] = Some(rav1e_weighted_sse_32x32_avx2);
193 out[BLOCK_32X64 as usize] = Some(rav1e_weighted_sse_32x64_avx2);
194 out[BLOCK_64X16 as usize] = Some(rav1e_weighted_sse_64x16_avx2);
195 out[BLOCK_64X32 as usize] = Some(rav1e_weighted_sse_64x32_avx2);
196 out[BLOCK_64X64 as usize] = Some(rav1e_weighted_sse_64x64_avx2);
197 out[BLOCK_64X128 as usize] = Some(rav1e_weighted_sse_64x128_avx2);
198 out[BLOCK_128X64 as usize] = Some(rav1e_weighted_sse_128x64_avx2);
199 out[BLOCK_128X128 as usize] = Some(rav1e_weighted_sse_128x128_avx2);
200
201 out
202 };
203
204 static SSE_HBD_FNS_SSE2: [Option<WeightedSseHBDFn>; DIST_FNS_LENGTH] = {
205 let mut out: [Option<WeightedSseHBDFn>; DIST_FNS_LENGTH] =
206 [None; DIST_FNS_LENGTH];
207
208 use BlockSize::*;
209 out[BLOCK_4X4 as usize] = Some(rav1e_weighted_sse_4x4_hbd_sse2);
210
211 out
212 };
213
214 cpu_function_lookup_table!(
215 SSE_FNS: [[Option<WeightedSseFn>; DIST_FNS_LENGTH]],
216 default: [None; DIST_FNS_LENGTH],
217 [SSSE3, AVX2]
218 );
219
220 cpu_function_lookup_table!(
221 SSE_HBD_FNS: [[Option<WeightedSseHBDFn>; DIST_FNS_LENGTH]],
222 default: [None; DIST_FNS_LENGTH],
223 [SSE2]
224 );