Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/dist/sse.rs

raw
1 // Copyright (c) 2023, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::asm::aarch64::dist::to_index;
11 use crate::asm::aarch64::dist::DIST_FNS_LENGTH;
12 use crate::cpu_features::CpuFeatureLevel;
13 use crate::dist::*;
14 use crate::encoder::IMPORTANCE_BLOCK_SIZE;
15 use crate::partition::BlockSize;
16 use crate::rdo::DistortionScale;
17 use crate::tiling::PlaneRegion;
18 use crate::util::*;
19
20 type WeightedSseFn = unsafe extern fn(
21 src: *const u8,
22 src_stride: isize,
23 dst: *const u8,
24 dst_stride: isize,
25 scale: *const u32,
26 scale_stride: isize,
27 ) -> u64;
28
29 type WeightedSseHBDFn = unsafe extern fn(
30 src: *const u16,
31 src_stride: isize,
32 dst: *const u16,
33 dst_stride: isize,
34 scale: *const u32,
35 scale_stride: isize,
36 ) -> u64;
37
38 macro_rules! declare_asm_sse_fn {
39 ($($name: ident),+) => (
40 $(
41 extern { fn $name (
42 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize, scale: *const u32, scale_stride: isize
43 ) -> u64; }
44 )+
45 )
46 }
47
48 macro_rules! declare_asm_hbd_sse_fn {
49 ($($name: ident),+) => (
50 $(
51 extern { fn $name (
52 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, scale: *const u32, scale_stride: isize
53 ) -> u64; }
54 )+
55 )
56 }
57
58 declare_asm_sse_fn![
59 rav1e_weighted_sse_4x4_neon,
60 rav1e_weighted_sse_4x8_neon,
61 rav1e_weighted_sse_4x16_neon,
62 rav1e_weighted_sse_8x4_neon,
63 rav1e_weighted_sse_8x8_neon,
64 rav1e_weighted_sse_8x16_neon,
65 rav1e_weighted_sse_8x32_neon,
66 rav1e_weighted_sse_16x4_neon,
67 rav1e_weighted_sse_16x8_neon,
68 rav1e_weighted_sse_16x16_neon,
69 rav1e_weighted_sse_16x32_neon,
70 rav1e_weighted_sse_16x64_neon,
71 rav1e_weighted_sse_32x8_neon,
72 rav1e_weighted_sse_32x16_neon,
73 rav1e_weighted_sse_32x32_neon,
74 rav1e_weighted_sse_32x64_neon,
75 rav1e_weighted_sse_64x16_neon,
76 rav1e_weighted_sse_64x32_neon,
77 rav1e_weighted_sse_64x64_neon,
78 rav1e_weighted_sse_64x128_neon,
79 rav1e_weighted_sse_128x64_neon,
80 rav1e_weighted_sse_128x128_neon
81 ];
82
83 declare_asm_hbd_sse_fn![
84 rav1e_weighted_sse_4x4_hbd_neon,
85 rav1e_weighted_sse_4x8_hbd_neon,
86 rav1e_weighted_sse_4x16_hbd_neon,
87 rav1e_weighted_sse_8x4_hbd_neon,
88 rav1e_weighted_sse_8x8_hbd_neon,
89 rav1e_weighted_sse_8x16_hbd_neon,
90 rav1e_weighted_sse_8x32_hbd_neon,
91 rav1e_weighted_sse_16x4_hbd_neon,
92 rav1e_weighted_sse_16x8_hbd_neon,
93 rav1e_weighted_sse_16x16_hbd_neon,
94 rav1e_weighted_sse_16x32_hbd_neon,
95 rav1e_weighted_sse_16x64_hbd_neon,
96 rav1e_weighted_sse_32x8_hbd_neon,
97 rav1e_weighted_sse_32x16_hbd_neon,
98 rav1e_weighted_sse_32x32_hbd_neon,
99 rav1e_weighted_sse_32x64_hbd_neon,
100 rav1e_weighted_sse_64x16_hbd_neon,
101 rav1e_weighted_sse_64x32_hbd_neon,
102 rav1e_weighted_sse_64x64_hbd_neon,
103 rav1e_weighted_sse_64x128_hbd_neon,
104 rav1e_weighted_sse_128x64_hbd_neon,
105 rav1e_weighted_sse_128x128_hbd_neon
106 ];
107
108 /// # Panics
109 ///
110 /// - If in `check_asm` mode, panics on mismatch between native and ASM results.
111 #[inline(always)]
112 #[allow(clippy::let_and_return)]
113 pub fn get_weighted_sse<T: Pixel>(
114 src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, scale: &[u32],
115 scale_stride: usize, w: usize, h: usize, bit_depth: usize,
116 cpu: CpuFeatureLevel,
117 ) -> u64 {
118 // Assembly breaks if imp block size changes.
119 assert_eq!(IMPORTANCE_BLOCK_SIZE >> 1, 4);
120
121 let bsize_opt = BlockSize::from_width_and_height_opt(w, h);
122
123 let call_rust = || -> u64 {
124 rust::get_weighted_sse(dst, src, scale, scale_stride, w, h, bit_depth, cpu)
125 };
126
127 #[cfg(feature = "check_asm")]
128 let ref_dist = call_rust();
129
130 let den =
131 DistortionScale::new(1, 1 << rust::GET_WEIGHTED_SSE_SHIFT).0 as u64;
132 let dist = match (bsize_opt, T::type_enum()) {
133 (Err(_), _) => call_rust(),
134 (Ok(bsize), PixelType::U8) => {
135 match SSE_FNS[cpu.as_index()][to_index(bsize)] {
136 // SAFETY: Calls Assembly code.
137 Some(func) => unsafe {
138 ((func)(
139 src.data_ptr() as *const _,
140 T::to_asm_stride(src.plane_cfg.stride),
141 dst.data_ptr() as *const _,
142 T::to_asm_stride(dst.plane_cfg.stride),
143 scale.as_ptr(),
144 (scale_stride * std::mem::size_of::<u32>()) as isize,
145 ) + (den >> 1))
146 / den
147 },
148 None => call_rust(),
149 }
150 }
151 (Ok(bsize), PixelType::U16) => {
152 match SSE_HBD_FNS[cpu.as_index()][to_index(bsize)] {
153 // SAFETY: Calls Assembly code.
154 Some(func) => unsafe {
155 ((func)(
156 src.data_ptr() as *const _,
157 T::to_asm_stride(src.plane_cfg.stride),
158 dst.data_ptr() as *const _,
159 T::to_asm_stride(dst.plane_cfg.stride),
160 scale.as_ptr(),
161 (scale_stride * std::mem::size_of::<u32>()) as isize,
162 ) + (den >> 1))
163 / den
164 },
165 None => call_rust(),
166 }
167 }
168 };
169
170 #[cfg(feature = "check_asm")]
171 assert_eq!(
172 dist, ref_dist,
173 "Weighted SSE {:?}: Assembly doesn't match reference code.",
174 bsize_opt
175 );
176
177 dist
178 }
179
180 static SSE_FNS_NEON: [Option<WeightedSseFn>; DIST_FNS_LENGTH] = {
181 let mut out: [Option<WeightedSseFn>; DIST_FNS_LENGTH] =
182 [None; DIST_FNS_LENGTH];
183
184 use BlockSize::*;
185 out[BLOCK_4X4 as usize] = Some(rav1e_weighted_sse_4x4_neon);
186 out[BLOCK_4X8 as usize] = Some(rav1e_weighted_sse_4x8_neon);
187 out[BLOCK_4X16 as usize] = Some(rav1e_weighted_sse_4x16_neon);
188 out[BLOCK_8X4 as usize] = Some(rav1e_weighted_sse_8x4_neon);
189 out[BLOCK_8X8 as usize] = Some(rav1e_weighted_sse_8x8_neon);
190 out[BLOCK_8X16 as usize] = Some(rav1e_weighted_sse_8x16_neon);
191 out[BLOCK_8X32 as usize] = Some(rav1e_weighted_sse_8x32_neon);
192 out[BLOCK_16X4 as usize] = Some(rav1e_weighted_sse_16x4_neon);
193 out[BLOCK_16X8 as usize] = Some(rav1e_weighted_sse_16x8_neon);
194 out[BLOCK_16X16 as usize] = Some(rav1e_weighted_sse_16x16_neon);
195 out[BLOCK_16X32 as usize] = Some(rav1e_weighted_sse_16x32_neon);
196 out[BLOCK_16X64 as usize] = Some(rav1e_weighted_sse_16x64_neon);
197 out[BLOCK_32X8 as usize] = Some(rav1e_weighted_sse_32x8_neon);
198 out[BLOCK_32X16 as usize] = Some(rav1e_weighted_sse_32x16_neon);
199 out[BLOCK_32X32 as usize] = Some(rav1e_weighted_sse_32x32_neon);
200 out[BLOCK_32X64 as usize] = Some(rav1e_weighted_sse_32x64_neon);
201 out[BLOCK_64X16 as usize] = Some(rav1e_weighted_sse_64x16_neon);
202 out[BLOCK_64X32 as usize] = Some(rav1e_weighted_sse_64x32_neon);
203 out[BLOCK_64X64 as usize] = Some(rav1e_weighted_sse_64x64_neon);
204 out[BLOCK_64X128 as usize] = Some(rav1e_weighted_sse_64x128_neon);
205 out[BLOCK_128X64 as usize] = Some(rav1e_weighted_sse_128x64_neon);
206 out[BLOCK_128X128 as usize] = Some(rav1e_weighted_sse_128x128_neon);
207
208 out
209 };
210
211 static SSE_HBD_FNS_NEON: [Option<WeightedSseHBDFn>; DIST_FNS_LENGTH] = {
212 let mut out: [Option<WeightedSseHBDFn>; DIST_FNS_LENGTH] =
213 [None; DIST_FNS_LENGTH];
214
215 use BlockSize::*;
216 out[BLOCK_4X4 as usize] = Some(rav1e_weighted_sse_4x4_hbd_neon);
217 out[BLOCK_4X8 as usize] = Some(rav1e_weighted_sse_4x8_hbd_neon);
218 out[BLOCK_4X16 as usize] = Some(rav1e_weighted_sse_4x16_hbd_neon);
219 out[BLOCK_8X4 as usize] = Some(rav1e_weighted_sse_8x4_hbd_neon);
220 out[BLOCK_8X8 as usize] = Some(rav1e_weighted_sse_8x8_hbd_neon);
221 out[BLOCK_8X16 as usize] = Some(rav1e_weighted_sse_8x16_hbd_neon);
222 out[BLOCK_8X32 as usize] = Some(rav1e_weighted_sse_8x32_hbd_neon);
223 out[BLOCK_16X4 as usize] = Some(rav1e_weighted_sse_16x4_hbd_neon);
224 out[BLOCK_16X8 as usize] = Some(rav1e_weighted_sse_16x8_hbd_neon);
225 out[BLOCK_16X16 as usize] = Some(rav1e_weighted_sse_16x16_hbd_neon);
226 out[BLOCK_16X32 as usize] = Some(rav1e_weighted_sse_16x32_hbd_neon);
227 out[BLOCK_16X64 as usize] = Some(rav1e_weighted_sse_16x64_hbd_neon);
228 out[BLOCK_32X8 as usize] = Some(rav1e_weighted_sse_32x8_hbd_neon);
229 out[BLOCK_32X16 as usize] = Some(rav1e_weighted_sse_32x16_hbd_neon);
230 out[BLOCK_32X32 as usize] = Some(rav1e_weighted_sse_32x32_hbd_neon);
231 out[BLOCK_32X64 as usize] = Some(rav1e_weighted_sse_32x64_hbd_neon);
232 out[BLOCK_64X16 as usize] = Some(rav1e_weighted_sse_64x16_hbd_neon);
233 out[BLOCK_64X32 as usize] = Some(rav1e_weighted_sse_64x32_hbd_neon);
234 out[BLOCK_64X64 as usize] = Some(rav1e_weighted_sse_64x64_hbd_neon);
235 out[BLOCK_64X128 as usize] = Some(rav1e_weighted_sse_64x128_hbd_neon);
236 out[BLOCK_128X64 as usize] = Some(rav1e_weighted_sse_128x64_hbd_neon);
237 out[BLOCK_128X128 as usize] = Some(rav1e_weighted_sse_128x128_hbd_neon);
238
239 out
240 };
241
242 cpu_function_lookup_table!(
243 SSE_FNS: [[Option<WeightedSseFn>; DIST_FNS_LENGTH]],
244 default: [None; DIST_FNS_LENGTH],
245 [NEON]
246 );
247
248 cpu_function_lookup_table!(
249 SSE_HBD_FNS: [[Option<WeightedSseHBDFn>; DIST_FNS_LENGTH]],
250 default: [None; DIST_FNS_LENGTH],
251 [NEON]
252 );