Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/mc.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::frame::*;
12 use crate::mc::FilterMode::*;
13 use crate::mc::*;
14 use crate::tiling::*;
15 use crate::util::*;
16
17 type PutFn = unsafe extern fn(
18 dst: *mut u8,
19 dst_stride: isize,
20 src: *const u8,
21 src_stride: isize,
22 width: i32,
23 height: i32,
24 col_frac: i32,
25 row_frac: i32,
26 );
27
28 type PutHBDFn = unsafe extern fn(
29 dst: *mut u16,
30 dst_stride: isize,
31 src: *const u16,
32 src_stride: isize,
33 width: i32,
34 height: i32,
35 col_frac: i32,
36 row_frac: i32,
37 bitdepth_max: i32,
38 );
39
40 type PrepFn = unsafe extern fn(
41 tmp: *mut i16,
42 src: *const u8,
43 src_stride: isize,
44 width: i32,
45 height: i32,
46 col_frac: i32,
47 row_frac: i32,
48 );
49
50 type PrepHBDFn = unsafe extern fn(
51 tmp: *mut i16,
52 src: *const u16,
53 src_stride: isize,
54 width: i32,
55 height: i32,
56 col_frac: i32,
57 row_frac: i32,
58 bitdepth_max: i32,
59 );
60
61 type AvgFn = unsafe extern fn(
62 dst: *mut u8,
63 dst_stride: isize,
64 tmp1: *const i16,
65 tmp2: *const i16,
66 width: i32,
67 height: i32,
68 );
69
70 type AvgHBDFn = unsafe extern fn(
71 dst: *mut u16,
72 dst_stride: isize,
73 tmp1: *const i16,
74 tmp2: *const i16,
75 width: i32,
76 height: i32,
77 bitdepth_max: i32,
78 );
79
80 // gets an index that can be mapped to a function for a pair of filter modes
81 #[inline]
82 const fn get_2d_mode_idx(mode_x: FilterMode, mode_y: FilterMode) -> usize {
83 (mode_x as usize + 4 * (mode_y as usize)) & 15
84 }
85
86 #[inline(always)]
87 pub fn put_8tap<T: Pixel>(
88 dst: &mut PlaneRegionMut<'_, T>, src: PlaneSlice<'_, T>, width: usize,
89 height: usize, col_frac: i32, row_frac: i32, mode_x: FilterMode,
90 mode_y: FilterMode, bit_depth: usize, cpu: CpuFeatureLevel,
91 ) {
92 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
93 rust::put_8tap(
94 dst, src, width, height, col_frac, row_frac, mode_x, mode_y, bit_depth,
95 cpu,
96 );
97 };
98 #[cfg(feature = "check_asm")]
99 let ref_dst = {
100 let mut copy = dst.scratch_copy();
101 call_rust(&mut copy.as_region_mut());
102 copy
103 };
104 unsafe {
105 // SAFETY: The assembly only supports even heights and valid uncropped
106 // widths
107 assert_eq!(height & 1, 0);
108 assert!(width.is_power_of_two() && (2..=128).contains(&width));
109
110 // SAFETY: Check bounds of dst
111 assert!(dst.rect().width >= width && dst.rect().height >= height);
112
113 // SAFETY: Check bounds of src
114 assert!(src.accessible(width + 4, height + 4));
115 assert!(src.accessible_neg(3, 3));
116
117 match T::type_enum() {
118 PixelType::U8 => {
119 match PUT_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
120 Some(func) => (func)(
121 dst.data_ptr_mut() as *mut _,
122 T::to_asm_stride(dst.plane_cfg.stride),
123 src.as_ptr() as *const _,
124 T::to_asm_stride(src.plane.cfg.stride),
125 width as i32,
126 height as i32,
127 col_frac,
128 row_frac,
129 ),
130 None => call_rust(dst),
131 }
132 }
133 PixelType::U16 if bit_depth > 8 => {
134 match PUT_HBD_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
135 Some(func) => (func)(
136 dst.data_ptr_mut() as *mut _,
137 T::to_asm_stride(dst.plane_cfg.stride),
138 src.as_ptr() as *const _,
139 T::to_asm_stride(src.plane.cfg.stride),
140 width as i32,
141 height as i32,
142 col_frac,
143 row_frac,
144 (1 << bit_depth) - 1,
145 ),
146 None => call_rust(dst),
147 }
148 }
149 _ => call_rust(dst),
150 }
151 }
152 #[cfg(feature = "check_asm")]
153 {
154 for (dst_row, ref_row) in
155 dst.rows_iter().zip(ref_dst.as_region().rows_iter())
156 {
157 for (dst, reference) in dst_row.iter().zip(ref_row) {
158 assert_eq!(*dst, *reference);
159 }
160 }
161 }
162 }
163
164 #[inline(always)]
165 pub fn prep_8tap<T: Pixel>(
166 tmp: &mut [i16], src: PlaneSlice<'_, T>, width: usize, height: usize,
167 col_frac: i32, row_frac: i32, mode_x: FilterMode, mode_y: FilterMode,
168 bit_depth: usize, cpu: CpuFeatureLevel,
169 ) {
170 // The assembly only supports even heights and valid uncropped widths
171 assert_eq!(height & 1, 0);
172 assert!(width.is_power_of_two() && (2 <= width || width <= 128));
173
174 let call_rust = |tmp: &mut [i16]| {
175 rust::prep_8tap(
176 tmp, src, width, height, col_frac, row_frac, mode_x, mode_y, bit_depth,
177 cpu,
178 );
179 };
180 #[cfg(feature = "check_asm")]
181 let ref_tmp = {
182 let mut copy = vec![0; width * height];
183 copy[..].copy_from_slice(&tmp[..width * height]);
184 call_rust(&mut copy);
185 copy
186 };
187 unsafe {
188 // SAFETY: The assembly only supports even heights and valid uncropped
189 // widths
190 assert_eq!(height & 1, 0);
191 assert!(width.is_power_of_two() && (2..=128).contains(&width));
192
193 // SAFETY: Check length of tmp
194 assert!(tmp.len() >= width * height);
195
196 // SAFETY: Check bounds of src
197 assert!(src.accessible(width + 4, height + 4));
198 assert!(src.accessible_neg(3, 3));
199 match T::type_enum() {
200 PixelType::U8 => {
201 match PREP_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
202 Some(func) => (func)(
203 tmp.as_mut_ptr(),
204 src.as_ptr() as *const _,
205 T::to_asm_stride(src.plane.cfg.stride),
206 width as i32,
207 height as i32,
208 col_frac,
209 row_frac,
210 ),
211 None => call_rust(tmp),
212 }
213 }
214 PixelType::U16 if bit_depth > 8 => {
215 match PREP_HBD_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
216 Some(func) => (func)(
217 tmp.as_mut_ptr() as *mut _,
218 src.as_ptr() as *const _,
219 T::to_asm_stride(src.plane.cfg.stride),
220 width as i32,
221 height as i32,
222 col_frac,
223 row_frac,
224 (1 << bit_depth) - 1,
225 ),
226 None => call_rust(tmp),
227 }
228 }
229 _ => call_rust(tmp),
230 }
231 }
232 #[cfg(feature = "check_asm")]
233 {
234 assert_eq!(&tmp[..width * height], &ref_tmp[..]);
235 }
236 }
237
238 pub fn mc_avg<T: Pixel>(
239 dst: &mut PlaneRegionMut<'_, T>, tmp1: &[i16], tmp2: &[i16], width: usize,
240 height: usize, bit_depth: usize, cpu: CpuFeatureLevel,
241 ) {
242 // The assembly only supports even heights and valid uncropped widths
243 assert_eq!(height & 1, 0);
244 assert!(width.is_power_of_two() && (2 <= width || width <= 128));
245
246 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
247 rust::mc_avg(dst, tmp1, tmp2, width, height, bit_depth, cpu);
248 };
249 #[cfg(feature = "check_asm")]
250 let ref_dst = {
251 let mut copy = dst.scratch_copy();
252 call_rust(&mut copy.as_region_mut());
253 copy
254 };
255 unsafe {
256 // SAFETY: The assembly only supports even heights and valid uncropped
257 // widths
258 assert_eq!(height & 1, 0);
259 assert!(width.is_power_of_two() && (2..=128).contains(&width));
260
261 // SAFETY: Check bounds of dst
262 assert!(dst.rect().width >= width && dst.rect().height >= height);
263
264 // SAFETY: Check length of tmp1 & tmp2
265 assert!(tmp1.len() >= width * height);
266 assert!(tmp2.len() >= width * height);
267 match T::type_enum() {
268 PixelType::U8 => match AVG_FNS[cpu.as_index()] {
269 Some(func) => (func)(
270 dst.data_ptr_mut() as *mut _,
271 T::to_asm_stride(dst.plane_cfg.stride),
272 tmp1.as_ptr(),
273 tmp2.as_ptr(),
274 width as i32,
275 height as i32,
276 ),
277 None => call_rust(dst),
278 },
279 PixelType::U16 if bit_depth > 8 => match AVG_HBD_FNS[cpu.as_index()] {
280 Some(func) => (func)(
281 dst.data_ptr_mut() as *mut _,
282 T::to_asm_stride(dst.plane_cfg.stride),
283 tmp1.as_ptr(),
284 tmp2.as_ptr(),
285 width as i32,
286 height as i32,
287 (1 << bit_depth) - 1,
288 ),
289 None => call_rust(dst),
290 },
291 _ => call_rust(dst),
292 }
293 }
294 #[cfg(feature = "check_asm")]
295 {
296 for (dst_row, ref_row) in
297 dst.rows_iter().zip(ref_dst.as_region().rows_iter())
298 {
299 for (dst, reference) in dst_row.iter().zip(ref_row) {
300 assert_eq!(*dst, *reference);
301 }
302 }
303 }
304 }
305
306 macro_rules! decl_mc_fns {
307 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
308 extern {
309 $(
310 fn $func_name(
311 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
312 w: i32, h: i32, mx: i32, my: i32,
313 );
314 )*
315 }
316
317 static PUT_FNS_NEON: [Option<PutFn>; 16] = {
318 let mut out: [Option<PutFn>; 16] = [None; 16];
319 $(
320 out[get_2d_mode_idx($mode_x, $mode_y)] = Some($func_name);
321 )*
322 out
323 };
324 }
325 }
326
327 decl_mc_fns!(
328 (REGULAR, REGULAR, rav1e_put_8tap_regular_8bpc_neon),
329 (REGULAR, SMOOTH, rav1e_put_8tap_regular_smooth_8bpc_neon),
330 (REGULAR, SHARP, rav1e_put_8tap_regular_sharp_8bpc_neon),
331 (SMOOTH, REGULAR, rav1e_put_8tap_smooth_regular_8bpc_neon),
332 (SMOOTH, SMOOTH, rav1e_put_8tap_smooth_8bpc_neon),
333 (SMOOTH, SHARP, rav1e_put_8tap_smooth_sharp_8bpc_neon),
334 (SHARP, REGULAR, rav1e_put_8tap_sharp_regular_8bpc_neon),
335 (SHARP, SMOOTH, rav1e_put_8tap_sharp_smooth_8bpc_neon),
336 (SHARP, SHARP, rav1e_put_8tap_sharp_8bpc_neon),
337 (BILINEAR, BILINEAR, rav1e_put_bilin_8bpc_neon)
338 );
339
340 cpu_function_lookup_table!(
341 PUT_FNS: [[Option<PutFn>; 16]],
342 default: [None; 16],
343 [NEON]
344 );
345
346 macro_rules! decl_mc_hbd_fns {
347 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
348 extern {
349 $(
350 fn $func_name(
351 dst: *mut u16, dst_stride: isize, src: *const u16, src_stride: isize,
352 w: i32, h: i32, mx: i32, my: i32, bitdepth_max: i32,
353 );
354 )*
355 }
356
357 static PUT_HBD_FNS_NEON: [Option<PutHBDFn>; 16] = {
358 let mut out: [Option<PutHBDFn>; 16] = [None; 16];
359 $(
360 out[get_2d_mode_idx($mode_x, $mode_y)] = Some($func_name);
361 )*
362 out
363 };
364 }
365 }
366
367 decl_mc_hbd_fns!(
368 (REGULAR, REGULAR, rav1e_put_8tap_regular_16bpc_neon),
369 (REGULAR, SMOOTH, rav1e_put_8tap_regular_smooth_16bpc_neon),
370 (REGULAR, SHARP, rav1e_put_8tap_regular_sharp_16bpc_neon),
371 (SMOOTH, REGULAR, rav1e_put_8tap_smooth_regular_16bpc_neon),
372 (SMOOTH, SMOOTH, rav1e_put_8tap_smooth_16bpc_neon),
373 (SMOOTH, SHARP, rav1e_put_8tap_smooth_sharp_16bpc_neon),
374 (SHARP, REGULAR, rav1e_put_8tap_sharp_regular_16bpc_neon),
375 (SHARP, SMOOTH, rav1e_put_8tap_sharp_smooth_16bpc_neon),
376 (SHARP, SHARP, rav1e_put_8tap_sharp_16bpc_neon),
377 (BILINEAR, BILINEAR, rav1e_put_bilin_16bpc_neon)
378 );
379
380 cpu_function_lookup_table!(
381 PUT_HBD_FNS: [[Option<PutHBDFn>; 16]],
382 default: [None; 16],
383 [NEON]
384 );
385
386 macro_rules! decl_mct_fns {
387 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
388 extern {
389 $(
390 fn $func_name(
391 tmp: *mut i16, src: *const u8, src_stride: libc::ptrdiff_t, w: i32,
392 h: i32, mx: i32, my: i32,
393 );
394 )*
395 }
396
397 static PREP_FNS_NEON: [Option<PrepFn>; 16] = {
398 let mut out: [Option<PrepFn>; 16] = [None; 16];
399 $(
400 out[get_2d_mode_idx($mode_x, $mode_y)] = Some($func_name);
401 )*
402 out
403 };
404 }
405 }
406
407 decl_mct_fns!(
408 (REGULAR, REGULAR, rav1e_prep_8tap_regular_8bpc_neon),
409 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth_8bpc_neon),
410 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp_8bpc_neon),
411 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular_8bpc_neon),
412 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth_8bpc_neon),
413 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp_8bpc_neon),
414 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular_8bpc_neon),
415 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth_8bpc_neon),
416 (SHARP, SHARP, rav1e_prep_8tap_sharp_8bpc_neon),
417 (BILINEAR, BILINEAR, rav1e_prep_bilin_8bpc_neon)
418 );
419
420 cpu_function_lookup_table!(
421 PREP_FNS: [[Option<PrepFn>; 16]],
422 default: [None; 16],
423 [NEON]
424 );
425
426 macro_rules! decl_mct_hbd_fns {
427 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
428 extern {
429 $(
430 fn $func_name(
431 tmp: *mut i16, src: *const u16, src_stride: libc::ptrdiff_t, w: i32,
432 h: i32, mx: i32, my: i32, bitdepth_max: i32,
433 );
434 )*
435 }
436
437 static PREP_HBD_FNS_NEON: [Option<PrepHBDFn>; 16] = {
438 let mut out: [Option<PrepHBDFn>; 16] = [None; 16];
439 $(
440 out[get_2d_mode_idx($mode_x, $mode_y)] = Some($func_name);
441 )*
442 out
443 };
444 }
445 }
446
447 decl_mct_hbd_fns!(
448 (REGULAR, REGULAR, rav1e_prep_8tap_regular_16bpc_neon),
449 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth_16bpc_neon),
450 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp_16bpc_neon),
451 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular_16bpc_neon),
452 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth_16bpc_neon),
453 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp_16bpc_neon),
454 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular_16bpc_neon),
455 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth_16bpc_neon),
456 (SHARP, SHARP, rav1e_prep_8tap_sharp_16bpc_neon),
457 (BILINEAR, BILINEAR, rav1e_prep_bilin_16bpc_neon)
458 );
459
460 cpu_function_lookup_table!(
461 PREP_HBD_FNS: [[Option<PrepHBDFn>; 16]],
462 default: [None; 16],
463 [NEON]
464 );
465
466 extern {
467 fn rav1e_avg_8bpc_neon(
468 dst: *mut u8, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
469 tmp2: *const i16, w: i32, h: i32,
470 );
471 }
472
473 cpu_function_lookup_table!(
474 AVG_FNS: [Option<AvgFn>],
475 default: None,
476 [(NEON, Some(rav1e_avg_8bpc_neon))]
477 );
478
479 extern {
480 fn rav1e_avg_16bpc_neon(
481 dst: *mut u16, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
482 tmp2: *const i16, w: i32, h: i32, bitdepth_max: i32,
483 );
484 }
485
486 cpu_function_lookup_table!(
487 AVG_HBD_FNS: [Option<AvgHBDFn>],
488 default: None,
489 [(NEON, Some(rav1e_avg_16bpc_neon))]
490 );