Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/mc.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::frame::*;
12 use crate::mc::FilterMode::*;
13 use crate::mc::*;
14 use crate::tiling::*;
15 use crate::util::*;
16
17 type PutFn = unsafe extern fn(
18 dst: *mut u8,
19 dst_stride: isize,
20 src: *const u8,
21 src_stride: isize,
22 width: i32,
23 height: i32,
24 col_frac: i32,
25 row_frac: i32,
26 );
27
28 type PutHBDFn = unsafe extern fn(
29 dst: *mut u16,
30 dst_stride: isize,
31 src: *const u16,
32 src_stride: isize,
33 width: i32,
34 height: i32,
35 col_frac: i32,
36 row_frac: i32,
37 bitdepth_max: i32,
38 );
39
40 type PrepFn = unsafe extern fn(
41 tmp: *mut i16,
42 src: *const u8,
43 src_stride: isize,
44 width: i32,
45 height: i32,
46 col_frac: i32,
47 row_frac: i32,
48 );
49
50 type PrepHBDFn = unsafe extern fn(
51 tmp: *mut i16,
52 src: *const u16,
53 src_stride: isize,
54 width: i32,
55 height: i32,
56 col_frac: i32,
57 row_frac: i32,
58 bitdepth_max: i32,
59 );
60
61 type AvgFn = unsafe extern fn(
62 dst: *mut u8,
63 dst_stride: isize,
64 tmp1: *const i16,
65 tmp2: *const i16,
66 width: i32,
67 height: i32,
68 );
69
70 type AvgHBDFn = unsafe extern fn(
71 dst: *mut u16,
72 dst_stride: isize,
73 tmp1: *const i16,
74 tmp2: *const i16,
75 width: i32,
76 height: i32,
77 bitdepth_max: i32,
78 );
79
80 // gets an index that can be mapped to a function for a pair of filter modes
81 #[inline]
82 const fn get_2d_mode_idx(mode_x: FilterMode, mode_y: FilterMode) -> usize {
83 (mode_x as usize + 4 * (mode_y as usize)) & 15
84 }
85
86 /// # Panics
87 ///
88 /// - If `width` is not a power of 2
89 /// - If `width` is not between 2 and 128
90 /// - If `height` is odd
91 /// - If `width * height` is greater than the length of `tmp1` or `tmp2`
92 /// - If `width` and `height` do not fit within the bounds of `src`
93 #[inline(always)]
94 pub fn put_8tap<T: Pixel>(
95 dst: &mut PlaneRegionMut<'_, T>, src: PlaneSlice<'_, T>, width: usize,
96 height: usize, col_frac: i32, row_frac: i32, mode_x: FilterMode,
97 mode_y: FilterMode, bit_depth: usize, cpu: CpuFeatureLevel,
98 ) {
99 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
100 rust::put_8tap(
101 dst, src, width, height, col_frac, row_frac, mode_x, mode_y, bit_depth,
102 cpu,
103 );
104 };
105 #[cfg(feature = "check_asm")]
106 let ref_dst = {
107 let mut copy = dst.scratch_copy();
108 call_rust(&mut copy.as_region_mut());
109 copy
110 };
111
112 // SAFETY: The assembly only supports even heights and valid uncropped
113 // widths
114 unsafe {
115 assert_eq!(height & 1, 0);
116 assert!(width.is_power_of_two() && (2..=128).contains(&width));
117
118 // SAFETY: Check bounds of dst
119 assert!(dst.rect().width >= width && dst.rect().height >= height);
120
121 // SAFETY: Check bounds of src
122 assert!(src.accessible(width + 4, height + 4));
123 assert!(src.accessible_neg(3, 3));
124
125 match T::type_enum() {
126 PixelType::U8 => {
127 match PUT_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
128 Some(func) => (func)(
129 dst.data_ptr_mut() as *mut _,
130 T::to_asm_stride(dst.plane_cfg.stride),
131 src.as_ptr() as *const _,
132 T::to_asm_stride(src.plane.cfg.stride),
133 width as i32,
134 height as i32,
135 col_frac,
136 row_frac,
137 ),
138 None => call_rust(dst),
139 }
140 }
141 PixelType::U16 => {
142 match PUT_HBD_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
143 Some(func) => (func)(
144 dst.data_ptr_mut() as *mut _,
145 T::to_asm_stride(dst.plane_cfg.stride),
146 src.as_ptr() as *const _,
147 T::to_asm_stride(src.plane.cfg.stride),
148 width as i32,
149 height as i32,
150 col_frac,
151 row_frac,
152 (1 << bit_depth) - 1,
153 ),
154 None => call_rust(dst),
155 }
156 }
157 }
158 }
159 #[cfg(feature = "check_asm")]
160 {
161 for (dst_row, ref_row) in
162 dst.rows_iter().zip(ref_dst.as_region().rows_iter())
163 {
164 for (dst, reference) in dst_row.iter().zip(ref_row) {
165 assert_eq!(*dst, *reference);
166 }
167 }
168 }
169 }
170
171 /// # Panics
172 ///
173 /// - If `width` is not a power of 2
174 /// - If `width` is not between 2 and 128
175 /// - If `height` is odd
176 /// - If `width * height` is greater than the length of `tmp1` or `tmp2`
177 /// - If `width` and `height` do not fit within the bounds of `src`
178 #[inline(always)]
179 pub fn prep_8tap<T: Pixel>(
180 tmp: &mut [i16], src: PlaneSlice<'_, T>, width: usize, height: usize,
181 col_frac: i32, row_frac: i32, mode_x: FilterMode, mode_y: FilterMode,
182 bit_depth: usize, cpu: CpuFeatureLevel,
183 ) {
184 let call_rust = |tmp: &mut [i16]| {
185 rust::prep_8tap(
186 tmp, src, width, height, col_frac, row_frac, mode_x, mode_y, bit_depth,
187 cpu,
188 );
189 };
190 #[cfg(feature = "check_asm")]
191 let ref_tmp = {
192 let mut copy = vec![0; width * height];
193 copy[..].copy_from_slice(&tmp[..width * height]);
194 call_rust(&mut copy);
195 copy
196 };
197
198 // SAFETY: The assembly only supports even heights and valid uncropped
199 // widths
200 unsafe {
201 assert_eq!(height & 1, 0);
202 assert!(width.is_power_of_two() && (2..=128).contains(&width));
203
204 // SAFETY: Check length of tmp
205 assert!(tmp.len() >= width * height);
206
207 // SAFETY: Check bounds of src
208 assert!(src.accessible(width + 4, height + 4));
209 assert!(src.accessible_neg(3, 3));
210
211 match T::type_enum() {
212 PixelType::U8 => {
213 match PREP_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
214 Some(func) => (func)(
215 tmp.as_mut_ptr(),
216 src.as_ptr() as *const _,
217 T::to_asm_stride(src.plane.cfg.stride),
218 width as i32,
219 height as i32,
220 col_frac,
221 row_frac,
222 ),
223 None => call_rust(tmp),
224 }
225 }
226 PixelType::U16 if bit_depth > 8 => {
227 match PREP_HBD_FNS[cpu.as_index()][get_2d_mode_idx(mode_x, mode_y)] {
228 Some(func) => (func)(
229 tmp.as_mut_ptr() as *mut _,
230 src.as_ptr() as *const _,
231 T::to_asm_stride(src.plane.cfg.stride),
232 width as i32,
233 height as i32,
234 col_frac,
235 row_frac,
236 (1 << bit_depth) - 1,
237 ),
238 None => call_rust(tmp),
239 }
240 }
241 _ => call_rust(tmp),
242 }
243 }
244 #[cfg(feature = "check_asm")]
245 {
246 assert_eq!(&tmp[..width * height], &ref_tmp[..]);
247 }
248 }
249
250 /// # Panics
251 ///
252 /// - If `width` is not a power of 2
253 /// - If `width` is not between 2 and 128
254 /// - If `width * height` is greater than the length of `tmp1` or `tmp2`
255 /// - If `width` and `height` do not fit within the bounds of `dst`
256 pub fn mc_avg<T: Pixel>(
257 dst: &mut PlaneRegionMut<'_, T>, tmp1: &[i16], tmp2: &[i16], width: usize,
258 height: usize, bit_depth: usize, cpu: CpuFeatureLevel,
259 ) {
260 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
261 rust::mc_avg(dst, tmp1, tmp2, width, height, bit_depth, cpu);
262 };
263 #[cfg(feature = "check_asm")]
264 let ref_dst = {
265 let mut copy = dst.scratch_copy();
266 call_rust(&mut copy.as_region_mut());
267 copy
268 };
269
270 // SAFETY: The assembly only supports even heights and valid uncropped
271 // widths
272 unsafe {
273 assert_eq!(height & 1, 0);
274 assert!(width.is_power_of_two() && (2..=128).contains(&width));
275
276 // SAFETY: Check bounds of dst
277 assert!(dst.rect().width >= width && dst.rect().height >= height);
278
279 // SAFETY: Check length of tmp1 & tmp2
280 assert!(tmp1.len() >= width * height);
281 assert!(tmp2.len() >= width * height);
282
283 match T::type_enum() {
284 PixelType::U8 => match AVG_FNS[cpu.as_index()] {
285 Some(func) => (func)(
286 dst.data_ptr_mut() as *mut _,
287 T::to_asm_stride(dst.plane_cfg.stride),
288 tmp1.as_ptr(),
289 tmp2.as_ptr(),
290 width as i32,
291 height as i32,
292 ),
293 None => call_rust(dst),
294 },
295 PixelType::U16 if bit_depth > 8 => match AVG_HBD_FNS[cpu.as_index()] {
296 Some(func) => (func)(
297 dst.data_ptr_mut() as *mut _,
298 T::to_asm_stride(dst.plane_cfg.stride),
299 tmp1.as_ptr(),
300 tmp2.as_ptr(),
301 width as i32,
302 height as i32,
303 (1 << bit_depth) - 1,
304 ),
305 None => call_rust(dst),
306 },
307 _ => call_rust(dst),
308 }
309 }
310 #[cfg(feature = "check_asm")]
311 {
312 for (dst_row, ref_row) in
313 dst.rows_iter().zip(ref_dst.as_region().rows_iter())
314 {
315 for (dst, reference) in dst_row.iter().zip(ref_row) {
316 assert_eq!(*dst, *reference);
317 }
318 }
319 }
320 }
321
322 macro_rules! decl_mc_fns {
323 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
324 paste::item! {
325 extern {
326 $(
327 fn [<$func_name _ssse3>](
328 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
329 w: i32, h: i32, mx: i32, my: i32
330 );
331
332 fn [<$func_name _avx2>](
333 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
334 w: i32, h: i32, mx: i32, my: i32
335 );
336
337 fn [<$func_name _avx512icl>](
338 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
339 w: i32, h: i32, mx: i32, my: i32
340 );
341 )*
342 }
343
344 static PUT_FNS_SSSE3: [Option<PutFn>; 16] = {
345 let mut out: [Option<PutFn>; 16] = [None; 16];
346 $(
347 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _ssse3>]);
348 )*
349 out
350 };
351
352 static PUT_FNS_AVX2: [Option<PutFn>; 16] = {
353 let mut out: [Option<PutFn>; 16] = [None; 16];
354 $(
355 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx2>]);
356 )*
357 out
358 };
359
360 static PUT_FNS_AVX512ICL: [Option<PutFn>; 16] = {
361 let mut out: [Option<PutFn>; 16] = [None; 16];
362 $(
363 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx512icl>]);
364 )*
365 out
366 };
367 }
368 }
369 }
370
371 decl_mc_fns!(
372 (REGULAR, REGULAR, rav1e_put_8tap_regular_8bpc),
373 (REGULAR, SMOOTH, rav1e_put_8tap_regular_smooth_8bpc),
374 (REGULAR, SHARP, rav1e_put_8tap_regular_sharp_8bpc),
375 (SMOOTH, REGULAR, rav1e_put_8tap_smooth_regular_8bpc),
376 (SMOOTH, SMOOTH, rav1e_put_8tap_smooth_8bpc),
377 (SMOOTH, SHARP, rav1e_put_8tap_smooth_sharp_8bpc),
378 (SHARP, REGULAR, rav1e_put_8tap_sharp_regular_8bpc),
379 (SHARP, SMOOTH, rav1e_put_8tap_sharp_smooth_8bpc),
380 (SHARP, SHARP, rav1e_put_8tap_sharp_8bpc),
381 (BILINEAR, BILINEAR, rav1e_put_bilin_8bpc)
382 );
383
384 cpu_function_lookup_table!(
385 PUT_FNS: [[Option<PutFn>; 16]],
386 default: [None; 16],
387 [SSSE3, AVX2, AVX512ICL]
388 );
389
390 macro_rules! decl_mc_hbd_fns {
391 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
392 paste::item! {
393 extern {
394 $(
395 fn [<$func_name _ssse3>](
396 dst: *mut u16, dst_stride: isize, src: *const u16, src_stride: isize,
397 w: i32, h: i32, mx: i32, my: i32, bitdepth_max: i32,
398 );
399
400 fn [<$func_name _avx2>](
401 dst: *mut u16, dst_stride: isize, src: *const u16, src_stride: isize,
402 w: i32, h: i32, mx: i32, my: i32, bitdepth_max: i32,
403 );
404 )*
405 }
406
407 static PUT_HBD_FNS_SSSE3: [Option<PutHBDFn>; 16] = {
408 let mut out: [Option<PutHBDFn>; 16] = [None; 16];
409 $(
410 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _ssse3>]);
411 )*
412 out
413 };
414
415 static PUT_HBD_FNS_AVX2: [Option<PutHBDFn>; 16] = {
416 let mut out: [Option<PutHBDFn>; 16] = [None; 16];
417 $(
418 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx2>]);
419 )*
420 out
421 };
422 }
423 }
424 }
425
426 decl_mc_hbd_fns!(
427 (REGULAR, REGULAR, rav1e_put_8tap_regular_16bpc),
428 (REGULAR, SMOOTH, rav1e_put_8tap_regular_smooth_16bpc),
429 (REGULAR, SHARP, rav1e_put_8tap_regular_sharp_16bpc),
430 (SMOOTH, REGULAR, rav1e_put_8tap_smooth_regular_16bpc),
431 (SMOOTH, SMOOTH, rav1e_put_8tap_smooth_16bpc),
432 (SMOOTH, SHARP, rav1e_put_8tap_smooth_sharp_16bpc),
433 (SHARP, REGULAR, rav1e_put_8tap_sharp_regular_16bpc),
434 (SHARP, SMOOTH, rav1e_put_8tap_sharp_smooth_16bpc),
435 (SHARP, SHARP, rav1e_put_8tap_sharp_16bpc),
436 (BILINEAR, BILINEAR, rav1e_put_bilin_16bpc)
437 );
438
439 cpu_function_lookup_table!(
440 PUT_HBD_FNS: [[Option<PutHBDFn>; 16]],
441 default: [None; 16],
442 [SSSE3, AVX2]
443 );
444
445 macro_rules! decl_mct_fns {
446 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
447 paste::item! {
448 extern {
449 $(
450 fn [<$func_name _sse2>](
451 tmp: *mut i16, src: *const u8, src_stride: libc::ptrdiff_t, w: i32,
452 h: i32, mx: i32, my: i32
453 );
454
455 fn [<$func_name _ssse3>](
456 tmp: *mut i16, src: *const u8, src_stride: libc::ptrdiff_t, w: i32,
457 h: i32, mx: i32, my: i32
458 );
459
460 fn [<$func_name _avx2>](
461 tmp: *mut i16, src: *const u8, src_stride: libc::ptrdiff_t, w: i32,
462 h: i32, mx: i32, my: i32
463 );
464
465 fn [<$func_name _avx512icl>](
466 tmp: *mut i16, src: *const u8, src_stride: libc::ptrdiff_t, w: i32,
467 h: i32, mx: i32, my: i32
468 );
469 )*
470 }
471
472 static PREP_FNS_SSE2: [Option<PrepFn>; 16] = {
473 let mut out: [Option<PrepFn>; 16] = [None; 16];
474 $(
475 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _sse2>]);
476 )*
477 out
478 };
479
480 static PREP_FNS_SSSE3: [Option<PrepFn>; 16] = {
481 let mut out: [Option<PrepFn>; 16] = [None; 16];
482 $(
483 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _ssse3>]);
484 )*
485 out
486 };
487
488 static PREP_FNS_AVX2: [Option<PrepFn>; 16] = {
489 let mut out: [Option<PrepFn>; 16] = [None; 16];
490 $(
491 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx2>]);
492 )*
493 out
494 };
495
496 static PREP_FNS_AVX512ICL: [Option<PrepFn>; 16] = {
497 let mut out: [Option<PrepFn>; 16] = [None; 16];
498 $(
499 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx512icl>]);
500 )*
501 out
502 };
503 }
504 }
505 }
506
507 decl_mct_fns!(
508 (REGULAR, REGULAR, rav1e_prep_8tap_regular_8bpc),
509 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth_8bpc),
510 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp_8bpc),
511 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular_8bpc),
512 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth_8bpc),
513 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp_8bpc),
514 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular_8bpc),
515 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth_8bpc),
516 (SHARP, SHARP, rav1e_prep_8tap_sharp_8bpc),
517 (BILINEAR, BILINEAR, rav1e_prep_bilin_8bpc)
518 );
519
520 cpu_function_lookup_table!(
521 PREP_FNS: [[Option<PrepFn>; 16]],
522 default: [None; 16],
523 [SSE2, SSSE3, AVX2, AVX512ICL]
524 );
525
526 macro_rules! decl_mct_hbd_fns {
527 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
528 paste::item! {
529 extern {
530 $(
531 fn [<$func_name _ssse3>](
532 tmp: *mut i16, src: *const u16, src_stride: libc::ptrdiff_t, w: i32,
533 h: i32, mx: i32, my: i32, bitdepth_max: i32,
534 );
535
536 fn [<$func_name _avx2>](
537 tmp: *mut i16, src: *const u16, src_stride: libc::ptrdiff_t, w: i32,
538 h: i32, mx: i32, my: i32, bitdepth_max: i32,
539 );
540 )*
541 }
542
543 static PREP_HBD_FNS_SSSE3: [Option<PrepHBDFn>; 16] = {
544 let mut out: [Option<PrepHBDFn>; 16] = [None; 16];
545 $(
546 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _ssse3>]);
547 )*
548 out
549 };
550
551 static PREP_HBD_FNS_AVX2: [Option<PrepHBDFn>; 16] = {
552 let mut out: [Option<PrepHBDFn>; 16] = [None; 16];
553 $(
554 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx2>]);
555 )*
556 out
557 };
558 }
559 }
560 }
561
562 decl_mct_hbd_fns!(
563 (REGULAR, REGULAR, rav1e_prep_8tap_regular_16bpc),
564 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth_16bpc),
565 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp_16bpc),
566 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular_16bpc),
567 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth_16bpc),
568 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp_16bpc),
569 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular_16bpc),
570 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth_16bpc),
571 (SHARP, SHARP, rav1e_prep_8tap_sharp_16bpc),
572 (BILINEAR, BILINEAR, rav1e_prep_bilin_16bpc)
573 );
574
575 cpu_function_lookup_table!(
576 PREP_HBD_FNS: [[Option<PrepHBDFn>; 16]],
577 default: [None; 16],
578 [SSSE3, AVX2]
579 );
580
581 extern {
582 fn rav1e_avg_8bpc_ssse3(
583 dst: *mut u8, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
584 tmp2: *const i16, w: i32, h: i32,
585 );
586
587 fn rav1e_avg_8bpc_avx2(
588 dst: *mut u8, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
589 tmp2: *const i16, w: i32, h: i32,
590 );
591
592 fn rav1e_avg_8bpc_avx512icl(
593 dst: *mut u8, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
594 tmp2: *const i16, w: i32, h: i32,
595 );
596
597 fn rav1e_avg_16bpc_ssse3(
598 dst: *mut u16, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
599 tmp2: *const i16, w: i32, h: i32, bitdepth_max: i32,
600 );
601
602 fn rav1e_avg_16bpc_avx2(
603 dst: *mut u16, dst_stride: libc::ptrdiff_t, tmp1: *const i16,
604 tmp2: *const i16, w: i32, h: i32, bitdepth_max: i32,
605 );
606 }
607
608 cpu_function_lookup_table!(
609 AVG_FNS: [Option<AvgFn>],
610 default: None,
611 [
612 (SSSE3, Some(rav1e_avg_8bpc_ssse3)),
613 (AVX2, Some(rav1e_avg_8bpc_avx2)),
614 (AVX512ICL, Some(rav1e_avg_8bpc_avx512icl))
615 ]
616 );
617
618 cpu_function_lookup_table!(
619 AVG_HBD_FNS: [Option<AvgHBDFn>],
620 default: None,
621 [(SSSE3, Some(rav1e_avg_16bpc_ssse3)), (AVX2, Some(rav1e_avg_16bpc_avx2))]
622 );
623
624 #[cfg(test)]
625 mod test {
626 use super::*;
627 use rand::random;
628 use std::str::FromStr;
629
630 macro_rules! test_put_fns {
631 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),*, $OPT:ident, $OPTLIT:tt, $BD:expr) => {
632 $(
633 paste::item! {
634 #[test]
635 fn [<test_ $func_name _bd_ $BD _ $OPT>]() {
636 if CpuFeatureLevel::default() < CpuFeatureLevel::from_str($OPTLIT).unwrap() {
637 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
638 return;
639 }
640
641 let test_mvs = [MotionVector { row: 0, col: 0 }, MotionVector { row: 4, col: 0 }, MotionVector { row: 0, col: 4 }, MotionVector { row: 4, col: 4 }];
642 if $BD > 8 {
643 // dynamic allocation: test
644 let mut src = Plane::from_slice(&vec![0u16; 64 * 64], 64);
645 for s in src.data.iter_mut() {
646 *s = random::<u8>() as u16 * $BD / 8;
647 }
648 // dynamic allocation: test
649 let mut dst1 = Plane::from_slice(&vec![0u16; 64 * 64], 64);
650 // dynamic allocation: test
651 let mut dst2 = Plane::from_slice(&vec![0u16; 64 * 64], 64);
652
653 for mv in &test_mvs {
654 let (row_frac, col_frac, src) = get_params(&src, PlaneOffset { x: 0, y: 0 }, *mv);
655 super::put_8tap(&mut dst1.as_region_mut(), src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::from_str($OPTLIT).unwrap());
656 super::put_8tap(&mut dst2.as_region_mut(), src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::RUST);
657
658 assert_eq!(&*dst1.data, &*dst2.data);
659 }
660 } else {
661 // dynamic allocation: test
662 let mut src = Plane::from_slice(&vec![0u8; 64 * 64], 64);
663 for s in src.data.iter_mut() {
664 *s = random::<u8>();
665 }
666 // dynamic allocation: test
667 let mut dst1 = Plane::from_slice(&vec![0u8; 64 * 64], 64);
668 // dynamic allocation: test
669 let mut dst2 = Plane::from_slice(&vec![0u8; 64 * 64], 64);
670
671 for mv in &test_mvs {
672 let (row_frac, col_frac, src) = get_params(&src, PlaneOffset { x: 0, y: 0 }, *mv);
673 super::put_8tap(&mut dst1.as_region_mut(), src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::from_str($OPTLIT).unwrap());
674 super::put_8tap(&mut dst2.as_region_mut(), src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::RUST);
675
676 assert_eq!(&*dst1.data, &*dst2.data);
677 }
678 };
679 }
680 }
681 )*
682 }
683 }
684
685 test_put_fns!(
686 (REGULAR, REGULAR, rav1e_put_8tap_regular),
687 (REGULAR, SMOOTH, rav1e_put_8tap_regular_smooth),
688 (REGULAR, SHARP, rav1e_put_8tap_regular_sharp),
689 (SMOOTH, REGULAR, rav1e_put_8tap_smooth_regular),
690 (SMOOTH, SMOOTH, rav1e_put_8tap_smooth),
691 (SMOOTH, SHARP, rav1e_put_8tap_smooth_sharp),
692 (SHARP, REGULAR, rav1e_put_8tap_sharp_regular),
693 (SHARP, SMOOTH, rav1e_put_8tap_sharp_smooth),
694 (SHARP, SHARP, rav1e_put_8tap_sharp),
695 (BILINEAR, BILINEAR, rav1e_put_bilin),
696 ssse3,
697 "ssse3",
698 8
699 );
700
701 test_put_fns!(
702 (REGULAR, REGULAR, rav1e_put_8tap_regular),
703 (REGULAR, SMOOTH, rav1e_put_8tap_regular_smooth),
704 (REGULAR, SHARP, rav1e_put_8tap_regular_sharp),
705 (SMOOTH, REGULAR, rav1e_put_8tap_smooth_regular),
706 (SMOOTH, SMOOTH, rav1e_put_8tap_smooth),
707 (SMOOTH, SHARP, rav1e_put_8tap_smooth_sharp),
708 (SHARP, REGULAR, rav1e_put_8tap_sharp_regular),
709 (SHARP, SMOOTH, rav1e_put_8tap_sharp_smooth),
710 (SHARP, SHARP, rav1e_put_8tap_sharp),
711 (BILINEAR, BILINEAR, rav1e_put_bilin),
712 avx2,
713 "avx2",
714 8
715 );
716
717 macro_rules! test_prep_fns {
718 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),*, $OPT:ident, $OPTLIT:tt, $BD:expr) => {
719 $(
720 paste::item! {
721 #[test]
722 fn [<test_ $func_name _bd_ $BD _ $OPT>]() {
723 if CpuFeatureLevel::default() < CpuFeatureLevel::from_str($OPTLIT).unwrap() {
724 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
725 return;
726 }
727
728 // dynamic allocation: test
729 let mut dst1 = Aligned::new([0i16; 128 * 128]);
730 // dynamic allocation: test
731 let mut dst2 = Aligned::new([0i16; 128 * 128]);
732 let test_mvs = [MotionVector { row: 0, col: 0 }, MotionVector { row: 4, col: 0 }, MotionVector { row: 0, col: 4 }, MotionVector { row: 4, col: 4 }];
733
734 if $BD > 8 {
735 // dynamic allocation: test
736 let mut src = Plane::from_slice(&vec![0u16; 64 * 64], 64);
737 for s in src.data.iter_mut() {
738 *s = random::<u8>() as u16 * $BD / 8;
739 }
740
741 for mv in &test_mvs {
742 let (row_frac, col_frac, src) = get_params(&src, PlaneOffset { x: 0, y: 0 }, *mv);
743 super::prep_8tap(&mut dst1.data, src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::from_str($OPTLIT).unwrap());
744 super::prep_8tap(&mut dst2.data, src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::RUST);
745 }
746 } else {
747 // dynamic allocation: test
748 let mut src = Plane::from_slice(&vec![0u8; 64 * 64], 64);
749 for s in src.data.iter_mut() {
750 *s = random::<u8>();
751 }
752
753 for mv in &test_mvs {
754 let (row_frac, col_frac, src) = get_params(&src, PlaneOffset { x: 0, y: 0 }, *mv);
755 super::prep_8tap(&mut dst1.data, src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::from_str($OPTLIT).unwrap());
756 super::prep_8tap(&mut dst2.data, src, 8, 8, col_frac, row_frac, $mode_x, $mode_y, 8, CpuFeatureLevel::RUST);
757 }
758 };
759
760 assert_eq!(&dst1.data.to_vec(), &dst2.data.to_vec());
761 }
762 }
763 )*
764 }
765 }
766
767 test_prep_fns!(
768 (REGULAR, REGULAR, rav1e_prep_8tap_regular),
769 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth),
770 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp),
771 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular),
772 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth),
773 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp),
774 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular),
775 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth),
776 (SHARP, SHARP, rav1e_prep_8tap_sharp),
777 (BILINEAR, BILINEAR, rav1e_prep_bilin),
778 ssse3,
779 "ssse3",
780 8
781 );
782
783 test_prep_fns!(
784 (REGULAR, REGULAR, rav1e_prep_8tap_regular),
785 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth),
786 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp),
787 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular),
788 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth),
789 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp),
790 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular),
791 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth),
792 (SHARP, SHARP, rav1e_prep_8tap_sharp),
793 (BILINEAR, BILINEAR, rav1e_prep_bilin),
794 avx2,
795 "avx2",
796 8
797 );
798
799 test_prep_fns!(
800 (REGULAR, REGULAR, rav1e_prep_8tap_regular),
801 (REGULAR, SMOOTH, rav1e_prep_8tap_regular_smooth),
802 (REGULAR, SHARP, rav1e_prep_8tap_regular_sharp),
803 (SMOOTH, REGULAR, rav1e_prep_8tap_smooth_regular),
804 (SMOOTH, SMOOTH, rav1e_prep_8tap_smooth),
805 (SMOOTH, SHARP, rav1e_prep_8tap_smooth_sharp),
806 (SHARP, REGULAR, rav1e_prep_8tap_sharp_regular),
807 (SHARP, SMOOTH, rav1e_prep_8tap_sharp_smooth),
808 (SHARP, SHARP, rav1e_prep_8tap_sharp),
809 (BILINEAR, BILINEAR, rav1e_prep_bilin),
810 avx512icl,
811 "avx512vpclmulqdq",
812 8
813 );
814
815 fn get_params<T: Pixel>(
816 rec_plane: &Plane<T>, po: PlaneOffset, mv: MotionVector,
817 ) -> (i32, i32, PlaneSlice<T>) {
818 let rec_cfg = &rec_plane.cfg;
819 let shift_row = 3 + rec_cfg.ydec;
820 let shift_col = 3 + rec_cfg.xdec;
821 let row_offset = mv.row as i32 >> shift_row;
822 let col_offset = mv.col as i32 >> shift_col;
823 let row_frac =
824 (mv.row as i32 - (row_offset << shift_row)) << (4 - shift_row);
825 let col_frac =
826 (mv.col as i32 - (col_offset << shift_col)) << (4 - shift_col);
827 let qo = PlaneOffset {
828 x: po.x + col_offset as isize - 3,
829 y: po.y + row_offset as isize - 3,
830 };
831 (row_frac, col_frac, rec_plane.slice(qo).clamp().subslice(3, 3))
832 }
833 }