Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/dist/mod.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 pub use self::cdef_dist::*;
11 pub use self::sse::*;
12 use crate::cpu_features::CpuFeatureLevel;
13 use crate::dist::*;
14 use crate::partition::BlockSize;
15 use crate::tiling::*;
16 use crate::util::*;
17
18 mod cdef_dist;
19 mod sse;
20
21 type SadFn = unsafe extern fn(
22 src: *const u8,
23 src_stride: isize,
24 dst: *const u8,
25 dst_stride: isize,
26 ) -> u32;
27
28 type SatdFn = SadFn;
29
30 type SadHBDFn = unsafe extern fn(
31 src: *const u16,
32 src_stride: isize,
33 dst: *const u16,
34 dst_stride: isize,
35 ) -> u32;
36
37 type SatdHBDFn = unsafe extern fn(
38 src: *const u16,
39 src_stride: isize,
40 dst: *const u16,
41 dst_stride: isize,
42 bdmax: u32,
43 ) -> u32;
44
45 macro_rules! declare_asm_dist_fn {
46 ($(($name: ident, $T: ident)),+) => (
47 $(
48 extern { fn $name (
49 src: *const $T, src_stride: isize, dst: *const $T, dst_stride: isize
50 ) -> u32; }
51 )+
52 )
53 }
54
55 macro_rules! declare_asm_satd_hbd_fn {
56 ($($name: ident),+) => (
57 $(
58 extern { pub(crate) fn $name (
59 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, bdmax: u32
60 ) -> u32; }
61 )+
62 )
63 }
64
65 #[cfg(target_arch = "x86_64")]
66 use std::arch::x86_64::*;
67
68 /// Horizontally sum 8 32-bit values in a YMM register
69 #[inline]
70 #[target_feature(enable = "avx2")]
71 unsafe fn mm256_sum_i32(ymm: __m256i) -> i32 {
72 // We split the vector in half and then add (2, 3) + (0, 1), and finally 0 + 1.
73 let m1 = _mm256_extracti128_si256(ymm, 1);
74 let m2 = _mm256_castsi256_si128(ymm);
75 let m2 = _mm_add_epi32(m2, m1);
76 let m1 = _mm_shuffle_epi32(m2, 0b11_10_11_10);
77 let m2 = _mm_add_epi32(m2, m1);
78 let m1 = _mm_shuffle_epi32(m2, 0b01_01_01_01);
79 let m2 = _mm_add_epi32(m2, m1);
80 _mm_cvtsi128_si32(m2)
81 }
82
83 /// Perform SAD over at most 256 (10 or 12-bit) values
84 ///
85 /// Returns a YMM register of 8 32-bit sums, which can
86 /// be used as an intermediate result for larger summations
87 ///
88 /// Limited to 256 values as any more 12-bit SADs may overflow
89 /// a 16-bit value.
90 ///
91 /// # Panics
92 /// - If `W` * `n_rows` > 256
93 /// - If `W` is not a multiple of 16
94 #[inline]
95 #[target_feature(enable = "avx2")]
96 unsafe fn rav1e_sad_wxh_hbd_avx2_inner<const W: usize>(
97 src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize,
98 n_rows: usize,
99 ) -> __m256i {
100 const LOADS_PER_REGISTER: usize = 32 / core::mem::size_of::<u16>(); // 16
101
102 // Check we can't overflow a 16-bit register
103 assert!(W * n_rows <= 256);
104 // Check we load the right number of values
105 assert_eq!(W % LOADS_PER_REGISTER, 0);
106
107 let mut sum16 = _mm256_setzero_si256();
108 (0..n_rows as isize).for_each(|n| {
109 (0..(W / LOADS_PER_REGISTER) as isize).for_each(|x| {
110 let res = _mm256_abs_epi16(_mm256_sub_epi16(
111 _mm256_loadu_si256(
112 src.offset(n * src_stride + x * LOADS_PER_REGISTER as isize * 2)
113 as *const _,
114 ),
115 _mm256_loadu_si256(
116 dst.offset(n * dst_stride + x * LOADS_PER_REGISTER as isize * 2)
117 as *const _,
118 ),
119 ));
120 sum16 = _mm256_add_epi16(sum16, res);
121 })
122 });
123
124 let b = _mm256_unpackhi_epi16(sum16, _mm256_setzero_si256());
125 let c = _mm256_unpacklo_epi16(sum16, _mm256_setzero_si256());
126
127 _mm256_add_epi32(b, c)
128 }
129
130 /// Sum of Absolute Differences
131 ///
132 /// By convention, `src_stride` and `dst_stride` are measured in bytes, not u16's
133 #[inline]
134 #[target_feature(enable = "avx2")]
135 unsafe fn rav1e_sad_wxh_hbd_avx2<const W: usize>(
136 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
137 n_rows: usize,
138 ) -> u32 {
139 const MAX_INNER_SIZE: usize = 256;
140 let src = src as *const u8;
141 let dst = dst as *const u8;
142
143 // Can we sum at least 1 entire row at once?
144 if W <= MAX_INNER_SIZE {
145 // If we can fit the entire area in one inner iteration, do it
146 if W * n_rows <= MAX_INNER_SIZE {
147 return mm256_sum_i32(rav1e_sad_wxh_hbd_avx2_inner::<W>(
148 src, src_stride, dst, dst_stride, n_rows,
149 )) as u32;
150 }
151 // Otherwise, sum rows in sets of 256
152 assert_eq!(MAX_INNER_SIZE % W, 0);
153 let row_step = (MAX_INNER_SIZE / W).min(n_rows);
154 assert_eq!(n_rows % row_step, 0);
155 let mut sum = _mm256_setzero_si256();
156 (0..(n_rows / row_step) as isize).for_each(|w| {
157 let src = src.offset(w * row_step as isize * src_stride);
158 let dst = dst.offset(w * row_step as isize * dst_stride);
159 let res = rav1e_sad_wxh_hbd_avx2_inner::<W>(
160 src, src_stride, dst, dst_stride, row_step,
161 );
162 sum = _mm256_add_epi32(sum, res)
163 });
164 mm256_sum_i32(sum) as u32
165 } else {
166 // If we can't sum an entire row in one shot, split each row up
167 assert_eq!(W % MAX_INNER_SIZE, 0);
168 let sum = (0..n_rows)
169 .flat_map(|_| {
170 (0..(W / MAX_INNER_SIZE)).map(move |h| {
171 let src = src.offset(src_stride + (h * MAX_INNER_SIZE * 2) as isize);
172 let dst = dst.offset(dst_stride + (h * MAX_INNER_SIZE * 2) as isize);
173 rav1e_sad_wxh_hbd_avx2_inner::<MAX_INNER_SIZE>(
174 src, src_stride, dst, dst_stride, 1,
175 )
176 })
177 })
178 .reduce(|a, b| unsafe { _mm256_add_epi32(a, b) })
179 .unwrap();
180 mm256_sum_i32(sum) as u32
181 }
182 }
183
184 declare_asm_dist_fn![
185 // SSSE3
186 (rav1e_sad_4x4_hbd_ssse3, u16),
187 (rav1e_sad_16x16_hbd_ssse3, u16),
188 (rav1e_satd_8x8_ssse3, u8),
189 // SSE2
190 (rav1e_sad4x4_sse2, u8),
191 (rav1e_sad4x8_sse2, u8),
192 (rav1e_sad4x16_sse2, u8),
193 (rav1e_sad8x4_sse2, u8),
194 (rav1e_sad8x8_sse2, u8),
195 (rav1e_sad8x16_sse2, u8),
196 (rav1e_sad8x32_sse2, u8),
197 (rav1e_sad16x4_sse2, u8),
198 (rav1e_sad16x8_sse2, u8),
199 (rav1e_sad16x16_sse2, u8),
200 (rav1e_sad16x32_sse2, u8),
201 (rav1e_sad16x64_sse2, u8),
202 (rav1e_sad32x8_sse2, u8),
203 (rav1e_sad32x16_sse2, u8),
204 (rav1e_sad32x32_sse2, u8),
205 (rav1e_sad32x64_sse2, u8),
206 (rav1e_sad64x16_sse2, u8),
207 (rav1e_sad64x32_sse2, u8),
208 (rav1e_sad64x64_sse2, u8),
209 (rav1e_sad64x128_sse2, u8),
210 (rav1e_sad128x64_sse2, u8),
211 (rav1e_sad128x128_sse2, u8),
212 // SSE4
213 (rav1e_satd_4x4_sse4, u8),
214 // AVX
215 (rav1e_sad32x8_avx2, u8),
216 (rav1e_sad32x16_avx2, u8),
217 (rav1e_sad32x32_avx2, u8),
218 (rav1e_sad32x64_avx2, u8),
219 (rav1e_sad64x16_avx2, u8),
220 (rav1e_sad64x32_avx2, u8),
221 (rav1e_sad64x64_avx2, u8),
222 (rav1e_sad64x128_avx2, u8),
223 (rav1e_sad128x64_avx2, u8),
224 (rav1e_sad128x128_avx2, u8),
225 (rav1e_satd_4x4_avx2, u8),
226 (rav1e_satd_8x8_avx2, u8),
227 (rav1e_satd_16x16_avx2, u8),
228 (rav1e_satd_32x32_avx2, u8),
229 (rav1e_satd_64x64_avx2, u8),
230 (rav1e_satd_128x128_avx2, u8),
231 (rav1e_satd_4x8_avx2, u8),
232 (rav1e_satd_8x4_avx2, u8),
233 (rav1e_satd_8x16_avx2, u8),
234 (rav1e_satd_16x8_avx2, u8),
235 (rav1e_satd_16x32_avx2, u8),
236 (rav1e_satd_32x16_avx2, u8),
237 (rav1e_satd_32x64_avx2, u8),
238 (rav1e_satd_64x32_avx2, u8),
239 (rav1e_satd_64x128_avx2, u8),
240 (rav1e_satd_128x64_avx2, u8),
241 (rav1e_satd_4x16_avx2, u8),
242 (rav1e_satd_16x4_avx2, u8),
243 (rav1e_satd_8x32_avx2, u8),
244 (rav1e_satd_32x8_avx2, u8),
245 (rav1e_satd_16x64_avx2, u8),
246 (rav1e_satd_64x16_avx2, u8)
247 ];
248
249 declare_asm_satd_hbd_fn![
250 rav1e_satd_4x4_hbd_avx2,
251 rav1e_satd_8x4_hbd_avx2,
252 rav1e_satd_4x8_hbd_avx2,
253 rav1e_satd_8x8_hbd_avx2,
254 rav1e_satd_16x8_hbd_avx2,
255 rav1e_satd_16x16_hbd_avx2,
256 rav1e_satd_32x32_hbd_avx2,
257 rav1e_satd_64x64_hbd_avx2,
258 rav1e_satd_128x128_hbd_avx2,
259 rav1e_satd_16x32_hbd_avx2,
260 rav1e_satd_16x64_hbd_avx2,
261 rav1e_satd_32x16_hbd_avx2,
262 rav1e_satd_32x64_hbd_avx2,
263 rav1e_satd_64x16_hbd_avx2,
264 rav1e_satd_64x32_hbd_avx2,
265 rav1e_satd_64x128_hbd_avx2,
266 rav1e_satd_128x64_hbd_avx2,
267 rav1e_satd_32x8_hbd_avx2,
268 rav1e_satd_8x16_hbd_avx2,
269 rav1e_satd_8x32_hbd_avx2,
270 rav1e_satd_16x4_hbd_avx2,
271 rav1e_satd_4x16_hbd_avx2
272 ];
273
274 // BlockSize::BLOCK_SIZES.next_power_of_two();
275 pub(crate) const DIST_FNS_LENGTH: usize = 32;
276
277 #[inline]
278 pub(crate) const fn to_index(bsize: BlockSize) -> usize {
279 bsize as usize & (DIST_FNS_LENGTH - 1)
280 }
281
282 /// # Panics
283 ///
284 /// - If in `check_asm` mode, panics on mismatch between native and ASM results.
285 #[inline(always)]
286 #[allow(clippy::let_and_return)]
287 pub fn get_sad<T: Pixel>(
288 src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, w: usize, h: usize,
289 bit_depth: usize, cpu: CpuFeatureLevel,
290 ) -> u32 {
291 let bsize_opt = BlockSize::from_width_and_height_opt(w, h);
292
293 let call_rust = || -> u32 { rust::get_sad(dst, src, w, h, bit_depth, cpu) };
294
295 #[cfg(feature = "check_asm")]
296 let ref_dist = call_rust();
297
298 let dist = match (bsize_opt, T::type_enum()) {
299 (Err(_), _) => call_rust(),
300 (Ok(bsize), PixelType::U8) => {
301 match SAD_FNS[cpu.as_index()][to_index(bsize)] {
302 // SAFETY: Calls Assembly code.
303 Some(func) => unsafe {
304 (func)(
305 src.data_ptr() as *const _,
306 T::to_asm_stride(src.plane_cfg.stride),
307 dst.data_ptr() as *const _,
308 T::to_asm_stride(dst.plane_cfg.stride),
309 )
310 },
311 None => call_rust(),
312 }
313 }
314 (Ok(bsize), PixelType::U16) => {
315 match SAD_HBD_FNS[cpu.as_index()][to_index(bsize)] {
316 // SAFETY: Calls Assembly code.
317 Some(func) => unsafe {
318 (func)(
319 src.data_ptr() as *const _,
320 T::to_asm_stride(src.plane_cfg.stride),
321 dst.data_ptr() as *const _,
322 T::to_asm_stride(dst.plane_cfg.stride),
323 )
324 },
325 None => call_rust(),
326 }
327 }
328 };
329
330 #[cfg(feature = "check_asm")]
331 assert_eq!(dist, ref_dist);
332
333 dist
334 }
335
336 /// # Panics
337 ///
338 /// - If in `check_asm` mode, panics on mismatch between native and ASM results.
339 #[inline(always)]
340 #[allow(clippy::let_and_return)]
341 pub fn get_satd<T: Pixel>(
342 src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, w: usize, h: usize,
343 bit_depth: usize, cpu: CpuFeatureLevel,
344 ) -> u32 {
345 let bsize_opt = BlockSize::from_width_and_height_opt(w, h);
346
347 let call_rust = || -> u32 { rust::get_satd(dst, src, w, h, bit_depth, cpu) };
348
349 #[cfg(feature = "check_asm")]
350 let ref_dist = call_rust();
351
352 let dist = match (bsize_opt, T::type_enum()) {
353 (Err(_), _) => call_rust(),
354 (Ok(bsize), PixelType::U8) => {
355 match SATD_FNS[cpu.as_index()][to_index(bsize)] {
356 // SAFETY: Calls Assembly code.
357 Some(func) => unsafe {
358 (func)(
359 src.data_ptr() as *const _,
360 T::to_asm_stride(src.plane_cfg.stride),
361 dst.data_ptr() as *const _,
362 T::to_asm_stride(dst.plane_cfg.stride),
363 )
364 },
365 None => call_rust(),
366 }
367 }
368 (Ok(bsize), PixelType::U16) => {
369 match SATD_HBD_FNS[cpu.as_index()][to_index(bsize)] {
370 // SAFETY: Calls Assembly code.
371 Some(func) => unsafe {
372 (func)(
373 src.data_ptr() as *const _,
374 T::to_asm_stride(src.plane_cfg.stride),
375 dst.data_ptr() as *const _,
376 T::to_asm_stride(dst.plane_cfg.stride),
377 (1 << bit_depth) - 1,
378 )
379 },
380 None => call_rust(),
381 }
382 }
383 };
384
385 #[cfg(feature = "check_asm")]
386 assert_eq!(dist, ref_dist);
387
388 dist
389 }
390
391 // We have hand-written ASM for 4x4 and 16x16 HBD blocks,
392 // so we can use those for other block sizes as well.
393 macro_rules! get_sad_hbd_ssse3 {
394 ($(($W:expr, $H:expr, $BS:expr)),*) => {
395 $(
396 paste::item! {
397 #[target_feature(enable = "ssse3")]
398 unsafe extern fn [<rav1e_sad_ $W x $H _hbd_ssse3>](
399 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
400 ) -> u32 {
401 let mut sum = 0;
402 for w in (0..$W).step_by($BS) {
403 for h in (0..$H).step_by($BS) {
404 sum += [<rav1e_sad_ $BS x $BS _hbd_ssse3>](
405 src.offset(w + h * src_stride / 2),
406 src_stride,
407 dst.offset(w + h * dst_stride / 2),
408 dst_stride
409 );
410 }
411 }
412 sum
413 }
414 }
415 )*
416 }
417 }
418
419 get_sad_hbd_ssse3!(
420 // 4x4 base
421 (8, 8, 4),
422 (4, 8, 4),
423 (8, 4, 4),
424 (8, 16, 4),
425 (16, 8, 4),
426 (4, 16, 4),
427 (16, 4, 4),
428 (8, 32, 4),
429 (32, 8, 4),
430 (32, 32, 16),
431 (64, 64, 16),
432 (128, 128, 16),
433 (16, 32, 16),
434 (32, 16, 16),
435 (32, 64, 16),
436 (64, 32, 16),
437 (64, 128, 16),
438 (128, 64, 16),
439 (16, 64, 16),
440 (64, 16, 16)
441 );
442
443 macro_rules! get_sad_hbd_avx2_WxH {
444 ($(($W:expr, $H:expr)),*) => {
445 $(
446 paste::item! {
447 #[target_feature(enable = "avx2")]
448 unsafe extern fn [<rav1e_sad_ $W x $H _hbd_avx2>](
449 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize,
450 ) -> u32 {
451 rav1e_sad_wxh_hbd_avx2::<$W>(
452 src,
453 src_stride,
454 dst,
455 dst_stride,
456 $H,
457 )
458 }
459 }
460 )*
461 }
462 }
463
464 get_sad_hbd_avx2_WxH!(
465 // No advantage to AVX below 16 width
466 (16, 4),
467 (16, 8),
468 (16, 16),
469 (16, 32),
470 (16, 64),
471 (32, 8),
472 (32, 16),
473 (32, 32),
474 (32, 64),
475 (64, 16),
476 (64, 32),
477 (64, 64),
478 (64, 128),
479 (128, 64),
480 (128, 128)
481 );
482
483 static SAD_FNS_SSE2: [Option<SadFn>; DIST_FNS_LENGTH] = {
484 let mut out: [Option<SadFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
485
486 use BlockSize::*;
487
488 out[BLOCK_4X4 as usize] = Some(rav1e_sad4x4_sse2);
489 out[BLOCK_4X8 as usize] = Some(rav1e_sad4x8_sse2);
490 out[BLOCK_4X16 as usize] = Some(rav1e_sad4x16_sse2);
491
492 out[BLOCK_8X4 as usize] = Some(rav1e_sad8x4_sse2);
493 out[BLOCK_8X8 as usize] = Some(rav1e_sad8x8_sse2);
494 out[BLOCK_8X16 as usize] = Some(rav1e_sad8x16_sse2);
495 out[BLOCK_8X32 as usize] = Some(rav1e_sad8x32_sse2);
496
497 out[BLOCK_16X4 as usize] = Some(rav1e_sad16x4_sse2);
498 out[BLOCK_16X8 as usize] = Some(rav1e_sad16x8_sse2);
499 out[BLOCK_16X16 as usize] = Some(rav1e_sad16x16_sse2);
500 out[BLOCK_16X32 as usize] = Some(rav1e_sad16x32_sse2);
501 out[BLOCK_16X64 as usize] = Some(rav1e_sad16x64_sse2);
502
503 out[BLOCK_32X8 as usize] = Some(rav1e_sad32x8_sse2);
504 out[BLOCK_32X16 as usize] = Some(rav1e_sad32x16_sse2);
505 out[BLOCK_32X32 as usize] = Some(rav1e_sad32x32_sse2);
506 out[BLOCK_32X64 as usize] = Some(rav1e_sad32x64_sse2);
507
508 out[BLOCK_64X16 as usize] = Some(rav1e_sad64x16_sse2);
509 out[BLOCK_64X32 as usize] = Some(rav1e_sad64x32_sse2);
510 out[BLOCK_64X64 as usize] = Some(rav1e_sad64x64_sse2);
511 out[BLOCK_64X128 as usize] = Some(rav1e_sad64x128_sse2);
512
513 out[BLOCK_128X64 as usize] = Some(rav1e_sad128x64_sse2);
514 out[BLOCK_128X128 as usize] = Some(rav1e_sad128x128_sse2);
515
516 out
517 };
518
519 static SAD_FNS_AVX2: [Option<SadFn>; DIST_FNS_LENGTH] = {
520 let mut out: [Option<SadFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
521
522 use BlockSize::*;
523
524 out[BLOCK_4X4 as usize] = Some(rav1e_sad4x4_sse2);
525 out[BLOCK_4X8 as usize] = Some(rav1e_sad4x8_sse2);
526 out[BLOCK_4X16 as usize] = Some(rav1e_sad4x16_sse2);
527
528 out[BLOCK_8X4 as usize] = Some(rav1e_sad8x4_sse2);
529 out[BLOCK_8X8 as usize] = Some(rav1e_sad8x8_sse2);
530 out[BLOCK_8X16 as usize] = Some(rav1e_sad8x16_sse2);
531 out[BLOCK_8X32 as usize] = Some(rav1e_sad8x32_sse2);
532
533 out[BLOCK_16X4 as usize] = Some(rav1e_sad16x4_sse2);
534 out[BLOCK_16X8 as usize] = Some(rav1e_sad16x8_sse2);
535 out[BLOCK_16X16 as usize] = Some(rav1e_sad16x16_sse2);
536 out[BLOCK_16X32 as usize] = Some(rav1e_sad16x32_sse2);
537 out[BLOCK_16X64 as usize] = Some(rav1e_sad16x64_sse2);
538
539 out[BLOCK_32X8 as usize] = Some(rav1e_sad32x8_avx2);
540 out[BLOCK_32X16 as usize] = Some(rav1e_sad32x16_avx2);
541 out[BLOCK_32X32 as usize] = Some(rav1e_sad32x32_avx2);
542 out[BLOCK_32X64 as usize] = Some(rav1e_sad32x64_avx2);
543
544 out[BLOCK_64X16 as usize] = Some(rav1e_sad64x16_avx2);
545 out[BLOCK_64X32 as usize] = Some(rav1e_sad64x32_avx2);
546 out[BLOCK_64X64 as usize] = Some(rav1e_sad64x64_avx2);
547 out[BLOCK_64X128 as usize] = Some(rav1e_sad64x128_avx2);
548
549 out[BLOCK_128X64 as usize] = Some(rav1e_sad128x64_avx2);
550 out[BLOCK_128X128 as usize] = Some(rav1e_sad128x128_avx2);
551
552 out
553 };
554
555 cpu_function_lookup_table!(
556 SAD_FNS: [[Option<SadFn>; DIST_FNS_LENGTH]],
557 default: [None; DIST_FNS_LENGTH],
558 [SSE2, AVX2]
559 );
560
561 static SAD_HBD_FNS_SSSE3: [Option<SadHBDFn>; DIST_FNS_LENGTH] = {
562 let mut out: [Option<SadHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
563
564 use BlockSize::*;
565
566 out[BLOCK_4X4 as usize] = Some(rav1e_sad_4x4_hbd_ssse3);
567 out[BLOCK_8X8 as usize] = Some(rav1e_sad_8x8_hbd_ssse3);
568 out[BLOCK_16X16 as usize] = Some(rav1e_sad_16x16_hbd_ssse3);
569 out[BLOCK_32X32 as usize] = Some(rav1e_sad_32x32_hbd_ssse3);
570 out[BLOCK_64X64 as usize] = Some(rav1e_sad_64x64_hbd_ssse3);
571 out[BLOCK_128X128 as usize] = Some(rav1e_sad_128x128_hbd_ssse3);
572
573 out[BLOCK_4X8 as usize] = Some(rav1e_sad_4x8_hbd_ssse3);
574 out[BLOCK_8X4 as usize] = Some(rav1e_sad_8x4_hbd_ssse3);
575 out[BLOCK_8X16 as usize] = Some(rav1e_sad_8x16_hbd_ssse3);
576 out[BLOCK_16X8 as usize] = Some(rav1e_sad_16x8_hbd_ssse3);
577 out[BLOCK_16X32 as usize] = Some(rav1e_sad_16x32_hbd_ssse3);
578 out[BLOCK_32X16 as usize] = Some(rav1e_sad_32x16_hbd_ssse3);
579 out[BLOCK_32X64 as usize] = Some(rav1e_sad_32x64_hbd_ssse3);
580 out[BLOCK_64X32 as usize] = Some(rav1e_sad_64x32_hbd_ssse3);
581 out[BLOCK_64X128 as usize] = Some(rav1e_sad_64x128_hbd_ssse3);
582 out[BLOCK_128X64 as usize] = Some(rav1e_sad_128x64_hbd_ssse3);
583
584 out[BLOCK_4X16 as usize] = Some(rav1e_sad_4x16_hbd_ssse3);
585 out[BLOCK_16X4 as usize] = Some(rav1e_sad_16x4_hbd_ssse3);
586 out[BLOCK_8X32 as usize] = Some(rav1e_sad_8x32_hbd_ssse3);
587 out[BLOCK_32X8 as usize] = Some(rav1e_sad_32x8_hbd_ssse3);
588 out[BLOCK_16X64 as usize] = Some(rav1e_sad_16x64_hbd_ssse3);
589 out[BLOCK_64X16 as usize] = Some(rav1e_sad_64x16_hbd_ssse3);
590
591 out
592 };
593
594 static SAD_HBD_FNS_AVX2: [Option<SadHBDFn>; DIST_FNS_LENGTH] = {
595 let mut out: [Option<SadHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
596
597 use BlockSize::*;
598
599 out[BLOCK_4X4 as usize] = Some(rav1e_sad_4x4_hbd_ssse3);
600 out[BLOCK_8X8 as usize] = Some(rav1e_sad_8x8_hbd_ssse3);
601 out[BLOCK_16X16 as usize] = Some(rav1e_sad_16x16_hbd_avx2);
602 out[BLOCK_32X32 as usize] = Some(rav1e_sad_32x32_hbd_avx2);
603 out[BLOCK_64X64 as usize] = Some(rav1e_sad_64x64_hbd_avx2);
604 out[BLOCK_128X128 as usize] = Some(rav1e_sad_128x128_hbd_avx2);
605
606 out[BLOCK_4X8 as usize] = Some(rav1e_sad_4x8_hbd_ssse3);
607 out[BLOCK_8X4 as usize] = Some(rav1e_sad_8x4_hbd_ssse3);
608 out[BLOCK_8X16 as usize] = Some(rav1e_sad_8x16_hbd_ssse3);
609 out[BLOCK_16X8 as usize] = Some(rav1e_sad_16x8_hbd_avx2);
610 out[BLOCK_16X32 as usize] = Some(rav1e_sad_16x32_hbd_avx2);
611 out[BLOCK_32X16 as usize] = Some(rav1e_sad_32x16_hbd_avx2);
612 out[BLOCK_32X64 as usize] = Some(rav1e_sad_32x64_hbd_avx2);
613 out[BLOCK_64X32 as usize] = Some(rav1e_sad_64x32_hbd_avx2);
614 out[BLOCK_64X128 as usize] = Some(rav1e_sad_64x128_hbd_avx2);
615 out[BLOCK_128X64 as usize] = Some(rav1e_sad_128x64_hbd_avx2);
616
617 out[BLOCK_4X16 as usize] = Some(rav1e_sad_4x16_hbd_ssse3);
618 out[BLOCK_16X4 as usize] = Some(rav1e_sad_16x4_hbd_avx2);
619 out[BLOCK_32X8 as usize] = Some(rav1e_sad_32x8_hbd_avx2);
620 out[BLOCK_16X64 as usize] = Some(rav1e_sad_16x64_hbd_avx2);
621 out[BLOCK_64X16 as usize] = Some(rav1e_sad_64x16_hbd_avx2);
622
623 out
624 };
625
626 cpu_function_lookup_table!(
627 SAD_HBD_FNS: [[Option<SadHBDFn>; DIST_FNS_LENGTH]],
628 default: [None; DIST_FNS_LENGTH],
629 [SSSE3, AVX2]
630 );
631
632 static SATD_FNS_SSSE3: [Option<SatdFn>; DIST_FNS_LENGTH] = {
633 let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
634
635 use BlockSize::*;
636
637 out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_ssse3);
638
639 out
640 };
641
642 static SATD_FNS_SSE4_1: [Option<SatdFn>; DIST_FNS_LENGTH] = {
643 let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
644
645 use BlockSize::*;
646
647 out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_sse4);
648 out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_ssse3);
649
650 out
651 };
652
653 static SATD_FNS_AVX2: [Option<SatdFn>; DIST_FNS_LENGTH] = {
654 let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
655
656 use BlockSize::*;
657
658 out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_avx2);
659 out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_avx2);
660 out[BLOCK_16X16 as usize] = Some(rav1e_satd_16x16_avx2);
661 out[BLOCK_32X32 as usize] = Some(rav1e_satd_32x32_avx2);
662 out[BLOCK_64X64 as usize] = Some(rav1e_satd_64x64_avx2);
663 out[BLOCK_128X128 as usize] = Some(rav1e_satd_128x128_avx2);
664
665 out[BLOCK_4X8 as usize] = Some(rav1e_satd_4x8_avx2);
666 out[BLOCK_8X4 as usize] = Some(rav1e_satd_8x4_avx2);
667 out[BLOCK_8X16 as usize] = Some(rav1e_satd_8x16_avx2);
668 out[BLOCK_16X8 as usize] = Some(rav1e_satd_16x8_avx2);
669 out[BLOCK_16X32 as usize] = Some(rav1e_satd_16x32_avx2);
670 out[BLOCK_32X16 as usize] = Some(rav1e_satd_32x16_avx2);
671 out[BLOCK_32X64 as usize] = Some(rav1e_satd_32x64_avx2);
672 out[BLOCK_64X32 as usize] = Some(rav1e_satd_64x32_avx2);
673 out[BLOCK_64X128 as usize] = Some(rav1e_satd_64x128_avx2);
674 out[BLOCK_128X64 as usize] = Some(rav1e_satd_128x64_avx2);
675
676 out[BLOCK_4X16 as usize] = Some(rav1e_satd_4x16_avx2);
677 out[BLOCK_16X4 as usize] = Some(rav1e_satd_16x4_avx2);
678 out[BLOCK_8X32 as usize] = Some(rav1e_satd_8x32_avx2);
679 out[BLOCK_32X8 as usize] = Some(rav1e_satd_32x8_avx2);
680 out[BLOCK_16X64 as usize] = Some(rav1e_satd_16x64_avx2);
681 out[BLOCK_64X16 as usize] = Some(rav1e_satd_64x16_avx2);
682
683 out
684 };
685
686 cpu_function_lookup_table!(
687 SATD_FNS: [[Option<SatdFn>; DIST_FNS_LENGTH]],
688 default: [None; DIST_FNS_LENGTH],
689 [SSSE3, SSE4_1, AVX2]
690 );
691
692 static SATD_HBD_FNS_AVX2: [Option<SatdHBDFn>; DIST_FNS_LENGTH] = {
693 let mut out: [Option<SatdHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
694
695 use BlockSize::*;
696
697 out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_hbd_avx2);
698 out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_hbd_avx2);
699 out[BLOCK_16X16 as usize] = Some(rav1e_satd_16x16_hbd_avx2);
700 out[BLOCK_32X32 as usize] = Some(rav1e_satd_32x32_hbd_avx2);
701 out[BLOCK_64X64 as usize] = Some(rav1e_satd_64x64_hbd_avx2);
702 out[BLOCK_128X128 as usize] = Some(rav1e_satd_128x128_hbd_avx2);
703
704 out[BLOCK_4X8 as usize] = Some(rav1e_satd_4x8_hbd_avx2);
705 out[BLOCK_8X4 as usize] = Some(rav1e_satd_8x4_hbd_avx2);
706 out[BLOCK_8X16 as usize] = Some(rav1e_satd_8x16_hbd_avx2);
707 out[BLOCK_16X8 as usize] = Some(rav1e_satd_16x8_hbd_avx2);
708 out[BLOCK_16X32 as usize] = Some(rav1e_satd_16x32_hbd_avx2);
709 out[BLOCK_32X16 as usize] = Some(rav1e_satd_32x16_hbd_avx2);
710 out[BLOCK_32X64 as usize] = Some(rav1e_satd_32x64_hbd_avx2);
711 out[BLOCK_64X32 as usize] = Some(rav1e_satd_64x32_hbd_avx2);
712 out[BLOCK_64X128 as usize] = Some(rav1e_satd_64x128_hbd_avx2);
713 out[BLOCK_128X64 as usize] = Some(rav1e_satd_128x64_hbd_avx2);
714
715 out[BLOCK_4X16 as usize] = Some(rav1e_satd_4x16_hbd_avx2);
716 out[BLOCK_16X4 as usize] = Some(rav1e_satd_16x4_hbd_avx2);
717 out[BLOCK_8X32 as usize] = Some(rav1e_satd_8x32_hbd_avx2);
718 out[BLOCK_32X8 as usize] = Some(rav1e_satd_32x8_hbd_avx2);
719 out[BLOCK_16X64 as usize] = Some(rav1e_satd_16x64_hbd_avx2);
720 out[BLOCK_64X16 as usize] = Some(rav1e_satd_64x16_hbd_avx2);
721
722 out
723 };
724
725 cpu_function_lookup_table!(
726 SATD_HBD_FNS: [[Option<SatdHBDFn>; DIST_FNS_LENGTH]],
727 default: [None; DIST_FNS_LENGTH],
728 [AVX2]
729 );
730
731 #[cfg(test)]
732 mod test {
733 use super::*;
734 use crate::frame::{AsRegion, Plane};
735 use rand::random;
736 use std::str::FromStr;
737
738 macro_rules! test_dist_fns {
739 ($(($W:expr, $H:expr)),*, $DIST_TY:ident, $BD:expr, $OPT:ident, $OPTLIT:tt) => {
740 $(
741 paste::item! {
742 #[test]
743 fn [<get_ $DIST_TY _ $W x $H _bd_ $BD _ $OPT>]() {
744 if CpuFeatureLevel::default() < CpuFeatureLevel::from_str($OPTLIT).unwrap() {
745 eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT);
746 return;
747 }
748
749 if $BD > 8 {
750 // dynamic allocation: test
751 let mut src = Plane::from_slice(&[0u16; $W * $H], $W);
752 // dynamic allocation: test
753 let mut dst = Plane::from_slice(&[0u16; $W * $H], $W);
754 for (s, d) in src.data.iter_mut().zip(dst.data.iter_mut()) {
755 *s = random::<u8>() as u16 * $BD / 8;
756 *d = random::<u8>() as u16 * $BD / 8;
757 }
758 let result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::from_str($OPTLIT).unwrap());
759 let rust_result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::RUST);
760
761 assert_eq!(rust_result, result);
762 } else {
763 // dynamic allocation: test
764 let mut src = Plane::from_slice(&[0u8; $W * $H], $W);
765 // dynamic allocation: test
766 let mut dst = Plane::from_slice(&[0u8; $W * $H], $W);
767 for (s, d) in src.data.iter_mut().zip(dst.data.iter_mut()) {
768 *s = random::<u8>();
769 *d = random::<u8>();
770 }
771 let result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::from_str($OPTLIT).unwrap());
772 let rust_result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::RUST);
773
774 assert_eq!(rust_result, result);
775 }
776 }
777 }
778 )*
779 }
780 }
781
782 test_dist_fns!(
783 (4, 4),
784 (16, 16),
785 (8, 8),
786 (4, 8),
787 (8, 4),
788 (8, 16),
789 (16, 8),
790 (4, 16),
791 (16, 4),
792 (8, 32),
793 (32, 8),
794 (32, 32),
795 (64, 64),
796 (128, 128),
797 (16, 32),
798 (32, 16),
799 (32, 64),
800 (64, 32),
801 (64, 128),
802 (128, 64),
803 (16, 64),
804 (64, 16),
805 sad,
806 10,
807 ssse3,
808 "ssse3"
809 );
810
811 test_dist_fns!(
812 (4, 4),
813 (16, 16),
814 (8, 8),
815 (4, 8),
816 (8, 4),
817 (8, 16),
818 (16, 8),
819 (4, 16),
820 (16, 4),
821 (8, 32),
822 (32, 8),
823 (32, 32),
824 (64, 64),
825 (128, 128),
826 (16, 32),
827 (32, 16),
828 (32, 64),
829 (64, 32),
830 (64, 128),
831 (128, 64),
832 (16, 64),
833 (64, 16),
834 sad,
835 10,
836 avx2,
837 "avx2"
838 );
839
840 test_dist_fns!(
841 (4, 4),
842 (4, 8),
843 (4, 16),
844 (8, 4),
845 (8, 8),
846 (8, 16),
847 (8, 32),
848 (16, 16),
849 (32, 32),
850 (64, 64),
851 (128, 128),
852 sad,
853 8,
854 sse2,
855 "sse2"
856 );
857
858 test_dist_fns!(
859 (16, 4),
860 (16, 8),
861 (16, 16),
862 (16, 32),
863 (16, 64),
864 (32, 8),
865 (32, 16),
866 (32, 32),
867 (32, 64),
868 (64, 16),
869 (64, 32),
870 (64, 64),
871 (64, 128),
872 (128, 64),
873 (128, 128),
874 sad,
875 8,
876 avx2,
877 "avx2"
878 );
879
880 test_dist_fns!((8, 8), satd, 8, ssse3, "ssse3");
881
882 test_dist_fns!((4, 4), satd, 8, sse4, "sse4.1");
883
884 test_dist_fns!(
885 (4, 4),
886 (8, 8),
887 (16, 16),
888 (32, 32),
889 (64, 64),
890 (128, 128),
891 (4, 8),
892 (8, 4),
893 (8, 16),
894 (16, 8),
895 (16, 32),
896 (32, 16),
897 (32, 64),
898 (64, 32),
899 (64, 128),
900 (128, 64),
901 (4, 16),
902 (16, 4),
903 (8, 32),
904 (32, 8),
905 (16, 64),
906 (64, 16),
907 satd,
908 8,
909 avx2,
910 "avx2"
911 );
912
913 test_dist_fns!(
914 (4, 4),
915 (8, 8),
916 (16, 16),
917 (32, 32),
918 (64, 64),
919 (128, 128),
920 (4, 8),
921 (8, 4),
922 (8, 16),
923 (16, 8),
924 (16, 32),
925 (32, 16),
926 (32, 64),
927 (64, 32),
928 (64, 128),
929 (128, 64),
930 (4, 16),
931 (16, 4),
932 (8, 32),
933 (32, 8),
934 (16, 64),
935 (64, 16),
936 satd,
937 10,
938 avx2,
939 "avx2"
940 );
941
942 test_dist_fns!(
943 (4, 4),
944 (8, 8),
945 (16, 16),
946 (32, 32),
947 (64, 64),
948 (128, 128),
949 (4, 8),
950 (8, 4),
951 (8, 16),
952 (16, 8),
953 (16, 32),
954 (32, 16),
955 (32, 64),
956 (64, 32),
957 (64, 128),
958 (128, 64),
959 (4, 16),
960 (16, 4),
961 (8, 32),
962 (32, 8),
963 (16, 64),
964 (64, 16),
965 satd,
966 12,
967 avx2,
968 "avx2"
969 );
970 }