| 1 |
// Copyright (c) 2019-2022, The rav1e contributors. All rights reserved |
| 2 |
// |
| 3 |
// This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
// was not distributed with this source code in the LICENSE file, you can |
| 6 |
// obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
// Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
// PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
pub use self::cdef_dist::*; |
| 11 |
pub use self::sse::*; |
| 12 |
use crate::cpu_features::CpuFeatureLevel; |
| 13 |
use crate::dist::*; |
| 14 |
use crate::partition::BlockSize; |
| 15 |
use crate::tiling::*; |
| 16 |
use crate::util::*; |
| 17 |
|
| 18 |
mod cdef_dist; |
| 19 |
mod sse; |
| 20 |
|
| 21 |
type SadFn = unsafe extern fn( |
| 22 |
src: *const u8, |
| 23 |
src_stride: isize, |
| 24 |
dst: *const u8, |
| 25 |
dst_stride: isize, |
| 26 |
) -> u32; |
| 27 |
|
| 28 |
type SatdFn = SadFn; |
| 29 |
|
| 30 |
type SadHBDFn = unsafe extern fn( |
| 31 |
src: *const u16, |
| 32 |
src_stride: isize, |
| 33 |
dst: *const u16, |
| 34 |
dst_stride: isize, |
| 35 |
) -> u32; |
| 36 |
|
| 37 |
type SatdHBDFn = unsafe extern fn( |
| 38 |
src: *const u16, |
| 39 |
src_stride: isize, |
| 40 |
dst: *const u16, |
| 41 |
dst_stride: isize, |
| 42 |
bdmax: u32, |
| 43 |
) -> u32; |
| 44 |
|
| 45 |
macro_rules! declare_asm_dist_fn { |
| 46 |
($(($name: ident, $T: ident)),+) => ( |
| 47 |
$( |
| 48 |
extern { fn $name ( |
| 49 |
src: *const $T, src_stride: isize, dst: *const $T, dst_stride: isize |
| 50 |
) -> u32; } |
| 51 |
)+ |
| 52 |
) |
| 53 |
} |
| 54 |
|
| 55 |
macro_rules! declare_asm_satd_hbd_fn { |
| 56 |
($($name: ident),+) => ( |
| 57 |
$( |
| 58 |
extern { pub(crate) fn $name ( |
| 59 |
src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, bdmax: u32 |
| 60 |
) -> u32; } |
| 61 |
)+ |
| 62 |
) |
| 63 |
} |
| 64 |
|
| 65 |
#[cfg(target_arch = "x86_64")] |
| 66 |
use std::arch::x86_64::*; |
| 67 |
|
| 68 |
/// Horizontally sum 8 32-bit values in a YMM register |
| 69 |
#[inline] |
| 70 |
#[target_feature(enable = "avx2")] |
| 71 |
unsafe fn mm256_sum_i32(ymm: __m256i) -> i32 { |
| 72 |
// We split the vector in half and then add (2, 3) + (0, 1), and finally 0 + 1. |
| 73 |
let m1 = _mm256_extracti128_si256(ymm, 1); |
| 74 |
let m2 = _mm256_castsi256_si128(ymm); |
| 75 |
let m2 = _mm_add_epi32(m2, m1); |
| 76 |
let m1 = _mm_shuffle_epi32(m2, 0b11_10_11_10); |
| 77 |
let m2 = _mm_add_epi32(m2, m1); |
| 78 |
let m1 = _mm_shuffle_epi32(m2, 0b01_01_01_01); |
| 79 |
let m2 = _mm_add_epi32(m2, m1); |
| 80 |
_mm_cvtsi128_si32(m2) |
| 81 |
} |
| 82 |
|
| 83 |
/// Perform SAD over at most 256 (10 or 12-bit) values |
| 84 |
/// |
| 85 |
/// Returns a YMM register of 8 32-bit sums, which can |
| 86 |
/// be used as an intermediate result for larger summations |
| 87 |
/// |
| 88 |
/// Limited to 256 values as any more 12-bit SADs may overflow |
| 89 |
/// a 16-bit value. |
| 90 |
/// |
| 91 |
/// # Panics |
| 92 |
/// - If `W` * `n_rows` > 256 |
| 93 |
/// - If `W` is not a multiple of 16 |
| 94 |
#[inline] |
| 95 |
#[target_feature(enable = "avx2")] |
| 96 |
unsafe fn rav1e_sad_wxh_hbd_avx2_inner<const W: usize>( |
| 97 |
src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize, |
| 98 |
n_rows: usize, |
| 99 |
) -> __m256i { |
| 100 |
const LOADS_PER_REGISTER: usize = 32 / core::mem::size_of::<u16>(); // 16 |
| 101 |
|
| 102 |
// Check we can't overflow a 16-bit register |
| 103 |
assert!(W * n_rows <= 256); |
| 104 |
// Check we load the right number of values |
| 105 |
assert_eq!(W % LOADS_PER_REGISTER, 0); |
| 106 |
|
| 107 |
let mut sum16 = _mm256_setzero_si256(); |
| 108 |
(0..n_rows as isize).for_each(|n| { |
| 109 |
(0..(W / LOADS_PER_REGISTER) as isize).for_each(|x| { |
| 110 |
let res = _mm256_abs_epi16(_mm256_sub_epi16( |
| 111 |
_mm256_loadu_si256( |
| 112 |
src.offset(n * src_stride + x * LOADS_PER_REGISTER as isize * 2) |
| 113 |
as *const _, |
| 114 |
), |
| 115 |
_mm256_loadu_si256( |
| 116 |
dst.offset(n * dst_stride + x * LOADS_PER_REGISTER as isize * 2) |
| 117 |
as *const _, |
| 118 |
), |
| 119 |
)); |
| 120 |
sum16 = _mm256_add_epi16(sum16, res); |
| 121 |
}) |
| 122 |
}); |
| 123 |
|
| 124 |
let b = _mm256_unpackhi_epi16(sum16, _mm256_setzero_si256()); |
| 125 |
let c = _mm256_unpacklo_epi16(sum16, _mm256_setzero_si256()); |
| 126 |
|
| 127 |
_mm256_add_epi32(b, c) |
| 128 |
} |
| 129 |
|
| 130 |
/// Sum of Absolute Differences |
| 131 |
/// |
| 132 |
/// By convention, `src_stride` and `dst_stride` are measured in bytes, not u16's |
| 133 |
#[inline] |
| 134 |
#[target_feature(enable = "avx2")] |
| 135 |
unsafe fn rav1e_sad_wxh_hbd_avx2<const W: usize>( |
| 136 |
src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, |
| 137 |
n_rows: usize, |
| 138 |
) -> u32 { |
| 139 |
const MAX_INNER_SIZE: usize = 256; |
| 140 |
let src = src as *const u8; |
| 141 |
let dst = dst as *const u8; |
| 142 |
|
| 143 |
// Can we sum at least 1 entire row at once? |
| 144 |
if W <= MAX_INNER_SIZE { |
| 145 |
// If we can fit the entire area in one inner iteration, do it |
| 146 |
if W * n_rows <= MAX_INNER_SIZE { |
| 147 |
return mm256_sum_i32(rav1e_sad_wxh_hbd_avx2_inner::<W>( |
| 148 |
src, src_stride, dst, dst_stride, n_rows, |
| 149 |
)) as u32; |
| 150 |
} |
| 151 |
// Otherwise, sum rows in sets of 256 |
| 152 |
assert_eq!(MAX_INNER_SIZE % W, 0); |
| 153 |
let row_step = (MAX_INNER_SIZE / W).min(n_rows); |
| 154 |
assert_eq!(n_rows % row_step, 0); |
| 155 |
let mut sum = _mm256_setzero_si256(); |
| 156 |
(0..(n_rows / row_step) as isize).for_each(|w| { |
| 157 |
let src = src.offset(w * row_step as isize * src_stride); |
| 158 |
let dst = dst.offset(w * row_step as isize * dst_stride); |
| 159 |
let res = rav1e_sad_wxh_hbd_avx2_inner::<W>( |
| 160 |
src, src_stride, dst, dst_stride, row_step, |
| 161 |
); |
| 162 |
sum = _mm256_add_epi32(sum, res) |
| 163 |
}); |
| 164 |
mm256_sum_i32(sum) as u32 |
| 165 |
} else { |
| 166 |
// If we can't sum an entire row in one shot, split each row up |
| 167 |
assert_eq!(W % MAX_INNER_SIZE, 0); |
| 168 |
let sum = (0..n_rows) |
| 169 |
.flat_map(|_| { |
| 170 |
(0..(W / MAX_INNER_SIZE)).map(move |h| { |
| 171 |
let src = src.offset(src_stride + (h * MAX_INNER_SIZE * 2) as isize); |
| 172 |
let dst = dst.offset(dst_stride + (h * MAX_INNER_SIZE * 2) as isize); |
| 173 |
rav1e_sad_wxh_hbd_avx2_inner::<MAX_INNER_SIZE>( |
| 174 |
src, src_stride, dst, dst_stride, 1, |
| 175 |
) |
| 176 |
}) |
| 177 |
}) |
| 178 |
.reduce(|a, b| unsafe { _mm256_add_epi32(a, b) }) |
| 179 |
.unwrap(); |
| 180 |
mm256_sum_i32(sum) as u32 |
| 181 |
} |
| 182 |
} |
| 183 |
|
| 184 |
declare_asm_dist_fn![ |
| 185 |
// SSSE3 |
| 186 |
(rav1e_sad_4x4_hbd_ssse3, u16), |
| 187 |
(rav1e_sad_16x16_hbd_ssse3, u16), |
| 188 |
(rav1e_satd_8x8_ssse3, u8), |
| 189 |
// SSE2 |
| 190 |
(rav1e_sad4x4_sse2, u8), |
| 191 |
(rav1e_sad4x8_sse2, u8), |
| 192 |
(rav1e_sad4x16_sse2, u8), |
| 193 |
(rav1e_sad8x4_sse2, u8), |
| 194 |
(rav1e_sad8x8_sse2, u8), |
| 195 |
(rav1e_sad8x16_sse2, u8), |
| 196 |
(rav1e_sad8x32_sse2, u8), |
| 197 |
(rav1e_sad16x4_sse2, u8), |
| 198 |
(rav1e_sad16x8_sse2, u8), |
| 199 |
(rav1e_sad16x16_sse2, u8), |
| 200 |
(rav1e_sad16x32_sse2, u8), |
| 201 |
(rav1e_sad16x64_sse2, u8), |
| 202 |
(rav1e_sad32x8_sse2, u8), |
| 203 |
(rav1e_sad32x16_sse2, u8), |
| 204 |
(rav1e_sad32x32_sse2, u8), |
| 205 |
(rav1e_sad32x64_sse2, u8), |
| 206 |
(rav1e_sad64x16_sse2, u8), |
| 207 |
(rav1e_sad64x32_sse2, u8), |
| 208 |
(rav1e_sad64x64_sse2, u8), |
| 209 |
(rav1e_sad64x128_sse2, u8), |
| 210 |
(rav1e_sad128x64_sse2, u8), |
| 211 |
(rav1e_sad128x128_sse2, u8), |
| 212 |
// SSE4 |
| 213 |
(rav1e_satd_4x4_sse4, u8), |
| 214 |
// AVX |
| 215 |
(rav1e_sad32x8_avx2, u8), |
| 216 |
(rav1e_sad32x16_avx2, u8), |
| 217 |
(rav1e_sad32x32_avx2, u8), |
| 218 |
(rav1e_sad32x64_avx2, u8), |
| 219 |
(rav1e_sad64x16_avx2, u8), |
| 220 |
(rav1e_sad64x32_avx2, u8), |
| 221 |
(rav1e_sad64x64_avx2, u8), |
| 222 |
(rav1e_sad64x128_avx2, u8), |
| 223 |
(rav1e_sad128x64_avx2, u8), |
| 224 |
(rav1e_sad128x128_avx2, u8), |
| 225 |
(rav1e_satd_4x4_avx2, u8), |
| 226 |
(rav1e_satd_8x8_avx2, u8), |
| 227 |
(rav1e_satd_16x16_avx2, u8), |
| 228 |
(rav1e_satd_32x32_avx2, u8), |
| 229 |
(rav1e_satd_64x64_avx2, u8), |
| 230 |
(rav1e_satd_128x128_avx2, u8), |
| 231 |
(rav1e_satd_4x8_avx2, u8), |
| 232 |
(rav1e_satd_8x4_avx2, u8), |
| 233 |
(rav1e_satd_8x16_avx2, u8), |
| 234 |
(rav1e_satd_16x8_avx2, u8), |
| 235 |
(rav1e_satd_16x32_avx2, u8), |
| 236 |
(rav1e_satd_32x16_avx2, u8), |
| 237 |
(rav1e_satd_32x64_avx2, u8), |
| 238 |
(rav1e_satd_64x32_avx2, u8), |
| 239 |
(rav1e_satd_64x128_avx2, u8), |
| 240 |
(rav1e_satd_128x64_avx2, u8), |
| 241 |
(rav1e_satd_4x16_avx2, u8), |
| 242 |
(rav1e_satd_16x4_avx2, u8), |
| 243 |
(rav1e_satd_8x32_avx2, u8), |
| 244 |
(rav1e_satd_32x8_avx2, u8), |
| 245 |
(rav1e_satd_16x64_avx2, u8), |
| 246 |
(rav1e_satd_64x16_avx2, u8) |
| 247 |
]; |
| 248 |
|
| 249 |
declare_asm_satd_hbd_fn![ |
| 250 |
rav1e_satd_4x4_hbd_avx2, |
| 251 |
rav1e_satd_8x4_hbd_avx2, |
| 252 |
rav1e_satd_4x8_hbd_avx2, |
| 253 |
rav1e_satd_8x8_hbd_avx2, |
| 254 |
rav1e_satd_16x8_hbd_avx2, |
| 255 |
rav1e_satd_16x16_hbd_avx2, |
| 256 |
rav1e_satd_32x32_hbd_avx2, |
| 257 |
rav1e_satd_64x64_hbd_avx2, |
| 258 |
rav1e_satd_128x128_hbd_avx2, |
| 259 |
rav1e_satd_16x32_hbd_avx2, |
| 260 |
rav1e_satd_16x64_hbd_avx2, |
| 261 |
rav1e_satd_32x16_hbd_avx2, |
| 262 |
rav1e_satd_32x64_hbd_avx2, |
| 263 |
rav1e_satd_64x16_hbd_avx2, |
| 264 |
rav1e_satd_64x32_hbd_avx2, |
| 265 |
rav1e_satd_64x128_hbd_avx2, |
| 266 |
rav1e_satd_128x64_hbd_avx2, |
| 267 |
rav1e_satd_32x8_hbd_avx2, |
| 268 |
rav1e_satd_8x16_hbd_avx2, |
| 269 |
rav1e_satd_8x32_hbd_avx2, |
| 270 |
rav1e_satd_16x4_hbd_avx2, |
| 271 |
rav1e_satd_4x16_hbd_avx2 |
| 272 |
]; |
| 273 |
|
| 274 |
// BlockSize::BLOCK_SIZES.next_power_of_two(); |
| 275 |
pub(crate) const DIST_FNS_LENGTH: usize = 32; |
| 276 |
|
| 277 |
#[inline] |
| 278 |
pub(crate) const fn to_index(bsize: BlockSize) -> usize { |
| 279 |
bsize as usize & (DIST_FNS_LENGTH - 1) |
| 280 |
} |
| 281 |
|
| 282 |
/// # Panics |
| 283 |
/// |
| 284 |
/// - If in `check_asm` mode, panics on mismatch between native and ASM results. |
| 285 |
#[inline(always)] |
| 286 |
#[allow(clippy::let_and_return)] |
| 287 |
pub fn get_sad<T: Pixel>( |
| 288 |
src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, w: usize, h: usize, |
| 289 |
bit_depth: usize, cpu: CpuFeatureLevel, |
| 290 |
) -> u32 { |
| 291 |
let bsize_opt = BlockSize::from_width_and_height_opt(w, h); |
| 292 |
|
| 293 |
let call_rust = || -> u32 { rust::get_sad(dst, src, w, h, bit_depth, cpu) }; |
| 294 |
|
| 295 |
#[cfg(feature = "check_asm")] |
| 296 |
let ref_dist = call_rust(); |
| 297 |
|
| 298 |
let dist = match (bsize_opt, T::type_enum()) { |
| 299 |
(Err(_), _) => call_rust(), |
| 300 |
(Ok(bsize), PixelType::U8) => { |
| 301 |
match SAD_FNS[cpu.as_index()][to_index(bsize)] { |
| 302 |
// SAFETY: Calls Assembly code. |
| 303 |
Some(func) => unsafe { |
| 304 |
(func)( |
| 305 |
src.data_ptr() as *const _, |
| 306 |
T::to_asm_stride(src.plane_cfg.stride), |
| 307 |
dst.data_ptr() as *const _, |
| 308 |
T::to_asm_stride(dst.plane_cfg.stride), |
| 309 |
) |
| 310 |
}, |
| 311 |
None => call_rust(), |
| 312 |
} |
| 313 |
} |
| 314 |
(Ok(bsize), PixelType::U16) => { |
| 315 |
match SAD_HBD_FNS[cpu.as_index()][to_index(bsize)] { |
| 316 |
// SAFETY: Calls Assembly code. |
| 317 |
Some(func) => unsafe { |
| 318 |
(func)( |
| 319 |
src.data_ptr() as *const _, |
| 320 |
T::to_asm_stride(src.plane_cfg.stride), |
| 321 |
dst.data_ptr() as *const _, |
| 322 |
T::to_asm_stride(dst.plane_cfg.stride), |
| 323 |
) |
| 324 |
}, |
| 325 |
None => call_rust(), |
| 326 |
} |
| 327 |
} |
| 328 |
}; |
| 329 |
|
| 330 |
#[cfg(feature = "check_asm")] |
| 331 |
assert_eq!(dist, ref_dist); |
| 332 |
|
| 333 |
dist |
| 334 |
} |
| 335 |
|
| 336 |
/// # Panics |
| 337 |
/// |
| 338 |
/// - If in `check_asm` mode, panics on mismatch between native and ASM results. |
| 339 |
#[inline(always)] |
| 340 |
#[allow(clippy::let_and_return)] |
| 341 |
pub fn get_satd<T: Pixel>( |
| 342 |
src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, w: usize, h: usize, |
| 343 |
bit_depth: usize, cpu: CpuFeatureLevel, |
| 344 |
) -> u32 { |
| 345 |
let bsize_opt = BlockSize::from_width_and_height_opt(w, h); |
| 346 |
|
| 347 |
let call_rust = || -> u32 { rust::get_satd(dst, src, w, h, bit_depth, cpu) }; |
| 348 |
|
| 349 |
#[cfg(feature = "check_asm")] |
| 350 |
let ref_dist = call_rust(); |
| 351 |
|
| 352 |
let dist = match (bsize_opt, T::type_enum()) { |
| 353 |
(Err(_), _) => call_rust(), |
| 354 |
(Ok(bsize), PixelType::U8) => { |
| 355 |
match SATD_FNS[cpu.as_index()][to_index(bsize)] { |
| 356 |
// SAFETY: Calls Assembly code. |
| 357 |
Some(func) => unsafe { |
| 358 |
(func)( |
| 359 |
src.data_ptr() as *const _, |
| 360 |
T::to_asm_stride(src.plane_cfg.stride), |
| 361 |
dst.data_ptr() as *const _, |
| 362 |
T::to_asm_stride(dst.plane_cfg.stride), |
| 363 |
) |
| 364 |
}, |
| 365 |
None => call_rust(), |
| 366 |
} |
| 367 |
} |
| 368 |
(Ok(bsize), PixelType::U16) => { |
| 369 |
match SATD_HBD_FNS[cpu.as_index()][to_index(bsize)] { |
| 370 |
// SAFETY: Calls Assembly code. |
| 371 |
Some(func) => unsafe { |
| 372 |
(func)( |
| 373 |
src.data_ptr() as *const _, |
| 374 |
T::to_asm_stride(src.plane_cfg.stride), |
| 375 |
dst.data_ptr() as *const _, |
| 376 |
T::to_asm_stride(dst.plane_cfg.stride), |
| 377 |
(1 << bit_depth) - 1, |
| 378 |
) |
| 379 |
}, |
| 380 |
None => call_rust(), |
| 381 |
} |
| 382 |
} |
| 383 |
}; |
| 384 |
|
| 385 |
#[cfg(feature = "check_asm")] |
| 386 |
assert_eq!(dist, ref_dist); |
| 387 |
|
| 388 |
dist |
| 389 |
} |
| 390 |
|
| 391 |
// We have hand-written ASM for 4x4 and 16x16 HBD blocks, |
| 392 |
// so we can use those for other block sizes as well. |
| 393 |
macro_rules! get_sad_hbd_ssse3 { |
| 394 |
($(($W:expr, $H:expr, $BS:expr)),*) => { |
| 395 |
$( |
| 396 |
paste::item! { |
| 397 |
#[target_feature(enable = "ssse3")] |
| 398 |
unsafe extern fn [<rav1e_sad_ $W x $H _hbd_ssse3>]( |
| 399 |
src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, |
| 400 |
) -> u32 { |
| 401 |
let mut sum = 0; |
| 402 |
for w in (0..$W).step_by($BS) { |
| 403 |
for h in (0..$H).step_by($BS) { |
| 404 |
sum += [<rav1e_sad_ $BS x $BS _hbd_ssse3>]( |
| 405 |
src.offset(w + h * src_stride / 2), |
| 406 |
src_stride, |
| 407 |
dst.offset(w + h * dst_stride / 2), |
| 408 |
dst_stride |
| 409 |
); |
| 410 |
} |
| 411 |
} |
| 412 |
sum |
| 413 |
} |
| 414 |
} |
| 415 |
)* |
| 416 |
} |
| 417 |
} |
| 418 |
|
| 419 |
get_sad_hbd_ssse3!( |
| 420 |
// 4x4 base |
| 421 |
(8, 8, 4), |
| 422 |
(4, 8, 4), |
| 423 |
(8, 4, 4), |
| 424 |
(8, 16, 4), |
| 425 |
(16, 8, 4), |
| 426 |
(4, 16, 4), |
| 427 |
(16, 4, 4), |
| 428 |
(8, 32, 4), |
| 429 |
(32, 8, 4), |
| 430 |
(32, 32, 16), |
| 431 |
(64, 64, 16), |
| 432 |
(128, 128, 16), |
| 433 |
(16, 32, 16), |
| 434 |
(32, 16, 16), |
| 435 |
(32, 64, 16), |
| 436 |
(64, 32, 16), |
| 437 |
(64, 128, 16), |
| 438 |
(128, 64, 16), |
| 439 |
(16, 64, 16), |
| 440 |
(64, 16, 16) |
| 441 |
); |
| 442 |
|
| 443 |
macro_rules! get_sad_hbd_avx2_WxH { |
| 444 |
($(($W:expr, $H:expr)),*) => { |
| 445 |
$( |
| 446 |
paste::item! { |
| 447 |
#[target_feature(enable = "avx2")] |
| 448 |
unsafe extern fn [<rav1e_sad_ $W x $H _hbd_avx2>]( |
| 449 |
src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, |
| 450 |
) -> u32 { |
| 451 |
rav1e_sad_wxh_hbd_avx2::<$W>( |
| 452 |
src, |
| 453 |
src_stride, |
| 454 |
dst, |
| 455 |
dst_stride, |
| 456 |
$H, |
| 457 |
) |
| 458 |
} |
| 459 |
} |
| 460 |
)* |
| 461 |
} |
| 462 |
} |
| 463 |
|
| 464 |
get_sad_hbd_avx2_WxH!( |
| 465 |
// No advantage to AVX below 16 width |
| 466 |
(16, 4), |
| 467 |
(16, 8), |
| 468 |
(16, 16), |
| 469 |
(16, 32), |
| 470 |
(16, 64), |
| 471 |
(32, 8), |
| 472 |
(32, 16), |
| 473 |
(32, 32), |
| 474 |
(32, 64), |
| 475 |
(64, 16), |
| 476 |
(64, 32), |
| 477 |
(64, 64), |
| 478 |
(64, 128), |
| 479 |
(128, 64), |
| 480 |
(128, 128) |
| 481 |
); |
| 482 |
|
| 483 |
static SAD_FNS_SSE2: [Option<SadFn>; DIST_FNS_LENGTH] = { |
| 484 |
let mut out: [Option<SadFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 485 |
|
| 486 |
use BlockSize::*; |
| 487 |
|
| 488 |
out[BLOCK_4X4 as usize] = Some(rav1e_sad4x4_sse2); |
| 489 |
out[BLOCK_4X8 as usize] = Some(rav1e_sad4x8_sse2); |
| 490 |
out[BLOCK_4X16 as usize] = Some(rav1e_sad4x16_sse2); |
| 491 |
|
| 492 |
out[BLOCK_8X4 as usize] = Some(rav1e_sad8x4_sse2); |
| 493 |
out[BLOCK_8X8 as usize] = Some(rav1e_sad8x8_sse2); |
| 494 |
out[BLOCK_8X16 as usize] = Some(rav1e_sad8x16_sse2); |
| 495 |
out[BLOCK_8X32 as usize] = Some(rav1e_sad8x32_sse2); |
| 496 |
|
| 497 |
out[BLOCK_16X4 as usize] = Some(rav1e_sad16x4_sse2); |
| 498 |
out[BLOCK_16X8 as usize] = Some(rav1e_sad16x8_sse2); |
| 499 |
out[BLOCK_16X16 as usize] = Some(rav1e_sad16x16_sse2); |
| 500 |
out[BLOCK_16X32 as usize] = Some(rav1e_sad16x32_sse2); |
| 501 |
out[BLOCK_16X64 as usize] = Some(rav1e_sad16x64_sse2); |
| 502 |
|
| 503 |
out[BLOCK_32X8 as usize] = Some(rav1e_sad32x8_sse2); |
| 504 |
out[BLOCK_32X16 as usize] = Some(rav1e_sad32x16_sse2); |
| 505 |
out[BLOCK_32X32 as usize] = Some(rav1e_sad32x32_sse2); |
| 506 |
out[BLOCK_32X64 as usize] = Some(rav1e_sad32x64_sse2); |
| 507 |
|
| 508 |
out[BLOCK_64X16 as usize] = Some(rav1e_sad64x16_sse2); |
| 509 |
out[BLOCK_64X32 as usize] = Some(rav1e_sad64x32_sse2); |
| 510 |
out[BLOCK_64X64 as usize] = Some(rav1e_sad64x64_sse2); |
| 511 |
out[BLOCK_64X128 as usize] = Some(rav1e_sad64x128_sse2); |
| 512 |
|
| 513 |
out[BLOCK_128X64 as usize] = Some(rav1e_sad128x64_sse2); |
| 514 |
out[BLOCK_128X128 as usize] = Some(rav1e_sad128x128_sse2); |
| 515 |
|
| 516 |
out |
| 517 |
}; |
| 518 |
|
| 519 |
static SAD_FNS_AVX2: [Option<SadFn>; DIST_FNS_LENGTH] = { |
| 520 |
let mut out: [Option<SadFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 521 |
|
| 522 |
use BlockSize::*; |
| 523 |
|
| 524 |
out[BLOCK_4X4 as usize] = Some(rav1e_sad4x4_sse2); |
| 525 |
out[BLOCK_4X8 as usize] = Some(rav1e_sad4x8_sse2); |
| 526 |
out[BLOCK_4X16 as usize] = Some(rav1e_sad4x16_sse2); |
| 527 |
|
| 528 |
out[BLOCK_8X4 as usize] = Some(rav1e_sad8x4_sse2); |
| 529 |
out[BLOCK_8X8 as usize] = Some(rav1e_sad8x8_sse2); |
| 530 |
out[BLOCK_8X16 as usize] = Some(rav1e_sad8x16_sse2); |
| 531 |
out[BLOCK_8X32 as usize] = Some(rav1e_sad8x32_sse2); |
| 532 |
|
| 533 |
out[BLOCK_16X4 as usize] = Some(rav1e_sad16x4_sse2); |
| 534 |
out[BLOCK_16X8 as usize] = Some(rav1e_sad16x8_sse2); |
| 535 |
out[BLOCK_16X16 as usize] = Some(rav1e_sad16x16_sse2); |
| 536 |
out[BLOCK_16X32 as usize] = Some(rav1e_sad16x32_sse2); |
| 537 |
out[BLOCK_16X64 as usize] = Some(rav1e_sad16x64_sse2); |
| 538 |
|
| 539 |
out[BLOCK_32X8 as usize] = Some(rav1e_sad32x8_avx2); |
| 540 |
out[BLOCK_32X16 as usize] = Some(rav1e_sad32x16_avx2); |
| 541 |
out[BLOCK_32X32 as usize] = Some(rav1e_sad32x32_avx2); |
| 542 |
out[BLOCK_32X64 as usize] = Some(rav1e_sad32x64_avx2); |
| 543 |
|
| 544 |
out[BLOCK_64X16 as usize] = Some(rav1e_sad64x16_avx2); |
| 545 |
out[BLOCK_64X32 as usize] = Some(rav1e_sad64x32_avx2); |
| 546 |
out[BLOCK_64X64 as usize] = Some(rav1e_sad64x64_avx2); |
| 547 |
out[BLOCK_64X128 as usize] = Some(rav1e_sad64x128_avx2); |
| 548 |
|
| 549 |
out[BLOCK_128X64 as usize] = Some(rav1e_sad128x64_avx2); |
| 550 |
out[BLOCK_128X128 as usize] = Some(rav1e_sad128x128_avx2); |
| 551 |
|
| 552 |
out |
| 553 |
}; |
| 554 |
|
| 555 |
cpu_function_lookup_table!( |
| 556 |
SAD_FNS: [[Option<SadFn>; DIST_FNS_LENGTH]], |
| 557 |
default: [None; DIST_FNS_LENGTH], |
| 558 |
[SSE2, AVX2] |
| 559 |
); |
| 560 |
|
| 561 |
static SAD_HBD_FNS_SSSE3: [Option<SadHBDFn>; DIST_FNS_LENGTH] = { |
| 562 |
let mut out: [Option<SadHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 563 |
|
| 564 |
use BlockSize::*; |
| 565 |
|
| 566 |
out[BLOCK_4X4 as usize] = Some(rav1e_sad_4x4_hbd_ssse3); |
| 567 |
out[BLOCK_8X8 as usize] = Some(rav1e_sad_8x8_hbd_ssse3); |
| 568 |
out[BLOCK_16X16 as usize] = Some(rav1e_sad_16x16_hbd_ssse3); |
| 569 |
out[BLOCK_32X32 as usize] = Some(rav1e_sad_32x32_hbd_ssse3); |
| 570 |
out[BLOCK_64X64 as usize] = Some(rav1e_sad_64x64_hbd_ssse3); |
| 571 |
out[BLOCK_128X128 as usize] = Some(rav1e_sad_128x128_hbd_ssse3); |
| 572 |
|
| 573 |
out[BLOCK_4X8 as usize] = Some(rav1e_sad_4x8_hbd_ssse3); |
| 574 |
out[BLOCK_8X4 as usize] = Some(rav1e_sad_8x4_hbd_ssse3); |
| 575 |
out[BLOCK_8X16 as usize] = Some(rav1e_sad_8x16_hbd_ssse3); |
| 576 |
out[BLOCK_16X8 as usize] = Some(rav1e_sad_16x8_hbd_ssse3); |
| 577 |
out[BLOCK_16X32 as usize] = Some(rav1e_sad_16x32_hbd_ssse3); |
| 578 |
out[BLOCK_32X16 as usize] = Some(rav1e_sad_32x16_hbd_ssse3); |
| 579 |
out[BLOCK_32X64 as usize] = Some(rav1e_sad_32x64_hbd_ssse3); |
| 580 |
out[BLOCK_64X32 as usize] = Some(rav1e_sad_64x32_hbd_ssse3); |
| 581 |
out[BLOCK_64X128 as usize] = Some(rav1e_sad_64x128_hbd_ssse3); |
| 582 |
out[BLOCK_128X64 as usize] = Some(rav1e_sad_128x64_hbd_ssse3); |
| 583 |
|
| 584 |
out[BLOCK_4X16 as usize] = Some(rav1e_sad_4x16_hbd_ssse3); |
| 585 |
out[BLOCK_16X4 as usize] = Some(rav1e_sad_16x4_hbd_ssse3); |
| 586 |
out[BLOCK_8X32 as usize] = Some(rav1e_sad_8x32_hbd_ssse3); |
| 587 |
out[BLOCK_32X8 as usize] = Some(rav1e_sad_32x8_hbd_ssse3); |
| 588 |
out[BLOCK_16X64 as usize] = Some(rav1e_sad_16x64_hbd_ssse3); |
| 589 |
out[BLOCK_64X16 as usize] = Some(rav1e_sad_64x16_hbd_ssse3); |
| 590 |
|
| 591 |
out |
| 592 |
}; |
| 593 |
|
| 594 |
static SAD_HBD_FNS_AVX2: [Option<SadHBDFn>; DIST_FNS_LENGTH] = { |
| 595 |
let mut out: [Option<SadHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 596 |
|
| 597 |
use BlockSize::*; |
| 598 |
|
| 599 |
out[BLOCK_4X4 as usize] = Some(rav1e_sad_4x4_hbd_ssse3); |
| 600 |
out[BLOCK_8X8 as usize] = Some(rav1e_sad_8x8_hbd_ssse3); |
| 601 |
out[BLOCK_16X16 as usize] = Some(rav1e_sad_16x16_hbd_avx2); |
| 602 |
out[BLOCK_32X32 as usize] = Some(rav1e_sad_32x32_hbd_avx2); |
| 603 |
out[BLOCK_64X64 as usize] = Some(rav1e_sad_64x64_hbd_avx2); |
| 604 |
out[BLOCK_128X128 as usize] = Some(rav1e_sad_128x128_hbd_avx2); |
| 605 |
|
| 606 |
out[BLOCK_4X8 as usize] = Some(rav1e_sad_4x8_hbd_ssse3); |
| 607 |
out[BLOCK_8X4 as usize] = Some(rav1e_sad_8x4_hbd_ssse3); |
| 608 |
out[BLOCK_8X16 as usize] = Some(rav1e_sad_8x16_hbd_ssse3); |
| 609 |
out[BLOCK_16X8 as usize] = Some(rav1e_sad_16x8_hbd_avx2); |
| 610 |
out[BLOCK_16X32 as usize] = Some(rav1e_sad_16x32_hbd_avx2); |
| 611 |
out[BLOCK_32X16 as usize] = Some(rav1e_sad_32x16_hbd_avx2); |
| 612 |
out[BLOCK_32X64 as usize] = Some(rav1e_sad_32x64_hbd_avx2); |
| 613 |
out[BLOCK_64X32 as usize] = Some(rav1e_sad_64x32_hbd_avx2); |
| 614 |
out[BLOCK_64X128 as usize] = Some(rav1e_sad_64x128_hbd_avx2); |
| 615 |
out[BLOCK_128X64 as usize] = Some(rav1e_sad_128x64_hbd_avx2); |
| 616 |
|
| 617 |
out[BLOCK_4X16 as usize] = Some(rav1e_sad_4x16_hbd_ssse3); |
| 618 |
out[BLOCK_16X4 as usize] = Some(rav1e_sad_16x4_hbd_avx2); |
| 619 |
out[BLOCK_32X8 as usize] = Some(rav1e_sad_32x8_hbd_avx2); |
| 620 |
out[BLOCK_16X64 as usize] = Some(rav1e_sad_16x64_hbd_avx2); |
| 621 |
out[BLOCK_64X16 as usize] = Some(rav1e_sad_64x16_hbd_avx2); |
| 622 |
|
| 623 |
out |
| 624 |
}; |
| 625 |
|
| 626 |
cpu_function_lookup_table!( |
| 627 |
SAD_HBD_FNS: [[Option<SadHBDFn>; DIST_FNS_LENGTH]], |
| 628 |
default: [None; DIST_FNS_LENGTH], |
| 629 |
[SSSE3, AVX2] |
| 630 |
); |
| 631 |
|
| 632 |
static SATD_FNS_SSSE3: [Option<SatdFn>; DIST_FNS_LENGTH] = { |
| 633 |
let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 634 |
|
| 635 |
use BlockSize::*; |
| 636 |
|
| 637 |
out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_ssse3); |
| 638 |
|
| 639 |
out |
| 640 |
}; |
| 641 |
|
| 642 |
static SATD_FNS_SSE4_1: [Option<SatdFn>; DIST_FNS_LENGTH] = { |
| 643 |
let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 644 |
|
| 645 |
use BlockSize::*; |
| 646 |
|
| 647 |
out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_sse4); |
| 648 |
out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_ssse3); |
| 649 |
|
| 650 |
out |
| 651 |
}; |
| 652 |
|
| 653 |
static SATD_FNS_AVX2: [Option<SatdFn>; DIST_FNS_LENGTH] = { |
| 654 |
let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 655 |
|
| 656 |
use BlockSize::*; |
| 657 |
|
| 658 |
out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_avx2); |
| 659 |
out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_avx2); |
| 660 |
out[BLOCK_16X16 as usize] = Some(rav1e_satd_16x16_avx2); |
| 661 |
out[BLOCK_32X32 as usize] = Some(rav1e_satd_32x32_avx2); |
| 662 |
out[BLOCK_64X64 as usize] = Some(rav1e_satd_64x64_avx2); |
| 663 |
out[BLOCK_128X128 as usize] = Some(rav1e_satd_128x128_avx2); |
| 664 |
|
| 665 |
out[BLOCK_4X8 as usize] = Some(rav1e_satd_4x8_avx2); |
| 666 |
out[BLOCK_8X4 as usize] = Some(rav1e_satd_8x4_avx2); |
| 667 |
out[BLOCK_8X16 as usize] = Some(rav1e_satd_8x16_avx2); |
| 668 |
out[BLOCK_16X8 as usize] = Some(rav1e_satd_16x8_avx2); |
| 669 |
out[BLOCK_16X32 as usize] = Some(rav1e_satd_16x32_avx2); |
| 670 |
out[BLOCK_32X16 as usize] = Some(rav1e_satd_32x16_avx2); |
| 671 |
out[BLOCK_32X64 as usize] = Some(rav1e_satd_32x64_avx2); |
| 672 |
out[BLOCK_64X32 as usize] = Some(rav1e_satd_64x32_avx2); |
| 673 |
out[BLOCK_64X128 as usize] = Some(rav1e_satd_64x128_avx2); |
| 674 |
out[BLOCK_128X64 as usize] = Some(rav1e_satd_128x64_avx2); |
| 675 |
|
| 676 |
out[BLOCK_4X16 as usize] = Some(rav1e_satd_4x16_avx2); |
| 677 |
out[BLOCK_16X4 as usize] = Some(rav1e_satd_16x4_avx2); |
| 678 |
out[BLOCK_8X32 as usize] = Some(rav1e_satd_8x32_avx2); |
| 679 |
out[BLOCK_32X8 as usize] = Some(rav1e_satd_32x8_avx2); |
| 680 |
out[BLOCK_16X64 as usize] = Some(rav1e_satd_16x64_avx2); |
| 681 |
out[BLOCK_64X16 as usize] = Some(rav1e_satd_64x16_avx2); |
| 682 |
|
| 683 |
out |
| 684 |
}; |
| 685 |
|
| 686 |
cpu_function_lookup_table!( |
| 687 |
SATD_FNS: [[Option<SatdFn>; DIST_FNS_LENGTH]], |
| 688 |
default: [None; DIST_FNS_LENGTH], |
| 689 |
[SSSE3, SSE4_1, AVX2] |
| 690 |
); |
| 691 |
|
| 692 |
static SATD_HBD_FNS_AVX2: [Option<SatdHBDFn>; DIST_FNS_LENGTH] = { |
| 693 |
let mut out: [Option<SatdHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH]; |
| 694 |
|
| 695 |
use BlockSize::*; |
| 696 |
|
| 697 |
out[BLOCK_4X4 as usize] = Some(rav1e_satd_4x4_hbd_avx2); |
| 698 |
out[BLOCK_8X8 as usize] = Some(rav1e_satd_8x8_hbd_avx2); |
| 699 |
out[BLOCK_16X16 as usize] = Some(rav1e_satd_16x16_hbd_avx2); |
| 700 |
out[BLOCK_32X32 as usize] = Some(rav1e_satd_32x32_hbd_avx2); |
| 701 |
out[BLOCK_64X64 as usize] = Some(rav1e_satd_64x64_hbd_avx2); |
| 702 |
out[BLOCK_128X128 as usize] = Some(rav1e_satd_128x128_hbd_avx2); |
| 703 |
|
| 704 |
out[BLOCK_4X8 as usize] = Some(rav1e_satd_4x8_hbd_avx2); |
| 705 |
out[BLOCK_8X4 as usize] = Some(rav1e_satd_8x4_hbd_avx2); |
| 706 |
out[BLOCK_8X16 as usize] = Some(rav1e_satd_8x16_hbd_avx2); |
| 707 |
out[BLOCK_16X8 as usize] = Some(rav1e_satd_16x8_hbd_avx2); |
| 708 |
out[BLOCK_16X32 as usize] = Some(rav1e_satd_16x32_hbd_avx2); |
| 709 |
out[BLOCK_32X16 as usize] = Some(rav1e_satd_32x16_hbd_avx2); |
| 710 |
out[BLOCK_32X64 as usize] = Some(rav1e_satd_32x64_hbd_avx2); |
| 711 |
out[BLOCK_64X32 as usize] = Some(rav1e_satd_64x32_hbd_avx2); |
| 712 |
out[BLOCK_64X128 as usize] = Some(rav1e_satd_64x128_hbd_avx2); |
| 713 |
out[BLOCK_128X64 as usize] = Some(rav1e_satd_128x64_hbd_avx2); |
| 714 |
|
| 715 |
out[BLOCK_4X16 as usize] = Some(rav1e_satd_4x16_hbd_avx2); |
| 716 |
out[BLOCK_16X4 as usize] = Some(rav1e_satd_16x4_hbd_avx2); |
| 717 |
out[BLOCK_8X32 as usize] = Some(rav1e_satd_8x32_hbd_avx2); |
| 718 |
out[BLOCK_32X8 as usize] = Some(rav1e_satd_32x8_hbd_avx2); |
| 719 |
out[BLOCK_16X64 as usize] = Some(rav1e_satd_16x64_hbd_avx2); |
| 720 |
out[BLOCK_64X16 as usize] = Some(rav1e_satd_64x16_hbd_avx2); |
| 721 |
|
| 722 |
out |
| 723 |
}; |
| 724 |
|
| 725 |
cpu_function_lookup_table!( |
| 726 |
SATD_HBD_FNS: [[Option<SatdHBDFn>; DIST_FNS_LENGTH]], |
| 727 |
default: [None; DIST_FNS_LENGTH], |
| 728 |
[AVX2] |
| 729 |
); |
| 730 |
|
| 731 |
#[cfg(test)] |
| 732 |
mod test { |
| 733 |
use super::*; |
| 734 |
use crate::frame::{AsRegion, Plane}; |
| 735 |
use rand::random; |
| 736 |
use std::str::FromStr; |
| 737 |
|
| 738 |
macro_rules! test_dist_fns { |
| 739 |
($(($W:expr, $H:expr)),*, $DIST_TY:ident, $BD:expr, $OPT:ident, $OPTLIT:tt) => { |
| 740 |
$( |
| 741 |
paste::item! { |
| 742 |
#[test] |
| 743 |
fn [<get_ $DIST_TY _ $W x $H _bd_ $BD _ $OPT>]() { |
| 744 |
if CpuFeatureLevel::default() < CpuFeatureLevel::from_str($OPTLIT).unwrap() { |
| 745 |
eprintln!("Ignoring {} test, not supported on this machine!", $OPTLIT); |
| 746 |
return; |
| 747 |
} |
| 748 |
|
| 749 |
if $BD > 8 { |
| 750 |
// dynamic allocation: test |
| 751 |
let mut src = Plane::from_slice(&[0u16; $W * $H], $W); |
| 752 |
// dynamic allocation: test |
| 753 |
let mut dst = Plane::from_slice(&[0u16; $W * $H], $W); |
| 754 |
for (s, d) in src.data.iter_mut().zip(dst.data.iter_mut()) { |
| 755 |
*s = random::<u8>() as u16 * $BD / 8; |
| 756 |
*d = random::<u8>() as u16 * $BD / 8; |
| 757 |
} |
| 758 |
let result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::from_str($OPTLIT).unwrap()); |
| 759 |
let rust_result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::RUST); |
| 760 |
|
| 761 |
assert_eq!(rust_result, result); |
| 762 |
} else { |
| 763 |
// dynamic allocation: test |
| 764 |
let mut src = Plane::from_slice(&[0u8; $W * $H], $W); |
| 765 |
// dynamic allocation: test |
| 766 |
let mut dst = Plane::from_slice(&[0u8; $W * $H], $W); |
| 767 |
for (s, d) in src.data.iter_mut().zip(dst.data.iter_mut()) { |
| 768 |
*s = random::<u8>(); |
| 769 |
*d = random::<u8>(); |
| 770 |
} |
| 771 |
let result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::from_str($OPTLIT).unwrap()); |
| 772 |
let rust_result = [<get_ $DIST_TY>](&src.as_region(), &dst.as_region(), $W, $H, $BD, CpuFeatureLevel::RUST); |
| 773 |
|
| 774 |
assert_eq!(rust_result, result); |
| 775 |
} |
| 776 |
} |
| 777 |
} |
| 778 |
)* |
| 779 |
} |
| 780 |
} |
| 781 |
|
| 782 |
test_dist_fns!( |
| 783 |
(4, 4), |
| 784 |
(16, 16), |
| 785 |
(8, 8), |
| 786 |
(4, 8), |
| 787 |
(8, 4), |
| 788 |
(8, 16), |
| 789 |
(16, 8), |
| 790 |
(4, 16), |
| 791 |
(16, 4), |
| 792 |
(8, 32), |
| 793 |
(32, 8), |
| 794 |
(32, 32), |
| 795 |
(64, 64), |
| 796 |
(128, 128), |
| 797 |
(16, 32), |
| 798 |
(32, 16), |
| 799 |
(32, 64), |
| 800 |
(64, 32), |
| 801 |
(64, 128), |
| 802 |
(128, 64), |
| 803 |
(16, 64), |
| 804 |
(64, 16), |
| 805 |
sad, |
| 806 |
10, |
| 807 |
ssse3, |
| 808 |
"ssse3" |
| 809 |
); |
| 810 |
|
| 811 |
test_dist_fns!( |
| 812 |
(4, 4), |
| 813 |
(16, 16), |
| 814 |
(8, 8), |
| 815 |
(4, 8), |
| 816 |
(8, 4), |
| 817 |
(8, 16), |
| 818 |
(16, 8), |
| 819 |
(4, 16), |
| 820 |
(16, 4), |
| 821 |
(8, 32), |
| 822 |
(32, 8), |
| 823 |
(32, 32), |
| 824 |
(64, 64), |
| 825 |
(128, 128), |
| 826 |
(16, 32), |
| 827 |
(32, 16), |
| 828 |
(32, 64), |
| 829 |
(64, 32), |
| 830 |
(64, 128), |
| 831 |
(128, 64), |
| 832 |
(16, 64), |
| 833 |
(64, 16), |
| 834 |
sad, |
| 835 |
10, |
| 836 |
avx2, |
| 837 |
"avx2" |
| 838 |
); |
| 839 |
|
| 840 |
test_dist_fns!( |
| 841 |
(4, 4), |
| 842 |
(4, 8), |
| 843 |
(4, 16), |
| 844 |
(8, 4), |
| 845 |
(8, 8), |
| 846 |
(8, 16), |
| 847 |
(8, 32), |
| 848 |
(16, 16), |
| 849 |
(32, 32), |
| 850 |
(64, 64), |
| 851 |
(128, 128), |
| 852 |
sad, |
| 853 |
8, |
| 854 |
sse2, |
| 855 |
"sse2" |
| 856 |
); |
| 857 |
|
| 858 |
test_dist_fns!( |
| 859 |
(16, 4), |
| 860 |
(16, 8), |
| 861 |
(16, 16), |
| 862 |
(16, 32), |
| 863 |
(16, 64), |
| 864 |
(32, 8), |
| 865 |
(32, 16), |
| 866 |
(32, 32), |
| 867 |
(32, 64), |
| 868 |
(64, 16), |
| 869 |
(64, 32), |
| 870 |
(64, 64), |
| 871 |
(64, 128), |
| 872 |
(128, 64), |
| 873 |
(128, 128), |
| 874 |
sad, |
| 875 |
8, |
| 876 |
avx2, |
| 877 |
"avx2" |
| 878 |
); |
| 879 |
|
| 880 |
test_dist_fns!((8, 8), satd, 8, ssse3, "ssse3"); |
| 881 |
|
| 882 |
test_dist_fns!((4, 4), satd, 8, sse4, "sse4.1"); |
| 883 |
|
| 884 |
test_dist_fns!( |
| 885 |
(4, 4), |
| 886 |
(8, 8), |
| 887 |
(16, 16), |
| 888 |
(32, 32), |
| 889 |
(64, 64), |
| 890 |
(128, 128), |
| 891 |
(4, 8), |
| 892 |
(8, 4), |
| 893 |
(8, 16), |
| 894 |
(16, 8), |
| 895 |
(16, 32), |
| 896 |
(32, 16), |
| 897 |
(32, 64), |
| 898 |
(64, 32), |
| 899 |
(64, 128), |
| 900 |
(128, 64), |
| 901 |
(4, 16), |
| 902 |
(16, 4), |
| 903 |
(8, 32), |
| 904 |
(32, 8), |
| 905 |
(16, 64), |
| 906 |
(64, 16), |
| 907 |
satd, |
| 908 |
8, |
| 909 |
avx2, |
| 910 |
"avx2" |
| 911 |
); |
| 912 |
|
| 913 |
test_dist_fns!( |
| 914 |
(4, 4), |
| 915 |
(8, 8), |
| 916 |
(16, 16), |
| 917 |
(32, 32), |
| 918 |
(64, 64), |
| 919 |
(128, 128), |
| 920 |
(4, 8), |
| 921 |
(8, 4), |
| 922 |
(8, 16), |
| 923 |
(16, 8), |
| 924 |
(16, 32), |
| 925 |
(32, 16), |
| 926 |
(32, 64), |
| 927 |
(64, 32), |
| 928 |
(64, 128), |
| 929 |
(128, 64), |
| 930 |
(4, 16), |
| 931 |
(16, 4), |
| 932 |
(8, 32), |
| 933 |
(32, 8), |
| 934 |
(16, 64), |
| 935 |
(64, 16), |
| 936 |
satd, |
| 937 |
10, |
| 938 |
avx2, |
| 939 |
"avx2" |
| 940 |
); |
| 941 |
|
| 942 |
test_dist_fns!( |
| 943 |
(4, 4), |
| 944 |
(8, 8), |
| 945 |
(16, 16), |
| 946 |
(32, 32), |
| 947 |
(64, 64), |
| 948 |
(128, 128), |
| 949 |
(4, 8), |
| 950 |
(8, 4), |
| 951 |
(8, 16), |
| 952 |
(16, 8), |
| 953 |
(16, 32), |
| 954 |
(32, 16), |
| 955 |
(32, 64), |
| 956 |
(64, 32), |
| 957 |
(64, 128), |
| 958 |
(128, 64), |
| 959 |
(4, 16), |
| 960 |
(16, 4), |
| 961 |
(8, 32), |
| 962 |
(32, 8), |
| 963 |
(16, 64), |
| 964 |
(64, 16), |
| 965 |
satd, |
| 966 |
12, |
| 967 |
avx2, |
| 968 |
"avx2" |
| 969 |
); |
| 970 |
} |