| 1 |
// Copyright (c) 2023, The rav1e contributors. All rights reserved |
| 2 |
// |
| 3 |
// This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
// was not distributed with this source code in the LICENSE file, you can |
| 6 |
// obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
// Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
// PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
use crate::asm::aarch64::dist::to_index; |
| 11 |
use crate::asm::aarch64::dist::DIST_FNS_LENGTH; |
| 12 |
use crate::cpu_features::CpuFeatureLevel; |
| 13 |
use crate::dist::*; |
| 14 |
use crate::encoder::IMPORTANCE_BLOCK_SIZE; |
| 15 |
use crate::partition::BlockSize; |
| 16 |
use crate::rdo::DistortionScale; |
| 17 |
use crate::tiling::PlaneRegion; |
| 18 |
use crate::util::*; |
| 19 |
|
| 20 |
type WeightedSseFn = unsafe extern fn( |
| 21 |
src: *const u8, |
| 22 |
src_stride: isize, |
| 23 |
dst: *const u8, |
| 24 |
dst_stride: isize, |
| 25 |
scale: *const u32, |
| 26 |
scale_stride: isize, |
| 27 |
) -> u64; |
| 28 |
|
| 29 |
type WeightedSseHBDFn = unsafe extern fn( |
| 30 |
src: *const u16, |
| 31 |
src_stride: isize, |
| 32 |
dst: *const u16, |
| 33 |
dst_stride: isize, |
| 34 |
scale: *const u32, |
| 35 |
scale_stride: isize, |
| 36 |
) -> u64; |
| 37 |
|
| 38 |
macro_rules! declare_asm_sse_fn { |
| 39 |
($($name: ident),+) => ( |
| 40 |
$( |
| 41 |
extern { fn $name ( |
| 42 |
src: *const u8, src_stride: isize, dst: *const u8, dst_stride: isize, scale: *const u32, scale_stride: isize |
| 43 |
) -> u64; } |
| 44 |
)+ |
| 45 |
) |
| 46 |
} |
| 47 |
|
| 48 |
macro_rules! declare_asm_hbd_sse_fn { |
| 49 |
($($name: ident),+) => ( |
| 50 |
$( |
| 51 |
extern { fn $name ( |
| 52 |
src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, scale: *const u32, scale_stride: isize |
| 53 |
) -> u64; } |
| 54 |
)+ |
| 55 |
) |
| 56 |
} |
| 57 |
|
| 58 |
declare_asm_sse_fn![ |
| 59 |
rav1e_weighted_sse_4x4_neon, |
| 60 |
rav1e_weighted_sse_4x8_neon, |
| 61 |
rav1e_weighted_sse_4x16_neon, |
| 62 |
rav1e_weighted_sse_8x4_neon, |
| 63 |
rav1e_weighted_sse_8x8_neon, |
| 64 |
rav1e_weighted_sse_8x16_neon, |
| 65 |
rav1e_weighted_sse_8x32_neon, |
| 66 |
rav1e_weighted_sse_16x4_neon, |
| 67 |
rav1e_weighted_sse_16x8_neon, |
| 68 |
rav1e_weighted_sse_16x16_neon, |
| 69 |
rav1e_weighted_sse_16x32_neon, |
| 70 |
rav1e_weighted_sse_16x64_neon, |
| 71 |
rav1e_weighted_sse_32x8_neon, |
| 72 |
rav1e_weighted_sse_32x16_neon, |
| 73 |
rav1e_weighted_sse_32x32_neon, |
| 74 |
rav1e_weighted_sse_32x64_neon, |
| 75 |
rav1e_weighted_sse_64x16_neon, |
| 76 |
rav1e_weighted_sse_64x32_neon, |
| 77 |
rav1e_weighted_sse_64x64_neon, |
| 78 |
rav1e_weighted_sse_64x128_neon, |
| 79 |
rav1e_weighted_sse_128x64_neon, |
| 80 |
rav1e_weighted_sse_128x128_neon |
| 81 |
]; |
| 82 |
|
| 83 |
declare_asm_hbd_sse_fn![ |
| 84 |
rav1e_weighted_sse_4x4_hbd_neon, |
| 85 |
rav1e_weighted_sse_4x8_hbd_neon, |
| 86 |
rav1e_weighted_sse_4x16_hbd_neon, |
| 87 |
rav1e_weighted_sse_8x4_hbd_neon, |
| 88 |
rav1e_weighted_sse_8x8_hbd_neon, |
| 89 |
rav1e_weighted_sse_8x16_hbd_neon, |
| 90 |
rav1e_weighted_sse_8x32_hbd_neon, |
| 91 |
rav1e_weighted_sse_16x4_hbd_neon, |
| 92 |
rav1e_weighted_sse_16x8_hbd_neon, |
| 93 |
rav1e_weighted_sse_16x16_hbd_neon, |
| 94 |
rav1e_weighted_sse_16x32_hbd_neon, |
| 95 |
rav1e_weighted_sse_16x64_hbd_neon, |
| 96 |
rav1e_weighted_sse_32x8_hbd_neon, |
| 97 |
rav1e_weighted_sse_32x16_hbd_neon, |
| 98 |
rav1e_weighted_sse_32x32_hbd_neon, |
| 99 |
rav1e_weighted_sse_32x64_hbd_neon, |
| 100 |
rav1e_weighted_sse_64x16_hbd_neon, |
| 101 |
rav1e_weighted_sse_64x32_hbd_neon, |
| 102 |
rav1e_weighted_sse_64x64_hbd_neon, |
| 103 |
rav1e_weighted_sse_64x128_hbd_neon, |
| 104 |
rav1e_weighted_sse_128x64_hbd_neon, |
| 105 |
rav1e_weighted_sse_128x128_hbd_neon |
| 106 |
]; |
| 107 |
|
| 108 |
/// # Panics |
| 109 |
/// |
| 110 |
/// - If in `check_asm` mode, panics on mismatch between native and ASM results. |
| 111 |
#[inline(always)] |
| 112 |
#[allow(clippy::let_and_return)] |
| 113 |
pub fn get_weighted_sse<T: Pixel>( |
| 114 |
src: &PlaneRegion<'_, T>, dst: &PlaneRegion<'_, T>, scale: &[u32], |
| 115 |
scale_stride: usize, w: usize, h: usize, bit_depth: usize, |
| 116 |
cpu: CpuFeatureLevel, |
| 117 |
) -> u64 { |
| 118 |
// Assembly breaks if imp block size changes. |
| 119 |
assert_eq!(IMPORTANCE_BLOCK_SIZE >> 1, 4); |
| 120 |
|
| 121 |
let bsize_opt = BlockSize::from_width_and_height_opt(w, h); |
| 122 |
|
| 123 |
let call_rust = || -> u64 { |
| 124 |
rust::get_weighted_sse(dst, src, scale, scale_stride, w, h, bit_depth, cpu) |
| 125 |
}; |
| 126 |
|
| 127 |
#[cfg(feature = "check_asm")] |
| 128 |
let ref_dist = call_rust(); |
| 129 |
|
| 130 |
let den = |
| 131 |
DistortionScale::new(1, 1 << rust::GET_WEIGHTED_SSE_SHIFT).0 as u64; |
| 132 |
let dist = match (bsize_opt, T::type_enum()) { |
| 133 |
(Err(_), _) => call_rust(), |
| 134 |
(Ok(bsize), PixelType::U8) => { |
| 135 |
match SSE_FNS[cpu.as_index()][to_index(bsize)] { |
| 136 |
// SAFETY: Calls Assembly code. |
| 137 |
Some(func) => unsafe { |
| 138 |
((func)( |
| 139 |
src.data_ptr() as *const _, |
| 140 |
T::to_asm_stride(src.plane_cfg.stride), |
| 141 |
dst.data_ptr() as *const _, |
| 142 |
T::to_asm_stride(dst.plane_cfg.stride), |
| 143 |
scale.as_ptr(), |
| 144 |
(scale_stride * std::mem::size_of::<u32>()) as isize, |
| 145 |
) + (den >> 1)) |
| 146 |
/ den |
| 147 |
}, |
| 148 |
None => call_rust(), |
| 149 |
} |
| 150 |
} |
| 151 |
(Ok(bsize), PixelType::U16) => { |
| 152 |
match SSE_HBD_FNS[cpu.as_index()][to_index(bsize)] { |
| 153 |
// SAFETY: Calls Assembly code. |
| 154 |
Some(func) => unsafe { |
| 155 |
((func)( |
| 156 |
src.data_ptr() as *const _, |
| 157 |
T::to_asm_stride(src.plane_cfg.stride), |
| 158 |
dst.data_ptr() as *const _, |
| 159 |
T::to_asm_stride(dst.plane_cfg.stride), |
| 160 |
scale.as_ptr(), |
| 161 |
(scale_stride * std::mem::size_of::<u32>()) as isize, |
| 162 |
) + (den >> 1)) |
| 163 |
/ den |
| 164 |
}, |
| 165 |
None => call_rust(), |
| 166 |
} |
| 167 |
} |
| 168 |
}; |
| 169 |
|
| 170 |
#[cfg(feature = "check_asm")] |
| 171 |
assert_eq!( |
| 172 |
dist, ref_dist, |
| 173 |
"Weighted SSE {:?}: Assembly doesn't match reference code.", |
| 174 |
bsize_opt |
| 175 |
); |
| 176 |
|
| 177 |
dist |
| 178 |
} |
| 179 |
|
| 180 |
static SSE_FNS_NEON: [Option<WeightedSseFn>; DIST_FNS_LENGTH] = { |
| 181 |
let mut out: [Option<WeightedSseFn>; DIST_FNS_LENGTH] = |
| 182 |
[None; DIST_FNS_LENGTH]; |
| 183 |
|
| 184 |
use BlockSize::*; |
| 185 |
out[BLOCK_4X4 as usize] = Some(rav1e_weighted_sse_4x4_neon); |
| 186 |
out[BLOCK_4X8 as usize] = Some(rav1e_weighted_sse_4x8_neon); |
| 187 |
out[BLOCK_4X16 as usize] = Some(rav1e_weighted_sse_4x16_neon); |
| 188 |
out[BLOCK_8X4 as usize] = Some(rav1e_weighted_sse_8x4_neon); |
| 189 |
out[BLOCK_8X8 as usize] = Some(rav1e_weighted_sse_8x8_neon); |
| 190 |
out[BLOCK_8X16 as usize] = Some(rav1e_weighted_sse_8x16_neon); |
| 191 |
out[BLOCK_8X32 as usize] = Some(rav1e_weighted_sse_8x32_neon); |
| 192 |
out[BLOCK_16X4 as usize] = Some(rav1e_weighted_sse_16x4_neon); |
| 193 |
out[BLOCK_16X8 as usize] = Some(rav1e_weighted_sse_16x8_neon); |
| 194 |
out[BLOCK_16X16 as usize] = Some(rav1e_weighted_sse_16x16_neon); |
| 195 |
out[BLOCK_16X32 as usize] = Some(rav1e_weighted_sse_16x32_neon); |
| 196 |
out[BLOCK_16X64 as usize] = Some(rav1e_weighted_sse_16x64_neon); |
| 197 |
out[BLOCK_32X8 as usize] = Some(rav1e_weighted_sse_32x8_neon); |
| 198 |
out[BLOCK_32X16 as usize] = Some(rav1e_weighted_sse_32x16_neon); |
| 199 |
out[BLOCK_32X32 as usize] = Some(rav1e_weighted_sse_32x32_neon); |
| 200 |
out[BLOCK_32X64 as usize] = Some(rav1e_weighted_sse_32x64_neon); |
| 201 |
out[BLOCK_64X16 as usize] = Some(rav1e_weighted_sse_64x16_neon); |
| 202 |
out[BLOCK_64X32 as usize] = Some(rav1e_weighted_sse_64x32_neon); |
| 203 |
out[BLOCK_64X64 as usize] = Some(rav1e_weighted_sse_64x64_neon); |
| 204 |
out[BLOCK_64X128 as usize] = Some(rav1e_weighted_sse_64x128_neon); |
| 205 |
out[BLOCK_128X64 as usize] = Some(rav1e_weighted_sse_128x64_neon); |
| 206 |
out[BLOCK_128X128 as usize] = Some(rav1e_weighted_sse_128x128_neon); |
| 207 |
|
| 208 |
out |
| 209 |
}; |
| 210 |
|
| 211 |
static SSE_HBD_FNS_NEON: [Option<WeightedSseHBDFn>; DIST_FNS_LENGTH] = { |
| 212 |
let mut out: [Option<WeightedSseHBDFn>; DIST_FNS_LENGTH] = |
| 213 |
[None; DIST_FNS_LENGTH]; |
| 214 |
|
| 215 |
use BlockSize::*; |
| 216 |
out[BLOCK_4X4 as usize] = Some(rav1e_weighted_sse_4x4_hbd_neon); |
| 217 |
out[BLOCK_4X8 as usize] = Some(rav1e_weighted_sse_4x8_hbd_neon); |
| 218 |
out[BLOCK_4X16 as usize] = Some(rav1e_weighted_sse_4x16_hbd_neon); |
| 219 |
out[BLOCK_8X4 as usize] = Some(rav1e_weighted_sse_8x4_hbd_neon); |
| 220 |
out[BLOCK_8X8 as usize] = Some(rav1e_weighted_sse_8x8_hbd_neon); |
| 221 |
out[BLOCK_8X16 as usize] = Some(rav1e_weighted_sse_8x16_hbd_neon); |
| 222 |
out[BLOCK_8X32 as usize] = Some(rav1e_weighted_sse_8x32_hbd_neon); |
| 223 |
out[BLOCK_16X4 as usize] = Some(rav1e_weighted_sse_16x4_hbd_neon); |
| 224 |
out[BLOCK_16X8 as usize] = Some(rav1e_weighted_sse_16x8_hbd_neon); |
| 225 |
out[BLOCK_16X16 as usize] = Some(rav1e_weighted_sse_16x16_hbd_neon); |
| 226 |
out[BLOCK_16X32 as usize] = Some(rav1e_weighted_sse_16x32_hbd_neon); |
| 227 |
out[BLOCK_16X64 as usize] = Some(rav1e_weighted_sse_16x64_hbd_neon); |
| 228 |
out[BLOCK_32X8 as usize] = Some(rav1e_weighted_sse_32x8_hbd_neon); |
| 229 |
out[BLOCK_32X16 as usize] = Some(rav1e_weighted_sse_32x16_hbd_neon); |
| 230 |
out[BLOCK_32X32 as usize] = Some(rav1e_weighted_sse_32x32_hbd_neon); |
| 231 |
out[BLOCK_32X64 as usize] = Some(rav1e_weighted_sse_32x64_hbd_neon); |
| 232 |
out[BLOCK_64X16 as usize] = Some(rav1e_weighted_sse_64x16_hbd_neon); |
| 233 |
out[BLOCK_64X32 as usize] = Some(rav1e_weighted_sse_64x32_hbd_neon); |
| 234 |
out[BLOCK_64X64 as usize] = Some(rav1e_weighted_sse_64x64_hbd_neon); |
| 235 |
out[BLOCK_64X128 as usize] = Some(rav1e_weighted_sse_64x128_hbd_neon); |
| 236 |
out[BLOCK_128X64 as usize] = Some(rav1e_weighted_sse_128x64_hbd_neon); |
| 237 |
out[BLOCK_128X128 as usize] = Some(rav1e_weighted_sse_128x128_hbd_neon); |
| 238 |
|
| 239 |
out |
| 240 |
}; |
| 241 |
|
| 242 |
cpu_function_lookup_table!( |
| 243 |
SSE_FNS: [[Option<WeightedSseFn>; DIST_FNS_LENGTH]], |
| 244 |
default: [None; DIST_FNS_LENGTH], |
| 245 |
[NEON] |
| 246 |
); |
| 247 |
|
| 248 |
cpu_function_lookup_table!( |
| 249 |
SSE_HBD_FNS: [[Option<WeightedSseHBDFn>; DIST_FNS_LENGTH]], |
| 250 |
default: [None; DIST_FNS_LENGTH], |
| 251 |
[NEON] |
| 252 |
); |