| 1 |
// Copyright (c) 2019-2022, The rav1e contributors. All rights reserved |
| 2 |
// |
| 3 |
// This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
// was not distributed with this source code in the LICENSE file, you can |
| 6 |
// obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
// Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
// PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
use crate::context::MAX_TX_SIZE; |
| 11 |
use crate::cpu_features::CpuFeatureLevel; |
| 12 |
use crate::partition::{BlockSize, IntraEdge}; |
| 13 |
use crate::predict::rust::{ |
| 14 |
dr_intra_derivative, select_ief_strength, select_ief_upsample, |
| 15 |
}; |
| 16 |
use crate::predict::{ |
| 17 |
rust, IntraEdgeFilterParameters, PredictionMode, PredictionVariant, |
| 18 |
}; |
| 19 |
use crate::tiling::{PlaneRegion, PlaneRegionMut}; |
| 20 |
use crate::transform::TxSize; |
| 21 |
use crate::{Pixel, PixelType}; |
| 22 |
use libc; |
| 23 |
use libc::{c_int, ptrdiff_t}; |
| 24 |
use std::mem::MaybeUninit; |
| 25 |
use PixelType::{U16, U8}; |
| 26 |
|
| 27 |
macro_rules! decl_cfl_ac_fn { |
| 28 |
($($f:ident),+) => { |
| 29 |
extern { |
| 30 |
$( |
| 31 |
fn $f( |
| 32 |
ac: *mut MaybeUninit<i16>, src: *const u8, stride: libc::ptrdiff_t, |
| 33 |
w_pad: libc::c_int, h_pad: libc::c_int, |
| 34 |
width: libc::c_int, height: libc::c_int, |
| 35 |
); |
| 36 |
)* |
| 37 |
} |
| 38 |
}; |
| 39 |
} |
| 40 |
|
| 41 |
decl_cfl_ac_fn! { |
| 42 |
rav1e_ipred_cfl_ac_420_8bpc_neon, |
| 43 |
rav1e_ipred_cfl_ac_422_8bpc_neon, |
| 44 |
rav1e_ipred_cfl_ac_444_8bpc_neon |
| 45 |
} |
| 46 |
|
| 47 |
macro_rules! decl_cfl_ac_hbd_fn { |
| 48 |
($($f:ident),+) => { |
| 49 |
extern { |
| 50 |
$( |
| 51 |
fn $f( |
| 52 |
ac: *mut MaybeUninit<i16>, src: *const u16, stride: libc::ptrdiff_t, |
| 53 |
w_pad: libc::c_int, h_pad: libc::c_int, |
| 54 |
width: libc::c_int, height: libc::c_int, |
| 55 |
); |
| 56 |
)* |
| 57 |
} |
| 58 |
}; |
| 59 |
} |
| 60 |
|
| 61 |
decl_cfl_ac_hbd_fn! { |
| 62 |
rav1e_ipred_cfl_ac_420_16bpc_neon, |
| 63 |
rav1e_ipred_cfl_ac_422_16bpc_neon, |
| 64 |
rav1e_ipred_cfl_ac_444_16bpc_neon |
| 65 |
} |
| 66 |
|
| 67 |
macro_rules! decl_angular_ipred_fn { |
| 68 |
($($f:ident),+) => { |
| 69 |
extern { |
| 70 |
$( |
| 71 |
fn $f( |
| 72 |
dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8, |
| 73 |
width: libc::c_int, height: libc::c_int, angle: libc::c_int, |
| 74 |
); |
| 75 |
)* |
| 76 |
} |
| 77 |
}; |
| 78 |
} |
| 79 |
|
| 80 |
decl_angular_ipred_fn! { |
| 81 |
rav1e_ipred_dc_8bpc_neon, |
| 82 |
rav1e_ipred_dc_128_8bpc_neon, |
| 83 |
rav1e_ipred_dc_left_8bpc_neon, |
| 84 |
rav1e_ipred_dc_top_8bpc_neon, |
| 85 |
rav1e_ipred_v_8bpc_neon, |
| 86 |
rav1e_ipred_h_8bpc_neon, |
| 87 |
rav1e_ipred_smooth_8bpc_neon, |
| 88 |
rav1e_ipred_smooth_v_8bpc_neon, |
| 89 |
rav1e_ipred_smooth_h_8bpc_neon, |
| 90 |
rav1e_ipred_paeth_8bpc_neon |
| 91 |
} |
| 92 |
|
| 93 |
macro_rules! decl_angular_ipred_hbd_fn { |
| 94 |
($($f:ident),+) => { |
| 95 |
extern { |
| 96 |
$( |
| 97 |
fn $f( |
| 98 |
dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16, |
| 99 |
width: libc::c_int, height: libc::c_int, angle: libc::c_int, |
| 100 |
max_width: libc::c_int, max_height: libc::c_int, |
| 101 |
bit_depth_max: libc::c_int, |
| 102 |
); |
| 103 |
)* |
| 104 |
} |
| 105 |
}; |
| 106 |
} |
| 107 |
|
| 108 |
decl_angular_ipred_hbd_fn! { |
| 109 |
rav1e_ipred_dc_16bpc_neon, |
| 110 |
rav1e_ipred_dc_128_16bpc_neon, |
| 111 |
rav1e_ipred_dc_left_16bpc_neon, |
| 112 |
rav1e_ipred_dc_top_16bpc_neon, |
| 113 |
rav1e_ipred_v_16bpc_neon, |
| 114 |
rav1e_ipred_h_16bpc_neon, |
| 115 |
rav1e_ipred_smooth_16bpc_neon, |
| 116 |
rav1e_ipred_smooth_v_16bpc_neon, |
| 117 |
rav1e_ipred_smooth_h_16bpc_neon, |
| 118 |
rav1e_ipred_paeth_16bpc_neon |
| 119 |
} |
| 120 |
|
| 121 |
macro_rules! decl_cfl_pred_fn { |
| 122 |
($($f:ident),+) => { |
| 123 |
extern { |
| 124 |
$( |
| 125 |
fn $f( |
| 126 |
dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8, |
| 127 |
width: libc::c_int, height: libc::c_int, ac: *const i16, |
| 128 |
alpha: libc::c_int, |
| 129 |
); |
| 130 |
)* |
| 131 |
} |
| 132 |
}; |
| 133 |
} |
| 134 |
|
| 135 |
decl_cfl_pred_fn! { |
| 136 |
rav1e_ipred_cfl_8bpc_neon, |
| 137 |
rav1e_ipred_cfl_128_8bpc_neon, |
| 138 |
rav1e_ipred_cfl_left_8bpc_neon, |
| 139 |
rav1e_ipred_cfl_top_8bpc_neon |
| 140 |
} |
| 141 |
|
| 142 |
macro_rules! decl_cfl_pred_hbd_fn { |
| 143 |
($($f:ident),+) => { |
| 144 |
extern { |
| 145 |
$( |
| 146 |
fn $f( |
| 147 |
dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16, |
| 148 |
width: libc::c_int, height: libc::c_int, ac: *const i16, |
| 149 |
alpha: libc::c_int, bit_depth_max: libc::c_int, |
| 150 |
); |
| 151 |
)* |
| 152 |
} |
| 153 |
}; |
| 154 |
} |
| 155 |
|
| 156 |
decl_cfl_pred_hbd_fn! { |
| 157 |
rav1e_ipred_cfl_16bpc_neon, |
| 158 |
rav1e_ipred_cfl_128_16bpc_neon, |
| 159 |
rav1e_ipred_cfl_left_16bpc_neon, |
| 160 |
rav1e_ipred_cfl_top_16bpc_neon |
| 161 |
} |
| 162 |
|
| 163 |
extern { |
| 164 |
fn rav1e_ipred_z1_upsample_edge_8bpc_neon( |
| 165 |
out: *mut u8, hsz: c_int, _in: *const u8, end: c_int, |
| 166 |
); |
| 167 |
fn rav1e_ipred_z1_upsample_edge_16bpc_neon( |
| 168 |
out: *mut u16, hsz: c_int, _in: *const u16, end: c_int, |
| 169 |
bit_depth_max: c_int, |
| 170 |
); |
| 171 |
fn rav1e_ipred_z1_filter_edge_8bpc_neon( |
| 172 |
out: *mut u8, sz: c_int, _in: *const u8, end: c_int, strength: c_int, |
| 173 |
); |
| 174 |
fn rav1e_ipred_z1_filter_edge_16bpc_neon( |
| 175 |
out: *mut u16, sz: c_int, _in: *const u16, end: c_int, strength: c_int, |
| 176 |
); |
| 177 |
fn rav1e_ipred_z1_fill1_8bpc_neon( |
| 178 |
dst: *mut u8, stride: ptrdiff_t, top: *const u8, width: c_int, |
| 179 |
height: c_int, dx: c_int, max_base_x: c_int, |
| 180 |
); |
| 181 |
fn rav1e_ipred_z1_fill1_16bpc_neon( |
| 182 |
dst: *mut u16, stride: ptrdiff_t, top: *const u16, width: c_int, |
| 183 |
height: c_int, dx: c_int, max_base_x: c_int, |
| 184 |
); |
| 185 |
fn rav1e_ipred_z1_fill2_8bpc_neon( |
| 186 |
dst: *mut u8, stride: ptrdiff_t, top: *const u8, width: c_int, |
| 187 |
height: c_int, dx: c_int, max_base_x: c_int, |
| 188 |
); |
| 189 |
fn rav1e_ipred_z1_fill2_16bpc_neon( |
| 190 |
dst: *mut u16, stride: ptrdiff_t, top: *const u16, width: c_int, |
| 191 |
height: c_int, dx: c_int, max_base_x: c_int, |
| 192 |
); |
| 193 |
fn rav1e_ipred_z2_upsample_edge_8bpc_neon( |
| 194 |
out: *mut u8, sz: c_int, _in: *const u8, |
| 195 |
); |
| 196 |
fn rav1e_ipred_z2_upsample_edge_16bpc_neon( |
| 197 |
out: *mut u16, sz: c_int, _in: *const u16, bit_depth_max: c_int, |
| 198 |
); |
| 199 |
fn rav1e_ipred_z2_fill1_8bpc_neon( |
| 200 |
dst: *mut u8, stride: ptrdiff_t, top: *const u8, left: *const u8, |
| 201 |
width: c_int, height: c_int, dx: c_int, dy: c_int, |
| 202 |
); |
| 203 |
fn rav1e_ipred_z2_fill1_16bpc_neon( |
| 204 |
dst: *mut u16, stride: ptrdiff_t, top: *const u16, left: *const u16, |
| 205 |
width: c_int, height: c_int, dx: c_int, dy: c_int, |
| 206 |
); |
| 207 |
fn rav1e_ipred_z2_fill2_8bpc_neon( |
| 208 |
dst: *mut u8, stride: ptrdiff_t, top: *const u8, left: *const u8, |
| 209 |
width: c_int, height: c_int, dx: c_int, dy: c_int, |
| 210 |
); |
| 211 |
fn rav1e_ipred_z2_fill2_16bpc_neon( |
| 212 |
dst: *mut u16, stride: ptrdiff_t, top: *const u16, left: *const u16, |
| 213 |
width: c_int, height: c_int, dx: c_int, dy: c_int, |
| 214 |
); |
| 215 |
fn rav1e_ipred_z2_fill3_8bpc_neon( |
| 216 |
dst: *mut u8, stride: ptrdiff_t, top: *const u8, left: *const u8, |
| 217 |
width: c_int, height: c_int, dx: c_int, dy: c_int, |
| 218 |
); |
| 219 |
fn rav1e_ipred_z2_fill3_16bpc_neon( |
| 220 |
dst: *mut u16, stride: ptrdiff_t, top: *const u16, left: *const u16, |
| 221 |
width: c_int, height: c_int, dx: c_int, dy: c_int, |
| 222 |
); |
| 223 |
fn rav1e_ipred_z3_fill1_8bpc_neon( |
| 224 |
dst: *mut u8, stride: ptrdiff_t, left: *const u8, width: c_int, |
| 225 |
height: c_int, dy: c_int, max_base_y: c_int, |
| 226 |
); |
| 227 |
fn rav1e_ipred_z3_fill1_16bpc_neon( |
| 228 |
dst: *mut u16, stride: ptrdiff_t, left: *const u16, width: c_int, |
| 229 |
height: c_int, dy: c_int, max_base_y: c_int, |
| 230 |
); |
| 231 |
fn rav1e_ipred_z3_fill2_8bpc_neon( |
| 232 |
dst: *mut u8, stride: ptrdiff_t, left: *const u8, width: c_int, |
| 233 |
height: c_int, dy: c_int, max_base_y: c_int, |
| 234 |
); |
| 235 |
fn rav1e_ipred_z3_fill2_16bpc_neon( |
| 236 |
dst: *mut u16, stride: ptrdiff_t, left: *const u16, width: c_int, |
| 237 |
height: c_int, dy: c_int, max_base_y: c_int, |
| 238 |
); |
| 239 |
fn rav1e_ipred_reverse_8bpc_neon(dst: *mut u8, src: *const u8, n: c_int); |
| 240 |
fn rav1e_ipred_reverse_16bpc_neon(dst: *mut u16, src: *const u16, n: c_int); |
| 241 |
} |
| 242 |
#[inline] |
| 243 |
unsafe fn ipred_z1_upsample_edge<T: Pixel>( |
| 244 |
o: *mut T, sz: c_int, i: *const T, n: c_int, m: c_int, |
| 245 |
) { |
| 246 |
match T::type_enum() { |
| 247 |
U8 => rav1e_ipred_z1_upsample_edge_8bpc_neon(o as _, sz, i as _, n), |
| 248 |
U16 => rav1e_ipred_z1_upsample_edge_16bpc_neon(o as _, sz, i as _, n, m), |
| 249 |
} |
| 250 |
} |
| 251 |
#[inline] |
| 252 |
unsafe fn ipred_z2_upsample_edge<T: Pixel>( |
| 253 |
o: *mut T, sz: c_int, i: *const T, m: c_int, |
| 254 |
) { |
| 255 |
match T::type_enum() { |
| 256 |
U8 => rav1e_ipred_z2_upsample_edge_8bpc_neon(o as _, sz, i as _), |
| 257 |
U16 => rav1e_ipred_z2_upsample_edge_16bpc_neon(o as _, sz, i as _, m), |
| 258 |
} |
| 259 |
} |
| 260 |
#[inline] |
| 261 |
unsafe fn ipred_z1_filter_edge<T: Pixel>( |
| 262 |
o: *mut T, sz: c_int, i: *const T, n: c_int, s: c_int, |
| 263 |
) { |
| 264 |
match T::type_enum() { |
| 265 |
U8 => rav1e_ipred_z1_filter_edge_8bpc_neon(o as _, sz, i as _, n, s), |
| 266 |
U16 => rav1e_ipred_z1_filter_edge_16bpc_neon(o as _, sz, i as _, n, s), |
| 267 |
} |
| 268 |
} |
| 269 |
#[inline] |
| 270 |
unsafe fn ipred_reverse<T: Pixel>(o: *mut T, i: *const T, n: c_int) { |
| 271 |
match T::type_enum() { |
| 272 |
U8 => rav1e_ipred_reverse_8bpc_neon(o as _, i as _, n), |
| 273 |
U16 => rav1e_ipred_reverse_16bpc_neon(o as _, i as _, n), |
| 274 |
} |
| 275 |
} |
| 276 |
|
| 277 |
#[rustfmt::skip] |
| 278 |
struct Fill( |
| 279 |
[unsafe extern fn(*mut u8, ptrdiff_t, *const u8, c_int, c_int, c_int, c_int); 2], |
| 280 |
[unsafe extern fn(*mut u16, ptrdiff_t, *const u16, c_int, c_int, c_int, c_int); 2], |
| 281 |
); |
| 282 |
impl Fill { |
| 283 |
unsafe fn ipred_fill<T: Pixel>( |
| 284 |
self, dst: *mut T, stride: ptrdiff_t, src: *const T, w: c_int, h: c_int, |
| 285 |
d: c_int, max_base: c_int, upsample: bool, |
| 286 |
) { |
| 287 |
let u = upsample as usize; |
| 288 |
match T::type_enum() { |
| 289 |
U8 => self.0[u](dst as _, stride, src as _, w, h, d, max_base), |
| 290 |
U16 => self.1[u](dst as _, stride, src as _, w, h, d, max_base), |
| 291 |
} |
| 292 |
} |
| 293 |
} |
| 294 |
const Z1: Fill = Fill( |
| 295 |
[rav1e_ipred_z1_fill1_8bpc_neon, rav1e_ipred_z1_fill2_8bpc_neon], |
| 296 |
[rav1e_ipred_z1_fill1_16bpc_neon, rav1e_ipred_z1_fill2_16bpc_neon], |
| 297 |
); |
| 298 |
const Z3: Fill = Fill( |
| 299 |
[rav1e_ipred_z3_fill1_8bpc_neon, rav1e_ipred_z3_fill2_8bpc_neon], |
| 300 |
[rav1e_ipred_z3_fill1_16bpc_neon, rav1e_ipred_z3_fill2_16bpc_neon], |
| 301 |
); |
| 302 |
|
| 303 |
#[rustfmt::skip] |
| 304 |
struct Fill2( |
| 305 |
[unsafe extern fn(*mut u8, ptrdiff_t, *const u8, *const u8, c_int, c_int, c_int, c_int); 3], |
| 306 |
[unsafe extern fn(*mut u16, ptrdiff_t, *const u16, *const u16, c_int, c_int, c_int, c_int); 3], |
| 307 |
); |
| 308 |
impl Fill2 { |
| 309 |
unsafe fn ipred_fill<T: Pixel>( |
| 310 |
self, dst: *mut T, stride: ptrdiff_t, top: *const T, left: *const T, |
| 311 |
w: c_int, h: c_int, dx: c_int, dy: c_int, upsample_above: bool, |
| 312 |
upsample_left: bool, |
| 313 |
) { |
| 314 |
let u = if upsample_left { 2 } else { upsample_above as usize }; |
| 315 |
match T::type_enum() { |
| 316 |
U8 => self.0[u](dst as _, stride, top as _, left as _, w, h, dx, dy), |
| 317 |
U16 => self.1[u](dst as _, stride, top as _, left as _, w, h, dx, dy), |
| 318 |
} |
| 319 |
} |
| 320 |
} |
| 321 |
const Z2: Fill2 = Fill2( |
| 322 |
[ |
| 323 |
rav1e_ipred_z2_fill1_8bpc_neon, |
| 324 |
rav1e_ipred_z2_fill2_8bpc_neon, |
| 325 |
rav1e_ipred_z2_fill3_8bpc_neon, |
| 326 |
], |
| 327 |
[ |
| 328 |
rav1e_ipred_z2_fill1_16bpc_neon, |
| 329 |
rav1e_ipred_z2_fill2_16bpc_neon, |
| 330 |
rav1e_ipred_z2_fill3_16bpc_neon, |
| 331 |
], |
| 332 |
); |
| 333 |
|
| 334 |
unsafe fn ipred_z1<T: Pixel>( |
| 335 |
dst: *mut T, stride: ptrdiff_t, src: *const T, angle: isize, w: c_int, |
| 336 |
h: c_int, bd_max: c_int, edge_filter: bool, smooth_filter: bool, |
| 337 |
) { |
| 338 |
let mut dx = dr_intra_derivative(angle as _) as c_int; |
| 339 |
let mut out = [MaybeUninit::<T>::uninit(); MAX_TX_SIZE * 4 + 15 * 2 + 16]; |
| 340 |
let out = out.as_mut_ptr() as *mut T; |
| 341 |
|
| 342 |
let upsample_above = edge_filter |
| 343 |
&& select_ief_upsample(w as _, h as _, smooth_filter, 90 - angle); |
| 344 |
let max_base_x = if upsample_above { |
| 345 |
ipred_z1_upsample_edge(out, w + h, src, w + w.min(h), bd_max); |
| 346 |
dx <<= 1; |
| 347 |
2 * (w + h) - 2 |
| 348 |
} else { |
| 349 |
let strength = |
| 350 |
select_ief_strength(w as _, h as _, smooth_filter, 90 - angle) as c_int; |
| 351 |
if strength != 0 { |
| 352 |
ipred_z1_filter_edge(out, w + h, src, w + w.min(h), strength); |
| 353 |
w + h - 1 |
| 354 |
} else { |
| 355 |
let n = w + w.min(h); |
| 356 |
out.copy_from_nonoverlapping(src.add(1), n as usize); |
| 357 |
n - 1 |
| 358 |
} |
| 359 |
}; |
| 360 |
|
| 361 |
let base_inc = 1 + upsample_above as c_int; |
| 362 |
let pad_pixels = w + 15; |
| 363 |
let fill_pixel = out.add(max_base_x as usize).read(); |
| 364 |
let base = out.add(max_base_x as usize + 1); |
| 365 |
for i in 0..(pad_pixels * base_inc) as usize { |
| 366 |
base.add(i).write(fill_pixel); |
| 367 |
} |
| 368 |
|
| 369 |
Z1.ipred_fill(dst, stride, out, w, h, dx, max_base_x, upsample_above); |
| 370 |
} |
| 371 |
|
| 372 |
unsafe fn ipred_z2<T: Pixel>( |
| 373 |
dst: *mut T, stride: ptrdiff_t, src: *const T, angle: isize, w: c_int, |
| 374 |
h: c_int, max_w: c_int, max_h: c_int, bd_max: c_int, edge_filter: bool, |
| 375 |
smooth_filter: bool, |
| 376 |
) { |
| 377 |
assert!(angle > 90 && angle < 180); |
| 378 |
let mut dx = dr_intra_derivative((180 - angle) as _) as c_int; |
| 379 |
let mut dy = dr_intra_derivative((angle - 90) as _) as c_int; |
| 380 |
let us_left = edge_filter |
| 381 |
&& select_ief_upsample(w as _, h as _, smooth_filter, 180 - angle); |
| 382 |
let us_above = edge_filter |
| 383 |
&& select_ief_upsample(w as _, h as _, smooth_filter, angle - 90); |
| 384 |
let mut out = [MaybeUninit::<T>::uninit(); 3 * (MAX_TX_SIZE * 4 + 1)]; |
| 385 |
let out = out.as_mut_ptr() as *mut T; |
| 386 |
let left = out.add(2 * (64 + 1)); |
| 387 |
let top = out.add(64 + 1); |
| 388 |
let flipped = out; |
| 389 |
if us_above { |
| 390 |
ipred_z2_upsample_edge(top, w, src, bd_max); |
| 391 |
dx <<= 1; |
| 392 |
} else { |
| 393 |
let strength = |
| 394 |
select_ief_strength(w as _, h as _, smooth_filter, angle - 90) as c_int; |
| 395 |
|
| 396 |
if edge_filter && strength != 0 { |
| 397 |
ipred_z1_filter_edge(top.add(1), max_w.min(w), src, w, strength); |
| 398 |
if max_w < w { |
| 399 |
top.add((1 + max_w) as _).copy_from_nonoverlapping( |
| 400 |
src.add((1 + max_w) as _), |
| 401 |
(w - max_w) as _, |
| 402 |
); |
| 403 |
} |
| 404 |
} else { |
| 405 |
top.add(1).copy_from_nonoverlapping(src.add(1), w as _); |
| 406 |
} |
| 407 |
} |
| 408 |
if us_left { |
| 409 |
flipped.write(src.read()); |
| 410 |
ipred_reverse(flipped.add(1), src, h); |
| 411 |
ipred_z2_upsample_edge(left, h, flipped, bd_max); |
| 412 |
dy <<= 1; |
| 413 |
} else { |
| 414 |
let strength = |
| 415 |
select_ief_strength(w as _, h as _, smooth_filter, 180 - angle) as c_int; |
| 416 |
|
| 417 |
if edge_filter && strength != 0 { |
| 418 |
flipped.write(src.read()); |
| 419 |
ipred_reverse(flipped.add(1), src, h); |
| 420 |
ipred_z1_filter_edge(left.add(1), max_h.min(h), flipped, h, strength); |
| 421 |
if max_h < h { |
| 422 |
left.add((1 + max_h) as _).copy_from_nonoverlapping( |
| 423 |
flipped.add((1 + max_h) as _), |
| 424 |
(h - max_h) as _, |
| 425 |
); |
| 426 |
} |
| 427 |
} else { |
| 428 |
ipred_reverse(left.add(1), src, h); |
| 429 |
} |
| 430 |
} |
| 431 |
let top_left = src.read(); |
| 432 |
top.write(top_left); |
| 433 |
left.write(top_left); |
| 434 |
|
| 435 |
assert!(!(us_above && us_left)); |
| 436 |
Z2.ipred_fill(dst, stride, top, left, w, h, dx, dy, us_above, us_left); |
| 437 |
} |
| 438 |
|
| 439 |
unsafe fn ipred_z3<T: Pixel>( |
| 440 |
dst: *mut T, stride: ptrdiff_t, src: *const T, angle: isize, w: c_int, |
| 441 |
h: c_int, bd_max: c_int, edge_filter: bool, smooth_filter: bool, |
| 442 |
) { |
| 443 |
assert!(angle > 180); |
| 444 |
let mut dy = dr_intra_derivative(270 - angle as usize) as c_int; |
| 445 |
let mut tmp = [MaybeUninit::<T>::uninit(); MAX_TX_SIZE * 4 + 16]; |
| 446 |
let mut out = [MaybeUninit::<T>::uninit(); MAX_TX_SIZE * 8 + 15 * 2]; |
| 447 |
let out = out.as_mut_ptr() as *mut T; |
| 448 |
let tmp = tmp.as_mut_ptr() as *mut T; |
| 449 |
|
| 450 |
let upsample_left = edge_filter |
| 451 |
&& select_ief_upsample(w as _, h as _, smooth_filter, angle - 180); |
| 452 |
let max_base_y = if upsample_left { |
| 453 |
tmp.write(src.read()); |
| 454 |
ipred_reverse(tmp.add(1), src, h + w.max(h)); |
| 455 |
ipred_z1_upsample_edge(out, w + h, tmp, h + w.min(h), bd_max); |
| 456 |
dy <<= 1; |
| 457 |
2 * (w + h) - 2 |
| 458 |
} else { |
| 459 |
let strength = |
| 460 |
select_ief_strength(w as _, h as _, smooth_filter, angle - 180) as c_int; |
| 461 |
if strength != 0 { |
| 462 |
tmp.write(src.read()); |
| 463 |
ipred_reverse(tmp.add(1), src, h + w.max(h)); |
| 464 |
ipred_z1_filter_edge(out, w + h, tmp, h + w.min(h), strength); |
| 465 |
w + h - 1 |
| 466 |
} else { |
| 467 |
let n = w + w.min(h); |
| 468 |
ipred_reverse(out, src, n); |
| 469 |
n - 1 |
| 470 |
} |
| 471 |
}; |
| 472 |
|
| 473 |
let base_inc = 1 + upsample_left as c_int; |
| 474 |
let pad_pixels = (h + 15).max(64 - max_base_y - 1); |
| 475 |
let fill_pixel = out.add(max_base_y as usize).read(); |
| 476 |
let base = out.add(max_base_y as usize + 1); |
| 477 |
for i in 0..(pad_pixels * base_inc) as usize { |
| 478 |
base.add(i).write(fill_pixel); |
| 479 |
} |
| 480 |
|
| 481 |
Z3.ipred_fill(dst, stride, out, w, h, dy, max_base_y, upsample_left); |
| 482 |
} |
| 483 |
|
| 484 |
#[inline(always)] |
| 485 |
pub fn dispatch_predict_intra<T: Pixel>( |
| 486 |
mode: PredictionMode, variant: PredictionVariant, |
| 487 |
dst: &mut PlaneRegionMut<'_, T>, tx_size: TxSize, bit_depth: usize, |
| 488 |
ac: &[i16], angle: isize, ief_params: Option<IntraEdgeFilterParameters>, |
| 489 |
edge_buf: &IntraEdge<T>, cpu: CpuFeatureLevel, |
| 490 |
) { |
| 491 |
let call_rust = |dst: &mut PlaneRegionMut<'_, T>| { |
| 492 |
rust::dispatch_predict_intra( |
| 493 |
mode, variant, dst, tx_size, bit_depth, ac, angle, ief_params, edge_buf, |
| 494 |
cpu, |
| 495 |
); |
| 496 |
}; |
| 497 |
|
| 498 |
if cpu < CpuFeatureLevel::NEON { |
| 499 |
return call_rust(dst); |
| 500 |
} |
| 501 |
|
| 502 |
unsafe { |
| 503 |
let dst_ptr = dst.data_ptr_mut() as *mut _; |
| 504 |
let dst_u16 = dst.data_ptr_mut() as *mut u16; |
| 505 |
let stride = T::to_asm_stride(dst.plane_cfg.stride) as libc::ptrdiff_t; |
| 506 |
let edge_ptr = edge_buf.top_left_ptr() as *const _; |
| 507 |
let edge_u16 = edge_buf.top_left_ptr() as *const u16; |
| 508 |
let w = tx_size.width() as libc::c_int; |
| 509 |
let h = tx_size.height() as libc::c_int; |
| 510 |
let angle = angle as libc::c_int; |
| 511 |
let bd_max = (1 << bit_depth) - 1; |
| 512 |
match T::type_enum() { |
| 513 |
PixelType::U8 => match mode { |
| 514 |
PredictionMode::DC_PRED => { |
| 515 |
(match variant { |
| 516 |
PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_neon, |
| 517 |
PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_neon, |
| 518 |
PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_neon, |
| 519 |
PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_neon, |
| 520 |
})(dst_ptr, stride, edge_ptr, w, h, angle); |
| 521 |
} |
| 522 |
PredictionMode::V_PRED if angle == 90 => { |
| 523 |
rav1e_ipred_v_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle); |
| 524 |
} |
| 525 |
PredictionMode::H_PRED if angle == 180 => { |
| 526 |
rav1e_ipred_h_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle); |
| 527 |
} |
| 528 |
PredictionMode::SMOOTH_PRED => { |
| 529 |
rav1e_ipred_smooth_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle); |
| 530 |
} |
| 531 |
PredictionMode::SMOOTH_V_PRED => { |
| 532 |
rav1e_ipred_smooth_v_8bpc_neon( |
| 533 |
dst_ptr, stride, edge_ptr, w, h, angle, |
| 534 |
); |
| 535 |
} |
| 536 |
PredictionMode::SMOOTH_H_PRED => { |
| 537 |
rav1e_ipred_smooth_h_8bpc_neon( |
| 538 |
dst_ptr, stride, edge_ptr, w, h, angle, |
| 539 |
); |
| 540 |
} |
| 541 |
PredictionMode::PAETH_PRED => { |
| 542 |
rav1e_ipred_paeth_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle); |
| 543 |
} |
| 544 |
PredictionMode::UV_CFL_PRED => { |
| 545 |
let ac_ptr = ac.as_ptr() as *const _; |
| 546 |
(match variant { |
| 547 |
PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_neon, |
| 548 |
PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_neon, |
| 549 |
PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_neon, |
| 550 |
PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_neon, |
| 551 |
})(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle); |
| 552 |
} |
| 553 |
_ => call_rust(dst), |
| 554 |
}, |
| 555 |
PixelType::U16 if bit_depth > 8 => match mode { |
| 556 |
PredictionMode::DC_PRED => { |
| 557 |
(match variant { |
| 558 |
PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_neon, |
| 559 |
PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_neon, |
| 560 |
PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_neon, |
| 561 |
PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_neon, |
| 562 |
})(dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max); |
| 563 |
} |
| 564 |
PredictionMode::V_PRED if angle == 90 => { |
| 565 |
rav1e_ipred_v_16bpc_neon( |
| 566 |
dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max, |
| 567 |
); |
| 568 |
} |
| 569 |
PredictionMode::H_PRED if angle == 180 => { |
| 570 |
rav1e_ipred_h_16bpc_neon( |
| 571 |
dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max, |
| 572 |
); |
| 573 |
} |
| 574 |
PredictionMode::H_PRED |
| 575 |
| PredictionMode::V_PRED |
| 576 |
| PredictionMode::D45_PRED |
| 577 |
| PredictionMode::D135_PRED |
| 578 |
| PredictionMode::D113_PRED |
| 579 |
| PredictionMode::D157_PRED |
| 580 |
| PredictionMode::D203_PRED |
| 581 |
| PredictionMode::D67_PRED => { |
| 582 |
let edge_filter = ief_params.is_some(); |
| 583 |
let smooth_filter = ief_params |
| 584 |
.map(IntraEdgeFilterParameters::use_smooth_filter) |
| 585 |
.unwrap_or_default(); |
| 586 |
if (90..=180).contains(&angle) { |
| 587 |
// From dav1d, bw and bh are the frame width and height rounded to 8px units |
| 588 |
let (bw, bh) = ( |
| 589 |
((dst.plane_cfg.width + 7) >> 3) << 3, |
| 590 |
((dst.plane_cfg.height + 7) >> 3) << 3, |
| 591 |
); |
| 592 |
// From dav1d, dx and dy are the distance from the predicted block to the frame edge |
| 593 |
let (dx, dy) = ( |
| 594 |
(bw as isize - dst.rect().x) as libc::c_int, |
| 595 |
(bh as isize - dst.rect().y) as libc::c_int, |
| 596 |
); |
| 597 |
return ipred_z2( |
| 598 |
dst.data_ptr_mut(), |
| 599 |
stride, |
| 600 |
edge_buf.top_left_ptr(), |
| 601 |
angle as isize, |
| 602 |
w, |
| 603 |
h, |
| 604 |
dx, |
| 605 |
dy, |
| 606 |
bd_max, |
| 607 |
edge_filter, |
| 608 |
smooth_filter, |
| 609 |
); |
| 610 |
} |
| 611 |
(if angle < 90 { ipred_z1 } else { ipred_z3 })( |
| 612 |
dst.data_ptr_mut(), |
| 613 |
stride, |
| 614 |
edge_buf.top_left_ptr(), |
| 615 |
angle as isize, |
| 616 |
w, |
| 617 |
h, |
| 618 |
bd_max, |
| 619 |
edge_filter, |
| 620 |
smooth_filter, |
| 621 |
); |
| 622 |
} |
| 623 |
PredictionMode::SMOOTH_PRED => { |
| 624 |
rav1e_ipred_smooth_16bpc_neon( |
| 625 |
dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max, |
| 626 |
); |
| 627 |
} |
| 628 |
PredictionMode::SMOOTH_V_PRED => { |
| 629 |
rav1e_ipred_smooth_v_16bpc_neon( |
| 630 |
dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max, |
| 631 |
); |
| 632 |
} |
| 633 |
PredictionMode::SMOOTH_H_PRED => { |
| 634 |
rav1e_ipred_smooth_h_16bpc_neon( |
| 635 |
dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max, |
| 636 |
); |
| 637 |
} |
| 638 |
PredictionMode::PAETH_PRED => { |
| 639 |
rav1e_ipred_paeth_16bpc_neon( |
| 640 |
dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max, |
| 641 |
); |
| 642 |
} |
| 643 |
PredictionMode::UV_CFL_PRED => { |
| 644 |
let ac_ptr = ac.as_ptr() as *const _; |
| 645 |
(match variant { |
| 646 |
PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_neon, |
| 647 |
PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_neon, |
| 648 |
PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_neon, |
| 649 |
PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_neon, |
| 650 |
})(dst_u16, stride, edge_u16, w, h, ac_ptr, angle, bd_max); |
| 651 |
} |
| 652 |
_ => call_rust(dst), |
| 653 |
}, |
| 654 |
_ => call_rust(dst), |
| 655 |
} |
| 656 |
} |
| 657 |
} |
| 658 |
|
| 659 |
/// It MUST initialize all `ac` elements. |
| 660 |
#[inline(always)] |
| 661 |
pub(crate) fn pred_cfl_ac<T: Pixel, const XDEC: usize, const YDEC: usize>( |
| 662 |
ac: &mut [MaybeUninit<i16>], luma: &PlaneRegion<'_, T>, bsize: BlockSize, |
| 663 |
w_pad: usize, h_pad: usize, cpu: CpuFeatureLevel, |
| 664 |
) { |
| 665 |
debug_assert_eq!(ac.len(), bsize.area()); |
| 666 |
|
| 667 |
if cpu < CpuFeatureLevel::NEON { |
| 668 |
return rust::pred_cfl_ac::<T, XDEC, YDEC>( |
| 669 |
ac, luma, bsize, w_pad, h_pad, cpu, |
| 670 |
); |
| 671 |
} |
| 672 |
|
| 673 |
let stride = T::to_asm_stride(luma.plane_cfg.stride) as libc::ptrdiff_t; |
| 674 |
let w = bsize.width() as libc::c_int; |
| 675 |
let h = bsize.height() as libc::c_int; |
| 676 |
let w_pad = w_pad as libc::c_int; |
| 677 |
let h_pad = h_pad as libc::c_int; |
| 678 |
|
| 679 |
// SAFETY: Calls Assembly code. |
| 680 |
unsafe { |
| 681 |
let ac_ptr = ac.as_mut_ptr(); |
| 682 |
match T::type_enum() { |
| 683 |
PixelType::U8 => { |
| 684 |
let luma_ptr = luma.data_ptr() as *const u8; |
| 685 |
(match (XDEC, YDEC) { |
| 686 |
(0, 0) => rav1e_ipred_cfl_ac_444_8bpc_neon, |
| 687 |
(1, 0) => rav1e_ipred_cfl_ac_422_8bpc_neon, |
| 688 |
_ => rav1e_ipred_cfl_ac_420_8bpc_neon, |
| 689 |
})(ac_ptr, luma_ptr, stride, w_pad, h_pad, w, h) |
| 690 |
} |
| 691 |
PixelType::U16 => { |
| 692 |
let luma_ptr = luma.data_ptr() as *const u16; |
| 693 |
(match (XDEC, YDEC) { |
| 694 |
(0, 0) => rav1e_ipred_cfl_ac_444_16bpc_neon, |
| 695 |
(1, 0) => rav1e_ipred_cfl_ac_422_16bpc_neon, |
| 696 |
_ => rav1e_ipred_cfl_ac_420_16bpc_neon, |
| 697 |
})(ac_ptr, luma_ptr, stride, w_pad, h_pad, w, h) |
| 698 |
} |
| 699 |
} |
| 700 |
} |
| 701 |
} |