Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/predict.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::context::MAX_TX_SIZE;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::partition::{BlockSize, IntraEdge};
13 use crate::predict::rust::{
14 dr_intra_derivative, select_ief_strength, select_ief_upsample,
15 };
16 use crate::predict::{
17 rust, IntraEdgeFilterParameters, PredictionMode, PredictionVariant,
18 };
19 use crate::tiling::{PlaneRegion, PlaneRegionMut};
20 use crate::transform::TxSize;
21 use crate::{Pixel, PixelType};
22 use libc;
23 use libc::{c_int, ptrdiff_t};
24 use std::mem::MaybeUninit;
25 use PixelType::{U16, U8};
26
27 macro_rules! decl_cfl_ac_fn {
28 ($($f:ident),+) => {
29 extern {
30 $(
31 fn $f(
32 ac: *mut MaybeUninit<i16>, src: *const u8, stride: libc::ptrdiff_t,
33 w_pad: libc::c_int, h_pad: libc::c_int,
34 width: libc::c_int, height: libc::c_int,
35 );
36 )*
37 }
38 };
39 }
40
41 decl_cfl_ac_fn! {
42 rav1e_ipred_cfl_ac_420_8bpc_neon,
43 rav1e_ipred_cfl_ac_422_8bpc_neon,
44 rav1e_ipred_cfl_ac_444_8bpc_neon
45 }
46
47 macro_rules! decl_cfl_ac_hbd_fn {
48 ($($f:ident),+) => {
49 extern {
50 $(
51 fn $f(
52 ac: *mut MaybeUninit<i16>, src: *const u16, stride: libc::ptrdiff_t,
53 w_pad: libc::c_int, h_pad: libc::c_int,
54 width: libc::c_int, height: libc::c_int,
55 );
56 )*
57 }
58 };
59 }
60
61 decl_cfl_ac_hbd_fn! {
62 rav1e_ipred_cfl_ac_420_16bpc_neon,
63 rav1e_ipred_cfl_ac_422_16bpc_neon,
64 rav1e_ipred_cfl_ac_444_16bpc_neon
65 }
66
67 macro_rules! decl_angular_ipred_fn {
68 ($($f:ident),+) => {
69 extern {
70 $(
71 fn $f(
72 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
73 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
74 );
75 )*
76 }
77 };
78 }
79
80 decl_angular_ipred_fn! {
81 rav1e_ipred_dc_8bpc_neon,
82 rav1e_ipred_dc_128_8bpc_neon,
83 rav1e_ipred_dc_left_8bpc_neon,
84 rav1e_ipred_dc_top_8bpc_neon,
85 rav1e_ipred_v_8bpc_neon,
86 rav1e_ipred_h_8bpc_neon,
87 rav1e_ipred_smooth_8bpc_neon,
88 rav1e_ipred_smooth_v_8bpc_neon,
89 rav1e_ipred_smooth_h_8bpc_neon,
90 rav1e_ipred_paeth_8bpc_neon
91 }
92
93 macro_rules! decl_angular_ipred_hbd_fn {
94 ($($f:ident),+) => {
95 extern {
96 $(
97 fn $f(
98 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
99 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
100 max_width: libc::c_int, max_height: libc::c_int,
101 bit_depth_max: libc::c_int,
102 );
103 )*
104 }
105 };
106 }
107
108 decl_angular_ipred_hbd_fn! {
109 rav1e_ipred_dc_16bpc_neon,
110 rav1e_ipred_dc_128_16bpc_neon,
111 rav1e_ipred_dc_left_16bpc_neon,
112 rav1e_ipred_dc_top_16bpc_neon,
113 rav1e_ipred_v_16bpc_neon,
114 rav1e_ipred_h_16bpc_neon,
115 rav1e_ipred_smooth_16bpc_neon,
116 rav1e_ipred_smooth_v_16bpc_neon,
117 rav1e_ipred_smooth_h_16bpc_neon,
118 rav1e_ipred_paeth_16bpc_neon
119 }
120
121 macro_rules! decl_cfl_pred_fn {
122 ($($f:ident),+) => {
123 extern {
124 $(
125 fn $f(
126 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
127 width: libc::c_int, height: libc::c_int, ac: *const i16,
128 alpha: libc::c_int,
129 );
130 )*
131 }
132 };
133 }
134
135 decl_cfl_pred_fn! {
136 rav1e_ipred_cfl_8bpc_neon,
137 rav1e_ipred_cfl_128_8bpc_neon,
138 rav1e_ipred_cfl_left_8bpc_neon,
139 rav1e_ipred_cfl_top_8bpc_neon
140 }
141
142 macro_rules! decl_cfl_pred_hbd_fn {
143 ($($f:ident),+) => {
144 extern {
145 $(
146 fn $f(
147 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
148 width: libc::c_int, height: libc::c_int, ac: *const i16,
149 alpha: libc::c_int, bit_depth_max: libc::c_int,
150 );
151 )*
152 }
153 };
154 }
155
156 decl_cfl_pred_hbd_fn! {
157 rav1e_ipred_cfl_16bpc_neon,
158 rav1e_ipred_cfl_128_16bpc_neon,
159 rav1e_ipred_cfl_left_16bpc_neon,
160 rav1e_ipred_cfl_top_16bpc_neon
161 }
162
163 extern {
164 fn rav1e_ipred_z1_upsample_edge_8bpc_neon(
165 out: *mut u8, hsz: c_int, _in: *const u8, end: c_int,
166 );
167 fn rav1e_ipred_z1_upsample_edge_16bpc_neon(
168 out: *mut u16, hsz: c_int, _in: *const u16, end: c_int,
169 bit_depth_max: c_int,
170 );
171 fn rav1e_ipred_z1_filter_edge_8bpc_neon(
172 out: *mut u8, sz: c_int, _in: *const u8, end: c_int, strength: c_int,
173 );
174 fn rav1e_ipred_z1_filter_edge_16bpc_neon(
175 out: *mut u16, sz: c_int, _in: *const u16, end: c_int, strength: c_int,
176 );
177 fn rav1e_ipred_z1_fill1_8bpc_neon(
178 dst: *mut u8, stride: ptrdiff_t, top: *const u8, width: c_int,
179 height: c_int, dx: c_int, max_base_x: c_int,
180 );
181 fn rav1e_ipred_z1_fill1_16bpc_neon(
182 dst: *mut u16, stride: ptrdiff_t, top: *const u16, width: c_int,
183 height: c_int, dx: c_int, max_base_x: c_int,
184 );
185 fn rav1e_ipred_z1_fill2_8bpc_neon(
186 dst: *mut u8, stride: ptrdiff_t, top: *const u8, width: c_int,
187 height: c_int, dx: c_int, max_base_x: c_int,
188 );
189 fn rav1e_ipred_z1_fill2_16bpc_neon(
190 dst: *mut u16, stride: ptrdiff_t, top: *const u16, width: c_int,
191 height: c_int, dx: c_int, max_base_x: c_int,
192 );
193 fn rav1e_ipred_z2_upsample_edge_8bpc_neon(
194 out: *mut u8, sz: c_int, _in: *const u8,
195 );
196 fn rav1e_ipred_z2_upsample_edge_16bpc_neon(
197 out: *mut u16, sz: c_int, _in: *const u16, bit_depth_max: c_int,
198 );
199 fn rav1e_ipred_z2_fill1_8bpc_neon(
200 dst: *mut u8, stride: ptrdiff_t, top: *const u8, left: *const u8,
201 width: c_int, height: c_int, dx: c_int, dy: c_int,
202 );
203 fn rav1e_ipred_z2_fill1_16bpc_neon(
204 dst: *mut u16, stride: ptrdiff_t, top: *const u16, left: *const u16,
205 width: c_int, height: c_int, dx: c_int, dy: c_int,
206 );
207 fn rav1e_ipred_z2_fill2_8bpc_neon(
208 dst: *mut u8, stride: ptrdiff_t, top: *const u8, left: *const u8,
209 width: c_int, height: c_int, dx: c_int, dy: c_int,
210 );
211 fn rav1e_ipred_z2_fill2_16bpc_neon(
212 dst: *mut u16, stride: ptrdiff_t, top: *const u16, left: *const u16,
213 width: c_int, height: c_int, dx: c_int, dy: c_int,
214 );
215 fn rav1e_ipred_z2_fill3_8bpc_neon(
216 dst: *mut u8, stride: ptrdiff_t, top: *const u8, left: *const u8,
217 width: c_int, height: c_int, dx: c_int, dy: c_int,
218 );
219 fn rav1e_ipred_z2_fill3_16bpc_neon(
220 dst: *mut u16, stride: ptrdiff_t, top: *const u16, left: *const u16,
221 width: c_int, height: c_int, dx: c_int, dy: c_int,
222 );
223 fn rav1e_ipred_z3_fill1_8bpc_neon(
224 dst: *mut u8, stride: ptrdiff_t, left: *const u8, width: c_int,
225 height: c_int, dy: c_int, max_base_y: c_int,
226 );
227 fn rav1e_ipred_z3_fill1_16bpc_neon(
228 dst: *mut u16, stride: ptrdiff_t, left: *const u16, width: c_int,
229 height: c_int, dy: c_int, max_base_y: c_int,
230 );
231 fn rav1e_ipred_z3_fill2_8bpc_neon(
232 dst: *mut u8, stride: ptrdiff_t, left: *const u8, width: c_int,
233 height: c_int, dy: c_int, max_base_y: c_int,
234 );
235 fn rav1e_ipred_z3_fill2_16bpc_neon(
236 dst: *mut u16, stride: ptrdiff_t, left: *const u16, width: c_int,
237 height: c_int, dy: c_int, max_base_y: c_int,
238 );
239 fn rav1e_ipred_reverse_8bpc_neon(dst: *mut u8, src: *const u8, n: c_int);
240 fn rav1e_ipred_reverse_16bpc_neon(dst: *mut u16, src: *const u16, n: c_int);
241 }
242 #[inline]
243 unsafe fn ipred_z1_upsample_edge<T: Pixel>(
244 o: *mut T, sz: c_int, i: *const T, n: c_int, m: c_int,
245 ) {
246 match T::type_enum() {
247 U8 => rav1e_ipred_z1_upsample_edge_8bpc_neon(o as _, sz, i as _, n),
248 U16 => rav1e_ipred_z1_upsample_edge_16bpc_neon(o as _, sz, i as _, n, m),
249 }
250 }
251 #[inline]
252 unsafe fn ipred_z2_upsample_edge<T: Pixel>(
253 o: *mut T, sz: c_int, i: *const T, m: c_int,
254 ) {
255 match T::type_enum() {
256 U8 => rav1e_ipred_z2_upsample_edge_8bpc_neon(o as _, sz, i as _),
257 U16 => rav1e_ipred_z2_upsample_edge_16bpc_neon(o as _, sz, i as _, m),
258 }
259 }
260 #[inline]
261 unsafe fn ipred_z1_filter_edge<T: Pixel>(
262 o: *mut T, sz: c_int, i: *const T, n: c_int, s: c_int,
263 ) {
264 match T::type_enum() {
265 U8 => rav1e_ipred_z1_filter_edge_8bpc_neon(o as _, sz, i as _, n, s),
266 U16 => rav1e_ipred_z1_filter_edge_16bpc_neon(o as _, sz, i as _, n, s),
267 }
268 }
269 #[inline]
270 unsafe fn ipred_reverse<T: Pixel>(o: *mut T, i: *const T, n: c_int) {
271 match T::type_enum() {
272 U8 => rav1e_ipred_reverse_8bpc_neon(o as _, i as _, n),
273 U16 => rav1e_ipred_reverse_16bpc_neon(o as _, i as _, n),
274 }
275 }
276
277 #[rustfmt::skip]
278 struct Fill(
279 [unsafe extern fn(*mut u8, ptrdiff_t, *const u8, c_int, c_int, c_int, c_int); 2],
280 [unsafe extern fn(*mut u16, ptrdiff_t, *const u16, c_int, c_int, c_int, c_int); 2],
281 );
282 impl Fill {
283 unsafe fn ipred_fill<T: Pixel>(
284 self, dst: *mut T, stride: ptrdiff_t, src: *const T, w: c_int, h: c_int,
285 d: c_int, max_base: c_int, upsample: bool,
286 ) {
287 let u = upsample as usize;
288 match T::type_enum() {
289 U8 => self.0[u](dst as _, stride, src as _, w, h, d, max_base),
290 U16 => self.1[u](dst as _, stride, src as _, w, h, d, max_base),
291 }
292 }
293 }
294 const Z1: Fill = Fill(
295 [rav1e_ipred_z1_fill1_8bpc_neon, rav1e_ipred_z1_fill2_8bpc_neon],
296 [rav1e_ipred_z1_fill1_16bpc_neon, rav1e_ipred_z1_fill2_16bpc_neon],
297 );
298 const Z3: Fill = Fill(
299 [rav1e_ipred_z3_fill1_8bpc_neon, rav1e_ipred_z3_fill2_8bpc_neon],
300 [rav1e_ipred_z3_fill1_16bpc_neon, rav1e_ipred_z3_fill2_16bpc_neon],
301 );
302
303 #[rustfmt::skip]
304 struct Fill2(
305 [unsafe extern fn(*mut u8, ptrdiff_t, *const u8, *const u8, c_int, c_int, c_int, c_int); 3],
306 [unsafe extern fn(*mut u16, ptrdiff_t, *const u16, *const u16, c_int, c_int, c_int, c_int); 3],
307 );
308 impl Fill2 {
309 unsafe fn ipred_fill<T: Pixel>(
310 self, dst: *mut T, stride: ptrdiff_t, top: *const T, left: *const T,
311 w: c_int, h: c_int, dx: c_int, dy: c_int, upsample_above: bool,
312 upsample_left: bool,
313 ) {
314 let u = if upsample_left { 2 } else { upsample_above as usize };
315 match T::type_enum() {
316 U8 => self.0[u](dst as _, stride, top as _, left as _, w, h, dx, dy),
317 U16 => self.1[u](dst as _, stride, top as _, left as _, w, h, dx, dy),
318 }
319 }
320 }
321 const Z2: Fill2 = Fill2(
322 [
323 rav1e_ipred_z2_fill1_8bpc_neon,
324 rav1e_ipred_z2_fill2_8bpc_neon,
325 rav1e_ipred_z2_fill3_8bpc_neon,
326 ],
327 [
328 rav1e_ipred_z2_fill1_16bpc_neon,
329 rav1e_ipred_z2_fill2_16bpc_neon,
330 rav1e_ipred_z2_fill3_16bpc_neon,
331 ],
332 );
333
334 unsafe fn ipred_z1<T: Pixel>(
335 dst: *mut T, stride: ptrdiff_t, src: *const T, angle: isize, w: c_int,
336 h: c_int, bd_max: c_int, edge_filter: bool, smooth_filter: bool,
337 ) {
338 let mut dx = dr_intra_derivative(angle as _) as c_int;
339 let mut out = [MaybeUninit::<T>::uninit(); MAX_TX_SIZE * 4 + 15 * 2 + 16];
340 let out = out.as_mut_ptr() as *mut T;
341
342 let upsample_above = edge_filter
343 && select_ief_upsample(w as _, h as _, smooth_filter, 90 - angle);
344 let max_base_x = if upsample_above {
345 ipred_z1_upsample_edge(out, w + h, src, w + w.min(h), bd_max);
346 dx <<= 1;
347 2 * (w + h) - 2
348 } else {
349 let strength =
350 select_ief_strength(w as _, h as _, smooth_filter, 90 - angle) as c_int;
351 if strength != 0 {
352 ipred_z1_filter_edge(out, w + h, src, w + w.min(h), strength);
353 w + h - 1
354 } else {
355 let n = w + w.min(h);
356 out.copy_from_nonoverlapping(src.add(1), n as usize);
357 n - 1
358 }
359 };
360
361 let base_inc = 1 + upsample_above as c_int;
362 let pad_pixels = w + 15;
363 let fill_pixel = out.add(max_base_x as usize).read();
364 let base = out.add(max_base_x as usize + 1);
365 for i in 0..(pad_pixels * base_inc) as usize {
366 base.add(i).write(fill_pixel);
367 }
368
369 Z1.ipred_fill(dst, stride, out, w, h, dx, max_base_x, upsample_above);
370 }
371
372 unsafe fn ipred_z2<T: Pixel>(
373 dst: *mut T, stride: ptrdiff_t, src: *const T, angle: isize, w: c_int,
374 h: c_int, max_w: c_int, max_h: c_int, bd_max: c_int, edge_filter: bool,
375 smooth_filter: bool,
376 ) {
377 assert!(angle > 90 && angle < 180);
378 let mut dx = dr_intra_derivative((180 - angle) as _) as c_int;
379 let mut dy = dr_intra_derivative((angle - 90) as _) as c_int;
380 let us_left = edge_filter
381 && select_ief_upsample(w as _, h as _, smooth_filter, 180 - angle);
382 let us_above = edge_filter
383 && select_ief_upsample(w as _, h as _, smooth_filter, angle - 90);
384 let mut out = [MaybeUninit::<T>::uninit(); 3 * (MAX_TX_SIZE * 4 + 1)];
385 let out = out.as_mut_ptr() as *mut T;
386 let left = out.add(2 * (64 + 1));
387 let top = out.add(64 + 1);
388 let flipped = out;
389 if us_above {
390 ipred_z2_upsample_edge(top, w, src, bd_max);
391 dx <<= 1;
392 } else {
393 let strength =
394 select_ief_strength(w as _, h as _, smooth_filter, angle - 90) as c_int;
395
396 if edge_filter && strength != 0 {
397 ipred_z1_filter_edge(top.add(1), max_w.min(w), src, w, strength);
398 if max_w < w {
399 top.add((1 + max_w) as _).copy_from_nonoverlapping(
400 src.add((1 + max_w) as _),
401 (w - max_w) as _,
402 );
403 }
404 } else {
405 top.add(1).copy_from_nonoverlapping(src.add(1), w as _);
406 }
407 }
408 if us_left {
409 flipped.write(src.read());
410 ipred_reverse(flipped.add(1), src, h);
411 ipred_z2_upsample_edge(left, h, flipped, bd_max);
412 dy <<= 1;
413 } else {
414 let strength =
415 select_ief_strength(w as _, h as _, smooth_filter, 180 - angle) as c_int;
416
417 if edge_filter && strength != 0 {
418 flipped.write(src.read());
419 ipred_reverse(flipped.add(1), src, h);
420 ipred_z1_filter_edge(left.add(1), max_h.min(h), flipped, h, strength);
421 if max_h < h {
422 left.add((1 + max_h) as _).copy_from_nonoverlapping(
423 flipped.add((1 + max_h) as _),
424 (h - max_h) as _,
425 );
426 }
427 } else {
428 ipred_reverse(left.add(1), src, h);
429 }
430 }
431 let top_left = src.read();
432 top.write(top_left);
433 left.write(top_left);
434
435 assert!(!(us_above && us_left));
436 Z2.ipred_fill(dst, stride, top, left, w, h, dx, dy, us_above, us_left);
437 }
438
439 unsafe fn ipred_z3<T: Pixel>(
440 dst: *mut T, stride: ptrdiff_t, src: *const T, angle: isize, w: c_int,
441 h: c_int, bd_max: c_int, edge_filter: bool, smooth_filter: bool,
442 ) {
443 assert!(angle > 180);
444 let mut dy = dr_intra_derivative(270 - angle as usize) as c_int;
445 let mut tmp = [MaybeUninit::<T>::uninit(); MAX_TX_SIZE * 4 + 16];
446 let mut out = [MaybeUninit::<T>::uninit(); MAX_TX_SIZE * 8 + 15 * 2];
447 let out = out.as_mut_ptr() as *mut T;
448 let tmp = tmp.as_mut_ptr() as *mut T;
449
450 let upsample_left = edge_filter
451 && select_ief_upsample(w as _, h as _, smooth_filter, angle - 180);
452 let max_base_y = if upsample_left {
453 tmp.write(src.read());
454 ipred_reverse(tmp.add(1), src, h + w.max(h));
455 ipred_z1_upsample_edge(out, w + h, tmp, h + w.min(h), bd_max);
456 dy <<= 1;
457 2 * (w + h) - 2
458 } else {
459 let strength =
460 select_ief_strength(w as _, h as _, smooth_filter, angle - 180) as c_int;
461 if strength != 0 {
462 tmp.write(src.read());
463 ipred_reverse(tmp.add(1), src, h + w.max(h));
464 ipred_z1_filter_edge(out, w + h, tmp, h + w.min(h), strength);
465 w + h - 1
466 } else {
467 let n = w + w.min(h);
468 ipred_reverse(out, src, n);
469 n - 1
470 }
471 };
472
473 let base_inc = 1 + upsample_left as c_int;
474 let pad_pixels = (h + 15).max(64 - max_base_y - 1);
475 let fill_pixel = out.add(max_base_y as usize).read();
476 let base = out.add(max_base_y as usize + 1);
477 for i in 0..(pad_pixels * base_inc) as usize {
478 base.add(i).write(fill_pixel);
479 }
480
481 Z3.ipred_fill(dst, stride, out, w, h, dy, max_base_y, upsample_left);
482 }
483
484 #[inline(always)]
485 pub fn dispatch_predict_intra<T: Pixel>(
486 mode: PredictionMode, variant: PredictionVariant,
487 dst: &mut PlaneRegionMut<'_, T>, tx_size: TxSize, bit_depth: usize,
488 ac: &[i16], angle: isize, ief_params: Option<IntraEdgeFilterParameters>,
489 edge_buf: &IntraEdge<T>, cpu: CpuFeatureLevel,
490 ) {
491 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
492 rust::dispatch_predict_intra(
493 mode, variant, dst, tx_size, bit_depth, ac, angle, ief_params, edge_buf,
494 cpu,
495 );
496 };
497
498 if cpu < CpuFeatureLevel::NEON {
499 return call_rust(dst);
500 }
501
502 unsafe {
503 let dst_ptr = dst.data_ptr_mut() as *mut _;
504 let dst_u16 = dst.data_ptr_mut() as *mut u16;
505 let stride = T::to_asm_stride(dst.plane_cfg.stride) as libc::ptrdiff_t;
506 let edge_ptr = edge_buf.top_left_ptr() as *const _;
507 let edge_u16 = edge_buf.top_left_ptr() as *const u16;
508 let w = tx_size.width() as libc::c_int;
509 let h = tx_size.height() as libc::c_int;
510 let angle = angle as libc::c_int;
511 let bd_max = (1 << bit_depth) - 1;
512 match T::type_enum() {
513 PixelType::U8 => match mode {
514 PredictionMode::DC_PRED => {
515 (match variant {
516 PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_neon,
517 PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_neon,
518 PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_neon,
519 PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_neon,
520 })(dst_ptr, stride, edge_ptr, w, h, angle);
521 }
522 PredictionMode::V_PRED if angle == 90 => {
523 rav1e_ipred_v_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
524 }
525 PredictionMode::H_PRED if angle == 180 => {
526 rav1e_ipred_h_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
527 }
528 PredictionMode::SMOOTH_PRED => {
529 rav1e_ipred_smooth_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
530 }
531 PredictionMode::SMOOTH_V_PRED => {
532 rav1e_ipred_smooth_v_8bpc_neon(
533 dst_ptr, stride, edge_ptr, w, h, angle,
534 );
535 }
536 PredictionMode::SMOOTH_H_PRED => {
537 rav1e_ipred_smooth_h_8bpc_neon(
538 dst_ptr, stride, edge_ptr, w, h, angle,
539 );
540 }
541 PredictionMode::PAETH_PRED => {
542 rav1e_ipred_paeth_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
543 }
544 PredictionMode::UV_CFL_PRED => {
545 let ac_ptr = ac.as_ptr() as *const _;
546 (match variant {
547 PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_neon,
548 PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_neon,
549 PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_neon,
550 PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_neon,
551 })(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
552 }
553 _ => call_rust(dst),
554 },
555 PixelType::U16 if bit_depth > 8 => match mode {
556 PredictionMode::DC_PRED => {
557 (match variant {
558 PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_neon,
559 PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_neon,
560 PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_neon,
561 PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_neon,
562 })(dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max);
563 }
564 PredictionMode::V_PRED if angle == 90 => {
565 rav1e_ipred_v_16bpc_neon(
566 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
567 );
568 }
569 PredictionMode::H_PRED if angle == 180 => {
570 rav1e_ipred_h_16bpc_neon(
571 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
572 );
573 }
574 PredictionMode::H_PRED
575 | PredictionMode::V_PRED
576 | PredictionMode::D45_PRED
577 | PredictionMode::D135_PRED
578 | PredictionMode::D113_PRED
579 | PredictionMode::D157_PRED
580 | PredictionMode::D203_PRED
581 | PredictionMode::D67_PRED => {
582 let edge_filter = ief_params.is_some();
583 let smooth_filter = ief_params
584 .map(IntraEdgeFilterParameters::use_smooth_filter)
585 .unwrap_or_default();
586 if (90..=180).contains(&angle) {
587 // From dav1d, bw and bh are the frame width and height rounded to 8px units
588 let (bw, bh) = (
589 ((dst.plane_cfg.width + 7) >> 3) << 3,
590 ((dst.plane_cfg.height + 7) >> 3) << 3,
591 );
592 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
593 let (dx, dy) = (
594 (bw as isize - dst.rect().x) as libc::c_int,
595 (bh as isize - dst.rect().y) as libc::c_int,
596 );
597 return ipred_z2(
598 dst.data_ptr_mut(),
599 stride,
600 edge_buf.top_left_ptr(),
601 angle as isize,
602 w,
603 h,
604 dx,
605 dy,
606 bd_max,
607 edge_filter,
608 smooth_filter,
609 );
610 }
611 (if angle < 90 { ipred_z1 } else { ipred_z3 })(
612 dst.data_ptr_mut(),
613 stride,
614 edge_buf.top_left_ptr(),
615 angle as isize,
616 w,
617 h,
618 bd_max,
619 edge_filter,
620 smooth_filter,
621 );
622 }
623 PredictionMode::SMOOTH_PRED => {
624 rav1e_ipred_smooth_16bpc_neon(
625 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
626 );
627 }
628 PredictionMode::SMOOTH_V_PRED => {
629 rav1e_ipred_smooth_v_16bpc_neon(
630 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
631 );
632 }
633 PredictionMode::SMOOTH_H_PRED => {
634 rav1e_ipred_smooth_h_16bpc_neon(
635 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
636 );
637 }
638 PredictionMode::PAETH_PRED => {
639 rav1e_ipred_paeth_16bpc_neon(
640 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
641 );
642 }
643 PredictionMode::UV_CFL_PRED => {
644 let ac_ptr = ac.as_ptr() as *const _;
645 (match variant {
646 PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_neon,
647 PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_neon,
648 PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_neon,
649 PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_neon,
650 })(dst_u16, stride, edge_u16, w, h, ac_ptr, angle, bd_max);
651 }
652 _ => call_rust(dst),
653 },
654 _ => call_rust(dst),
655 }
656 }
657 }
658
659 /// It MUST initialize all `ac` elements.
660 #[inline(always)]
661 pub(crate) fn pred_cfl_ac<T: Pixel, const XDEC: usize, const YDEC: usize>(
662 ac: &mut [MaybeUninit<i16>], luma: &PlaneRegion<'_, T>, bsize: BlockSize,
663 w_pad: usize, h_pad: usize, cpu: CpuFeatureLevel,
664 ) {
665 debug_assert_eq!(ac.len(), bsize.area());
666
667 if cpu < CpuFeatureLevel::NEON {
668 return rust::pred_cfl_ac::<T, XDEC, YDEC>(
669 ac, luma, bsize, w_pad, h_pad, cpu,
670 );
671 }
672
673 let stride = T::to_asm_stride(luma.plane_cfg.stride) as libc::ptrdiff_t;
674 let w = bsize.width() as libc::c_int;
675 let h = bsize.height() as libc::c_int;
676 let w_pad = w_pad as libc::c_int;
677 let h_pad = h_pad as libc::c_int;
678
679 // SAFETY: Calls Assembly code.
680 unsafe {
681 let ac_ptr = ac.as_mut_ptr();
682 match T::type_enum() {
683 PixelType::U8 => {
684 let luma_ptr = luma.data_ptr() as *const u8;
685 (match (XDEC, YDEC) {
686 (0, 0) => rav1e_ipred_cfl_ac_444_8bpc_neon,
687 (1, 0) => rav1e_ipred_cfl_ac_422_8bpc_neon,
688 _ => rav1e_ipred_cfl_ac_420_8bpc_neon,
689 })(ac_ptr, luma_ptr, stride, w_pad, h_pad, w, h)
690 }
691 PixelType::U16 => {
692 let luma_ptr = luma.data_ptr() as *const u16;
693 (match (XDEC, YDEC) {
694 (0, 0) => rav1e_ipred_cfl_ac_444_16bpc_neon,
695 (1, 0) => rav1e_ipred_cfl_ac_422_16bpc_neon,
696 _ => rav1e_ipred_cfl_ac_420_16bpc_neon,
697 })(ac_ptr, luma_ptr, stride, w_pad, h_pad, w, h)
698 }
699 }
700 }
701 }