Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/predict.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::partition::{BlockSize, IntraEdge};
12 use crate::predict::{
13 rust, IntraEdgeFilterParameters, PredictionMode, PredictionVariant,
14 };
15 use crate::tiling::{PlaneRegion, PlaneRegionMut};
16 use crate::transform::TxSize;
17 use crate::Pixel;
18 use std::mem::MaybeUninit;
19 use v_frame::pixel::PixelType;
20
21 macro_rules! decl_angular_ipred_fn {
22 ($($f:ident),+) => {
23 extern {
24 $(
25 fn $f(
26 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
27 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
28 );
29 )*
30 }
31 };
32 }
33
34 decl_angular_ipred_fn! {
35 rav1e_ipred_h_8bpc_ssse3,
36 rav1e_ipred_h_8bpc_avx2,
37 rav1e_ipred_h_8bpc_avx512icl,
38 rav1e_ipred_v_8bpc_ssse3,
39 rav1e_ipred_v_8bpc_avx2,
40 rav1e_ipred_v_8bpc_avx512icl,
41 rav1e_ipred_dc_8bpc_ssse3,
42 rav1e_ipred_dc_8bpc_avx2,
43 rav1e_ipred_dc_8bpc_avx512icl,
44 rav1e_ipred_dc_left_8bpc_ssse3,
45 rav1e_ipred_dc_left_8bpc_avx2,
46 rav1e_ipred_dc_left_8bpc_avx512icl,
47 rav1e_ipred_dc_128_8bpc_ssse3,
48 rav1e_ipred_dc_128_8bpc_avx2,
49 rav1e_ipred_dc_128_8bpc_avx512icl,
50 rav1e_ipred_dc_top_8bpc_ssse3,
51 rav1e_ipred_dc_top_8bpc_avx2,
52 rav1e_ipred_dc_top_8bpc_avx512icl,
53 rav1e_ipred_smooth_v_8bpc_ssse3,
54 rav1e_ipred_smooth_v_8bpc_avx2,
55 rav1e_ipred_smooth_v_8bpc_avx512icl,
56 rav1e_ipred_smooth_h_8bpc_ssse3,
57 rav1e_ipred_smooth_h_8bpc_avx2,
58 rav1e_ipred_smooth_h_8bpc_avx512icl,
59 rav1e_ipred_smooth_8bpc_ssse3,
60 rav1e_ipred_smooth_8bpc_avx2,
61 rav1e_ipred_smooth_8bpc_avx512icl,
62 rav1e_ipred_z1_8bpc_ssse3,
63 rav1e_ipred_z1_8bpc_avx2,
64 rav1e_ipred_z3_8bpc_ssse3,
65 rav1e_ipred_z3_8bpc_avx2,
66 rav1e_ipred_paeth_8bpc_ssse3,
67 rav1e_ipred_paeth_8bpc_avx2,
68 rav1e_ipred_paeth_8bpc_avx512icl
69 }
70
71 macro_rules! decl_angular_ipred_hbd_fn {
72 ($($f:ident),+) => {
73 extern {
74 $(
75 fn $f(
76 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
77 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
78 max_width: libc::c_int, max_height: libc::c_int,
79 bit_depth_max: libc::c_int,
80 );
81 )*
82 }
83 };
84 }
85
86 decl_angular_ipred_hbd_fn! {
87 rav1e_ipred_h_16bpc_ssse3,
88 rav1e_ipred_h_16bpc_avx2,
89 rav1e_ipred_v_16bpc_ssse3,
90 rav1e_ipred_v_16bpc_avx2,
91 rav1e_ipred_dc_16bpc_ssse3,
92 rav1e_ipred_dc_16bpc_avx2,
93 rav1e_ipred_dc_left_16bpc_ssse3,
94 rav1e_ipred_dc_left_16bpc_avx2,
95 rav1e_ipred_dc_128_16bpc_ssse3,
96 rav1e_ipred_dc_128_16bpc_avx2,
97 rav1e_ipred_dc_top_16bpc_ssse3,
98 rav1e_ipred_dc_top_16bpc_avx2,
99 rav1e_ipred_smooth_v_16bpc_ssse3,
100 rav1e_ipred_smooth_v_16bpc_avx2,
101 rav1e_ipred_smooth_v_16bpc_avx512icl,
102 rav1e_ipred_smooth_h_16bpc_ssse3,
103 rav1e_ipred_smooth_h_16bpc_avx2,
104 rav1e_ipred_smooth_h_16bpc_avx512icl,
105 rav1e_ipred_smooth_16bpc_ssse3,
106 rav1e_ipred_smooth_16bpc_avx2,
107 rav1e_ipred_smooth_16bpc_avx512icl,
108 rav1e_ipred_z1_16bpc_ssse3,
109 rav1e_ipred_z1_16bpc_avx2,
110 rav1e_ipred_z2_16bpc_ssse3,
111 rav1e_ipred_z3_16bpc_ssse3,
112 rav1e_ipred_z3_16bpc_avx2,
113 rav1e_ipred_paeth_16bpc_ssse3,
114 rav1e_ipred_paeth_16bpc_avx2,
115 rav1e_ipred_paeth_16bpc_avx512icl
116 }
117
118 // For z2 prediction, we need to provide extra parameters, dx and dy, which indicate
119 // the distance between the predicted block's top-left pixel and the frame's edge.
120 // It is required for the intra edge filtering process.
121 extern {
122
123 fn rav1e_ipred_z2_8bpc_ssse3(
124 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
125 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
126 dx: libc::c_int, dy: libc::c_int,
127 );
128
129 fn rav1e_ipred_z2_8bpc_avx2(
130 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
131 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
132 dx: libc::c_int, dy: libc::c_int,
133 );
134
135 fn rav1e_ipred_z2_16bpc_avx2(
136 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
137 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
138 dx: libc::c_int, dy: libc::c_int, bit_depth_max: libc::c_int,
139 );
140 }
141
142 macro_rules! decl_cfl_ac_fn {
143 ($($f:ident),+) => {
144 extern {
145 $(
146 fn $f(
147 ac: *mut MaybeUninit<i16>, src: *const u8, stride: libc::ptrdiff_t,
148 w_pad: libc::c_int, h_pad: libc::c_int,
149 width: libc::c_int, height: libc::c_int,
150 );
151 )*
152 }
153 };
154 }
155
156 decl_cfl_ac_fn! {
157 rav1e_ipred_cfl_ac_420_8bpc_avx2,
158 rav1e_ipred_cfl_ac_420_8bpc_ssse3,
159 rav1e_ipred_cfl_ac_422_8bpc_avx2,
160 rav1e_ipred_cfl_ac_422_8bpc_ssse3,
161 rav1e_ipred_cfl_ac_444_8bpc_avx2,
162 rav1e_ipred_cfl_ac_444_8bpc_ssse3
163 }
164
165 macro_rules! decl_cfl_ac_hbd_fn {
166 ($($f:ident),+) => {
167 extern {
168 $(
169 fn $f(
170 ac: *mut MaybeUninit<i16>, src: *const u16, stride: libc::ptrdiff_t,
171 w_pad: libc::c_int, h_pad: libc::c_int,
172 width: libc::c_int, height: libc::c_int,
173 );
174 )*
175 }
176 };
177 }
178
179 decl_cfl_ac_hbd_fn! {
180 rav1e_ipred_cfl_ac_420_16bpc_ssse3,
181 rav1e_ipred_cfl_ac_420_16bpc_avx2,
182 rav1e_ipred_cfl_ac_422_16bpc_ssse3,
183 rav1e_ipred_cfl_ac_422_16bpc_avx2,
184 rav1e_ipred_cfl_ac_444_16bpc_ssse3,
185 rav1e_ipred_cfl_ac_444_16bpc_avx2
186 }
187
188 macro_rules! decl_cfl_pred_fn {
189 ($($f:ident),+) => {
190 extern {
191 $(
192 fn $f(
193 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
194 width: libc::c_int, height: libc::c_int, ac: *const i16,
195 alpha: libc::c_int,
196 );
197 )*
198 }
199 };
200 }
201
202 decl_cfl_pred_fn! {
203 rav1e_ipred_cfl_8bpc_ssse3,
204 rav1e_ipred_cfl_8bpc_avx2,
205 rav1e_ipred_cfl_left_8bpc_ssse3,
206 rav1e_ipred_cfl_left_8bpc_avx2,
207 rav1e_ipred_cfl_top_8bpc_ssse3,
208 rav1e_ipred_cfl_top_8bpc_avx2,
209 rav1e_ipred_cfl_128_8bpc_ssse3,
210 rav1e_ipred_cfl_128_8bpc_avx2
211 }
212
213 macro_rules! decl_cfl_pred_hbd_fn {
214 ($($f:ident),+) => {
215 extern {
216 $(
217 fn $f(
218 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
219 width: libc::c_int, height: libc::c_int, ac: *const i16,
220 alpha: libc::c_int, bit_depth_max: libc::c_int,
221 );
222 )*
223 }
224 };
225 }
226
227 decl_cfl_pred_hbd_fn! {
228 rav1e_ipred_cfl_16bpc_ssse3,
229 rav1e_ipred_cfl_16bpc_avx2,
230 rav1e_ipred_cfl_128_16bpc_ssse3,
231 rav1e_ipred_cfl_128_16bpc_avx2,
232 rav1e_ipred_cfl_left_16bpc_ssse3,
233 rav1e_ipred_cfl_left_16bpc_avx2,
234 rav1e_ipred_cfl_top_16bpc_ssse3,
235 rav1e_ipred_cfl_top_16bpc_avx2
236 }
237
238 #[inline(always)]
239 pub fn dispatch_predict_intra<T: Pixel>(
240 mode: PredictionMode, variant: PredictionVariant,
241 dst: &mut PlaneRegionMut<'_, T>, tx_size: TxSize, bit_depth: usize,
242 ac: &[i16], angle: isize, ief_params: Option<IntraEdgeFilterParameters>,
243 edge_buf: &IntraEdge<T>, cpu: CpuFeatureLevel,
244 ) {
245 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
246 rust::dispatch_predict_intra(
247 mode, variant, dst, tx_size, bit_depth, ac, angle, ief_params, edge_buf,
248 cpu,
249 );
250 };
251
252 // SAFETY: Calls Assembly code.
253 unsafe {
254 let stride = T::to_asm_stride(dst.plane_cfg.stride) as libc::ptrdiff_t;
255 let w = tx_size.width() as libc::c_int;
256 let h = tx_size.height() as libc::c_int;
257 let angle = angle as libc::c_int;
258
259 match T::type_enum() {
260 PixelType::U8 => {
261 let dst_ptr = dst.data_ptr_mut() as *mut _;
262 let edge_ptr = edge_buf.top_left_ptr() as *const _;
263 if cpu >= CpuFeatureLevel::AVX512ICL {
264 match mode {
265 PredictionMode::DC_PRED => {
266 (match variant {
267 PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_avx512icl,
268 PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_avx512icl,
269 PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_avx512icl,
270 PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_avx512icl,
271 })(dst_ptr, stride, edge_ptr, w, h, angle);
272 }
273 PredictionMode::V_PRED if angle == 90 => {
274 rav1e_ipred_v_8bpc_avx512icl(
275 dst_ptr, stride, edge_ptr, w, h, angle,
276 );
277 }
278 PredictionMode::H_PRED if angle == 180 => {
279 rav1e_ipred_h_8bpc_avx512icl(
280 dst_ptr, stride, edge_ptr, w, h, angle,
281 );
282 }
283 PredictionMode::V_PRED
284 | PredictionMode::H_PRED
285 | PredictionMode::D45_PRED
286 | PredictionMode::D135_PRED
287 | PredictionMode::D113_PRED
288 | PredictionMode::D157_PRED
289 | PredictionMode::D203_PRED
290 | PredictionMode::D67_PRED => {
291 let (enable_ief, ief_smooth_filter) =
292 if let Some(params) = ief_params {
293 (
294 true as libc::c_int,
295 params.use_smooth_filter() as libc::c_int,
296 )
297 } else {
298 (false as libc::c_int, false as libc::c_int)
299 };
300
301 // dav1d assembly uses the unused integer bits to hold IEF parameters
302 let angle_arg =
303 angle | (enable_ief << 10) | (ief_smooth_filter << 9);
304
305 // From dav1d, bw and bh are the frame width and height rounded to 8px units
306 let (bw, bh) = (
307 ((dst.plane_cfg.width + 7) >> 3) << 3,
308 ((dst.plane_cfg.height + 7) >> 3) << 3,
309 );
310 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
311 let (dx, dy) = (
312 (bw as isize - dst.rect().x) as libc::c_int,
313 (bh as isize - dst.rect().y) as libc::c_int,
314 );
315
316 if angle <= 90 {
317 rav1e_ipred_z1_8bpc_avx2(
318 dst_ptr, stride, edge_ptr, w, h, angle_arg,
319 );
320 } else if angle < 180 {
321 rav1e_ipred_z2_8bpc_avx2(
322 dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy,
323 );
324 } else {
325 rav1e_ipred_z3_8bpc_avx2(
326 dst_ptr, stride, edge_ptr, w, h, angle_arg,
327 );
328 }
329 }
330 PredictionMode::SMOOTH_PRED => {
331 rav1e_ipred_smooth_8bpc_avx512icl(
332 dst_ptr, stride, edge_ptr, w, h, angle,
333 );
334 }
335 PredictionMode::SMOOTH_V_PRED => {
336 rav1e_ipred_smooth_v_8bpc_avx512icl(
337 dst_ptr, stride, edge_ptr, w, h, angle,
338 );
339 }
340 PredictionMode::SMOOTH_H_PRED => {
341 rav1e_ipred_smooth_h_8bpc_avx512icl(
342 dst_ptr, stride, edge_ptr, w, h, angle,
343 );
344 }
345 PredictionMode::PAETH_PRED => {
346 rav1e_ipred_paeth_8bpc_avx512icl(
347 dst_ptr, stride, edge_ptr, w, h, angle,
348 );
349 }
350 PredictionMode::UV_CFL_PRED => {
351 let ac_ptr = ac.as_ptr() as *const _;
352 (match variant {
353 PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_avx2,
354 PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_avx2,
355 PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_avx2,
356 PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_avx2,
357 })(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
358 }
359 _ => call_rust(dst),
360 }
361 } else if cpu >= CpuFeatureLevel::AVX2 {
362 match mode {
363 PredictionMode::DC_PRED => {
364 (match variant {
365 PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_avx2,
366 PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_avx2,
367 PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_avx2,
368 PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_avx2,
369 })(dst_ptr, stride, edge_ptr, w, h, angle);
370 }
371 PredictionMode::V_PRED if angle == 90 => {
372 rav1e_ipred_v_8bpc_avx2(dst_ptr, stride, edge_ptr, w, h, angle);
373 }
374 PredictionMode::H_PRED if angle == 180 => {
375 rav1e_ipred_h_8bpc_avx2(dst_ptr, stride, edge_ptr, w, h, angle);
376 }
377 PredictionMode::V_PRED
378 | PredictionMode::H_PRED
379 | PredictionMode::D45_PRED
380 | PredictionMode::D135_PRED
381 | PredictionMode::D113_PRED
382 | PredictionMode::D157_PRED
383 | PredictionMode::D203_PRED
384 | PredictionMode::D67_PRED => {
385 let (enable_ief, ief_smooth_filter) =
386 if let Some(params) = ief_params {
387 (
388 true as libc::c_int,
389 params.use_smooth_filter() as libc::c_int,
390 )
391 } else {
392 (false as libc::c_int, false as libc::c_int)
393 };
394
395 // dav1d assembly uses the unused integer bits to hold IEF parameters
396 let angle_arg =
397 angle | (enable_ief << 10) | (ief_smooth_filter << 9);
398
399 // From dav1d, bw and bh are the frame width and height rounded to 8px units
400 let (bw, bh) = (
401 ((dst.plane_cfg.width + 7) >> 3) << 3,
402 ((dst.plane_cfg.height + 7) >> 3) << 3,
403 );
404 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
405 let (dx, dy) = (
406 (bw as isize - dst.rect().x) as libc::c_int,
407 (bh as isize - dst.rect().y) as libc::c_int,
408 );
409
410 if angle <= 90 {
411 rav1e_ipred_z1_8bpc_avx2(
412 dst_ptr, stride, edge_ptr, w, h, angle_arg,
413 );
414 } else if angle < 180 {
415 rav1e_ipred_z2_8bpc_avx2(
416 dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy,
417 );
418 } else {
419 rav1e_ipred_z3_8bpc_avx2(
420 dst_ptr, stride, edge_ptr, w, h, angle_arg,
421 );
422 }
423 }
424 PredictionMode::SMOOTH_PRED => {
425 rav1e_ipred_smooth_8bpc_avx2(
426 dst_ptr, stride, edge_ptr, w, h, angle,
427 );
428 }
429 PredictionMode::SMOOTH_V_PRED => {
430 rav1e_ipred_smooth_v_8bpc_avx2(
431 dst_ptr, stride, edge_ptr, w, h, angle,
432 );
433 }
434 PredictionMode::SMOOTH_H_PRED => {
435 rav1e_ipred_smooth_h_8bpc_avx2(
436 dst_ptr, stride, edge_ptr, w, h, angle,
437 );
438 }
439 PredictionMode::PAETH_PRED => {
440 rav1e_ipred_paeth_8bpc_avx2(
441 dst_ptr, stride, edge_ptr, w, h, angle,
442 );
443 }
444 PredictionMode::UV_CFL_PRED => {
445 let ac_ptr = ac.as_ptr() as *const _;
446 (match variant {
447 PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_avx2,
448 PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_avx2,
449 PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_avx2,
450 PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_avx2,
451 })(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
452 }
453 _ => call_rust(dst),
454 }
455 } else if cpu >= CpuFeatureLevel::SSSE3 {
456 match mode {
457 PredictionMode::DC_PRED => {
458 (match variant {
459 PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_ssse3,
460 PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_ssse3,
461 PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_ssse3,
462 PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_ssse3,
463 })(dst_ptr, stride, edge_ptr, w, h, angle);
464 }
465 PredictionMode::V_PRED if angle == 90 => {
466 rav1e_ipred_v_8bpc_ssse3(dst_ptr, stride, edge_ptr, w, h, angle);
467 }
468 PredictionMode::H_PRED if angle == 180 => {
469 rav1e_ipred_h_8bpc_ssse3(dst_ptr, stride, edge_ptr, w, h, angle);
470 }
471 PredictionMode::V_PRED
472 | PredictionMode::H_PRED
473 | PredictionMode::D45_PRED
474 | PredictionMode::D135_PRED
475 | PredictionMode::D113_PRED
476 | PredictionMode::D157_PRED
477 | PredictionMode::D203_PRED
478 | PredictionMode::D67_PRED => {
479 let (enable_ief, ief_smooth_filter) =
480 if let Some(params) = ief_params {
481 (
482 true as libc::c_int,
483 params.use_smooth_filter() as libc::c_int,
484 )
485 } else {
486 (false as libc::c_int, false as libc::c_int)
487 };
488
489 // dav1d assembly uses the unused integer bits to hold IEF parameters
490 let angle_arg =
491 angle | (enable_ief << 10) | (ief_smooth_filter << 9);
492
493 // From dav1d, bw and bh are the frame width and height rounded to 8px units
494 let (bw, bh) = (
495 ((dst.plane_cfg.width + 7) >> 3) << 3,
496 ((dst.plane_cfg.height + 7) >> 3) << 3,
497 );
498 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
499 let (dx, dy) = (
500 (bw as isize - dst.rect().x) as libc::c_int,
501 (bh as isize - dst.rect().y) as libc::c_int,
502 );
503
504 if angle <= 90 {
505 rav1e_ipred_z1_8bpc_ssse3(
506 dst_ptr, stride, edge_ptr, w, h, angle_arg,
507 );
508 } else if angle < 180 {
509 rav1e_ipred_z2_8bpc_ssse3(
510 dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy,
511 );
512 } else {
513 rav1e_ipred_z3_8bpc_ssse3(
514 dst_ptr, stride, edge_ptr, w, h, angle_arg,
515 );
516 }
517 }
518 PredictionMode::SMOOTH_PRED => {
519 rav1e_ipred_smooth_8bpc_ssse3(
520 dst_ptr, stride, edge_ptr, w, h, angle,
521 );
522 }
523 PredictionMode::SMOOTH_V_PRED => {
524 rav1e_ipred_smooth_v_8bpc_ssse3(
525 dst_ptr, stride, edge_ptr, w, h, angle,
526 );
527 }
528 PredictionMode::SMOOTH_H_PRED => {
529 rav1e_ipred_smooth_h_8bpc_ssse3(
530 dst_ptr, stride, edge_ptr, w, h, angle,
531 );
532 }
533 PredictionMode::PAETH_PRED => {
534 rav1e_ipred_paeth_8bpc_ssse3(
535 dst_ptr, stride, edge_ptr, w, h, angle,
536 );
537 }
538 PredictionMode::UV_CFL_PRED => {
539 let ac_ptr = ac.as_ptr() as *const _;
540 (match variant {
541 PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_ssse3,
542 PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_ssse3,
543 PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_ssse3,
544 PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_ssse3,
545 })(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
546 }
547 _ => call_rust(dst),
548 }
549 } else {
550 call_rust(dst)
551 }
552 }
553 PixelType::U16 => {
554 let dst_ptr = dst.data_ptr_mut() as *mut _;
555 let edge_ptr = edge_buf.top_left_ptr() as *const _;
556 let bd_max = (1 << bit_depth) - 1;
557 if cpu >= CpuFeatureLevel::AVX512ICL {
558 match mode {
559 PredictionMode::DC_PRED => {
560 (match variant {
561 PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_avx2,
562 PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_avx2,
563 PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_avx2,
564 PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_avx2,
565 })(
566 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max
567 );
568 }
569 PredictionMode::V_PRED if angle == 90 => {
570 rav1e_ipred_v_16bpc_avx2(
571 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
572 );
573 }
574 PredictionMode::H_PRED if angle == 180 => {
575 rav1e_ipred_h_16bpc_avx2(
576 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
577 );
578 }
579 PredictionMode::V_PRED
580 | PredictionMode::H_PRED
581 | PredictionMode::D45_PRED
582 | PredictionMode::D135_PRED
583 | PredictionMode::D113_PRED
584 | PredictionMode::D157_PRED
585 | PredictionMode::D203_PRED
586 | PredictionMode::D67_PRED => {
587 let (enable_ief, ief_smooth_filter) =
588 if let Some(params) = ief_params {
589 (
590 true as libc::c_int,
591 params.use_smooth_filter() as libc::c_int,
592 )
593 } else {
594 (false as libc::c_int, false as libc::c_int)
595 };
596
597 // dav1d assembly uses the unused integer bits to hold IEF parameters
598 let angle_arg =
599 angle | (enable_ief << 10) | (ief_smooth_filter << 9);
600
601 // From dav1d, bw and bh are the frame width and height rounded to 8px units
602 let (bw, bh) = (
603 ((dst.plane_cfg.width + 7) >> 3) << 3,
604 ((dst.plane_cfg.height + 7) >> 3) << 3,
605 );
606 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
607 let (dx, dy) = (
608 (bw as isize - dst.rect().x) as libc::c_int,
609 (bh as isize - dst.rect().y) as libc::c_int,
610 );
611
612 if angle <= 90 {
613 rav1e_ipred_z1_16bpc_avx2(
614 dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
615 );
616 } else if angle < 180 {
617 rav1e_ipred_z2_16bpc_avx2(
618 dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy, bd_max,
619 );
620 } else {
621 rav1e_ipred_z3_16bpc_avx2(
622 dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
623 );
624 }
625 }
626 PredictionMode::SMOOTH_PRED => {
627 rav1e_ipred_smooth_16bpc_avx512icl(
628 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
629 );
630 }
631 PredictionMode::SMOOTH_V_PRED => {
632 rav1e_ipred_smooth_v_16bpc_avx512icl(
633 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
634 );
635 }
636 PredictionMode::SMOOTH_H_PRED => {
637 rav1e_ipred_smooth_h_16bpc_avx512icl(
638 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
639 );
640 }
641 PredictionMode::PAETH_PRED => {
642 rav1e_ipred_paeth_16bpc_avx512icl(
643 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
644 );
645 }
646 PredictionMode::UV_CFL_PRED => {
647 let ac_ptr = ac.as_ptr() as *const _;
648 (match variant {
649 PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_avx2,
650 PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_avx2,
651 PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_avx2,
652 PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_avx2,
653 })(
654 dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle, bd_max
655 );
656 }
657 _ => call_rust(dst),
658 }
659 } else if cpu >= CpuFeatureLevel::AVX2 {
660 match mode {
661 PredictionMode::DC_PRED => {
662 (match variant {
663 PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_avx2,
664 PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_avx2,
665 PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_avx2,
666 PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_avx2,
667 })(
668 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max
669 );
670 }
671 PredictionMode::V_PRED if angle == 90 => {
672 rav1e_ipred_v_16bpc_avx2(
673 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
674 );
675 }
676 PredictionMode::H_PRED if angle == 180 => {
677 rav1e_ipred_h_16bpc_avx2(
678 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
679 );
680 }
681 PredictionMode::V_PRED
682 | PredictionMode::H_PRED
683 | PredictionMode::D45_PRED
684 | PredictionMode::D135_PRED
685 | PredictionMode::D113_PRED
686 | PredictionMode::D157_PRED
687 | PredictionMode::D203_PRED
688 | PredictionMode::D67_PRED => {
689 let (enable_ief, ief_smooth_filter) =
690 if let Some(params) = ief_params {
691 (
692 true as libc::c_int,
693 params.use_smooth_filter() as libc::c_int,
694 )
695 } else {
696 (false as libc::c_int, false as libc::c_int)
697 };
698
699 // dav1d assembly uses the unused integer bits to hold IEF parameters
700 let angle_arg =
701 angle | (enable_ief << 10) | (ief_smooth_filter << 9);
702
703 // From dav1d, bw and bh are the frame width and height rounded to 8px units
704 let (bw, bh) = (
705 ((dst.plane_cfg.width + 7) >> 3) << 3,
706 ((dst.plane_cfg.height + 7) >> 3) << 3,
707 );
708 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
709 let (dx, dy) = (
710 (bw as isize - dst.rect().x) as libc::c_int,
711 (bh as isize - dst.rect().y) as libc::c_int,
712 );
713
714 if angle <= 90 {
715 rav1e_ipred_z1_16bpc_avx2(
716 dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
717 );
718 } else if angle < 180 {
719 rav1e_ipred_z2_16bpc_avx2(
720 dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy, bd_max,
721 );
722 } else {
723 rav1e_ipred_z3_16bpc_avx2(
724 dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
725 );
726 }
727 }
728 PredictionMode::SMOOTH_PRED => {
729 rav1e_ipred_smooth_16bpc_avx2(
730 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
731 );
732 }
733 PredictionMode::SMOOTH_V_PRED => {
734 rav1e_ipred_smooth_v_16bpc_avx2(
735 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
736 );
737 }
738 PredictionMode::SMOOTH_H_PRED => {
739 rav1e_ipred_smooth_h_16bpc_avx2(
740 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
741 );
742 }
743 PredictionMode::PAETH_PRED => {
744 rav1e_ipred_paeth_16bpc_avx2(
745 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
746 );
747 }
748 PredictionMode::UV_CFL_PRED => {
749 let ac_ptr = ac.as_ptr() as *const _;
750 (match variant {
751 PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_avx2,
752 PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_avx2,
753 PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_avx2,
754 PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_avx2,
755 })(
756 dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle, bd_max
757 );
758 }
759 _ => call_rust(dst),
760 }
761 } else if cpu >= CpuFeatureLevel::SSSE3 {
762 match mode {
763 PredictionMode::DC_PRED => {
764 (match variant {
765 PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_ssse3,
766 PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_ssse3,
767 PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_ssse3,
768 PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_ssse3,
769 })(
770 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max
771 );
772 }
773 PredictionMode::V_PRED if angle == 90 => {
774 rav1e_ipred_v_16bpc_ssse3(
775 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
776 );
777 }
778 PredictionMode::H_PRED if angle == 180 => {
779 rav1e_ipred_h_16bpc_ssse3(
780 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
781 );
782 }
783 PredictionMode::V_PRED
784 | PredictionMode::H_PRED
785 | PredictionMode::D45_PRED
786 | PredictionMode::D135_PRED
787 | PredictionMode::D113_PRED
788 | PredictionMode::D157_PRED
789 | PredictionMode::D203_PRED
790 | PredictionMode::D67_PRED => {
791 let (enable_ief, ief_smooth_filter) =
792 if let Some(params) = ief_params {
793 (
794 true as libc::c_int,
795 params.use_smooth_filter() as libc::c_int,
796 )
797 } else {
798 (false as libc::c_int, false as libc::c_int)
799 };
800
801 // dav1d assembly uses the unused integer bits to hold IEF parameters
802 let angle_arg =
803 angle | (enable_ief << 10) | (ief_smooth_filter << 9);
804
805 // From dav1d, bw and bh are the frame width and height rounded to 8px units
806 let (bw, bh) = (
807 ((dst.plane_cfg.width + 7) >> 3) << 3,
808 ((dst.plane_cfg.height + 7) >> 3) << 3,
809 );
810 // From dav1d, dx and dy are the distance from the predicted block to the frame edge
811 let (dx, dy) = (
812 (bw as isize - dst.rect().x) as libc::c_int,
813 (bh as isize - dst.rect().y) as libc::c_int,
814 );
815
816 if angle <= 90 {
817 rav1e_ipred_z1_16bpc_ssse3(
818 dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
819 );
820 } else if angle < 180 {
821 rav1e_ipred_z2_16bpc_ssse3(
822 dst_ptr, stride, edge_ptr, w, h, angle_arg, dx, dy, bd_max,
823 );
824 } else {
825 rav1e_ipred_z3_16bpc_ssse3(
826 dst_ptr, stride, edge_ptr, w, h, angle_arg, 0, 0, bd_max,
827 );
828 }
829 }
830 PredictionMode::SMOOTH_PRED => {
831 rav1e_ipred_smooth_16bpc_ssse3(
832 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
833 );
834 }
835 PredictionMode::SMOOTH_V_PRED => {
836 rav1e_ipred_smooth_v_16bpc_ssse3(
837 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
838 );
839 }
840 PredictionMode::SMOOTH_H_PRED => {
841 rav1e_ipred_smooth_h_16bpc_ssse3(
842 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
843 );
844 }
845 PredictionMode::PAETH_PRED => {
846 rav1e_ipred_paeth_16bpc_ssse3(
847 dst_ptr, stride, edge_ptr, w, h, angle, 0, 0, bd_max,
848 );
849 }
850 PredictionMode::UV_CFL_PRED => {
851 let ac_ptr = ac.as_ptr() as *const _;
852 (match variant {
853 PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_ssse3,
854 PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_ssse3,
855 PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_ssse3,
856 PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_ssse3,
857 })(
858 dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle, bd_max
859 );
860 }
861 _ => call_rust(dst),
862 }
863 } else {
864 call_rust(dst)
865 }
866 }
867 }
868 }
869 }
870
871 // The implementation MUST inititialize all `ac` elements
872 #[inline(always)]
873 pub(crate) fn pred_cfl_ac<T: Pixel, const XDEC: usize, const YDEC: usize>(
874 ac: &mut [MaybeUninit<i16>], luma: &PlaneRegion<'_, T>, bsize: BlockSize,
875 w_pad: usize, h_pad: usize, cpu: CpuFeatureLevel,
876 ) {
877 debug_assert_eq!(ac.len(), bsize.area());
878
879 let call_rust = |ac: &mut [MaybeUninit<i16>]| {
880 rust::pred_cfl_ac::<T, XDEC, YDEC>(ac, luma, bsize, w_pad, h_pad, cpu);
881 };
882
883 let stride = T::to_asm_stride(luma.plane_cfg.stride) as libc::ptrdiff_t;
884 let w = bsize.width() as libc::c_int;
885 let h = bsize.height() as libc::c_int;
886 let w_pad = w_pad as libc::c_int;
887 let h_pad = h_pad as libc::c_int;
888
889 // SAFETY: Calls Assembly code.
890 unsafe {
891 let ac_ptr = ac.as_mut_ptr();
892 match T::type_enum() {
893 PixelType::U8 if cpu >= CpuFeatureLevel::SSSE3 => {
894 let luma_ptr = luma.data_ptr() as *const u8;
895 (if cpu >= CpuFeatureLevel::AVX2 {
896 match (XDEC, YDEC) {
897 (0, 0) => rav1e_ipred_cfl_ac_444_8bpc_avx2,
898 (1, 0) => rav1e_ipred_cfl_ac_422_8bpc_avx2,
899 _ => rav1e_ipred_cfl_ac_420_8bpc_avx2,
900 }
901 } else {
902 match (XDEC, YDEC) {
903 (0, 0) => rav1e_ipred_cfl_ac_444_8bpc_ssse3,
904 (1, 0) => rav1e_ipred_cfl_ac_422_8bpc_ssse3,
905 _ => rav1e_ipred_cfl_ac_420_8bpc_ssse3,
906 }
907 })(ac_ptr, luma_ptr, stride, w_pad, h_pad, w, h)
908 }
909 PixelType::U16 if cpu >= CpuFeatureLevel::SSSE3 => {
910 let luma_ptr = luma.data_ptr() as *const u16;
911 (if cpu >= CpuFeatureLevel::AVX2 {
912 match (XDEC, YDEC) {
913 (0, 0) => rav1e_ipred_cfl_ac_444_16bpc_avx2,
914 (1, 0) => rav1e_ipred_cfl_ac_422_16bpc_avx2,
915 _ => rav1e_ipred_cfl_ac_420_16bpc_avx2,
916 }
917 } else {
918 match (XDEC, YDEC) {
919 (0, 0) => rav1e_ipred_cfl_ac_444_16bpc_ssse3,
920 (1, 0) => rav1e_ipred_cfl_ac_422_16bpc_ssse3,
921 _ => rav1e_ipred_cfl_ac_420_16bpc_ssse3,
922 }
923 })(ac_ptr, luma_ptr, stride, w_pad, h_pad, w, h)
924 }
925 _ => call_rust(ac),
926 }
927 }
928 }