Debian Codegraph @

.cargo/vendor/av-scenechange/src/data/mc/simd_x86.rs

raw
1 use v_frame::{
2 pixel::{Pixel, PixelType},
3 plane::PlaneSlice,
4 };
5
6 use crate::{
7 cpu::CpuFeatureLevel,
8 data::{
9 mc::{FilterMode, FilterMode::*},
10 plane::PlaneRegionMut,
11 },
12 };
13
14 #[allow(clippy::too_many_arguments)]
15 pub fn put_8tap_internal<T: Pixel>(
16 dst: &mut PlaneRegionMut<'_, T>,
17 src: PlaneSlice<'_, T>,
18 width: usize,
19 height: usize,
20 col_frac: i32,
21 row_frac: i32,
22 bit_depth: usize,
23 cpu: CpuFeatureLevel,
24 ) {
25 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
26 super::rust::put_8tap_internal(dst, src, width, height, col_frac, row_frac, bit_depth, cpu);
27 };
28
29 // SAFETY: The assembly only supports even heights and valid uncropped
30 // widths
31 unsafe {
32 assert_eq!(height & 1, 0);
33 assert!(width.is_power_of_two() && (2..=128).contains(&width));
34
35 // SAFETY: Check bounds of dst
36 assert!(dst.rect().width >= width && dst.rect().height >= height);
37
38 // SAFETY: Check bounds of src
39 assert!(src.accessible(width + 4, height + 4));
40 assert!(src.accessible_neg(3, 3));
41
42 match T::type_enum() {
43 PixelType::U8 => match PUT_FNS[cpu.as_index()][get_2d_mode_idx(REGULAR, REGULAR)] {
44 Some(func) => func(
45 dst.data_ptr_mut() as *mut _,
46 T::to_asm_stride(dst.plane_cfg.stride),
47 src.as_ptr() as *const _,
48 T::to_asm_stride(src.plane.cfg.stride),
49 width as i32,
50 height as i32,
51 col_frac,
52 row_frac,
53 ),
54 None => call_rust(dst),
55 },
56 PixelType::U16 => {
57 match PUT_HBD_FNS[cpu.as_index()][get_2d_mode_idx(REGULAR, REGULAR)] {
58 Some(func) => func(
59 dst.data_ptr_mut() as *mut _,
60 T::to_asm_stride(dst.plane_cfg.stride),
61 src.as_ptr() as *const _,
62 T::to_asm_stride(src.plane.cfg.stride),
63 width as i32,
64 height as i32,
65 col_frac,
66 row_frac,
67 (1 << bit_depth) - 1,
68 ),
69 None => call_rust(dst),
70 }
71 }
72 }
73 }
74 }
75
76 type PutFn = unsafe extern "C" fn(
77 dst: *mut u8,
78 dst_stride: isize,
79 src: *const u8,
80 src_stride: isize,
81 width: i32,
82 height: i32,
83 col_frac: i32,
84 row_frac: i32,
85 );
86
87 type PutHBDFn = unsafe extern "C" fn(
88 dst: *mut u16,
89 dst_stride: isize,
90 src: *const u16,
91 src_stride: isize,
92 width: i32,
93 height: i32,
94 col_frac: i32,
95 row_frac: i32,
96 bitdepth_max: i32,
97 );
98
99 // gets an index that can be mapped to a function for a pair of filter modes
100 const fn get_2d_mode_idx(mode_x: FilterMode, mode_y: FilterMode) -> usize {
101 (mode_x as usize + 4 * (mode_y as usize)) & 15
102 }
103
104 macro_rules! decl_mc_fns {
105 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
106 pastey::item! {
107 extern "C" {
108 $(
109 fn [<$func_name _ssse3>](
110 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
111 w: i32, h: i32, mx: i32, my: i32
112 );
113
114 fn [<$func_name _avx2>](
115 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
116 w: i32, h: i32, mx: i32, my: i32
117 );
118
119 fn [<$func_name _avx512icl>](
120 dst: *mut u8, dst_stride: isize, src: *const u8, src_stride: isize,
121 w: i32, h: i32, mx: i32, my: i32
122 );
123 )*
124 }
125
126 static PUT_FNS_SSSE3: [Option<PutFn>; 16] = {
127 let mut out: [Option<PutFn>; 16] = [None; 16];
128 $(
129 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _ssse3>]);
130 )*
131 out
132 };
133
134 static PUT_FNS_AVX2: [Option<PutFn>; 16] = {
135 let mut out: [Option<PutFn>; 16] = [None; 16];
136 $(
137 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx2>]);
138 )*
139 out
140 };
141
142 static PUT_FNS_AVX512ICL: [Option<PutFn>; 16] = {
143 let mut out: [Option<PutFn>; 16] = [None; 16];
144 $(
145 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx512icl>]);
146 )*
147 out
148 };
149 }
150 }
151 }
152
153 decl_mc_fns!(
154 (REGULAR, REGULAR, avsc_put_8tap_regular_8bpc),
155 (REGULAR, SMOOTH, avsc_put_8tap_regular_smooth_8bpc),
156 (REGULAR, SHARP, avsc_put_8tap_regular_sharp_8bpc),
157 (SMOOTH, REGULAR, avsc_put_8tap_smooth_regular_8bpc),
158 (SMOOTH, SMOOTH, avsc_put_8tap_smooth_8bpc),
159 (SMOOTH, SHARP, avsc_put_8tap_smooth_sharp_8bpc),
160 (SHARP, REGULAR, avsc_put_8tap_sharp_regular_8bpc),
161 (SHARP, SMOOTH, avsc_put_8tap_sharp_smooth_8bpc),
162 (SHARP, SHARP, avsc_put_8tap_sharp_8bpc),
163 (BILINEAR, BILINEAR, avsc_put_bilin_8bpc)
164 );
165
166 cpu_function_lookup_table!(
167 PUT_FNS: [[Option<PutFn>; 16]],
168 default: [None; 16],
169 [SSSE3, AVX2, AVX512ICL]
170 );
171
172 macro_rules! decl_mc_hbd_fns {
173 ($(($mode_x:expr, $mode_y:expr, $func_name:ident)),+) => {
174 pastey::item! {
175 extern "C" {
176 $(
177 fn [<$func_name _ssse3>](
178 dst: *mut u16, dst_stride: isize, src: *const u16, src_stride: isize,
179 w: i32, h: i32, mx: i32, my: i32, bitdepth_max: i32,
180 );
181
182 fn [<$func_name _avx2>](
183 dst: *mut u16, dst_stride: isize, src: *const u16, src_stride: isize,
184 w: i32, h: i32, mx: i32, my: i32, bitdepth_max: i32,
185 );
186 )*
187 }
188
189 static PUT_HBD_FNS_SSSE3: [Option<PutHBDFn>; 16] = {
190 let mut out: [Option<PutHBDFn>; 16] = [None; 16];
191 $(
192 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _ssse3>]);
193 )*
194 out
195 };
196
197 static PUT_HBD_FNS_AVX2: [Option<PutHBDFn>; 16] = {
198 let mut out: [Option<PutHBDFn>; 16] = [None; 16];
199 $(
200 out[get_2d_mode_idx($mode_x, $mode_y)] = Some([<$func_name _avx2>]);
201 )*
202 out
203 };
204 }
205 }
206 }
207
208 decl_mc_hbd_fns!(
209 (REGULAR, REGULAR, avsc_put_8tap_regular_16bpc),
210 (REGULAR, SMOOTH, avsc_put_8tap_regular_smooth_16bpc),
211 (REGULAR, SHARP, avsc_put_8tap_regular_sharp_16bpc),
212 (SMOOTH, REGULAR, avsc_put_8tap_smooth_regular_16bpc),
213 (SMOOTH, SMOOTH, avsc_put_8tap_smooth_16bpc),
214 (SMOOTH, SHARP, avsc_put_8tap_smooth_sharp_16bpc),
215 (SHARP, REGULAR, avsc_put_8tap_sharp_regular_16bpc),
216 (SHARP, SMOOTH, avsc_put_8tap_sharp_smooth_16bpc),
217 (SHARP, SHARP, avsc_put_8tap_sharp_16bpc),
218 (BILINEAR, BILINEAR, avsc_put_bilin_16bpc)
219 );
220
221 cpu_function_lookup_table!(
222 PUT_HBD_FNS: [[Option<PutHBDFn>; 16]],
223 default: [None; 16],
224 [SSSE3, AVX2]
225 );