Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/cdef.rs

raw
1 // Copyright (c) 2020-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cdef::*;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::frame::*;
13 use crate::tiling::PlaneRegionMut;
14 use crate::util::*;
15
16 type CdefPaddingFn = unsafe extern fn(
17 tmp: *mut u16,
18 src: *const u8,
19 src_stride: isize,
20 left: *const [u8; 2],
21 top: *const u8,
22 bottom: *const u8,
23 h: i32,
24 edges: isize,
25 );
26
27 type CdefPaddingHBDFn = unsafe extern fn(
28 tmp: *mut u16,
29 src: *const u16,
30 src_stride: isize,
31 left: *const [u16; 2],
32 top: *const u16,
33 bottom: *const u16,
34 h: i32,
35 edges: isize,
36 );
37
38 type CdefFilterFn = unsafe extern fn(
39 dst: *mut u8,
40 dst_stride: isize,
41 tmp: *const u16,
42 pri_strength: i32,
43 sec_strength: i32,
44 dir: i32,
45 damping: i32,
46 h: i32,
47 edges: isize,
48 );
49
50 type CdefFilterHBDFn = unsafe extern fn(
51 dst: *mut u16,
52 dst_stride: isize,
53 tmp: *const u16,
54 pri_strength: i32,
55 sec_strength: i32,
56 dir: i32,
57 damping: i32,
58 h: i32,
59 edges: isize,
60 bitdepth_max: i32,
61 );
62
63 #[inline(always)]
64 const fn decimate_index(xdec: usize, ydec: usize) -> usize {
65 ((ydec << 1) | xdec) & 3
66 }
67
68 pub(crate) unsafe fn cdef_filter_block<T: Pixel>(
69 dst: &mut PlaneRegionMut<'_, T>, src: *const T, src_stride: isize,
70 pri_strength: i32, sec_strength: i32, dir: usize, damping: i32,
71 bit_depth: usize, xdec: usize, ydec: usize, edges: u8, cpu: CpuFeatureLevel,
72 ) {
73 let call_rust = |dst: &mut PlaneRegionMut<T>| {
74 rust::cdef_filter_block(
75 dst,
76 src,
77 src_stride,
78 pri_strength,
79 sec_strength,
80 dir,
81 damping,
82 bit_depth,
83 xdec,
84 ydec,
85 edges,
86 cpu,
87 );
88 };
89
90 #[cfg(feature = "check_asm")]
91 let ref_dst = {
92 let mut copy = dst.scratch_copy();
93 call_rust(&mut copy.as_region_mut());
94 copy
95 };
96 match T::type_enum() {
97 PixelType::U8 => {
98 match (
99 CDEF_PAD_FNS[cpu.as_index()][decimate_index(xdec, ydec)],
100 CDEF_FILTER_FNS[cpu.as_index()][decimate_index(xdec, ydec)],
101 ) {
102 (Some(pad), Some(func)) => {
103 let h = if ydec == 1 { 4 } else { 8 };
104 let tmpstride = if xdec == 1 { 8 } else { 16 } as isize;
105 const MAXTMPSTRIDE: isize = 16;
106 const TMPSIZE: usize = (12 * MAXTMPSTRIDE + 8) as usize;
107 let mut tmp: Aligned<[u16; TMPSIZE]> =
108 Aligned::new([CDEF_VERY_LARGE; TMPSIZE]);
109 let top = src.offset(-2 * src_stride);
110 let bottom = src.offset(h as isize * src_stride);
111 let mut left: Aligned<[[u8; 2]; 8]> = Aligned::new([[0; 2]; 8]);
112
113 // Rather than modify the dav1d assembly, just swallow our
114 // pride and copy relevant portions of src into a left
115 // array. The array is a monolithic, packed [x=2][y=8],
116 // though it is aligned to start on a 16-byte boundary.
117 if edges & CDEF_HAVE_LEFT != 0 {
118 let mut wptr = src.offset(-2) as *const u8;
119 for i in 0..h {
120 left.data[i as usize][0] = *wptr;
121 left.data[i as usize][1] = *wptr.add(1);
122 wptr = wptr.offset(src_stride);
123 }
124 }
125
126 // dav1d's implicit indexing as of this version: tmp array
127 // working pointer points to the upper-left of the current
128 // coded block, not the upper-left of the tmp array storage,
129 // with an adjustment to ensure UL of the block is 16-byte
130 // aligned. src, similarly, points to upper left of src
131 // block being coded. top points to coded block minus two
132 // rows (that is, src.x, src.y-2). It does _not_ point to
133 // src.x-2, src.y-2.
134 (pad)(
135 tmp.data.as_mut_ptr().offset(2 * tmpstride + 8),
136 src as *const u8,
137 T::to_asm_stride(src_stride as usize),
138 left.data.as_ptr(),
139 top as *const u8,
140 bottom as *const u8,
141 8 >> ydec,
142 edges.into(),
143 );
144
145 (func)(
146 dst.data_ptr_mut() as *mut u8,
147 T::to_asm_stride(dst.plane_cfg.stride),
148 tmp.data.as_ptr().offset(2 * tmpstride + 8),
149 pri_strength,
150 sec_strength,
151 dir as i32,
152 damping,
153 8 >> ydec,
154 edges.into(),
155 );
156 }
157 _ => call_rust(dst),
158 }
159 }
160 PixelType::U16 => {
161 match (
162 CDEF_PAD_HBD_FNS[cpu.as_index()][decimate_index(xdec, ydec)],
163 CDEF_FILTER_HBD_FNS[cpu.as_index()][decimate_index(xdec, ydec)],
164 ) {
165 // almost exactly as above, but the call convention isn't
166 // quite what we'd need to roll 8 bit and HBD together in one
167 // clause using Rust macros. See comments above for
168 // indexing/addressing notes.
169 (Some(pad), Some(func)) => {
170 let h = if ydec == 1 { 4 } else { 8 };
171 let tmpstride = if xdec == 1 { 8 } else { 16 } as isize;
172 const MAXTMPSTRIDE: isize = 16;
173 const TMPSIZE: usize = (12 * MAXTMPSTRIDE + 8) as usize;
174 let mut tmp: Aligned<[u16; TMPSIZE]> =
175 Aligned::new([CDEF_VERY_LARGE; TMPSIZE]);
176 let top = src.offset(-2 * src_stride);
177 let bottom = src.offset(h as isize * src_stride);
178 let mut left: Aligned<[[u16; 2]; 8]> = Aligned::new([[0; 2]; 8]);
179
180 if edges & CDEF_HAVE_LEFT != 0 {
181 let mut wptr = src.offset(-2) as *const u16;
182 for i in 0..h {
183 left.data[i as usize][0] = *wptr;
184 left.data[i as usize][1] = *wptr.add(1);
185 wptr = wptr.offset(src_stride);
186 }
187 }
188
189 (pad)(
190 tmp.data.as_mut_ptr().offset(2 * tmpstride + 8),
191 src as *const u16,
192 T::to_asm_stride(src_stride as usize),
193 left.data.as_ptr(),
194 top as *const u16,
195 bottom as *const u16,
196 8 >> ydec,
197 edges.into(),
198 );
199
200 (func)(
201 dst.data_ptr_mut() as *mut u16,
202 T::to_asm_stride(dst.plane_cfg.stride),
203 tmp.data.as_ptr().offset(2 * tmpstride + 8),
204 pri_strength,
205 sec_strength,
206 dir as i32,
207 damping,
208 8 >> ydec,
209 edges.into(),
210 (1 << bit_depth) - 1,
211 );
212 }
213 _ => call_rust(dst),
214 }
215 }
216 }
217 #[cfg(feature = "check_asm")]
218 {
219 for (dst_row, ref_row) in
220 dst.rows_iter().zip(ref_dst.as_region().rows_iter())
221 {
222 for (dst, reference) in dst_row.iter().zip(ref_row) {
223 assert_eq!(*dst, *reference);
224 }
225 }
226 }
227 }
228
229 extern {
230 fn rav1e_cdef_filter4_8bpc_neon(
231 dst: *mut u8, dst_stride: isize, tmp: *const u16, pri_strength: i32,
232 sec_strength: i32, dir: i32, damping: i32, h: i32, edges: isize,
233 );
234
235 fn rav1e_cdef_padding4_8bpc_neon(
236 tmp: *mut u16, src: *const u8, src_stride: isize, left: *const [u8; 2],
237 top: *const u8, bottom: *const u8, h: i32, edges: isize,
238 );
239
240 fn rav1e_cdef_filter8_8bpc_neon(
241 dst: *mut u8, dst_stride: isize, tmp: *const u16, pri_strength: i32,
242 sec_strength: i32, dir: i32, damping: i32, h: i32, edges: isize,
243 );
244
245 fn rav1e_cdef_padding8_8bpc_neon(
246 tmp: *mut u16, src: *const u8, src_stride: isize, left: *const [u8; 2],
247 top: *const u8, bottom: *const u8, h: i32, edges: isize,
248 );
249
250 fn rav1e_cdef_filter4_16bpc_neon(
251 dst: *mut u16, dst_stride: isize, tmp: *const u16, pri_strength: i32,
252 sec_strength: i32, dir: i32, damping: i32, h: i32, edges: isize, bd: i32,
253 );
254
255 fn rav1e_cdef_padding4_16bpc_neon(
256 tmp: *mut u16, src: *const u16, src_stride: isize, left: *const [u16; 2],
257 top: *const u16, bottom: *const u16, h: i32, edges: isize,
258 );
259
260 fn rav1e_cdef_filter8_16bpc_neon(
261 dst: *mut u16, dst_stride: isize, tmp: *const u16, pri_strength: i32,
262 sec_strength: i32, dir: i32, damping: i32, h: i32, edges: isize, bd: i32,
263 );
264
265 fn rav1e_cdef_padding8_16bpc_neon(
266 tmp: *mut u16, src: *const u16, src_stride: isize, left: *const [u16; 2],
267 top: *const u16, bottom: *const u16, h: i32, edges: isize,
268 );
269 }
270
271 static CDEF_PAD_FNS_NEON: [Option<CdefPaddingFn>; 4] = {
272 let mut out: [Option<CdefPaddingFn>; 4] = [None; 4];
273 out[decimate_index(1, 1)] = Some(rav1e_cdef_padding4_8bpc_neon);
274 out[decimate_index(1, 0)] = Some(rav1e_cdef_padding4_8bpc_neon);
275 out[decimate_index(0, 0)] = Some(rav1e_cdef_padding8_8bpc_neon);
276 out
277 };
278
279 static CDEF_FILTER_FNS_NEON: [Option<CdefFilterFn>; 4] = {
280 let mut out: [Option<CdefFilterFn>; 4] = [None; 4];
281 out[decimate_index(1, 1)] = Some(rav1e_cdef_filter4_8bpc_neon);
282 out[decimate_index(1, 0)] = Some(rav1e_cdef_filter4_8bpc_neon);
283 out[decimate_index(0, 0)] = Some(rav1e_cdef_filter8_8bpc_neon);
284 out
285 };
286
287 static CDEF_PAD_HBD_FNS_NEON: [Option<CdefPaddingHBDFn>; 4] = {
288 let mut out: [Option<CdefPaddingHBDFn>; 4] = [None; 4];
289 out[decimate_index(1, 1)] = Some(rav1e_cdef_padding4_16bpc_neon);
290 out[decimate_index(1, 0)] = Some(rav1e_cdef_padding4_16bpc_neon);
291 out[decimate_index(0, 0)] = Some(rav1e_cdef_padding8_16bpc_neon);
292 out
293 };
294
295 static CDEF_FILTER_HBD_FNS_NEON: [Option<CdefFilterHBDFn>; 4] = {
296 let mut out: [Option<CdefFilterHBDFn>; 4] = [None; 4];
297 out[decimate_index(1, 1)] = Some(rav1e_cdef_filter4_16bpc_neon);
298 out[decimate_index(1, 0)] = Some(rav1e_cdef_filter4_16bpc_neon);
299 out[decimate_index(0, 0)] = Some(rav1e_cdef_filter8_16bpc_neon);
300 out
301 };
302
303 cpu_function_lookup_table!(
304 CDEF_PAD_FNS: [[Option<CdefPaddingFn>; 4]],
305 default: [None; 4],
306 [NEON]
307 );
308
309 cpu_function_lookup_table!(
310 CDEF_FILTER_FNS: [[Option<CdefFilterFn>; 4]],
311 default: [None; 4],
312 [NEON]
313 );
314
315 cpu_function_lookup_table!(
316 CDEF_PAD_HBD_FNS: [[Option<CdefPaddingHBDFn>; 4]],
317 default: [None; 4],
318 [NEON]
319 );
320
321 cpu_function_lookup_table!(
322 CDEF_FILTER_HBD_FNS: [[Option<CdefFilterHBDFn>; 4]],
323 default: [None; 4],
324 [NEON]
325 );
326
327 type CdefDirLBDFn =
328 unsafe extern fn(tmp: *const u8, tmp_stride: isize, var: *mut u32) -> i32;
329 type CdefDirHBDFn = unsafe extern fn(
330 tmp: *const u16,
331 tmp_stride: isize,
332 var: *mut u32,
333 bitdepth_max: i32,
334 ) -> i32;
335
336 #[inline(always)]
337 #[allow(clippy::let_and_return)]
338 pub(crate) fn cdef_find_dir<T: Pixel>(
339 img: &PlaneSlice<'_, T>, var: &mut u32, coeff_shift: usize,
340 cpu: CpuFeatureLevel,
341 ) -> i32 {
342 let call_rust =
343 |var: &mut u32| rust::cdef_find_dir::<T>(img, var, coeff_shift, cpu);
344
345 #[cfg(feature = "check_asm")]
346 let (ref_dir, ref_var) = {
347 let mut var: u32 = 0;
348 let dir = call_rust(&mut var);
349 (dir, var)
350 };
351
352 let dir = match T::type_enum() {
353 PixelType::U8 => {
354 if let Some(func) = CDEF_DIR_LBD_FNS[cpu.as_index()] {
355 unsafe {
356 (func)(
357 img.as_ptr() as *const _,
358 T::to_asm_stride(img.plane.cfg.stride),
359 var as *mut u32,
360 )
361 }
362 } else {
363 call_rust(var)
364 }
365 }
366 PixelType::U16 if coeff_shift > 0 => {
367 if let Some(func) = CDEF_DIR_HBD_FNS[cpu.as_index()] {
368 unsafe {
369 (func)(
370 img.as_ptr() as *const _,
371 T::to_asm_stride(img.plane.cfg.stride),
372 var as *mut u32,
373 (1 << (coeff_shift + 8)) - 1,
374 )
375 }
376 } else {
377 call_rust(var)
378 }
379 }
380 _ => call_rust(var),
381 };
382
383 #[cfg(feature = "check_asm")]
384 {
385 assert_eq!(dir, ref_dir);
386 assert_eq!(*var, ref_var);
387 }
388
389 dir
390 }
391
392 extern {
393 fn rav1e_cdef_find_dir_8bpc_neon(
394 tmp: *const u8, tmp_stride: isize, var: *mut u32,
395 ) -> i32;
396 }
397
398 extern {
399 fn rav1e_cdef_find_dir_16bpc_neon(
400 tmp: *const u16, tmp_stride: isize, var: *mut u32, max_bitdepth: i32,
401 ) -> i32;
402 }
403
404 cpu_function_lookup_table!(
405 CDEF_DIR_LBD_FNS: [Option<CdefDirLBDFn>],
406 default: None,
407 [(NEON, Some(rav1e_cdef_find_dir_8bpc_neon))]
408 );
409
410 cpu_function_lookup_table!(
411 CDEF_DIR_HBD_FNS: [Option<CdefDirHBDFn>],
412 default: None,
413 [(NEON, Some(rav1e_cdef_find_dir_16bpc_neon))]
414 );