Debian Codegraph @

.cargo/vendor/av-scenechange/src/data/satd/simd_x86.rs

raw
1 use v_frame::pixel::{Pixel, PixelType};
2
3 use super::{to_index, DIST_FNS_LENGTH};
4 use crate::{
5 cpu::CpuFeatureLevel,
6 data::{block::BlockSize, plane::PlaneRegion},
7 };
8
9 type SatdFn = unsafe extern "C" fn(
10 src: *const u8,
11 src_stride: isize,
12 dst: *const u8,
13 dst_stride: isize,
14 ) -> u32;
15
16 type SatdHBDFn = unsafe extern "C" fn(
17 src: *const u16,
18 src_stride: isize,
19 dst: *const u16,
20 dst_stride: isize,
21 bdmax: u32,
22 ) -> u32;
23
24 macro_rules! declare_asm_dist_fn {
25 ($(($name: ident, $T: ident)),+) => (
26 $(
27 extern "C" { fn $name (
28 src: *const $T, src_stride: isize, dst: *const $T, dst_stride: isize
29 ) -> u32; }
30 )+
31 )
32 }
33
34 macro_rules! declare_asm_satd_hbd_fn {
35 ($($name: ident),+) => (
36 $(
37 extern "C" { pub(crate) fn $name (
38 src: *const u16, src_stride: isize, dst: *const u16, dst_stride: isize, bdmax: u32
39 ) -> u32; }
40 )+
41 )
42 }
43
44 declare_asm_dist_fn![
45 // SSSE3
46 (avsc_satd_8x8_ssse3, u8),
47 // SSE4
48 (avsc_satd_4x4_sse4, u8),
49 // AVX
50 (avsc_satd_4x4_avx2, u8),
51 (avsc_satd_8x8_avx2, u8),
52 (avsc_satd_16x16_avx2, u8),
53 (avsc_satd_32x32_avx2, u8),
54 (avsc_satd_64x64_avx2, u8),
55 (avsc_satd_128x128_avx2, u8),
56 (avsc_satd_4x8_avx2, u8),
57 (avsc_satd_8x4_avx2, u8),
58 (avsc_satd_8x16_avx2, u8),
59 (avsc_satd_16x8_avx2, u8),
60 (avsc_satd_16x32_avx2, u8),
61 (avsc_satd_32x16_avx2, u8),
62 (avsc_satd_32x64_avx2, u8),
63 (avsc_satd_64x32_avx2, u8),
64 (avsc_satd_64x128_avx2, u8),
65 (avsc_satd_128x64_avx2, u8),
66 (avsc_satd_4x16_avx2, u8),
67 (avsc_satd_16x4_avx2, u8),
68 (avsc_satd_8x32_avx2, u8),
69 (avsc_satd_32x8_avx2, u8),
70 (avsc_satd_16x64_avx2, u8),
71 (avsc_satd_64x16_avx2, u8)
72 ];
73
74 declare_asm_satd_hbd_fn![
75 avsc_satd_4x4_hbd_avx2,
76 avsc_satd_8x4_hbd_avx2,
77 avsc_satd_4x8_hbd_avx2,
78 avsc_satd_8x8_hbd_avx2,
79 avsc_satd_16x8_hbd_avx2,
80 avsc_satd_16x16_hbd_avx2,
81 avsc_satd_32x32_hbd_avx2,
82 avsc_satd_64x64_hbd_avx2,
83 avsc_satd_128x128_hbd_avx2,
84 avsc_satd_16x32_hbd_avx2,
85 avsc_satd_16x64_hbd_avx2,
86 avsc_satd_32x16_hbd_avx2,
87 avsc_satd_32x64_hbd_avx2,
88 avsc_satd_64x16_hbd_avx2,
89 avsc_satd_64x32_hbd_avx2,
90 avsc_satd_64x128_hbd_avx2,
91 avsc_satd_128x64_hbd_avx2,
92 avsc_satd_32x8_hbd_avx2,
93 avsc_satd_8x16_hbd_avx2,
94 avsc_satd_8x32_hbd_avx2,
95 avsc_satd_16x4_hbd_avx2,
96 avsc_satd_4x16_hbd_avx2
97 ];
98
99 static SATD_FNS_SSSE3: [Option<SatdFn>; DIST_FNS_LENGTH] = {
100 let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
101
102 use BlockSize::*;
103
104 out[BLOCK_8X8 as usize] = Some(avsc_satd_8x8_ssse3);
105
106 out
107 };
108
109 static SATD_FNS_SSE4_1: [Option<SatdFn>; DIST_FNS_LENGTH] = {
110 let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
111
112 use BlockSize::*;
113
114 out[BLOCK_4X4 as usize] = Some(avsc_satd_4x4_sse4);
115 out[BLOCK_8X8 as usize] = Some(avsc_satd_8x8_ssse3);
116
117 out
118 };
119
120 static SATD_FNS_AVX2: [Option<SatdFn>; DIST_FNS_LENGTH] = {
121 let mut out: [Option<SatdFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
122
123 use BlockSize::*;
124
125 out[BLOCK_4X4 as usize] = Some(avsc_satd_4x4_avx2);
126 out[BLOCK_8X8 as usize] = Some(avsc_satd_8x8_avx2);
127 out[BLOCK_16X16 as usize] = Some(avsc_satd_16x16_avx2);
128 out[BLOCK_32X32 as usize] = Some(avsc_satd_32x32_avx2);
129 out[BLOCK_64X64 as usize] = Some(avsc_satd_64x64_avx2);
130 out[BLOCK_128X128 as usize] = Some(avsc_satd_128x128_avx2);
131
132 out[BLOCK_4X8 as usize] = Some(avsc_satd_4x8_avx2);
133 out[BLOCK_8X4 as usize] = Some(avsc_satd_8x4_avx2);
134 out[BLOCK_8X16 as usize] = Some(avsc_satd_8x16_avx2);
135 out[BLOCK_16X8 as usize] = Some(avsc_satd_16x8_avx2);
136 out[BLOCK_16X32 as usize] = Some(avsc_satd_16x32_avx2);
137 out[BLOCK_32X16 as usize] = Some(avsc_satd_32x16_avx2);
138 out[BLOCK_32X64 as usize] = Some(avsc_satd_32x64_avx2);
139 out[BLOCK_64X32 as usize] = Some(avsc_satd_64x32_avx2);
140 out[BLOCK_64X128 as usize] = Some(avsc_satd_64x128_avx2);
141 out[BLOCK_128X64 as usize] = Some(avsc_satd_128x64_avx2);
142
143 out[BLOCK_4X16 as usize] = Some(avsc_satd_4x16_avx2);
144 out[BLOCK_16X4 as usize] = Some(avsc_satd_16x4_avx2);
145 out[BLOCK_8X32 as usize] = Some(avsc_satd_8x32_avx2);
146 out[BLOCK_32X8 as usize] = Some(avsc_satd_32x8_avx2);
147 out[BLOCK_16X64 as usize] = Some(avsc_satd_16x64_avx2);
148 out[BLOCK_64X16 as usize] = Some(avsc_satd_64x16_avx2);
149
150 out
151 };
152
153 cpu_function_lookup_table!(
154 SATD_FNS: [[Option<SatdFn>; DIST_FNS_LENGTH]],
155 default: [None; DIST_FNS_LENGTH],
156 [SSSE3, SSE4_1, AVX2]
157 );
158
159 static SATD_HBD_FNS_AVX2: [Option<SatdHBDFn>; DIST_FNS_LENGTH] = {
160 let mut out: [Option<SatdHBDFn>; DIST_FNS_LENGTH] = [None; DIST_FNS_LENGTH];
161
162 use BlockSize::*;
163
164 out[BLOCK_4X4 as usize] = Some(avsc_satd_4x4_hbd_avx2);
165 out[BLOCK_8X8 as usize] = Some(avsc_satd_8x8_hbd_avx2);
166 out[BLOCK_16X16 as usize] = Some(avsc_satd_16x16_hbd_avx2);
167 out[BLOCK_32X32 as usize] = Some(avsc_satd_32x32_hbd_avx2);
168 out[BLOCK_64X64 as usize] = Some(avsc_satd_64x64_hbd_avx2);
169 out[BLOCK_128X128 as usize] = Some(avsc_satd_128x128_hbd_avx2);
170
171 out[BLOCK_4X8 as usize] = Some(avsc_satd_4x8_hbd_avx2);
172 out[BLOCK_8X4 as usize] = Some(avsc_satd_8x4_hbd_avx2);
173 out[BLOCK_8X16 as usize] = Some(avsc_satd_8x16_hbd_avx2);
174 out[BLOCK_16X8 as usize] = Some(avsc_satd_16x8_hbd_avx2);
175 out[BLOCK_16X32 as usize] = Some(avsc_satd_16x32_hbd_avx2);
176 out[BLOCK_32X16 as usize] = Some(avsc_satd_32x16_hbd_avx2);
177 out[BLOCK_32X64 as usize] = Some(avsc_satd_32x64_hbd_avx2);
178 out[BLOCK_64X32 as usize] = Some(avsc_satd_64x32_hbd_avx2);
179 out[BLOCK_64X128 as usize] = Some(avsc_satd_64x128_hbd_avx2);
180 out[BLOCK_128X64 as usize] = Some(avsc_satd_128x64_hbd_avx2);
181
182 out[BLOCK_4X16 as usize] = Some(avsc_satd_4x16_hbd_avx2);
183 out[BLOCK_16X4 as usize] = Some(avsc_satd_16x4_hbd_avx2);
184 out[BLOCK_8X32 as usize] = Some(avsc_satd_8x32_hbd_avx2);
185 out[BLOCK_32X8 as usize] = Some(avsc_satd_32x8_hbd_avx2);
186 out[BLOCK_16X64 as usize] = Some(avsc_satd_16x64_hbd_avx2);
187 out[BLOCK_64X16 as usize] = Some(avsc_satd_64x16_hbd_avx2);
188
189 out
190 };
191
192 cpu_function_lookup_table!(
193 SATD_HBD_FNS: [[Option<SatdHBDFn>; DIST_FNS_LENGTH]],
194 default: [None; DIST_FNS_LENGTH],
195 [AVX2]
196 );
197
198 pub(super) fn get_satd_internal<T: Pixel>(
199 src: &PlaneRegion<'_, T>,
200 dst: &PlaneRegion<'_, T>,
201 w: usize,
202 h: usize,
203 bit_depth: usize,
204 cpu: CpuFeatureLevel,
205 ) -> u32 {
206 let bsize_opt = BlockSize::from_width_and_height_opt(w, h);
207
208 let call_rust = || -> u32 { super::rust::get_satd_internal(dst, src, w, h, bit_depth, cpu) };
209
210 match (bsize_opt, T::type_enum()) {
211 (Err(_), _) => call_rust(),
212 (Ok(bsize), PixelType::U8) => {
213 match SATD_FNS[cpu.as_index()][to_index(bsize)] {
214 // SAFETY: Calls Assembly code.
215 Some(func) => unsafe {
216 func(
217 src.data_ptr() as *const _,
218 T::to_asm_stride(src.plane_cfg.stride),
219 dst.data_ptr() as *const _,
220 T::to_asm_stride(dst.plane_cfg.stride),
221 )
222 },
223 None => call_rust(),
224 }
225 }
226 (Ok(bsize), PixelType::U16) => {
227 match SATD_HBD_FNS[cpu.as_index()][to_index(bsize)] {
228 // SAFETY: Calls Assembly code.
229 Some(func) => unsafe {
230 func(
231 src.data_ptr() as *const _,
232 T::to_asm_stride(src.plane_cfg.stride),
233 dst.data_ptr() as *const _,
234 T::to_asm_stride(dst.plane_cfg.stride),
235 (1 << bit_depth) - 1,
236 )
237 },
238 None => call_rust(),
239 }
240 }
241 }
242 }