Debian Codegraph @

.cargo/vendor/rav1e/src/asm/aarch64/transform/inverse.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::tiling::PlaneRegionMut;
12 use crate::transform::inverse::*;
13 use crate::transform::*;
14 use crate::{Pixel, PixelType};
15
16 use crate::asm::shared::transform::inverse::*;
17
18 pub fn inverse_transform_add<T: Pixel>(
19 input: &[T::Coeff], output: &mut PlaneRegionMut<'_, T>, eob: u16,
20 tx_size: TxSize, tx_type: TxType, bd: usize, cpu: CpuFeatureLevel,
21 ) {
22 match T::type_enum() {
23 PixelType::U8 => {
24 if let Some(func) = INV_TXFM_FNS[cpu.as_index()][tx_size][tx_type] {
25 return call_inverse_func(
26 func,
27 input,
28 output,
29 eob,
30 tx_size.width(),
31 tx_size.height(),
32 bd,
33 );
34 }
35 }
36 PixelType::U16 if bd == 10 => {
37 if let Some(func) = INV_TXFM_HBD_FNS[cpu.as_index()][tx_size][tx_type] {
38 return call_inverse_hbd_func(
39 func,
40 input,
41 output,
42 eob,
43 tx_size.width(),
44 tx_size.height(),
45 bd,
46 );
47 }
48 }
49 PixelType::U16 => {}
50 };
51
52 rust::inverse_transform_add(input, output, eob, tx_size, tx_type, bd, cpu);
53 }
54
55 macro_rules! decl_itx_fns {
56 // Takes a 2d list of tx types for W and H
57 ([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr,
58 $OPT_LOWER:ident, $OPT_UPPER:ident) => {
59 paste::item! {
60 // For each tx type, declare an function for the current WxH
61 $(
62 $(
63 extern {
64 // Note: type1 and type2 are flipped
65 fn [<rav1e_inv_txfm_add_ $TYPE2 _$TYPE1 _$W x $H _8bpc_$OPT_LOWER>](
66 dst: *mut u8, dst_stride: libc::ptrdiff_t, coeff: *mut i16,
67 eob: i32
68 );
69 }
70 )*
71 )*
72 // Create a lookup table for the tx types declared above
73 const [<INV_TXFM_FNS_$W _$H _$OPT_UPPER>]: [Option<InvTxfmFunc>; TX_TYPES_PLUS_LL] = {
74 let mut out: [Option<InvTxfmFunc>; TX_TYPES_PLUS_LL] = [None; TX_TYPES_PLUS_LL];
75 $(
76 $(
77 out[$ENUM as usize] = Some([<rav1e_inv_txfm_add_$TYPE2 _$TYPE1 _$W x $H _8bpc_$OPT_LOWER>]);
78 )*
79 )*
80 out
81 };
82 }
83 };
84 }
85
86 macro_rules! decl_itx_hbd_fns {
87 // Takes a 2d list of tx types for W and H
88 ([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr,
89 $OPT_LOWER:ident, $OPT_UPPER:ident) => {
90 paste::item! {
91 // For each tx type, declare an function for the current WxH
92 $(
93 $(
94 extern {
95 // Note: type1 and type2 are flipped
96 fn [<rav1e_inv_txfm_add_ $TYPE2 _$TYPE1 _$W x $H _16bpc_$OPT_LOWER>](
97 dst: *mut u16, dst_stride: libc::ptrdiff_t, coeff: *mut i16,
98 eob: i32, bitdepth_max: i32,
99 );
100 }
101 )*
102 )*
103 // Create a lookup table for the tx types declared above
104 const [<INV_TXFM_HBD_FNS_$W _$H _$OPT_UPPER>]: [Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL] = {
105 let mut out: [Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL] = [None; TX_TYPES_PLUS_LL];
106 $(
107 $(
108 out[$ENUM as usize] = Some([<rav1e_inv_txfm_add_$TYPE2 _$TYPE1 _$W x $H _16bpc_$OPT_LOWER>]);
109 )*
110 )*
111 out
112 };
113 }
114 };
115 }
116
117 macro_rules! create_wxh_tables {
118 // Create a lookup table for each cpu feature
119 ([$([$(($W:expr, $H:expr)),*]),*], $OPT_LOWER:ident, $OPT_UPPER:ident) => {
120 paste::item! {
121 const [<INV_TXFM_FNS_$OPT_UPPER>]: [[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] = {
122 let mut out: [[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] =
123 [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL];
124 // For each dimension, add an entry to the table
125 $(
126 $(
127 out[TxSize::[<TX_ $W X $H>] as usize] = [<INV_TXFM_FNS_$W _$H _$OPT_UPPER>];
128 )*
129 )*
130 out
131 };
132 }
133 };
134
135 // Loop through cpu features
136 ($DIMS:tt, [$(($OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
137 $(
138 create_wxh_tables!($DIMS, $OPT_LOWER, $OPT_UPPER);
139 )*
140 };
141 }
142
143 macro_rules! create_wxh_hbd_tables {
144 // Create a lookup table for each cpu feature
145 ([$([$(($W:expr, $H:expr)),*]),*], $OPT_LOWER:ident, $OPT_UPPER:ident) => {
146 paste::item! {
147 const [<INV_TXFM_HBD_FNS_$OPT_UPPER>]: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] = {
148 let mut out: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] =
149 [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL];
150 // For each dimension, add an entry to the table
151 $(
152 $(
153 out[TxSize::[<TX_ $W X $H>] as usize] = [<INV_TXFM_HBD_FNS_$W _$H _$OPT_UPPER>];
154 )*
155 )*
156 out
157 };
158 }
159 };
160
161 // Loop through cpu features
162 ($DIMS:tt, [$(($OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
163 $(
164 create_wxh_hbd_tables!($DIMS, $OPT_LOWER, $OPT_UPPER);
165 )*
166 };
167 }
168
169 macro_rules! impl_itx_fns {
170 ($TYPES:tt, $W:expr, $H:expr, [$(($OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
171 $(
172 decl_itx_fns!($TYPES, $W, $H, $OPT_LOWER, $OPT_UPPER);
173 )*
174 };
175
176 // Loop over a list of dimensions
177 ($TYPES_VALID:tt, [$(($W:expr, $H:expr)),*], $OPT:tt) => {
178 $(
179 impl_itx_fns!($TYPES_VALID, $W, $H, $OPT);
180 )*
181 };
182
183 ($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
184 $TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt, $OPT:tt) => {
185 // Make 2d list of tx types for each set of dimensions. Each set of
186 // dimensions uses a superset of the previous set of tx types.
187 impl_itx_fns!([$TYPES64], $DIMS64, $OPT);
188 impl_itx_fns!([$TYPES64, $TYPES32], $DIMS32, $OPT);
189 impl_itx_fns!([$TYPES64, $TYPES32, $TYPES16], $DIMS16, $OPT);
190 impl_itx_fns!(
191 [$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84, $OPT
192 );
193 impl_itx_fns!(
194 [$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4, $OPT
195 );
196
197 // Pool all of the dimensions together to create a table for each cpu
198 // feature level.
199 create_wxh_tables!(
200 [$DIMS64, $DIMS32, $DIMS16, $DIMS84, $DIMS4], $OPT
201 );
202 };
203 }
204
205 impl_itx_fns!(
206 // 64x
207 [(TxType::DCT_DCT, dct, dct)],
208 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
209 // 32x
210 [(TxType::IDTX, identity, identity)],
211 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
212 // 16x16
213 [
214 (TxType::DCT_ADST, dct, adst),
215 (TxType::ADST_DCT, adst, dct),
216 (TxType::DCT_FLIPADST, dct, flipadst),
217 (TxType::FLIPADST_DCT, flipadst, dct),
218 (TxType::V_DCT, dct, identity),
219 (TxType::H_DCT, identity, dct),
220 (TxType::ADST_ADST, adst, adst),
221 (TxType::ADST_FLIPADST, adst, flipadst),
222 (TxType::FLIPADST_ADST, flipadst, adst),
223 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
224 ],
225 [(16, 16)],
226 // 8x, 4x and 16x (minus 16x16 and 4x4)
227 [
228 (TxType::V_ADST, adst, identity),
229 (TxType::H_ADST, identity, adst),
230 (TxType::V_FLIPADST, flipadst, identity),
231 (TxType::H_FLIPADST, identity, flipadst)
232 ],
233 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8)],
234 // 4x4
235 [(TxType::WHT_WHT, wht, wht)],
236 [(4, 4)],
237 [(neon, NEON)]
238 );
239
240 cpu_function_lookup_table!(
241 INV_TXFM_FNS: [[[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
242 default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
243 [NEON]
244 );
245
246 macro_rules! impl_itx_hbd_fns {
247
248 ($TYPES:tt, $W:expr, $H:expr, [$(($OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
249 $(
250 decl_itx_hbd_fns!($TYPES, $W, $H, $OPT_LOWER, $OPT_UPPER);
251 )*
252 };
253
254 // Loop over a list of dimensions
255 ($TYPES_VALID:tt, [$(($W:expr, $H:expr)),*], $OPT:tt) => {
256 $(
257 impl_itx_hbd_fns!($TYPES_VALID, $W, $H, $OPT);
258 )*
259 };
260
261 ($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
262 $TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt, $OPT:tt) => {
263 // Make 2d list of tx types for each set of dimensions. Each set of
264 // dimensions uses a superset of the previous set of tx types.
265 impl_itx_hbd_fns!([$TYPES64], $DIMS64, $OPT);
266 impl_itx_hbd_fns!([$TYPES64, $TYPES32], $DIMS32, $OPT);
267 impl_itx_hbd_fns!([$TYPES64, $TYPES32, $TYPES16], $DIMS16, $OPT);
268 impl_itx_hbd_fns!(
269 [$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84, $OPT
270 );
271 impl_itx_hbd_fns!(
272 [$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4, $OPT
273 );
274
275 // Pool all of the dimensions together to create a table for each cpu
276 // feature level.
277 create_wxh_hbd_tables!(
278 [$DIMS64, $DIMS32, $DIMS16, $DIMS84, $DIMS4], $OPT
279 );
280 };
281 }
282
283 impl_itx_hbd_fns!(
284 // 64x
285 [(TxType::DCT_DCT, dct, dct)],
286 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
287 // 32x
288 [(TxType::IDTX, identity, identity)],
289 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
290 // 16x16
291 [
292 (TxType::DCT_ADST, dct, adst),
293 (TxType::ADST_DCT, adst, dct),
294 (TxType::DCT_FLIPADST, dct, flipadst),
295 (TxType::FLIPADST_DCT, flipadst, dct),
296 (TxType::V_DCT, dct, identity),
297 (TxType::H_DCT, identity, dct),
298 (TxType::ADST_ADST, adst, adst),
299 (TxType::ADST_FLIPADST, adst, flipadst),
300 (TxType::FLIPADST_ADST, flipadst, adst),
301 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
302 ],
303 [(16, 16)],
304 // 8x, 4x and 16x (minus 16x16 and 4x4)
305 [
306 (TxType::V_ADST, adst, identity),
307 (TxType::H_ADST, identity, adst),
308 (TxType::V_FLIPADST, flipadst, identity),
309 (TxType::H_FLIPADST, identity, flipadst)
310 ],
311 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8)],
312 // 4x4
313 [(TxType::WHT_WHT, wht, wht)],
314 [(4, 4)],
315 [(neon, NEON)]
316 );
317
318 cpu_function_lookup_table!(
319 INV_TXFM_HBD_FNS: [[[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
320 default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
321 [NEON]
322 );