Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/transform/inverse.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::tiling::PlaneRegionMut;
12 use crate::transform::inverse::*;
13 use crate::transform::*;
14 use crate::{Pixel, PixelType};
15
16 use crate::asm::shared::transform::inverse::*;
17
18 pub fn inverse_transform_add<T: Pixel>(
19 input: &[T::Coeff], output: &mut PlaneRegionMut<'_, T>, eob: u16,
20 tx_size: TxSize, tx_type: TxType, bd: usize, cpu: CpuFeatureLevel,
21 ) {
22 match T::type_enum() {
23 PixelType::U8 => {
24 if let Some(func) = INV_TXFM_FNS[cpu.as_index()][tx_size][tx_type] {
25 return call_inverse_func(
26 func,
27 input,
28 output,
29 eob,
30 tx_size.width(),
31 tx_size.height(),
32 bd,
33 );
34 }
35 }
36 PixelType::U16 if bd == 10 => {
37 if let Some(func) = INV_TXFM_HBD_FNS_10[cpu.as_index()][tx_size][tx_type]
38 {
39 return call_inverse_hbd_func(
40 func,
41 input,
42 output,
43 eob,
44 tx_size.width(),
45 tx_size.height(),
46 bd,
47 );
48 }
49 }
50 PixelType::U16 => {
51 if let Some(func) = INV_TXFM_HBD_FNS_12[cpu.as_index()][tx_size][tx_type]
52 {
53 return call_inverse_hbd_func(
54 func,
55 input,
56 output,
57 eob,
58 tx_size.width(),
59 tx_size.height(),
60 bd,
61 );
62 }
63 }
64 };
65
66 rust::inverse_transform_add(input, output, eob, tx_size, tx_type, bd, cpu);
67 }
68
69 macro_rules! decl_itx_fns {
70 // Takes a 2d list of tx types for W and H
71 ([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr,
72 $OPT_LOWER:ident, $OPT_UPPER:ident) => {
73 paste::item! {
74 // For each tx type, declare an function for the current WxH
75 $(
76 $(
77 extern {
78 // Note: type1 and type2 are flipped
79 fn [<rav1e_inv_txfm_add_ $TYPE2 _$TYPE1 _$W x $H _8bpc_$OPT_LOWER>](
80 dst: *mut u8, dst_stride: libc::ptrdiff_t, coeff: *mut i16,
81 eob: i32
82 );
83 }
84 )*
85 )*
86 // Create a lookup table for the tx types declared above
87 const [<INV_TXFM_FNS_$W _$H _$OPT_UPPER>]: [Option<InvTxfmFunc>; TX_TYPES_PLUS_LL] = {
88 let mut out: [Option<InvTxfmFunc>; TX_TYPES_PLUS_LL] = [None; TX_TYPES_PLUS_LL];
89 $(
90 $(
91 out[$ENUM as usize] = Some([<rav1e_inv_txfm_add_$TYPE2 _$TYPE1 _$W x $H _8bpc_$OPT_LOWER>]);
92 )*
93 )*
94 out
95 };
96 }
97 };
98 }
99
100 macro_rules! decl_itx_hbd_fns {
101 // Takes a 2d list of tx types for W and H
102 ([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr, $BPC:expr,
103 $OPT_LOWER:ident, $OPT_UPPER:ident) => {
104 paste::item! {
105 // For each tx type, declare an function for the current WxH
106 $(
107 $(
108 extern {
109 // Note: type1 and type2 are flipped
110 fn [<rav1e_inv_txfm_add_ $TYPE2 _$TYPE1 _$W x $H _ $BPC bpc_$OPT_LOWER>](
111 dst: *mut u16, dst_stride: libc::ptrdiff_t, coeff: *mut i16,
112 eob: i32, bitdepth_max: i32,
113 );
114 }
115 )*
116 )*
117 // Create a lookup table for the tx types declared above
118 const [<INV_TXFM_HBD_FNS_$W _$H _$BPC _$OPT_UPPER>]: [Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL] = {
119 #[allow(unused_mut)]
120 let mut out: [Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL] = [None; TX_TYPES_PLUS_LL];
121 $(
122 $(
123 out[$ENUM as usize] = Some([<rav1e_inv_txfm_add_$TYPE2 _$TYPE1 _$W x $H _ $BPC bpc_$OPT_LOWER>]);
124 )*
125 )*
126 out
127 };
128 }
129 };
130 }
131
132 macro_rules! create_wxh_tables {
133 // Create a lookup table for each cpu feature
134 ([$([$(($W:expr, $H:expr)),*]),*], $OPT_LOWER:ident, $OPT_UPPER:ident) => {
135 paste::item! {
136 const [<INV_TXFM_FNS_$OPT_UPPER>]: [[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] = {
137 let mut out: [[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] =
138 [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL];
139 // For each dimension, add an entry to the table
140 $(
141 $(
142 out[TxSize::[<TX_ $W X $H>] as usize] = [<INV_TXFM_FNS_$W _$H _$OPT_UPPER>];
143 )*
144 )*
145 out
146 };
147 }
148 };
149
150 // Loop through cpu features
151 ($DIMS:tt, [$(($OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
152 $(
153 create_wxh_tables!($DIMS, $OPT_LOWER, $OPT_UPPER);
154 )*
155 };
156 }
157
158 macro_rules! create_wxh_hbd_tables {
159 // Create a lookup table for each cpu feature
160 ([$([$(($W:expr, $H:expr)),*]),*], $EXT:ident, $BPC:expr, $OPT_LOWER:ident, $OPT_UPPER:ident) => {
161 paste::item! {
162 const [<INV_TXFM_HBD_FNS $EXT _$OPT_UPPER>]: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] = {
163 let mut out: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] =
164 [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL];
165 // For each dimension, add an entry to the table
166 $(
167 $(
168 out[TxSize::[<TX_ $W X $H>] as usize] = [<INV_TXFM_HBD_FNS_$W _$H _$BPC _$OPT_UPPER>];
169 )*
170 )*
171 out
172 };
173 }
174 };
175
176 // Loop through cpu features
177 ($DIMS:tt, $EXT:ident, [$(($BPC:expr, $OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
178 $(
179 create_wxh_hbd_tables!($DIMS, $EXT, $BPC, $OPT_LOWER, $OPT_UPPER);
180 )*
181 };
182 }
183
184 macro_rules! impl_itx_fns {
185 ($TYPES:tt, $W:expr, $H:expr, [$(($OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
186 $(
187 decl_itx_fns!($TYPES, $W, $H, $OPT_LOWER, $OPT_UPPER);
188 )*
189 };
190
191 // Loop over a list of dimensions
192 ($TYPES_VALID:tt, [$(($W:expr, $H:expr)),*], $OPT:tt) => {
193 $(
194 impl_itx_fns!($TYPES_VALID, $W, $H, $OPT);
195 )*
196 };
197
198 ($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
199 $TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt, $OPT:tt) => {
200 // Make 2d list of tx types for each set of dimensions. Each set of
201 // dimensions uses a superset of the previous set of tx types.
202 impl_itx_fns!([$TYPES64], $DIMS64, $OPT);
203 impl_itx_fns!([$TYPES64, $TYPES32], $DIMS32, $OPT);
204 impl_itx_fns!([$TYPES64, $TYPES32, $TYPES16], $DIMS16, $OPT);
205 impl_itx_fns!(
206 [$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84, $OPT
207 );
208 impl_itx_fns!(
209 [$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4, $OPT
210 );
211
212 // Pool all of the dimensions together to create a table for each cpu
213 // feature level.
214 create_wxh_tables!(
215 [$DIMS64, $DIMS32, $DIMS16, $DIMS84, $DIMS4], $OPT
216 );
217 };
218 }
219
220 impl_itx_fns!(
221 // 64x
222 [(TxType::DCT_DCT, dct, dct)],
223 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
224 // 32x
225 [(TxType::IDTX, identity, identity)],
226 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
227 // 16x16
228 [
229 (TxType::DCT_ADST, dct, adst),
230 (TxType::ADST_DCT, adst, dct),
231 (TxType::DCT_FLIPADST, dct, flipadst),
232 (TxType::FLIPADST_DCT, flipadst, dct),
233 (TxType::V_DCT, dct, identity),
234 (TxType::H_DCT, identity, dct),
235 (TxType::ADST_ADST, adst, adst),
236 (TxType::ADST_FLIPADST, adst, flipadst),
237 (TxType::FLIPADST_ADST, flipadst, adst),
238 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
239 ],
240 [(16, 16)],
241 // 8x, 4x and 16x (minus 16x16 and 4x4)
242 [
243 (TxType::V_ADST, adst, identity),
244 (TxType::H_ADST, identity, adst),
245 (TxType::V_FLIPADST, flipadst, identity),
246 (TxType::H_FLIPADST, identity, flipadst)
247 ],
248 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8)],
249 // 4x4
250 [(TxType::WHT_WHT, wht, wht)],
251 [(4, 4)],
252 [(avx2, AVX2)]
253 );
254
255 impl_itx_fns!(
256 // 64x
257 [(TxType::DCT_DCT, dct, dct)],
258 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
259 // 32x
260 [(TxType::IDTX, identity, identity)],
261 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
262 // 16x16
263 [
264 (TxType::DCT_ADST, dct, adst),
265 (TxType::ADST_DCT, adst, dct),
266 (TxType::DCT_FLIPADST, dct, flipadst),
267 (TxType::FLIPADST_DCT, flipadst, dct),
268 (TxType::V_DCT, dct, identity),
269 (TxType::H_DCT, identity, dct),
270 (TxType::ADST_ADST, adst, adst),
271 (TxType::ADST_FLIPADST, adst, flipadst),
272 (TxType::FLIPADST_ADST, flipadst, adst),
273 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
274 ],
275 [(16, 16)],
276 // 8x, 4x and 16x (minus 16x16)
277 [
278 (TxType::V_ADST, adst, identity),
279 (TxType::H_ADST, identity, adst),
280 (TxType::V_FLIPADST, flipadst, identity),
281 (TxType::H_FLIPADST, identity, flipadst)
282 ],
283 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
284 // 4x4
285 [],
286 [],
287 [(avx512icl, AVX512ICL), (ssse3, SSSE3)]
288 );
289
290 impl_itx_fns!(
291 // 64x
292 [],
293 [],
294 // 32x
295 [],
296 [],
297 // 16x16
298 [],
299 [],
300 // 8x, 4x and 16x (minus 16x16 and 4x4)
301 [],
302 [],
303 // 4x4
304 [(TxType::WHT_WHT, wht, wht)],
305 [(4, 4)],
306 [(sse2, SSE2)]
307 );
308
309 cpu_function_lookup_table!(
310 INV_TXFM_FNS: [[[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
311 default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
312 [SSE2, SSSE3, AVX2, AVX512ICL]
313 );
314
315 macro_rules! impl_itx_hbd_fns {
316
317 ($TYPES:tt, $W:expr, $H:expr, [$(($BPC:expr, $OPT_LOWER:ident, $OPT_UPPER:ident)),+]) => {
318 $(
319 decl_itx_hbd_fns!($TYPES, $W, $H, $BPC, $OPT_LOWER, $OPT_UPPER);
320 )*
321 };
322
323 // Loop over a list of dimensions
324 ($TYPES_VALID:tt, [$(($W:expr, $H:expr)),*], $OPT:tt) => {
325 $(
326 impl_itx_hbd_fns!($TYPES_VALID, $W, $H, $OPT);
327 )*
328 };
329
330 ($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
331 $TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt, $EXT:ident, $OPT:tt) => {
332 // Make 2d list of tx types for each set of dimensions. Each set of
333 // dimensions uses a superset of the previous set of tx types.
334 impl_itx_hbd_fns!([$TYPES64], $DIMS64, $OPT);
335 impl_itx_hbd_fns!([$TYPES64, $TYPES32], $DIMS32, $OPT);
336 impl_itx_hbd_fns!([$TYPES64, $TYPES32, $TYPES16], $DIMS16, $OPT);
337 impl_itx_hbd_fns!(
338 [$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84, $OPT
339 );
340 impl_itx_hbd_fns!(
341 [$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4, $OPT
342 );
343
344 // Pool all of the dimensions together to create a table for each cpu
345 // feature level.
346 create_wxh_hbd_tables!(
347 [$DIMS64, $DIMS32, $DIMS16, $DIMS84, $DIMS4], $EXT, $OPT
348 );
349 };
350 }
351
352 impl_itx_hbd_fns!(
353 // 64x
354 [(TxType::DCT_DCT, dct, dct)],
355 [(64, 64), (64, 32), (32, 64), (64, 16), (16, 64)],
356 // 32x
357 [(TxType::IDTX, identity, identity)],
358 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
359 // 16x16
360 [
361 (TxType::DCT_ADST, dct, adst),
362 (TxType::ADST_DCT, adst, dct),
363 (TxType::DCT_FLIPADST, dct, flipadst),
364 (TxType::FLIPADST_DCT, flipadst, dct),
365 (TxType::V_DCT, dct, identity),
366 (TxType::H_DCT, identity, dct),
367 (TxType::ADST_ADST, adst, adst),
368 (TxType::ADST_FLIPADST, adst, flipadst),
369 (TxType::FLIPADST_ADST, flipadst, adst),
370 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
371 ],
372 [(16, 16)],
373 // 8x, 4x and 16x (minus 16x16)
374 [
375 (TxType::V_ADST, adst, identity),
376 (TxType::H_ADST, identity, adst),
377 (TxType::V_FLIPADST, flipadst, identity),
378 (TxType::H_FLIPADST, identity, flipadst)
379 ],
380 [(16, 8), (8, 16), (8, 8)],
381 // 4x4
382 [],
383 [],
384 _10,
385 [(10, avx512icl, AVX512ICL)]
386 );
387
388 impl_itx_hbd_fns!(
389 // 64x
390 [(TxType::DCT_DCT, dct, dct)],
391 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
392 // 32x
393 [(TxType::IDTX, identity, identity)],
394 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
395 // 16x16
396 [
397 (TxType::DCT_ADST, dct, adst),
398 (TxType::ADST_DCT, adst, dct),
399 (TxType::DCT_FLIPADST, dct, flipadst),
400 (TxType::FLIPADST_DCT, flipadst, dct),
401 (TxType::V_DCT, dct, identity),
402 (TxType::H_DCT, identity, dct),
403 (TxType::ADST_ADST, adst, adst),
404 (TxType::ADST_FLIPADST, adst, flipadst),
405 (TxType::FLIPADST_ADST, flipadst, adst),
406 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
407 ],
408 [(16, 16)],
409 // 8x, 4x and 16x (minus 16x16)
410 [
411 (TxType::V_ADST, adst, identity),
412 (TxType::H_ADST, identity, adst),
413 (TxType::V_FLIPADST, flipadst, identity),
414 (TxType::H_FLIPADST, identity, flipadst)
415 ],
416 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
417 // 4x4
418 [],
419 [],
420 _10_,
421 [(10, avx2, AVX2)]
422 );
423
424 impl_itx_hbd_fns!(
425 // 64x
426 [(TxType::DCT_DCT, dct, dct)],
427 [(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
428 // 32x
429 [(TxType::IDTX, identity, identity)],
430 [(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
431 // 16x16
432 [
433 (TxType::DCT_ADST, dct, adst),
434 (TxType::ADST_DCT, adst, dct),
435 (TxType::DCT_FLIPADST, dct, flipadst),
436 (TxType::FLIPADST_DCT, flipadst, dct),
437 (TxType::V_DCT, dct, identity),
438 (TxType::H_DCT, identity, dct),
439 (TxType::ADST_ADST, adst, adst),
440 (TxType::ADST_FLIPADST, adst, flipadst),
441 (TxType::FLIPADST_ADST, flipadst, adst),
442 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
443 ],
444 [(16, 16)],
445 // 8x, 4x and 16x (minus 16x16)
446 [
447 (TxType::V_ADST, adst, identity),
448 (TxType::H_ADST, identity, adst),
449 (TxType::V_FLIPADST, flipadst, identity),
450 (TxType::H_FLIPADST, identity, flipadst)
451 ],
452 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
453 // 4x4
454 [],
455 [],
456 _10,
457 [(16, sse4, SSE4_1)]
458 );
459
460 impl_itx_hbd_fns!(
461 // 64x
462 [],
463 [],
464 // 32x
465 [],
466 [],
467 // 16x16
468 [],
469 [],
470 // 8x, 4x and 16x (minus 16x16 and 4x4)
471 [],
472 [],
473 // 4x4
474 [(TxType::WHT_WHT, wht, wht)],
475 [(4, 4)],
476 _16,
477 [(16, sse2, SSE2), (16, avx2, AVX2)]
478 );
479
480 const INV_TXFM_HBD_FNS_10_SSE2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
481 TxSize::TX_SIZES_ALL] = INV_TXFM_HBD_FNS_16_SSE2;
482 const INV_TXFM_HBD_FNS_12_SSE2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
483 TxSize::TX_SIZES_ALL] = INV_TXFM_HBD_FNS_16_SSE2;
484
485 const fn merge_hbd_fns(
486 a: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
487 b: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
488 ) -> [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] {
489 let mut out = b;
490 let mut tx_size = 0;
491 loop {
492 let mut tx_type = 0;
493 loop {
494 if a[tx_size][tx_type].is_some() {
495 out[tx_size][tx_type] = a[tx_size][tx_type];
496 }
497 tx_type += 1;
498 if tx_type == TX_TYPES_PLUS_LL {
499 break;
500 }
501 }
502 tx_size += 1;
503 if tx_size == TxSize::TX_SIZES_ALL {
504 break;
505 }
506 }
507 out
508 }
509
510 const INV_TXFM_HBD_FNS_10_AVX2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
511 TxSize::TX_SIZES_ALL] =
512 merge_hbd_fns(INV_TXFM_HBD_FNS_10__AVX2, INV_TXFM_HBD_FNS_16_AVX2);
513
514 cpu_function_lookup_table!(
515 INV_TXFM_HBD_FNS_10: [[[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
516 default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
517 [SSE2, SSE4_1, AVX2, AVX512ICL]
518 );
519
520 impl_itx_hbd_fns!(
521 // 32x (DCT and IDTX swapped due to incomplete DCT implementation)
522 [(TxType::IDTX, identity, identity)],
523 [(32, 32), (32, 16), (16, 32)],
524 [(TxType::DCT_DCT, dct, dct)],
525 [(32, 8), (8, 32)],
526 // 16x16
527 [
528 (TxType::DCT_ADST, dct, adst),
529 (TxType::ADST_DCT, adst, dct),
530 (TxType::DCT_FLIPADST, dct, flipadst),
531 (TxType::FLIPADST_DCT, flipadst, dct),
532 (TxType::V_DCT, dct, identity),
533 (TxType::H_DCT, identity, dct),
534 (TxType::ADST_ADST, adst, adst),
535 (TxType::ADST_FLIPADST, adst, flipadst),
536 (TxType::FLIPADST_ADST, flipadst, adst),
537 (TxType::FLIPADST_FLIPADST, flipadst, flipadst)
538 ],
539 [(16, 16)],
540 // 8x, 4x and 16x (minus 16x16)
541 [
542 (TxType::V_ADST, adst, identity),
543 (TxType::H_ADST, identity, adst),
544 (TxType::V_FLIPADST, flipadst, identity),
545 (TxType::H_FLIPADST, identity, flipadst)
546 ],
547 [(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
548 // 4x4
549 [],
550 [],
551 _12_,
552 [(12, avx2, AVX2)]
553 );
554
555 const INV_TXFM_HBD_FNS_12_AVX2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
556 TxSize::TX_SIZES_ALL] =
557 merge_hbd_fns(INV_TXFM_HBD_FNS_12__AVX2, INV_TXFM_HBD_FNS_16_AVX2);
558
559 cpu_function_lookup_table!(
560 INV_TXFM_HBD_FNS_12: [[[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
561 default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
562 [SSE2, AVX2]
563 );