Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/itx16.S

raw
1 /******************************************************************************
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2020, Martin Storsjo
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *****************************************************************************/
27
28 #include "src/arm/asm.S"
29 #include "util.S"
30
31 // The exported functions in this file have got the following signature:
32 // void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob,
33 // int bitdepth_max);
34
35 // Most of the functions use the following register layout:
36 // x0-x3 external parameters
37 // x4 function pointer to first transform
38 // x5 function pointer to second transform
39 // x6 output parameter for helper function
40 // x7 input parameter for helper function
41 // x8 input stride for helper function
42 // x9-x12 scratch variables for helper functions
43 // x13 pointer to list of eob thresholds
44 // x14 return pointer for helper function
45 // x15 return pointer for main function
46
47 // The SIMD registers most often use the following layout:
48 // v0-v1 multiplication coefficients
49 // v2-v7 scratch registers
50 // v8-v15 unused
51 // v16-v31 inputs/outputs of transforms
52
53 const idct_coeffs, align=4
54 // idct4
55 .int 2896, 2896*8*(1<<16), 1567, 3784
56 // idct8
57 .int 799, 4017, 3406, 2276
58 // idct16
59 .int 401, 4076, 3166, 2598
60 .int 1931, 3612, 3920, 1189
61 // idct32
62 .int 201, 4091, 3035, 2751
63 .int 1751, 3703, 3857, 1380
64 .int 995, 3973, 3513, 2106
65 .int 2440, 3290, 4052, 601
66 endconst
67
68 const idct64_coeffs, align=4
69 .int 101*8*(1<<16), 4095*8*(1<<16), 2967*8*(1<<16), -2824*8*(1<<16)
70 .int 1660*8*(1<<16), 3745*8*(1<<16), 3822*8*(1<<16), -1474*8*(1<<16)
71 .int 4076, 401, 4017, 799
72
73 .int 4036*8*(1<<16), -700*8*(1<<16), 2359*8*(1<<16), 3349*8*(1<<16)
74 .int 3461*8*(1<<16), -2191*8*(1<<16), 897*8*(1<<16), 3996*8*(1<<16)
75 .int -3166, -2598, -799, -4017
76
77 .int 501*8*(1<<16), 4065*8*(1<<16), 3229*8*(1<<16), -2520*8*(1<<16)
78 .int 2019*8*(1<<16), 3564*8*(1<<16), 3948*8*(1<<16), -1092*8*(1<<16)
79 .int 3612, 1931, 2276, 3406
80
81 .int 4085*8*(1<<16), -301*8*(1<<16), 2675*8*(1<<16), 3102*8*(1<<16)
82 .int 3659*8*(1<<16), -1842*8*(1<<16), 1285*8*(1<<16), 3889*8*(1<<16)
83 .int -3920, -1189, -3406, -2276
84 endconst
85
86 const iadst4_coeffs, align=4
87 .int 1321, 3803, 2482, 3344
88 endconst
89
90 const iadst8_coeffs, align=4
91 .int 4076, 401, 3612, 1931
92 .int 2598, 3166, 1189, 3920
93 // idct_coeffs
94 .int 2896, 0, 1567, 3784
95 endconst
96
97 const iadst16_coeffs, align=4
98 .int 4091, 201, 3973, 995
99 .int 3703, 1751, 3290, 2440
100 .int 2751, 3035, 2106, 3513
101 .int 1380, 3857, 601, 4052
102 endconst
103
104 .macro mul_mla d, s0, s1, c0, c1
105 mul \d\().4s, \s0\().4s, \c0
106 mla \d\().4s, \s1\().4s, \c1
107 .endm
108
109 .macro mul_mls d, s0, s1, c0, c1
110 mul \d\().4s, \s0\().4s, \c0
111 mls \d\().4s, \s1\().4s, \c1
112 .endm
113
114 .macro scale_input sz, c, r0, r1, r2 r3, r4, r5, r6, r7
115 sqrdmulh \r0\sz, \r0\sz, \c
116 sqrdmulh \r1\sz, \r1\sz, \c
117 sqrdmulh \r2\sz, \r2\sz, \c
118 sqrdmulh \r3\sz, \r3\sz, \c
119 .ifnb \r4
120 sqrdmulh \r4\sz, \r4\sz, \c
121 sqrdmulh \r5\sz, \r5\sz, \c
122 sqrdmulh \r6\sz, \r6\sz, \c
123 sqrdmulh \r7\sz, \r7\sz, \c
124 .endif
125 .endm
126
127 .macro smin_4s r0, r1, r2
128 smin \r0\().4s, \r1\().4s, \r2\().4s
129 .endm
130 .macro smax_4s r0, r1, r2
131 smax \r0\().4s, \r1\().4s, \r2\().4s
132 .endm
133
134 .macro load_add_store load, shift, addsrc, adddst, min, store, dst, src, shiftbits=4
135 .ifnb \load
136 ld1 {\load}, [\src], x1
137 .endif
138 .ifnb \shift
139 srshr \shift, \shift, #\shiftbits
140 .endif
141 .ifnb \addsrc
142 usqadd \adddst, \addsrc
143 .endif
144 .ifnb \min
145 smin \min, \min, v7.8h
146 .endif
147 .ifnb \store
148 st1 {\store}, [\dst], x1
149 .endif
150 .endm
151 .macro load_add_store_8x16 dst, src
152 mov \src, \dst
153 mvni v7.8h, #0xfc, lsl #8 // 0x3ff
154 load_add_store v2.8h, v16.8h, , , , , \dst, \src
155 load_add_store v3.8h, v17.8h, , , , , \dst, \src
156 load_add_store v4.8h, v18.8h, v16.8h, v2.8h, , , \dst, \src
157 load_add_store v5.8h, v19.8h, v17.8h, v3.8h, v2.8h, , \dst, \src
158 load_add_store v16.8h, v20.8h, v18.8h, v4.8h, v3.8h, v2.8h, \dst, \src
159 load_add_store v17.8h, v21.8h, v19.8h, v5.8h, v4.8h, v3.8h, \dst, \src
160 load_add_store v18.8h, v22.8h, v20.8h, v16.8h, v5.8h, v4.8h, \dst, \src
161 load_add_store v19.8h, v23.8h, v21.8h, v17.8h, v16.8h, v5.8h, \dst, \src
162 load_add_store v20.8h, v24.8h, v22.8h, v18.8h, v17.8h, v16.8h, \dst, \src
163 load_add_store v21.8h, v25.8h, v23.8h, v19.8h, v18.8h, v17.8h, \dst, \src
164 load_add_store v22.8h, v26.8h, v24.8h, v20.8h, v19.8h, v18.8h, \dst, \src
165 load_add_store v23.8h, v27.8h, v25.8h, v21.8h, v20.8h, v19.8h, \dst, \src
166 load_add_store v24.8h, v28.8h, v26.8h, v22.8h, v21.8h, v20.8h, \dst, \src
167 load_add_store v25.8h, v29.8h, v27.8h, v23.8h, v22.8h, v21.8h, \dst, \src
168 load_add_store v26.8h, v30.8h, v28.8h, v24.8h, v23.8h, v22.8h, \dst, \src
169 load_add_store v27.8h, v31.8h, v29.8h, v25.8h, v24.8h, v23.8h, \dst, \src
170 load_add_store , , v30.8h, v26.8h, v25.8h, v24.8h, \dst, \src
171 load_add_store , , v31.8h, v27.8h, v26.8h, v25.8h, \dst, \src
172 load_add_store , , , , v27.8h, v26.8h, \dst, \src
173 load_add_store , , , , , v27.8h, \dst, \src
174 .endm
175 .macro load_add_store_8x8 dst, src, shiftbits=4
176 mov \src, \dst
177 mvni v7.8h, #0xfc, lsl #8 // 0x3ff
178 load_add_store v2.8h, v16.8h, , , , , \dst, \src, \shiftbits
179 load_add_store v3.8h, v17.8h, , , , , \dst, \src, \shiftbits
180 load_add_store v4.8h, v18.8h, v16.8h, v2.8h, , , \dst, \src, \shiftbits
181 load_add_store v5.8h, v19.8h, v17.8h, v3.8h, v2.8h, , \dst, \src, \shiftbits
182 load_add_store v16.8h, v20.8h, v18.8h, v4.8h, v3.8h, v2.8h, \dst, \src, \shiftbits
183 load_add_store v17.8h, v21.8h, v19.8h, v5.8h, v4.8h, v3.8h, \dst, \src, \shiftbits
184 load_add_store v18.8h, v22.8h, v20.8h, v16.8h, v5.8h, v4.8h, \dst, \src, \shiftbits
185 load_add_store v19.8h, v23.8h, v21.8h, v17.8h, v16.8h, v5.8h, \dst, \src, \shiftbits
186 load_add_store , , v22.8h, v18.8h, v17.8h, v16.8h, \dst, \src, \shiftbits
187 load_add_store , , v23.8h, v19.8h, v18.8h, v17.8h, \dst, \src, \shiftbits
188 load_add_store , , , , v19.8h, v18.8h, \dst, \src, \shiftbits
189 load_add_store , , , , , v19.8h, \dst, \src, \shiftbits
190 .endm
191 .macro load_add_store_8x4 dst, src, shiftbits=4
192 mov \src, \dst
193 mvni v7.8h, #0xfc, lsl #8 // 0x3ff
194 load_add_store v2.8h, v16.8h, , , , , \dst, \src, \shiftbits
195 load_add_store v3.8h, v17.8h, , , , , \dst, \src, \shiftbits
196 load_add_store v4.8h, v18.8h, v16.8h, v2.8h, , , \dst, \src, \shiftbits
197 load_add_store v5.8h, v19.8h, v17.8h, v3.8h, v2.8h, , \dst, \src, \shiftbits
198 load_add_store , , v18.8h, v4.8h, v3.8h, v2.8h, \dst, \src, \shiftbits
199 load_add_store , , v19.8h, v5.8h, v4.8h, v3.8h, \dst, \src, \shiftbits
200 load_add_store , , , , v5.8h, v4.8h, \dst, \src, \shiftbits
201 load_add_store , , , , , v5.8h, \dst, \src, \shiftbits
202 .endm
203 .macro load_add_store4 load, inssrc, insdst, shift, addsrc, adddst, min, store, dst, src
204 .ifnb \load
205 ld1 {\load}[0], [\src], x1
206 .endif
207 .ifnb \inssrc
208 ins \insdst\().d[1], \inssrc\().d[0]
209 .endif
210 .ifnb \shift
211 srshr \shift, \shift, #4
212 .endif
213 .ifnb \load
214 ld1 {\load}[1], [\src], x1
215 .endif
216 .ifnb \addsrc
217 usqadd \adddst, \addsrc
218 .endif
219 .ifnb \store
220 st1 {\store}[0], [\dst], x1
221 .endif
222 .ifnb \min
223 smin \min, \min, v7.8h
224 .endif
225 .ifnb \store
226 st1 {\store}[1], [\dst], x1
227 .endif
228 .endm
229 .macro load_add_store_4x16 dst, src
230 mov \src, \dst
231 mvni v7.8h, #0xfc, lsl #8 // 0x3ff
232 load_add_store4 v0.d, v17, v16, , , , , , \dst, \src
233 load_add_store4 v1.d, v19, v18, , , , , , \dst, \src
234 load_add_store4 v2.d, v21, v20, v16.8h, , , , , \dst, \src
235 load_add_store4 v3.d, v23, v22, v18.8h, v16.8h, v0.8h, , , \dst, \src
236 load_add_store4 v17.d, v25, v24, v20.8h, v18.8h, v1.8h, v0.8h, , \dst, \src
237 load_add_store4 v19.d, v27, v26, v22.8h, v20.8h, v2.8h, v1.8h, v0.d, \dst, \src
238 load_add_store4 v21.d, v29, v28, v24.8h, v22.8h, v3.8h, v2.8h, v1.d, \dst, \src
239 load_add_store4 v23.d, v31, v30, v26.8h, v24.8h, v17.8h, v3.8h, v2.d, \dst, \src
240 load_add_store4 , , , v28.8h, v26.8h, v19.8h, v17.8h, v3.d, \dst, \src
241 load_add_store4 , , , v30.8h, v28.8h, v21.8h, v19.8h, v17.d, \dst, \src
242 load_add_store4 , , , , v30.8h, v23.8h, v21.8h, v19.d, \dst, \src
243 load_add_store4 , , , , , , v23.8h, v21.d, \dst, \src
244 load_add_store4 , , , , , , , v23.d, \dst, \src
245 .endm
246 .macro load_add_store_4x8 dst, src
247 mov \src, \dst
248 mvni v7.8h, #0xfc, lsl #8 // 0x3ff
249 load_add_store4 v0.d, v17, v16, , , , , , \dst, \src
250 load_add_store4 v1.d, v19, v18, , , , , , \dst, \src
251 load_add_store4 v2.d, v21, v20, v16.8h, , , , , \dst, \src
252 load_add_store4 v3.d, v23, v22, v18.8h, v16.8h, v0.8h, , , \dst, \src
253 load_add_store4 , , , v20.8h, v18.8h, v1.8h, v0.8h, , \dst, \src
254 load_add_store4 , , , v22.8h, v20.8h, v2.8h, v1.8h, v0.d, \dst, \src
255 load_add_store4 , , , , v22.8h, v3.8h, v2.8h, v1.d, \dst, \src
256 load_add_store4 , , , , , , v3.8h, v2.d, \dst, \src
257 load_add_store4 , , , , , , , v3.d, \dst, \src
258 .endm
259
260 .macro idct_dc w, h, shift
261 cbnz w3, 1f
262 movz w16, #2896*8, lsl #16
263 ld1r {v16.4s}, [x2]
264 dup v0.2s, w16
265 sqrdmulh v20.4s, v16.4s, v0.s[0]
266 str wzr, [x2]
267 .if (\w == 2*\h) || (2*\w == \h)
268 sqrdmulh v20.4s, v20.4s, v0.s[0]
269 .endif
270 .if \shift > 0
271 sqrshrn v16.4h, v20.4s, #\shift
272 sqrshrn2 v16.8h, v20.4s, #\shift
273 .else
274 sqxtn v16.4h, v20.4s
275 sqxtn2 v16.8h, v20.4s
276 .endif
277 sqrdmulh v16.8h, v16.8h, v0.h[1]
278 srshr v16.8h, v16.8h, #4
279 mov w4, #\h
280 b idct_dc_w\w\()_neon
281 1:
282 .endm
283
284 function idct_dc_w4_neon
285 mvni v31.8h, #0xfc, lsl #8 // 0x3ff
286 1:
287 ld1 {v0.d}[0], [x0], x1
288 ld1 {v0.d}[1], [x0], x1
289 ld1 {v1.d}[0], [x0], x1
290 subs w4, w4, #4
291 ld1 {v1.d}[1], [x0], x1
292 usqadd v0.8h, v16.8h
293 sub x0, x0, x1, lsl #2
294 usqadd v1.8h, v16.8h
295 smin v0.8h, v0.8h, v31.8h
296 st1 {v0.d}[0], [x0], x1
297 smin v1.8h, v1.8h, v31.8h
298 st1 {v0.d}[1], [x0], x1
299 st1 {v1.d}[0], [x0], x1
300 st1 {v1.d}[1], [x0], x1
301 b.gt 1b
302 ret
303 endfunc
304
305 function idct_dc_w8_neon
306 mvni v31.8h, #0xfc, lsl #8 // 0x3ff
307 1:
308 ld1 {v0.8h}, [x0], x1
309 subs w4, w4, #4
310 ld1 {v1.8h}, [x0], x1
311 usqadd v0.8h, v16.8h
312 ld1 {v2.8h}, [x0], x1
313 usqadd v1.8h, v16.8h
314 ld1 {v3.8h}, [x0], x1
315 usqadd v2.8h, v16.8h
316 usqadd v3.8h, v16.8h
317 sub x0, x0, x1, lsl #2
318 smin v0.8h, v0.8h, v31.8h
319 smin v1.8h, v1.8h, v31.8h
320 st1 {v0.8h}, [x0], x1
321 smin v2.8h, v2.8h, v31.8h
322 st1 {v1.8h}, [x0], x1
323 smin v3.8h, v3.8h, v31.8h
324 st1 {v2.8h}, [x0], x1
325 st1 {v3.8h}, [x0], x1
326 b.gt 1b
327 ret
328 endfunc
329
330 function idct_dc_w16_neon
331 mvni v31.8h, #0xfc, lsl #8 // 0x3ff
332 1:
333 ld1 {v0.8h, v1.8h}, [x0], x1
334 subs w4, w4, #2
335 ld1 {v2.8h, v3.8h}, [x0], x1
336 usqadd v0.8h, v16.8h
337 usqadd v1.8h, v16.8h
338 sub x0, x0, x1, lsl #1
339 usqadd v2.8h, v16.8h
340 usqadd v3.8h, v16.8h
341 smin v0.8h, v0.8h, v31.8h
342 smin v1.8h, v1.8h, v31.8h
343 smin v2.8h, v2.8h, v31.8h
344 st1 {v0.8h, v1.8h}, [x0], x1
345 smin v3.8h, v3.8h, v31.8h
346 st1 {v2.8h, v3.8h}, [x0], x1
347 b.gt 1b
348 ret
349 endfunc
350
351 function idct_dc_w32_neon
352 mvni v31.8h, #0xfc, lsl #8 // 0x3ff
353 1:
354 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0]
355 subs w4, w4, #1
356 usqadd v0.8h, v16.8h
357 usqadd v1.8h, v16.8h
358 usqadd v2.8h, v16.8h
359 usqadd v3.8h, v16.8h
360 smin v0.8h, v0.8h, v31.8h
361 smin v1.8h, v1.8h, v31.8h
362 smin v2.8h, v2.8h, v31.8h
363 smin v3.8h, v3.8h, v31.8h
364 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1
365 b.gt 1b
366 ret
367 endfunc
368
369 function idct_dc_w64_neon
370 mvni v31.8h, #0xfc, lsl #8 // 0x3ff
371 sub x1, x1, #64
372 1:
373 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
374 subs w4, w4, #1
375 usqadd v0.8h, v16.8h
376 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0]
377 usqadd v1.8h, v16.8h
378 sub x0, x0, #64
379 usqadd v2.8h, v16.8h
380 usqadd v3.8h, v16.8h
381 usqadd v4.8h, v16.8h
382 usqadd v5.8h, v16.8h
383 usqadd v6.8h, v16.8h
384 usqadd v7.8h, v16.8h
385 smin v0.8h, v0.8h, v31.8h
386 smin v1.8h, v1.8h, v31.8h
387 smin v2.8h, v2.8h, v31.8h
388 smin v3.8h, v3.8h, v31.8h
389 smin v4.8h, v4.8h, v31.8h
390 smin v5.8h, v5.8h, v31.8h
391 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
392 smin v6.8h, v6.8h, v31.8h
393 smin v7.8h, v7.8h, v31.8h
394 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1
395 b.gt 1b
396 ret
397 endfunc
398
399 .macro iwht4
400 add v16.4s, v16.4s, v17.4s
401 sub v21.4s, v18.4s, v19.4s
402 sub v20.4s, v16.4s, v21.4s
403 sshr v20.4s, v20.4s, #1
404 sub v18.4s, v20.4s, v17.4s
405 sub v17.4s, v20.4s, v19.4s
406 add v19.4s, v21.4s, v18.4s
407 sub v16.4s, v16.4s, v17.4s
408 .endm
409
410 .macro idct_4 r0, r1, r2, r3
411 mul_mla v6, \r1, \r3, v0.s[3], v0.s[2]
412 mul_mla v2, \r0, \r2, v0.s[0], v0.s[0]
413 mul_mls v4, \r1, \r3, v0.s[2], v0.s[3]
414 mul_mls v3, \r0, \r2, v0.s[0], v0.s[0]
415 srshr v6.4s, v6.4s, #12
416 srshr v2.4s, v2.4s, #12
417 srshr v7.4s, v4.4s, #12
418 srshr v3.4s, v3.4s, #12
419 sqadd \r0\().4s, v2.4s, v6.4s
420 sqsub \r3\().4s, v2.4s, v6.4s
421 sqadd \r1\().4s, v3.4s, v7.4s
422 sqsub \r2\().4s, v3.4s, v7.4s
423 .endm
424
425 function inv_dct_4s_x4_neon
426 AARCH64_VALID_CALL_TARGET
427 movrel x16, idct_coeffs
428 ld1 {v0.4s}, [x16]
429 idct_4 v16, v17, v18, v19
430 ret
431 endfunc
432
433 .macro iadst_4x4 o0, o1, o2, o3
434 movrel x16, iadst4_coeffs
435 ld1 {v0.4s}, [x16]
436
437 sub v3.4s, v16.4s, v18.4s
438 mul v4.4s, v16.4s, v0.s[0]
439 mla v4.4s, v18.4s, v0.s[1]
440 mla v4.4s, v19.4s, v0.s[2]
441 mul v7.4s, v17.4s, v0.s[3]
442 add v3.4s, v3.4s, v19.4s
443 mul v5.4s, v16.4s, v0.s[2]
444 mls v5.4s, v18.4s, v0.s[0]
445 mls v5.4s, v19.4s, v0.s[1]
446
447 add \o3\().4s, v4.4s, v5.4s
448 mul \o2\().4s, v3.4s, v0.s[3]
449 add \o0\().4s, v4.4s, v7.4s
450 add \o1\().4s, v5.4s, v7.4s
451 sub \o3\().4s, \o3\().4s, v7.4s
452
453 srshr \o0\().4s, \o0\().4s, #12
454 srshr \o2\().4s, \o2\().4s, #12
455 srshr \o1\().4s, \o1\().4s, #12
456 srshr \o3\().4s, \o3\().4s, #12
457 .endm
458
459 function inv_adst_4s_x4_neon
460 AARCH64_VALID_CALL_TARGET
461 iadst_4x4 v16, v17, v18, v19
462 ret
463 endfunc
464
465 function inv_flipadst_4s_x4_neon
466 AARCH64_VALID_CALL_TARGET
467 iadst_4x4 v19, v18, v17, v16
468 ret
469 endfunc
470
471 function inv_identity_4s_x4_neon
472 AARCH64_VALID_CALL_TARGET
473 movz w16, #(5793-4096)*8, lsl #16
474 dup v0.2s, w16
475 sqrdmulh v4.4s, v16.4s, v0.s[0]
476 sqrdmulh v5.4s, v17.4s, v0.s[0]
477 sqrdmulh v6.4s, v18.4s, v0.s[0]
478 sqrdmulh v7.4s, v19.4s, v0.s[0]
479 sqadd v16.4s, v16.4s, v4.4s
480 sqadd v17.4s, v17.4s, v5.4s
481 sqadd v18.4s, v18.4s, v6.4s
482 sqadd v19.4s, v19.4s, v7.4s
483 ret
484 endfunc
485
486 function inv_txfm_add_wht_wht_4x4_16bpc_neon, export=1
487 mov x15, x30
488 movi v30.4s, #0
489 movi v31.4s, #0
490 ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2]
491 st1 {v30.4s, v31.4s}, [x2], #32
492
493 sshr v16.4s, v16.4s, #2
494 sshr v17.4s, v17.4s, #2
495 sshr v18.4s, v18.4s, #2
496 sshr v19.4s, v19.4s, #2
497
498 iwht4
499
500 st1 {v30.4s, v31.4s}, [x2], #32
501 transpose_4x4s v16, v17, v18, v19, v20, v21, v22, v23
502
503 iwht4
504
505 ld1 {v0.d}[0], [x0], x1
506 sqxtn v16.4h, v16.4s
507 ld1 {v0.d}[1], [x0], x1
508 sqxtn2 v16.8h, v17.4s
509 ld1 {v1.d}[0], [x0], x1
510 sqxtn v18.4h, v18.4s
511 ld1 {v1.d}[1], [x0], x1
512 sqxtn2 v18.8h, v19.4s
513
514 b L(itx_4x4_end)
515 endfunc
516
517 function inv_txfm_add_4x4_neon
518 movi v30.4s, #0
519 movi v31.4s, #0
520 ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2]
521 st1 {v30.4s, v31.4s}, [x2], #32
522
523 blr x4
524
525 st1 {v30.4s, v31.4s}, [x2], #32
526 sqxtn v16.4h, v16.4s
527 sqxtn v17.4h, v17.4s
528 sqxtn v18.4h, v18.4s
529 sqxtn v19.4h, v19.4s
530 transpose_4x4h v16, v17, v18, v19, v20, v21, v22, v23
531
532 blr x5
533
534 ld1 {v0.d}[0], [x0], x1
535 ld1 {v0.d}[1], [x0], x1
536 ins v16.d[1], v17.d[0]
537 ins v18.d[1], v19.d[0]
538 ld1 {v1.d}[0], [x0], x1
539 ld1 {v1.d}[1], [x0], x1
540 srshr v16.8h, v16.8h, #4
541 srshr v18.8h, v18.8h, #4
542
543 L(itx_4x4_end):
544 mvni v31.8h, #0xfc, lsl #8 // 0x3ff
545 sub x0, x0, x1, lsl #2
546 usqadd v0.8h, v16.8h
547 usqadd v1.8h, v18.8h
548 smin v0.8h, v0.8h, v31.8h
549 st1 {v0.d}[0], [x0], x1
550 smin v1.8h, v1.8h, v31.8h
551 st1 {v0.d}[1], [x0], x1
552 st1 {v1.d}[0], [x0], x1
553 st1 {v1.d}[1], [x0], x1
554
555 ret x15
556 endfunc
557
558 .macro def_fn_4x4 txfm1, txfm2
559 function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_16bpc_neon, export=1
560 mov x15, x30
561
562 .ifc \txfm1\()_\txfm2, dct_dct
563 cbnz w3, 1f
564 movz w16, #2896*8, lsl #16
565 ld1r {v16.4s}, [x2]
566 dup v4.2s, w16
567 str wzr, [x2]
568 sqrdmulh v16.4s, v16.4s, v4.s[0]
569 ld1 {v0.d}[0], [x0], x1
570 sqxtn v20.4h, v16.4s
571 sqxtn2 v20.8h, v16.4s
572 ld1 {v0.d}[1], [x0], x1
573 sqrdmulh v20.8h, v20.8h, v4.h[1]
574 ld1 {v1.d}[0], [x0], x1
575 srshr v16.8h, v20.8h, #4
576 ld1 {v1.d}[1], [x0], x1
577 srshr v18.8h, v20.8h, #4
578 movi v30.8h, #0
579 b L(itx_4x4_end)
580 1:
581 .endif
582 adr x4, inv_\txfm1\()_4s_x4_neon
583 movrel x5, X(inv_\txfm2\()_4h_x4_neon)
584 b inv_txfm_add_4x4_neon
585 endfunc
586 .endm
587
588 def_fn_4x4 dct, dct
589 def_fn_4x4 identity, identity
590 def_fn_4x4 dct, adst
591 def_fn_4x4 dct, flipadst
592 def_fn_4x4 dct, identity
593 def_fn_4x4 adst, dct
594 def_fn_4x4 adst, adst
595 def_fn_4x4 adst, flipadst
596 def_fn_4x4 flipadst, dct
597 def_fn_4x4 flipadst, adst
598 def_fn_4x4 flipadst, flipadst
599 def_fn_4x4 identity, dct
600
601 def_fn_4x4 adst, identity
602 def_fn_4x4 flipadst, identity
603 def_fn_4x4 identity, adst
604 def_fn_4x4 identity, flipadst
605
606 .macro idct_8 r0, r1, r2, r3, r4, r5, r6, r7
607 idct_4 \r0, \r2, \r4, \r6
608
609 movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff
610 mvni v4.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
611 .irp r, \r0, \r2, \r4, \r6
612 smin_4s \r, \r, v5
613 .endr
614 .irp r, \r0, \r2, \r4, \r6
615 smax_4s \r, \r, v4
616 .endr
617
618 mul_mls v2, \r1, \r7, v1.s[0], v1.s[1] // -> t4a
619 mul_mla v3, \r1, \r7, v1.s[1], v1.s[0] // -> t7a
620 mul_mls v6, \r5, \r3, v1.s[2], v1.s[3] // -> t5a
621 mul_mla v7, \r5, \r3, v1.s[3], v1.s[2] // -> t6a
622 srshr \r1\().4s, v2.4s, #12 // t4a
623 srshr \r7\().4s, v3.4s, #12 // t7a
624 srshr \r3\().4s, v6.4s, #12 // t5a
625 srshr \r5\().4s, v7.4s, #12 // t6a
626
627 sqadd v2.4s, \r1\().4s, \r3\().4s // t4
628 sqsub \r1\().4s, \r1\().4s, \r3\().4s // t5a
629 sqadd v3.4s, \r7\().4s, \r5\().4s // t7
630 sqsub \r3\().4s, \r7\().4s, \r5\().4s // t6a
631
632 .irp r, v2, \r1, v3, \r3
633 smin_4s \r, \r, v5
634 .endr
635 .irp r, v2, \r1, v3, \r3
636 smax_4s \r, \r, v4
637 .endr
638
639 mul_mls v7, \r3, \r1, v0.s[0], v0.s[0] // -> t5
640 mul_mla v6, \r3, \r1, v0.s[0], v0.s[0] // -> t6
641 srshr v7.4s, v7.4s, #12 // t5
642 srshr v6.4s, v6.4s, #12 // t6
643
644 sqsub \r7\().4s, \r0\().4s, v3.4s // out7
645 sqadd \r0\().4s, \r0\().4s, v3.4s // out0
646 sqadd \r1\().4s, \r2\().4s, v6.4s // out1
647 sqsub v6.4s, \r2\().4s, v6.4s // out6
648 sqadd \r2\().4s, \r4\().4s, v7.4s // out2
649 sqsub \r5\().4s, \r4\().4s, v7.4s // out5
650 sqadd \r3\().4s, \r6\().4s, v2.4s // out3
651 sqsub \r4\().4s, \r6\().4s, v2.4s // out4
652 mov \r6\().16b, v6.16b // out6
653 .endm
654
655 function inv_dct_4s_x8_neon
656 AARCH64_VALID_CALL_TARGET
657 movrel x16, idct_coeffs
658 ld1 {v0.4s, v1.4s}, [x16]
659 idct_8 v16, v17, v18, v19, v20, v21, v22, v23
660 ret
661 endfunc
662
663 .macro iadst_8 o0, o1, o2, o3, o4, o5, o6, o7
664 movrel x16, iadst8_coeffs
665 ld1 {v0.4s, v1.4s}, [x16], #32
666
667 mul_mla v2, v23, v16, v0.s[0], v0.s[1]
668 mul_mls v4, v23, v16, v0.s[1], v0.s[0]
669 mul_mla v6, v21, v18, v0.s[2], v0.s[3]
670 srshr v16.4s, v2.4s, #12 // t0a
671 srshr v23.4s, v4.4s, #12 // t1a
672 mul_mls v2, v21, v18, v0.s[3], v0.s[2]
673 mul_mla v4, v19, v20, v1.s[0], v1.s[1]
674 srshr v18.4s, v6.4s, #12 // t2a
675 srshr v21.4s, v2.4s, #12 // t3a
676 mul_mls v6, v19, v20, v1.s[1], v1.s[0]
677 mul_mla v2, v17, v22, v1.s[2], v1.s[3]
678 srshr v20.4s, v4.4s, #12 // t4a
679 srshr v19.4s, v6.4s, #12 // t5a
680 mul_mls v4, v17, v22, v1.s[3], v1.s[2]
681 srshr v22.4s, v2.4s, #12 // t6a
682 srshr v17.4s, v4.4s, #12 // t7a
683
684 ld1 {v0.4s}, [x16]
685
686 movi v1.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff
687
688 sqadd v2.4s, v16.4s, v20.4s // t0
689 sqsub v3.4s, v16.4s, v20.4s // t4
690 mvni v20.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
691 sqadd v4.4s, v23.4s, v19.4s // t1
692 sqsub v5.4s, v23.4s, v19.4s // t5
693 sqadd v6.4s, v18.4s, v22.4s // t2
694 sqsub v7.4s, v18.4s, v22.4s // t6
695 sqadd v18.4s, v21.4s, v17.4s // t3
696 sqsub v19.4s, v21.4s, v17.4s // t7
697
698 .irp r, v2, v3, v4, v5, v6, v7, v18, v19
699 smin_4s \r, \r, v1
700 .endr
701 .irp r, v2, v3, v4, v5, v6, v7, v18, v19
702 smax_4s \r, \r, v20
703 .endr
704
705 mul_mla v16, v3, v5, v0.s[3], v0.s[2]
706 mul_mls v20, v3, v5, v0.s[2], v0.s[3]
707 mul_mls v22, v19, v7, v0.s[3], v0.s[2]
708
709 srshr v3.4s, v16.4s, #12 // t4a
710 srshr v5.4s, v20.4s, #12 // t5a
711
712 mul_mla v16, v19, v7, v0.s[2], v0.s[3]
713
714 srshr v7.4s, v22.4s, #12 // t6a
715 srshr v19.4s, v16.4s, #12 // t7a
716
717 sqadd \o0\().4s, v2.4s, v6.4s // out0
718 sqsub v2.4s, v2.4s, v6.4s // t2
719 sqadd \o7\().4s, v4.4s, v18.4s // out7
720 sqsub v4.4s, v4.4s, v18.4s // t3
721
722 mvni v18.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
723
724 sqadd \o1\().4s, v3.4s, v7.4s // out1
725 sqsub v3.4s, v3.4s, v7.4s // t6
726 sqadd \o6\().4s, v5.4s, v19.4s // out6
727 sqsub v5.4s, v5.4s, v19.4s // t7
728
729 // Not clipping the output registers, as they will be downshifted and
730 // narrowed afterwards anyway.
731 .irp r, v2, v4, v3, v5
732 smin_4s \r, \r, v1
733 .endr
734 .irp r, v2, v4, v3, v5
735 smax_4s \r, \r, v18
736 .endr
737
738 sqneg \o7\().4s, \o7\().4s // out7
739 sqneg \o1\().4s, \o1\().4s // out1
740
741 mul_mla v18, v2, v4, v0.s[0], v0.s[0] // -> out3 (v19 or v20)
742 mul_mls v6, v2, v4, v0.s[0], v0.s[0] // -> out4 (v20 or v19)
743 mul_mls v20, v3, v5, v0.s[0], v0.s[0] // -> out5 (v21 or v18)
744 srshr v2.4s, v18.4s, #12 // out3
745 mul_mla v18, v3, v5, v0.s[0], v0.s[0] // -> out2 (v18 or v21)
746 srshr v3.4s, v20.4s, #12 // out5
747 srshr \o2\().4s, v18.4s, #12 // out2 (v18 or v21)
748 srshr \o4\().4s, v6.4s, #12 // out4 (v20 or v19)
749
750 sqneg \o3\().4s, v2.4s // out3
751 sqneg \o5\().4s, v3.4s // out5
752 .endm
753
754 function inv_adst_4s_x8_neon
755 AARCH64_VALID_CALL_TARGET
756 iadst_8 v16, v17, v18, v19, v20, v21, v22, v23
757 ret
758 endfunc
759
760 function inv_flipadst_4s_x8_neon
761 AARCH64_VALID_CALL_TARGET
762 iadst_8 v23, v22, v21, v20, v19, v18, v17, v16
763 ret
764 endfunc
765
766 function inv_identity_4s_x8_neon
767 AARCH64_VALID_CALL_TARGET
768 sqshl v16.4s, v16.4s, #1
769 sqshl v17.4s, v17.4s, #1
770 sqshl v18.4s, v18.4s, #1
771 sqshl v19.4s, v19.4s, #1
772 sqshl v20.4s, v20.4s, #1
773 sqshl v21.4s, v21.4s, #1
774 sqshl v22.4s, v22.4s, #1
775 sqshl v23.4s, v23.4s, #1
776 ret
777 endfunc
778
779 function inv_txfm_add_8x8_neon
780 movi v31.4s, #0
781
782 cmp w3, w13
783 mov x11, #32
784 b.lt 1f
785
786 add x6, x2, #16
787 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
788 ld1 {\i}, [x6]
789 st1 {v31.4s}, [x6], x11
790 .endr
791
792 blr x4
793
794 sqrshrn v24.4h, v16.4s, #1
795 sqrshrn v25.4h, v17.4s, #1
796 sqrshrn v26.4h, v18.4s, #1
797 sqrshrn v27.4h, v19.4s, #1
798 sqrshrn2 v24.8h, v20.4s, #1
799 sqrshrn2 v25.8h, v21.4s, #1
800 sqrshrn2 v26.8h, v22.4s, #1
801 sqrshrn2 v27.8h, v23.4s, #1
802
803 transpose_4x8h v24, v25, v26, v27, v2, v3, v4, v5
804
805 b 2f
806
807 1:
808 .irp i, v24.8h, v25.8h, v26.8h, v27.8h
809 movi \i, #0
810 .endr
811
812 2:
813
814 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
815 ld1 {\i}, [x2]
816 st1 {v31.4s}, [x2], x11
817 .endr
818
819 blr x4
820
821 sqrshrn v16.4h, v16.4s, #1
822 sqrshrn v17.4h, v17.4s, #1
823 sqrshrn v18.4h, v18.4s, #1
824 sqrshrn v19.4h, v19.4s, #1
825 sqrshrn2 v16.8h, v20.4s, #1
826 sqrshrn2 v17.8h, v21.4s, #1
827 sqrshrn2 v18.8h, v22.4s, #1
828 sqrshrn2 v19.8h, v23.4s, #1
829
830 transpose_4x8h v16, v17, v18, v19, v20, v21, v22, v23
831
832 mov v20.16b, v24.16b
833 mov v21.16b, v25.16b
834 mov v22.16b, v26.16b
835 mov v23.16b, v27.16b
836
837 blr x5
838
839 load_add_store_8x8 x0, x7
840 ret x15
841 endfunc
842
843 .macro def_fn_8x8 txfm1, txfm2, eob_half
844 function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_16bpc_neon, export=1
845 mov x15, x30
846
847 .ifc \txfm1\()_\txfm2, dct_dct
848 idct_dc 8, 8, 1
849 .endif
850 movrel x5, X(inv_\txfm2\()_8h_x8_neon)
851 mov w13, #\eob_half
852 adr x4, inv_\txfm1\()_4s_x8_neon
853 b inv_txfm_add_8x8_neon
854 endfunc
855 .endm
856
857 def_fn_8x8 dct, dct, 10
858 def_fn_8x8 identity, identity, 10
859 def_fn_8x8 dct, adst, 10
860 def_fn_8x8 dct, flipadst, 10
861 def_fn_8x8 dct, identity, 4
862 def_fn_8x8 adst, dct, 10
863 def_fn_8x8 adst, adst, 10
864 def_fn_8x8 adst, flipadst, 10
865 def_fn_8x8 flipadst, dct, 10
866 def_fn_8x8 flipadst, adst, 10
867 def_fn_8x8 flipadst, flipadst, 10
868 def_fn_8x8 identity, dct, 4
869 def_fn_8x8 adst, identity, 4
870 def_fn_8x8 flipadst, identity, 4
871 def_fn_8x8 identity, adst, 4
872 def_fn_8x8 identity, flipadst, 4
873
874 function inv_txfm_add_8x4_neon
875 movi v28.4s, #0
876 movi v29.4s, #0
877 movi v30.4s, #0
878 movi v31.4s, #0
879 ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2]
880 st1 {v28.4s,v29.4s,v30.4s,v31.4s}, [x2], #64
881 movz w16, #2896*8, lsl #16
882 dup v0.2s, w16
883 ld1 {v20.4s,v21.4s,v22.4s,v23.4s}, [x2]
884 st1 {v28.4s,v29.4s,v30.4s,v31.4s}, [x2]
885
886 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
887
888 blr x4
889
890 sqxtn v16.4h, v16.4s
891 sqxtn v17.4h, v17.4s
892 sqxtn v18.4h, v18.4s
893 sqxtn v19.4h, v19.4s
894 sqxtn v20.4h, v20.4s
895 sqxtn v21.4h, v21.4s
896 sqxtn v22.4h, v22.4s
897 sqxtn v23.4h, v23.4s
898
899 transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7
900 transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7
901 ins v16.d[1], v20.d[0]
902 ins v17.d[1], v21.d[0]
903 ins v18.d[1], v22.d[0]
904 ins v19.d[1], v23.d[0]
905
906 blr x5
907
908 load_add_store_8x4 x0, x7
909 ret x15
910 endfunc
911
912 function inv_txfm_add_4x8_neon
913 movz w16, #2896*8, lsl #16
914 movi v31.4s, #0
915 dup v30.2s, w16
916
917 cmp w3, w13
918 mov x11, #32
919 b.lt 1f
920
921 add x6, x2, #16
922 .irp i, v16.4s, v17.4s, v18.4s, v19.4s
923 ld1 {\i}, [x6]
924 st1 {v31.4s}, [x6], x11
925 .endr
926 scale_input .4s, v30.s[0], v16, v17, v18, v19
927 blr x4
928 sqxtn v20.4h, v16.4s
929 sqxtn v21.4h, v17.4s
930 sqxtn v22.4h, v18.4s
931 sqxtn v23.4h, v19.4s
932 transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7
933
934 b 2f
935
936 1:
937 .irp i, v20, v21, v22, v23
938 movi \i\().4h, #0
939 .endr
940
941 2:
942
943 .irp i, v16.4s, v17.4s, v18.4s, v19.4s
944 ld1 {\i}, [x2]
945 st1 {v31.4s}, [x2], x11
946 .endr
947 scale_input .4s, v30.s[0], v16, v17, v18, v19
948 blr x4
949 sqxtn v16.4h, v16.4s
950 sqxtn v17.4h, v17.4s
951 sqxtn v18.4h, v18.4s
952 sqxtn v19.4h, v19.4s
953 transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7
954
955 blr x5
956
957 load_add_store_4x8 x0, x7
958 ret x15
959 endfunc
960
961 .macro def_fn_48 w, h, txfm1, txfm2, eob_half
962 function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1
963 mov x15, x30
964
965 .ifc \txfm1\()_\txfm2, dct_dct
966 idct_dc \w, \h, 0
967 .endif
968 adr x4, inv_\txfm1\()_4s_x\w\()_neon
969 .if \w == 4
970 mov w13, #\eob_half
971 .endif
972 movrel x5, X(inv_\txfm2\()_\w\()h_x\h\()_neon)
973 b inv_txfm_add_\w\()x\h\()_neon
974 endfunc
975 .endm
976
977 .macro def_fns_48 w, h
978 def_fn_48 \w, \h, dct, dct, 13
979 def_fn_48 \w, \h, identity, identity, 13
980 def_fn_48 \w, \h, dct, adst, 13
981 def_fn_48 \w, \h, dct, flipadst, 13
982 def_fn_48 \w, \h, dct, identity, 4
983 def_fn_48 \w, \h, adst, dct, 13
984 def_fn_48 \w, \h, adst, adst, 13
985 def_fn_48 \w, \h, adst, flipadst, 13
986 def_fn_48 \w, \h, flipadst, dct, 13
987 def_fn_48 \w, \h, flipadst, adst, 13
988 def_fn_48 \w, \h, flipadst, flipadst, 13
989 def_fn_48 \w, \h, identity, dct, 16
990 def_fn_48 \w, \h, adst, identity, 4
991 def_fn_48 \w, \h, flipadst, identity, 4
992 def_fn_48 \w, \h, identity, adst, 16
993 def_fn_48 \w, \h, identity, flipadst, 16
994 .endm
995
996 def_fns_48 4, 8
997 def_fns_48 8, 4
998
999
1000 function inv_dct_4s_x16_neon
1001 AARCH64_VALID_CALL_TARGET
1002 movrel x16, idct_coeffs
1003 ld1 {v0.4s, v1.4s}, [x16], #32
1004
1005 idct_8 v16, v18, v20, v22, v24, v26, v28, v30
1006
1007 // idct_8 leaves the row_clip_max/min constants in v5 and v4
1008 .irp r, v16, v18, v20, v22, v24, v26, v28, v30
1009 smin \r\().4s, \r\().4s, v5.4s
1010 .endr
1011 .irp r, v16, v18, v20, v22, v24, v26, v28, v30
1012 smax \r\().4s, \r\().4s, v4.4s
1013 .endr
1014
1015 ld1 {v0.4s, v1.4s}, [x16]
1016 sub x16, x16, #32
1017
1018 mul_mls v2, v17, v31, v0.s[0], v0.s[1] // -> t8a
1019 mul_mla v3, v17, v31, v0.s[1], v0.s[0] // -> t15a
1020 mul_mls v6, v25, v23, v0.s[2], v0.s[3] // -> t9a
1021 srshr v17.4s, v2.4s, #12 // t8a
1022 srshr v31.4s, v3.4s, #12 // t15a
1023 mul_mla v2, v25, v23, v0.s[3], v0.s[2] // -> t14a
1024 mul_mls v3, v21, v27, v1.s[0], v1.s[1] // -> t10a
1025 srshr v23.4s, v6.4s, #12 // t9a
1026 srshr v25.4s, v2.4s, #12 // t14a
1027 mul_mla v6, v21, v27, v1.s[1], v1.s[0] // -> t13a
1028 mul_mls v2, v29, v19, v1.s[2], v1.s[3] // -> t11a
1029 srshr v21.4s, v3.4s, #12 // t10a
1030 srshr v27.4s, v6.4s, #12 // t13a
1031 mul_mla v3, v29, v19, v1.s[3], v1.s[2] // -> t12a
1032 srshr v19.4s, v2.4s, #12 // t11a
1033 srshr v29.4s, v3.4s, #12 // t12a
1034
1035 ld1 {v0.4s}, [x16]
1036
1037 sqsub v2.4s, v17.4s, v23.4s // t9
1038 sqadd v17.4s, v17.4s, v23.4s // t8
1039 sqsub v3.4s, v31.4s, v25.4s // t14
1040 sqadd v31.4s, v31.4s, v25.4s // t15
1041 sqsub v23.4s, v19.4s, v21.4s // t10
1042 sqadd v19.4s, v19.4s, v21.4s // t11
1043 sqadd v25.4s, v29.4s, v27.4s // t12
1044 sqsub v29.4s, v29.4s, v27.4s // t13
1045
1046 .irp r, v2, v17, v3, v31, v23, v19, v25, v29
1047 smin \r\().4s, \r\().4s, v5.4s
1048 .endr
1049 .irp r, v2, v17, v3, v31, v23, v19, v25, v29
1050 smax \r\().4s, \r\().4s, v4.4s
1051 .endr
1052
1053 mul_mls v7, v3, v2, v0.s[2], v0.s[3] // -> t9a
1054 mul_mla v6, v3, v2, v0.s[3], v0.s[2] // -> t14a
1055 srshr v21.4s, v7.4s, #12 // t9a
1056 srshr v27.4s, v6.4s, #12 // t14a
1057
1058 mul_mls v7, v29, v23, v0.s[2], v0.s[3] // -> t13a
1059 mul_mla v6, v29, v23, v0.s[3], v0.s[2] // -> t10a
1060 srshr v29.4s, v7.4s, #12 // t13a
1061 neg v6.4s, v6.4s
1062 srshr v23.4s, v6.4s, #12 // t10a
1063
1064 sqsub v2.4s, v17.4s, v19.4s // t11a
1065 sqadd v17.4s, v17.4s, v19.4s // t8a
1066 sqsub v3.4s, v31.4s, v25.4s // t12a
1067 sqadd v31.4s, v31.4s, v25.4s // t15a
1068 sqadd v19.4s, v21.4s, v23.4s // t9
1069 sqsub v21.4s, v21.4s, v23.4s // t10
1070 sqsub v25.4s, v27.4s, v29.4s // t13
1071 sqadd v27.4s, v27.4s, v29.4s // t14
1072
1073 .irp r, v2, v17, v3, v31, v19, v21, v25, v27
1074 smin \r\().4s, \r\().4s, v5.4s
1075 .endr
1076 .irp r, v2, v17, v3, v31, v19, v21, v25, v27
1077 smax \r\().4s, \r\().4s, v4.4s
1078 .endr
1079
1080 mul_mls v7, v3, v2, v0.s[0], v0.s[0] // -> t11
1081 mul_mla v6, v3, v2, v0.s[0], v0.s[0] // -> t12
1082 mul_mls v2, v25, v21, v0.s[0], v0.s[0] // -> t10a
1083
1084 srshr v7.4s, v7.4s, #12 // t11
1085 srshr v6.4s, v6.4s, #12 // t12
1086 mul_mla v3, v25, v21, v0.s[0], v0.s[0] // -> t13a
1087 srshr v2.4s, v2.4s, #12 // t10a
1088 srshr v3.4s, v3.4s, #12 // t13a
1089
1090 sqadd v1.4s, v16.4s, v31.4s // out0
1091 sqsub v31.4s, v16.4s, v31.4s // out15
1092 mov v16.16b, v1.16b
1093 sqadd v23.4s, v30.4s, v17.4s // out7
1094 sqsub v1.4s, v30.4s, v17.4s // out8
1095 sqadd v17.4s, v18.4s, v27.4s // out1
1096 sqsub v30.4s, v18.4s, v27.4s // out14
1097 sqadd v18.4s, v20.4s, v3.4s // out2
1098 sqsub v29.4s, v20.4s, v3.4s // out13
1099 sqadd v3.4s, v28.4s, v19.4s // out6
1100 sqsub v25.4s, v28.4s, v19.4s // out9
1101 sqadd v19.4s, v22.4s, v6.4s // out3
1102 sqsub v28.4s, v22.4s, v6.4s // out12
1103 sqadd v20.4s, v24.4s, v7.4s // out4
1104 sqsub v27.4s, v24.4s, v7.4s // out11
1105 sqadd v21.4s, v26.4s, v2.4s // out5
1106 sqsub v26.4s, v26.4s, v2.4s // out10
1107 mov v24.16b, v1.16b
1108 mov v22.16b, v3.16b
1109
1110 ret
1111 endfunc
1112
1113 .macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15
1114 movrel x16, iadst16_coeffs
1115 ld1 {v0.4s, v1.4s}, [x16], #32
1116
1117 mul_mla v2, v31, v16, v0.s[0], v0.s[1] // -> t0
1118 mul_mls v4, v31, v16, v0.s[1], v0.s[0] // -> t1
1119 mul_mla v6, v29, v18, v0.s[2], v0.s[3] // -> t2
1120 srshr v16.4s, v2.4s, #12 // t0
1121 srshr v31.4s, v4.4s, #12 // t1
1122 mul_mls v2, v29, v18, v0.s[3], v0.s[2] // -> t3
1123 mul_mla v4, v27, v20, v1.s[0], v1.s[1] // -> t4
1124 srshr v18.4s, v6.4s, #12 // t2
1125 srshr v29.4s, v2.4s, #12 // t3
1126 mul_mls v6, v27, v20, v1.s[1], v1.s[0] // -> t5
1127 mul_mla v2, v25, v22, v1.s[2], v1.s[3] // -> t6
1128 srshr v20.4s, v4.4s, #12 // t4
1129 srshr v27.4s, v6.4s, #12 // t5
1130 mul_mls v4, v25, v22, v1.s[3], v1.s[2] // -> t7
1131 ld1 {v0.4s, v1.4s}, [x16]
1132 movrel x16, idct_coeffs
1133 mul_mla v6, v23, v24, v0.s[0], v0.s[1] // -> t8
1134 srshr v22.4s, v2.4s, #12 // t6
1135 srshr v25.4s, v4.4s, #12 // t7
1136 mul_mls v2, v23, v24, v0.s[1], v0.s[0] // -> t9
1137 mul_mla v4, v21, v26, v0.s[2], v0.s[3] // -> t10
1138 srshr v23.4s, v6.4s, #12 // t8
1139 srshr v24.4s, v2.4s, #12 // t9
1140 mul_mls v6, v21, v26, v0.s[3], v0.s[2] // -> t11
1141 mul_mla v2, v19, v28, v1.s[0], v1.s[1] // -> t12
1142 srshr v21.4s, v4.4s, #12 // t10
1143 srshr v26.4s, v6.4s, #12 // t11
1144 mul_mls v4, v19, v28, v1.s[1], v1.s[0] // -> t13
1145 mul_mla v6, v17, v30, v1.s[2], v1.s[3] // -> t14
1146 srshr v19.4s, v2.4s, #12 // t12
1147 srshr v28.4s, v4.4s, #12 // t13
1148 mul_mls v2, v17, v30, v1.s[3], v1.s[2] // -> t15
1149 srshr v17.4s, v6.4s, #12 // t14
1150 srshr v30.4s, v2.4s, #12 // t15
1151
1152 ld1 {v0.4s, v1.4s}, [x16]
1153
1154 movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff
1155 mvni v7.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
1156
1157 sqsub v2.4s, v16.4s, v23.4s // t8a
1158 sqadd v16.4s, v16.4s, v23.4s // t0a
1159 sqsub v3.4s, v31.4s, v24.4s // t9a
1160 sqadd v31.4s, v31.4s, v24.4s // t1a
1161 sqadd v23.4s, v18.4s, v21.4s // t2a
1162 sqsub v18.4s, v18.4s, v21.4s // t10a
1163 sqadd v24.4s, v29.4s, v26.4s // t3a
1164 sqsub v29.4s, v29.4s, v26.4s // t11a
1165 sqadd v21.4s, v20.4s, v19.4s // t4a
1166 sqsub v20.4s, v20.4s, v19.4s // t12a
1167 sqadd v26.4s, v27.4s, v28.4s // t5a
1168 sqsub v27.4s, v27.4s, v28.4s // t13a
1169 sqadd v19.4s, v22.4s, v17.4s // t6a
1170 sqsub v22.4s, v22.4s, v17.4s // t14a
1171 sqadd v28.4s, v25.4s, v30.4s // t7a
1172 sqsub v25.4s, v25.4s, v30.4s // t15a
1173
1174 .irp r, v2, v16, v3, v31, v23, v18, v24, v29, v21, v20, v26, v27, v19, v22, v28, v25
1175 smin_4s \r, \r, v5
1176 .endr
1177 .irp r, v2, v16, v3, v31, v23, v18, v24, v29, v21, v20, v26, v27, v19, v22, v28, v25
1178 smax_4s \r, \r, v7
1179 .endr
1180
1181 mul_mla v4, v2, v3, v1.s[1], v1.s[0] // -> t8
1182 mul_mls v6, v2, v3, v1.s[0], v1.s[1] // -> t9
1183 mul_mla v2, v18, v29, v1.s[3], v1.s[2] // -> t10
1184 srshr v17.4s, v4.4s, #12 // t8
1185 srshr v30.4s, v6.4s, #12 // t9
1186 mul_mls v4, v18, v29, v1.s[2], v1.s[3] // -> t11
1187 mul_mls v6, v27, v20, v1.s[1], v1.s[0] // -> t12
1188 srshr v18.4s, v2.4s, #12 // t10
1189 srshr v29.4s, v4.4s, #12 // t11
1190 mul_mla v2, v27, v20, v1.s[0], v1.s[1] // -> t13
1191 mul_mls v4, v25, v22, v1.s[3], v1.s[2] // -> t14
1192 srshr v27.4s, v6.4s, #12 // t12
1193 srshr v20.4s, v2.4s, #12 // t13
1194 mul_mla v6, v25, v22, v1.s[2], v1.s[3] // -> t15
1195 srshr v25.4s, v4.4s, #12 // t14
1196 srshr v22.4s, v6.4s, #12 // t15
1197
1198 sqsub v2.4s, v16.4s, v21.4s // t4
1199 sqadd v16.4s, v16.4s, v21.4s // t0
1200 sqsub v3.4s, v31.4s, v26.4s // t5
1201 sqadd v31.4s, v31.4s, v26.4s // t1
1202 sqadd v21.4s, v23.4s, v19.4s // t2
1203 sqsub v23.4s, v23.4s, v19.4s // t6
1204 sqadd v26.4s, v24.4s, v28.4s // t3
1205 sqsub v24.4s, v24.4s, v28.4s // t7
1206 sqadd v19.4s, v17.4s, v27.4s // t8a
1207 sqsub v17.4s, v17.4s, v27.4s // t12a
1208 sqadd v28.4s, v30.4s, v20.4s // t9a
1209 sqsub v30.4s, v30.4s, v20.4s // t13a
1210 sqadd v27.4s, v18.4s, v25.4s // t10a
1211 sqsub v18.4s, v18.4s, v25.4s // t14a
1212 sqadd v20.4s, v29.4s, v22.4s // t11a
1213 sqsub v29.4s, v29.4s, v22.4s // t15a
1214
1215 .irp r, v2, v16, v3, v31, v21, v23, v26, v24, v19, v17, v28, v30, v27, v18, v20, v29
1216 smin_4s \r, \r, v5
1217 .endr
1218 .irp r, v2, v16, v3, v31, v21, v23, v26, v24, v19, v17, v28, v30, v27, v18, v20, v29
1219 smax_4s \r, \r, v7
1220 .endr
1221
1222 mul_mla v4, v2, v3, v0.s[3], v0.s[2] // -> t4a
1223 mul_mls v6, v2, v3, v0.s[2], v0.s[3] // -> t5a
1224 mul_mls v2, v24, v23, v0.s[3], v0.s[2] // -> t6a
1225 srshr v22.4s, v4.4s, #12 // t4a
1226 srshr v25.4s, v6.4s, #12 // t5a
1227 mul_mla v4, v24, v23, v0.s[2], v0.s[3] // -> t7a
1228 mul_mla v6, v17, v30, v0.s[3], v0.s[2] // -> t12
1229 srshr v24.4s, v2.4s, #12 // t6a
1230 srshr v23.4s, v4.4s, #12 // t7a
1231 mul_mls v2, v17, v30, v0.s[2], v0.s[3] // -> t13
1232 mul_mls v4, v29, v18, v0.s[3], v0.s[2] // -> t14
1233 srshr v17.4s, v6.4s, #12 // t12
1234 mul_mla v6, v29, v18, v0.s[2], v0.s[3] // -> t15
1235 srshr v29.4s, v2.4s, #12 // t13
1236 srshr v30.4s, v4.4s, #12 // t14
1237 srshr v18.4s, v6.4s, #12 // t15
1238
1239 sqsub v2.4s, v16.4s, v21.4s // t2a
1240 .ifc \o0, v16
1241 sqadd \o0\().4s, v16.4s, v21.4s // out0
1242 sqsub v21.4s, v31.4s, v26.4s // t3a
1243 sqadd \o15\().4s, v31.4s, v26.4s // out15
1244 .else
1245 sqadd v4.4s, v16.4s, v21.4s // out0
1246 sqsub v21.4s, v31.4s, v26.4s // t3a
1247 sqadd \o15\().4s, v31.4s, v26.4s // out15
1248 mov \o0\().16b, v4.16b
1249 .endif
1250
1251 sqsub v3.4s, v29.4s, v18.4s // t15a
1252 sqadd \o13\().4s, v29.4s, v18.4s // out13
1253 sqadd \o2\().4s, v17.4s, v30.4s // out2
1254 sqsub v26.4s, v17.4s, v30.4s // t14a
1255
1256 sqadd \o1\().4s, v19.4s, v27.4s // out1
1257 sqsub v27.4s, v19.4s, v27.4s // t10
1258 sqadd \o14\().4s, v28.4s, v20.4s // out14
1259 sqsub v20.4s, v28.4s, v20.4s // t11
1260
1261 sqadd \o3\().4s, v22.4s, v24.4s // out3
1262 sqsub v22.4s, v22.4s, v24.4s // t6
1263 sqadd \o12\().4s, v25.4s, v23.4s // out12
1264 sqsub v23.4s, v25.4s, v23.4s // t7
1265
1266 // Not clipping the output registers, as they will be downshifted and
1267 // narrowed afterwards anyway.
1268 .irp r, v2, v21, v3, v26, v27, v20, v22, v23
1269 smin_4s \r, \r, v5
1270 .endr
1271 .irp r, v2, v21, v3, v26, v27, v20, v22, v23
1272 smax_4s \r, \r, v7
1273 .endr
1274
1275 sqneg \o15\().4s, \o15\().4s // out15
1276 sqneg \o13\().4s, \o13\().4s // out13
1277 sqneg \o1\().4s, \o1\().4s // out1
1278 sqneg \o3\().4s, \o3\().4s // out3
1279
1280 mul_mls v24, v2, v21, v0.s[0], v0.s[0] // -> out8 (v24 or v23)
1281 mul_mla v4, v2, v21, v0.s[0], v0.s[0] // -> out7 (v23 or v24)
1282 mul_mla v6, v26, v3, v0.s[0], v0.s[0] // -> out5 (v21 or v26)
1283
1284 srshr v24.4s, v24.4s, #12 // out8
1285 srshr v4.4s, v4.4s, #12 // out7
1286 srshr v5.4s, v6.4s, #12 // out5
1287 mul_mls v6, v26, v3, v0.s[0], v0.s[0] // -> out10 (v26 or v21)
1288 mul_mla v2, v22, v23, v0.s[0], v0.s[0] // -> out4 (v20 or v27)
1289 srshr v26.4s, v6.4s, #12 // out10
1290
1291 mul_mls v6, v22, v23, v0.s[0], v0.s[0] // -> out11 (v27 or v20)
1292 mul_mla v22, v27, v20, v0.s[0], v0.s[0] // -> out6 (v22 or v25)
1293 mul_mls v21, v27, v20, v0.s[0], v0.s[0] // -> out9 (v25 or v22)
1294
1295 srshr \o4\().4s, v2.4s, #12 // out4
1296 srshr v6.4s, v6.4s, #12 // out11
1297 srshr v7.4s, v21.4s, #12 // out9
1298 srshr \o6\().4s, v22.4s, #12 // out6
1299
1300 .ifc \o8, v23
1301 mov \o8\().16b, v24.16b
1302 mov \o10\().16b, v26.16b
1303 .endif
1304
1305 sqneg \o7\().4s, v4.4s // out7
1306 sqneg \o5\().4s, v5.4s // out5
1307 sqneg \o11\().4s, v6.4s // out11
1308 sqneg \o9\().4s, v7.4s // out9
1309 .endm
1310
1311 function inv_adst_4s_x16_neon
1312 AARCH64_VALID_CALL_TARGET
1313 iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31
1314 ret
1315 endfunc
1316
1317 function inv_flipadst_4s_x16_neon
1318 AARCH64_VALID_CALL_TARGET
1319 iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16
1320 ret
1321 endfunc
1322
1323 function inv_identity_4s_x16_neon
1324 AARCH64_VALID_CALL_TARGET
1325 movz w16, #2*(5793-4096)*8, lsl #16
1326 dup v0.2s, w16
1327 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
1328 sqrdmulh v2.4s, v\i\().4s, v0.s[0]
1329 sqadd v\i\().4s, v\i\().4s, v\i\().4s
1330 sqadd v\i\().4s, v\i\().4s, v2.4s
1331 .endr
1332 ret
1333 endfunc
1334
1335 .macro identity_4x16_shift1 c
1336 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
1337 sqrdmulh v3.4s, \i, \c
1338 srshr v3.4s, v3.4s, #1
1339 sqadd \i, \i, v3.4s
1340 .endr
1341 .endm
1342
1343 .macro identity_4x16 c
1344 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
1345 sqrdmulh v3.4s, \i, \c
1346 sqadd \i, \i, \i
1347 sqadd \i, \i, v3.4s
1348 .endr
1349 .endm
1350
1351 .macro def_horz_16 scale=0, shift=2, suffix
1352 function inv_txfm_horz\suffix\()_16x4_neon
1353 mov x14, x30
1354 movi v7.4s, #0
1355 .if \scale
1356 movz w16, #2896*8, lsl #16
1357 dup v0.2s, w16
1358 .endif
1359 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
1360 ld1 {\i}, [x7]
1361 st1 {v7.4s}, [x7], x8
1362 .endr
1363 .if \scale
1364 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1365 scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31
1366 .endif
1367 blr x4
1368 sqrshrn v16.4h, v16.4s, #\shift
1369 sqrshrn v17.4h, v17.4s, #\shift
1370 sqrshrn v18.4h, v18.4s, #\shift
1371 sqrshrn v19.4h, v19.4s, #\shift
1372 sqrshrn2 v16.8h, v20.4s, #\shift
1373 sqrshrn2 v17.8h, v21.4s, #\shift
1374 sqrshrn2 v18.8h, v22.4s, #\shift
1375 sqrshrn2 v19.8h, v23.4s, #\shift
1376 sqrshrn v20.4h, v24.4s, #\shift
1377 sqrshrn v21.4h, v25.4s, #\shift
1378 sqrshrn v22.4h, v26.4s, #\shift
1379 sqrshrn v23.4h, v27.4s, #\shift
1380 sqrshrn2 v20.8h, v28.4s, #\shift
1381 sqrshrn2 v21.8h, v29.4s, #\shift
1382 sqrshrn2 v22.8h, v30.4s, #\shift
1383 sqrshrn2 v23.8h, v31.4s, #\shift
1384 transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7
1385 transpose_4x8h v20, v21, v22, v23, v4, v5, v6, v7
1386
1387 .irp i, v16.8h, v20.8h, v17.8h, v21.8h, v18.8h, v22.8h, v19.8h, v23.8h
1388 st1 {\i}, [x6], #16
1389 .endr
1390
1391 ret x14
1392 endfunc
1393 .endm
1394
1395 def_horz_16 scale=0, shift=2
1396 def_horz_16 scale=1, shift=1, suffix=_scale
1397
1398 function inv_txfm_add_vert_8x16_neon
1399 mov x14, x30
1400 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
1401 ld1 {v\i\().8h}, [x7], x8
1402 .endr
1403 blr x5
1404 load_add_store_8x16 x6, x7
1405 ret x14
1406 endfunc
1407
1408 function inv_txfm_add_16x16_neon
1409 mov x15, x30
1410 sub sp, sp, #512
1411 ldrh w12, [x13], #2
1412 .irp i, 0, 4, 8, 12
1413 add x6, sp, #(\i*16*2)
1414 .if \i > 0
1415 mov w8, #(16 - \i)
1416 cmp w3, w12
1417 b.lt 1f
1418 .if \i < 12
1419 ldrh w12, [x13], #2
1420 .endif
1421 .endif
1422 add x7, x2, #(\i*4)
1423 mov x8, #16*4
1424 bl inv_txfm_horz_16x4_neon
1425 .endr
1426 b 3f
1427 1:
1428 movi v4.8h, #0
1429 movi v5.8h, #0
1430 movi v6.8h, #0
1431 movi v7.8h, #0
1432 2:
1433 subs w8, w8, #4
1434 .rept 2
1435 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
1436 .endr
1437 b.gt 2b
1438 3:
1439 .irp i, 0, 8
1440 add x6, x0, #(\i*2)
1441 add x7, sp, #(\i*2)
1442 mov x8, #32
1443 bl inv_txfm_add_vert_8x16_neon
1444 .endr
1445
1446 add sp, sp, #512
1447 ret x15
1448 endfunc
1449
1450 const eob_16x16
1451 .short 10, 36, 78, 256
1452 endconst
1453
1454 const eob_16x16_identity
1455 .short 4, 8, 12, 256
1456 endconst
1457
1458 .macro def_fn_16x16 txfm1, txfm2
1459 function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_16bpc_neon, export=1
1460 .ifc \txfm1\()_\txfm2, dct_dct
1461 idct_dc 16, 16, 2
1462 .endif
1463 adr x4, inv_\txfm1\()_4s_x16_neon
1464 movrel x5, X(inv_\txfm2\()_8h_x16_neon)
1465 .ifc \txfm1, identity
1466 .ifc \txfm2, identity
1467 movrel x13, eob_16x16
1468 .else
1469 movrel x13, eob_16x16_identity
1470 .endif
1471 .else
1472 .ifc \txfm2, identity
1473 movrel x13, eob_16x16_identity
1474 .else
1475 movrel x13, eob_16x16
1476 .endif
1477 .endif
1478 b inv_txfm_add_16x16_neon
1479 endfunc
1480 .endm
1481
1482 def_fn_16x16 dct, dct
1483 def_fn_16x16 identity, identity
1484 def_fn_16x16 dct, adst
1485 def_fn_16x16 dct, flipadst
1486 def_fn_16x16 dct, identity
1487 def_fn_16x16 adst, dct
1488 def_fn_16x16 adst, adst
1489 def_fn_16x16 adst, flipadst
1490 def_fn_16x16 flipadst, dct
1491 def_fn_16x16 flipadst, adst
1492 def_fn_16x16 flipadst, flipadst
1493 def_fn_16x16 identity, dct
1494
1495 function inv_txfm_add_16x4_neon
1496 mov x15, x30
1497 movi v4.4s, #0
1498
1499 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
1500 ld1 {\i}, [x2]
1501 st1 {v4.4s}, [x2], #16
1502 .endr
1503
1504 blr x4
1505
1506 sqrshrn v16.4h, v16.4s, #1
1507 sqrshrn v17.4h, v17.4s, #1
1508 sqrshrn v18.4h, v18.4s, #1
1509 sqrshrn v19.4h, v19.4s, #1
1510 sqrshrn2 v16.8h, v20.4s, #1
1511 sqrshrn2 v17.8h, v21.4s, #1
1512 sqrshrn2 v18.8h, v22.4s, #1
1513 sqrshrn2 v19.8h, v23.4s, #1
1514 transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5
1515 blr x5
1516 mov x6, x0
1517 load_add_store_8x4 x6, x7
1518
1519 sqrshrn v16.4h, v24.4s, #1
1520 sqrshrn v17.4h, v25.4s, #1
1521 sqrshrn v18.4h, v26.4s, #1
1522 sqrshrn v19.4h, v27.4s, #1
1523 sqrshrn2 v16.8h, v28.4s, #1
1524 sqrshrn2 v17.8h, v29.4s, #1
1525 sqrshrn2 v18.8h, v30.4s, #1
1526 sqrshrn2 v19.8h, v31.4s, #1
1527 transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5
1528 blr x5
1529 add x6, x0, #16
1530 load_add_store_8x4 x6, x7
1531
1532 ret x15
1533 endfunc
1534
1535 function inv_txfm_add_4x16_neon
1536 ldrh w12, [x13, #4]
1537 mov x15, x30
1538
1539 mov x11, #64
1540
1541 cmp w3, w12
1542 ldrh w12, [x13, #2]
1543 b.lt 1f
1544
1545 add x6, x2, #48
1546 movi v2.4s, #0
1547 .irp i, v16.4s, v17.4s, v18.4s, v19.4s
1548 ld1 {\i}, [x6]
1549 st1 {v2.4s}, [x6], x11
1550 .endr
1551 blr x4
1552 sqrshrn v28.4h, v16.4s, #1
1553 sqrshrn v29.4h, v17.4s, #1
1554 sqrshrn v30.4h, v18.4s, #1
1555 sqrshrn v31.4h, v19.4s, #1
1556 transpose_4x4h v28, v29, v30, v31, v4, v5, v6, v7
1557
1558 b 2f
1559 1:
1560 .irp i, v28.4h, v29.4h, v30.4h, v31.4h
1561 movi \i, #0
1562 .endr
1563 2:
1564 cmp w3, w12
1565 ldrh w12, [x13, #0]
1566 b.lt 1f
1567
1568 add x6, x2, #32
1569 movi v2.4s, #0
1570 .irp i, v16.4s, v17.4s, v18.4s, v19.4s
1571 ld1 {\i}, [x6]
1572 st1 {v2.4s}, [x6], x11
1573 .endr
1574 blr x4
1575 sqrshrn v24.4h, v16.4s, #1
1576 sqrshrn v25.4h, v17.4s, #1
1577 sqrshrn v26.4h, v18.4s, #1
1578 sqrshrn v27.4h, v19.4s, #1
1579 transpose_4x4h v24, v25, v26, v27, v4, v5, v6, v7
1580
1581 b 2f
1582 1:
1583 .irp i, v24.4h, v25.4h, v26.4h, v27.4h
1584 movi \i, #0
1585 .endr
1586 2:
1587 cmp w3, w12
1588 b.lt 1f
1589
1590 add x6, x2, #16
1591 movi v2.4s, #0
1592 .irp i, v16.4s, v17.4s, v18.4s, v19.4s
1593 ld1 {\i}, [x6]
1594 st1 {v2.4s}, [x6], x11
1595 .endr
1596 blr x4
1597 sqrshrn v20.4h, v16.4s, #1
1598 sqrshrn v21.4h, v17.4s, #1
1599 sqrshrn v22.4h, v18.4s, #1
1600 sqrshrn v23.4h, v19.4s, #1
1601 transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7
1602
1603 b 2f
1604 1:
1605 .irp i, v20.4h, v21.4h, v22.4h, v23.4h
1606 movi \i, #0
1607 .endr
1608 2:
1609
1610 movi v2.4s, #0
1611 .irp i, v16.4s, v17.4s, v18.4s, v19.4s
1612 ld1 {\i}, [x2]
1613 st1 {v2.4s}, [x2], x11
1614 .endr
1615 blr x4
1616 sqrshrn v16.4h, v16.4s, #1
1617 sqrshrn v17.4h, v17.4s, #1
1618 sqrshrn v18.4h, v18.4s, #1
1619 sqrshrn v19.4h, v19.4s, #1
1620 transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7
1621
1622 blr x5
1623
1624 load_add_store_4x16 x0, x6
1625
1626 ret x15
1627 endfunc
1628
1629 const eob_4x16
1630 .short 13, 29, 45, 64
1631 endconst
1632
1633 const eob_4x16_identity1
1634 .short 16, 32, 48, 64
1635 endconst
1636
1637 const eob_4x16_identity2
1638 .short 4, 8, 12, 64
1639 endconst
1640
1641 .macro def_fn_416 w, h, txfm1, txfm2
1642 function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1
1643 .ifc \txfm1\()_\txfm2, dct_dct
1644 idct_dc \w, \h, 1
1645 .endif
1646 .if \w == 4
1647 adr x4, inv_\txfm1\()_4s_x\w\()_neon
1648 movrel x5, X(inv_\txfm2\()_4h_x\h\()_neon)
1649 .ifc \txfm1, identity
1650 .ifc \txfm2, identity
1651 movrel x13, eob_4x16
1652 .else
1653 movrel x13, eob_4x16_identity1
1654 .endif
1655 .else
1656 .ifc \txfm2, identity
1657 movrel x13, eob_4x16_identity2
1658 .else
1659 movrel x13, eob_4x16
1660 .endif
1661 .endif
1662 .else
1663 adr x4, inv_\txfm1\()_4s_x\w\()_neon
1664 movrel x5, X(inv_\txfm2\()_8h_x\h\()_neon)
1665 .endif
1666 b inv_txfm_add_\w\()x\h\()_neon
1667 endfunc
1668 .endm
1669
1670 .macro def_fns_416 w, h
1671 def_fn_416 \w, \h, dct, dct
1672 def_fn_416 \w, \h, identity, identity
1673 def_fn_416 \w, \h, dct, adst
1674 def_fn_416 \w, \h, dct, flipadst
1675 def_fn_416 \w, \h, dct, identity
1676 def_fn_416 \w, \h, adst, dct
1677 def_fn_416 \w, \h, adst, adst
1678 def_fn_416 \w, \h, adst, flipadst
1679 def_fn_416 \w, \h, flipadst, dct
1680 def_fn_416 \w, \h, flipadst, adst
1681 def_fn_416 \w, \h, flipadst, flipadst
1682 def_fn_416 \w, \h, identity, dct
1683 def_fn_416 \w, \h, adst, identity
1684 def_fn_416 \w, \h, flipadst, identity
1685 def_fn_416 \w, \h, identity, adst
1686 def_fn_416 \w, \h, identity, flipadst
1687 .endm
1688
1689 def_fns_416 4, 16
1690 def_fns_416 16, 4
1691
1692
1693 function inv_txfm_add_16x8_neon
1694 mov x15, x30
1695 stp d8, d9, [sp, #-0x40]!
1696 stp d10, d11, [sp, #0x10]
1697 stp d12, d13, [sp, #0x20]
1698 stp d14, d15, [sp, #0x30]
1699
1700 cmp w3, w13
1701 mov x11, #32
1702 b.lt 1f
1703
1704 movi v4.4s, #0
1705 movz w16, #2896*8, lsl #16
1706 dup v0.2s, w16
1707
1708 add x6, x2, #16
1709 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
1710 ld1 {\i}, [x6]
1711 st1 {v4.4s}, [x6], x11
1712 .endr
1713
1714 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1715 scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31
1716 blr x4
1717
1718 sqrshrn v8.4h, v16.4s, #1
1719 sqrshrn v9.4h, v17.4s, #1
1720 sqrshrn v10.4h, v18.4s, #1
1721 sqrshrn v11.4h, v19.4s, #1
1722 sqrshrn2 v8.8h, v20.4s, #1
1723 sqrshrn2 v9.8h, v21.4s, #1
1724 sqrshrn2 v10.8h, v22.4s, #1
1725 sqrshrn2 v11.8h, v23.4s, #1
1726 sqrshrn v12.4h, v24.4s, #1
1727 sqrshrn v13.4h, v25.4s, #1
1728 sqrshrn v14.4h, v26.4s, #1
1729 sqrshrn v15.4h, v27.4s, #1
1730 sqrshrn2 v12.8h, v28.4s, #1
1731 sqrshrn2 v13.8h, v29.4s, #1
1732 sqrshrn2 v14.8h, v30.4s, #1
1733 sqrshrn2 v15.8h, v31.4s, #1
1734
1735 transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5
1736 transpose_4x8h v12, v13, v14, v15, v2, v3, v4, v5
1737
1738 b 2f
1739 1:
1740 .irp i, v8.8h, v9.8h, v10.8h, v11.8h, v12.8h, v13.8h, v14.8h, v15.8h
1741 movi \i, #0
1742 .endr
1743 2:
1744 movz w16, #2896*8, lsl #16
1745 dup v0.2s, w16
1746
1747 movi v4.4s, #0
1748 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
1749 ld1 {\i}, [x2]
1750 st1 {v4.4s}, [x2], x11
1751 .endr
1752
1753 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1754 scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31
1755 blr x4
1756
1757 sqrshrn v16.4h, v16.4s, #1
1758 sqrshrn v17.4h, v17.4s, #1
1759 sqrshrn v18.4h, v18.4s, #1
1760 sqrshrn v19.4h, v19.4s, #1
1761 sqrshrn2 v16.8h, v20.4s, #1
1762 sqrshrn2 v17.8h, v21.4s, #1
1763 sqrshrn2 v18.8h, v22.4s, #1
1764 sqrshrn2 v19.8h, v23.4s, #1
1765
1766 mov v20.16b, v8.16b
1767 mov v21.16b, v9.16b
1768 mov v22.16b, v10.16b
1769 mov v23.16b, v11.16b
1770
1771 transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5
1772
1773 sqrshrn v8.4h, v24.4s, #1
1774 sqrshrn v9.4h, v25.4s, #1
1775 sqrshrn v10.4h, v26.4s, #1
1776 sqrshrn v11.4h, v27.4s, #1
1777 sqrshrn2 v8.8h, v28.4s, #1
1778 sqrshrn2 v9.8h, v29.4s, #1
1779 sqrshrn2 v10.8h, v30.4s, #1
1780 sqrshrn2 v11.8h, v31.4s, #1
1781
1782 transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5
1783
1784 blr x5
1785
1786 mov x6, x0
1787 load_add_store_8x8 x6, x7
1788
1789 mov v16.16b, v8.16b
1790 mov v17.16b, v9.16b
1791 mov v18.16b, v10.16b
1792 mov v19.16b, v11.16b
1793 mov v20.16b, v12.16b
1794 mov v21.16b, v13.16b
1795 mov v22.16b, v14.16b
1796 mov v23.16b, v15.16b
1797
1798 blr x5
1799
1800 add x0, x0, #16
1801 load_add_store_8x8 x0, x7
1802
1803 ldp d14, d15, [sp, #0x30]
1804 ldp d12, d13, [sp, #0x20]
1805 ldp d10, d11, [sp, #0x10]
1806 ldp d8, d9, [sp], 0x40
1807 ret x15
1808 endfunc
1809
1810 function inv_txfm_add_8x16_neon
1811 mov x15, x30
1812 stp d8, d9, [sp, #-0x20]!
1813 stp d10, d11, [sp, #0x10]
1814 ldrh w12, [x13, #4]
1815
1816 mov x11, #64
1817
1818 cmp w3, w12
1819 ldrh w12, [x13, #2]
1820 b.lt 1f
1821
1822 add x6, x2, #48
1823 movi v4.4s, #0
1824 movz w16, #2896*8, lsl #16
1825 dup v0.2s, w16
1826 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
1827 ld1 {\i}, [x6]
1828 st1 {v4.4s}, [x6], x11
1829 .endr
1830 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1831 blr x4
1832
1833 sqrshrn v28.4h, v16.4s, #1
1834 sqrshrn v29.4h, v17.4s, #1
1835 sqrshrn v30.4h, v18.4s, #1
1836 sqrshrn v31.4h, v19.4s, #1
1837 sqrshrn2 v28.8h, v20.4s, #1
1838 sqrshrn2 v29.8h, v21.4s, #1
1839 sqrshrn2 v30.8h, v22.4s, #1
1840 sqrshrn2 v31.8h, v23.4s, #1
1841 transpose_4x8h v28, v29, v30, v31, v2, v3, v4, v5
1842
1843 b 2f
1844
1845 1:
1846 .irp i, v28.8h, v29.8h, v30.8h, v31.8h
1847 movi \i, #0
1848 .endr
1849
1850 2:
1851 cmp w3, w12
1852 ldrh w12, [x13, #0]
1853 b.lt 1f
1854
1855 add x6, x2, #32
1856 movi v4.4s, #0
1857 movz w16, #2896*8, lsl #16
1858 dup v0.2s, w16
1859 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
1860 ld1 {\i}, [x6]
1861 st1 {v4.4s}, [x6], x11
1862 .endr
1863 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1864 blr x4
1865
1866 sqrshrn v24.4h, v16.4s, #1
1867 sqrshrn v25.4h, v17.4s, #1
1868 sqrshrn v26.4h, v18.4s, #1
1869 sqrshrn v27.4h, v19.4s, #1
1870 sqrshrn2 v24.8h, v20.4s, #1
1871 sqrshrn2 v25.8h, v21.4s, #1
1872 sqrshrn2 v26.8h, v22.4s, #1
1873 sqrshrn2 v27.8h, v23.4s, #1
1874 transpose_4x8h v24, v25, v26, v27, v2, v3, v4, v5
1875
1876 b 2f
1877
1878 1:
1879 .irp i, v24.8h, v25.8h, v26.8h, v27.8h
1880 movi \i, #0
1881 .endr
1882
1883 2:
1884 cmp w3, w12
1885 b.lt 1f
1886
1887 add x6, x2, #16
1888 movi v4.4s, #0
1889 movz w16, #2896*8, lsl #16
1890 dup v0.2s, w16
1891 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
1892 ld1 {\i}, [x6]
1893 st1 {v4.4s}, [x6], x11
1894 .endr
1895 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1896 blr x4
1897
1898 sqrshrn v8.4h, v16.4s, #1
1899 sqrshrn v9.4h, v17.4s, #1
1900 sqrshrn v10.4h, v18.4s, #1
1901 sqrshrn v11.4h, v19.4s, #1
1902 sqrshrn2 v8.8h, v20.4s, #1
1903 sqrshrn2 v9.8h, v21.4s, #1
1904 sqrshrn2 v10.8h, v22.4s, #1
1905 sqrshrn2 v11.8h, v23.4s, #1
1906 transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5
1907
1908 b 2f
1909
1910 1:
1911 .irp i, v8.8h, v9.8h, v10.8h, v11.8h
1912 movi \i, #0
1913 .endr
1914
1915 2:
1916 movi v4.4s, #0
1917 movz w16, #2896*8, lsl #16
1918 dup v0.2s, w16
1919 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
1920 ld1 {\i}, [x2]
1921 st1 {v4.4s}, [x2], x11
1922 .endr
1923 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
1924 blr x4
1925
1926 sqrshrn v16.4h, v16.4s, #1
1927 sqrshrn v17.4h, v17.4s, #1
1928 sqrshrn v18.4h, v18.4s, #1
1929 sqrshrn v19.4h, v19.4s, #1
1930 sqrshrn2 v16.8h, v20.4s, #1
1931 sqrshrn2 v17.8h, v21.4s, #1
1932 sqrshrn2 v18.8h, v22.4s, #1
1933 sqrshrn2 v19.8h, v23.4s, #1
1934 transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5
1935
1936 mov v20.16b, v8.16b
1937 mov v21.16b, v9.16b
1938 mov v22.16b, v10.16b
1939 mov v23.16b, v11.16b
1940
1941 blr x5
1942
1943 load_add_store_8x16 x0, x6
1944
1945 ldp d10, d11, [sp, #0x10]
1946 ldp d8, d9, [sp], 0x20
1947
1948 ret x15
1949 endfunc
1950
1951 const eob_8x16
1952 .short 10, 43, 75, 128
1953 endconst
1954
1955 const eob_8x16_identity1
1956 .short 4, 64, 96, 128
1957 endconst
1958
1959 const eob_8x16_identity2
1960 .short 4, 8, 12, 128
1961 endconst
1962
1963 .macro def_fn_816 w, h, txfm1, txfm2
1964 function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1
1965 .ifc \txfm1\()_\txfm2, dct_dct
1966 idct_dc \w, \h, 1
1967 .endif
1968 adr x4, inv_\txfm1\()_4s_x\w\()_neon
1969 movrel x5, X(inv_\txfm2\()_8h_x\h\()_neon)
1970 .ifc \txfm1, identity
1971 .ifc \txfm2, identity
1972 movrel x13, eob_8x16
1973 .else
1974 movrel x13, eob_8x16_identity1
1975 .endif
1976 .else
1977 .ifc \txfm2, identity
1978 movrel x13, eob_8x16_identity2
1979 .else
1980 movrel x13, eob_8x16
1981 .endif
1982 .endif
1983 .if \h == 8
1984 ldrh w13, [x13]
1985 .endif
1986 b inv_txfm_add_\w\()x\h\()_neon
1987 endfunc
1988 .endm
1989
1990 .macro def_fns_816 w, h
1991 def_fn_816 \w, \h, dct, dct
1992 def_fn_816 \w, \h, identity, identity
1993 def_fn_816 \w, \h, dct, adst
1994 def_fn_816 \w, \h, dct, flipadst
1995 def_fn_816 \w, \h, dct, identity
1996 def_fn_816 \w, \h, adst, dct
1997 def_fn_816 \w, \h, adst, adst
1998 def_fn_816 \w, \h, adst, flipadst
1999 def_fn_816 \w, \h, flipadst, dct
2000 def_fn_816 \w, \h, flipadst, adst
2001 def_fn_816 \w, \h, flipadst, flipadst
2002 def_fn_816 \w, \h, identity, dct
2003 def_fn_816 \w, \h, adst, identity
2004 def_fn_816 \w, \h, flipadst, identity
2005 def_fn_816 \w, \h, identity, adst
2006 def_fn_816 \w, \h, identity, flipadst
2007 .endm
2008
2009 def_fns_816 8, 16
2010 def_fns_816 16, 8
2011
2012 function inv_dct32_odd_4s_x16_neon
2013 movrel x16, idct_coeffs, 4*16
2014 ld1 {v0.4s, v1.4s}, [x16], #32
2015
2016 mul_mls v2, v16, v31, v0.s[0], v0.s[1] // -> t16a
2017 mul_mla v4, v16, v31, v0.s[1], v0.s[0] // -> t31a
2018 mul_mls v6, v24, v23, v0.s[2], v0.s[3] // -> t17a
2019 srshr v16.4s, v2.4s, #12 // t16a
2020 srshr v31.4s, v4.4s, #12 // t31a
2021 mul_mla v2, v24, v23, v0.s[3], v0.s[2] // -> t30a
2022 mul_mls v4, v20, v27, v1.s[0], v1.s[1] // -> t18a
2023 srshr v24.4s, v6.4s, #12 // t17a
2024 srshr v23.4s, v2.4s, #12 // t30a
2025 mul_mla v6, v20, v27, v1.s[1], v1.s[0] // -> t29a
2026 mul_mls v2, v28, v19, v1.s[2], v1.s[3] // -> t19a
2027 srshr v20.4s, v4.4s, #12 // t18a
2028 srshr v27.4s, v6.4s, #12 // t29a
2029 mul_mla v4, v28, v19, v1.s[3], v1.s[2] // -> t28a
2030 ld1 {v0.4s, v1.4s}, [x16]
2031 sub x16, x16, #4*24
2032 mul_mls v6, v18, v29, v0.s[0], v0.s[1] // -> t20a
2033 srshr v28.4s, v2.4s, #12 // t19a
2034 srshr v19.4s, v4.4s, #12 // t28a
2035 mul_mla v2, v18, v29, v0.s[1], v0.s[0] // -> t27a
2036 mul_mls v4, v26, v21, v0.s[2], v0.s[3] // -> t21a
2037 srshr v18.4s, v6.4s, #12 // t20a
2038 srshr v29.4s, v2.4s, #12 // t27a
2039 mul_mla v6, v26, v21, v0.s[3], v0.s[2] // -> t26a
2040 mul_mls v2, v22, v25, v1.s[0], v1.s[1] // -> t22a
2041 srshr v26.4s, v4.4s, #12 // t21a
2042 srshr v21.4s, v6.4s, #12 // t26a
2043 mul_mla v4, v22, v25, v1.s[1], v1.s[0] // -> t25a
2044 mul_mls v6, v30, v17, v1.s[2], v1.s[3] // -> t23a
2045 srshr v22.4s, v2.4s, #12 // t22a
2046 srshr v25.4s, v4.4s, #12 // t25a
2047 mul_mla v2, v30, v17, v1.s[3], v1.s[2] // -> t24a
2048 srshr v30.4s, v6.4s, #12 // t23a
2049 srshr v17.4s, v2.4s, #12 // t24a
2050
2051 ld1 {v0.4s, v1.4s}, [x16]
2052
2053 movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff
2054 mvni v4.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
2055
2056 sqsub v2.4s, v16.4s, v24.4s // t17
2057 sqadd v16.4s, v16.4s, v24.4s // t16
2058 sqsub v3.4s, v31.4s, v23.4s // t30
2059 sqadd v31.4s, v31.4s, v23.4s // t31
2060 sqsub v24.4s, v28.4s, v20.4s // t18
2061 sqadd v28.4s, v28.4s, v20.4s // t19
2062 sqadd v23.4s, v18.4s, v26.4s // t20
2063 sqsub v18.4s, v18.4s, v26.4s // t21
2064 sqsub v20.4s, v30.4s, v22.4s // t22
2065 sqadd v30.4s, v30.4s, v22.4s // t23
2066 sqadd v26.4s, v17.4s, v25.4s // t24
2067 sqsub v17.4s, v17.4s, v25.4s // t25
2068 sqsub v22.4s, v29.4s, v21.4s // t26
2069 sqadd v29.4s, v29.4s, v21.4s // t27
2070 sqadd v25.4s, v19.4s, v27.4s // t28
2071 sqsub v19.4s, v19.4s, v27.4s // t29
2072
2073 .irp r, v2, v16, v3, v31, v24, v28, v23, v18, v20, v30, v26, v17, v22, v29, v25, v19
2074 smin \r\().4s, \r\().4s, v5.4s
2075 .endr
2076 .irp r, v2, v16, v3, v31, v24, v28, v23, v18, v20, v30, v26, v17, v22, v29, v25, v19
2077 smax \r\().4s, \r\().4s, v4.4s
2078 .endr
2079
2080 mul_mls v7, v3, v2, v1.s[0], v1.s[1] // -> t17a
2081 mul_mla v6, v3, v2, v1.s[1], v1.s[0] // -> t30a
2082 mul_mla v2, v19, v24, v1.s[1], v1.s[0] // -> t18a
2083 srshr v21.4s, v7.4s, #12 // t17a
2084 srshr v27.4s, v6.4s, #12 // t30a
2085 neg v2.4s, v2.4s // -> t18a
2086 mul_mls v7, v19, v24, v1.s[0], v1.s[1] // -> t29a
2087 mul_mls v6, v22, v18, v1.s[2], v1.s[3] // -> t21a
2088 srshr v19.4s, v2.4s, #12 // t18a
2089 srshr v24.4s, v7.4s, #12 // t29a
2090 mul_mla v2, v22, v18, v1.s[3], v1.s[2] // -> t26a
2091 mul_mla v7, v17, v20, v1.s[3], v1.s[2] // -> t22a
2092 srshr v22.4s, v6.4s, #12 // t21a
2093 srshr v18.4s, v2.4s, #12 // t26a
2094 neg v7.4s, v7.4s // -> t22a
2095 mul_mls v6, v17, v20, v1.s[2], v1.s[3] // -> t25a
2096 srshr v17.4s, v7.4s, #12 // t22a
2097 srshr v20.4s, v6.4s, #12 // t25a
2098
2099 sqsub v2.4s, v27.4s, v24.4s // t29
2100 sqadd v27.4s, v27.4s, v24.4s // t30
2101 sqsub v3.4s, v21.4s, v19.4s // t18
2102 sqadd v21.4s, v21.4s, v19.4s // t17
2103 sqsub v24.4s, v16.4s, v28.4s // t19a
2104 sqadd v16.4s, v16.4s, v28.4s // t16a
2105 sqsub v19.4s, v30.4s, v23.4s // t20a
2106 sqadd v30.4s, v30.4s, v23.4s // t23a
2107 sqsub v28.4s, v17.4s, v22.4s // t21
2108 sqadd v17.4s, v17.4s, v22.4s // t22
2109 sqadd v23.4s, v26.4s, v29.4s // t24a
2110 sqsub v26.4s, v26.4s, v29.4s // t27a
2111 sqadd v22.4s, v20.4s, v18.4s // t25
2112 sqsub v20.4s, v20.4s, v18.4s // t26
2113 sqsub v29.4s, v31.4s, v25.4s // t28a
2114 sqadd v31.4s, v31.4s, v25.4s // t31a
2115
2116 .irp r, v2, v27, v3, v21, v24, v16, v19, v30, v28, v17, v23, v26, v22, v20, v29, v31
2117 smin \r\().4s, \r\().4s, v5.4s
2118 .endr
2119 .irp r, v2, v27, v3, v21, v24, v16, v19, v30, v28, v17, v23, v26, v22, v20, v29, v31
2120 smax \r\().4s, \r\().4s, v4.4s
2121 .endr
2122
2123 mul_mls v7, v2, v3, v0.s[2], v0.s[3] // -> t18a
2124 mul_mla v6, v2, v3, v0.s[3], v0.s[2] // -> t29a
2125 mul_mls v2, v29, v24, v0.s[2], v0.s[3] // -> t19
2126 srshr v18.4s, v7.4s, #12 // t18a
2127 srshr v25.4s, v6.4s, #12 // t29a
2128 mul_mla v7, v29, v24, v0.s[3], v0.s[2] // -> t28
2129 mul_mla v6, v26, v19, v0.s[3], v0.s[2] // -> t20
2130 srshr v29.4s, v2.4s, #12 // t19
2131 srshr v24.4s, v7.4s, #12 // t28
2132 neg v6.4s, v6.4s // -> t20
2133 mul_mls v2, v26, v19, v0.s[2], v0.s[3] // -> t27
2134 mul_mla v7, v20, v28, v0.s[3], v0.s[2] // -> t21a
2135 srshr v26.4s, v6.4s, #12 // t20
2136 srshr v19.4s, v2.4s, #12 // t27
2137 neg v7.4s, v7.4s // -> t21a
2138 mul_mls v6, v20, v28, v0.s[2], v0.s[3] // -> t26a
2139 srshr v20.4s, v7.4s, #12 // t21a
2140 srshr v28.4s, v6.4s, #12 // t26a
2141
2142 sqsub v2.4s, v16.4s, v30.4s // t23
2143 sqadd v16.4s, v16.4s, v30.4s // t16 = out16
2144 sqsub v3.4s, v31.4s, v23.4s // t24
2145 sqadd v31.4s, v31.4s, v23.4s // t31 = out31
2146 sqsub v23.4s, v21.4s, v17.4s // t22a
2147 sqadd v17.4s, v21.4s, v17.4s // t17a = out17
2148 sqadd v30.4s, v27.4s, v22.4s // t30a = out30
2149 sqsub v21.4s, v27.4s, v22.4s // t25a
2150 sqsub v27.4s, v18.4s, v20.4s // t21
2151 sqadd v18.4s, v18.4s, v20.4s // t18 = out18
2152 sqadd v7.4s, v29.4s, v26.4s // t19a = out19
2153 sqsub v26.4s, v29.4s, v26.4s // t20a
2154 sqadd v29.4s, v25.4s, v28.4s // t29 = out29
2155 sqsub v25.4s, v25.4s, v28.4s // t26
2156 sqadd v28.4s, v24.4s, v19.4s // t28a = out28
2157 sqsub v24.4s, v24.4s, v19.4s // t27a
2158 mov v19.16b, v7.16b // out19
2159
2160 .irp r, v2, v16, v3, v31, v23, v17, v30, v21, v27, v18, v19, v26, v29, v25, v28, v24
2161 smin \r\().4s, \r\().4s, v5.4s
2162 .endr
2163 .irp r, v2, v16, v3, v31, v23, v17, v30, v21, v27, v18, v19, v26, v29, v25, v28, v24
2164 smax \r\().4s, \r\().4s, v4.4s
2165 .endr
2166
2167 mul_mls v7, v24, v26, v0.s[0], v0.s[0] // -> t20
2168 mul_mla v6, v24, v26, v0.s[0], v0.s[0] // -> t27
2169 srshr v20.4s, v7.4s, #12 // t20
2170 srshr v22.4s, v6.4s, #12 // t27
2171
2172 mul_mla v7, v25, v27, v0.s[0], v0.s[0] // -> t26a
2173 mul_mls v6, v25, v27, v0.s[0], v0.s[0] // -> t21a
2174 mov v27.16b, v22.16b // t27
2175 srshr v26.4s, v7.4s, #12 // t26a
2176
2177 mul_mls v24, v21, v23, v0.s[0], v0.s[0] // -> t22
2178 mul_mla v7, v21, v23, v0.s[0], v0.s[0] // -> t25
2179 srshr v21.4s, v6.4s, #12 // t21a
2180 srshr v22.4s, v24.4s, #12 // t22
2181 srshr v25.4s, v7.4s, #12 // t25
2182
2183 mul_mls v7, v3, v2, v0.s[0], v0.s[0] // -> t23a
2184 mul_mla v6, v3, v2, v0.s[0], v0.s[0] // -> t24a
2185 srshr v23.4s, v7.4s, #12 // t23a
2186 srshr v24.4s, v6.4s, #12 // t24a
2187
2188 ret
2189 endfunc
2190
2191 .macro def_horz_32 scale=0, shift=2, suffix
2192 function inv_txfm_horz\suffix\()_dct_32x4_neon
2193 mov x14, x30
2194 movi v7.4s, #0
2195 lsl x8, x8, #1
2196 .if \scale
2197 movz w16, #2896*8, lsl #16
2198 dup v0.2s, w16
2199 .endif
2200
2201 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
2202 ld1 {\i}, [x7]
2203 st1 {v7.4s}, [x7], x8
2204 .endr
2205 sub x7, x7, x8, lsl #4
2206 add x7, x7, x8, lsr #1
2207 .if \scale
2208 scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
2209 scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31
2210 .endif
2211 bl inv_dct_4s_x16_neon
2212
2213 // idct_16 leaves the row_clip_max/min constants in v5 and v4
2214 .irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31
2215 smin_4s \r, \r, v5
2216 .endr
2217 .irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31
2218 smax_4s \r, \r, v4
2219 .endr
2220
2221 transpose_4x4s v16, v17, v18, v19, v2, v3, v4, v5
2222 transpose_4x4s v20, v21, v22, v23, v2, v3, v4, v5
2223 transpose_4x4s v24, v25, v26, v27, v2, v3, v4, v5
2224 transpose_4x4s v28, v29, v30, v31, v2, v3, v4, v5
2225
2226 .macro store1 r0, r1, r2, r3
2227 st1 {\r0}, [x6], #16
2228 st1 {\r1}, [x6], #16
2229 st1 {\r2}, [x6], #16
2230 st1 {\r3}, [x6], #16
2231 .endm
2232 store1 v16.4s, v20.4s, v24.4s, v28.4s
2233 store1 v17.4s, v21.4s, v25.4s, v29.4s
2234 store1 v18.4s, v22.4s, v26.4s, v30.4s
2235 store1 v19.4s, v23.4s, v27.4s, v31.4s
2236 .purgem store1
2237 sub x6, x6, #64*4
2238
2239 movi v7.4s, #0
2240 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
2241 ld1 {\i}, [x7]
2242 st1 {v7.4s}, [x7], x8
2243 .endr
2244 .if \scale
2245 // This relies on the fact that the idct also leaves the right coeff in v0.s[1]
2246 scale_input .4s, v0.s[1], v16, v17, v18, v19, v20, v21, v22, v23
2247 scale_input .4s, v0.s[1], v24, v25, v26, v27, v28, v29, v30, v31
2248 .endif
2249 bl inv_dct32_odd_4s_x16_neon
2250 transpose_4x4s v31, v30, v29, v28, v2, v3, v4, v5
2251 transpose_4x4s v27, v26, v25, v24, v2, v3, v4, v5
2252 transpose_4x4s v23, v22, v21, v20, v2, v3, v4, v5
2253 transpose_4x4s v19, v18, v17, v16, v2, v3, v4, v5
2254 .macro store2 r0, r1, r2, r3, shift
2255 ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x6]
2256 sqsub v4.4s, v0.4s, \r0
2257 sqadd v0.4s, v0.4s, \r0
2258 sqsub v5.4s, v1.4s, \r1
2259 sqadd v1.4s, v1.4s, \r1
2260 sqsub v6.4s, v2.4s, \r2
2261 sqadd v2.4s, v2.4s, \r2
2262 sqsub v7.4s, v3.4s, \r3
2263 sqadd v3.4s, v3.4s, \r3
2264 sqrshrn v0.4h, v0.4s, #\shift
2265 sqrshrn2 v0.8h, v1.4s, #\shift
2266 sqrshrn v1.4h, v2.4s, #\shift
2267 sqrshrn2 v1.8h, v3.4s, #\shift
2268 sqrshrn v2.4h, v7.4s, #\shift
2269 sqrshrn2 v2.8h, v6.4s, #\shift
2270 sqrshrn v3.4h, v5.4s, #\shift
2271 sqrshrn2 v3.8h, v4.4s, #\shift
2272 st1 {v0.8h, v1.8h}, [x6], #32
2273 rev64 v2.8h, v2.8h
2274 rev64 v3.8h, v3.8h
2275 st1 {v2.8h, v3.8h}, [x6], #32
2276 .endm
2277
2278 store2 v31.4s, v27.4s, v23.4s, v19.4s, \shift
2279 store2 v30.4s, v26.4s, v22.4s, v18.4s, \shift
2280 store2 v29.4s, v25.4s, v21.4s, v17.4s, \shift
2281 store2 v28.4s, v24.4s, v20.4s, v16.4s, \shift
2282 .purgem store2
2283 ret x14
2284 endfunc
2285 .endm
2286
2287 def_horz_32 scale=0, shift=2
2288 def_horz_32 scale=1, shift=1, suffix=_scale
2289
2290 function inv_txfm_add_vert_dct_8x32_neon
2291 mov x14, x30
2292 lsl x8, x8, #1
2293
2294 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
2295 ld1 {v\i\().8h}, [x7], x8
2296 .endr
2297 sub x7, x7, x8, lsl #4
2298
2299 bl X(inv_dct_8h_x16_neon)
2300
2301 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
2302 st1 {v\i\().8h}, [x7], x8
2303 .endr
2304 sub x7, x7, x8, lsl #4
2305 add x7, x7, x8, lsr #1
2306
2307 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
2308 ld1 {v\i\().8h}, [x7], x8
2309 .endr
2310 sub x7, x7, x8, lsl #4
2311 sub x7, x7, x8, lsr #1
2312 bl X(inv_dct32_odd_8h_x16_neon)
2313
2314 neg x9, x8
2315 mov x10, x6
2316 mvni v1.8h, #0xfc, lsl #8 // 0x3ff
2317 .macro combine r0, r1, r2, r3, op, stride
2318 ld1 {v5.8h}, [x7], \stride
2319 ld1 {v2.8h}, [x10], x1
2320 ld1 {v6.8h}, [x7], \stride
2321 ld1 {v3.8h}, [x10], x1
2322 \op v5.8h, v5.8h, \r0
2323 ld1 {v7.8h}, [x7], \stride
2324 ld1 {v4.8h}, [x10], x1
2325 srshr v5.8h, v5.8h, #4
2326 \op v6.8h, v6.8h, \r1
2327 usqadd v2.8h, v5.8h
2328 srshr v6.8h, v6.8h, #4
2329 \op v7.8h, v7.8h, \r2
2330 ld1 {v5.8h}, [x7], \stride
2331 usqadd v3.8h, v6.8h
2332 smin v2.8h, v2.8h, v1.8h
2333 srshr v7.8h, v7.8h, #4
2334 \op v5.8h, v5.8h, \r3
2335 st1 {v2.8h}, [x6], x1
2336 ld1 {v2.8h}, [x10], x1
2337 usqadd v4.8h, v7.8h
2338 smin v3.8h, v3.8h, v1.8h
2339 srshr v5.8h, v5.8h, #4
2340 st1 {v3.8h}, [x6], x1
2341 usqadd v2.8h, v5.8h
2342 smin v4.8h, v4.8h, v1.8h
2343 st1 {v4.8h}, [x6], x1
2344 smin v2.8h, v2.8h, v1.8h
2345 st1 {v2.8h}, [x6], x1
2346 .endm
2347 combine v31.8h, v30.8h, v29.8h, v28.8h, sqadd, x8
2348 combine v27.8h, v26.8h, v25.8h, v24.8h, sqadd, x8
2349 combine v23.8h, v22.8h, v21.8h, v20.8h, sqadd, x8
2350 combine v19.8h, v18.8h, v17.8h, v16.8h, sqadd, x8
2351 sub x7, x7, x8
2352 combine v16.8h, v17.8h, v18.8h, v19.8h, sqsub, x9
2353 combine v20.8h, v21.8h, v22.8h, v23.8h, sqsub, x9
2354 combine v24.8h, v25.8h, v26.8h, v27.8h, sqsub, x9
2355 combine v28.8h, v29.8h, v30.8h, v31.8h, sqsub, x9
2356 .purgem combine
2357
2358 ret x14
2359 endfunc
2360
2361 const eob_32x32
2362 .short 10, 36, 78, 136, 210, 300, 406, 1024
2363 endconst
2364
2365 const eob_16x32
2366 .short 10, 36, 78, 151, 215, 279, 343, 512
2367 endconst
2368
2369 const eob_16x32_shortside
2370 .short 10, 36, 78, 512
2371 endconst
2372
2373 const eob_8x32
2374 .short 10, 43, 75, 107, 139, 171, 203, 256
2375 endconst
2376
2377 function inv_txfm_add_identity_identity_32x32_16bpc_neon, export=1
2378 movi v0.8h, #0
2379 movi v1.8h, #0
2380 movrel x13, eob_32x32, 2
2381
2382 mov x8, #4*32
2383 1:
2384 mov w9, #0
2385 movrel x12, eob_32x32, 2
2386 2:
2387 add w9, w9, #8
2388 ld1 {v16.4s, v17.4s}, [x2]
2389 st1 {v0.4s, v1.4s}, [x2], x8
2390 ld1 {v18.4s, v19.4s}, [x2]
2391 st1 {v0.4s, v1.4s}, [x2], x8
2392 ld1 {v20.4s, v21.4s}, [x2]
2393 st1 {v0.4s, v1.4s}, [x2], x8
2394 ld1 {v22.4s, v23.4s}, [x2]
2395 st1 {v0.4s, v1.4s}, [x2], x8
2396 ld1 {v24.4s, v25.4s}, [x2]
2397 st1 {v0.4s, v1.4s}, [x2], x8
2398 ld1 {v26.4s, v27.4s}, [x2]
2399 st1 {v0.4s, v1.4s}, [x2], x8
2400 ld1 {v28.4s, v29.4s}, [x2]
2401 st1 {v0.4s, v1.4s}, [x2], x8
2402 ld1 {v30.4s, v31.4s}, [x2]
2403 st1 {v0.4s, v1.4s}, [x2], x8
2404 sqxtn v16.4h, v16.4s
2405 sqxtn2 v16.8h, v17.4s
2406 sqxtn v17.4h, v18.4s
2407 sqxtn2 v17.8h, v19.4s
2408 sqxtn v18.4h, v20.4s
2409 sqxtn2 v18.8h, v21.4s
2410 sqxtn v19.4h, v22.4s
2411 sqxtn2 v19.8h, v23.4s
2412 sqxtn v20.4h, v24.4s
2413 sqxtn2 v20.8h, v25.4s
2414 sqxtn v21.4h, v26.4s
2415 sqxtn2 v21.8h, v27.4s
2416 sqxtn v22.4h, v28.4s
2417 sqxtn2 v22.8h, v29.4s
2418 sqxtn v23.4h, v30.4s
2419 sqxtn2 v23.8h, v31.4s
2420 transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
2421
2422 load_add_store_8x8 x0, x7, shiftbits=2
2423 ldrh w11, [x12], #4
2424 sub x0, x0, x1, lsl #3
2425 add x0, x0, #2*8
2426 cmp w3, w11
2427 b.ge 2b
2428
2429 ldrh w11, [x13], #4
2430 cmp w3, w11
2431 b.lt 9f
2432
2433 sub x0, x0, w9, uxtw #1
2434 add x0, x0, x1, lsl #3
2435 msub x2, x8, x9, x2
2436 add x2, x2, #4*8
2437 b 1b
2438 9:
2439 ret
2440 endfunc
2441
2442 .macro shift_16_regs op, shift
2443 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
2444 \op \i, \i, #\shift
2445 .endr
2446 .endm
2447
2448 .macro def_identity_1632 w, h, wshort, hshort
2449 function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1
2450 movz w16, #2896*8, lsl #16
2451 movz w17, #2*(5793-4096)*8, lsl #16
2452 movi v0.4s, #0
2453 movi v1.4s, #0
2454 movrel x13, eob_16x32\hshort, 2
2455
2456 mov x8, #4*\h
2457 1:
2458 mov w9, #0
2459 movrel x12, eob_16x32\wshort, 2
2460 2:
2461 add w9, w9, #8
2462 ld1 {v16.4s, v17.4s}, [x2]
2463 st1 {v0.4s, v1.4s}, [x2], x8
2464 dup v2.2s, w16
2465 ld1 {v18.4s, v19.4s}, [x2]
2466 st1 {v0.4s, v1.4s}, [x2], x8
2467 mov v2.s[1], w17
2468 ld1 {v20.4s, v21.4s}, [x2]
2469 st1 {v0.4s, v1.4s}, [x2], x8
2470 ld1 {v22.4s, v23.4s}, [x2]
2471 st1 {v0.4s, v1.4s}, [x2], x8
2472 ld1 {v24.4s, v25.4s}, [x2]
2473 st1 {v0.4s, v1.4s}, [x2], x8
2474 ld1 {v26.4s, v27.4s}, [x2]
2475 st1 {v0.4s, v1.4s}, [x2], x8
2476 ld1 {v28.4s, v29.4s}, [x2]
2477 st1 {v0.4s, v1.4s}, [x2], x8
2478 ld1 {v30.4s, v31.4s}, [x2]
2479 st1 {v0.4s, v1.4s}, [x2], x8
2480 scale_input .4s, v2.s[0], v16, v17, v18, v19, v20, v21, v22, v23
2481 scale_input .4s, v2.s[0], v24, v25, v26, v27, v28, v29, v30, v31
2482
2483 .if \w == 16
2484 // 16x32
2485 identity_4x16_shift1 v2.s[1]
2486 .else
2487 // 32x16
2488 shift_16_regs sqshl, 1
2489 identity_4x16 v2.s[1]
2490 .endif
2491 sqxtn v16.4h, v16.4s
2492 sqxtn2 v16.8h, v17.4s
2493 sqxtn v17.4h, v18.4s
2494 sqxtn2 v17.8h, v19.4s
2495 sqxtn v18.4h, v20.4s
2496 sqxtn2 v18.8h, v21.4s
2497 sqxtn v19.4h, v22.4s
2498 sqxtn2 v19.8h, v23.4s
2499 sqxtn v20.4h, v24.4s
2500 sqxtn2 v20.8h, v25.4s
2501 sqxtn v21.4h, v26.4s
2502 sqxtn2 v21.8h, v27.4s
2503 sqxtn v22.4h, v28.4s
2504 sqxtn2 v22.8h, v29.4s
2505 sqxtn v23.4h, v30.4s
2506 sqxtn2 v23.8h, v31.4s
2507
2508 transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
2509
2510 .if \w == 16
2511 load_add_store_8x8 x0, x7, shiftbits=2
2512 .else
2513 load_add_store_8x8 x0, x7, shiftbits=4
2514 .endif
2515 ldrh w11, [x12], #4
2516 sub x0, x0, x1, lsl #3
2517 add x0, x0, #16
2518 cmp w3, w11
2519 b.ge 2b
2520
2521 ldrh w11, [x13], #4
2522 cmp w3, w11
2523 b.lt 9f
2524
2525 sub x0, x0, w9, uxtw #1
2526 add x0, x0, x1, lsl #3
2527 msub x2, x8, x9, x2
2528 add x2, x2, #4*8
2529 b 1b
2530 9:
2531 ret
2532 endfunc
2533 .endm
2534
2535 def_identity_1632 16, 32, _shortside,
2536 def_identity_1632 32, 16, , _shortside
2537
2538 .macro def_identity_832 w, h
2539 function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1
2540 movi v0.4s, #0
2541 movi v1.4s, #0
2542 // Working on 8x8 blocks, read every other entry from eob_8x32
2543 movrel x13, eob_8x32, 2
2544
2545 mov w8, #4*\h
2546 1:
2547 // Working on 8x8 blocks, read every other entry from eob_8x32
2548 ldrh w12, [x13], #4
2549 ld1 {v16.4s, v17.4s}, [x2]
2550 st1 {v0.4s, v1.4s}, [x2], x8
2551 ld1 {v18.4s, v19.4s}, [x2]
2552 st1 {v0.4s, v1.4s}, [x2], x8
2553 ld1 {v20.4s, v21.4s}, [x2]
2554 st1 {v0.4s, v1.4s}, [x2], x8
2555 ld1 {v22.4s, v23.4s}, [x2]
2556 st1 {v0.4s, v1.4s}, [x2], x8
2557 ld1 {v24.4s, v25.4s}, [x2]
2558 st1 {v0.4s, v1.4s}, [x2], x8
2559 ld1 {v26.4s, v27.4s}, [x2]
2560 st1 {v0.4s, v1.4s}, [x2], x8
2561 ld1 {v28.4s, v29.4s}, [x2]
2562 st1 {v0.4s, v1.4s}, [x2], x8
2563 ld1 {v30.4s, v31.4s}, [x2]
2564 st1 {v0.4s, v1.4s}, [x2], x8
2565
2566 .if \w == 8
2567 sqrshrn v16.4h, v16.4s, #1
2568 sqrshrn2 v16.8h, v17.4s, #1
2569 sqrshrn v17.4h, v18.4s, #1
2570 sqrshrn2 v17.8h, v19.4s, #1
2571 sqrshrn v18.4h, v20.4s, #1
2572 sqrshrn2 v18.8h, v21.4s, #1
2573 sqrshrn v19.4h, v22.4s, #1
2574 sqrshrn2 v19.8h, v23.4s, #1
2575 sqrshrn v20.4h, v24.4s, #1
2576 sqrshrn2 v20.8h, v25.4s, #1
2577 sqrshrn v21.4h, v26.4s, #1
2578 sqrshrn2 v21.8h, v27.4s, #1
2579 sqrshrn v22.4h, v28.4s, #1
2580 sqrshrn2 v22.8h, v29.4s, #1
2581 sqrshrn v23.4h, v30.4s, #1
2582 sqrshrn2 v23.8h, v31.4s, #1
2583 .else
2584 sqxtn v16.4h, v16.4s
2585 sqxtn2 v16.8h, v17.4s
2586 sqxtn v17.4h, v18.4s
2587 sqxtn2 v17.8h, v19.4s
2588 sqxtn v18.4h, v20.4s
2589 sqxtn2 v18.8h, v21.4s
2590 sqxtn v19.4h, v22.4s
2591 sqxtn2 v19.8h, v23.4s
2592 sqxtn v20.4h, v24.4s
2593 sqxtn2 v20.8h, v25.4s
2594 sqxtn v21.4h, v26.4s
2595 sqxtn2 v21.8h, v27.4s
2596 sqxtn v22.4h, v28.4s
2597 sqxtn2 v22.8h, v29.4s
2598 sqxtn v23.4h, v30.4s
2599 sqxtn2 v23.8h, v31.4s
2600 .endif
2601
2602 transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
2603
2604
2605 cmp w3, w12
2606 .if \w == 8
2607 load_add_store_8x8 x0, x7, shiftbits=2
2608 .else
2609 load_add_store_8x8 x0, x7, shiftbits=3
2610 .endif
2611
2612 b.lt 9f
2613 .if \w == 8
2614 sub x2, x2, x8, lsl #3
2615 add x2, x2, #4*8
2616 .else
2617 sub x0, x0, x1, lsl #3
2618 add x0, x0, #2*8
2619 .endif
2620 b 1b
2621
2622 9:
2623 ret
2624 endfunc
2625 .endm
2626
2627 def_identity_832 8, 32
2628 def_identity_832 32, 8
2629
2630 function inv_txfm_add_dct_dct_32x32_16bpc_neon, export=1
2631 idct_dc 32, 32, 2
2632
2633 mov x15, x30
2634 sub sp, sp, #2048
2635 movrel x13, eob_32x32
2636 ldrh w12, [x13], #2
2637
2638 .irp i, 0, 4, 8, 12, 16, 20, 24, 28
2639 add x6, sp, #(\i*32*2)
2640 .if \i > 0
2641 mov w8, #(32 - \i)
2642 cmp w3, w12
2643 b.lt 1f
2644 .if \i < 28
2645 ldrh w12, [x13], #2
2646 .endif
2647 .endif
2648 add x7, x2, #(\i*4)
2649 mov x8, #32*4
2650 bl inv_txfm_horz_dct_32x4_neon
2651 .endr
2652 b 3f
2653
2654 1:
2655 movi v4.8h, #0
2656 movi v5.8h, #0
2657 movi v6.8h, #0
2658 movi v7.8h, #0
2659 2:
2660 subs w8, w8, #4
2661 .rept 4
2662 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
2663 .endr
2664 b.gt 2b
2665
2666 3:
2667 .irp i, 0, 8, 16, 24
2668 add x6, x0, #(\i*2)
2669 add x7, sp, #(\i*2)
2670 mov x8, #32*2
2671 bl inv_txfm_add_vert_dct_8x32_neon
2672 .endr
2673
2674 add sp, sp, #2048
2675 ret x15
2676 endfunc
2677
2678 function inv_txfm_add_dct_dct_16x32_16bpc_neon, export=1
2679 idct_dc 16, 32, 1
2680
2681 mov x15, x30
2682 sub sp, sp, #1024
2683 movrel x13, eob_16x32
2684 ldrh w12, [x13], #2
2685 adr x4, inv_dct_4s_x16_neon
2686
2687 .irp i, 0, 4, 8, 12, 16, 20, 24, 28
2688 add x6, sp, #(\i*16*2)
2689 add x7, x2, #(\i*4)
2690 .if \i > 0
2691 mov w8, #(32 - \i)
2692 cmp w3, w12
2693 b.lt 1f
2694 .if \i < 28
2695 ldrh w12, [x13], #2
2696 .endif
2697 .endif
2698 mov x8, #4*32
2699 bl inv_txfm_horz_scale_16x4_neon
2700 .endr
2701 b 3f
2702
2703 1:
2704 movi v4.8h, #0
2705 movi v5.8h, #0
2706 movi v6.8h, #0
2707 movi v7.8h, #0
2708 2:
2709 subs w8, w8, #4
2710 .rept 2
2711 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
2712 .endr
2713 b.gt 2b
2714
2715 3:
2716 .irp i, 0, 8
2717 add x6, x0, #(\i*2)
2718 add x7, sp, #(\i*2)
2719 mov x8, #16*2
2720 bl inv_txfm_add_vert_dct_8x32_neon
2721 .endr
2722
2723 add sp, sp, #1024
2724 ret x15
2725 endfunc
2726
2727 function inv_txfm_add_dct_dct_32x16_16bpc_neon, export=1
2728 idct_dc 32, 16, 1
2729
2730 mov x15, x30
2731 sub sp, sp, #1024
2732
2733 movrel x13, eob_16x32
2734 movrel x5, X(inv_dct_8h_x16_neon)
2735 ldrh w12, [x13], #2
2736
2737 .irp i, 0, 4, 8, 12
2738 add x6, sp, #(\i*32*2)
2739 add x7, x2, #(\i*4)
2740 .if \i > 0
2741 mov w8, #(16 - \i)
2742 cmp w3, w12
2743 b.lt 1f
2744 .if \i < 12
2745 ldrh w12, [x13], #2
2746 .endif
2747 .endif
2748 mov x8, #4*16
2749 bl inv_txfm_horz_scale_dct_32x4_neon
2750 .endr
2751 b 3f
2752
2753 1:
2754 movi v4.8h, #0
2755 movi v5.8h, #0
2756 movi v6.8h, #0
2757 movi v7.8h, #0
2758 2:
2759 subs w8, w8, #4
2760 .rept 4
2761 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
2762 .endr
2763 b.gt 2b
2764
2765 3:
2766 .irp i, 0, 8, 16, 24
2767 add x6, x0, #(\i*2)
2768 add x7, sp, #(\i*2)
2769 mov x8, #32*2
2770 bl inv_txfm_add_vert_8x16_neon
2771 .endr
2772
2773 add sp, sp, #1024
2774 ret x15
2775 endfunc
2776
2777 function inv_txfm_add_dct_dct_8x32_16bpc_neon, export=1
2778 idct_dc 8, 32, 2
2779
2780 mov x15, x30
2781 sub sp, sp, #512
2782
2783 movrel x13, eob_8x32
2784
2785 movi v28.4s, #0
2786 mov x8, #4*32
2787 mov w9, #32
2788 mov x6, sp
2789 mov x7, x2
2790 1:
2791 .irp i, 16, 17, 18, 19, 20, 21, 22, 23
2792 ld1 {v\i\().4s}, [x7]
2793 st1 {v28.4s}, [x7], x8
2794 .endr
2795 ldrh w12, [x13], #2
2796 sub w9, w9, #4
2797 sub x7, x7, x8, lsl #3
2798 add x7, x7, #4*4
2799
2800 bl inv_dct_4s_x8_neon
2801
2802 sqrshrn v16.4h, v16.4s, #2
2803 sqrshrn v17.4h, v17.4s, #2
2804 sqrshrn v18.4h, v18.4s, #2
2805 sqrshrn v19.4h, v19.4s, #2
2806 sqrshrn2 v16.8h, v20.4s, #2
2807 sqrshrn2 v17.8h, v21.4s, #2
2808 sqrshrn2 v18.8h, v22.4s, #2
2809 sqrshrn2 v19.8h, v23.4s, #2
2810
2811 transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5
2812
2813 cmp w3, w12
2814 st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64
2815
2816 b.ge 1b
2817 cbz w9, 3f
2818
2819 movi v29.8h, #0
2820 movi v30.8h, #0
2821 movi v31.8h, #0
2822 2:
2823 subs w9, w9, #4
2824 st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x6], #64
2825 b.gt 2b
2826
2827 3:
2828 mov x6, x0
2829 mov x7, sp
2830 mov x8, #8*2
2831 bl inv_txfm_add_vert_dct_8x32_neon
2832
2833 add sp, sp, #512
2834 ret x15
2835 endfunc
2836
2837 function inv_txfm_add_dct_dct_32x8_16bpc_neon, export=1
2838 idct_dc 32, 8, 2
2839
2840 mov x15, x30
2841 sub sp, sp, #512
2842
2843 .irp i, 0, 4
2844 add x6, sp, #(\i*32*2)
2845 add x7, x2, #(\i*4)
2846 .if \i > 0
2847 cmp w3, #10
2848 b.lt 1f
2849 .endif
2850 mov x8, #8*4
2851 bl inv_txfm_horz_dct_32x4_neon
2852 .endr
2853 b 2f
2854
2855 1:
2856 movi v4.8h, #0
2857 movi v5.8h, #0
2858 movi v6.8h, #0
2859 movi v7.8h, #0
2860 .rept 4
2861 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
2862 .endr
2863
2864 2:
2865 mov x8, #2*32
2866 mov w9, #0
2867 1:
2868 add x6, x0, x9, lsl #1
2869 add x7, sp, x9, lsl #1 // #(\i*2)
2870
2871 .irp i, 16, 17, 18, 19, 20, 21, 22, 23
2872 ld1 {v\i\().8h}, [x7], x8
2873 .endr
2874 add w9, w9, #8
2875
2876 bl X(inv_dct_8h_x8_neon)
2877
2878 cmp w9, #32
2879
2880 load_add_store_8x8 x6, x7
2881
2882 b.lt 1b
2883
2884 add sp, sp, #512
2885 ret x15
2886 endfunc
2887
2888 function inv_dct64_step1_neon
2889 // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a
2890 // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a
2891 // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a
2892 // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a
2893
2894 ld1 {v0.4s, v1.4s}, [x17], #32
2895
2896 sqrdmulh v23.4s, v16.4s, v0.s[1] // t63a
2897 sqrdmulh v16.4s, v16.4s, v0.s[0] // t32a
2898 sqrdmulh v22.4s, v17.4s, v0.s[2] // t62a
2899 sqrdmulh v17.4s, v17.4s, v0.s[3] // t33a
2900 sqrdmulh v21.4s, v18.4s, v1.s[1] // t61a
2901 sqrdmulh v18.4s, v18.4s, v1.s[0] // t34a
2902 sqrdmulh v20.4s, v19.4s, v1.s[2] // t60a
2903 sqrdmulh v19.4s, v19.4s, v1.s[3] // t35a
2904
2905 ld1 {v0.4s}, [x17], #16
2906
2907 sqadd v24.4s, v16.4s, v17.4s // t32
2908 sqsub v25.4s, v16.4s, v17.4s // t33
2909 sqsub v26.4s, v19.4s, v18.4s // t34
2910 sqadd v27.4s, v19.4s, v18.4s // t35
2911 sqadd v28.4s, v20.4s, v21.4s // t60
2912 sqsub v29.4s, v20.4s, v21.4s // t61
2913 sqsub v30.4s, v23.4s, v22.4s // t62
2914 sqadd v31.4s, v23.4s, v22.4s // t63
2915
2916 .irp r, v24, v25, v26, v27, v28, v29, v30, v31
2917 smin_4s \r, \r, v5
2918 .endr
2919 .irp r, v24, v25, v26, v27, v28, v29, v30, v31
2920 smax_4s \r, \r, v4
2921 .endr
2922
2923 mul_mla v2, v29, v26, v0.s[0], v0.s[1] // -> t34a
2924 mul_mls v7, v29, v26, v0.s[1], v0.s[0] // -> t61a
2925 neg v2.4s, v2.4s // t34a
2926 mul_mls v6, v30, v25, v0.s[1], v0.s[0] // -> t33a
2927 srshr v26.4s, v2.4s, #12 // t34a
2928 mul_mla v2, v30, v25, v0.s[0], v0.s[1] // -> t62a
2929 srshr v29.4s, v7.4s, #12 // t61a
2930 srshr v25.4s, v6.4s, #12 // t33a
2931 srshr v30.4s, v2.4s, #12 // t62a
2932
2933 sqadd v16.4s, v24.4s, v27.4s // t32a
2934 sqsub v19.4s, v24.4s, v27.4s // t35a
2935 sqadd v17.4s, v25.4s, v26.4s // t33
2936 sqsub v18.4s, v25.4s, v26.4s // t34
2937 sqsub v20.4s, v31.4s, v28.4s // t60a
2938 sqadd v23.4s, v31.4s, v28.4s // t63a
2939 sqsub v21.4s, v30.4s, v29.4s // t61
2940 sqadd v22.4s, v30.4s, v29.4s // t62
2941
2942 .irp r, v16, v19, v17, v18, v20, v23, v21, v22
2943 smin_4s \r, \r, v5
2944 .endr
2945 .irp r, v16, v19, v17, v18, v20, v23, v21, v22
2946 smax_4s \r, \r, v4
2947 .endr
2948
2949 mul_mla v2, v21, v18, v0.s[2], v0.s[3] // -> t61a
2950 mul_mls v7, v21, v18, v0.s[3], v0.s[2] // -> t34a
2951 mul_mla v6, v20, v19, v0.s[2], v0.s[3] // -> t60
2952 srshr v21.4s, v2.4s, #12 // t61a
2953 srshr v18.4s, v7.4s, #12 // t34a
2954 mul_mls v2, v20, v19, v0.s[3], v0.s[2] // -> t35
2955 srshr v20.4s, v6.4s, #12 // t60
2956 srshr v19.4s, v2.4s, #12 // t35
2957
2958 st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x6], #64
2959 st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x6], #64
2960
2961 ret
2962 endfunc
2963
2964 function inv_dct64_step2_neon
2965 movrel x16, idct_coeffs
2966 ld1 {v0.4s}, [x16]
2967 1:
2968 // t32a/33/34a/35/60/61a/62/63a
2969 // t56a/57/58a/59/36/37a/38/39a
2970 // t40a/41/42a/43/52/53a/54/55a
2971 // t48a/49/50a/51/44/45a/46/47a
2972 ldr q16, [x6, #4*4*0] // t32a
2973 ldr q17, [x9, #4*4*8] // t39a
2974 ldr q18, [x9, #4*4*0] // t63a
2975 ldr q19, [x6, #4*4*8] // t56a
2976 ldr q20, [x6, #4*4*16] // t40a
2977 ldr q21, [x9, #4*4*24] // t47a
2978 ldr q22, [x9, #4*4*16] // t55a
2979 ldr q23, [x6, #4*4*24] // t48a
2980
2981 sqadd v24.4s, v16.4s, v17.4s // t32
2982 sqsub v25.4s, v16.4s, v17.4s // t39
2983 sqadd v26.4s, v18.4s, v19.4s // t63
2984 sqsub v27.4s, v18.4s, v19.4s // t56
2985 sqsub v28.4s, v21.4s, v20.4s // t40
2986 sqadd v29.4s, v21.4s, v20.4s // t47
2987 sqadd v30.4s, v23.4s, v22.4s // t48
2988 sqsub v31.4s, v23.4s, v22.4s // t55
2989
2990 .irp r, v24, v25, v26, v27, v28, v29, v30, v31
2991 smin_4s \r, \r, v5
2992 .endr
2993 .irp r, v24, v25, v26, v27, v28, v29, v30, v31
2994 smax_4s \r, \r, v4
2995 .endr
2996
2997 mul_mla v2, v27, v25, v0.s[3], v0.s[2] // -> t56a
2998 mul_mls v7, v27, v25, v0.s[2], v0.s[3] // -> t39a
2999 mul_mla v6, v31, v28, v0.s[3], v0.s[2] // -> t40a
3000 srshr v25.4s, v2.4s, #12 // t56a
3001 srshr v27.4s, v7.4s, #12 // t39a
3002 neg v6.4s, v6.4s // t40a
3003 mul_mls v2, v31, v28, v0.s[2], v0.s[3] // -> t55a
3004 srshr v31.4s, v6.4s, #12 // t40a
3005 srshr v28.4s, v2.4s, #12 // t55a
3006
3007 sqadd v16.4s, v24.4s, v29.4s // t32a
3008 sqsub v19.4s, v24.4s, v29.4s // t47a
3009 sqadd v17.4s, v27.4s, v31.4s // t39
3010 sqsub v18.4s, v27.4s, v31.4s // t40
3011 sqsub v20.4s, v26.4s, v30.4s // t48a
3012 sqadd v23.4s, v26.4s, v30.4s // t63a
3013 sqsub v21.4s, v25.4s, v28.4s // t55
3014 sqadd v22.4s, v25.4s, v28.4s // t56
3015
3016 .irp r, v16, v19, v17, v18, v20, v23, v21, v22
3017 smin_4s \r, \r, v5
3018 .endr
3019 .irp r, v16, v19, v17, v18, v20, v23, v21, v22
3020 smax_4s \r, \r, v4
3021 .endr
3022
3023 mul_mls v2, v21, v18, v0.s[0], v0.s[0] // -> t40a
3024 mul_mla v7, v21, v18, v0.s[0], v0.s[0] // -> t55a
3025 mul_mls v6, v20, v19, v0.s[0], v0.s[0] // -> t47
3026 srshr v18.4s, v2.4s, #12 // t40a
3027 srshr v21.4s, v7.4s, #12 // t55a
3028 mul_mla v2, v20, v19, v0.s[0], v0.s[0] // -> t48
3029 srshr v19.4s, v6.4s, #12 // t47
3030 srshr v20.4s, v2.4s, #12 // t48
3031
3032 str q16, [x6, #4*4*0] // t32a
3033 str q17, [x9, #4*4*0] // t39
3034 str q18, [x6, #4*4*8] // t40a
3035 str q19, [x9, #4*4*8] // t47
3036 str q20, [x6, #4*4*16] // t48
3037 str q21, [x9, #4*4*16] // t55a
3038 str q22, [x6, #4*4*24] // t56
3039 str q23, [x9, #4*4*24] // t63a
3040
3041 add x6, x6, #4*4
3042 sub x9, x9, #4*4
3043 cmp x6, x9
3044 b.lt 1b
3045 ret
3046 endfunc
3047
3048 .macro load8 src, strd, zero, clear
3049 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s
3050 .if \clear
3051 ld1 {\i}, [\src]
3052 st1 {\zero}, [\src], \strd
3053 .else
3054 ld1 {\i}, [\src], \strd
3055 .endif
3056 .endr
3057 .endm
3058
3059 .macro store16 dst
3060 .irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
3061 st1 {\i}, [\dst], #16
3062 .endr
3063 .endm
3064
3065 .macro clear_upper8
3066 .irp i, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s
3067 movi \i, #0
3068 .endr
3069 .endm
3070
3071 .macro movi_if reg, val, cond
3072 .if \cond
3073 movi \reg, \val
3074 .endif
3075 .endm
3076
3077 .macro movz16dup_if reg, gpr, val, cond
3078 .if \cond
3079 movz \gpr, \val, lsl #16
3080 dup \reg, \gpr
3081 .endif
3082 .endm
3083
3084 .macro st1_if regs, dst, cond
3085 .if \cond
3086 st1 \regs, \dst
3087 .endif
3088 .endm
3089
3090 .macro str_if reg, dst, cond
3091 .if \cond
3092 str \reg, \dst
3093 .endif
3094 .endm
3095
3096 .macro stroff_if reg, dst, dstoff, cond
3097 .if \cond
3098 str \reg, \dst, \dstoff
3099 .endif
3100 .endm
3101
3102 .macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7
3103 .if \cond
3104 scale_input .4s, \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7
3105 .endif
3106 .endm
3107
3108 .macro def_dct64_func suffix, clear=0, scale=0
3109 function inv_txfm_dct\suffix\()_4s_x64_neon
3110 mov x14, x30
3111 mov x6, sp
3112 lsl x8, x8, #2
3113
3114 movz16dup_if v0.2s, w16, #2896*8, \scale
3115 movi_if v7.4s, #0, \clear
3116 load8 x7, x8, v7.4s, \clear
3117 clear_upper8
3118 sub x7, x7, x8, lsl #3
3119 add x7, x7, x8, lsr #1
3120 scale_if \scale, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
3121
3122 bl inv_dct_4s_x16_neon
3123
3124 // idct_16 leaves the row_clip_max/min constants in v5 and v4
3125 .irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31
3126 smin_4s \r, \r, v5
3127 .endr
3128 .irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31
3129 smax_4s \r, \r, v4
3130 .endr
3131
3132 store16 x6
3133
3134 movz16dup_if v0.2s, w16, #2896*8, \scale
3135 movi_if v7.8h, #0, \clear
3136 load8 x7, x8, v7.4s, \clear
3137 clear_upper8
3138 sub x7, x7, x8, lsl #3
3139 lsr x8, x8, #1
3140 sub x7, x7, x8, lsr #1
3141 scale_if \scale, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23
3142
3143 bl inv_dct32_odd_4s_x16_neon
3144
3145 add x10, x6, #16*15
3146 sub x6, x6, #16*16
3147
3148 mov x9, #-16
3149
3150 movi v1.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff
3151 mvni v0.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
3152
3153 .macro store_addsub r0, r1, r2, r3
3154 ld1 {v2.4s}, [x6], #16
3155 ld1 {v3.4s}, [x6], #16
3156 sqadd v6.4s, v2.4s, \r0
3157 sqsub \r0, v2.4s, \r0
3158 ld1 {v4.4s}, [x6], #16
3159 sqadd v7.4s, v3.4s, \r1
3160 sqsub \r1, v3.4s, \r1
3161 smin v6.4s, v6.4s, v1.4s
3162 smin \r0, \r0, v1.4s
3163 ld1 {v5.4s}, [x6], #16
3164 sqadd v2.4s, v4.4s, \r2
3165 sub x6, x6, #16*4
3166 smax v6.4s, v6.4s, v0.4s
3167 smax \r0, \r0, v0.4s
3168 sqsub \r2, v4.4s, \r2
3169 smin v7.4s, v7.4s, v1.4s
3170 smin \r1, \r1, v1.4s
3171 st1 {v6.4s}, [x6], #16
3172 st1 {\r0}, [x10], x9
3173 smin v2.4s, v2.4s, v1.4s
3174 smin \r2, \r2, v1.4s
3175 smax v7.4s, v7.4s, v0.4s
3176 smax \r1, \r1, v0.4s
3177 sqadd v3.4s, v5.4s, \r3
3178 sqsub \r3, v5.4s, \r3
3179 smax v2.4s, v2.4s, v0.4s
3180 smax \r2, \r2, v0.4s
3181 smin v3.4s, v3.4s, v1.4s
3182 smin \r3, \r3, v1.4s
3183 st1 {v7.4s}, [x6], #16
3184 st1 {\r1}, [x10], x9
3185 smax v3.4s, v3.4s, v0.4s
3186 smax \r3, \r3, v0.4s
3187 st1 {v2.4s}, [x6], #16
3188 st1 {\r2}, [x10], x9
3189 st1 {v3.4s}, [x6], #16
3190 st1 {\r3}, [x10], x9
3191 .endm
3192 store_addsub v31.4s, v30.4s, v29.4s, v28.4s
3193 store_addsub v27.4s, v26.4s, v25.4s, v24.4s
3194 store_addsub v23.4s, v22.4s, v21.4s, v20.4s
3195 store_addsub v19.4s, v18.4s, v17.4s, v16.4s
3196 .purgem store_addsub
3197
3198 add x6, x6, #4*4*16
3199
3200 movrel x17, idct64_coeffs
3201 movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff
3202 mvni v4.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000
3203 movz16dup_if v0.2s, w16, #2896*8, \scale
3204 movi_if v7.4s, #0, \clear
3205 add x9, x7, x8, lsl #4 // offset 16
3206 add x10, x7, x8, lsl #3 // offset 8
3207 sub x9, x9, x8 // offset 15
3208 sub x11, x10, x8 // offset 7
3209 ld1 {v16.4s}, [x7] // in1 (offset 0)
3210 ld1 {v17.4s}, [x9] // in31 (offset 15)
3211 ld1 {v18.4s}, [x10] // in17 (offset 8)
3212 ld1 {v19.4s}, [x11] // in15 (offset 7)
3213 st1_if {v7.4s}, [x7], \clear
3214 st1_if {v7.4s}, [x9], \clear
3215 st1_if {v7.4s}, [x10], \clear
3216 st1_if {v7.4s}, [x11], \clear
3217 scale_if \scale, v0.s[0], v16, v17, v18, v19
3218 bl inv_dct64_step1_neon
3219 movz16dup_if v0.2s, w16, #2896*8, \scale
3220 movi_if v7.4s, #0, \clear
3221 add x7, x7, x8, lsl #2 // offset 4
3222 sub x9, x9, x8, lsl #2 // offset 11
3223 sub x10, x7, x8 // offset 3
3224 add x11, x9, x8 // offset 12
3225 ld1 {v16.4s}, [x10] // in7 (offset 3)
3226 ld1 {v17.4s}, [x11] // in25 (offset 12)
3227 ld1 {v18.4s}, [x9] // in23 (offset 11)
3228 ld1 {v19.4s}, [x7] // in9 (offset 4)
3229 st1_if {v7.4s}, [x7], \clear
3230 st1_if {v7.4s}, [x9], \clear
3231 st1_if {v7.4s}, [x10], \clear
3232 st1_if {v7.4s}, [x11], \clear
3233 scale_if \scale, v0.s[0], v16, v17, v18, v19
3234 bl inv_dct64_step1_neon
3235 movz16dup_if v0.2s, w16, #2896*8, \scale
3236 movi_if v7.4s, #0, \clear
3237 sub x10, x10, x8, lsl #1 // offset 1
3238 sub x9, x9, x8, lsl #1 // offset 9
3239 add x7, x7, x8 // offset 5
3240 add x11, x11, x8 // offset 13
3241 ldr q16, [x10, x8] // in5 (offset 2)
3242 ldr q17, [x11] // in27 (offset 13)
3243 ldr q18, [x9, x8] // in21 (offset 10)
3244 ldr q19, [x7] // in11 (offset 5)
3245 stroff_if q7, [x10, x8], \clear
3246 str_if q7, [x11], \clear
3247 stroff_if q7, [x9, x8], \clear
3248 str_if q7, [x7], \clear
3249 scale_if \scale, v0.s[0], v16, v17, v18, v19
3250 bl inv_dct64_step1_neon
3251 movz16dup_if v0.2s, w16, #2896*8, \scale
3252 movi_if v7.4s, #0, \clear
3253 ldr q16, [x10] // in3 (offset 1)
3254 ldr q17, [x11, x8] // in29 (offset 14)
3255 ldr q18, [x9] // in19 (offset 9)
3256 ldr q19, [x7, x8] // in13 (offset 6)
3257 str_if q7, [x10], \clear
3258 stroff_if q7, [x11, x8], \clear
3259 str_if q7, [x9], \clear
3260 stroff_if q7, [x7, x8], \clear
3261 scale_if \scale, v0.s[0], v16, v17, v18, v19
3262 bl inv_dct64_step1_neon
3263
3264 sub x6, x6, #4*4*32
3265 add x9, x6, #4*4*7
3266
3267 bl inv_dct64_step2_neon
3268
3269 ret x14
3270 endfunc
3271 .endm
3272
3273 def_dct64_func _clear, clear=1
3274 def_dct64_func _clear_scale, clear=1, scale=1
3275
3276
3277 function inv_txfm_horz_dct_64x4_neon
3278 mov x14, x30
3279
3280 mov x7, sp
3281 add x8, sp, #4*4*(64 - 4)
3282 add x9, x6, #2*56
3283 mov x10, #2*64
3284 mov x11, #-4*4*4
3285
3286 dup v7.4s, w12
3287 1:
3288 ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x7], #64
3289 ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x8], x11
3290 ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x7], #64
3291 ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x8], x11
3292 transpose_4x4s v16, v17, v18, v19, v2, v3, v4, v5
3293 transpose_4x4s v20, v21, v22, v23, v2, v3, v4, v5
3294 transpose_4x4s v31, v30, v29, v28, v2, v3, v4, v5
3295 transpose_4x4s v27, v26, v25, v24, v2, v3, v4, v5
3296
3297 .macro store_addsub src0, src1, src2, src3
3298 sqsub v1.4s, \src0, \src1
3299 sqadd v0.4s, \src0, \src1
3300 sqsub v3.4s, \src2, \src3
3301 srshl v1.4s, v1.4s, v7.4s
3302 sqadd v2.4s, \src2, \src3
3303 srshl v3.4s, v3.4s, v7.4s
3304 srshl v0.4s, v0.4s, v7.4s
3305 srshl v2.4s, v2.4s, v7.4s
3306 sqxtn v3.4h, v3.4s
3307 sqxtn2 v3.8h, v1.4s
3308 sqxtn v0.4h, v0.4s
3309 sqxtn2 v0.8h, v2.4s
3310 rev64 v3.8h, v3.8h
3311 st1 {v0.8h}, [x6], x10
3312 st1 {v3.8h}, [x9], x10
3313 .endm
3314 store_addsub v16.4s, v31.4s, v20.4s, v27.4s
3315 store_addsub v17.4s, v30.4s, v21.4s, v26.4s
3316 store_addsub v18.4s, v29.4s, v22.4s, v25.4s
3317 store_addsub v19.4s, v28.4s, v23.4s, v24.4s
3318 .purgem store_addsub
3319 sub x6, x6, x10, lsl #2
3320 sub x9, x9, x10, lsl #2
3321 add x6, x6, #16
3322 sub x9, x9, #16
3323
3324 cmp x7, x8
3325 b.lt 1b
3326 ret x14
3327 endfunc
3328
3329 function inv_txfm_add_vert_dct_8x64_neon
3330 mov x14, x30
3331 lsl x8, x8, #1
3332
3333 mov x7, sp
3334 add x8, sp, #2*8*(64 - 4)
3335 add x9, x6, x1, lsl #6
3336 sub x9, x9, x1
3337 neg x10, x1
3338 mov x11, #-2*8*4
3339
3340 1:
3341 ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64
3342 ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11
3343 ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64
3344 ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11
3345
3346 mvni v7.8h, #0xfc, lsl #8 // 0x3ff
3347 .macro add_dest_addsub src0, src1, src2, src3
3348 ld1 {v0.8h}, [x6], x1
3349 ld1 {v1.8h}, [x9], x10
3350 sqadd v4.8h, \src0, \src1
3351 ld1 {v2.8h}, [x6]
3352 sqsub \src0, \src0, \src1
3353 ld1 {v3.8h}, [x9]
3354 sqadd v5.8h, \src2, \src3
3355 sqsub \src2, \src2, \src3
3356 sub x6, x6, x1
3357 sub x9, x9, x10
3358 srshr v4.8h, v4.8h, #4
3359 srshr v5.8h, v5.8h, #4
3360 srshr \src0, \src0, #4
3361 usqadd v0.8h, v4.8h
3362 srshr \src2, \src2, #4
3363 usqadd v1.8h, \src0
3364 usqadd v2.8h, v5.8h
3365 smin v0.8h, v0.8h, v7.8h
3366 usqadd v3.8h, \src2
3367 smin v1.8h, v1.8h, v7.8h
3368 st1 {v0.8h}, [x6], x1
3369 smin v2.8h, v2.8h, v7.8h
3370 st1 {v1.8h}, [x9], x10
3371 smin v3.8h, v3.8h, v7.8h
3372 st1 {v2.8h}, [x6], x1
3373 st1 {v3.8h}, [x9], x10
3374 .endm
3375 add_dest_addsub v16.8h, v31.8h, v17.8h, v30.8h
3376 add_dest_addsub v18.8h, v29.8h, v19.8h, v28.8h
3377 add_dest_addsub v20.8h, v27.8h, v21.8h, v26.8h
3378 add_dest_addsub v22.8h, v25.8h, v23.8h, v24.8h
3379 .purgem add_dest_addsub
3380 cmp x7, x8
3381 b.lt 1b
3382
3383 ret x14
3384 endfunc
3385
3386 function inv_txfm_add_dct_dct_64x64_16bpc_neon, export=1
3387 idct_dc 64, 64, 2
3388
3389 mov x15, x30
3390
3391 sub_sp 64*32*2+64*4*4
3392 add x5, sp, #64*4*4
3393
3394 movrel x13, eob_32x32
3395
3396 .irp i, 0, 4, 8, 12, 16, 20, 24, 28
3397 add x6, x5, #(\i*64*2)
3398 .if \i > 0
3399 mov w8, #(32 - \i)
3400 cmp w3, w12
3401 b.lt 1f
3402 .endif
3403 add x7, x2, #(\i*4)
3404 mov x8, #32*4
3405 mov x12, #-2 // shift
3406 bl inv_txfm_dct_clear_4s_x64_neon
3407 add x6, x5, #(\i*64*2)
3408 bl inv_txfm_horz_dct_64x4_neon
3409 .if \i < 28
3410 ldrh w12, [x13], #2
3411 .endif
3412 .endr
3413 b 3f
3414
3415 1:
3416 movi v4.8h, #0
3417 movi v5.8h, #0
3418 movi v6.8h, #0
3419 movi v7.8h, #0
3420 2:
3421 subs w8, w8, #2
3422 .rept 4
3423 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
3424 .endr
3425 b.gt 2b
3426
3427 3:
3428 .irp i, 0, 8, 16, 24, 32, 40, 48, 56
3429 add x7, x5, #(\i*2)
3430 mov x8, #64*2
3431 bl X(inv_txfm_dct_8h_x64_neon)
3432 add x6, x0, #(\i*2)
3433 bl inv_txfm_add_vert_dct_8x64_neon
3434 .endr
3435
3436 add sp, x5, #64*32*2
3437 ret x15
3438 endfunc
3439
3440 function inv_txfm_add_dct_dct_64x32_16bpc_neon, export=1
3441 idct_dc 64, 32, 1
3442
3443 mov x15, x30
3444
3445 sub_sp 64*32*2+64*4*4
3446 add x5, sp, #64*4*4
3447
3448 movrel x13, eob_32x32
3449
3450 .irp i, 0, 4, 8, 12, 16, 20, 24, 28
3451 add x6, x5, #(\i*64*2)
3452 .if \i > 0
3453 mov w8, #(32 - \i)
3454 cmp w3, w12
3455 b.lt 1f
3456 .endif
3457 add x7, x2, #(\i*4)
3458 mov x8, #32*4
3459 mov x12, #-1 // shift
3460 bl inv_txfm_dct_clear_scale_4s_x64_neon
3461 add x6, x5, #(\i*64*2)
3462 bl inv_txfm_horz_dct_64x4_neon
3463 .if \i < 28
3464 ldrh w12, [x13], #2
3465 .endif
3466 .endr
3467 b 3f
3468
3469 1:
3470 movi v4.8h, #0
3471 movi v5.8h, #0
3472 movi v6.8h, #0
3473 movi v7.8h, #0
3474 2:
3475 subs w8, w8, #2
3476 .rept 4
3477 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
3478 .endr
3479 b.gt 2b
3480
3481 3:
3482 .irp i, 0, 8, 16, 24, 32, 40, 48, 56
3483 add x6, x0, #(\i*2)
3484 add x7, x5, #(\i*2)
3485 mov x8, #64*2
3486 bl inv_txfm_add_vert_dct_8x32_neon
3487 .endr
3488
3489 add sp, x5, #64*32*2
3490 ret x15
3491 endfunc
3492
3493 function inv_txfm_add_dct_dct_32x64_16bpc_neon, export=1
3494 idct_dc 32, 64, 1
3495
3496 mov x15, x30
3497
3498 sub_sp 32*32*2+64*8*2
3499 add x5, sp, #64*8*2
3500
3501 movrel x13, eob_32x32
3502 ldrh w12, [x13], #2
3503
3504 .irp i, 0, 4, 8, 12, 16, 20, 24, 28
3505 add x6, x5, #(\i*32*2)
3506 .if \i > 0
3507 mov w8, #(32 - \i)
3508 cmp w3, w12
3509 b.lt 1f
3510 ldrh w12, [x13], #2
3511 .endif
3512 add x7, x2, #(\i*4)
3513 mov x8, #32*4
3514 bl inv_txfm_horz_scale_dct_32x4_neon
3515 .endr
3516 b 3f
3517
3518 1:
3519 movi v4.8h, #0
3520 movi v5.8h, #0
3521 movi v6.8h, #0
3522 movi v7.8h, #0
3523 2:
3524 subs w8, w8, #4
3525 .rept 4
3526 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
3527 .endr
3528 b.gt 2b
3529
3530 3:
3531 .irp i, 0, 8, 16, 24
3532 add x7, x5, #(\i*2)
3533 mov x8, #32*2
3534 bl X(inv_txfm_dct_8h_x64_neon)
3535 add x6, x0, #(\i*2)
3536 bl inv_txfm_add_vert_dct_8x64_neon
3537 .endr
3538
3539 add sp, x5, #32*32*2
3540 ret x15
3541 endfunc
3542
3543 function inv_txfm_add_dct_dct_64x16_16bpc_neon, export=1
3544 idct_dc 64, 16, 2
3545
3546 mov x15, x30
3547
3548 sub_sp 64*16*2+64*4*4
3549 add x4, sp, #64*4*4
3550
3551 movrel x13, eob_16x32
3552
3553 .irp i, 0, 4, 8, 12
3554 add x6, x4, #(\i*64*2)
3555 .if \i > 0
3556 mov w8, #(16 - \i)
3557 cmp w3, w12
3558 b.lt 1f
3559 .endif
3560 add x7, x2, #(\i*4)
3561 mov x8, #16*4
3562 mov x12, #-2 // shift
3563 bl inv_txfm_dct_clear_4s_x64_neon
3564 add x6, x4, #(\i*64*2)
3565 bl inv_txfm_horz_dct_64x4_neon
3566 .if \i < 12
3567 ldrh w12, [x13], #2
3568 .endif
3569 .endr
3570 b 3f
3571
3572 1:
3573 movi v4.8h, #0
3574 movi v5.8h, #0
3575 movi v6.8h, #0
3576 movi v7.8h, #0
3577 2:
3578 subs w8, w8, #2
3579 .rept 4
3580 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
3581 .endr
3582 b.gt 2b
3583
3584 3:
3585 movrel x5, X(inv_dct_8h_x16_neon)
3586 .irp i, 0, 8, 16, 24, 32, 40, 48, 56
3587 add x6, x0, #(\i*2)
3588 add x7, x4, #(\i*2)
3589 mov x8, #64*2
3590 bl inv_txfm_add_vert_8x16_neon
3591 .endr
3592
3593 add sp, x4, #64*16*2
3594 ret x15
3595 endfunc
3596
3597 function inv_txfm_add_dct_dct_16x64_16bpc_neon, export=1
3598 idct_dc 16, 64, 2
3599
3600 mov x15, x30
3601
3602 sub_sp 16*32*2+64*8*2
3603 add x5, sp, #64*8*2
3604
3605 movrel x13, eob_16x32
3606 ldrh w12, [x13], #2
3607
3608 adr x4, inv_dct_4s_x16_neon
3609 .irp i, 0, 4, 8, 12, 16, 20, 24, 28
3610 add x6, x5, #(\i*16*2)
3611 .if \i > 0
3612 mov w8, #(32 - \i)
3613 cmp w3, w12
3614 b.lt 1f
3615 .if \i < 28
3616 ldrh w12, [x13], #2
3617 .endif
3618 .endif
3619 add x7, x2, #(\i*4)
3620 mov x8, #32*4
3621 bl inv_txfm_horz_16x4_neon
3622 .endr
3623 b 3f
3624
3625 1:
3626 movi v4.8h, #0
3627 movi v5.8h, #0
3628 movi v6.8h, #0
3629 movi v7.8h, #0
3630 2:
3631 subs w8, w8, #4
3632 .rept 2
3633 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
3634 .endr
3635 b.gt 2b
3636
3637 3:
3638 .irp i, 0, 8
3639 add x7, x5, #(\i*2)
3640 mov x8, #16*2
3641 bl X(inv_txfm_dct_8h_x64_neon)
3642 add x6, x0, #(\i*2)
3643 bl inv_txfm_add_vert_dct_8x64_neon
3644 .endr
3645
3646 add sp, x5, #16*32*2
3647 ret x15
3648 endfunc