Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/x86/ipred16_avx512.asm

raw
1 ; Copyright © 2022, VideoLAN and dav1d authors
2 ; Copyright © 2022, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "src/asm/x86/x86inc.asm"
28
29 %if ARCH_X86_64
30
31 SECTION_RODATA 64
32
33 ipred_shuf: db 14, 15, 14, 15, 0, 1, 2, 3, 6, 7, 6, 7, 0, 1, 2, 3
34 db 10, 11, 10, 11, 8, 9, 10, 11, 2, 3, 2, 3, 8, 9, 10, 11
35 db 12, 13, 12, 13, 4, 5, 6, 7, 4, 5, 4, 5, 4, 5, 6, 7
36 db 8, 9, 8, 9, 12, 13, 14, 15, 0, 1, 0, 1, 12, 13, 14, 15
37 smooth_perm: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30
38 db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62
39 db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94
40 db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126
41 pal_pred_perm: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39
42 db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47
43 db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55
44 db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
45 filter_permA: times 4 db 6, 7, 8, 9, 14, 15, 4, 5
46 times 4 db 10, 11, 12, 13, 2, 3, -1, -1
47 filter_permB: times 4 db 22, 23, 24, 25, 30, 31, 6, 7
48 times 4 db 26, 27, 28, 29, 14, 15, -1, -1
49 filter_permC: dd 8 ; dq 8, 10, 1, 11, 0, 9
50 pw_1: times 2 dw 1
51 dd 10
52 filter_rnd: dd 32
53 dd 1
54 dd 8
55 dd 11
56 filter_shift: times 2 dw 6
57 dd 0
58 times 2 dw 4
59 dd 9
60
61 %macro JMP_TABLE 3-*
62 %xdefine %1_%2_table (%%table - 2*4)
63 %xdefine %%base mangle(private_prefix %+ _%1_%2)
64 %%table:
65 %rep %0 - 2
66 dd %%base %+ .%3 - (%%table - 2*4)
67 %rotate 1
68 %endrep
69 %endmacro
70
71 JMP_TABLE ipred_paeth_16bpc, avx512icl, w4, w8, w16, w32, w64
72 JMP_TABLE ipred_smooth_16bpc, avx512icl, w4, w8, w16, w32, w64
73 JMP_TABLE ipred_smooth_h_16bpc, avx512icl, w4, w8, w16, w32, w64
74 JMP_TABLE ipred_smooth_v_16bpc, avx512icl, w4, w8, w16, w32, w64
75 JMP_TABLE pal_pred_16bpc, avx512icl, w4, w8, w16, w32, w64
76
77 cextern smooth_weights_1d_16bpc
78 cextern smooth_weights_2d_16bpc
79 cextern filter_intra_taps
80
81 SECTION .text
82
83 %macro PAETH 3 ; top, signed_ldiff, ldiff
84 paddw m0, m%2, m2
85 psubw m1, m0, m3 ; tldiff
86 psubw m0, m%1 ; tdiff
87 pabsw m1, m1
88 pabsw m0, m0
89 pcmpgtw k1, m0, m1
90 pminsw m0, m1
91 pcmpgtw k2, m%3, m0
92 vpblendmw m0{k1}, m%1, m3
93 vpblendmw m0{k2}, m2, m0
94 %endmacro
95
96 INIT_ZMM avx512icl
97 cglobal ipred_paeth_16bpc, 3, 7, 10, dst, stride, tl, w, h
98 %define base r6-ipred_paeth_16bpc_avx512icl_table
99 lea r6, [ipred_paeth_16bpc_avx512icl_table]
100 tzcnt wd, wm
101 movifnidn hd, hm
102 movsxd wq, [r6+wq*4]
103 vpbroadcastw m3, [tlq] ; topleft
104 add wq, r6
105 jmp wq
106 .w4:
107 vpbroadcastq m4, [tlq+2] ; top
108 movsldup m7, [base+ipred_shuf]
109 lea r6, [strideq*3]
110 psubw m5, m4, m3
111 pabsw m6, m5
112 .w4_loop:
113 sub tlq, 16
114 vbroadcasti32x4 m2, [tlq]
115 pshufb m2, m7 ; left
116 PAETH 4, 5, 6
117 vextracti32x4 xm1, m0, 2
118 vextracti32x4 xm8, ym0, 1
119 vextracti32x4 xm9, m0, 3
120 movq [dstq+strideq*0], xm0
121 movq [dstq+strideq*1], xm1
122 movq [dstq+strideq*2], xm8
123 movq [dstq+r6 ], xm9
124 sub hd, 8
125 jl .w4_end
126 lea dstq, [dstq+strideq*4]
127 movhps [dstq+strideq*0], xm0
128 movhps [dstq+strideq*1], xm1
129 movhps [dstq+strideq*2], xm8
130 movhps [dstq+r6 ], xm9
131 lea dstq, [dstq+strideq*4]
132 jg .w4_loop
133 .w4_end:
134 RET
135 .w8:
136 vbroadcasti32x4 m4, [tlq+2]
137 movsldup m7, [base+ipred_shuf]
138 lea r6, [strideq*3]
139 psubw m5, m4, m3
140 pabsw m6, m5
141 .w8_loop:
142 sub tlq, 8
143 vpbroadcastq m2, [tlq]
144 pshufb m2, m7
145 PAETH 4, 5, 6
146 mova [dstq+strideq*0], xm0
147 vextracti32x4 [dstq+strideq*1], m0, 2
148 vextracti32x4 [dstq+strideq*2], ym0, 1
149 vextracti32x4 [dstq+r6 ], m0, 3
150 lea dstq, [dstq+strideq*4]
151 sub hd, 4
152 jg .w8_loop
153 RET
154 .w16:
155 vbroadcasti32x8 m4, [tlq+2]
156 movsldup m7, [base+ipred_shuf]
157 psubw m5, m4, m3
158 pabsw m6, m5
159 .w16_loop:
160 sub tlq, 4
161 vpbroadcastd m2, [tlq]
162 pshufb m2, m7
163 PAETH 4, 5, 6
164 mova [dstq+strideq*0], ym0
165 vextracti32x8 [dstq+strideq*1], m0, 1
166 lea dstq, [dstq+strideq*2]
167 sub hd, 2
168 jg .w16_loop
169 RET
170 .w32:
171 movu m4, [tlq+2]
172 psubw m5, m4, m3
173 pabsw m6, m5
174 .w32_loop:
175 sub tlq, 2
176 vpbroadcastw m2, [tlq]
177 PAETH 4, 5, 6
178 mova [dstq], m0
179 add dstq, strideq
180 dec hd
181 jg .w32_loop
182 RET
183 .w64:
184 movu m4, [tlq+ 2]
185 movu m7, [tlq+66]
186 psubw m5, m4, m3
187 psubw m8, m7, m3
188 pabsw m6, m5
189 pabsw m9, m8
190 .w64_loop:
191 sub tlq, 2
192 vpbroadcastw m2, [tlq]
193 PAETH 4, 5, 6
194 mova [dstq+64*0], m0
195 PAETH 7, 8, 9
196 mova [dstq+64*1], m0
197 add dstq, strideq
198 dec hd
199 jg .w64_loop
200 RET
201
202 cglobal ipred_smooth_v_16bpc, 3, 7, 7, dst, stride, tl, w, h, weights, stride3
203 %define base r6-$$
204 lea r6, [$$]
205 tzcnt wd, wm
206 mov hd, hm
207 movsxd wq, [base+ipred_smooth_v_16bpc_avx512icl_table+wq*4]
208 lea weightsq, [base+smooth_weights_1d_16bpc+hq*4]
209 neg hq
210 vpbroadcastw m6, [tlq+hq*2] ; bottom
211 lea wq, [base+ipred_smooth_v_16bpc_avx512icl_table+wq]
212 lea stride3q, [strideq*3]
213 jmp wq
214 .w4:
215 vpbroadcastq m5, [tlq+2] ; top
216 movsldup m4, [ipred_shuf]
217 psubw m5, m6 ; top - bottom
218 .w4_loop:
219 vbroadcasti32x4 m3, [weightsq+hq*2]
220 pshufb m3, m4
221 pmulhrsw m3, m5
222 paddw m3, m6
223 vextracti32x4 xm0, m3, 3
224 vextracti32x4 xm1, ym3, 1
225 vextracti32x4 xm2, m3, 2
226 movhps [dstq+strideq*0], xm0
227 movhps [dstq+strideq*1], xm1
228 movhps [dstq+strideq*2], xm2
229 movhps [dstq+stride3q ], xm3
230 add hq, 8
231 jg .end
232 lea dstq, [dstq+strideq*4]
233 movq [dstq+strideq*0], xm0
234 movq [dstq+strideq*1], xm1
235 movq [dstq+strideq*2], xm2
236 movq [dstq+stride3q ], xm3
237 lea dstq, [dstq+strideq*4]
238 jl .w4_loop
239 .end:
240 RET
241 .w8:
242 vbroadcasti32x4 m5, [tlq+2] ; top
243 movsldup m4, [ipred_shuf]
244 psubw m5, m6 ; top - bottom
245 .w8_loop:
246 vpbroadcastq m0, [weightsq+hq*2]
247 pshufb m0, m4
248 pmulhrsw m0, m5
249 paddw m0, m6
250 vextracti32x4 [dstq+strideq*0], m0, 3
251 vextracti32x4 [dstq+strideq*1], ym0, 1
252 vextracti32x4 [dstq+strideq*2], m0, 2
253 mova [dstq+stride3q ], xm0
254 lea dstq, [dstq+strideq*4]
255 add hq, 4
256 jl .w8_loop
257 RET
258 .w16:
259 vbroadcasti32x8 m5, [tlq+2] ; top
260 movsldup m4, [ipred_shuf]
261 psubw m5, m6 ; top - bottom
262 .w16_loop:
263 vpbroadcastd m0, [weightsq+hq*2+0]
264 vpbroadcastd m1, [weightsq+hq*2+4]
265 pshufb m0, m4
266 pshufb m1, m4
267 pmulhrsw m0, m5
268 pmulhrsw m1, m5
269 paddw m0, m6
270 paddw m1, m6
271 vextracti32x8 [dstq+strideq*0], m0, 1
272 mova [dstq+strideq*1], ym0
273 vextracti32x8 [dstq+strideq*2], m1, 1
274 mova [dstq+stride3q ], ym1
275 lea dstq, [dstq+strideq*4]
276 add hq, 4
277 jl .w16_loop
278 RET
279 .w32:
280 movu m5, [tlq+2]
281 psubw m5, m6
282 .w32_loop:
283 vpbroadcastw m0, [weightsq+hq*2+0]
284 vpbroadcastw m1, [weightsq+hq*2+2]
285 vpbroadcastw m2, [weightsq+hq*2+4]
286 vpbroadcastw m3, [weightsq+hq*2+6]
287 REPX {pmulhrsw x, m5}, m0, m1, m2, m3
288 REPX {paddw x, m6}, m0, m1, m2, m3
289 mova [dstq+strideq*0], m0
290 mova [dstq+strideq*1], m1
291 mova [dstq+strideq*2], m2
292 mova [dstq+stride3q ], m3
293 lea dstq, [dstq+strideq*4]
294 add hq, 4
295 jl .w32_loop
296 RET
297 .w64:
298 movu m4, [tlq+ 2]
299 movu m5, [tlq+66]
300 psubw m4, m6
301 psubw m5, m6
302 .w64_loop:
303 vpbroadcastw m1, [weightsq+hq*2+0]
304 vpbroadcastw m3, [weightsq+hq*2+2]
305 pmulhrsw m0, m4, m1
306 pmulhrsw m1, m5
307 pmulhrsw m2, m4, m3
308 pmulhrsw m3, m5
309 REPX {paddw x, m6}, m0, m1, m2, m3
310 mova [dstq+strideq*0+64*0], m0
311 mova [dstq+strideq*0+64*1], m1
312 mova [dstq+strideq*1+64*0], m2
313 mova [dstq+strideq*1+64*1], m3
314 lea dstq, [dstq+strideq*2]
315 add hq, 2
316 jl .w64_loop
317 RET
318
319 cglobal ipred_smooth_h_16bpc, 3, 7, 7, dst, stride, tl, w, h, stride3
320 lea r6, [$$]
321 mov wd, wm
322 movifnidn hd, hm
323 vpbroadcastw m6, [tlq+wq*2] ; right
324 tzcnt wd, wd
325 add hd, hd
326 movsxd wq, [base+ipred_smooth_h_16bpc_avx512icl_table+wq*4]
327 sub tlq, hq
328 lea stride3q, [strideq*3]
329 lea wq, [base+ipred_smooth_h_16bpc_avx512icl_table+wq]
330 jmp wq
331 .w4:
332 movsldup m4, [base+ipred_shuf]
333 vpbroadcastq m5, [base+smooth_weights_1d_16bpc+4*2]
334 .w4_loop:
335 vbroadcasti32x4 m0, [tlq+hq-16] ; left
336 pshufb m0, m4
337 psubw m0, m6 ; left - right
338 pmulhrsw m0, m5
339 paddw m0, m6
340 vextracti32x4 xm1, m0, 2
341 vextracti32x4 xm2, ym0, 1
342 vextracti32x4 xm3, m0, 3
343 movq [dstq+strideq*0], xm0
344 movq [dstq+strideq*1], xm1
345 movq [dstq+strideq*2], xm2
346 movq [dstq+stride3q ], xm3
347 sub hd, 8*2
348 jl .end
349 lea dstq, [dstq+strideq*4]
350 movhps [dstq+strideq*0], xm0
351 movhps [dstq+strideq*1], xm1
352 movhps [dstq+strideq*2], xm2
353 movhps [dstq+stride3q ], xm3
354 lea dstq, [dstq+strideq*4]
355 jg .w4_loop
356 .end:
357 RET
358 .w8:
359 movsldup m4, [base+ipred_shuf]
360 vbroadcasti32x4 m5, [base+smooth_weights_1d_16bpc+8*2]
361 .w8_loop:
362 vpbroadcastq m0, [tlq+hq-8] ; left
363 pshufb m0, m4
364 psubw m0, m6 ; left - right
365 pmulhrsw m0, m5
366 paddw m0, m6
367 mova [dstq+strideq*0], xm0
368 vextracti32x4 [dstq+strideq*1], m0, 2
369 vextracti32x4 [dstq+strideq*2], ym0, 1
370 vextracti32x4 [dstq+stride3q ], m0, 3
371 lea dstq, [dstq+strideq*4]
372 sub hd, 4*2
373 jg .w8_loop
374 RET
375 .w16:
376 movsldup m4, [base+ipred_shuf]
377 vbroadcasti32x8 m5, [base+smooth_weights_1d_16bpc+16*2]
378 .w16_loop:
379 vpbroadcastd m0, [tlq+hq-4]
380 vpbroadcastd m1, [tlq+hq-8]
381 pshufb m0, m4
382 pshufb m1, m4
383 psubw m0, m6
384 psubw m1, m6
385 pmulhrsw m0, m5
386 pmulhrsw m1, m5
387 paddw m0, m6
388 paddw m1, m6
389 mova [dstq+strideq*0], ym0
390 vextracti32x8 [dstq+strideq*1], m0, 1
391 mova [dstq+strideq*2], ym1
392 vextracti32x8 [dstq+stride3q ], m1, 1
393 lea dstq, [dstq+strideq*4]
394 sub hq, 4*2
395 jg .w16_loop
396 RET
397 .w32:
398 movu m5, [base+smooth_weights_1d_16bpc+32*2]
399 .w32_loop:
400 vpbroadcastq m3, [tlq+hq-8]
401 punpcklwd m3, m3
402 psubw m3, m6
403 pshufd m0, m3, q3333
404 pshufd m1, m3, q2222
405 pshufd m2, m3, q1111
406 pshufd m3, m3, q0000
407 REPX {pmulhrsw x, m5}, m0, m1, m2, m3
408 REPX {paddw x, m6}, m0, m1, m2, m3
409 mova [dstq+strideq*0], m0
410 mova [dstq+strideq*1], m1
411 mova [dstq+strideq*2], m2
412 mova [dstq+stride3q ], m3
413 lea dstq, [dstq+strideq*4]
414 sub hq, 4*2
415 jg .w32_loop
416 RET
417 .w64:
418 movu m4, [base+smooth_weights_1d_16bpc+64*2]
419 movu m5, [base+smooth_weights_1d_16bpc+64*3]
420 .w64_loop:
421 vpbroadcastw m1, [tlq+hq-2]
422 vpbroadcastw m3, [tlq+hq-4]
423 psubw m1, m6
424 psubw m3, m6
425 pmulhrsw m0, m4, m1
426 pmulhrsw m1, m5
427 pmulhrsw m2, m4, m3
428 pmulhrsw m3, m5
429 REPX {paddw x, m6}, m0, m1, m2, m3
430 mova [dstq+strideq*0+64*0], m0
431 mova [dstq+strideq*0+64*1], m1
432 mova [dstq+strideq*1+64*0], m2
433 mova [dstq+strideq*1+64*1], m3
434 lea dstq, [dstq+strideq*2]
435 sub hq, 2*2
436 jg .w64_loop
437 RET
438
439 cglobal ipred_smooth_16bpc, 3, 7, 16, dst, stride, tl, w, h, v_weights, stride3
440 lea r6, [$$]
441 mov wd, wm
442 movifnidn hd, hm
443 vpbroadcastw m13, [tlq+wq*2] ; right
444 tzcnt wd, wd
445 add hd, hd
446 movsxd wq, [base+ipred_smooth_16bpc_avx512icl_table+wq*4]
447 mov r5d, 0x55555555
448 sub tlq, hq
449 mova m14, [base+smooth_perm]
450 kmovd k1, r5d
451 vpbroadcastw m0, [tlq] ; bottom
452 mov r5, 0x3333333333333333
453 pxor m15, m15
454 lea wq, [base+ipred_smooth_16bpc_avx512icl_table+wq]
455 kmovq k2, r5
456 lea v_weightsq, [base+smooth_weights_2d_16bpc+hq*2]
457 jmp wq
458 .w4:
459 vpbroadcastq m5, [tlq+hq+2]
460 movshdup m3, [base+ipred_shuf]
461 movsldup m4, [base+ipred_shuf]
462 vbroadcasti32x4 m6, [base+smooth_weights_2d_16bpc+4*4]
463 lea stride3q, [strideq*3]
464 punpcklwd m5, m0 ; top, bottom
465 .w4_loop:
466 vbroadcasti32x4 m0, [v_weightsq]
467 vpbroadcastq m2, [tlq+hq-8]
468 mova m1, m13
469 pshufb m0, m3
470 pmaddwd m0, m5
471 pshufb m1{k2}, m2, m4 ; left, right
472 vpdpwssd m0, m1, m6
473 vpermb m0, m14, m0
474 pavgw ym0, ym15
475 vextracti32x4 xm1, ym0, 1
476 movq [dstq+strideq*0], xm0
477 movq [dstq+strideq*1], xm1
478 movhps [dstq+strideq*2], xm0
479 movhps [dstq+stride3q ], xm1
480 lea dstq, [dstq+strideq*4]
481 add v_weightsq, 4*4
482 sub hd, 4*2
483 jg .w4_loop
484 RET
485 .w8:
486 vbroadcasti32x4 ym5, [tlq+hq+2]
487 movshdup m6, [base+ipred_shuf]
488 movsldup m7, [base+ipred_shuf]
489 pmovzxwd m5, ym5
490 vbroadcasti32x8 m8, [base+smooth_weights_2d_16bpc+8*4]
491 lea stride3q, [strideq*3]
492 vpblendmw m5{k1}, m0, m5 ; top, bottom
493 .w8_loop:
494 vpbroadcastq m0, [v_weightsq+0]
495 vpbroadcastq m1, [v_weightsq+8]
496 vpbroadcastd m3, [tlq+hq-4]
497 vpbroadcastd m4, [tlq+hq-8]
498 pshufb m0, m6
499 pmaddwd m0, m5
500 pshufb m1, m6
501 pmaddwd m1, m5
502 mova m2, m13
503 pshufb m2{k2}, m3, m7 ; left, right
504 mova m3, m13
505 pshufb m3{k2}, m4, m7
506 vpdpwssd m0, m2, m8
507 vpdpwssd m1, m3, m8
508 add v_weightsq, 4*4
509 vpermt2b m0, m14, m1
510 pavgw m0, m15
511 mova [dstq+strideq*0], xm0
512 vextracti32x4 [dstq+strideq*1], ym0, 1
513 vextracti32x4 [dstq+strideq*2], m0, 2
514 vextracti32x4 [dstq+stride3q ], m0, 3
515 lea dstq, [dstq+strideq*4]
516 sub hd, 4*2
517 jg .w8_loop
518 RET
519 .w16:
520 pmovzxwd m5, [tlq+hq+2]
521 mova m6, [base+smooth_weights_2d_16bpc+16*4]
522 vpblendmw m5{k1}, m0, m5 ; top, bottom
523 .w16_loop:
524 vpbroadcastd m0, [v_weightsq+0]
525 vpbroadcastd m1, [v_weightsq+4]
526 pmaddwd m0, m5
527 pmaddwd m1, m5
528 mova m2, m13
529 vpbroadcastw m2{k1}, [tlq+hq-2] ; left, right
530 mova m3, m13
531 vpbroadcastw m3{k1}, [tlq+hq-4]
532 vpdpwssd m0, m2, m6
533 vpdpwssd m1, m3, m6
534 add v_weightsq, 2*4
535 vpermt2b m0, m14, m1
536 pavgw m0, m15
537 mova [dstq+strideq*0], ym0
538 vextracti32x8 [dstq+strideq*1], m0, 1
539 lea dstq, [dstq+strideq*2]
540 sub hq, 2*2
541 jg .w16_loop
542 RET
543 .w32:
544 pmovzxwd m5, [tlq+hq+ 2]
545 pmovzxwd m6, [tlq+hq+34]
546 mova m7, [base+smooth_weights_2d_16bpc+32*4]
547 mova m8, [base+smooth_weights_2d_16bpc+32*6]
548 vpblendmw m5{k1}, m0, m5 ; top, bottom
549 vpblendmw m6{k1}, m0, m6
550 .w32_loop:
551 vpbroadcastd m2, [v_weightsq+0]
552 vpbroadcastd m3, [v_weightsq+4]
553 pmaddwd m0, m5, m2
554 pmaddwd m2, m6
555 pmaddwd m1, m5, m3
556 pmaddwd m3, m6
557 mova m4, m13
558 vpbroadcastw m4{k1}, [tlq+hq-2] ; left, right
559 vpdpwssd m0, m4, m7
560 vpdpwssd m2, m4, m8
561 mova m4, m13
562 vpbroadcastw m4{k1}, [tlq+hq-4]
563 vpdpwssd m1, m4, m7
564 vpdpwssd m3, m4, m8
565 add v_weightsq, 2*4
566 vpermt2b m0, m14, m2
567 vpermt2b m1, m14, m3
568 pavgw m0, m15
569 pavgw m1, m15
570 mova [dstq+strideq*0], m0
571 mova [dstq+strideq*1], m1
572 lea dstq, [dstq+strideq*2]
573 sub hq, 2*2
574 jg .w32_loop
575 RET
576 .w64:
577 pmovzxwd m5, [tlq+hq+ 2]
578 pmovzxwd m6, [tlq+hq+34]
579 pmovzxwd m7, [tlq+hq+66]
580 pmovzxwd m8, [tlq+hq+98]
581 mova m9, [base+smooth_weights_2d_16bpc+64*4]
582 vpblendmw m5{k1}, m0, m5 ; top, bottom
583 mova m10, [base+smooth_weights_2d_16bpc+64*5]
584 vpblendmw m6{k1}, m0, m6
585 mova m11, [base+smooth_weights_2d_16bpc+64*6]
586 vpblendmw m7{k1}, m0, m7
587 mova m12, [base+smooth_weights_2d_16bpc+64*7]
588 vpblendmw m8{k1}, m0, m8
589 .w64_loop:
590 vpbroadcastd m3, [v_weightsq]
591 mova m4, m13
592 vpbroadcastw m4{k1}, [tlq+hq-2] ; left, right
593 pmaddwd m0, m5, m3
594 pmaddwd m2, m6, m3
595 pmaddwd m1, m7, m3
596 pmaddwd m3, m8
597 vpdpwssd m0, m4, m9
598 vpdpwssd m2, m4, m10
599 vpdpwssd m1, m4, m11
600 vpdpwssd m3, m4, m12
601 add v_weightsq, 1*4
602 vpermt2b m0, m14, m2
603 vpermt2b m1, m14, m3
604 pavgw m0, m15
605 pavgw m1, m15
606 mova [dstq+64*0], m0
607 mova [dstq+64*1], m1
608 add dstq, strideq
609 sub hd, 1*2
610 jg .w64_loop
611 RET
612
613 cglobal pal_pred_16bpc, 4, 7, 4, dst, stride, pal, idx, w, h, stride3
614 lea r6, [pal_pred_16bpc_avx512icl_table]
615 tzcnt wd, wm
616 mova m2, [pal_pred_perm]
617 movsxd wq, [r6+wq*4]
618 mova xm3, [palq]
619 movifnidn hd, hm
620 add wq, r6
621 lea stride3q, [strideq*3]
622 jmp wq
623 .w4:
624 pmovzxbw ym0, [idxq]
625 add idxq, 16
626 vpermw ym0, ym0, ym3
627 vextracti32x4 xm1, ym0, 1
628 movq [dstq+strideq*0], xm0
629 movhps [dstq+strideq*1], xm0
630 movq [dstq+strideq*2], xm1
631 movhps [dstq+stride3q ], xm1
632 lea dstq, [dstq+strideq*4]
633 sub hd, 4
634 jg .w4
635 RET
636 .w8:
637 pmovzxbw m0, [idxq]
638 add idxq, 32
639 vpermw m0, m0, m3
640 mova [dstq+strideq*0], xm0
641 vextracti32x4 [dstq+strideq*1], ym0, 1
642 vextracti32x4 [dstq+strideq*2], m0, 2
643 vextracti32x4 [dstq+stride3q ], m0, 3
644 lea dstq, [dstq+strideq*4]
645 sub hd, 4
646 jg .w8
647 RET
648 .w16:
649 vpermb m1, m2, [idxq]
650 add idxq, 64
651 vpermw m0, m1, m3
652 psrlw m1, 8
653 vpermw m1, m1, m3
654 mova [dstq+strideq*0], ym0
655 vextracti32x8 [dstq+strideq*1], m0, 1
656 mova [dstq+strideq*2], ym1
657 vextracti32x8 [dstq+stride3q ], m1, 1
658 lea dstq, [dstq+strideq*4]
659 sub hd, 4
660 jg .w16
661 RET
662 .w32:
663 vpermb m1, m2, [idxq]
664 add idxq, 64
665 vpermw m0, m1, m3
666 psrlw m1, 8
667 vpermw m1, m1, m3
668 mova [dstq+strideq*0], m0
669 mova [dstq+strideq*1], m1
670 lea dstq, [dstq+strideq*2]
671 sub hd, 2
672 jg .w32
673 RET
674 .w64:
675 vpermb m1, m2, [idxq]
676 add idxq, 64
677 vpermw m0, m1, m3
678 psrlw m1, 8
679 vpermw m1, m1, m3
680 mova [dstq+64*0], m0
681 mova [dstq+64*1], m1
682 add dstq, strideq
683 dec hd
684 jg .w64
685 RET
686
687 ; The ipred_filter SIMD processes 4x2 blocks in the following order which
688 ; increases parallelism compared to doing things row by row.
689 ; w4 w8 w16 w32
690 ; 1 1 2 1 2 5 6 1 2 5 6 9 a d e
691 ; 2 2 3 2 3 6 7 2 3 6 7 a b e f
692 ; 3 3 4 3 4 7 8 3 4 7 8 b c f g
693 ; 4 4 5 4 5 8 9 4 5 8 9 c d g h
694
695 cglobal ipred_filter_16bpc, 4, 7, 14, dst, stride, tl, w, h, filter, top
696 %define base r6-$$
697 lea r6, [$$]
698 %ifidn filterd, filterm
699 movzx filterd, filterb
700 %else
701 movzx filterd, byte filterm
702 %endif
703 shl filterd, 6
704 movifnidn hd, hm
705 movu xm0, [tlq-6]
706 pmovsxbw m7, [base+filter_intra_taps+filterq+32*0]
707 pmovsxbw m8, [base+filter_intra_taps+filterq+32*1]
708 mov r5d, r8m ; bitdepth_max
709 movsldup m9, [base+filter_permA]
710 movshdup m10, [base+filter_permA]
711 shr r5d, 11 ; is_12bpc
712 jnz .12bpc
713 psllw m7, 2 ; upshift multipliers so that packusdw
714 psllw m8, 2 ; will perform clipping for free
715 .12bpc:
716 vpbroadcastd m5, [base+filter_rnd+r5*8]
717 vpbroadcastd m6, [base+filter_shift+r5*8]
718 sub wd, 8
719 jl .w4
720 .w8:
721 call .main4
722 movsldup m11, [filter_permB]
723 lea r5d, [hq*2+2]
724 movshdup m12, [filter_permB]
725 lea topq, [tlq+2]
726 mova m13, [filter_permC]
727 sub hd, 4
728 vinserti32x4 ym0, [topq], 1 ; a0 b0 t0 t1
729 sub tlq, r5
730 %if WIN64
731 push r7
732 push r8
733 %endif
734 mov r7, dstq
735 mov r8d, hd
736 .w8_loop:
737 movlps xm4, xm0, [tlq+hq*2]
738 call .main8
739 lea dstq, [dstq+strideq*2]
740 sub hd, 2
741 jge .w8_loop
742 test wd, wd
743 jz .end
744 mov r2d, 0x0d
745 kmovb k1, r2d
746 lea r2, [strideq*3]
747 .w16:
748 movd xmm0, [r7+strideq*1+12]
749 vpblendd xmm0, [topq+8], 0x0e ; t1 t2
750 pinsrw xm4, xmm0, [r7+strideq*0+14], 2
751 call .main8
752 add r7, 16
753 vinserti32x4 ym0, [topq+16], 1 ; a2 b2 t2 t3
754 mov hd, r8d
755 mov dstq, r7
756 add topq, 16
757 .w16_loop:
758 movd xmm1, [dstq+strideq*2-4]
759 punpcklwd xm4, xmm1, xmm0
760 movd xmm0, [dstq+r2-4]
761 shufps xm4{k1}, xmm0, xm0, q3210
762 call .main8
763 lea dstq, [dstq+strideq*2]
764 sub hd, 2
765 jge .w16_loop
766 sub wd, 8
767 jg .w16
768 .end:
769 vpermb m2, m11, m0
770 mova ym1, ym5
771 vpdpwssd m1, m2, m7
772 vpermb m2, m12, m0
773 vpdpwssd m1, m2, m8
774 %if WIN64
775 pop r8
776 pop r7
777 %endif
778 vextracti32x8 ym2, m1, 1
779 paddd ym1, ym2
780 packusdw ym1, ym1
781 vpsrlvw ym1, ym6
782 vpermt2q m0, m13, m1
783 vextracti32x4 [dstq+strideq*0], m0, 2
784 vextracti32x4 [dstq+strideq*1], ym0, 1
785 RET
786 .w4_loop:
787 movlps xm0, [tlq-10]
788 lea dstq, [dstq+strideq*2]
789 sub tlq, 4
790 .w4:
791 call .main4
792 movq [dstq+strideq*0], xm0
793 movhps [dstq+strideq*1], xm0
794 sub hd, 2
795 jg .w4_loop
796 RET
797 ALIGN function_align
798 .main4:
799 vpermb m2, m9, m0
800 mova ym1, ym5
801 vpdpwssd m1, m2, m7
802 vpermb m0, m10, m0
803 vpdpwssd m1, m0, m8
804 vextracti32x8 ym0, m1, 1
805 paddd ym0, ym1
806 vextracti32x4 xm1, ym0, 1
807 packusdw xm0, xm1 ; clip
808 vpsrlvw xm0, xm6
809 ret
810 ALIGN function_align
811 .main8:
812 vpermb m3, m11, m0
813 mova ym2, ym5
814 vpdpwssd m2, m3, m7
815 vpermb m3, m9, m4
816 mova ym1, ym5
817 vpdpwssd m1, m3, m7
818 vpermb m3, m12, m0
819 vpdpwssd m2, m3, m8
820 vpermb m3, m10, m4
821 vpdpwssd m1, m3, m8
822 vextracti32x8 ym4, m2, 1
823 vextracti32x8 ym3, m1, 1
824 paddd ym2, ym4
825 paddd ym1, ym3
826 packusdw ym1, ym2 ; clip
827 vpsrlvw ym1, ym6
828 vpermt2q m0, m13, m1 ; c0 d0 b0 b1 a0 a1
829 vextracti32x4 [dstq+strideq*0], m0, 2
830 vextracti32x4 [dstq+strideq*1], ym0, 1
831 ret
832
833 %endif