Debian Codegraph @

.cargo/vendor/rav1e/src/x86/mc16_avx2.asm

raw
1 ; Copyright © 2021, VideoLAN and dav1d authors
2 ; Copyright © 2021, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28
29 %if ARCH_X86_64
30
31 SECTION_RODATA 64
32
33 ; dav1d_obmc_masks[] * -512
34 const obmc_masks_avx2
35 dw 0, 0, -9728, 0, -12800, -7168, -2560, 0
36 dw -14336, -11264, -8192, -5632, -3584, -1536, 0, 0
37 dw -15360, -13824, -12288, -10752, -9216, -7680, -6144, -5120
38 dw -4096, -3072, -2048, -1536, 0, 0, 0, 0
39 dw -15872, -14848, -14336, -13312, -12288, -11776, -10752, -10240
40 dw -9728, -8704, -8192, -7168, -6656, -6144, -5632, -4608
41 dw -4096, -3584, -3072, -2560, -2048, -2048, -1536, -1024
42 dw 0, 0, 0, 0, 0, 0, 0, 0
43
44 deint_shuf: dd 0, 4, 1, 5, 2, 6, 3, 7
45 subpel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9
46 subpel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13
47 subpel_h_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9
48 subpel_s_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7
49 subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
50 rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7
51 rescale_mul2: dd 0, 1, 4, 5, 2, 3, 6, 7
52 resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7
53 db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15
54 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
55 wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
56 bdct_lb_q: times 8 db 0
57 times 8 db 4
58 times 8 db 8
59 times 8 db 12
60
61 prep_mul: dw 16, 16, 4, 4
62 put_bilin_h_rnd: dw 8, 8, 10, 10
63 put_8tap_h_rnd: dd 34, 40
64 s_8tap_h_rnd: dd 2, 8
65 s_8tap_h_sh: dd 2, 4
66 put_s_8tap_v_rnd: dd 512, 128
67 put_s_8tap_v_sh: dd 10, 8
68 prep_8tap_1d_rnd: dd 8 - (8192 << 4)
69 prep_8tap_2d_rnd: dd 32 - (8192 << 5)
70 warp8x8t_rnd: dd 16384 - (8192 << 15)
71 warp8x8_shift: dd 5, 3
72 warp8x8_rnd: dw 4096, 4096, 16384, 16384
73 bidir_rnd: dw -16400, -16400, -16388, -16388
74 bidir_mul: dw 2048, 2048, 8192, 8192
75
76 %define pw_16 prep_mul
77 %define pd_512 put_s_8tap_v_rnd
78
79 pw_2: times 2 dw 2
80 pw_64: times 2 dw 64
81 pw_2048: times 2 dw 2048
82 pw_8192: times 2 dw 8192
83 pw_27615: times 2 dw 27615
84 pw_32766: times 2 dw 32766
85 pw_m512: times 2 dw -512
86 pd_32: dd 32
87 pd_63: dd 63
88 pd_64: dd 64
89 pd_32768: dd 32768
90 pd_65538: dd 65538
91 pd_m524256: dd -524256 ; -8192 << 6 + 32
92 pd_0x3ff: dd 0x3ff
93 pq_0x40000000: dq 0x40000000
94 dd 0
95
96 %macro BIDIR_JMP_TABLE 2-*
97 %xdefine %1_%2_table (%%table - 2*%3)
98 %xdefine %%base %1_%2_table
99 %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2)
100 %%table:
101 %rep %0 - 2
102 dd %%prefix %+ .w%3 - %%base
103 %rotate 1
104 %endrep
105 %endmacro
106
107 BIDIR_JMP_TABLE avg, avx2, 4, 8, 16, 32, 64, 128
108 BIDIR_JMP_TABLE w_avg, avx2, 4, 8, 16, 32, 64, 128
109 BIDIR_JMP_TABLE mask, avx2, 4, 8, 16, 32, 64, 128
110 BIDIR_JMP_TABLE w_mask_420, avx2, 4, 8, 16, 32, 64, 128
111 BIDIR_JMP_TABLE w_mask_422, avx2, 4, 8, 16, 32, 64, 128
112 BIDIR_JMP_TABLE w_mask_444, avx2, 4, 8, 16, 32, 64, 128
113 BIDIR_JMP_TABLE blend, avx2, 4, 8, 16, 32
114 BIDIR_JMP_TABLE blend_v, avx2, 2, 4, 8, 16, 32
115 BIDIR_JMP_TABLE blend_h, avx2, 2, 4, 8, 16, 32, 64, 128
116
117 %macro BASE_JMP_TABLE 3-*
118 %xdefine %1_%2_table (%%table - %3)
119 %xdefine %%base %1_%2
120 %%table:
121 %rep %0 - 2
122 dw %%base %+ _w%3 - %%base
123 %rotate 1
124 %endrep
125 %endmacro
126
127 %xdefine put_avx2 mangle(private_prefix %+ _put_bilin_16bpc_avx2.put)
128 %xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_16bpc_avx2.prep)
129
130 BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128
131 BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128
132
133 %macro HV_JMP_TABLE 5-*
134 %xdefine %%prefix mangle(private_prefix %+ _%1_%2_16bpc_%3)
135 %xdefine %%base %1_%3
136 %assign %%types %4
137 %if %%types & 1
138 %xdefine %1_%2_h_%3_table (%%h - %5)
139 %%h:
140 %rep %0 - 4
141 dw %%prefix %+ .h_w%5 - %%base
142 %rotate 1
143 %endrep
144 %rotate 4
145 %endif
146 %if %%types & 2
147 %xdefine %1_%2_v_%3_table (%%v - %5)
148 %%v:
149 %rep %0 - 4
150 dw %%prefix %+ .v_w%5 - %%base
151 %rotate 1
152 %endrep
153 %rotate 4
154 %endif
155 %if %%types & 4
156 %xdefine %1_%2_hv_%3_table (%%hv - %5)
157 %%hv:
158 %rep %0 - 4
159 dw %%prefix %+ .hv_w%5 - %%base
160 %rotate 1
161 %endrep
162 %endif
163 %endmacro
164
165 HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128
166 HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128
167
168 %macro SCALED_JMP_TABLE 2-*
169 %xdefine %1_%2_table (%%table - %3)
170 %xdefine %%base mangle(private_prefix %+ _%1_16bpc_%2)
171 %%table:
172 %rep %0 - 2
173 dw %%base %+ .w%3 - %%base
174 %rotate 1
175 %endrep
176 %rotate 2
177 %%dy_1024:
178 %xdefine %1_%2_dy1_table (%%dy_1024 - %3)
179 %rep %0 - 2
180 dw %%base %+ .dy1_w%3 - %%base
181 %rotate 1
182 %endrep
183 %rotate 2
184 %%dy_2048:
185 %xdefine %1_%2_dy2_table (%%dy_2048 - %3)
186 %rep %0 - 2
187 dw %%base %+ .dy2_w%3 - %%base
188 %rotate 1
189 %endrep
190 %endmacro
191
192 SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128
193 SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128
194
195 %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
196
197 cextern mc_subpel_filters
198 %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
199
200 cextern mc_warp_filter
201 cextern resize_filter
202
203 SECTION .text
204
205 INIT_XMM avx2
206 cglobal put_bilin_16bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
207 mov mxyd, r6m ; mx
208 lea r7, [put_avx2]
209 %if UNIX64
210 DECLARE_REG_TMP 8
211 %define org_w r8d
212 mov r8d, wd
213 %else
214 DECLARE_REG_TMP 7
215 %define org_w wm
216 %endif
217 tzcnt wd, wm
218 movifnidn hd, hm
219 test mxyd, mxyd
220 jnz .h
221 mov mxyd, r7m ; my
222 test mxyd, mxyd
223 jnz .v
224 .put:
225 movzx wd, word [r7+wq*2+table_offset(put,)]
226 add wq, r7
227 jmp wq
228 .put_w2:
229 mov r6d, [srcq+ssq*0]
230 mov r7d, [srcq+ssq*1]
231 lea srcq, [srcq+ssq*2]
232 mov [dstq+dsq*0], r6d
233 mov [dstq+dsq*1], r7d
234 lea dstq, [dstq+dsq*2]
235 sub hd, 2
236 jg .put_w2
237 RET
238 .put_w4:
239 mov r6, [srcq+ssq*0]
240 mov r7, [srcq+ssq*1]
241 lea srcq, [srcq+ssq*2]
242 mov [dstq+dsq*0], r6
243 mov [dstq+dsq*1], r7
244 lea dstq, [dstq+dsq*2]
245 sub hd, 2
246 jg .put_w4
247 RET
248 .put_w8:
249 movu m0, [srcq+ssq*0]
250 movu m1, [srcq+ssq*1]
251 lea srcq, [srcq+ssq*2]
252 mova [dstq+dsq*0], m0
253 mova [dstq+dsq*1], m1
254 lea dstq, [dstq+dsq*2]
255 sub hd, 2
256 jg .put_w8
257 RET
258 INIT_YMM avx2
259 .put_w16:
260 movu m0, [srcq+ssq*0]
261 movu m1, [srcq+ssq*1]
262 lea srcq, [srcq+ssq*2]
263 mova [dstq+dsq*0], m0
264 mova [dstq+dsq*1], m1
265 lea dstq, [dstq+dsq*2]
266 sub hd, 2
267 jg .put_w16
268 RET
269 .put_w32:
270 movu m0, [srcq+ssq*0+32*0]
271 movu m1, [srcq+ssq*0+32*1]
272 movu m2, [srcq+ssq*1+32*0]
273 movu m3, [srcq+ssq*1+32*1]
274 lea srcq, [srcq+ssq*2]
275 mova [dstq+dsq*0+32*0], m0
276 mova [dstq+dsq*0+32*1], m1
277 mova [dstq+dsq*1+32*0], m2
278 mova [dstq+dsq*1+32*1], m3
279 lea dstq, [dstq+dsq*2]
280 sub hd, 2
281 jg .put_w32
282 RET
283 .put_w64:
284 movu m0, [srcq+32*0]
285 movu m1, [srcq+32*1]
286 movu m2, [srcq+32*2]
287 movu m3, [srcq+32*3]
288 add srcq, ssq
289 mova [dstq+32*0], m0
290 mova [dstq+32*1], m1
291 mova [dstq+32*2], m2
292 mova [dstq+32*3], m3
293 add dstq, dsq
294 dec hd
295 jg .put_w64
296 RET
297 .put_w128:
298 movu m0, [srcq+32*0]
299 movu m1, [srcq+32*1]
300 movu m2, [srcq+32*2]
301 movu m3, [srcq+32*3]
302 mova [dstq+32*0], m0
303 mova [dstq+32*1], m1
304 mova [dstq+32*2], m2
305 mova [dstq+32*3], m3
306 movu m0, [srcq+32*4]
307 movu m1, [srcq+32*5]
308 movu m2, [srcq+32*6]
309 movu m3, [srcq+32*7]
310 add srcq, ssq
311 mova [dstq+32*4], m0
312 mova [dstq+32*5], m1
313 mova [dstq+32*6], m2
314 mova [dstq+32*7], m3
315 add dstq, dsq
316 dec hd
317 jg .put_w128
318 RET
319 .h:
320 movd xm5, mxyd
321 mov mxyd, r7m ; my
322 vpbroadcastd m4, [pw_16]
323 vpbroadcastw m5, xm5
324 psubw m4, m5
325 test mxyd, mxyd
326 jnz .hv
327 ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v
328 movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)]
329 mov r6d, r8m ; bitdepth_max
330 add wq, r7
331 shr r6d, 11
332 vpbroadcastd m3, [r7-put_avx2+put_bilin_h_rnd+r6*4]
333 jmp wq
334 .h_w2:
335 movq xm1, [srcq+ssq*0]
336 movhps xm1, [srcq+ssq*1]
337 lea srcq, [srcq+ssq*2]
338 pmullw xm0, xm4, xm1
339 psrlq xm1, 16
340 pmullw xm1, xm5
341 paddw xm0, xm3
342 paddw xm0, xm1
343 psrlw xm0, 4
344 movd [dstq+dsq*0], xm0
345 pextrd [dstq+dsq*1], xm0, 2
346 lea dstq, [dstq+dsq*2]
347 sub hd, 2
348 jg .h_w2
349 RET
350 .h_w4:
351 movq xm0, [srcq+ssq*0]
352 movhps xm0, [srcq+ssq*1]
353 movq xm1, [srcq+ssq*0+2]
354 movhps xm1, [srcq+ssq*1+2]
355 lea srcq, [srcq+ssq*2]
356 pmullw xm0, xm4
357 pmullw xm1, xm5
358 paddw xm0, xm3
359 paddw xm0, xm1
360 psrlw xm0, 4
361 movq [dstq+dsq*0], xm0
362 movhps [dstq+dsq*1], xm0
363 lea dstq, [dstq+dsq*2]
364 sub hd, 2
365 jg .h_w4
366 RET
367 .h_w8:
368 movu xm0, [srcq+ssq*0]
369 vinserti128 m0, [srcq+ssq*1], 1
370 movu xm1, [srcq+ssq*0+2]
371 vinserti128 m1, [srcq+ssq*1+2], 1
372 lea srcq, [srcq+ssq*2]
373 pmullw m0, m4
374 pmullw m1, m5
375 paddw m0, m3
376 paddw m0, m1
377 psrlw m0, 4
378 mova [dstq+dsq*0], xm0
379 vextracti128 [dstq+dsq*1], m0, 1
380 lea dstq, [dstq+dsq*2]
381 sub hd, 2
382 jg .h_w8
383 RET
384 .h_w16:
385 pmullw m0, m4, [srcq+ssq*0]
386 pmullw m1, m5, [srcq+ssq*0+2]
387 paddw m0, m3
388 paddw m0, m1
389 pmullw m1, m4, [srcq+ssq*1]
390 pmullw m2, m5, [srcq+ssq*1+2]
391 lea srcq, [srcq+ssq*2]
392 paddw m1, m3
393 paddw m1, m2
394 psrlw m0, 4
395 psrlw m1, 4
396 mova [dstq+dsq*0], m0
397 mova [dstq+dsq*1], m1
398 lea dstq, [dstq+dsq*2]
399 sub hd, 2
400 jg .h_w16
401 RET
402 .h_w32:
403 pmullw m0, m4, [srcq+32*0]
404 pmullw m1, m5, [srcq+32*0+2]
405 paddw m0, m3
406 paddw m0, m1
407 pmullw m1, m4, [srcq+32*1]
408 pmullw m2, m5, [srcq+32*1+2]
409 add srcq, ssq
410 paddw m1, m3
411 paddw m1, m2
412 psrlw m0, 4
413 psrlw m1, 4
414 mova [dstq+32*0], m0
415 mova [dstq+32*1], m1
416 add dstq, dsq
417 dec hd
418 jg .h_w32
419 RET
420 .h_w64:
421 .h_w128:
422 movifnidn t0d, org_w
423 .h_w64_loop0:
424 mov r6d, t0d
425 .h_w64_loop:
426 pmullw m0, m4, [srcq+r6*2-32*1]
427 pmullw m1, m5, [srcq+r6*2-32*1+2]
428 paddw m0, m3
429 paddw m0, m1
430 pmullw m1, m4, [srcq+r6*2-32*2]
431 pmullw m2, m5, [srcq+r6*2-32*2+2]
432 paddw m1, m3
433 paddw m1, m2
434 psrlw m0, 4
435 psrlw m1, 4
436 mova [dstq+r6*2-32*1], m0
437 mova [dstq+r6*2-32*2], m1
438 sub r6d, 32
439 jg .h_w64_loop
440 add srcq, ssq
441 add dstq, dsq
442 dec hd
443 jg .h_w64_loop0
444 RET
445 .v:
446 movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)]
447 shl mxyd, 11
448 movd xm5, mxyd
449 add wq, r7
450 vpbroadcastw m5, xm5
451 jmp wq
452 .v_w2:
453 movd xm0, [srcq+ssq*0]
454 .v_w2_loop:
455 movd xm1, [srcq+ssq*1]
456 lea srcq, [srcq+ssq*2]
457 punpckldq xm2, xm0, xm1
458 movd xm0, [srcq+ssq*0]
459 punpckldq xm1, xm0
460 psubw xm1, xm2
461 pmulhrsw xm1, xm5
462 paddw xm1, xm2
463 movd [dstq+dsq*0], xm1
464 pextrd [dstq+dsq*1], xm1, 1
465 lea dstq, [dstq+dsq*2]
466 sub hd, 2
467 jg .v_w2_loop
468 RET
469 .v_w4:
470 movq xm0, [srcq+ssq*0]
471 .v_w4_loop:
472 movq xm1, [srcq+ssq*1]
473 lea srcq, [srcq+ssq*2]
474 punpcklqdq xm2, xm0, xm1
475 movq xm0, [srcq+ssq*0]
476 punpcklqdq xm1, xm0
477 psubw xm1, xm2
478 pmulhrsw xm1, xm5
479 paddw xm1, xm2
480 movq [dstq+dsq*0], xm1
481 movhps [dstq+dsq*1], xm1
482 lea dstq, [dstq+dsq*2]
483 sub hd, 2
484 jg .v_w4_loop
485 RET
486 .v_w8:
487 movu xm0, [srcq+ssq*0]
488 .v_w8_loop:
489 vbroadcasti128 m1, [srcq+ssq*1]
490 lea srcq, [srcq+ssq*2]
491 vpblendd m2, m0, m1, 0xf0
492 vbroadcasti128 m0, [srcq+ssq*0]
493 vpblendd m1, m0, 0xf0
494 psubw m1, m2
495 pmulhrsw m1, m5
496 paddw m1, m2
497 mova [dstq+dsq*0], xm1
498 vextracti128 [dstq+dsq*1], m1, 1
499 lea dstq, [dstq+dsq*2]
500 sub hd, 2
501 jg .v_w8_loop
502 RET
503 .v_w32:
504 movu m0, [srcq+ssq*0+32*0]
505 movu m1, [srcq+ssq*0+32*1]
506 .v_w32_loop:
507 movu m2, [srcq+ssq*1+32*0]
508 movu m3, [srcq+ssq*1+32*1]
509 lea srcq, [srcq+ssq*2]
510 psubw m4, m2, m0
511 pmulhrsw m4, m5
512 paddw m4, m0
513 movu m0, [srcq+ssq*0+32*0]
514 mova [dstq+dsq*0+32*0], m4
515 psubw m4, m3, m1
516 pmulhrsw m4, m5
517 paddw m4, m1
518 movu m1, [srcq+ssq*0+32*1]
519 mova [dstq+dsq*0+32*1], m4
520 psubw m4, m0, m2
521 pmulhrsw m4, m5
522 paddw m4, m2
523 mova [dstq+dsq*1+32*0], m4
524 psubw m4, m1, m3
525 pmulhrsw m4, m5
526 paddw m4, m3
527 mova [dstq+dsq*1+32*1], m4
528 lea dstq, [dstq+dsq*2]
529 sub hd, 2
530 jg .v_w32_loop
531 RET
532 .v_w16:
533 .v_w64:
534 .v_w128:
535 movifnidn t0d, org_w
536 add t0d, t0d
537 mov r4, srcq
538 lea r6d, [hq+t0*8-256]
539 mov r7, dstq
540 .v_w16_loop0:
541 movu m0, [srcq+ssq*0]
542 .v_w16_loop:
543 movu m3, [srcq+ssq*1]
544 lea srcq, [srcq+ssq*2]
545 psubw m1, m3, m0
546 pmulhrsw m1, m5
547 paddw m1, m0
548 movu m0, [srcq+ssq*0]
549 psubw m2, m0, m3
550 pmulhrsw m2, m5
551 paddw m2, m3
552 mova [dstq+dsq*0], m1
553 mova [dstq+dsq*1], m2
554 lea dstq, [dstq+dsq*2]
555 sub hd, 2
556 jg .v_w16_loop
557 add r4, 32
558 add r7, 32
559 movzx hd, r6b
560 mov srcq, r4
561 mov dstq, r7
562 sub r6d, 1<<8
563 jg .v_w16_loop0
564 RET
565 .hv:
566 movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)]
567 WIN64_SPILL_XMM 8
568 shl mxyd, 11
569 vpbroadcastd m3, [pw_2]
570 movd xm6, mxyd
571 vpbroadcastd m7, [pw_8192]
572 add wq, r7
573 vpbroadcastw m6, xm6
574 test dword r8m, 0x800
575 jnz .hv_12bpc
576 psllw m4, 2
577 psllw m5, 2
578 vpbroadcastd m7, [pw_2048]
579 .hv_12bpc:
580 jmp wq
581 .hv_w2:
582 vpbroadcastq xm1, [srcq+ssq*0]
583 pmullw xm0, xm4, xm1
584 psrlq xm1, 16
585 pmullw xm1, xm5
586 paddw xm0, xm3
587 paddw xm0, xm1
588 psrlw xm0, 2
589 .hv_w2_loop:
590 movq xm2, [srcq+ssq*1]
591 lea srcq, [srcq+ssq*2]
592 movhps xm2, [srcq+ssq*0]
593 pmullw xm1, xm4, xm2
594 psrlq xm2, 16
595 pmullw xm2, xm5
596 paddw xm1, xm3
597 paddw xm1, xm2
598 psrlw xm1, 2 ; 1 _ 2 _
599 shufpd xm2, xm0, xm1, 0x01 ; 0 _ 1 _
600 mova xm0, xm1
601 psubw xm1, xm2
602 paddw xm1, xm1
603 pmulhw xm1, xm6
604 paddw xm1, xm2
605 pmulhrsw xm1, xm7
606 movd [dstq+dsq*0], xm1
607 pextrd [dstq+dsq*1], xm1, 2
608 lea dstq, [dstq+dsq*2]
609 sub hd, 2
610 jg .hv_w2_loop
611 RET
612 .hv_w4:
613 pmullw xm0, xm4, [srcq+ssq*0-8]
614 pmullw xm1, xm5, [srcq+ssq*0-6]
615 paddw xm0, xm3
616 paddw xm0, xm1
617 psrlw xm0, 2
618 .hv_w4_loop:
619 movq xm1, [srcq+ssq*1]
620 movq xm2, [srcq+ssq*1+2]
621 lea srcq, [srcq+ssq*2]
622 movhps xm1, [srcq+ssq*0]
623 movhps xm2, [srcq+ssq*0+2]
624 pmullw xm1, xm4
625 pmullw xm2, xm5
626 paddw xm1, xm3
627 paddw xm1, xm2
628 psrlw xm1, 2 ; 1 2
629 shufpd xm2, xm0, xm1, 0x01 ; 0 1
630 mova xm0, xm1
631 psubw xm1, xm2
632 paddw xm1, xm1
633 pmulhw xm1, xm6
634 paddw xm1, xm2
635 pmulhrsw xm1, xm7
636 movq [dstq+dsq*0], xm1
637 movhps [dstq+dsq*1], xm1
638 lea dstq, [dstq+dsq*2]
639 sub hd, 2
640 jg .hv_w4_loop
641 RET
642 .hv_w8:
643 pmullw xm0, xm4, [srcq+ssq*0]
644 pmullw xm1, xm5, [srcq+ssq*0+2]
645 paddw xm0, xm3
646 paddw xm0, xm1
647 psrlw xm0, 2
648 vinserti128 m0, xm0, 1
649 .hv_w8_loop:
650 movu xm1, [srcq+ssq*1]
651 movu xm2, [srcq+ssq*1+2]
652 lea srcq, [srcq+ssq*2]
653 vinserti128 m1, [srcq+ssq*0], 1
654 vinserti128 m2, [srcq+ssq*0+2], 1
655 pmullw m1, m4
656 pmullw m2, m5
657 paddw m1, m3
658 paddw m1, m2
659 psrlw m1, 2 ; 1 2
660 vperm2i128 m2, m0, m1, 0x21 ; 0 1
661 mova m0, m1
662 psubw m1, m2
663 paddw m1, m1
664 pmulhw m1, m6
665 paddw m1, m2
666 pmulhrsw m1, m7
667 mova [dstq+dsq*0], xm1
668 vextracti128 [dstq+dsq*1], m1, 1
669 lea dstq, [dstq+dsq*2]
670 sub hd, 2
671 jg .hv_w8_loop
672 RET
673 .hv_w16:
674 .hv_w32:
675 .hv_w64:
676 .hv_w128:
677 %if UNIX64
678 lea r6d, [r8*2-32]
679 %else
680 mov r6d, wm
681 lea r6d, [r6*2-32]
682 %endif
683 mov r4, srcq
684 lea r6d, [hq+r6*8]
685 mov r7, dstq
686 .hv_w16_loop0:
687 pmullw m0, m4, [srcq+ssq*0]
688 pmullw m1, m5, [srcq+ssq*0+2]
689 paddw m0, m3
690 paddw m0, m1
691 psrlw m0, 2
692 .hv_w16_loop:
693 pmullw m1, m4, [srcq+ssq*1]
694 pmullw m2, m5, [srcq+ssq*1+2]
695 lea srcq, [srcq+ssq*2]
696 paddw m1, m3
697 paddw m1, m2
698 psrlw m1, 2
699 psubw m2, m1, m0
700 paddw m2, m2
701 pmulhw m2, m6
702 paddw m2, m0
703 pmulhrsw m2, m7
704 mova [dstq+dsq*0], m2
705 pmullw m0, m4, [srcq+ssq*0]
706 pmullw m2, m5, [srcq+ssq*0+2]
707 paddw m0, m3
708 paddw m0, m2
709 psrlw m0, 2
710 psubw m2, m0, m1
711 paddw m2, m2
712 pmulhw m2, m6
713 paddw m2, m1
714 pmulhrsw m2, m7
715 mova [dstq+dsq*1], m2
716 lea dstq, [dstq+dsq*2]
717 sub hd, 2
718 jg .hv_w16_loop
719 add r4, 32
720 add r7, 32
721 movzx hd, r6b
722 mov srcq, r4
723 mov dstq, r7
724 sub r6d, 1<<8
725 jg .hv_w16_loop0
726 RET
727
728 cglobal prep_bilin_16bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
729 movifnidn mxyd, r5m ; mx
730 lea r6, [prep_avx2]
731 %if UNIX64
732 DECLARE_REG_TMP 7
733 %define org_w r7d
734 %else
735 DECLARE_REG_TMP 6
736 %define org_w r5m
737 %endif
738 mov org_w, wd
739 tzcnt wd, wm
740 movifnidn hd, hm
741 test mxyd, mxyd
742 jnz .h
743 mov mxyd, r6m ; my
744 test mxyd, mxyd
745 jnz .v
746 .prep:
747 movzx wd, word [r6+wq*2+table_offset(prep,)]
748 mov r5d, r7m ; bitdepth_max
749 vpbroadcastd m5, [r6-prep_avx2+pw_8192]
750 add wq, r6
751 shr r5d, 11
752 vpbroadcastd m4, [r6-prep_avx2+prep_mul+r5*4]
753 lea stride3q, [strideq*3]
754 jmp wq
755 .prep_w4:
756 movq xm0, [srcq+strideq*0]
757 movhps xm0, [srcq+strideq*1]
758 vpbroadcastq m1, [srcq+strideq*2]
759 vpbroadcastq m2, [srcq+stride3q ]
760 lea srcq, [srcq+strideq*4]
761 vpblendd m0, m1, 0x30
762 vpblendd m0, m2, 0xc0
763 pmullw m0, m4
764 psubw m0, m5
765 mova [tmpq], m0
766 add tmpq, 32
767 sub hd, 4
768 jg .prep_w4
769 RET
770 .prep_w8:
771 movu xm0, [srcq+strideq*0]
772 vinserti128 m0, [srcq+strideq*1], 1
773 movu xm1, [srcq+strideq*2]
774 vinserti128 m1, [srcq+stride3q ], 1
775 lea srcq, [srcq+strideq*4]
776 pmullw m0, m4
777 pmullw m1, m4
778 psubw m0, m5
779 psubw m1, m5
780 mova [tmpq+32*0], m0
781 mova [tmpq+32*1], m1
782 add tmpq, 32*2
783 sub hd, 4
784 jg .prep_w8
785 RET
786 .prep_w16:
787 pmullw m0, m4, [srcq+strideq*0]
788 pmullw m1, m4, [srcq+strideq*1]
789 pmullw m2, m4, [srcq+strideq*2]
790 pmullw m3, m4, [srcq+stride3q ]
791 lea srcq, [srcq+strideq*4]
792 psubw m0, m5
793 psubw m1, m5
794 psubw m2, m5
795 psubw m3, m5
796 mova [tmpq+32*0], m0
797 mova [tmpq+32*1], m1
798 mova [tmpq+32*2], m2
799 mova [tmpq+32*3], m3
800 add tmpq, 32*4
801 sub hd, 4
802 jg .prep_w16
803 RET
804 .prep_w32:
805 pmullw m0, m4, [srcq+strideq*0+32*0]
806 pmullw m1, m4, [srcq+strideq*0+32*1]
807 pmullw m2, m4, [srcq+strideq*1+32*0]
808 pmullw m3, m4, [srcq+strideq*1+32*1]
809 lea srcq, [srcq+strideq*2]
810 psubw m0, m5
811 psubw m1, m5
812 psubw m2, m5
813 psubw m3, m5
814 mova [tmpq+32*0], m0
815 mova [tmpq+32*1], m1
816 mova [tmpq+32*2], m2
817 mova [tmpq+32*3], m3
818 add tmpq, 32*4
819 sub hd, 2
820 jg .prep_w32
821 RET
822 .prep_w64:
823 pmullw m0, m4, [srcq+32*0]
824 pmullw m1, m4, [srcq+32*1]
825 pmullw m2, m4, [srcq+32*2]
826 pmullw m3, m4, [srcq+32*3]
827 add srcq, strideq
828 psubw m0, m5
829 psubw m1, m5
830 psubw m2, m5
831 psubw m3, m5
832 mova [tmpq+32*0], m0
833 mova [tmpq+32*1], m1
834 mova [tmpq+32*2], m2
835 mova [tmpq+32*3], m3
836 add tmpq, 32*4
837 dec hd
838 jg .prep_w64
839 RET
840 .prep_w128:
841 pmullw m0, m4, [srcq+32*0]
842 pmullw m1, m4, [srcq+32*1]
843 pmullw m2, m4, [srcq+32*2]
844 pmullw m3, m4, [srcq+32*3]
845 psubw m0, m5
846 psubw m1, m5
847 psubw m2, m5
848 psubw m3, m5
849 mova [tmpq+32*0], m0
850 mova [tmpq+32*1], m1
851 mova [tmpq+32*2], m2
852 mova [tmpq+32*3], m3
853 pmullw m0, m4, [srcq+32*4]
854 pmullw m1, m4, [srcq+32*5]
855 pmullw m2, m4, [srcq+32*6]
856 pmullw m3, m4, [srcq+32*7]
857 add tmpq, 32*8
858 add srcq, strideq
859 psubw m0, m5
860 psubw m1, m5
861 psubw m2, m5
862 psubw m3, m5
863 mova [tmpq-32*4], m0
864 mova [tmpq-32*3], m1
865 mova [tmpq-32*2], m2
866 mova [tmpq-32*1], m3
867 dec hd
868 jg .prep_w128
869 RET
870 .h:
871 movd xm5, mxyd
872 mov mxyd, r6m ; my
873 vpbroadcastd m4, [pw_16]
874 vpbroadcastw m5, xm5
875 vpbroadcastd m3, [pw_32766]
876 psubw m4, m5
877 test dword r7m, 0x800
878 jnz .h_12bpc
879 psllw m4, 2
880 psllw m5, 2
881 .h_12bpc:
882 test mxyd, mxyd
883 jnz .hv
884 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)]
885 add wq, r6
886 lea stride3q, [strideq*3]
887 jmp wq
888 .h_w4:
889 movu xm1, [srcq+strideq*0]
890 vinserti128 m1, [srcq+strideq*2], 1
891 movu xm2, [srcq+strideq*1]
892 vinserti128 m2, [srcq+stride3q ], 1
893 lea srcq, [srcq+strideq*4]
894 punpcklqdq m0, m1, m2
895 psrldq m1, 2
896 pslldq m2, 6
897 pmullw m0, m4
898 vpblendd m1, m2, 0xcc
899 pmullw m1, m5
900 psubw m0, m3
901 paddw m0, m1
902 psraw m0, 2
903 mova [tmpq], m0
904 add tmpq, 32
905 sub hd, 4
906 jg .h_w4
907 RET
908 .h_w8:
909 movu xm0, [srcq+strideq*0]
910 vinserti128 m0, [srcq+strideq*1], 1
911 movu xm1, [srcq+strideq*0+2]
912 vinserti128 m1, [srcq+strideq*1+2], 1
913 lea srcq, [srcq+strideq*2]
914 pmullw m0, m4
915 pmullw m1, m5
916 psubw m0, m3
917 paddw m0, m1
918 psraw m0, 2
919 mova [tmpq], m0
920 add tmpq, 32
921 sub hd, 2
922 jg .h_w8
923 RET
924 .h_w16:
925 pmullw m0, m4, [srcq+strideq*0]
926 pmullw m1, m5, [srcq+strideq*0+2]
927 psubw m0, m3
928 paddw m0, m1
929 pmullw m1, m4, [srcq+strideq*1]
930 pmullw m2, m5, [srcq+strideq*1+2]
931 lea srcq, [srcq+strideq*2]
932 psubw m1, m3
933 paddw m1, m2
934 psraw m0, 2
935 psraw m1, 2
936 mova [tmpq+32*0], m0
937 mova [tmpq+32*1], m1
938 add tmpq, 32*2
939 sub hd, 2
940 jg .h_w16
941 RET
942 .h_w32:
943 .h_w64:
944 .h_w128:
945 movifnidn t0d, org_w
946 .h_w32_loop0:
947 mov r3d, t0d
948 .h_w32_loop:
949 pmullw m0, m4, [srcq+r3*2-32*1]
950 pmullw m1, m5, [srcq+r3*2-32*1+2]
951 psubw m0, m3
952 paddw m0, m1
953 pmullw m1, m4, [srcq+r3*2-32*2]
954 pmullw m2, m5, [srcq+r3*2-32*2+2]
955 psubw m1, m3
956 paddw m1, m2
957 psraw m0, 2
958 psraw m1, 2
959 mova [tmpq+r3*2-32*1], m0
960 mova [tmpq+r3*2-32*2], m1
961 sub r3d, 32
962 jg .h_w32_loop
963 add srcq, strideq
964 lea tmpq, [tmpq+t0*2]
965 dec hd
966 jg .h_w32_loop0
967 RET
968 .v:
969 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)]
970 movd xm5, mxyd
971 vpbroadcastd m4, [pw_16]
972 vpbroadcastw m5, xm5
973 vpbroadcastd m3, [pw_32766]
974 add wq, r6
975 lea stride3q, [strideq*3]
976 psubw m4, m5
977 test dword r7m, 0x800
978 jnz .v_12bpc
979 psllw m4, 2
980 psllw m5, 2
981 .v_12bpc:
982 jmp wq
983 .v_w4:
984 movq xm0, [srcq+strideq*0]
985 .v_w4_loop:
986 vpbroadcastq m2, [srcq+strideq*2]
987 vpbroadcastq xm1, [srcq+strideq*1]
988 vpblendd m2, m0, 0x03 ; 0 2 2 2
989 vpbroadcastq m0, [srcq+stride3q ]
990 lea srcq, [srcq+strideq*4]
991 vpblendd m1, m0, 0xf0 ; 1 1 3 3
992 vpbroadcastq m0, [srcq+strideq*0]
993 vpblendd m1, m2, 0x33 ; 0 1 2 3
994 vpblendd m0, m2, 0x0c ; 4 2 4 4
995 punpckhqdq m2, m1, m0 ; 1 2 3 4
996 pmullw m1, m4
997 pmullw m2, m5
998 psubw m1, m3
999 paddw m1, m2
1000 psraw m1, 2
1001 mova [tmpq], m1
1002 add tmpq, 32
1003 sub hd, 4
1004 jg .v_w4_loop
1005 RET
1006 .v_w8:
1007 movu xm0, [srcq+strideq*0]
1008 .v_w8_loop:
1009 vbroadcasti128 m2, [srcq+strideq*1]
1010 lea srcq, [srcq+strideq*2]
1011 vpblendd m1, m0, m2, 0xf0 ; 0 1
1012 vbroadcasti128 m0, [srcq+strideq*0]
1013 vpblendd m2, m0, 0xf0 ; 1 2
1014 pmullw m1, m4
1015 pmullw m2, m5
1016 psubw m1, m3
1017 paddw m1, m2
1018 psraw m1, 2
1019 mova [tmpq], m1
1020 add tmpq, 32
1021 sub hd, 2
1022 jg .v_w8_loop
1023 RET
1024 .v_w16:
1025 movu m0, [srcq+strideq*0]
1026 .v_w16_loop:
1027 movu m2, [srcq+strideq*1]
1028 lea srcq, [srcq+strideq*2]
1029 pmullw m0, m4
1030 pmullw m1, m5, m2
1031 psubw m0, m3
1032 paddw m1, m0
1033 movu m0, [srcq+strideq*0]
1034 psraw m1, 2
1035 pmullw m2, m4
1036 mova [tmpq+32*0], m1
1037 pmullw m1, m5, m0
1038 psubw m2, m3
1039 paddw m1, m2
1040 psraw m1, 2
1041 mova [tmpq+32*1], m1
1042 add tmpq, 32*2
1043 sub hd, 2
1044 jg .v_w16_loop
1045 RET
1046 .v_w32:
1047 .v_w64:
1048 .v_w128:
1049 %if WIN64
1050 PUSH r7
1051 %endif
1052 movifnidn r7d, org_w
1053 add r7d, r7d
1054 mov r3, srcq
1055 lea r6d, [hq+r7*8-256]
1056 mov r5, tmpq
1057 .v_w32_loop0:
1058 movu m0, [srcq+strideq*0]
1059 .v_w32_loop:
1060 movu m2, [srcq+strideq*1]
1061 lea srcq, [srcq+strideq*2]
1062 pmullw m0, m4
1063 pmullw m1, m5, m2
1064 psubw m0, m3
1065 paddw m1, m0
1066 movu m0, [srcq+strideq*0]
1067 psraw m1, 2
1068 pmullw m2, m4
1069 mova [tmpq+r7*0], m1
1070 pmullw m1, m5, m0
1071 psubw m2, m3
1072 paddw m1, m2
1073 psraw m1, 2
1074 mova [tmpq+r7*1], m1
1075 lea tmpq, [tmpq+r7*2]
1076 sub hd, 2
1077 jg .v_w32_loop
1078 add r3, 32
1079 add r5, 32
1080 movzx hd, r6b
1081 mov srcq, r3
1082 mov tmpq, r5
1083 sub r6d, 1<<8
1084 jg .v_w32_loop0
1085 %if WIN64
1086 POP r7
1087 %endif
1088 RET
1089 .hv:
1090 WIN64_SPILL_XMM 7
1091 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)]
1092 shl mxyd, 11
1093 movd xm6, mxyd
1094 add wq, r6
1095 lea stride3q, [strideq*3]
1096 vpbroadcastw m6, xm6
1097 jmp wq
1098 .hv_w4:
1099 movu xm1, [srcq+strideq*0]
1100 %if WIN64
1101 movaps [rsp+24], xmm7
1102 %endif
1103 pmullw xm0, xm4, xm1
1104 psrldq xm1, 2
1105 pmullw xm1, xm5
1106 psubw xm0, xm3
1107 paddw xm0, xm1
1108 psraw xm0, 2
1109 vpbroadcastq m0, xm0
1110 .hv_w4_loop:
1111 movu xm1, [srcq+strideq*1]
1112 vinserti128 m1, [srcq+stride3q ], 1
1113 movu xm2, [srcq+strideq*2]
1114 lea srcq, [srcq+strideq*4]
1115 vinserti128 m2, [srcq+strideq*0], 1
1116 punpcklqdq m7, m1, m2
1117 psrldq m1, 2
1118 pslldq m2, 6
1119 pmullw m7, m4
1120 vpblendd m1, m2, 0xcc
1121 pmullw m1, m5
1122 psubw m7, m3
1123 paddw m1, m7
1124 psraw m1, 2 ; 1 2 3 4
1125 vpblendd m0, m1, 0x3f
1126 vpermq m2, m0, q2103 ; 0 1 2 3
1127 mova m0, m1
1128 psubw m1, m2
1129 pmulhrsw m1, m6
1130 paddw m1, m2
1131 mova [tmpq], m1
1132 add tmpq, 32
1133 sub hd, 4
1134 jg .hv_w4_loop
1135 %if WIN64
1136 movaps xmm7, [rsp+24]
1137 %endif
1138 RET
1139 .hv_w8:
1140 pmullw xm0, xm4, [srcq+strideq*0]
1141 pmullw xm1, xm5, [srcq+strideq*0+2]
1142 psubw xm0, xm3
1143 paddw xm0, xm1
1144 psraw xm0, 2
1145 vinserti128 m0, xm0, 1
1146 .hv_w8_loop:
1147 movu xm1, [srcq+strideq*1]
1148 movu xm2, [srcq+strideq*1+2]
1149 lea srcq, [srcq+strideq*2]
1150 vinserti128 m1, [srcq+strideq*0], 1
1151 vinserti128 m2, [srcq+strideq*0+2], 1
1152 pmullw m1, m4
1153 pmullw m2, m5
1154 psubw m1, m3
1155 paddw m1, m2
1156 psraw m1, 2 ; 1 2
1157 vperm2i128 m2, m0, m1, 0x21 ; 0 1
1158 mova m0, m1
1159 psubw m1, m2
1160 pmulhrsw m1, m6
1161 paddw m1, m2
1162 mova [tmpq], m1
1163 add tmpq, 32
1164 sub hd, 2
1165 jg .hv_w8_loop
1166 RET
1167 .hv_w16:
1168 .hv_w32:
1169 .hv_w64:
1170 .hv_w128:
1171 %if WIN64
1172 PUSH r7
1173 %endif
1174 movifnidn r7d, org_w
1175 add r7d, r7d
1176 mov r3, srcq
1177 lea r6d, [hq+r7*8-256]
1178 mov r5, tmpq
1179 .hv_w16_loop0:
1180 pmullw m0, m4, [srcq]
1181 pmullw m1, m5, [srcq+2]
1182 psubw m0, m3
1183 paddw m0, m1
1184 psraw m0, 2
1185 .hv_w16_loop:
1186 pmullw m1, m4, [srcq+strideq*1]
1187 pmullw m2, m5, [srcq+strideq*1+2]
1188 lea srcq, [srcq+strideq*2]
1189 psubw m1, m3
1190 paddw m1, m2
1191 psraw m1, 2
1192 psubw m2, m1, m0
1193 pmulhrsw m2, m6
1194 paddw m2, m0
1195 mova [tmpq+r7*0], m2
1196 pmullw m0, m4, [srcq+strideq*0]
1197 pmullw m2, m5, [srcq+strideq*0+2]
1198 psubw m0, m3
1199 paddw m0, m2
1200 psraw m0, 2
1201 psubw m2, m0, m1
1202 pmulhrsw m2, m6
1203 paddw m2, m1
1204 mova [tmpq+r7*1], m2
1205 lea tmpq, [tmpq+r7*2]
1206 sub hd, 2
1207 jg .hv_w16_loop
1208 add r3, 32
1209 add r5, 32
1210 movzx hd, r6b
1211 mov srcq, r3
1212 mov tmpq, r5
1213 sub r6d, 1<<8
1214 jg .hv_w16_loop0
1215 %if WIN64
1216 POP r7
1217 %endif
1218 RET
1219
1220 ; int8_t subpel_filters[5][15][8]
1221 %assign FILTER_REGULAR (0*15 << 16) | 3*15
1222 %assign FILTER_SMOOTH (1*15 << 16) | 4*15
1223 %assign FILTER_SHARP (2*15 << 16) | 3*15
1224
1225 %macro FN 4 ; prefix, type, type_h, type_v
1226 cglobal %1_%2_16bpc
1227 mov t0d, FILTER_%3
1228 %ifidn %3, %4
1229 mov t1d, t0d
1230 %else
1231 mov t1d, FILTER_%4
1232 %endif
1233 %ifnidn %2, regular ; skip the jump in the last filter
1234 jmp mangle(private_prefix %+ _%1_16bpc %+ SUFFIX)
1235 %endif
1236 %endmacro
1237
1238 %if WIN64
1239 DECLARE_REG_TMP 4, 5
1240 %else
1241 DECLARE_REG_TMP 7, 8
1242 %endif
1243
1244 %define PUT_8TAP_FN FN put_8tap,
1245 PUT_8TAP_FN sharp, SHARP, SHARP
1246 PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH
1247 PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP
1248 PUT_8TAP_FN smooth, SMOOTH, SMOOTH
1249 PUT_8TAP_FN sharp_regular, SHARP, REGULAR
1250 PUT_8TAP_FN regular_sharp, REGULAR, SHARP
1251 PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR
1252 PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH
1253 PUT_8TAP_FN regular, REGULAR, REGULAR
1254
1255 cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my
1256 %define base r8-put_avx2
1257 imul mxd, mxm, 0x010101
1258 add mxd, t0d ; 8tap_h, mx, 4tap_h
1259 imul myd, mym, 0x010101
1260 add myd, t1d ; 8tap_v, my, 4tap_v
1261 lea r8, [put_avx2]
1262 movifnidn wd, wm
1263 movifnidn hd, hm
1264 test mxd, 0xf00
1265 jnz .h
1266 test myd, 0xf00
1267 jnz .v
1268 tzcnt wd, wd
1269 movzx wd, word [r8+wq*2+table_offset(put,)]
1270 add wq, r8
1271 %if WIN64
1272 pop r8
1273 %endif
1274 jmp wq
1275 .h_w2:
1276 movzx mxd, mxb
1277 sub srcq, 2
1278 mova xm2, [subpel_h_shuf2]
1279 vpbroadcastd xm3, [base+subpel_filters+mxq*8+2]
1280 pmovsxbw xm3, xm3
1281 .h_w2_loop:
1282 movu xm0, [srcq+ssq*0]
1283 movu xm1, [srcq+ssq*1]
1284 lea srcq, [srcq+ssq*2]
1285 pshufb xm0, xm2
1286 pshufb xm1, xm2
1287 pmaddwd xm0, xm3
1288 pmaddwd xm1, xm3
1289 phaddd xm0, xm1
1290 paddd xm0, xm4
1291 psrad xm0, 6
1292 packusdw xm0, xm0
1293 pminsw xm0, xm5
1294 movd [dstq+dsq*0], xm0
1295 pextrd [dstq+dsq*1], xm0, 1
1296 lea dstq, [dstq+dsq*2]
1297 sub hd, 2
1298 jg .h_w2_loop
1299 RET
1300 .h_w4:
1301 movzx mxd, mxb
1302 sub srcq, 2
1303 pmovsxbw xm3, [base+subpel_filters+mxq*8]
1304 WIN64_SPILL_XMM 8
1305 vbroadcasti128 m6, [subpel_h_shufA]
1306 vbroadcasti128 m7, [subpel_h_shufB]
1307 pshufd xm3, xm3, q2211
1308 vpbroadcastq m2, xm3
1309 vpermq m3, m3, q1111
1310 .h_w4_loop:
1311 movu xm1, [srcq+ssq*0]
1312 vinserti128 m1, [srcq+ssq*1], 1
1313 lea srcq, [srcq+ssq*2]
1314 pshufb m0, m1, m6 ; 0 1 1 2 2 3 3 4
1315 pshufb m1, m7 ; 2 3 3 4 4 5 5 6
1316 pmaddwd m0, m2
1317 pmaddwd m1, m3
1318 paddd m0, m4
1319 paddd m0, m1
1320 psrad m0, 6
1321 vextracti128 xm1, m0, 1
1322 packusdw xm0, xm1
1323 pminsw xm0, xm5
1324 movq [dstq+dsq*0], xm0
1325 movhps [dstq+dsq*1], xm0
1326 lea dstq, [dstq+dsq*2]
1327 sub hd, 2
1328 jg .h_w4_loop
1329 RET
1330 .h:
1331 test myd, 0xf00
1332 jnz .hv
1333 mov r7d, r8m
1334 vpbroadcastw m5, r8m
1335 shr r7d, 11
1336 vpbroadcastd m4, [base+put_8tap_h_rnd+r7*4]
1337 cmp wd, 4
1338 je .h_w4
1339 jl .h_w2
1340 %assign stack_offset stack_offset - stack_size_padded
1341 WIN64_SPILL_XMM 13
1342 shr mxd, 16
1343 sub srcq, 6
1344 vpbroadcastq m0, [base+subpel_filters+mxq*8]
1345 vbroadcasti128 m6, [subpel_h_shufA]
1346 vbroadcasti128 m7, [subpel_h_shufB]
1347 punpcklbw m0, m0
1348 psraw m0, 8 ; sign-extend
1349 pshufd m8, m0, q0000
1350 pshufd m9, m0, q1111
1351 pshufd m10, m0, q2222
1352 pshufd m11, m0, q3333
1353 cmp wd, 8
1354 jg .h_w16
1355 .h_w8:
1356 %macro PUT_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2]
1357 pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6
1358 pshufb m%1, m6 ; 0 1 1 2 2 3 3 4
1359 pmaddwd m%5, m9, m%4 ; abcd1
1360 pmaddwd m%1, m8 ; abcd0
1361 pshufb m%2, m7 ; 6 7 7 8 8 9 9 a
1362 shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8
1363 paddd m%5, m4
1364 paddd m%1, m%5
1365 pmaddwd m%5, m11, m%2 ; abcd3
1366 paddd m%1, m%5
1367 pmaddwd m%5, m10, m%4 ; abcd2
1368 pshufb m%3, m7 ; a b b c c d d e
1369 pmaddwd m%4, m8 ; efgh0
1370 paddd m%1, m%5
1371 pmaddwd m%5, m9, m%2 ; efgh1
1372 shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c
1373 pmaddwd m%3, m11 ; efgh3
1374 pmaddwd m%2, m10 ; efgh2
1375 paddd m%4, m4
1376 paddd m%4, m%5
1377 paddd m%3, m%4
1378 paddd m%2, m%3
1379 psrad m%1, 6
1380 psrad m%2, 6
1381 packusdw m%1, m%2
1382 pminsw m%1, m5
1383 %endmacro
1384 movu xm0, [srcq+ssq*0+ 0]
1385 vinserti128 m0, [srcq+ssq*1+ 0], 1
1386 movu xm2, [srcq+ssq*0+16]
1387 vinserti128 m2, [srcq+ssq*1+16], 1
1388 lea srcq, [srcq+ssq*2]
1389 shufpd m1, m0, m2, 0x05
1390 PUT_8TAP_H 0, 1, 2, 3, 12
1391 mova [dstq+dsq*0], xm0
1392 vextracti128 [dstq+dsq*1], m0, 1
1393 lea dstq, [dstq+dsq*2]
1394 sub hd, 2
1395 jg .h_w8
1396 RET
1397 .h_w16:
1398 mov r6d, wd
1399 .h_w16_loop:
1400 movu m0, [srcq+r6*2-32]
1401 movu m1, [srcq+r6*2-24]
1402 movu m2, [srcq+r6*2-16]
1403 PUT_8TAP_H 0, 1, 2, 3, 12
1404 mova [dstq+r6*2-32], m0
1405 sub r6d, 16
1406 jg .h_w16_loop
1407 add srcq, ssq
1408 add dstq, dsq
1409 dec hd
1410 jg .h_w16
1411 RET
1412 .v:
1413 movzx mxd, myb
1414 shr myd, 16
1415 cmp hd, 4
1416 cmovle myd, mxd
1417 vpbroadcastq m0, [base+subpel_filters+myq*8]
1418 %assign stack_offset stack_offset - stack_size_padded
1419 WIN64_SPILL_XMM 15
1420 vpbroadcastd m6, [pd_32]
1421 vpbroadcastw m7, r8m
1422 lea r6, [ssq*3]
1423 sub srcq, r6
1424 punpcklbw m0, m0
1425 psraw m0, 8 ; sign-extend
1426 pshufd m8, m0, q0000
1427 pshufd m9, m0, q1111
1428 pshufd m10, m0, q2222
1429 pshufd m11, m0, q3333
1430 cmp wd, 4
1431 jg .v_w8
1432 je .v_w4
1433 .v_w2:
1434 movd xm2, [srcq+ssq*0]
1435 pinsrd xm2, [srcq+ssq*1], 1
1436 pinsrd xm2, [srcq+ssq*2], 2
1437 pinsrd xm2, [srcq+r6 ], 3 ; 0 1 2 3
1438 lea srcq, [srcq+ssq*4]
1439 movd xm3, [srcq+ssq*0]
1440 vpbroadcastd xm1, [srcq+ssq*1]
1441 vpbroadcastd xm0, [srcq+ssq*2]
1442 add srcq, r6
1443 vpblendd xm3, xm1, 0x02 ; 4 5
1444 vpblendd xm1, xm0, 0x02 ; 5 6
1445 palignr xm4, xm3, xm2, 4 ; 1 2 3 4
1446 punpcklwd xm3, xm1 ; 45 56
1447 punpcklwd xm1, xm2, xm4 ; 01 12
1448 punpckhwd xm2, xm4 ; 23 34
1449 .v_w2_loop:
1450 vpbroadcastd xm4, [srcq+ssq*0]
1451 pmaddwd xm5, xm8, xm1 ; a0 b0
1452 mova xm1, xm2
1453 pmaddwd xm2, xm9 ; a1 b1
1454 paddd xm5, xm6
1455 paddd xm5, xm2
1456 mova xm2, xm3
1457 pmaddwd xm3, xm10 ; a2 b2
1458 paddd xm5, xm3
1459 vpblendd xm3, xm0, xm4, 0x02 ; 6 7
1460 vpbroadcastd xm0, [srcq+ssq*1]
1461 lea srcq, [srcq+ssq*2]
1462 vpblendd xm4, xm0, 0x02 ; 7 8
1463 punpcklwd xm3, xm4 ; 67 78
1464 pmaddwd xm4, xm11, xm3 ; a3 b3
1465 paddd xm5, xm4
1466 psrad xm5, 6
1467 packusdw xm5, xm5
1468 pminsw xm5, xm7
1469 movd [dstq+dsq*0], xm5
1470 pextrd [dstq+dsq*1], xm5, 1
1471 lea dstq, [dstq+dsq*2]
1472 sub hd, 2
1473 jg .v_w2_loop
1474 RET
1475 .v_w4:
1476 movq xm1, [srcq+ssq*0]
1477 vpbroadcastq m0, [srcq+ssq*1]
1478 vpbroadcastq m2, [srcq+ssq*2]
1479 vpbroadcastq m4, [srcq+r6 ]
1480 lea srcq, [srcq+ssq*4]
1481 vpbroadcastq m3, [srcq+ssq*0]
1482 vpbroadcastq m5, [srcq+ssq*1]
1483 vpblendd m1, m0, 0x30
1484 vpblendd m0, m2, 0x30
1485 punpcklwd m1, m0 ; 01 12
1486 vpbroadcastq m0, [srcq+ssq*2]
1487 add srcq, r6
1488 vpblendd m2, m4, 0x30
1489 vpblendd m4, m3, 0x30
1490 punpcklwd m2, m4 ; 23 34
1491 vpblendd m3, m5, 0x30
1492 vpblendd m5, m0, 0x30
1493 punpcklwd m3, m5 ; 45 56
1494 .v_w4_loop:
1495 vpbroadcastq m4, [srcq+ssq*0]
1496 pmaddwd m5, m8, m1 ; a0 b0
1497 mova m1, m2
1498 pmaddwd m2, m9 ; a1 b1
1499 paddd m5, m6
1500 paddd m5, m2
1501 mova m2, m3
1502 pmaddwd m3, m10 ; a2 b2
1503 paddd m5, m3
1504 vpblendd m3, m0, m4, 0x30
1505 vpbroadcastq m0, [srcq+ssq*1]
1506 lea srcq, [srcq+ssq*2]
1507 vpblendd m4, m0, 0x30
1508 punpcklwd m3, m4 ; 67 78
1509 pmaddwd m4, m11, m3 ; a3 b3
1510 paddd m5, m4
1511 psrad m5, 6
1512 vextracti128 xm4, m5, 1
1513 packusdw xm5, xm4
1514 pminsw xm5, xm7
1515 movq [dstq+dsq*0], xm5
1516 movhps [dstq+dsq*1], xm5
1517 lea dstq, [dstq+dsq*2]
1518 sub hd, 2
1519 jg .v_w4_loop
1520 RET
1521 .v_w8:
1522 shl wd, 5
1523 mov r7, srcq
1524 mov r8, dstq
1525 lea wd, [hq+wq-256]
1526 .v_w8_loop0:
1527 vbroadcasti128 m4, [srcq+ssq*0]
1528 vbroadcasti128 m5, [srcq+ssq*1]
1529 vbroadcasti128 m0, [srcq+r6 ]
1530 vbroadcasti128 m6, [srcq+ssq*2]
1531 lea srcq, [srcq+ssq*4]
1532 vbroadcasti128 m1, [srcq+ssq*0]
1533 vbroadcasti128 m2, [srcq+ssq*1]
1534 vbroadcasti128 m3, [srcq+ssq*2]
1535 add srcq, r6
1536 shufpd m4, m0, 0x0c
1537 shufpd m5, m1, 0x0c
1538 punpcklwd m1, m4, m5 ; 01
1539 punpckhwd m4, m5 ; 34
1540 shufpd m6, m2, 0x0c
1541 punpcklwd m2, m5, m6 ; 12
1542 punpckhwd m5, m6 ; 45
1543 shufpd m0, m3, 0x0c
1544 punpcklwd m3, m6, m0 ; 23
1545 punpckhwd m6, m0 ; 56
1546 .v_w8_loop:
1547 vbroadcasti128 m14, [srcq+ssq*0]
1548 pmaddwd m12, m8, m1 ; a0
1549 pmaddwd m13, m8, m2 ; b0
1550 mova m1, m3
1551 mova m2, m4
1552 pmaddwd m3, m9 ; a1
1553 pmaddwd m4, m9 ; b1
1554 paddd m12, m3
1555 paddd m13, m4
1556 mova m3, m5
1557 mova m4, m6
1558 pmaddwd m5, m10 ; a2
1559 pmaddwd m6, m10 ; b2
1560 paddd m12, m5
1561 vbroadcasti128 m5, [srcq+ssq*1]
1562 lea srcq, [srcq+ssq*2]
1563 paddd m13, m6
1564 shufpd m6, m0, m14, 0x0d
1565 shufpd m0, m14, m5, 0x0c
1566 punpcklwd m5, m6, m0 ; 67
1567 punpckhwd m6, m0 ; 78
1568 pmaddwd m14, m11, m5 ; a3
1569 paddd m12, m14
1570 pmaddwd m14, m11, m6 ; b3
1571 paddd m13, m14
1572 psrad m12, 5
1573 psrad m13, 5
1574 packusdw m12, m13
1575 pxor m13, m13
1576 pavgw m12, m13
1577 pminsw m12, m7
1578 vpermq m12, m12, q3120
1579 mova [dstq+dsq*0], xm12
1580 vextracti128 [dstq+dsq*1], m12, 1
1581 lea dstq, [dstq+dsq*2]
1582 sub hd, 2
1583 jg .v_w8_loop
1584 add r7, 16
1585 add r8, 16
1586 movzx hd, wb
1587 mov srcq, r7
1588 mov dstq, r8
1589 sub wd, 1<<8
1590 jg .v_w8_loop0
1591 RET
1592 .hv:
1593 %assign stack_offset stack_offset - stack_size_padded
1594 WIN64_SPILL_XMM 16
1595 vpbroadcastw m15, r8m
1596 cmp wd, 4
1597 jg .hv_w8
1598 movzx mxd, mxb
1599 vpbroadcastd m0, [base+subpel_filters+mxq*8+2]
1600 movzx mxd, myb
1601 shr myd, 16
1602 cmp hd, 4
1603 cmovle myd, mxd
1604 vpbroadcastq m1, [base+subpel_filters+myq*8]
1605 vpbroadcastd m6, [pd_512]
1606 lea r6, [ssq*3]
1607 sub srcq, 2
1608 sub srcq, r6
1609 pxor m7, m7
1610 punpcklbw m7, m0
1611 punpcklbw m1, m1
1612 psraw m1, 8 ; sign-extend
1613 test dword r8m, 0x800
1614 jz .hv_10bit
1615 psraw m7, 2
1616 psllw m1, 2
1617 .hv_10bit:
1618 pshufd m11, m1, q0000
1619 pshufd m12, m1, q1111
1620 pshufd m13, m1, q2222
1621 pshufd m14, m1, q3333
1622 cmp wd, 4
1623 je .hv_w4
1624 vbroadcasti128 m9, [subpel_h_shuf2]
1625 vbroadcasti128 m1, [srcq+r6 ] ; 3 3
1626 movu xm3, [srcq+ssq*2]
1627 movu xm0, [srcq+ssq*0]
1628 movu xm2, [srcq+ssq*1]
1629 lea srcq, [srcq+ssq*4]
1630 vinserti128 m3, [srcq+ssq*0], 1 ; 2 4
1631 vinserti128 m0, [srcq+ssq*1], 1 ; 0 5
1632 vinserti128 m2, [srcq+ssq*2], 1 ; 1 6
1633 add srcq, r6
1634 pshufb m1, m9
1635 pshufb m3, m9
1636 pshufb m0, m9
1637 pshufb m2, m9
1638 pmaddwd m1, m7
1639 pmaddwd m3, m7
1640 pmaddwd m0, m7
1641 pmaddwd m2, m7
1642 phaddd m1, m3
1643 phaddd m0, m2
1644 paddd m1, m6
1645 paddd m0, m6
1646 psrad m1, 10
1647 psrad m0, 10
1648 packssdw m1, m0 ; 3 2 0 1
1649 vextracti128 xm0, m1, 1 ; 3 4 5 6
1650 pshufd xm2, xm1, q1301 ; 2 3 1 2
1651 pshufd xm3, xm0, q2121 ; 4 5 4 5
1652 punpckhwd xm1, xm2 ; 01 12
1653 punpcklwd xm2, xm0 ; 23 34
1654 punpckhwd xm3, xm0 ; 45 56
1655 .hv_w2_loop:
1656 movu xm4, [srcq+ssq*0]
1657 movu xm5, [srcq+ssq*1]
1658 lea srcq, [srcq+ssq*2]
1659 pshufb xm4, xm9
1660 pshufb xm5, xm9
1661 pmaddwd xm4, xm7
1662 pmaddwd xm5, xm7
1663 phaddd xm4, xm5
1664 pmaddwd xm5, xm11, xm1 ; a0 b0
1665 mova xm1, xm2
1666 pmaddwd xm2, xm12 ; a1 b1
1667 paddd xm5, xm2
1668 mova xm2, xm3
1669 pmaddwd xm3, xm13 ; a2 b2
1670 paddd xm5, xm3
1671 paddd xm4, xm6
1672 psrad xm4, 10
1673 packssdw xm4, xm4
1674 palignr xm3, xm4, xm0, 12
1675 mova xm0, xm4
1676 punpcklwd xm3, xm0 ; 67 78
1677 pmaddwd xm4, xm14, xm3 ; a3 b3
1678 paddd xm5, xm6
1679 paddd xm5, xm4
1680 psrad xm5, 10
1681 packusdw xm5, xm5
1682 pminsw xm5, xm15
1683 movd [dstq+dsq*0], xm5
1684 pextrd [dstq+dsq*1], xm5, 1
1685 lea dstq, [dstq+dsq*2]
1686 sub hd, 2
1687 jg .hv_w2_loop
1688 RET
1689 .hv_w4:
1690 vbroadcasti128 m9, [subpel_h_shufA]
1691 vbroadcasti128 m10, [subpel_h_shufB]
1692 pshufd m8, m7, q1111
1693 pshufd m7, m7, q0000
1694 movu xm1, [srcq+ssq*0]
1695 vinserti128 m1, [srcq+ssq*1], 1 ; 0 1
1696 vbroadcasti128 m0, [srcq+r6 ]
1697 vinserti128 m2, m0, [srcq+ssq*2], 0 ; 2 3
1698 lea srcq, [srcq+ssq*4]
1699 vinserti128 m0, [srcq+ssq*0], 1 ; 3 4
1700 movu xm3, [srcq+ssq*1]
1701 vinserti128 m3, [srcq+ssq*2], 1 ; 5 6
1702 add srcq, r6
1703 pshufb m4, m1, m9
1704 pshufb m1, m10
1705 pmaddwd m4, m7
1706 pmaddwd m1, m8
1707 pshufb m5, m2, m9
1708 pshufb m2, m10
1709 pmaddwd m5, m7
1710 pmaddwd m2, m8
1711 paddd m4, m6
1712 paddd m1, m4
1713 pshufb m4, m0, m9
1714 pshufb m0, m10
1715 pmaddwd m4, m7
1716 pmaddwd m0, m8
1717 paddd m5, m6
1718 paddd m2, m5
1719 pshufb m5, m3, m9
1720 pshufb m3, m10
1721 pmaddwd m5, m7
1722 pmaddwd m3, m8
1723 paddd m4, m6
1724 paddd m4, m0
1725 paddd m5, m6
1726 paddd m5, m3
1727 vperm2i128 m0, m1, m2, 0x21
1728 psrld m1, 10
1729 psrld m2, 10
1730 vperm2i128 m3, m4, m5, 0x21
1731 pslld m4, 6
1732 pslld m5, 6
1733 pblendw m2, m4, 0xaa ; 23 34
1734 pslld m0, 6
1735 pblendw m1, m0, 0xaa ; 01 12
1736 psrld m3, 10
1737 pblendw m3, m5, 0xaa ; 45 56
1738 psrad m0, m5, 16
1739 .hv_w4_loop:
1740 movu xm4, [srcq+ssq*0]
1741 vinserti128 m4, [srcq+ssq*1], 1
1742 lea srcq, [srcq+ssq*2]
1743 pmaddwd m5, m11, m1 ; a0 b0
1744 mova m1, m2
1745 pmaddwd m2, m12 ; a1 b1
1746 paddd m5, m6
1747 paddd m5, m2
1748 mova m2, m3
1749 pmaddwd m3, m13 ; a2 b2
1750 paddd m5, m3
1751 pshufb m3, m4, m9
1752 pshufb m4, m10
1753 pmaddwd m3, m7
1754 pmaddwd m4, m8
1755 paddd m3, m6
1756 paddd m4, m3
1757 psrad m4, 10
1758 packssdw m0, m4 ; _ 7 6 8
1759 vpermq m3, m0, q1122 ; _ 6 _ 7
1760 punpckhwd m3, m0 ; 67 78
1761 mova m0, m4
1762 pmaddwd m4, m14, m3 ; a3 b3
1763 paddd m4, m5
1764 psrad m4, 10
1765 vextracti128 xm5, m4, 1
1766 packusdw xm4, xm5
1767 pminsw xm4, xm15
1768 movq [dstq+dsq*0], xm4
1769 movhps [dstq+dsq*1], xm4
1770 lea dstq, [dstq+dsq*2]
1771 sub hd, 2
1772 jg .hv_w4_loop
1773 RET
1774 .hv_w8:
1775 shr mxd, 16
1776 vpbroadcastq m2, [base+subpel_filters+mxq*8]
1777 movzx mxd, myb
1778 shr myd, 16
1779 cmp hd, 4
1780 cmovle myd, mxd
1781 pmovsxbw xm1, [base+subpel_filters+myq*8]
1782 shl wd, 5
1783 lea r6, [ssq*3]
1784 sub srcq, 6
1785 sub srcq, r6
1786 pxor m0, m0
1787 punpcklbw m0, m2
1788 mov r7, srcq
1789 mov r8, dstq
1790 lea wd, [hq+wq-256]
1791 test dword r8m, 0x800
1792 jz .hv_w8_10bit
1793 psraw m0, 2
1794 psllw xm1, 2
1795 .hv_w8_10bit:
1796 pshufd m11, m0, q0000
1797 pshufd m12, m0, q1111
1798 pshufd m13, m0, q2222
1799 pshufd m14, m0, q3333
1800 %if WIN64
1801 %define v_mul (rsp+stack_offset+40) ; r4m
1802 %else
1803 %define v_mul (rsp-24) ; red zone
1804 %endif
1805 mova [v_mul], xm1
1806 .hv_w8_loop0:
1807 %macro PUT_8TAP_HV_H 3 ; dst/src+0, src+8, src+16
1808 pshufb m2, m%1, m9 ; 2 3 3 4 4 5 5 6
1809 pshufb m%1, m8 ; 0 1 1 2 2 3 3 4
1810 pmaddwd m3, m12, m2
1811 pmaddwd m%1, m11
1812 pshufb m%2, m9 ; 6 7 7 8 8 9 9 a
1813 shufpd m2, m%2, 0x05 ; 4 5 5 6 6 7 7 8
1814 paddd m3, m10
1815 paddd m%1, m3
1816 pmaddwd m3, m14, m%2
1817 paddd m%1, m3
1818 pmaddwd m3, m13, m2
1819 pshufb m%3, m9 ; a b b c c d d e
1820 pmaddwd m2, m11
1821 paddd m%1, m3
1822 pmaddwd m3, m12, m%2
1823 shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c
1824 pmaddwd m%3, m14
1825 pmaddwd m%2, m13
1826 paddd m2, m10
1827 paddd m2, m3
1828 paddd m%3, m2
1829 paddd m%2, m%3
1830 psrad m%1, 10
1831 psrad m%2, 10
1832 packssdw m%1, m%2
1833 %endmacro
1834 movu xm4, [srcq+r6 *1+ 0]
1835 vbroadcasti128 m8, [subpel_h_shufA]
1836 movu xm6, [srcq+r6 *1+ 8]
1837 vbroadcasti128 m9, [subpel_h_shufB]
1838 movu xm0, [srcq+r6 *1+16]
1839 vpbroadcastd m10, [pd_512]
1840 movu xm5, [srcq+ssq*0+ 0]
1841 vinserti128 m5, [srcq+ssq*4+ 0], 1
1842 movu xm1, [srcq+ssq*0+16]
1843 vinserti128 m1, [srcq+ssq*4+16], 1
1844 shufpd m7, m5, m1, 0x05
1845 INIT_XMM avx2
1846 PUT_8TAP_HV_H 4, 6, 0 ; 3
1847 INIT_YMM avx2
1848 PUT_8TAP_HV_H 5, 7, 1 ; 0 4
1849 movu xm0, [srcq+ssq*2+ 0]
1850 vinserti128 m0, [srcq+r6 *2+ 0], 1
1851 movu xm1, [srcq+ssq*2+16]
1852 vinserti128 m1, [srcq+r6 *2+16], 1
1853 shufpd m7, m0, m1, 0x05
1854 PUT_8TAP_HV_H 0, 7, 1 ; 2 6
1855 movu xm6, [srcq+ssq*1+ 0]
1856 movu xm1, [srcq+ssq*1+16]
1857 lea srcq, [srcq+ssq*4]
1858 vinserti128 m6, [srcq+ssq*1+ 0], 1
1859 vinserti128 m1, [srcq+ssq*1+16], 1
1860 add srcq, r6
1861 shufpd m7, m6, m1, 0x05
1862 PUT_8TAP_HV_H 6, 7, 1 ; 1 5
1863 vpermq m4, m4, q1100
1864 vpermq m5, m5, q3120
1865 vpermq m6, m6, q3120
1866 vpermq m7, m0, q3120
1867 punpcklwd m3, m7, m4 ; 23
1868 punpckhwd m4, m5 ; 34
1869 punpcklwd m1, m5, m6 ; 01
1870 punpckhwd m5, m6 ; 45
1871 punpcklwd m2, m6, m7 ; 12
1872 punpckhwd m6, m7 ; 56
1873 .hv_w8_loop:
1874 vpbroadcastd m9, [v_mul+4*0]
1875 vpbroadcastd m7, [v_mul+4*1]
1876 vpbroadcastd m10, [v_mul+4*2]
1877 pmaddwd m8, m9, m1 ; a0
1878 pmaddwd m9, m2 ; b0
1879 mova m1, m3
1880 mova m2, m4
1881 pmaddwd m3, m7 ; a1
1882 pmaddwd m4, m7 ; b1
1883 paddd m8, m3
1884 paddd m9, m4
1885 mova m3, m5
1886 mova m4, m6
1887 pmaddwd m5, m10 ; a2
1888 pmaddwd m6, m10 ; b2
1889 paddd m8, m5
1890 paddd m9, m6
1891 movu xm5, [srcq+ssq*0]
1892 vinserti128 m5, [srcq+ssq*1], 1
1893 vbroadcasti128 m7, [subpel_h_shufA]
1894 vbroadcasti128 m10, [subpel_h_shufB]
1895 movu xm6, [srcq+ssq*0+16]
1896 vinserti128 m6, [srcq+ssq*1+16], 1
1897 vextracti128 [dstq], m0, 1
1898 pshufb m0, m5, m7 ; 01
1899 pshufb m5, m10 ; 23
1900 pmaddwd m0, m11
1901 pmaddwd m5, m12
1902 paddd m0, m5
1903 pshufb m5, m6, m7 ; 89
1904 pshufb m6, m10 ; ab
1905 pmaddwd m5, m13
1906 pmaddwd m6, m14
1907 paddd m6, m5
1908 movu xm5, [srcq+ssq*0+8]
1909 vinserti128 m5, [srcq+ssq*1+8], 1
1910 lea srcq, [srcq+ssq*2]
1911 pshufb m7, m5, m7
1912 pshufb m5, m10
1913 pmaddwd m10, m13, m7
1914 pmaddwd m7, m11
1915 paddd m0, m10
1916 vpbroadcastd m10, [pd_512]
1917 paddd m6, m7
1918 pmaddwd m7, m14, m5
1919 pmaddwd m5, m12
1920 paddd m0, m7
1921 paddd m5, m6
1922 vbroadcasti128 m6, [dstq]
1923 paddd m8, m10
1924 paddd m9, m10
1925 paddd m0, m10
1926 paddd m5, m10
1927 vpbroadcastd m10, [v_mul+4*3]
1928 psrad m0, 10
1929 psrad m5, 10
1930 packssdw m0, m5
1931 vpermq m7, m0, q3120 ; 7 8
1932 shufpd m6, m7, 0x04 ; 6 7
1933 punpcklwd m5, m6, m7 ; 67
1934 punpckhwd m6, m7 ; 78
1935 pmaddwd m7, m10, m5 ; a3
1936 pmaddwd m10, m6 ; b3
1937 paddd m7, m8
1938 paddd m9, m10
1939 psrad m7, 10
1940 psrad m9, 10
1941 packusdw m7, m9
1942 pminsw m7, m15
1943 vpermq m7, m7, q3120
1944 mova [dstq+dsq*0], xm7
1945 vextracti128 [dstq+dsq*1], m7, 1
1946 lea dstq, [dstq+dsq*2]
1947 sub hd, 2
1948 jg .hv_w8_loop
1949 add r7, 16
1950 add r8, 16
1951 movzx hd, wb
1952 mov srcq, r7
1953 mov dstq, r8
1954 sub wd, 1<<8
1955 jg .hv_w8_loop0
1956 RET
1957
1958 %if WIN64
1959 DECLARE_REG_TMP 6, 4
1960 %else
1961 DECLARE_REG_TMP 6, 7
1962 %endif
1963
1964 %define PREP_8TAP_FN FN prep_8tap,
1965 PREP_8TAP_FN sharp, SHARP, SHARP
1966 PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH
1967 PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP
1968 PREP_8TAP_FN smooth, SMOOTH, SMOOTH
1969 PREP_8TAP_FN sharp_regular, SHARP, REGULAR
1970 PREP_8TAP_FN regular_sharp, REGULAR, SHARP
1971 PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR
1972 PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH
1973 PREP_8TAP_FN regular, REGULAR, REGULAR
1974
1975 cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my
1976 %define base r7-prep_avx2
1977 imul mxd, mxm, 0x010101
1978 add mxd, t0d ; 8tap_h, mx, 4tap_h
1979 imul myd, mym, 0x010101
1980 add myd, t1d ; 8tap_v, my, 4tap_v
1981 lea r7, [prep_avx2]
1982 movifnidn hd, hm
1983 test mxd, 0xf00
1984 jnz .h
1985 test myd, 0xf00
1986 jnz .v
1987 tzcnt wd, wd
1988 mov r6d, r7m ; bitdepth_max
1989 movzx wd, word [r7+wq*2+table_offset(prep,)]
1990 vpbroadcastd m5, [r7-prep_avx2+pw_8192]
1991 shr r6d, 11
1992 add wq, r7
1993 vpbroadcastd m4, [base+prep_mul+r6*4]
1994 lea r6, [strideq*3]
1995 %if WIN64
1996 pop r7
1997 %endif
1998 jmp wq
1999 .h_w4:
2000 movzx mxd, mxb
2001 sub srcq, 2
2002 pmovsxbw xm0, [base+subpel_filters+mxq*8]
2003 vbroadcasti128 m3, [subpel_h_shufA]
2004 vbroadcasti128 m4, [subpel_h_shufB]
2005 WIN64_SPILL_XMM 8
2006 pshufd xm0, xm0, q2211
2007 test dword r7m, 0x800
2008 jnz .h_w4_12bpc
2009 psllw xm0, 2
2010 .h_w4_12bpc:
2011 vpbroadcastq m6, xm0
2012 vpermq m7, m0, q1111
2013 .h_w4_loop:
2014 movu xm1, [srcq+strideq*0]
2015 vinserti128 m1, [srcq+strideq*2], 1
2016 movu xm2, [srcq+strideq*1]
2017 vinserti128 m2, [srcq+r6 ], 1
2018 lea srcq, [srcq+strideq*4]
2019 pshufb m0, m1, m3 ; 0 1 1 2 2 3 3 4
2020 pshufb m1, m4 ; 2 3 3 4 4 5 5 6
2021 pmaddwd m0, m6
2022 pmaddwd m1, m7
2023 paddd m0, m5
2024 paddd m0, m1
2025 pshufb m1, m2, m3
2026 pshufb m2, m4
2027 pmaddwd m1, m6
2028 pmaddwd m2, m7
2029 paddd m1, m5
2030 paddd m1, m2
2031 psrad m0, 4
2032 psrad m1, 4
2033 packssdw m0, m1
2034 mova [tmpq], m0
2035 add tmpq, 32
2036 sub hd, 4
2037 jg .h_w4_loop
2038 RET
2039 .h:
2040 test myd, 0xf00
2041 jnz .hv
2042 vpbroadcastd m5, [prep_8tap_1d_rnd] ; 8 - (8192 << 4)
2043 lea r6, [strideq*3]
2044 cmp wd, 4
2045 je .h_w4
2046 shr mxd, 16
2047 sub srcq, 6
2048 vpbroadcastq m0, [base+subpel_filters+mxq*8]
2049 %assign stack_offset stack_offset - stack_size_padded
2050 WIN64_SPILL_XMM 12
2051 vbroadcasti128 m6, [subpel_h_shufA]
2052 vbroadcasti128 m7, [subpel_h_shufB]
2053 punpcklbw m0, m0
2054 psraw m0, 8 ; sign-extend
2055 test dword r7m, 0x800
2056 jnz .h_12bpc
2057 psllw m0, 2
2058 .h_12bpc:
2059 pshufd m8, m0, q0000
2060 pshufd m9, m0, q1111
2061 pshufd m10, m0, q2222
2062 pshufd m11, m0, q3333
2063 cmp wd, 8
2064 jg .h_w16
2065 .h_w8:
2066 %macro PREP_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2]
2067 pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6
2068 pshufb m%1, m6 ; 0 1 1 2 2 3 3 4
2069 pmaddwd m%5, m9, m%4 ; abcd1
2070 pmaddwd m%1, m8 ; abcd0
2071 pshufb m%2, m7 ; 6 7 7 8 8 9 9 a
2072 shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8
2073 paddd m%5, m5
2074 paddd m%1, m%5
2075 pmaddwd m%5, m11, m%2 ; abcd3
2076 paddd m%1, m%5
2077 pmaddwd m%5, m10, m%4 ; abcd2
2078 pshufb m%3, m7 ; a b b c c d d e
2079 pmaddwd m%4, m8 ; efgh0
2080 paddd m%1, m%5
2081 pmaddwd m%5, m9, m%2 ; efgh1
2082 shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c
2083 pmaddwd m%3, m11 ; efgh3
2084 pmaddwd m%2, m10 ; efgh2
2085 paddd m%4, m5
2086 paddd m%4, m%5
2087 paddd m%3, m%4
2088 paddd m%2, m%3
2089 psrad m%1, 4
2090 psrad m%2, 4
2091 packssdw m%1, m%2
2092 %endmacro
2093 movu xm0, [srcq+strideq*0+ 0]
2094 vinserti128 m0, [srcq+strideq*1+ 0], 1
2095 movu xm2, [srcq+strideq*0+16]
2096 vinserti128 m2, [srcq+strideq*1+16], 1
2097 lea srcq, [srcq+strideq*2]
2098 shufpd m1, m0, m2, 0x05
2099 PREP_8TAP_H 0, 1, 2, 3, 4
2100 mova [tmpq], m0
2101 add tmpq, 32
2102 sub hd, 2
2103 jg .h_w8
2104 RET
2105 .h_w16:
2106 add wd, wd
2107 .h_w16_loop0:
2108 mov r6d, wd
2109 .h_w16_loop:
2110 movu m0, [srcq+r6-32]
2111 movu m1, [srcq+r6-24]
2112 movu m2, [srcq+r6-16]
2113 PREP_8TAP_H 0, 1, 2, 3, 4
2114 mova [tmpq+r6-32], m0
2115 sub r6d, 32
2116 jg .h_w16_loop
2117 add srcq, strideq
2118 add tmpq, wq
2119 dec hd
2120 jg .h_w16_loop0
2121 RET
2122 .v:
2123 movzx mxd, myb
2124 shr myd, 16
2125 cmp hd, 4
2126 cmovle myd, mxd
2127 vpbroadcastq m0, [base+subpel_filters+myq*8]
2128 %assign stack_offset stack_offset - stack_size_padded
2129 WIN64_SPILL_XMM 15
2130 vpbroadcastd m7, [prep_8tap_1d_rnd]
2131 lea r6, [strideq*3]
2132 sub srcq, r6
2133 punpcklbw m0, m0
2134 psraw m0, 8 ; sign-extend
2135 test dword r7m, 0x800
2136 jnz .v_12bpc
2137 psllw m0, 2
2138 .v_12bpc:
2139 pshufd m8, m0, q0000
2140 pshufd m9, m0, q1111
2141 pshufd m10, m0, q2222
2142 pshufd m11, m0, q3333
2143 cmp wd, 4
2144 jg .v_w8
2145 .v_w4:
2146 movq xm1, [srcq+strideq*0]
2147 vpbroadcastq m0, [srcq+strideq*1]
2148 vpbroadcastq m2, [srcq+strideq*2]
2149 vpbroadcastq m4, [srcq+r6 ]
2150 lea srcq, [srcq+strideq*4]
2151 vpbroadcastq m3, [srcq+strideq*0]
2152 vpbroadcastq m5, [srcq+strideq*1]
2153 vpblendd m1, m0, 0x30
2154 vpblendd m0, m2, 0x30
2155 punpcklwd m1, m0 ; 01 12
2156 vpbroadcastq m0, [srcq+strideq*2]
2157 add srcq, r6
2158 vpblendd m2, m4, 0x30
2159 vpblendd m4, m3, 0x30
2160 punpcklwd m2, m4 ; 23 34
2161 vpblendd m3, m5, 0x30
2162 vpblendd m5, m0, 0x30
2163 punpcklwd m3, m5 ; 45 56
2164 .v_w4_loop:
2165 vpbroadcastq m4, [srcq+strideq*0]
2166 pmaddwd m5, m8, m1 ; a0 b0
2167 mova m1, m2
2168 pmaddwd m2, m9 ; a1 b1
2169 paddd m5, m7
2170 paddd m5, m2
2171 mova m2, m3
2172 pmaddwd m3, m10 ; a2 b2
2173 paddd m5, m3
2174 vpblendd m3, m0, m4, 0x30
2175 vpbroadcastq m0, [srcq+strideq*1]
2176 lea srcq, [srcq+strideq*2]
2177 vpblendd m4, m0, 0x30
2178 punpcklwd m3, m4 ; 67 78
2179 pmaddwd m4, m11, m3 ; a3 b3
2180 paddd m5, m4
2181 psrad m5, 4
2182 vextracti128 xm4, m5, 1
2183 packssdw xm5, xm4
2184 mova [tmpq], xm5
2185 add tmpq, 16
2186 sub hd, 2
2187 jg .v_w4_loop
2188 RET
2189 .v_w8:
2190 %if WIN64
2191 push r8
2192 %endif
2193 mov r8d, wd
2194 shl wd, 5
2195 mov r5, srcq
2196 mov r7, tmpq
2197 lea wd, [hq+wq-256]
2198 .v_w8_loop0:
2199 vbroadcasti128 m4, [srcq+strideq*0]
2200 vbroadcasti128 m5, [srcq+strideq*1]
2201 vbroadcasti128 m0, [srcq+r6 ]
2202 vbroadcasti128 m6, [srcq+strideq*2]
2203 lea srcq, [srcq+strideq*4]
2204 vbroadcasti128 m1, [srcq+strideq*0]
2205 vbroadcasti128 m2, [srcq+strideq*1]
2206 vbroadcasti128 m3, [srcq+strideq*2]
2207 add srcq, r6
2208 shufpd m4, m0, 0x0c
2209 shufpd m5, m1, 0x0c
2210 punpcklwd m1, m4, m5 ; 01
2211 punpckhwd m4, m5 ; 34
2212 shufpd m6, m2, 0x0c
2213 punpcklwd m2, m5, m6 ; 12
2214 punpckhwd m5, m6 ; 45
2215 shufpd m0, m3, 0x0c
2216 punpcklwd m3, m6, m0 ; 23
2217 punpckhwd m6, m0 ; 56
2218 .v_w8_loop:
2219 vbroadcasti128 m14, [srcq+strideq*0]
2220 pmaddwd m12, m8, m1 ; a0
2221 pmaddwd m13, m8, m2 ; b0
2222 mova m1, m3
2223 mova m2, m4
2224 pmaddwd m3, m9 ; a1
2225 pmaddwd m4, m9 ; b1
2226 paddd m12, m7
2227 paddd m13, m7
2228 paddd m12, m3
2229 paddd m13, m4
2230 mova m3, m5
2231 mova m4, m6
2232 pmaddwd m5, m10 ; a2
2233 pmaddwd m6, m10 ; b2
2234 paddd m12, m5
2235 vbroadcasti128 m5, [srcq+strideq*1]
2236 lea srcq, [srcq+strideq*2]
2237 paddd m13, m6
2238 shufpd m6, m0, m14, 0x0d
2239 shufpd m0, m14, m5, 0x0c
2240 punpcklwd m5, m6, m0 ; 67
2241 punpckhwd m6, m0 ; 78
2242 pmaddwd m14, m11, m5 ; a3
2243 paddd m12, m14
2244 pmaddwd m14, m11, m6 ; b3
2245 paddd m13, m14
2246 psrad m12, 4
2247 psrad m13, 4
2248 packssdw m12, m13
2249 vpermq m12, m12, q3120
2250 mova [tmpq+r8*0], xm12
2251 vextracti128 [tmpq+r8*2], m12, 1
2252 lea tmpq, [tmpq+r8*4]
2253 sub hd, 2
2254 jg .v_w8_loop
2255 add r5, 16
2256 add r7, 16
2257 movzx hd, wb
2258 mov srcq, r5
2259 mov tmpq, r7
2260 sub wd, 1<<8
2261 jg .v_w8_loop0
2262 %if WIN64
2263 pop r8
2264 %endif
2265 RET
2266 .hv:
2267 %assign stack_offset stack_offset - stack_size_padded
2268 WIN64_SPILL_XMM 16
2269 vpbroadcastd m15, [prep_8tap_2d_rnd]
2270 cmp wd, 4
2271 jg .hv_w8
2272 movzx mxd, mxb
2273 vpbroadcastd m0, [base+subpel_filters+mxq*8+2]
2274 movzx mxd, myb
2275 shr myd, 16
2276 cmp hd, 4
2277 cmovle myd, mxd
2278 vpbroadcastq m1, [base+subpel_filters+myq*8]
2279 lea r6, [strideq*3]
2280 sub srcq, 2
2281 sub srcq, r6
2282 pxor m7, m7
2283 punpcklbw m7, m0
2284 punpcklbw m1, m1
2285 psraw m7, 4
2286 psraw m1, 8
2287 test dword r7m, 0x800
2288 jz .hv_w4_10bit
2289 psraw m7, 2
2290 .hv_w4_10bit:
2291 pshufd m11, m1, q0000
2292 pshufd m12, m1, q1111
2293 pshufd m13, m1, q2222
2294 pshufd m14, m1, q3333
2295 .hv_w4:
2296 vbroadcasti128 m9, [subpel_h_shufA]
2297 vbroadcasti128 m10, [subpel_h_shufB]
2298 pshufd m8, m7, q1111
2299 pshufd m7, m7, q0000
2300 movu xm1, [srcq+strideq*0]
2301 vinserti128 m1, [srcq+strideq*1], 1 ; 0 1
2302 vbroadcasti128 m0, [srcq+r6 ]
2303 vinserti128 m2, m0, [srcq+strideq*2], 0 ; 2 3
2304 lea srcq, [srcq+strideq*4]
2305 vinserti128 m0, [srcq+strideq*0], 1 ; 3 4
2306 movu xm3, [srcq+strideq*1]
2307 vinserti128 m3, [srcq+strideq*2], 1 ; 5 6
2308 add srcq, r6
2309 pshufb m4, m1, m9
2310 pshufb m1, m10
2311 pmaddwd m4, m7
2312 pmaddwd m1, m8
2313 pshufb m5, m2, m9
2314 pshufb m2, m10
2315 pmaddwd m5, m7
2316 pmaddwd m2, m8
2317 paddd m4, m15
2318 paddd m1, m4
2319 pshufb m4, m0, m9
2320 pshufb m0, m10
2321 pmaddwd m4, m7
2322 pmaddwd m0, m8
2323 paddd m5, m15
2324 paddd m2, m5
2325 pshufb m5, m3, m9
2326 pshufb m3, m10
2327 pmaddwd m5, m7
2328 pmaddwd m3, m8
2329 paddd m4, m15
2330 paddd m4, m0
2331 paddd m5, m15
2332 paddd m5, m3
2333 vperm2i128 m0, m1, m2, 0x21
2334 psrld m1, 6
2335 psrld m2, 6
2336 vperm2i128 m3, m4, m5, 0x21
2337 pslld m4, 10
2338 pslld m5, 10
2339 pblendw m2, m4, 0xaa ; 23 34
2340 pslld m0, 10
2341 pblendw m1, m0, 0xaa ; 01 12
2342 psrld m3, 6
2343 pblendw m3, m5, 0xaa ; 45 56
2344 psrad m0, m5, 16
2345 .hv_w4_loop:
2346 movu xm4, [srcq+strideq*0]
2347 vinserti128 m4, [srcq+strideq*1], 1
2348 lea srcq, [srcq+strideq*2]
2349 pmaddwd m5, m11, m1 ; a0 b0
2350 mova m1, m2
2351 pmaddwd m2, m12 ; a1 b1
2352 paddd m5, m15
2353 paddd m5, m2
2354 mova m2, m3
2355 pmaddwd m3, m13 ; a2 b2
2356 paddd m5, m3
2357 pshufb m3, m4, m9
2358 pshufb m4, m10
2359 pmaddwd m3, m7
2360 pmaddwd m4, m8
2361 paddd m3, m15
2362 paddd m4, m3
2363 psrad m4, 6
2364 packssdw m0, m4 ; _ 7 6 8
2365 vpermq m3, m0, q1122 ; _ 6 _ 7
2366 punpckhwd m3, m0 ; 67 78
2367 mova m0, m4
2368 pmaddwd m4, m14, m3 ; a3 b3
2369 paddd m4, m5
2370 psrad m4, 6
2371 vextracti128 xm5, m4, 1
2372 packssdw xm4, xm5
2373 mova [tmpq], xm4
2374 add tmpq, 16
2375 sub hd, 2
2376 jg .hv_w4_loop
2377 RET
2378 .hv_w8:
2379 shr mxd, 16
2380 vpbroadcastq m2, [base+subpel_filters+mxq*8]
2381 movzx mxd, myb
2382 shr myd, 16
2383 cmp hd, 4
2384 cmovle myd, mxd
2385 pmovsxbw xm1, [base+subpel_filters+myq*8]
2386 %if WIN64
2387 PUSH r8
2388 %endif
2389 mov r8d, wd
2390 shl wd, 5
2391 lea r6, [strideq*3]
2392 sub srcq, 6
2393 sub srcq, r6
2394 mov r5, srcq
2395 mov r7, tmpq
2396 lea wd, [hq+wq-256]
2397 pxor m0, m0
2398 punpcklbw m0, m2
2399 mova [v_mul], xm1
2400 psraw m0, 4
2401 test dword r7m, 0x800
2402 jz .hv_w8_10bit
2403 psraw m0, 2
2404 .hv_w8_10bit:
2405 pshufd m11, m0, q0000
2406 pshufd m12, m0, q1111
2407 pshufd m13, m0, q2222
2408 pshufd m14, m0, q3333
2409 .hv_w8_loop0:
2410 %macro PREP_8TAP_HV_H 3 ; dst/src+0, src+8, src+16
2411 pshufb m2, m%1, m9 ; 2 3 3 4 4 5 5 6
2412 pshufb m%1, m8 ; 0 1 1 2 2 3 3 4
2413 pmaddwd m3, m12, m2
2414 pmaddwd m%1, m11
2415 pshufb m%2, m9 ; 6 7 7 8 8 9 9 a
2416 shufpd m2, m%2, 0x05 ; 4 5 5 6 6 7 7 8
2417 paddd m3, m15
2418 paddd m%1, m3
2419 pmaddwd m3, m14, m%2
2420 paddd m%1, m3
2421 pmaddwd m3, m13, m2
2422 pshufb m%3, m9 ; a b b c c d d e
2423 pmaddwd m2, m11
2424 paddd m%1, m3
2425 pmaddwd m3, m12, m%2
2426 shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c
2427 pmaddwd m%3, m14
2428 pmaddwd m%2, m13
2429 paddd m2, m15
2430 paddd m2, m3
2431 paddd m2, m%3
2432 paddd m2, m%2
2433 psrad m%1, 6
2434 psrad m2, 6
2435 packssdw m%1, m2
2436 %endmacro
2437 movu xm4, [srcq+r6 + 0]
2438 vbroadcasti128 m8, [subpel_h_shufA]
2439 movu xm6, [srcq+r6 + 8]
2440 vbroadcasti128 m9, [subpel_h_shufB]
2441 movu xm0, [srcq+r6 +16]
2442 movu xm5, [srcq+strideq*0+ 0]
2443 vinserti128 m5, [srcq+strideq*4+ 0], 1
2444 movu xm1, [srcq+strideq*0+16]
2445 vinserti128 m1, [srcq+strideq*4+16], 1
2446 shufpd m7, m5, m1, 0x05
2447 INIT_XMM avx2
2448 PREP_8TAP_HV_H 4, 6, 0 ; 3
2449 INIT_YMM avx2
2450 PREP_8TAP_HV_H 5, 7, 1 ; 0 4
2451 movu xm0, [srcq+strideq*2+ 0]
2452 vinserti128 m0, [srcq+r6 *2+ 0], 1
2453 movu xm1, [srcq+strideq*2+16]
2454 vinserti128 m1, [srcq+r6 *2+16], 1
2455 shufpd m7, m0, m1, 0x05
2456 PREP_8TAP_HV_H 0, 7, 1 ; 2 6
2457 movu xm6, [srcq+strideq*1+ 0]
2458 movu xm1, [srcq+strideq*1+16]
2459 lea srcq, [srcq+strideq*4]
2460 vinserti128 m6, [srcq+strideq*1+ 0], 1
2461 vinserti128 m1, [srcq+strideq*1+16], 1
2462 add srcq, r6
2463 shufpd m7, m6, m1, 0x05
2464 PREP_8TAP_HV_H 6, 7, 1 ; 1 5
2465 vpermq m4, m4, q1100
2466 vpermq m5, m5, q3120
2467 vpermq m6, m6, q3120
2468 vpermq m7, m0, q3120
2469 punpcklwd m3, m7, m4 ; 23
2470 punpckhwd m4, m5 ; 34
2471 punpcklwd m1, m5, m6 ; 01
2472 punpckhwd m5, m6 ; 45
2473 punpcklwd m2, m6, m7 ; 12
2474 punpckhwd m6, m7 ; 56
2475 .hv_w8_loop:
2476 vpbroadcastd m9, [v_mul+4*0]
2477 vpbroadcastd m7, [v_mul+4*1]
2478 vpbroadcastd m10, [v_mul+4*2]
2479 pmaddwd m8, m9, m1 ; a0
2480 pmaddwd m9, m2 ; b0
2481 mova m1, m3
2482 mova m2, m4
2483 pmaddwd m3, m7 ; a1
2484 pmaddwd m4, m7 ; b1
2485 paddd m8, m15
2486 paddd m9, m15
2487 paddd m8, m3
2488 paddd m9, m4
2489 mova m3, m5
2490 mova m4, m6
2491 pmaddwd m5, m10 ; a2
2492 pmaddwd m6, m10 ; b2
2493 paddd m8, m5
2494 paddd m9, m6
2495 movu xm5, [srcq+strideq*0]
2496 vinserti128 m5, [srcq+strideq*1], 1
2497 vbroadcasti128 m7, [subpel_h_shufA]
2498 vbroadcasti128 m10, [subpel_h_shufB]
2499 movu xm6, [srcq+strideq*0+16]
2500 vinserti128 m6, [srcq+strideq*1+16], 1
2501 vextracti128 [tmpq], m0, 1
2502 pshufb m0, m5, m7 ; 01
2503 pshufb m5, m10 ; 23
2504 pmaddwd m0, m11
2505 pmaddwd m5, m12
2506 paddd m0, m15
2507 paddd m0, m5
2508 pshufb m5, m6, m7 ; 89
2509 pshufb m6, m10 ; ab
2510 pmaddwd m5, m13
2511 pmaddwd m6, m14
2512 paddd m5, m15
2513 paddd m6, m5
2514 movu xm5, [srcq+strideq*0+8]
2515 vinserti128 m5, [srcq+strideq*1+8], 1
2516 lea srcq, [srcq+strideq*2]
2517 pshufb m7, m5, m7
2518 pshufb m5, m10
2519 pmaddwd m10, m13, m7
2520 pmaddwd m7, m11
2521 paddd m0, m10
2522 paddd m6, m7
2523 pmaddwd m7, m14, m5
2524 pmaddwd m5, m12
2525 paddd m0, m7
2526 paddd m5, m6
2527 vbroadcasti128 m6, [tmpq]
2528 vpbroadcastd m10, [v_mul+4*3]
2529 psrad m0, 6
2530 psrad m5, 6
2531 packssdw m0, m5
2532 vpermq m7, m0, q3120 ; 7 8
2533 shufpd m6, m7, 0x04 ; 6 7
2534 punpcklwd m5, m6, m7 ; 67
2535 punpckhwd m6, m7 ; 78
2536 pmaddwd m7, m10, m5 ; a3
2537 pmaddwd m10, m6 ; b3
2538 paddd m7, m8
2539 paddd m9, m10
2540 psrad m7, 6
2541 psrad m9, 6
2542 packssdw m7, m9
2543 vpermq m7, m7, q3120
2544 mova [tmpq+r8*0], xm7
2545 vextracti128 [tmpq+r8*2], m7, 1
2546 lea tmpq, [tmpq+r8*4]
2547 sub hd, 2
2548 jg .hv_w8_loop
2549 add r5, 16
2550 add r7, 16
2551 movzx hd, wb
2552 mov srcq, r5
2553 mov tmpq, r7
2554 sub wd, 1<<8
2555 jg .hv_w8_loop0
2556 %if WIN64
2557 POP r8
2558 %endif
2559 RET
2560
2561 %macro movifprep 2
2562 %if isprep
2563 mov %1, %2
2564 %endif
2565 %endmacro
2566
2567 %macro REMAP_REG 2
2568 %xdefine r%1 r%2
2569 %xdefine r%1q r%2q
2570 %xdefine r%1d r%2d
2571 %endmacro
2572
2573 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0
2574 %if isprep
2575 %xdefine r14_save r14
2576 %assign %%i 14
2577 %rep 14
2578 %assign %%j %%i-1
2579 REMAP_REG %%i, %%j
2580 %assign %%i %%i-1
2581 %endrep
2582 %endif
2583 %endmacro
2584
2585 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0
2586 %if isprep
2587 %assign %%i 1
2588 %rep 13
2589 %assign %%j %%i+1
2590 REMAP_REG %%i, %%j
2591 %assign %%i %%i+1
2592 %endrep
2593 %xdefine r14 r14_save
2594 %undef r14_save
2595 %endif
2596 %endmacro
2597
2598 %macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged
2599 MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
2600 RET
2601 %if %1
2602 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2603 %endif
2604 %endmacro
2605
2606 %macro MC_8TAP_SCALED_H 8-9 0 ; dst, tmp[0-6], load_hrnd
2607 movu xm%1, [srcq+ r4*2]
2608 movu xm%2, [srcq+ r6*2]
2609 movu xm%3, [srcq+ r7*2]
2610 movu xm%4, [srcq+ r9*2]
2611 vinserti128 m%1, [srcq+r10*2], 1
2612 vinserti128 m%2, [srcq+r11*2], 1
2613 vinserti128 m%3, [srcq+r13*2], 1
2614 vinserti128 m%4, [srcq+ rX*2], 1
2615 add srcq, ssq
2616 movu xm%5, [srcq+ r4*2]
2617 movu xm%6, [srcq+ r6*2]
2618 movu xm%7, [srcq+ r7*2]
2619 movu xm%8, [srcq+ r9*2]
2620 vinserti128 m%5, [srcq+r10*2], 1
2621 vinserti128 m%6, [srcq+r11*2], 1
2622 vinserti128 m%7, [srcq+r13*2], 1
2623 vinserti128 m%8, [srcq+ rX*2], 1
2624 add srcq, ssq
2625 pmaddwd m%1, m12
2626 pmaddwd m%2, m13
2627 pmaddwd m%3, m14
2628 pmaddwd m%4, m15
2629 pmaddwd m%5, m12
2630 pmaddwd m%6, m13
2631 pmaddwd m%7, m14
2632 pmaddwd m%8, m15
2633 phaddd m%1, m%2
2634 %if %9
2635 mova m10, [rsp+0x00]
2636 %endif
2637 phaddd m%3, m%4
2638 phaddd m%5, m%6
2639 phaddd m%7, m%8
2640 phaddd m%1, m%3
2641 phaddd m%5, m%7
2642 paddd m%1, m10
2643 paddd m%5, m10
2644 psrad m%1, xm11
2645 psrad m%5, xm11
2646 packssdw m%1, m%5
2647 %endmacro
2648
2649 %macro MC_8TAP_SCALED 1
2650 %ifidn %1, put
2651 %assign isput 1
2652 %assign isprep 0
2653 cglobal put_8tap_scaled_16bpc, 4, 14, 16, 0xe0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax
2654 %xdefine base_reg r12
2655 mov r7d, pxmaxm
2656 %else
2657 %assign isput 0
2658 %assign isprep 1
2659 cglobal prep_8tap_scaled_16bpc, 4, 14, 16, 0xe0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax
2660 %define tmp_stridem qword [rsp+0xd0]
2661 %xdefine base_reg r11
2662 %endif
2663 lea base_reg, [%1_8tap_scaled_16bpc_avx2]
2664 %define base base_reg-%1_8tap_scaled_16bpc_avx2
2665 tzcnt wd, wm
2666 vpbroadcastd m8, dxm
2667 %if isprep && UNIX64
2668 movd xm10, mxd
2669 vpbroadcastd m10, xm10
2670 mov r5d, t0d
2671 DECLARE_REG_TMP 5, 7
2672 mov r6d, pxmaxm
2673 %else
2674 vpbroadcastd m10, mxm
2675 %if isput
2676 vpbroadcastw m11, pxmaxm
2677 %else
2678 mov r6d, pxmaxm
2679 %endif
2680 %endif
2681 mov dyd, dym
2682 %if isput
2683 %if WIN64
2684 mov r8d, hm
2685 DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3
2686 %define hm r5m
2687 %define dxm r8m
2688 %else
2689 DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3
2690 %define hm r6m
2691 %endif
2692 %define dsm [rsp+0x98]
2693 %define rX r1
2694 %define rXd r1d
2695 %else ; prep
2696 %if WIN64
2697 mov r7d, hm
2698 DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3
2699 %define hm r4m
2700 %define dxm r7m
2701 %else
2702 DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3
2703 %define hm [rsp+0x98]
2704 %endif
2705 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2706 %define rX r14
2707 %define rXd r14d
2708 %endif
2709 shr r7d, 11
2710 vpbroadcastd m6, [base+pd_0x3ff]
2711 vpbroadcastd m12, [base+s_8tap_h_rnd+r7*4]
2712 movd xm7, [base+s_8tap_h_sh+r7*4]
2713 %if isput
2714 vpbroadcastd m13, [base+put_s_8tap_v_rnd+r7*4]
2715 pinsrd xm7, [base+put_s_8tap_v_sh+r7*4], 2
2716 %else
2717 vpbroadcastd m13, [base+pd_m524256]
2718 %endif
2719 pxor m9, m9
2720 lea ss3q, [ssq*3]
2721 movzx r7d, t1b
2722 shr t1d, 16
2723 cmp hd, 6
2724 cmovs t1d, r7d
2725 sub srcq, ss3q
2726 cmp dyd, 1024
2727 je .dy1
2728 cmp dyd, 2048
2729 je .dy2
2730 movzx wd, word [base+%1_8tap_scaled_avx2_table+wq*2]
2731 add wq, base_reg
2732 jmp wq
2733 %if isput
2734 .w2:
2735 mov myd, mym
2736 movzx t0d, t0b
2737 sub srcq, 2
2738 movd xm15, t0d
2739 punpckldq m8, m9, m8
2740 paddd m10, m8 ; mx+dx*[0,1]
2741 vpbroadcastd xm14, [base+pq_0x40000000+2]
2742 vpbroadcastd xm15, xm15
2743 pand xm8, xm10, xm6
2744 psrld xm8, 6
2745 paddd xm15, xm8
2746 movd r4d, xm15
2747 pextrd r6d, xm15, 1
2748 vbroadcasti128 m5, [base+bdct_lb_q]
2749 vbroadcasti128 m6, [base+subpel_s_shuf2]
2750 vpbroadcastd xm15, [base+subpel_filters+r4*8+2]
2751 vpbroadcastd xm4, [base+subpel_filters+r6*8+2]
2752 pcmpeqd xm8, xm9
2753 psrld m10, 10
2754 paddd m10, m10
2755 movu xm0, [srcq+ssq*0]
2756 movu xm1, [srcq+ssq*1]
2757 movu xm2, [srcq+ssq*2]
2758 movu xm3, [srcq+ss3q ]
2759 lea srcq, [srcq+ssq*4]
2760 pshufb m10, m5
2761 paddb m10, m6
2762 vpblendd xm15, xm4, 0xa
2763 pblendvb xm15, xm14, xm8
2764 pmovsxbw m15, xm15
2765 vinserti128 m0, [srcq+ssq*0], 1 ; 0 4
2766 vinserti128 m1, [srcq+ssq*1], 1 ; 1 5
2767 vinserti128 m2, [srcq+ssq*2], 1 ; 2 6
2768 vinserti128 m3, [srcq+ss3q ], 1 ; 3 7
2769 lea srcq, [srcq+ssq*4]
2770 REPX {pshufb x, m10}, m0, m1, m2, m3
2771 REPX {pmaddwd x, m15}, m0, m1, m2, m3
2772 phaddd m0, m1
2773 phaddd m2, m3
2774 paddd m0, m12
2775 paddd m2, m12
2776 psrad m0, xm7
2777 psrad m2, xm7
2778 packssdw m0, m2 ; 0 1 2 3 4 5 6 7
2779 vextracti128 xm1, m0, 1
2780 palignr xm2, xm1, xm0, 4 ; 1 2 3 4
2781 punpcklwd xm3, xm0, xm2 ; 01 12
2782 punpckhwd xm0, xm2 ; 23 34
2783 pshufd xm4, xm1, q0321 ; 5 6 7 _
2784 punpcklwd xm2, xm1, xm4 ; 45 56
2785 punpckhwd xm4, xm1, xm4 ; 67 __
2786 .w2_loop:
2787 and myd, 0x3ff
2788 mov r6d, 64 << 24
2789 mov r4d, myd
2790 shr r4d, 6
2791 lea r4d, [t1+r4]
2792 cmovnz r6q, [base+subpel_filters+r4*8]
2793 movq xm14, r6q
2794 pmovsxbw xm14, xm14
2795 pshufd xm8, xm14, q0000
2796 pshufd xm9, xm14, q1111
2797 pmaddwd xm5, xm3, xm8
2798 pmaddwd xm6, xm0, xm9
2799 pshufd xm8, xm14, q2222
2800 pshufd xm14, xm14, q3333
2801 paddd xm5, xm6
2802 pmaddwd xm6, xm2, xm8
2803 pmaddwd xm8, xm4, xm14
2804 psrldq xm9, xm7, 8
2805 paddd xm5, xm6
2806 paddd xm5, xm13
2807 paddd xm5, xm8
2808 psrad xm5, xm9
2809 packusdw xm5, xm5
2810 pminsw xm5, xm11
2811 movd [dstq], xm5
2812 add dstq, dsq
2813 dec hd
2814 jz .ret
2815 add myd, dyd
2816 test myd, ~0x3ff
2817 jz .w2_loop
2818 movu xm5, [srcq]
2819 test myd, 0x400
2820 jz .w2_skip_line
2821 add srcq, ssq
2822 shufps xm3, xm0, q1032 ; 01 12
2823 shufps xm0, xm2, q1032 ; 23 34
2824 shufps xm2, xm4, q1032 ; 45 56
2825 pshufb xm5, xm10
2826 pmaddwd xm5, xm15
2827 phaddd xm5, xm5
2828 paddd xm5, xm12
2829 psrad xm5, xm7
2830 packssdw xm5, xm5
2831 palignr xm1, xm5, xm1, 12
2832 punpcklqdq xm1, xm1 ; 6 7 6 7
2833 punpcklwd xm4, xm1, xm5 ; 67 __
2834 jmp .w2_loop
2835 .w2_skip_line:
2836 movu xm6, [srcq+ssq*1]
2837 lea srcq, [srcq+ssq*2]
2838 mova xm3, xm0 ; 01 12
2839 mova xm0, xm2 ; 23 34
2840 pshufb xm5, xm10
2841 pshufb xm6, xm10
2842 pmaddwd xm5, xm15
2843 pmaddwd xm6, xm15
2844 phaddd xm5, xm6
2845 paddd xm5, xm12
2846 psrad xm5, xm7
2847 packssdw xm5, xm5 ; 6 7 6 7
2848 palignr xm1, xm5, xm1, 8 ; 4 5 6 7
2849 pshufd xm5, xm1, q0321 ; 5 6 7 _
2850 punpcklwd xm2, xm1, xm5 ; 45 56
2851 punpckhwd xm4, xm1, xm5 ; 67 __
2852 jmp .w2_loop
2853 %endif
2854 .w4:
2855 mov myd, mym
2856 mova [rsp+0x00], m12
2857 %if isput
2858 mova [rsp+0x20], xm13
2859 %else
2860 SWAP m11, m13
2861 %endif
2862 mova [rsp+0x30], xm7
2863 vbroadcasti128 m7, [base+rescale_mul]
2864 movzx t0d, t0b
2865 sub srcq, 2
2866 movd xm15, t0d
2867 pmaddwd m8, m7
2868 vpbroadcastq m2, [base+pq_0x40000000+1]
2869 vpbroadcastd xm15, xm15
2870 SWAP m13, m10
2871 paddd m13, m8 ; mx+dx*[0-3]
2872 pand m6, m13
2873 psrld m6, 6
2874 paddd xm15, xm6
2875 movd r4d, xm15
2876 pextrd r6d, xm15, 1
2877 pextrd r11d, xm15, 2
2878 pextrd r13d, xm15, 3
2879 vbroadcasti128 m5, [base+bdct_lb_q+ 0]
2880 vbroadcasti128 m1, [base+bdct_lb_q+16]
2881 vbroadcasti128 m0, [base+subpel_s_shuf2]
2882 vpbroadcastd xm14, [base+subpel_filters+r4*8+2]
2883 vpbroadcastd xm7, [base+subpel_filters+r6*8+2]
2884 vpbroadcastd xm15, [base+subpel_filters+r11*8+2]
2885 vpbroadcastd xm8, [base+subpel_filters+r13*8+2]
2886 pcmpeqd m6, m9
2887 punpckldq m10, m6, m6
2888 punpckhdq m6, m6
2889 psrld m13, 10
2890 paddd m13, m13
2891 vpblendd xm14, xm7, 0xa
2892 vpblendd xm15, xm8, 0xa
2893 pmovsxbw m14, xm14
2894 pmovsxbw m15, xm15
2895 pblendvb m14, m2, m10
2896 pblendvb m15, m2, m6
2897 pextrd r4, xm13, 2
2898 pshufb m12, m13, m5
2899 pshufb m13, m1
2900 lea r6, [r4+ssq*1]
2901 lea r11, [r4+ssq*2]
2902 lea r13, [r4+ss3q ]
2903 movu xm7, [srcq+ssq*0]
2904 movu xm9, [srcq+ssq*1]
2905 movu xm8, [srcq+ssq*2]
2906 movu xm10, [srcq+ss3q ]
2907 movu xm1, [srcq+r4 ]
2908 movu xm3, [srcq+r6 ]
2909 movu xm2, [srcq+r11 ]
2910 movu xm4, [srcq+r13 ]
2911 lea srcq, [srcq+ssq*4]
2912 vinserti128 m7, [srcq+ssq*0], 1
2913 vinserti128 m9, [srcq+ssq*1], 1
2914 vinserti128 m8, [srcq+ssq*2], 1
2915 vinserti128 m10, [srcq+ss3q ], 1
2916 vinserti128 m1, [srcq+r4 ], 1
2917 vinserti128 m3, [srcq+r6 ], 1
2918 vinserti128 m2, [srcq+r11 ], 1
2919 vinserti128 m4, [srcq+r13 ], 1
2920 lea srcq, [srcq+ssq*4]
2921 vpbroadcastb m5, xm13
2922 psubb m13, m5
2923 paddb m12, m0
2924 paddb m13, m0
2925 REPX {pshufb x, m12}, m7, m9, m8, m10
2926 REPX {pmaddwd x, m14}, m7, m9, m8, m10
2927 REPX {pshufb x, m13}, m1, m2, m3, m4
2928 REPX {pmaddwd x, m15}, m1, m2, m3, m4
2929 mova m5, [rsp+0x00]
2930 movd xm6, [rsp+0x30]
2931 phaddd m7, m1
2932 phaddd m9, m3
2933 phaddd m8, m2
2934 phaddd m10, m4
2935 REPX {paddd x, m5}, m7, m9, m8, m10
2936 REPX {psrad x, xm6}, m7, m9, m8, m10
2937 packssdw m7, m9 ; 0 1 4 5
2938 packssdw m8, m10 ; 2 3 6 7
2939 vextracti128 xm9, m7, 1 ; 4 5
2940 vextracti128 xm3, m8, 1 ; 6 7
2941 shufps xm4, xm7, xm8, q1032 ; 1 2
2942 shufps xm5, xm8, xm9, q1032 ; 3 4
2943 shufps xm6, xm9, xm3, q1032 ; 5 6
2944 psrldq xm10, xm3, 8 ; 7 _
2945 punpcklwd xm0, xm7, xm4 ; 01
2946 punpckhwd xm7, xm4 ; 12
2947 punpcklwd xm1, xm8, xm5 ; 23
2948 punpckhwd xm8, xm5 ; 34
2949 punpcklwd xm2, xm9, xm6 ; 45
2950 punpckhwd xm9, xm6 ; 56
2951 punpcklwd xm3, xm10 ; 67
2952 mova [rsp+0x40], xm7
2953 mova [rsp+0x50], xm8
2954 mova [rsp+0x60], xm9
2955 .w4_loop:
2956 and myd, 0x3ff
2957 mov r11d, 64 << 24
2958 mov r13d, myd
2959 shr r13d, 6
2960 lea r13d, [t1+r13]
2961 cmovnz r11q, [base+subpel_filters+r13*8]
2962 movq xm9, r11q
2963 pmovsxbw xm9, xm9
2964 pshufd xm7, xm9, q0000
2965 pshufd xm8, xm9, q1111
2966 pmaddwd xm4, xm0, xm7
2967 pmaddwd xm5, xm1, xm8
2968 pshufd xm7, xm9, q2222
2969 pshufd xm9, xm9, q3333
2970 pmaddwd xm6, xm2, xm7
2971 pmaddwd xm8, xm3, xm9
2972 %if isput
2973 mova xm7, [rsp+0x20]
2974 movd xm9, [rsp+0x38]
2975 %else
2976 SWAP m7, m11
2977 %endif
2978 paddd xm4, xm5
2979 paddd xm6, xm8
2980 paddd xm4, xm6
2981 paddd xm4, xm7
2982 %if isput
2983 psrad xm4, xm9
2984 packusdw xm4, xm4
2985 pminuw xm4, xm11
2986 movq [dstq], xm4
2987 add dstq, dsq
2988 %else
2989 SWAP m11, m7
2990 psrad xm4, 6
2991 packssdw xm4, xm4
2992 movq [tmpq], xm4
2993 add tmpq, 8
2994 %endif
2995 dec hd
2996 jz .ret
2997 add myd, dyd
2998 test myd, ~0x3ff
2999 jz .w4_loop
3000 mova xm8, [rsp+0x00]
3001 movd xm9, [rsp+0x30]
3002 movu xm4, [srcq]
3003 movu xm5, [srcq+r4]
3004 test myd, 0x400
3005 jz .w4_skip_line
3006 mova xm0, [rsp+0x40]
3007 mova [rsp+0x40], xm1
3008 mova xm1, [rsp+0x50]
3009 mova [rsp+0x50], xm2
3010 mova xm2, [rsp+0x60]
3011 mova [rsp+0x60], xm3
3012 pshufb xm4, xm12
3013 pshufb xm5, xm13
3014 pmaddwd xm4, xm14
3015 pmaddwd xm5, xm15
3016 phaddd xm4, xm5
3017 paddd xm4, xm8
3018 psrad xm4, xm9
3019 packssdw xm4, xm4
3020 punpcklwd xm3, xm10, xm4
3021 mova xm10, xm4
3022 add srcq, ssq
3023 jmp .w4_loop
3024 .w4_skip_line:
3025 movu xm6, [srcq+ssq*1]
3026 movu xm7, [srcq+r6]
3027 movu m0, [rsp+0x50]
3028 pshufb xm4, xm12
3029 pshufb xm6, xm12
3030 pshufb xm5, xm13
3031 pshufb xm7, xm13
3032 pmaddwd xm4, xm14
3033 pmaddwd xm6, xm14
3034 pmaddwd xm5, xm15
3035 pmaddwd xm7, xm15
3036 mova [rsp+0x40], m0
3037 phaddd xm4, xm5
3038 phaddd xm6, xm7
3039 paddd xm4, xm8
3040 paddd xm6, xm8
3041 psrad xm4, xm9
3042 psrad xm6, xm9
3043 packssdw xm4, xm6
3044 punpcklwd xm9, xm10, xm4
3045 mova [rsp+0x60], xm9
3046 psrldq xm10, xm4, 8
3047 mova xm0, xm1
3048 mova xm1, xm2
3049 mova xm2, xm3
3050 punpcklwd xm3, xm4, xm10
3051 lea srcq, [srcq+ssq*2]
3052 jmp .w4_loop
3053 SWAP m10, m13
3054 %if isprep
3055 SWAP m13, m11
3056 %endif
3057 .w8:
3058 mov dword [rsp+0x80], 1
3059 movifprep tmp_stridem, 16
3060 jmp .w_start
3061 .w16:
3062 mov dword [rsp+0x80], 2
3063 movifprep tmp_stridem, 32
3064 jmp .w_start
3065 .w32:
3066 mov dword [rsp+0x80], 4
3067 movifprep tmp_stridem, 64
3068 jmp .w_start
3069 .w64:
3070 mov dword [rsp+0x80], 8
3071 movifprep tmp_stridem, 128
3072 jmp .w_start
3073 .w128:
3074 mov dword [rsp+0x80], 16
3075 movifprep tmp_stridem, 256
3076 .w_start:
3077 SWAP m10, m12, m1
3078 SWAP m11, m7
3079 ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free
3080 %if isput
3081 movifnidn dsm, dsq
3082 mova [rsp+0xb0], xm7
3083 %endif
3084 mova [rsp+0x00], m10
3085 mova [rsp+0x20], m13
3086 shr t0d, 16
3087 sub srcq, 6
3088 pmaddwd m8, [base+rescale_mul2]
3089 movd xm15, t0d
3090 mov [rsp+0x84], t0d
3091 mov [rsp+0x88], srcq
3092 mov [rsp+0x90], r0q ; dstq / tmpq
3093 %if UNIX64
3094 mov hm, hd
3095 %endif
3096 shl dword dxm, 3 ; dx*8
3097 vpbroadcastd m15, xm15
3098 paddd m1, m8 ; mx+dx*[0-7]
3099 jmp .hloop
3100 .hloop_prep:
3101 dec dword [rsp+0x80]
3102 jz .ret
3103 add qword [rsp+0x90], 16
3104 mov hd, hm
3105 vpbroadcastd m8, dxm
3106 vpbroadcastd m6, [base+pd_0x3ff]
3107 paddd m1, m8, [rsp+0x40]
3108 vpbroadcastd m15, [rsp+0x84]
3109 pxor m9, m9
3110 mov srcq, [rsp+0x88]
3111 mov r0q, [rsp+0x90] ; dstq / tmpq
3112 .hloop:
3113 vpbroadcastq xm2, [base+pq_0x40000000]
3114 pand m5, m1, m6
3115 psrld m5, 6
3116 paddd m15, m5
3117 pcmpeqd m5, m9
3118 vextracti128 xm7, m15, 1
3119 movq r6, xm15
3120 pextrq r9, xm15, 1
3121 movq r11, xm7
3122 pextrq rX, xm7, 1
3123 mov r4d, r6d
3124 shr r6, 32
3125 mov r7d, r9d
3126 shr r9, 32
3127 mov r10d, r11d
3128 shr r11, 32
3129 mov r13d, rXd
3130 shr rX, 32
3131 mova [rsp+0x40], m1
3132 movq xm12, [base+subpel_filters+ r4*8]
3133 movq xm13, [base+subpel_filters+ r6*8]
3134 movhps xm12, [base+subpel_filters+ r7*8]
3135 movhps xm13, [base+subpel_filters+ r9*8]
3136 movq xm14, [base+subpel_filters+r10*8]
3137 movq xm15, [base+subpel_filters+r11*8]
3138 movhps xm14, [base+subpel_filters+r13*8]
3139 movhps xm15, [base+subpel_filters+ rX*8]
3140 psrld m1, 10
3141 vextracti128 xm7, m1, 1
3142 vextracti128 xm6, m5, 1
3143 movq [rsp+0xa0], xm1
3144 movq [rsp+0xa8], xm7
3145 movq r6, xm1
3146 pextrq r11, xm1, 1
3147 movq r9, xm7
3148 pextrq rX, xm7, 1
3149 mov r4d, r6d
3150 shr r6, 32
3151 mov r10d, r11d
3152 shr r11, 32
3153 mov r7d, r9d
3154 shr r9, 32
3155 mov r13d, rXd
3156 shr rX, 32
3157 pshufd xm4, xm5, q2200
3158 pshufd xm5, xm5, q3311
3159 pshufd xm7, xm6, q2200
3160 pshufd xm6, xm6, q3311
3161 pblendvb xm12, xm2, xm4
3162 pblendvb xm13, xm2, xm5
3163 pblendvb xm14, xm2, xm7
3164 pblendvb xm15, xm2, xm6
3165 pmovsxbw m12, xm12
3166 pmovsxbw m13, xm13
3167 pmovsxbw m14, xm14
3168 pmovsxbw m15, xm15
3169 MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b
3170 mova [rsp+0x60], m0
3171 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b
3172 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b
3173 MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b
3174 mova m0, [rsp+0x60]
3175 vbroadcasti128 m9, [base+subpel_s_shuf8]
3176 mov myd, mym
3177 mov dyd, dym
3178 pshufb m0, m9 ; 01a 01b
3179 pshufb m1, m9 ; 23a 23b
3180 pshufb m2, m9 ; 45a 45b
3181 pshufb m3, m9 ; 67a 67b
3182 .vloop:
3183 and myd, 0x3ff
3184 mov r6d, 64 << 24
3185 mov r4d, myd
3186 shr r4d, 6
3187 lea r4d, [t1+r4]
3188 cmovnz r6q, [base+subpel_filters+r4*8]
3189 movq xm9, r6q
3190 punpcklqdq xm9, xm9
3191 pmovsxbw m9, xm9
3192 pshufd m8, m9, q0000
3193 pshufd m7, m9, q1111
3194 pmaddwd m4, m0, m8
3195 pmaddwd m5, m1, m7
3196 pshufd m8, m9, q2222
3197 pshufd m9, m9, q3333
3198 pmaddwd m6, m2, m8
3199 pmaddwd m7, m3, m9
3200 %if isput
3201 psrldq xm8, xm11, 8
3202 %endif
3203 paddd m4, [rsp+0x20]
3204 paddd m6, m7
3205 paddd m4, m5
3206 paddd m4, m6
3207 %if isput
3208 psrad m4, xm8
3209 vextracti128 xm5, m4, 1
3210 packusdw xm4, xm5
3211 pminsw xm4, [rsp+0xb0]
3212 mova [dstq], xm4
3213 add dstq, dsm
3214 %else
3215 psrad m4, 6
3216 vextracti128 xm5, m4, 1
3217 packssdw xm4, xm5
3218 mova [tmpq], xm4
3219 add tmpq, tmp_stridem
3220 %endif
3221 dec hd
3222 jz .hloop_prep
3223 add myd, dyd
3224 test myd, ~0x3ff
3225 jz .vloop
3226 test myd, 0x400
3227 mov [rsp+0x60], myd
3228 mov r4d, [rsp+0xa0]
3229 mov r6d, [rsp+0xa4]
3230 mov r7d, [rsp+0xa8]
3231 mov r9d, [rsp+0xac]
3232 jz .skip_line
3233 vbroadcasti128 m9, [base+wswap]
3234 movu xm4, [srcq+ r4*2]
3235 movu xm5, [srcq+ r6*2]
3236 movu xm6, [srcq+ r7*2]
3237 movu xm7, [srcq+ r9*2]
3238 vinserti128 m4, [srcq+r10*2], 1
3239 vinserti128 m5, [srcq+r11*2], 1
3240 vinserti128 m6, [srcq+r13*2], 1
3241 vinserti128 m7, [srcq+ rX*2], 1
3242 add srcq, ssq
3243 mov myd, [rsp+0x60]
3244 mov dyd, dym
3245 pshufb m0, m9
3246 pshufb m1, m9
3247 pshufb m2, m9
3248 pshufb m3, m9
3249 pmaddwd m4, m12
3250 pmaddwd m5, m13
3251 pmaddwd m6, m14
3252 pmaddwd m7, m15
3253 phaddd m4, m5
3254 phaddd m6, m7
3255 phaddd m4, m6
3256 paddd m4, m10
3257 psrad m4, xm11
3258 pslld m4, 16
3259 pblendw m0, m1, 0xaa
3260 pblendw m1, m2, 0xaa
3261 pblendw m2, m3, 0xaa
3262 pblendw m3, m4, 0xaa
3263 jmp .vloop
3264 .skip_line:
3265 mova m0, m1
3266 mova m1, m2
3267 mova m2, m3
3268 MC_8TAP_SCALED_H 3, 10, 4, 5, 6, 7, 8, 9, 1
3269 vbroadcasti128 m9, [base+subpel_s_shuf8]
3270 mov myd, [rsp+0x60]
3271 mov dyd, dym
3272 pshufb m3, m9
3273 jmp .vloop
3274 SWAP m1, m12, m10
3275 SWAP m7, m11
3276 .dy1:
3277 movzx wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2]
3278 add wq, base_reg
3279 jmp wq
3280 %if isput
3281 .dy1_w2:
3282 mov myd, mym
3283 movzx t0d, t0b
3284 sub srcq, 2
3285 movd xm15, t0d
3286 punpckldq m8, m9, m8
3287 paddd m10, m8 ; mx+dx*[0-1]
3288 vpbroadcastd xm14, [base+pq_0x40000000+2]
3289 vpbroadcastd xm15, xm15
3290 pand xm8, xm10, xm6
3291 psrld xm8, 6
3292 paddd xm15, xm8
3293 movd r4d, xm15
3294 pextrd r6d, xm15, 1
3295 vbroadcasti128 m5, [base+bdct_lb_q]
3296 vbroadcasti128 m6, [base+subpel_s_shuf2]
3297 vpbroadcastd m15, [base+subpel_filters+r4*8+2]
3298 vpbroadcastd m4, [base+subpel_filters+r6*8+2]
3299 pcmpeqd xm8, xm9
3300 psrld m10, 10
3301 paddd m10, m10
3302 movu xm0, [srcq+ssq*0]
3303 movu xm1, [srcq+ssq*1]
3304 movu xm2, [srcq+ssq*2]
3305 movu xm3, [srcq+ss3q ]
3306 lea srcq, [srcq+ssq*4]
3307 shr myd, 6
3308 mov r4d, 64 << 24
3309 lea myd, [t1+myq]
3310 cmovnz r4q, [base+subpel_filters+myq*8]
3311 pshufb m10, m5
3312 paddb m10, m6
3313 vpblendd xm15, xm4, 0xa
3314 pblendvb xm15, xm14, xm8
3315 pmovsxbw m15, xm15
3316 vinserti128 m0, [srcq+ssq*0], 1
3317 vinserti128 m1, [srcq+ssq*1], 1
3318 vinserti128 m2, [srcq+ssq*2], 1
3319 add srcq, ss3q
3320 movq xm6, r4q
3321 pmovsxbw xm6, xm6
3322 pshufd xm8, xm6, q0000
3323 pshufd xm9, xm6, q1111
3324 pshufd xm14, xm6, q2222
3325 pshufd xm6, xm6, q3333
3326 REPX {pshufb x, m10}, m0, m1, m2
3327 pshufb xm3, xm10
3328 REPX {pmaddwd x, m15}, m0, m1, m2
3329 pmaddwd xm3, xm15
3330 phaddd m0, m1
3331 phaddd m2, m3
3332 paddd m0, m12
3333 paddd m2, m12
3334 psrad m0, xm7
3335 psrad m2, xm7
3336 packssdw m0, m2
3337 vextracti128 xm1, m0, 1
3338 palignr xm2, xm1, xm0, 4
3339 pshufd xm4, xm1, q2121
3340 punpcklwd xm3, xm0, xm2 ; 01 12
3341 punpckhwd xm0, xm2 ; 23 34
3342 punpcklwd xm2, xm1, xm4 ; 45 56
3343 .dy1_w2_loop:
3344 movu xm1, [srcq+ssq*0]
3345 movu xm5, [srcq+ssq*1]
3346 lea srcq, [srcq+ssq*2]
3347 pshufb xm1, xm10
3348 pshufb xm5, xm10
3349 pmaddwd xm1, xm15
3350 pmaddwd xm5, xm15
3351 phaddd xm1, xm5
3352 pmaddwd xm5, xm3, xm8
3353 mova xm3, xm0
3354 pmaddwd xm0, xm9
3355 paddd xm1, xm12
3356 psrad xm1, xm7
3357 packssdw xm1, xm1
3358 paddd xm5, xm0
3359 mova xm0, xm2
3360 pmaddwd xm2, xm14
3361 paddd xm5, xm2
3362 palignr xm2, xm1, xm4, 12
3363 punpcklwd xm2, xm1 ; 67 78
3364 pmaddwd xm4, xm2, xm6
3365 paddd xm5, xm13
3366 paddd xm5, xm4
3367 mova xm4, xm1
3368 psrldq xm1, xm7, 8
3369 psrad xm5, xm1
3370 packusdw xm5, xm5
3371 pminsw xm5, xm11
3372 movd [dstq+dsq*0], xm5
3373 pextrd [dstq+dsq*1], xm5, 1
3374 lea dstq, [dstq+dsq*2]
3375 sub hd, 2
3376 jg .dy1_w2_loop
3377 RET
3378 %endif
3379 .dy1_w4:
3380 mov myd, mym
3381 %if isput
3382 mova [rsp+0x50], xm11
3383 %endif
3384 mova [rsp+0x00], m12
3385 mova [rsp+0x20], m13
3386 mova [rsp+0x40], xm7
3387 vbroadcasti128 m7, [base+rescale_mul]
3388 movzx t0d, t0b
3389 sub srcq, 2
3390 movd xm15, t0d
3391 pmaddwd m8, m7
3392 vpbroadcastq m2, [base+pq_0x40000000+1]
3393 vpbroadcastd xm15, xm15
3394 SWAP m13, m10
3395 paddd m13, m8 ; mx+dx*[0-3]
3396 pand m6, m13
3397 psrld m6, 6
3398 paddd xm15, xm6
3399 movd r4d, xm15
3400 pextrd r6d, xm15, 1
3401 pextrd r11d, xm15, 2
3402 pextrd r13d, xm15, 3
3403 vbroadcasti128 m5, [base+bdct_lb_q+ 0]
3404 vbroadcasti128 m1, [base+bdct_lb_q+16]
3405 vbroadcasti128 m4, [base+subpel_s_shuf2]
3406 vpbroadcastd xm14, [base+subpel_filters+r4*8+2]
3407 vpbroadcastd xm7, [base+subpel_filters+r6*8+2]
3408 vpbroadcastd xm15, [base+subpel_filters+r11*8+2]
3409 vpbroadcastd xm8, [base+subpel_filters+r13*8+2]
3410 pcmpeqd m6, m9
3411 punpckldq m10, m6, m6
3412 punpckhdq m6, m6
3413 psrld m13, 10
3414 paddd m13, m13
3415 vpblendd xm14, xm7, 0xa
3416 vpblendd xm15, xm8, 0xa
3417 pmovsxbw m14, xm14
3418 pmovsxbw m15, xm15
3419 pblendvb m14, m2, m10
3420 pblendvb m15, m2, m6
3421 pextrd r4, xm13, 2
3422 pshufb m12, m13, m5
3423 pshufb m13, m1
3424 lea r6, [r4+ssq*2]
3425 lea r11, [r4+ssq*1]
3426 lea r13, [r4+ss3q ]
3427 movu xm0, [srcq+ssq*0]
3428 movu xm7, [srcq+r4 ]
3429 movu xm1, [srcq+ssq*2]
3430 movu xm8, [srcq+r6 ]
3431 vinserti128 m0, [srcq+ssq*1], 1 ; 0 1
3432 vinserti128 m7, [srcq+r11 ], 1
3433 vinserti128 m1, [srcq+ss3q ], 1 ; 2 3
3434 vinserti128 m8, [srcq+r13 ], 1
3435 lea srcq, [srcq+ssq*4]
3436 movu xm2, [srcq+ssq*0]
3437 movu xm9, [srcq+r4 ]
3438 movu xm3, [srcq+ssq*2] ; 6 _
3439 movu xm10, [srcq+r6 ]
3440 vinserti128 m2, [srcq+ssq*1], 1 ; 4 5
3441 vinserti128 m9, [srcq+r11 ], 1
3442 lea srcq, [srcq+ss3q ]
3443 vpbroadcastb m5, xm13
3444 psubb m13, m5
3445 paddb m12, m4
3446 paddb m13, m4
3447 mova m5, [rsp+0x00]
3448 movd xm6, [rsp+0x40]
3449 pshufb m0, m12
3450 pshufb m1, m12
3451 pmaddwd m0, m14
3452 pmaddwd m1, m14
3453 pshufb m7, m13
3454 pshufb m8, m13
3455 pmaddwd m7, m15
3456 pmaddwd m8, m15
3457 pshufb m2, m12
3458 pshufb xm3, xm12
3459 pmaddwd m2, m14
3460 pmaddwd xm3, xm14
3461 pshufb m9, m13
3462 pshufb xm10, xm13
3463 pmaddwd m9, m15
3464 pmaddwd xm10, xm15
3465 phaddd m0, m7
3466 phaddd m1, m8
3467 phaddd m2, m9
3468 phaddd xm3, xm10
3469 paddd m0, m5
3470 paddd m1, m5
3471 paddd m2, m5
3472 paddd xm3, xm5
3473 psrad m0, xm6
3474 psrad m1, xm6
3475 psrad m2, xm6
3476 psrad xm3, xm6
3477 vperm2i128 m4, m0, m1, 0x21 ; 1 2
3478 vperm2i128 m5, m1, m2, 0x21 ; 3 4
3479 vperm2i128 m6, m2, m3, 0x21 ; 5 6
3480 shr myd, 6
3481 mov r13d, 64 << 24
3482 lea myd, [t1+myq]
3483 cmovnz r13q, [base+subpel_filters+myq*8]
3484 pslld m4, 16
3485 pslld m5, 16
3486 pslld m6, 16
3487 pblendw m0, m4, 0xaa ; 01 12
3488 pblendw m1, m5, 0xaa ; 23 34
3489 pblendw m2, m6, 0xaa ; 45 56
3490 movq xm10, r13q
3491 punpcklqdq xm10, xm10
3492 pmovsxbw m10, xm10
3493 pshufd m7, m10, q0000
3494 pshufd m8, m10, q1111
3495 pshufd m9, m10, q2222
3496 pshufd m10, m10, q3333
3497 .dy1_w4_loop:
3498 movu xm11, [srcq+ssq*0]
3499 movu xm6, [srcq+r4 ]
3500 vinserti128 m11, [srcq+ssq*1], 1
3501 vinserti128 m6, [srcq+r11 ], 1
3502 lea srcq, [srcq+ssq*2]
3503 pmaddwd m4, m0, m7
3504 pmaddwd m5, m1, m8
3505 pshufb m11, m12
3506 pshufb m6, m13
3507 pmaddwd m11, m14
3508 pmaddwd m6, m15
3509 paddd m4, [rsp+0x20]
3510 phaddd m11, m6
3511 pmaddwd m6, m2, m9
3512 paddd m11, [rsp+0x00]
3513 psrad m11, [rsp+0x40]
3514 mova m0, m1
3515 mova m1, m2
3516 paddd m5, m6
3517 paddd m4, m5
3518 vinserti128 m2, m3, xm11, 1
3519 pslld m3, m11, 16
3520 pblendw m2, m3, 0xaa ; 67 78
3521 pmaddwd m5, m2, m10
3522 vextracti128 xm3, m11, 1
3523 paddd m4, m5
3524 %if isput
3525 psrad m4, [rsp+0x48]
3526 vextracti128 xm5, m4, 1
3527 packusdw xm4, xm5
3528 pminsw xm4, [rsp+0x50]
3529 movq [dstq+dsq*0], xm4
3530 movhps [dstq+dsq*1], xm4
3531 lea dstq, [dstq+dsq*2]
3532 %else
3533 psrad m4, 6
3534 vextracti128 xm5, m4, 1
3535 packssdw xm4, xm5
3536 mova [tmpq], xm4
3537 add tmpq, 16
3538 %endif
3539 sub hd, 2
3540 jg .dy1_w4_loop
3541 MC_8TAP_SCALED_RET
3542 SWAP m10, m13
3543 .dy1_w8:
3544 mov dword [rsp+0xa0], 1
3545 movifprep tmp_stridem, 16
3546 jmp .dy1_w_start
3547 .dy1_w16:
3548 mov dword [rsp+0xa0], 2
3549 movifprep tmp_stridem, 32
3550 jmp .dy1_w_start
3551 .dy1_w32:
3552 mov dword [rsp+0xa0], 4
3553 movifprep tmp_stridem, 64
3554 jmp .dy1_w_start
3555 .dy1_w64:
3556 mov dword [rsp+0xa0], 8
3557 movifprep tmp_stridem, 128
3558 jmp .dy1_w_start
3559 .dy1_w128:
3560 mov dword [rsp+0xa0], 16
3561 movifprep tmp_stridem, 256
3562 .dy1_w_start:
3563 SWAP m10, m12, m1
3564 SWAP m11, m7
3565 ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free
3566 mov myd, mym
3567 %if isput
3568 %define dsm [rsp+0xb8]
3569 movifnidn dsm, dsq
3570 mova [rsp+0xc0], xm7
3571 %else
3572 %if UNIX64
3573 %define hm [rsp+0xb8]
3574 %endif
3575 %endif
3576 mova [rsp+0x00], m10
3577 mova [rsp+0x20], m13
3578 mova [rsp+0x40], xm11
3579 shr t0d, 16
3580 sub srcq, 6
3581 shr myd, 6
3582 mov r4d, 64 << 24
3583 lea myd, [t1+myq]
3584 cmovnz r4q, [base+subpel_filters+myq*8]
3585 pmaddwd m8, [base+rescale_mul2]
3586 movd xm15, t0d
3587 mov [rsp+0xa4], t0d
3588 mov [rsp+0xa8], srcq
3589 mov [rsp+0xb0], r0q ; dstq / tmpq
3590 %if UNIX64
3591 mov hm, hd
3592 %endif
3593 shl dword dxm, 3 ; dx*8
3594 vpbroadcastd m15, xm15
3595 paddd m1, m8 ; mx+dx*[0-7]
3596 movq xm0, r4q
3597 pmovsxbw xm0, xm0
3598 mova [rsp+0x50], xm0
3599 jmp .dy1_hloop
3600 .dy1_hloop_prep:
3601 dec dword [rsp+0xa0]
3602 jz .ret
3603 add qword [rsp+0xb0], 16
3604 mov hd, hm
3605 vpbroadcastd m8, dxm
3606 vpbroadcastd m6, [base+pd_0x3ff]
3607 paddd m1, m8, [rsp+0x60]
3608 vpbroadcastd m15, [rsp+0xa4]
3609 pxor m9, m9
3610 mov srcq, [rsp+0xa8]
3611 mov r0q, [rsp+0xb0] ; dstq / tmpq
3612 mova m10, [rsp+0x00]
3613 mova xm11, [rsp+0x40]
3614 .dy1_hloop:
3615 vpbroadcastq xm2, [base+pq_0x40000000]
3616 pand m5, m1, m6
3617 psrld m5, 6
3618 paddd m15, m5
3619 pcmpeqd m5, m9
3620 vextracti128 xm7, m15, 1
3621 movq r6, xm15
3622 pextrq r9, xm15, 1
3623 movq r11, xm7
3624 pextrq rX, xm7, 1
3625 mov r4d, r6d
3626 shr r6, 32
3627 mov r7d, r9d
3628 shr r9, 32
3629 mov r10d, r11d
3630 shr r11, 32
3631 mov r13d, rXd
3632 shr rX, 32
3633 mova [rsp+0x60], m1
3634 movq xm12, [base+subpel_filters+ r4*8]
3635 movq xm13, [base+subpel_filters+ r6*8]
3636 movhps xm12, [base+subpel_filters+ r7*8]
3637 movhps xm13, [base+subpel_filters+ r9*8]
3638 movq xm14, [base+subpel_filters+r10*8]
3639 movq xm15, [base+subpel_filters+r11*8]
3640 movhps xm14, [base+subpel_filters+r13*8]
3641 movhps xm15, [base+subpel_filters+ rX*8]
3642 psrld m1, 10
3643 vextracti128 xm7, m1, 1
3644 vextracti128 xm6, m5, 1
3645 movq r6, xm1
3646 pextrq r11, xm1, 1
3647 movq r9, xm7
3648 pextrq rX, xm7, 1
3649 mov r4d, r6d
3650 shr r6, 32
3651 mov r10d, r11d
3652 shr r11, 32
3653 mov r7d, r9d
3654 shr r9, 32
3655 mov r13d, rXd
3656 shr rX, 32
3657 pshufd xm4, xm5, q2200
3658 pshufd xm5, xm5, q3311
3659 pshufd xm7, xm6, q2200
3660 pshufd xm6, xm6, q3311
3661 pblendvb xm12, xm2, xm4
3662 pblendvb xm13, xm2, xm5
3663 pblendvb xm14, xm2, xm7
3664 pblendvb xm15, xm2, xm6
3665 pmovsxbw m12, xm12
3666 pmovsxbw m13, xm13
3667 pmovsxbw m14, xm14
3668 pmovsxbw m15, xm15
3669 MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b
3670 mova [rsp+0x80], m0
3671 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b
3672 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b
3673 MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b
3674 mova m0, [rsp+0x80]
3675 vbroadcasti128 m7, [base+subpel_s_shuf8]
3676 vpbroadcastd m8, [rsp+0x50]
3677 vpbroadcastd m9, [rsp+0x54]
3678 vpbroadcastd m10, [rsp+0x58]
3679 vpbroadcastd m11, [rsp+0x5c]
3680 pshufb m0, m7 ; 01a 01b
3681 pshufb m1, m7 ; 23a 23b
3682 pshufb m2, m7 ; 45a 45b
3683 pshufb m3, m7 ; 67a 67b
3684 .dy1_vloop:
3685 pmaddwd m4, m0, m8
3686 pmaddwd m5, m1, m9
3687 pmaddwd m6, m2, m10
3688 pmaddwd m7, m3, m11
3689 paddd m4, [rsp+0x20]
3690 paddd m6, m7
3691 paddd m4, m5
3692 paddd m4, m6
3693 %if isput
3694 psrad m4, [rsp+0x48]
3695 vextracti128 xm5, m4, 1
3696 packusdw xm4, xm5
3697 pminsw xm4, [rsp+0xc0]
3698 mova [dstq], xm4
3699 add dstq, dsm
3700 %else
3701 psrad m4, 6
3702 vextracti128 xm5, m4, 1
3703 packssdw xm4, xm5
3704 mova [tmpq], xm4
3705 add tmpq, tmp_stridem
3706 %endif
3707 dec hd
3708 jz .dy1_hloop_prep
3709 vbroadcasti128 m7, [base+wswap]
3710 pshufb m0, m7
3711 pshufb m1, m7
3712 pshufb m2, m7
3713 pshufb m3, m7
3714 movu xm4, [srcq+ r4*2]
3715 movu xm5, [srcq+ r6*2]
3716 movu xm6, [srcq+ r7*2]
3717 movu xm7, [srcq+ r9*2]
3718 vinserti128 m4, [srcq+r10*2], 1
3719 vinserti128 m5, [srcq+r11*2], 1
3720 vinserti128 m6, [srcq+r13*2], 1
3721 vinserti128 m7, [srcq+ rX*2], 1
3722 add srcq, ssq
3723 pmaddwd m4, m12
3724 pmaddwd m5, m13
3725 pmaddwd m6, m14
3726 pmaddwd m7, m15
3727 phaddd m4, m5
3728 phaddd m6, m7
3729 phaddd m4, m6
3730 paddd m4, [rsp+0x00]
3731 psrad m4, [rsp+0x40]
3732 pslld m4, 16
3733 pblendw m0, m1, 0xaa
3734 pblendw m1, m2, 0xaa
3735 pblendw m2, m3, 0xaa
3736 pblendw m3, m4, 0xaa
3737 jmp .dy1_vloop
3738 SWAP m1, m12, m10
3739 SWAP m7, m11
3740 .dy2:
3741 movzx wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2]
3742 add wq, base_reg
3743 jmp wq
3744 %if isput
3745 .dy2_w2:
3746 mov myd, mym
3747 movzx t0d, t0b
3748 sub srcq, 2
3749 movd xm15, t0d
3750 punpckldq m8, m9, m8
3751 paddd m10, m8 ; mx+dx*[0-1]
3752 vpbroadcastd xm14, [base+pq_0x40000000+2]
3753 vpbroadcastd xm15, xm15
3754 pand xm8, xm10, xm6
3755 psrld xm8, 6
3756 paddd xm15, xm8
3757 movd r4d, xm15
3758 pextrd r6d, xm15, 1
3759 vbroadcasti128 m5, [base+bdct_lb_q]
3760 vbroadcasti128 m6, [base+subpel_s_shuf2]
3761 vpbroadcastd xm15, [base+subpel_filters+r4*8+2]
3762 vpbroadcastd xm4, [base+subpel_filters+r6*8+2]
3763 pcmpeqd xm8, xm9
3764 psrld m10, 10
3765 paddd m10, m10
3766 movu xm0, [srcq+ssq*0]
3767 movu xm1, [srcq+ssq*2]
3768 movu xm2, [srcq+ssq*4]
3769 pshufb m10, m5
3770 paddb m10, m6
3771 vpblendd xm15, xm4, 0xa
3772 pblendvb xm15, xm14, xm8
3773 pmovsxbw m15, xm15
3774 vinserti128 m0, [srcq+ssq*1], 1 ; 0 1
3775 vinserti128 m1, [srcq+ss3q ], 1 ; 2 3
3776 lea srcq, [srcq+ssq*4]
3777 vinserti128 m2, [srcq+ssq*1], 1 ; 4 5
3778 lea srcq, [srcq+ssq*2]
3779 shr myd, 6
3780 mov r4d, 64 << 24
3781 lea myd, [t1+myq]
3782 cmovnz r4q, [base+subpel_filters+myq*8]
3783 pshufb m0, m10
3784 pshufb m1, m10
3785 pshufb m2, m10
3786 pmaddwd m0, m15
3787 pmaddwd m1, m15
3788 pmaddwd m2, m15
3789 movq xm6, r4q
3790 pmovsxbw xm6, xm6
3791 phaddd m0, m1
3792 phaddd m1, m2
3793 paddd m0, m12
3794 paddd m1, m12
3795 psrad m0, xm7
3796 psrad m1, xm7
3797 packssdw m0, m1 ; 0 2 2 4 1 3 3 5
3798 vextracti128 xm1, m0, 1
3799 pshufd xm8, xm6, q0000
3800 pshufd xm9, xm6, q1111
3801 pshufd xm14, xm6, q2222
3802 pshufd xm6, xm6, q3333
3803 punpcklwd xm2, xm0, xm1 ; 01 23
3804 punpckhwd xm1, xm0, xm1 ; 23 45
3805 .dy2_w2_loop:
3806 movu xm3, [srcq+ssq*0]
3807 movu xm5, [srcq+ssq*2]
3808 vinserti128 m3, [srcq+ssq*1], 1 ; 6 7
3809 vinserti128 m5, [srcq+ss3q ], 1 ; 8 9
3810 lea srcq, [srcq+ssq*4]
3811 pmaddwd xm4, xm2, xm8
3812 pmaddwd xm1, xm9
3813 pshufb m3, m10
3814 pshufb m5, m10
3815 pmaddwd m3, m15
3816 pmaddwd m5, m15
3817 phaddd m3, m5
3818 paddd xm4, xm1
3819 paddd m3, m12
3820 psrad m3, xm7
3821 packssdw m3, m3
3822 pshufd m3, m3, q2100
3823 palignr m0, m3, m0, 12 ; 4 6 6 8 5 7 7 9
3824 vextracti128 xm1, m0, 1
3825 punpcklwd xm2, xm0, xm1 ; 45 67
3826 punpckhwd xm1, xm0, xm1 ; 67 89
3827 pmaddwd xm3, xm2, xm14
3828 pmaddwd xm5, xm1, xm6
3829 paddd xm4, xm13
3830 paddd xm4, xm3
3831 psrldq xm3, xm7, 8
3832 paddd xm4, xm5
3833 psrad xm4, xm3
3834 packusdw xm4, xm4
3835 pminsw xm4, xm11
3836 movd [dstq+dsq*0], xm4
3837 pextrd [dstq+dsq*1], xm4, 1
3838 lea dstq, [dstq+dsq*2]
3839 sub hd, 2
3840 jg .dy2_w2_loop
3841 RET
3842 %endif
3843 .dy2_w4:
3844 mov myd, mym
3845 %if isput
3846 mova [rsp+0x50], xm11
3847 %endif
3848 mova [rsp+0x00], m12
3849 mova [rsp+0x20], m13
3850 mova [rsp+0x40], xm7
3851 vbroadcasti128 m7, [base+rescale_mul]
3852 movzx t0d, t0b
3853 sub srcq, 2
3854 movd xm15, t0d
3855 pmaddwd m8, m7
3856 vpbroadcastq m2, [base+pq_0x40000000+1]
3857 vpbroadcastd xm15, xm15
3858 SWAP m13, m10
3859 paddd m13, m8 ; mx+dx*[0-3]
3860 pand m6, m13
3861 psrld m6, 6
3862 paddd xm15, xm6
3863 movd r4d, xm15
3864 pextrd r6d, xm15, 1
3865 pextrd r11d, xm15, 2
3866 pextrd r13d, xm15, 3
3867 vbroadcasti128 m5, [base+bdct_lb_q+ 0]
3868 vbroadcasti128 m1, [base+bdct_lb_q+16]
3869 vbroadcasti128 m4, [base+subpel_s_shuf2]
3870 vpbroadcastd xm14, [base+subpel_filters+r4*8+2]
3871 vpbroadcastd xm7, [base+subpel_filters+r6*8+2]
3872 vpbroadcastd xm15, [base+subpel_filters+r11*8+2]
3873 vpbroadcastd xm8, [base+subpel_filters+r13*8+2]
3874 shr myd, 6
3875 mov r13d, 64 << 24
3876 lea myd, [t1+myq]
3877 cmovnz r13q, [base+subpel_filters+myq*8]
3878 pcmpeqd m6, m9
3879 punpckldq m11, m6, m6
3880 punpckhdq m6, m6
3881 psrld m13, 10
3882 paddd m13, m13
3883 vpblendd xm14, xm7, 0xa
3884 vpblendd xm15, xm8, 0xa
3885 pmovsxbw m14, xm14
3886 pmovsxbw m15, xm15
3887 movq xm10, r13q
3888 pblendvb m14, m2, m11
3889 pblendvb m15, m2, m6
3890 pextrd r4, xm13, 2
3891 pshufb m12, m13, m5
3892 pshufb m13, m1
3893 lea r6, [r4+ssq*1]
3894 lea r11, [r4+ssq*2]
3895 lea r13, [r4+ss3q ]
3896 movu xm0, [srcq+ssq*0]
3897 movu xm7, [srcq+r4 ]
3898 movu xm1, [srcq+ssq*1]
3899 movu xm8, [srcq+r6 ]
3900 vinserti128 m0, [srcq+ssq*2], 1 ; 0 2
3901 vinserti128 m7, [srcq+r11 ], 1
3902 vinserti128 m1, [srcq+ss3q ], 1 ; 1 3
3903 vinserti128 m8, [srcq+r13 ], 1
3904 lea srcq, [srcq+ssq*4]
3905 movu xm2, [srcq+ssq*0]
3906 movu xm9, [srcq+r4 ]
3907 vinserti128 m2, [srcq+ssq*1], 1 ; 4 5
3908 vinserti128 m9, [srcq+r6 ], 1
3909 lea srcq, [srcq+ssq*2]
3910 vpbroadcastb m5, xm13
3911 psubb m13, m5
3912 paddb m12, m4
3913 paddb m13, m4
3914 mova m5, [rsp+0x00]
3915 movd xm6, [rsp+0x40]
3916 pshufb m0, m12
3917 pshufb m1, m12
3918 pshufb m2, m12
3919 pmaddwd m0, m14
3920 pmaddwd m1, m14
3921 pmaddwd m2, m14
3922 pshufb m7, m13
3923 pshufb m8, m13
3924 pshufb m9, m13
3925 pmaddwd m7, m15
3926 pmaddwd m8, m15
3927 pmaddwd m9, m15
3928 punpcklqdq xm10, xm10
3929 pmovsxbw m10, xm10
3930 phaddd m0, m7
3931 phaddd m1, m8
3932 phaddd m2, m9
3933 paddd m0, m5
3934 paddd m1, m5
3935 paddd m2, m5
3936 psrad m0, xm6
3937 psrad m1, xm6
3938 psrad m2, xm6
3939 vperm2i128 m3, m0, m2, 0x21 ; 2 4
3940 vperm2i128 m2, m1, 0x13 ; 3 5
3941 pshufd m7, m10, q0000
3942 pshufd m8, m10, q1111
3943 pshufd m9, m10, q2222
3944 pshufd m10, m10, q3333
3945 packssdw m0, m3 ; 0 2 2 4
3946 packssdw m1, m2 ; 1 3 3 5
3947 punpckhwd m2, m0, m1 ; 23 45
3948 punpcklwd m0, m1 ; 01 23
3949 .dy2_w4_loop:
3950 movu xm1, [srcq+ssq*0]
3951 movu xm6, [srcq+r4 ]
3952 movu xm3, [srcq+ssq*1]
3953 movu xm11, [srcq+r6 ]
3954 vinserti128 m1, [srcq+ssq*2], 1 ; 6 8
3955 vinserti128 m6, [srcq+r11 ], 1
3956 vinserti128 m3, [srcq+ss3q ], 1 ; 7 9
3957 vinserti128 m11, [srcq+r13 ], 1
3958 lea srcq, [srcq+ssq*4]
3959 pmaddwd m4, m0, m7
3960 pmaddwd m5, m2, m8
3961 pshufb m1, m12
3962 pshufb m3, m12
3963 pmaddwd m1, m14
3964 pmaddwd m3, m14
3965 mova m0, [rsp+0x00]
3966 pshufb m6, m13
3967 pshufb m11, m13
3968 pmaddwd m6, m15
3969 pmaddwd m11, m15
3970 paddd m4, m5
3971 movd xm5, [rsp+0x40]
3972 phaddd m1, m6
3973 phaddd m3, m11
3974 paddd m1, m0
3975 paddd m3, m0
3976 psrad m1, xm5
3977 psrad m3, xm5
3978 pslld m3, 16
3979 pblendw m1, m3, 0xaa ; 67 89
3980 vperm2i128 m0, m2, m1, 0x21 ; 45 67
3981 paddd m4, [rsp+0x20]
3982 mova m2, m1
3983 pmaddwd m5, m0, m9
3984 pmaddwd m6, m2, m10
3985 paddd m4, m5
3986 paddd m4, m6
3987 %if isput
3988 psrad m4, [rsp+0x48]
3989 vextracti128 xm5, m4, 1
3990 packusdw xm4, xm5
3991 pminsw xm4, [rsp+0x50]
3992 movq [dstq+dsq*0], xm4
3993 movhps [dstq+dsq*1], xm4
3994 lea dstq, [dstq+dsq*2]
3995 %else
3996 psrad m4, 6
3997 vextracti128 xm5, m4, 1
3998 packssdw xm4, xm5
3999 mova [tmpq], xm4
4000 add tmpq, 16
4001 %endif
4002 sub hd, 2
4003 jg .dy2_w4_loop
4004 MC_8TAP_SCALED_RET
4005 SWAP m10, m13
4006 .dy2_w8:
4007 mov dword [rsp+0xa0], 1
4008 movifprep tmp_stridem, 16
4009 jmp .dy2_w_start
4010 .dy2_w16:
4011 mov dword [rsp+0xa0], 2
4012 movifprep tmp_stridem, 32
4013 jmp .dy2_w_start
4014 .dy2_w32:
4015 mov dword [rsp+0xa0], 4
4016 movifprep tmp_stridem, 64
4017 jmp .dy2_w_start
4018 .dy2_w64:
4019 mov dword [rsp+0xa0], 8
4020 movifprep tmp_stridem, 128
4021 jmp .dy2_w_start
4022 .dy2_w128:
4023 mov dword [rsp+0xa0], 16
4024 movifprep tmp_stridem, 256
4025 .dy2_w_start:
4026 SWAP m10, m12, m1
4027 SWAP m11, m7
4028 ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free
4029 mov myd, mym
4030 %if isput
4031 movifnidn dsm, dsq
4032 mova [rsp+0xc0], xm7
4033 %endif
4034 mova [rsp+0x00], m10
4035 mova [rsp+0x20], m13
4036 mova [rsp+0x40], xm11
4037 shr t0d, 16
4038 sub srcq, 6
4039 shr myd, 6
4040 mov r4d, 64 << 24
4041 lea myd, [t1+myq]
4042 cmovnz r4q, [base+subpel_filters+myq*8]
4043 pmaddwd m8, [base+rescale_mul2]
4044 movd xm15, t0d
4045 mov [rsp+0xa4], t0d
4046 mov [rsp+0xa8], srcq
4047 mov [rsp+0xb0], r0q ; dstq / tmpq
4048 %if UNIX64
4049 mov hm, hd
4050 %endif
4051 shl dword dxm, 3 ; dx*8
4052 vpbroadcastd m15, xm15
4053 paddd m1, m8 ; mx+dx*[0-7]
4054 movq xm0, r4q
4055 pmovsxbw xm0, xm0
4056 mova [rsp+0x50], xm0
4057 jmp .dy2_hloop
4058 .dy2_hloop_prep:
4059 dec dword [rsp+0xa0]
4060 jz .ret
4061 add qword [rsp+0xb0], 16
4062 mov hd, hm
4063 vpbroadcastd m8, dxm
4064 vpbroadcastd m6, [base+pd_0x3ff]
4065 paddd m1, m8, [rsp+0x60]
4066 vpbroadcastd m15, [rsp+0xa4]
4067 pxor m9, m9
4068 mov srcq, [rsp+0xa8]
4069 mov r0q, [rsp+0xb0] ; dstq / tmpq
4070 mova m10, [rsp+0x00]
4071 mova xm11, [rsp+0x40]
4072 .dy2_hloop:
4073 vpbroadcastq xm2, [base+pq_0x40000000]
4074 pand m5, m1, m6
4075 psrld m5, 6
4076 paddd m15, m5
4077 pcmpeqd m5, m9
4078 vextracti128 xm7, m15, 1
4079 movq r6, xm15
4080 pextrq r9, xm15, 1
4081 movq r11, xm7
4082 pextrq rX, xm7, 1
4083 mov r4d, r6d
4084 shr r6, 32
4085 mov r7d, r9d
4086 shr r9, 32
4087 mov r10d, r11d
4088 shr r11, 32
4089 mov r13d, rXd
4090 shr rX, 32
4091 mova [rsp+0x60], m1
4092 movq xm12, [base+subpel_filters+ r4*8]
4093 movq xm13, [base+subpel_filters+ r6*8]
4094 movhps xm12, [base+subpel_filters+ r7*8]
4095 movhps xm13, [base+subpel_filters+ r9*8]
4096 movq xm14, [base+subpel_filters+r10*8]
4097 movq xm15, [base+subpel_filters+r11*8]
4098 movhps xm14, [base+subpel_filters+r13*8]
4099 movhps xm15, [base+subpel_filters+ rX*8]
4100 psrld m1, 10
4101 vextracti128 xm7, m1, 1
4102 vextracti128 xm6, m5, 1
4103 movq r6, xm1
4104 pextrq r11, xm1, 1
4105 movq r9, xm7
4106 pextrq rX, xm7, 1
4107 mov r4d, r6d
4108 shr r6, 32
4109 mov r10d, r11d
4110 shr r11, 32
4111 mov r7d, r9d
4112 shr r9, 32
4113 mov r13d, rXd
4114 shr rX, 32
4115 pshufd xm4, xm5, q2200
4116 pshufd xm5, xm5, q3311
4117 pshufd xm7, xm6, q2200
4118 pshufd xm6, xm6, q3311
4119 pblendvb xm12, xm2, xm4
4120 pblendvb xm13, xm2, xm5
4121 pblendvb xm14, xm2, xm7
4122 pblendvb xm15, xm2, xm6
4123 pmovsxbw m12, xm12
4124 pmovsxbw m13, xm13
4125 pmovsxbw m14, xm14
4126 pmovsxbw m15, xm15
4127 MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b
4128 mova [rsp+0x80], m0
4129 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b
4130 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b
4131 MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b
4132 mova m0, [rsp+0x80]
4133 vbroadcasti128 m7, [base+subpel_s_shuf8]
4134 vpbroadcastd m8, [rsp+0x50]
4135 vpbroadcastd m9, [rsp+0x54]
4136 vpbroadcastd m10, [rsp+0x58]
4137 vpbroadcastd m11, [rsp+0x5c]
4138 pshufb m0, m7 ; 01a 01b
4139 pshufb m1, m7 ; 23a 23b
4140 pshufb m2, m7 ; 45a 45b
4141 pshufb m3, m7 ; 67a 67b
4142 .dy2_vloop:
4143 pmaddwd m4, m0, m8
4144 pmaddwd m5, m1, m9
4145 pmaddwd m6, m2, m10
4146 pmaddwd m7, m3, m11
4147 paddd m4, [rsp+0x20]
4148 paddd m6, m7
4149 paddd m4, m5
4150 paddd m4, m6
4151 %if isput
4152 psrad m4, [rsp+0x48]
4153 vextracti128 xm5, m4, 1
4154 packusdw xm4, xm5
4155 pminsw xm4, [rsp+0xc0]
4156 mova [dstq], xm4
4157 add dstq, dsm
4158 %else
4159 psrad m4, 6
4160 vextracti128 xm5, m4, 1
4161 packssdw xm4, xm5
4162 mova [tmpq], xm4
4163 add tmpq, tmp_stridem
4164 %endif
4165 dec hd
4166 jz .dy2_hloop_prep
4167 mova m0, m1
4168 mova m1, m2
4169 mova m2, m3
4170 movu xm3, [srcq+ r4*2]
4171 movu xm4, [srcq+ r6*2]
4172 movu xm5, [srcq+ r7*2]
4173 movu xm6, [srcq+ r9*2]
4174 vinserti128 m3, [srcq+r10*2], 1
4175 vinserti128 m4, [srcq+r11*2], 1
4176 vinserti128 m5, [srcq+r13*2], 1
4177 vinserti128 m6, [srcq+ rX*2], 1
4178 add srcq, ssq
4179 pmaddwd m3, m12
4180 pmaddwd m4, m13
4181 pmaddwd m5, m14
4182 pmaddwd m6, m15
4183 phaddd m3, m4
4184 phaddd m5, m6
4185 phaddd m3, m5
4186 movu xm4, [srcq+ r4*2]
4187 movu xm5, [srcq+ r6*2]
4188 movu xm6, [srcq+ r7*2]
4189 movu xm7, [srcq+ r9*2]
4190 vinserti128 m4, [srcq+r10*2], 1
4191 vinserti128 m5, [srcq+r11*2], 1
4192 vinserti128 m6, [srcq+r13*2], 1
4193 vinserti128 m7, [srcq+ rX*2], 1
4194 add srcq, ssq
4195 pmaddwd m4, m12
4196 pmaddwd m5, m13
4197 pmaddwd m6, m14
4198 pmaddwd m7, m15
4199 phaddd m4, m5
4200 phaddd m6, m7
4201 mova m5, [rsp+0x00]
4202 movd xm7, [rsp+0x40]
4203 phaddd m4, m6
4204 paddd m3, m5
4205 paddd m4, m5
4206 psrad m3, xm7
4207 psrad m4, xm7
4208 pslld m4, 16
4209 pblendw m3, m4, 0xaa
4210 jmp .dy2_vloop
4211 .ret:
4212 MC_8TAP_SCALED_RET 0
4213 %undef isput
4214 %undef isprep
4215 %endmacro
4216
4217 %macro BILIN_SCALED_FN 1
4218 cglobal %1_bilin_scaled_16bpc
4219 mov t0d, (5*15 << 16) | 5*15
4220 mov t1d, t0d
4221 jmp mangle(private_prefix %+ _%1_8tap_scaled_16bpc %+ SUFFIX)
4222 %endmacro
4223
4224 %if WIN64
4225 DECLARE_REG_TMP 6, 5
4226 %else
4227 DECLARE_REG_TMP 6, 8
4228 %endif
4229
4230 %define PUT_8TAP_SCALED_FN FN put_8tap_scaled,
4231 BILIN_SCALED_FN put
4232 PUT_8TAP_SCALED_FN sharp, SHARP, SHARP
4233 PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH
4234 PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP
4235 PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH
4236 PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR
4237 PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP
4238 PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR
4239 PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH
4240 PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR
4241 MC_8TAP_SCALED put
4242
4243 %if WIN64
4244 DECLARE_REG_TMP 5, 4
4245 %else
4246 DECLARE_REG_TMP 6, 7
4247 %endif
4248
4249 %define PREP_8TAP_SCALED_FN FN prep_8tap_scaled,
4250 BILIN_SCALED_FN prep
4251 PREP_8TAP_SCALED_FN sharp, SHARP, SHARP
4252 PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH
4253 PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP
4254 PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH
4255 PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR
4256 PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP
4257 PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR
4258 PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH
4259 PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR
4260 MC_8TAP_SCALED prep
4261
4262 %macro WARP_V 5 ; dst, 01, 23, 45, 67
4263 lea tmp1d, [myq+deltaq*4]
4264 lea tmp2d, [myq+deltaq*1]
4265 shr myd, 10
4266 shr tmp1d, 10
4267 movq xm8, [filterq+myq *8]
4268 vinserti128 m8, [filterq+tmp1q*8], 1 ; a e
4269 lea tmp1d, [tmp2q+deltaq*4]
4270 lea myd, [tmp2q+deltaq*1]
4271 shr tmp2d, 10
4272 shr tmp1d, 10
4273 movq xm0, [filterq+tmp2q*8]
4274 vinserti128 m0, [filterq+tmp1q*8], 1 ; b f
4275 lea tmp1d, [myq+deltaq*4]
4276 lea tmp2d, [myq+deltaq*1]
4277 shr myd, 10
4278 shr tmp1d, 10
4279 movq xm9, [filterq+myq *8]
4280 vinserti128 m9, [filterq+tmp1q*8], 1 ; c g
4281 lea tmp1d, [tmp2q+deltaq*4]
4282 lea myd, [tmp2q+gammaq] ; my += gamma
4283 punpcklwd m8, m0
4284 shr tmp2d, 10
4285 shr tmp1d, 10
4286 movq xm0, [filterq+tmp2q*8]
4287 vinserti128 m0, [filterq+tmp1q*8], 1 ; d h
4288 punpcklwd m0, m9, m0
4289 punpckldq m9, m8, m0
4290 punpckhdq m0, m8, m0
4291 punpcklbw m8, m11, m9 ; a0 a1 b0 b1 c0 c1 d0 d1 << 8
4292 punpckhbw m9, m11, m9 ; a2 a3 b2 b3 c2 c3 d2 d3 << 8
4293 pmaddwd m%2, m8
4294 pmaddwd m9, m%3
4295 punpcklbw m8, m11, m0 ; a4 a5 b4 b5 c4 c5 d4 d5 << 8
4296 punpckhbw m0, m11, m0 ; a6 a7 b6 b7 c6 c7 d6 d7 << 8
4297 pmaddwd m8, m%4
4298 pmaddwd m0, m%5
4299 paddd m9, m%2
4300 mova m%2, m%3
4301 paddd m0, m8
4302 mova m%3, m%4
4303 mova m%4, m%5
4304 paddd m%1, m0, m9
4305 %endmacro
4306
4307 cglobal warp_affine_8x8t_16bpc, 4, 14, 16, tmp, ts
4308 mov r6d, r7m
4309 lea r9, [$$]
4310 shr r6d, 11
4311 vpbroadcastd m13, [r9-$$+warp8x8_shift+r6*4]
4312 vpbroadcastd m14, [warp8x8t_rnd]
4313 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx2).main
4314 jmp .start
4315 .loop:
4316 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx2).main2
4317 lea tmpq, [tmpq+tsq*4]
4318 .start:
4319 paddd m7, m14
4320 paddd m0, m14
4321 psrad m7, 15
4322 psrad m0, 15
4323 packssdw m7, m0
4324 vpermq m7, m7, q3120
4325 mova [tmpq+tsq*0], xm7
4326 vextracti128 [tmpq+tsq*2], m7, 1
4327 dec r4d
4328 jg .loop
4329 .end:
4330 RET
4331
4332 cglobal warp_affine_8x8_16bpc, 4, 14, 16, dst, ds, src, ss, abcd, mx, tmp2, \
4333 alpha, beta, filter, tmp1, delta, \
4334 my, gamma
4335 mov r6d, r7m
4336 lea filterq, [$$]
4337 shr r6d, 11
4338 vpbroadcastd m13, [filterq-$$+warp8x8_shift+r6*4]
4339 vpbroadcastd m14, [filterq-$$+warp8x8_rnd +r6*4]
4340 vpbroadcastw m15, r7m ; pixel_max
4341 call .main
4342 jmp .start
4343 .loop:
4344 call .main2
4345 lea dstq, [dstq+dsq*2]
4346 .start:
4347 psrad m7, 16
4348 psrad m0, 16
4349 packusdw m7, m0
4350 pmulhrsw m7, m14
4351 pminsw m7, m15
4352 vpermq m7, m7, q3120
4353 mova [dstq+dsq*0], xm7
4354 vextracti128 [dstq+dsq*1], m7, 1
4355 dec r4d
4356 jg .loop
4357 .end:
4358 RET
4359 ALIGN function_align
4360 .main:
4361 ; Stack args offset by one (r4m -> r5m etc.) due to call
4362 %if WIN64
4363 mov abcdq, r5m
4364 mov mxd, r6m
4365 %endif
4366 movsx alphad, word [abcdq+2*0]
4367 movsx betad, word [abcdq+2*1]
4368 vpbroadcastd m12, [pd_32768]
4369 pxor m11, m11
4370 add filterq, mc_warp_filter-$$
4371 lea tmp1q, [ssq*3]
4372 add mxd, 512+(64<<10)
4373 lea tmp2d, [alphaq*3]
4374 sub srcq, tmp1q ; src -= src_stride*3
4375 sub betad, tmp2d ; beta -= alpha*3
4376 mov myd, r7m
4377 call .h
4378 psrld m1, m0, 16
4379 call .h
4380 pblendw m1, m0, 0xaa ; 01
4381 psrld m2, m0, 16
4382 call .h
4383 pblendw m2, m0, 0xaa ; 12
4384 psrld m3, m0, 16
4385 call .h
4386 pblendw m3, m0, 0xaa ; 23
4387 psrld m4, m0, 16
4388 call .h
4389 pblendw m4, m0, 0xaa ; 34
4390 psrld m5, m0, 16
4391 call .h
4392 pblendw m5, m0, 0xaa ; 45
4393 psrld m6, m0, 16
4394 call .h
4395 pblendw m6, m0, 0xaa ; 56
4396 movsx deltad, word [abcdq+2*2]
4397 movsx gammad, word [abcdq+2*3]
4398 add myd, 512+(64<<10)
4399 mov r4d, 4
4400 lea tmp1d, [deltaq*3]
4401 sub gammad, tmp1d ; gamma -= delta*3
4402 .main2:
4403 call .h
4404 psrld m7, m6, 16
4405 pblendw m7, m0, 0xaa ; 67
4406 WARP_V 7, 1, 3, 5, 7
4407 call .h
4408 psrld m10, m5, 16
4409 pblendw m10, m0, 0xaa ; 78
4410 WARP_V 0, 2, 4, 6, 10
4411 ret
4412 ALIGN function_align
4413 .h:
4414 lea tmp1d, [mxq+alphaq*4]
4415 lea tmp2d, [mxq+alphaq*1]
4416 movu xm10, [srcq-6]
4417 vinserti128 m10, [srcq+2], 1
4418 shr mxd, 10 ; 0
4419 shr tmp1d, 10 ; 4
4420 movq xm0, [filterq+mxq *8]
4421 vinserti128 m0, [filterq+tmp1q*8], 1
4422 lea tmp1d, [tmp2q+alphaq*4]
4423 lea mxd, [tmp2q+alphaq*1]
4424 movu xm8, [srcq-4]
4425 vinserti128 m8, [srcq+4], 1
4426 shr tmp2d, 10 ; 1
4427 shr tmp1d, 10 ; 5
4428 movq xm9, [filterq+tmp2q*8]
4429 vinserti128 m9, [filterq+tmp1q*8], 1
4430 lea tmp1d, [mxq+alphaq*4]
4431 lea tmp2d, [mxq+alphaq*1]
4432 shr mxd, 10 ; 2
4433 shr tmp1d, 10 ; 6
4434 punpcklbw m0, m11, m0
4435 pmaddwd m0, m10
4436 movu xm10, [srcq-2]
4437 vinserti128 m10, [srcq+6], 1
4438 punpcklbw m9, m11, m9
4439 pmaddwd m9, m8
4440 movq xm8, [filterq+mxq *8]
4441 vinserti128 m8, [filterq+tmp1q*8], 1
4442 lea tmp1d, [tmp2q+alphaq*4]
4443 lea mxd, [tmp2q+betaq] ; mx += beta
4444 phaddd m0, m9 ; 0 1 4 5
4445 movu xm9, [srcq+0]
4446 vinserti128 m9, [srcq+8], 1
4447 shr tmp2d, 10 ; 3
4448 shr tmp1d, 10 ; 7
4449 punpcklbw m8, m11, m8
4450 pmaddwd m8, m10
4451 movq xm10, [filterq+tmp2q*8]
4452 vinserti128 m10, [filterq+tmp1q*8], 1
4453 punpcklbw m10, m11, m10
4454 pmaddwd m9, m10
4455 add srcq, ssq
4456 phaddd m8, m9 ; 2 3 6 7
4457 phaddd m0, m8 ; 0 1 2 3 4 5 6 7
4458 vpsllvd m0, m13
4459 paddd m0, m12 ; rounded 14-bit result in upper 16 bits of dword
4460 ret
4461
4462 %macro BIDIR_FN 0
4463 call .main
4464 lea stride3q, [strideq*3]
4465 jmp wq
4466 .w4:
4467 movq [dstq ], xm0
4468 movhps [dstq+strideq*1], xm0
4469 vextracti128 xm0, m0, 1
4470 movq [dstq+strideq*2], xm0
4471 movhps [dstq+stride3q ], xm0
4472 cmp hd, 4
4473 je .ret
4474 lea dstq, [dstq+strideq*4]
4475 movq [dstq ], xm1
4476 movhps [dstq+strideq*1], xm1
4477 vextracti128 xm1, m1, 1
4478 movq [dstq+strideq*2], xm1
4479 movhps [dstq+stride3q ], xm1
4480 cmp hd, 8
4481 je .ret
4482 lea dstq, [dstq+strideq*4]
4483 movq [dstq ], xm2
4484 movhps [dstq+strideq*1], xm2
4485 vextracti128 xm2, m2, 1
4486 movq [dstq+strideq*2], xm2
4487 movhps [dstq+stride3q ], xm2
4488 lea dstq, [dstq+strideq*4]
4489 movq [dstq ], xm3
4490 movhps [dstq+strideq*1], xm3
4491 vextracti128 xm3, m3, 1
4492 movq [dstq+strideq*2], xm3
4493 movhps [dstq+stride3q ], xm3
4494 .ret:
4495 RET
4496 .w8:
4497 mova [dstq+strideq*0], xm0
4498 vextracti128 [dstq+strideq*1], m0, 1
4499 mova [dstq+strideq*2], xm1
4500 vextracti128 [dstq+stride3q ], m1, 1
4501 cmp hd, 4
4502 jne .w8_loop_start
4503 RET
4504 .w8_loop:
4505 call .main
4506 lea dstq, [dstq+strideq*4]
4507 mova [dstq+strideq*0], xm0
4508 vextracti128 [dstq+strideq*1], m0, 1
4509 mova [dstq+strideq*2], xm1
4510 vextracti128 [dstq+stride3q ], m1, 1
4511 .w8_loop_start:
4512 lea dstq, [dstq+strideq*4]
4513 mova [dstq+strideq*0], xm2
4514 vextracti128 [dstq+strideq*1], m2, 1
4515 mova [dstq+strideq*2], xm3
4516 vextracti128 [dstq+stride3q ], m3, 1
4517 sub hd, 8
4518 jg .w8_loop
4519 RET
4520 .w16_loop:
4521 call .main
4522 lea dstq, [dstq+strideq*4]
4523 .w16:
4524 mova [dstq+strideq*0], m0
4525 mova [dstq+strideq*1], m1
4526 mova [dstq+strideq*2], m2
4527 mova [dstq+stride3q ], m3
4528 sub hd, 4
4529 jg .w16_loop
4530 RET
4531 .w32_loop:
4532 call .main
4533 lea dstq, [dstq+strideq*2]
4534 .w32:
4535 mova [dstq+strideq*0+32*0], m0
4536 mova [dstq+strideq*0+32*1], m1
4537 mova [dstq+strideq*1+32*0], m2
4538 mova [dstq+strideq*1+32*1], m3
4539 sub hd, 2
4540 jg .w32_loop
4541 RET
4542 .w64_loop:
4543 call .main
4544 add dstq, strideq
4545 .w64:
4546 mova [dstq+32*0], m0
4547 mova [dstq+32*1], m1
4548 mova [dstq+32*2], m2
4549 mova [dstq+32*3], m3
4550 dec hd
4551 jg .w64_loop
4552 RET
4553 .w128_loop:
4554 call .main
4555 add dstq, strideq
4556 .w128:
4557 mova [dstq+32*0], m0
4558 mova [dstq+32*1], m1
4559 mova [dstq+32*2], m2
4560 mova [dstq+32*3], m3
4561 call .main
4562 mova [dstq+32*4], m0
4563 mova [dstq+32*5], m1
4564 mova [dstq+32*6], m2
4565 mova [dstq+32*7], m3
4566 dec hd
4567 jg .w128_loop
4568 RET
4569 %endmacro
4570
4571 %if WIN64
4572 DECLARE_REG_TMP 5
4573 %else
4574 DECLARE_REG_TMP 7
4575 %endif
4576
4577 cglobal avg_16bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
4578 %define base r6-avg_avx2_table
4579 lea r6, [avg_avx2_table]
4580 tzcnt wd, wm
4581 mov t0d, r6m ; pixel_max
4582 movsxd wq, [r6+wq*4]
4583 shr t0d, 11
4584 vpbroadcastd m4, [base+bidir_rnd+t0*4]
4585 vpbroadcastd m5, [base+bidir_mul+t0*4]
4586 movifnidn hd, hm
4587 add wq, r6
4588 BIDIR_FN
4589 ALIGN function_align
4590 .main:
4591 mova m0, [tmp1q+32*0]
4592 paddsw m0, [tmp2q+32*0]
4593 mova m1, [tmp1q+32*1]
4594 paddsw m1, [tmp2q+32*1]
4595 mova m2, [tmp1q+32*2]
4596 paddsw m2, [tmp2q+32*2]
4597 mova m3, [tmp1q+32*3]
4598 paddsw m3, [tmp2q+32*3]
4599 add tmp1q, 32*4
4600 add tmp2q, 32*4
4601 pmaxsw m0, m4
4602 pmaxsw m1, m4
4603 pmaxsw m2, m4
4604 pmaxsw m3, m4
4605 psubsw m0, m4
4606 psubsw m1, m4
4607 psubsw m2, m4
4608 psubsw m3, m4
4609 pmulhw m0, m5
4610 pmulhw m1, m5
4611 pmulhw m2, m5
4612 pmulhw m3, m5
4613 ret
4614
4615 cglobal w_avg_16bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, stride3
4616 lea r6, [w_avg_avx2_table]
4617 tzcnt wd, wm
4618 mov t0d, r6m ; weight
4619 vpbroadcastw m8, r7m ; pixel_max
4620 vpbroadcastd m7, [r6-w_avg_avx2_table+pd_65538]
4621 movsxd wq, [r6+wq*4]
4622 paddw m7, m8
4623 add wq, r6
4624 lea r6d, [t0-16]
4625 shl t0d, 16
4626 sub t0d, r6d ; 16-weight, weight
4627 pslld m7, 7
4628 rorx r6d, t0d, 30 ; << 2
4629 test dword r7m, 0x800
4630 cmovz r6d, t0d
4631 movifnidn hd, hm
4632 movd xm6, r6d
4633 vpbroadcastd m6, xm6
4634 BIDIR_FN
4635 ALIGN function_align
4636 .main:
4637 mova m4, [tmp1q+32*0]
4638 mova m0, [tmp2q+32*0]
4639 punpckhwd m5, m0, m4
4640 punpcklwd m0, m4
4641 mova m4, [tmp1q+32*1]
4642 mova m1, [tmp2q+32*1]
4643 pmaddwd m5, m6
4644 pmaddwd m0, m6
4645 paddd m5, m7
4646 paddd m0, m7
4647 psrad m5, 8
4648 psrad m0, 8
4649 packusdw m0, m5
4650 punpckhwd m5, m1, m4
4651 punpcklwd m1, m4
4652 mova m4, [tmp1q+32*2]
4653 mova m2, [tmp2q+32*2]
4654 pmaddwd m5, m6
4655 pmaddwd m1, m6
4656 paddd m5, m7
4657 paddd m1, m7
4658 psrad m5, 8
4659 psrad m1, 8
4660 packusdw m1, m5
4661 punpckhwd m5, m2, m4
4662 punpcklwd m2, m4
4663 mova m4, [tmp1q+32*3]
4664 mova m3, [tmp2q+32*3]
4665 add tmp1q, 32*4
4666 add tmp2q, 32*4
4667 pmaddwd m5, m6
4668 pmaddwd m2, m6
4669 paddd m5, m7
4670 paddd m2, m7
4671 psrad m5, 8
4672 psrad m2, 8
4673 packusdw m2, m5
4674 punpckhwd m5, m3, m4
4675 punpcklwd m3, m4
4676 pmaddwd m5, m6
4677 pmaddwd m3, m6
4678 paddd m5, m7
4679 paddd m3, m7
4680 psrad m5, 8
4681 psrad m3, 8
4682 packusdw m3, m5
4683 pminsw m0, m8
4684 pminsw m1, m8
4685 pminsw m2, m8
4686 pminsw m3, m8
4687 ret
4688
4689 cglobal mask_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3
4690 %define base r7-mask_avx2_table
4691 lea r7, [mask_avx2_table]
4692 tzcnt wd, wm
4693 mov r6d, r7m ; pixel_max
4694 movifnidn hd, hm
4695 shr r6d, 11
4696 movsxd wq, [r7+wq*4]
4697 vpbroadcastd m8, [base+pw_64]
4698 vpbroadcastd m9, [base+bidir_rnd+r6*4]
4699 vpbroadcastd m10, [base+bidir_mul+r6*4]
4700 mov maskq, maskmp
4701 add wq, r7
4702 BIDIR_FN
4703 ALIGN function_align
4704 .main:
4705 %macro MASK 1
4706 pmovzxbw m5, [maskq+16*%1]
4707 mova m%1, [tmp1q+32*%1]
4708 mova m6, [tmp2q+32*%1]
4709 punpckhwd m4, m%1, m6
4710 punpcklwd m%1, m6
4711 psubw m7, m8, m5
4712 punpckhwd m6, m5, m7 ; m, 64-m
4713 punpcklwd m5, m7
4714 pmaddwd m4, m6 ; tmp1 * m + tmp2 * (64-m)
4715 pmaddwd m%1, m5
4716 psrad m4, 5
4717 psrad m%1, 5
4718 packssdw m%1, m4
4719 pmaxsw m%1, m9
4720 psubsw m%1, m9
4721 pmulhw m%1, m10
4722 %endmacro
4723 MASK 0
4724 MASK 1
4725 MASK 2
4726 MASK 3
4727 add maskq, 16*4
4728 add tmp1q, 32*4
4729 add tmp2q, 32*4
4730 ret
4731
4732 cglobal w_mask_420_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
4733 %define base r7-w_mask_420_avx2_table
4734 lea r7, [w_mask_420_avx2_table]
4735 tzcnt wd, wm
4736 mov r6d, r8m ; pixel_max
4737 movd xm0, r7m ; sign
4738 movifnidn hd, hm
4739 shr r6d, 11
4740 movsxd wq, [r7+wq*4]
4741 vpbroadcastd m10, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
4742 vpbroadcastd m11, [base+pw_64]
4743 vpbroadcastd m12, [base+bidir_rnd+r6*4]
4744 vpbroadcastd m13, [base+bidir_mul+r6*4]
4745 movd xm14, [base+pw_2]
4746 mov maskq, maskmp
4747 psubw xm14, xm0
4748 vpbroadcastw m14, xm14
4749 add wq, r7
4750 call .main
4751 lea stride3q, [strideq*3]
4752 jmp wq
4753 .w4:
4754 phaddd m4, m5
4755 paddw m4, m14
4756 psrlw m4, 2
4757 packuswb m4, m4
4758 vextracti128 xm5, m4, 1
4759 punpcklwd xm4, xm5
4760 movq [dstq+strideq*0], xm0
4761 movhps [dstq+strideq*1], xm0
4762 vextracti128 xm0, m0, 1
4763 movq [dstq+strideq*2], xm0
4764 movhps [dstq+stride3q ], xm0
4765 mova [maskq], xm4
4766 cmp hd, 8
4767 jl .w4_end
4768 lea dstq, [dstq+strideq*4]
4769 movq [dstq+strideq*0], xm1
4770 movhps [dstq+strideq*1], xm1
4771 vextracti128 xm1, m1, 1
4772 movq [dstq+strideq*2], xm1
4773 movhps [dstq+stride3q ], xm1
4774 je .w4_end
4775 lea dstq, [dstq+strideq*4]
4776 movq [dstq+strideq*0], xm2
4777 movhps [dstq+strideq*1], xm2
4778 vextracti128 xm2, m2, 1
4779 movq [dstq+strideq*2], xm2
4780 movhps [dstq+stride3q ], xm2
4781 lea dstq, [dstq+strideq*4]
4782 movq [dstq+strideq*0], xm3
4783 movhps [dstq+strideq*1], xm3
4784 vextracti128 xm3, m3, 1
4785 movq [dstq+strideq*2], xm3
4786 movhps [dstq+stride3q ], xm3
4787 .w4_end:
4788 RET
4789 .w8_loop:
4790 call .main
4791 lea dstq, [dstq+strideq*4]
4792 add maskq, 16
4793 .w8:
4794 vperm2i128 m6, m4, m5, 0x21
4795 vpblendd m4, m5, 0xf0
4796 paddw m4, m14
4797 paddw m4, m6
4798 psrlw m4, 2
4799 vextracti128 xm5, m4, 1
4800 packuswb xm4, xm5
4801 mova [dstq+strideq*0], xm0
4802 vextracti128 [dstq+strideq*1], m0, 1
4803 mova [dstq+strideq*2], xm1
4804 vextracti128 [dstq+stride3q ], m1, 1
4805 mova [maskq], xm4
4806 sub hd, 8
4807 jl .w8_end
4808 lea dstq, [dstq+strideq*4]
4809 mova [dstq+strideq*0], xm2
4810 vextracti128 [dstq+strideq*1], m2, 1
4811 mova [dstq+strideq*2], xm3
4812 vextracti128 [dstq+stride3q ], m3, 1
4813 jg .w8_loop
4814 .w8_end:
4815 RET
4816 .w16_loop:
4817 call .main
4818 lea dstq, [dstq+strideq*4]
4819 add maskq, 16
4820 .w16:
4821 punpcklqdq m6, m4, m5
4822 punpckhqdq m4, m5
4823 paddw m6, m14
4824 paddw m4, m6
4825 psrlw m4, 2
4826 vextracti128 xm5, m4, 1
4827 packuswb xm4, xm5
4828 pshufd xm4, xm4, q3120
4829 mova [dstq+strideq*0], m0
4830 mova [dstq+strideq*1], m1
4831 mova [dstq+strideq*2], m2
4832 mova [dstq+stride3q ], m3
4833 mova [maskq], xm4
4834 sub hd, 4
4835 jg .w16_loop
4836 RET
4837 .w32_loop:
4838 call .main
4839 lea dstq, [dstq+strideq*4]
4840 add maskq, 32
4841 .w32:
4842 paddw m4, m14
4843 paddw m4, m5
4844 psrlw m15, m4, 2
4845 mova [dstq+strideq*0+32*0], m0
4846 mova [dstq+strideq*0+32*1], m1
4847 mova [dstq+strideq*1+32*0], m2
4848 mova [dstq+strideq*1+32*1], m3
4849 call .main
4850 mova m6, [deint_shuf]
4851 paddw m4, m14
4852 paddw m4, m5
4853 psrlw m4, 2
4854 packuswb m15, m4
4855 vpermd m4, m6, m15
4856 mova [dstq+strideq*2+32*0], m0
4857 mova [dstq+strideq*2+32*1], m1
4858 mova [dstq+stride3q +32*0], m2
4859 mova [dstq+stride3q +32*1], m3
4860 mova [maskq], m4
4861 sub hd, 4
4862 jg .w32_loop
4863 RET
4864 .w64_loop:
4865 call .main
4866 lea dstq, [dstq+strideq*2]
4867 add maskq, 32
4868 .w64:
4869 paddw m4, m14
4870 paddw m15, m14, m5
4871 mova [dstq+strideq*0+32*0], m0
4872 mova [dstq+strideq*0+32*1], m1
4873 mova [dstq+strideq*0+32*2], m2
4874 mova [dstq+strideq*0+32*3], m3
4875 mova [maskq], m4 ; no available registers
4876 call .main
4877 paddw m4, [maskq]
4878 mova m6, [deint_shuf]
4879 paddw m5, m15
4880 psrlw m4, 2
4881 psrlw m5, 2
4882 packuswb m4, m5 ; 0 2 4 6 1 3 5 7
4883 vpermd m4, m6, m4
4884 mova [dstq+strideq*1+32*0], m0
4885 mova [dstq+strideq*1+32*1], m1
4886 mova [dstq+strideq*1+32*2], m2
4887 mova [dstq+strideq*1+32*3], m3
4888 mova [maskq], m4
4889 sub hd, 2
4890 jg .w64_loop
4891 RET
4892 .w128_loop:
4893 call .main
4894 lea dstq, [dstq+strideq*2]
4895 add maskq, 64
4896 .w128:
4897 paddw m4, m14
4898 paddw m5, m14
4899 mova [dstq+strideq*0+32*0], m0
4900 mova [dstq+strideq*0+32*1], m1
4901 mova [dstq+strideq*0+32*2], m2
4902 mova [dstq+strideq*0+32*3], m3
4903 mova [maskq+32*0], m4
4904 mova [dstq+strideq], m5
4905 call .main
4906 paddw m4, m14
4907 paddw m15, m14, m5
4908 mova [dstq+strideq*0+32*4], m0
4909 mova [dstq+strideq*0+32*5], m1
4910 mova [dstq+strideq*0+32*6], m2
4911 mova [dstq+strideq*0+32*7], m3
4912 mova [maskq+32*1], m4
4913 call .main
4914 paddw m4, [maskq+32*0]
4915 paddw m5, [dstq+strideq]
4916 mova m6, [deint_shuf]
4917 psrlw m4, 2
4918 psrlw m5, 2
4919 packuswb m4, m5
4920 vpermd m4, m6, m4
4921 mova [dstq+strideq*1+32*0], m0
4922 mova [dstq+strideq*1+32*1], m1
4923 mova [dstq+strideq*1+32*2], m2
4924 mova [dstq+strideq*1+32*3], m3
4925 mova [maskq+32*0], m4
4926 call .main
4927 paddw m4, [maskq+32*1]
4928 mova m6, [deint_shuf]
4929 paddw m5, m15
4930 psrlw m4, 2
4931 psrlw m5, 2
4932 packuswb m4, m5
4933 vpermd m4, m6, m4
4934 mova [dstq+strideq*1+32*4], m0
4935 mova [dstq+strideq*1+32*5], m1
4936 mova [dstq+strideq*1+32*6], m2
4937 mova [dstq+strideq*1+32*7], m3
4938 mova [maskq+32*1], m4
4939 sub hd, 2
4940 jg .w128_loop
4941 RET
4942 ALIGN function_align
4943 .main:
4944 %macro W_MASK 2-6 11, 12, 13 ; dst/src1, mask/src2, pw_64, rnd, mul
4945 mova m%1, [tmp1q+32*%1]
4946 mova m%2, [tmp2q+32*%1]
4947 punpcklwd m8, m%2, m%1
4948 punpckhwd m9, m%2, m%1
4949 psubsw m%1, m%2
4950 pabsw m%1, m%1
4951 psubusw m7, m10, m%1
4952 psrlw m7, 10 ; 64-m
4953 psubw m%2, m%3, m7 ; m
4954 punpcklwd m%1, m7, m%2
4955 punpckhwd m7, m%2
4956 pmaddwd m%1, m8
4957 pmaddwd m7, m9
4958 psrad m%1, 5
4959 psrad m7, 5
4960 packssdw m%1, m7
4961 pmaxsw m%1, m%4
4962 psubsw m%1, m%4
4963 pmulhw m%1, m%5
4964 %endmacro
4965 W_MASK 0, 4
4966 W_MASK 1, 5
4967 phaddw m4, m5
4968 W_MASK 2, 5
4969 W_MASK 3, 6
4970 phaddw m5, m6
4971 add tmp1q, 32*4
4972 add tmp2q, 32*4
4973 ret
4974
4975 cglobal w_mask_422_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
4976 %define base r7-w_mask_422_avx2_table
4977 lea r7, [w_mask_422_avx2_table]
4978 tzcnt wd, wm
4979 mov r6d, r8m ; pixel_max
4980 vpbroadcastb m14, r7m ; sign
4981 movifnidn hd, hm
4982 shr r6d, 11
4983 movsxd wq, [r7+wq*4]
4984 vpbroadcastd m10, [base+pw_27615]
4985 vpbroadcastd m11, [base+pw_64]
4986 vpbroadcastd m12, [base+bidir_rnd+r6*4]
4987 vpbroadcastd m13, [base+bidir_mul+r6*4]
4988 mova m15, [base+deint_shuf]
4989 mov maskq, maskmp
4990 add wq, r7
4991 call .main
4992 lea stride3q, [strideq*3]
4993 jmp wq
4994 .w4:
4995 movq [dstq+strideq*0], xm0
4996 movhps [dstq+strideq*1], xm0
4997 vextracti128 xm0, m0, 1
4998 movq [dstq+strideq*2], xm0
4999 movhps [dstq+stride3q ], xm0
5000 cmp hd, 8
5001 jl .w4_end
5002 lea dstq, [dstq+strideq*4]
5003 movq [dstq+strideq*0], xm1
5004 movhps [dstq+strideq*1], xm1
5005 vextracti128 xm1, m1, 1
5006 movq [dstq+strideq*2], xm1
5007 movhps [dstq+stride3q ], xm1
5008 je .w4_end
5009 lea dstq, [dstq+strideq*4]
5010 movq [dstq+strideq*0], xm2
5011 movhps [dstq+strideq*1], xm2
5012 vextracti128 xm2, m2, 1
5013 movq [dstq+strideq*2], xm2
5014 movhps [dstq+stride3q ], xm2
5015 lea dstq, [dstq+strideq*4]
5016 movq [dstq+strideq*0], xm3
5017 movhps [dstq+strideq*1], xm3
5018 vextracti128 xm3, m3, 1
5019 movq [dstq+strideq*2], xm3
5020 movhps [dstq+stride3q ], xm3
5021 .w4_end:
5022 RET
5023 .w8_loop:
5024 call .main
5025 lea dstq, [dstq+strideq*4]
5026 .w8:
5027 mova [dstq+strideq*0], xm0
5028 vextracti128 [dstq+strideq*1], m0, 1
5029 mova [dstq+strideq*2], xm1
5030 vextracti128 [dstq+stride3q ], m1, 1
5031 sub hd, 8
5032 jl .w8_end
5033 lea dstq, [dstq+strideq*4]
5034 mova [dstq+strideq*0], xm2
5035 vextracti128 [dstq+strideq*1], m2, 1
5036 mova [dstq+strideq*2], xm3
5037 vextracti128 [dstq+stride3q ], m3, 1
5038 jg .w8_loop
5039 .w8_end:
5040 RET
5041 .w16_loop:
5042 call .main
5043 lea dstq, [dstq+strideq*4]
5044 .w16:
5045 mova [dstq+strideq*0], m0
5046 mova [dstq+strideq*1], m1
5047 mova [dstq+strideq*2], m2
5048 mova [dstq+stride3q ], m3
5049 sub hd, 4
5050 jg .w16_loop
5051 RET
5052 .w32_loop:
5053 call .main
5054 lea dstq, [dstq+strideq*2]
5055 .w32:
5056 mova [dstq+strideq*0+32*0], m0
5057 mova [dstq+strideq*0+32*1], m1
5058 mova [dstq+strideq*1+32*0], m2
5059 mova [dstq+strideq*1+32*1], m3
5060 sub hd, 2
5061 jg .w32_loop
5062 RET
5063 .w64_loop:
5064 call .main
5065 add dstq, strideq
5066 .w64:
5067 mova [dstq+32*0], m0
5068 mova [dstq+32*1], m1
5069 mova [dstq+32*2], m2
5070 mova [dstq+32*3], m3
5071 dec hd
5072 jg .w64_loop
5073 RET
5074 .w128_loop:
5075 call .main
5076 add dstq, strideq
5077 .w128:
5078 mova [dstq+32*0], m0
5079 mova [dstq+32*1], m1
5080 mova [dstq+32*2], m2
5081 mova [dstq+32*3], m3
5082 call .main
5083 mova [dstq+32*4], m0
5084 mova [dstq+32*5], m1
5085 mova [dstq+32*6], m2
5086 mova [dstq+32*7], m3
5087 dec hd
5088 jg .w128_loop
5089 RET
5090 ALIGN function_align
5091 .main:
5092 W_MASK 0, 4
5093 W_MASK 1, 5
5094 phaddw m4, m5
5095 W_MASK 2, 5
5096 W_MASK 3, 6
5097 phaddw m5, m6
5098 add tmp1q, 32*4
5099 add tmp2q, 32*4
5100 packuswb m4, m5
5101 pxor m5, m5
5102 psubb m4, m14
5103 pavgb m4, m5
5104 vpermd m4, m15, m4
5105 mova [maskq], m4
5106 add maskq, 32
5107 ret
5108
5109 cglobal w_mask_444_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3
5110 %define base r7-w_mask_444_avx2_table
5111 lea r7, [w_mask_444_avx2_table]
5112 tzcnt wd, wm
5113 mov r6d, r8m ; pixel_max
5114 movifnidn hd, hm
5115 shr r6d, 11
5116 movsxd wq, [r7+wq*4]
5117 vpbroadcastd m10, [base+pw_27615]
5118 vpbroadcastd m4, [base+pw_64]
5119 vpbroadcastd m5, [base+bidir_rnd+r6*4]
5120 vpbroadcastd m6, [base+bidir_mul+r6*4]
5121 mov maskq, maskmp
5122 add wq, r7
5123 call .main
5124 lea stride3q, [strideq*3]
5125 jmp wq
5126 .w4:
5127 movq [dstq+strideq*0], xm0
5128 movhps [dstq+strideq*1], xm0
5129 vextracti128 xm0, m0, 1
5130 movq [dstq+strideq*2], xm0
5131 movhps [dstq+stride3q ], xm0
5132 cmp hd, 8
5133 jl .w4_end
5134 lea dstq, [dstq+strideq*4]
5135 movq [dstq+strideq*0], xm1
5136 movhps [dstq+strideq*1], xm1
5137 vextracti128 xm1, m1, 1
5138 movq [dstq+strideq*2], xm1
5139 movhps [dstq+stride3q ], xm1
5140 je .w4_end
5141 call .main
5142 lea dstq, [dstq+strideq*4]
5143 movq [dstq+strideq*0], xm0
5144 movhps [dstq+strideq*1], xm0
5145 vextracti128 xm0, m0, 1
5146 movq [dstq+strideq*2], xm0
5147 movhps [dstq+stride3q ], xm0
5148 lea dstq, [dstq+strideq*4]
5149 movq [dstq+strideq*0], xm1
5150 movhps [dstq+strideq*1], xm1
5151 vextracti128 xm1, m1, 1
5152 movq [dstq+strideq*2], xm1
5153 movhps [dstq+stride3q ], xm1
5154 .w4_end:
5155 RET
5156 .w8_loop:
5157 call .main
5158 lea dstq, [dstq+strideq*4]
5159 .w8:
5160 mova [dstq+strideq*0], xm0
5161 vextracti128 [dstq+strideq*1], m0, 1
5162 mova [dstq+strideq*2], xm1
5163 vextracti128 [dstq+stride3q ], m1, 1
5164 sub hd, 4
5165 jg .w8_loop
5166 .w8_end:
5167 RET
5168 .w16_loop:
5169 call .main
5170 lea dstq, [dstq+strideq*2]
5171 .w16:
5172 mova [dstq+strideq*0], m0
5173 mova [dstq+strideq*1], m1
5174 sub hd, 2
5175 jg .w16_loop
5176 RET
5177 .w32_loop:
5178 call .main
5179 add dstq, strideq
5180 .w32:
5181 mova [dstq+32*0], m0
5182 mova [dstq+32*1], m1
5183 dec hd
5184 jg .w32_loop
5185 RET
5186 .w64_loop:
5187 call .main
5188 add dstq, strideq
5189 .w64:
5190 mova [dstq+32*0], m0
5191 mova [dstq+32*1], m1
5192 call .main
5193 mova [dstq+32*2], m0
5194 mova [dstq+32*3], m1
5195 dec hd
5196 jg .w64_loop
5197 RET
5198 .w128_loop:
5199 call .main
5200 add dstq, strideq
5201 .w128:
5202 mova [dstq+32*0], m0
5203 mova [dstq+32*1], m1
5204 call .main
5205 mova [dstq+32*2], m0
5206 mova [dstq+32*3], m1
5207 call .main
5208 mova [dstq+32*4], m0
5209 mova [dstq+32*5], m1
5210 call .main
5211 mova [dstq+32*6], m0
5212 mova [dstq+32*7], m1
5213 dec hd
5214 jg .w128_loop
5215 RET
5216 ALIGN function_align
5217 .main:
5218 W_MASK 0, 2, 4, 5, 6
5219 W_MASK 1, 3, 4, 5, 6
5220 packuswb m2, m3
5221 vpermq m2, m2, q3120
5222 add tmp1q, 32*2
5223 add tmp2q, 32*2
5224 mova [maskq], m2
5225 add maskq, 32
5226 ret
5227
5228 ; (a * (64 - m) + b * m + 32) >> 6
5229 ; = (((b - a) * m + 32) >> 6) + a
5230 ; = (((b - a) * (m << 9) + 16384) >> 15) + a
5231 ; except m << 9 overflows int16_t when m == 64 (which is possible),
5232 ; but if we negate m it works out (-64 << 9 == -32768).
5233 ; = (((a - b) * (m * -512) + 16384) >> 15) + a
5234 cglobal blend_16bpc, 3, 7, 7, dst, ds, tmp, w, h, mask
5235 %define base r6-blend_avx2_table
5236 lea r6, [blend_avx2_table]
5237 tzcnt wd, wm
5238 movifnidn hd, hm
5239 movsxd wq, [r6+wq*4]
5240 movifnidn maskq, maskmp
5241 vpbroadcastd m6, [base+pw_m512]
5242 add wq, r6
5243 lea r6, [dsq*3]
5244 jmp wq
5245 .w4:
5246 pmovzxbw m3, [maskq]
5247 movq xm0, [dstq+dsq*0]
5248 movhps xm0, [dstq+dsq*1]
5249 vpbroadcastq m1, [dstq+dsq*2]
5250 vpbroadcastq m2, [dstq+r6 ]
5251 vpblendd m0, m1, 0x30
5252 vpblendd m0, m2, 0xc0
5253 psubw m1, m0, [tmpq]
5254 add maskq, 16
5255 add tmpq, 32
5256 pmullw m3, m6
5257 pmulhrsw m1, m3
5258 paddw m0, m1
5259 vextracti128 xm1, m0, 1
5260 movq [dstq+dsq*0], xm0
5261 movhps [dstq+dsq*1], xm0
5262 movq [dstq+dsq*2], xm1
5263 movhps [dstq+r6 ], xm1
5264 lea dstq, [dstq+dsq*4]
5265 sub hd, 4
5266 jg .w4
5267 RET
5268 .w8:
5269 pmovzxbw m4, [maskq+16*0]
5270 pmovzxbw m5, [maskq+16*1]
5271 mova xm0, [dstq+dsq*0]
5272 vinserti128 m0, [dstq+dsq*1], 1
5273 mova xm1, [dstq+dsq*2]
5274 vinserti128 m1, [dstq+r6 ], 1
5275 psubw m2, m0, [tmpq+32*0]
5276 psubw m3, m1, [tmpq+32*1]
5277 add maskq, 16*2
5278 add tmpq, 32*2
5279 pmullw m4, m6
5280 pmullw m5, m6
5281 pmulhrsw m2, m4
5282 pmulhrsw m3, m5
5283 paddw m0, m2
5284 paddw m1, m3
5285 mova [dstq+dsq*0], xm0
5286 vextracti128 [dstq+dsq*1], m0, 1
5287 mova [dstq+dsq*2], xm1
5288 vextracti128 [dstq+r6 ], m1, 1
5289 lea dstq, [dstq+dsq*4]
5290 sub hd, 4
5291 jg .w8
5292 RET
5293 .w16:
5294 pmovzxbw m4, [maskq+16*0]
5295 pmovzxbw m5, [maskq+16*1]
5296 mova m0, [dstq+dsq*0]
5297 psubw m2, m0, [tmpq+ 32*0]
5298 mova m1, [dstq+dsq*1]
5299 psubw m3, m1, [tmpq+ 32*1]
5300 add maskq, 16*2
5301 add tmpq, 32*2
5302 pmullw m4, m6
5303 pmullw m5, m6
5304 pmulhrsw m2, m4
5305 pmulhrsw m3, m5
5306 paddw m0, m2
5307 paddw m1, m3
5308 mova [dstq+dsq*0], m0
5309 mova [dstq+dsq*1], m1
5310 lea dstq, [dstq+dsq*2]
5311 sub hd, 2
5312 jg .w16
5313 RET
5314 .w32:
5315 pmovzxbw m4, [maskq+16*0]
5316 pmovzxbw m5, [maskq+16*1]
5317 mova m0, [dstq+32*0]
5318 psubw m2, m0, [tmpq+32*0]
5319 mova m1, [dstq+32*1]
5320 psubw m3, m1, [tmpq+32*1]
5321 add maskq, 16*2
5322 add tmpq, 32*2
5323 pmullw m4, m6
5324 pmullw m5, m6
5325 pmulhrsw m2, m4
5326 pmulhrsw m3, m5
5327 paddw m0, m2
5328 paddw m1, m3
5329 mova [dstq+32*0], m0
5330 mova [dstq+32*1], m1
5331 add dstq, dsq
5332 dec hd
5333 jg .w32
5334 RET
5335
5336 INIT_XMM avx2
5337 cglobal blend_v_16bpc, 3, 6, 6, dst, ds, tmp, w, h
5338 %define base r5-blend_v_avx2_table
5339 lea r5, [blend_v_avx2_table]
5340 tzcnt wd, wm
5341 movifnidn hd, hm
5342 movsxd wq, [r5+wq*4]
5343 add wq, r5
5344 jmp wq
5345 .w2:
5346 vpbroadcastd m2, [base+obmc_masks_avx2+2*2]
5347 .w2_loop:
5348 movd m0, [dstq+dsq*0]
5349 pinsrd m0, [dstq+dsq*1], 1
5350 movq m1, [tmpq]
5351 add tmpq, 4*2
5352 psubw m1, m0, m1
5353 pmulhrsw m1, m2
5354 paddw m0, m1
5355 movd [dstq+dsq*0], m0
5356 pextrd [dstq+dsq*1], m0, 1
5357 lea dstq, [dstq+dsq*2]
5358 sub hd, 2
5359 jg .w2_loop
5360 RET
5361 .w4:
5362 vpbroadcastq m2, [base+obmc_masks_avx2+4*2]
5363 .w4_loop:
5364 movq m0, [dstq+dsq*0]
5365 movhps m0, [dstq+dsq*1]
5366 psubw m1, m0, [tmpq]
5367 add tmpq, 8*2
5368 pmulhrsw m1, m2
5369 paddw m0, m1
5370 movq [dstq+dsq*0], m0
5371 movhps [dstq+dsq*1], m0
5372 lea dstq, [dstq+dsq*2]
5373 sub hd, 2
5374 jg .w4_loop
5375 RET
5376 INIT_YMM avx2
5377 .w8:
5378 vbroadcasti128 m2, [base+obmc_masks_avx2+8*2]
5379 .w8_loop:
5380 mova xm0, [dstq+dsq*0]
5381 vinserti128 m0, [dstq+dsq*1], 1
5382 psubw m1, m0, [tmpq]
5383 add tmpq, 16*2
5384 pmulhrsw m1, m2
5385 paddw m0, m1
5386 mova [dstq+dsq*0], xm0
5387 vextracti128 [dstq+dsq*1], m0, 1
5388 lea dstq, [dstq+dsq*2]
5389 sub hd, 2
5390 jg .w8_loop
5391 RET
5392 .w16:
5393 mova m4, [base+obmc_masks_avx2+16*2]
5394 .w16_loop:
5395 mova m0, [dstq+dsq*0]
5396 psubw m2, m0, [tmpq+ 32*0]
5397 mova m1, [dstq+dsq*1]
5398 psubw m3, m1, [tmpq+ 32*1]
5399 add tmpq, 32*2
5400 pmulhrsw m2, m4
5401 pmulhrsw m3, m4
5402 paddw m0, m2
5403 paddw m1, m3
5404 mova [dstq+dsq*0], m0
5405 mova [dstq+dsq*1], m1
5406 lea dstq, [dstq+dsq*2]
5407 sub hd, 2
5408 jg .w16_loop
5409 RET
5410 .w32:
5411 %if WIN64
5412 movaps [rsp+ 8], xmm6
5413 movaps [rsp+24], xmm7
5414 %endif
5415 mova m6, [base+obmc_masks_avx2+32*2]
5416 vbroadcasti128 m7, [base+obmc_masks_avx2+32*3]
5417 .w32_loop:
5418 mova m0, [dstq+dsq*0+32*0]
5419 psubw m3, m0, [tmpq +32*0]
5420 mova xm2, [dstq+dsq*0+32*1]
5421 mova xm5, [tmpq +32*1]
5422 mova m1, [dstq+dsq*1+32*0]
5423 psubw m4, m1, [tmpq +32*2]
5424 vinserti128 m2, [dstq+dsq*1+32*1], 1
5425 vinserti128 m5, [tmpq +32*3], 1
5426 add tmpq, 32*4
5427 psubw m5, m2, m5
5428 pmulhrsw m3, m6
5429 pmulhrsw m4, m6
5430 pmulhrsw m5, m7
5431 paddw m0, m3
5432 paddw m1, m4
5433 paddw m2, m5
5434 mova [dstq+dsq*0+32*0], m0
5435 mova [dstq+dsq*1+32*0], m1
5436 mova [dstq+dsq*0+32*1], xm2
5437 vextracti128 [dstq+dsq*1+32*1], m2, 1
5438 lea dstq, [dstq+dsq*2]
5439 sub hd, 2
5440 jg .w32_loop
5441 %if WIN64
5442 movaps xmm6, [rsp+ 8]
5443 movaps xmm7, [rsp+24]
5444 %endif
5445 RET
5446
5447 %macro BLEND_H_ROW 2-3 0; dst_off, tmp_off, inc_tmp
5448 mova m0, [dstq+32*(%1+0)]
5449 psubw m2, m0, [tmpq+32*(%2+0)]
5450 mova m1, [dstq+32*(%1+1)]
5451 psubw m3, m1, [tmpq+32*(%2+1)]
5452 %if %3
5453 add tmpq, 32*%3
5454 %endif
5455 pmulhrsw m2, m4
5456 pmulhrsw m3, m4
5457 paddw m0, m2
5458 paddw m1, m3
5459 mova [dstq+32*(%1+0)], m0
5460 mova [dstq+32*(%1+1)], m1
5461 %endmacro
5462
5463 INIT_XMM avx2
5464 cglobal blend_h_16bpc, 3, 6, 6, dst, ds, tmp, w, h, mask
5465 %define base r5-blend_h_avx2_table
5466 lea r5, [blend_h_avx2_table]
5467 tzcnt wd, wm
5468 mov hd, hm
5469 movsxd wq, [r5+wq*4]
5470 add wq, r5
5471 lea maskq, [base+obmc_masks_avx2+hq*2]
5472 lea hd, [hq*3]
5473 shr hd, 2 ; h * 3/4
5474 lea maskq, [maskq+hq*2]
5475 neg hq
5476 jmp wq
5477 .w2:
5478 movd m0, [dstq+dsq*0]
5479 pinsrd m0, [dstq+dsq*1], 1
5480 movd m2, [maskq+hq*2]
5481 movq m1, [tmpq]
5482 add tmpq, 4*2
5483 punpcklwd m2, m2
5484 psubw m1, m0, m1
5485 pmulhrsw m1, m2
5486 paddw m0, m1
5487 movd [dstq+dsq*0], m0
5488 pextrd [dstq+dsq*1], m0, 1
5489 lea dstq, [dstq+dsq*2]
5490 add hq, 2
5491 jl .w2
5492 RET
5493 .w4:
5494 mova m3, [blend_shuf]
5495 .w4_loop:
5496 movq m0, [dstq+dsq*0]
5497 movhps m0, [dstq+dsq*1]
5498 movd m2, [maskq+hq*2]
5499 psubw m1, m0, [tmpq]
5500 add tmpq, 8*2
5501 pshufb m2, m3
5502 pmulhrsw m1, m2
5503 paddw m0, m1
5504 movq [dstq+dsq*0], m0
5505 movhps [dstq+dsq*1], m0
5506 lea dstq, [dstq+dsq*2]
5507 add hq, 2
5508 jl .w4_loop
5509 RET
5510 INIT_YMM avx2
5511 .w8:
5512 vbroadcasti128 m3, [blend_shuf]
5513 shufpd m3, m3, 0x0c
5514 .w8_loop:
5515 mova xm0, [dstq+dsq*0]
5516 vinserti128 m0, [dstq+dsq*1], 1
5517 vpbroadcastd m2, [maskq+hq*2]
5518 psubw m1, m0, [tmpq]
5519 add tmpq, 16*2
5520 pshufb m2, m3
5521 pmulhrsw m1, m2
5522 paddw m0, m1
5523 mova [dstq+dsq*0], xm0
5524 vextracti128 [dstq+dsq*1], m0, 1
5525 lea dstq, [dstq+dsq*2]
5526 add hq, 2
5527 jl .w8_loop
5528 RET
5529 .w16:
5530 vpbroadcastw m4, [maskq+hq*2]
5531 vpbroadcastw m5, [maskq+hq*2+2]
5532 mova m0, [dstq+dsq*0]
5533 psubw m2, m0, [tmpq+ 32*0]
5534 mova m1, [dstq+dsq*1]
5535 psubw m3, m1, [tmpq+ 32*1]
5536 add tmpq, 32*2
5537 pmulhrsw m2, m4
5538 pmulhrsw m3, m5
5539 paddw m0, m2
5540 paddw m1, m3
5541 mova [dstq+dsq*0], m0
5542 mova [dstq+dsq*1], m1
5543 lea dstq, [dstq+dsq*2]
5544 add hq, 2
5545 jl .w16
5546 RET
5547 .w32:
5548 vpbroadcastw m4, [maskq+hq*2]
5549 BLEND_H_ROW 0, 0, 2
5550 add dstq, dsq
5551 inc hq
5552 jl .w32
5553 RET
5554 .w64:
5555 vpbroadcastw m4, [maskq+hq*2]
5556 BLEND_H_ROW 0, 0
5557 BLEND_H_ROW 2, 2, 4
5558 add dstq, dsq
5559 inc hq
5560 jl .w64
5561 RET
5562 .w128:
5563 vpbroadcastw m4, [maskq+hq*2]
5564 BLEND_H_ROW 0, 0
5565 BLEND_H_ROW 2, 2, 8
5566 BLEND_H_ROW 4, -4
5567 BLEND_H_ROW 6, -2
5568 add dstq, dsq
5569 inc hq
5570 jl .w128
5571 RET
5572
5573 cglobal emu_edge_16bpc, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \
5574 bottomext, rightext
5575 ; we assume that the buffer (stride) is larger than width, so we can
5576 ; safely overwrite by a few bytes
5577
5578 ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride)
5579 xor r12d, r12d
5580 lea r10, [ihq-1]
5581 cmp yq, ihq
5582 cmovs r10, yq
5583 test yq, yq
5584 cmovs r10, r12
5585 imul r10, sstrideq
5586 add srcq, r10
5587
5588 ; ref += iclip(x, 0, iw - 1)
5589 lea r10, [iwq-1]
5590 cmp xq, iwq
5591 cmovs r10, xq
5592 test xq, xq
5593 cmovs r10, r12
5594 lea srcq, [srcq+r10*2]
5595
5596 ; bottom_ext = iclip(y + bh - ih, 0, bh - 1)
5597 lea bottomextq, [yq+bhq]
5598 sub bottomextq, ihq
5599 lea r3, [bhq-1]
5600 cmovs bottomextq, r12
5601
5602 DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \
5603 bottomext, rightext
5604
5605 ; top_ext = iclip(-y, 0, bh - 1)
5606 neg topextq
5607 cmovs topextq, r12
5608 cmp bottomextq, bhq
5609 cmovns bottomextq, r3
5610 cmp topextq, bhq
5611 cmovg topextq, r3
5612
5613 ; right_ext = iclip(x + bw - iw, 0, bw - 1)
5614 lea rightextq, [xq+bwq]
5615 sub rightextq, iwq
5616 lea r2, [bwq-1]
5617 cmovs rightextq, r12
5618
5619 DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \
5620 bottomext, rightext
5621
5622 ; left_ext = iclip(-x, 0, bw - 1)
5623 neg leftextq
5624 cmovs leftextq, r12
5625 cmp rightextq, bwq
5626 cmovns rightextq, r2
5627 cmp leftextq, bwq
5628 cmovns leftextq, r2
5629
5630 DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \
5631 dst, dstride, src, sstride, bottomext, rightext
5632
5633 ; center_h = bh - top_ext - bottom_ext
5634 lea r3, [bottomextq+topextq]
5635 sub centerhq, r3
5636
5637 ; blk += top_ext * PXSTRIDE(dst_stride)
5638 mov r2, topextq
5639 imul r2, dstrideq
5640 add dstq, r2
5641 mov r9m, dstq
5642
5643 ; center_w = bw - left_ext - right_ext
5644 mov centerwq, bwq
5645 lea r3, [rightextq+leftextq]
5646 sub centerwq, r3
5647
5648 %macro v_loop 3 ; need_left_ext, need_right_ext, suffix
5649 .v_loop_%3:
5650 %if %1
5651 ; left extension
5652 xor r3, r3
5653 vpbroadcastw m0, [srcq]
5654 .left_loop_%3:
5655 mova [dstq+r3*2], m0
5656 add r3, 16
5657 cmp r3, leftextq
5658 jl .left_loop_%3
5659
5660 ; body
5661 lea r12, [dstq+leftextq*2]
5662 %endif
5663 xor r3, r3
5664 .body_loop_%3:
5665 movu m0, [srcq+r3*2]
5666 %if %1
5667 movu [r12+r3*2], m0
5668 %else
5669 movu [dstq+r3*2], m0
5670 %endif
5671 add r3, 16
5672 cmp r3, centerwq
5673 jl .body_loop_%3
5674
5675 %if %2
5676 ; right extension
5677 %if %1
5678 lea r12, [r12+centerwq*2]
5679 %else
5680 lea r12, [dstq+centerwq*2]
5681 %endif
5682 xor r3, r3
5683 vpbroadcastw m0, [srcq+centerwq*2-2]
5684 .right_loop_%3:
5685 movu [r12+r3*2], m0
5686 add r3, 16
5687 cmp r3, rightextq
5688 jl .right_loop_%3
5689
5690 %endif
5691 add dstq, dstrideq
5692 add srcq, sstrideq
5693 dec centerhq
5694 jg .v_loop_%3
5695 %endmacro
5696
5697 test leftextq, leftextq
5698 jnz .need_left_ext
5699 test rightextq, rightextq
5700 jnz .need_right_ext
5701 v_loop 0, 0, 0
5702 jmp .body_done
5703
5704 .need_left_ext:
5705 test rightextq, rightextq
5706 jnz .need_left_right_ext
5707 v_loop 1, 0, 1
5708 jmp .body_done
5709
5710 .need_left_right_ext:
5711 v_loop 1, 1, 2
5712 jmp .body_done
5713
5714 .need_right_ext:
5715 v_loop 0, 1, 3
5716
5717 .body_done:
5718 ; bottom edge extension
5719 test bottomextq, bottomextq
5720 jz .top
5721 mov srcq, dstq
5722 sub srcq, dstrideq
5723 xor r1, r1
5724 .bottom_x_loop:
5725 mova m0, [srcq+r1*2]
5726 lea r3, [dstq+r1*2]
5727 mov r4, bottomextq
5728 .bottom_y_loop:
5729 mova [r3], m0
5730 add r3, dstrideq
5731 dec r4
5732 jg .bottom_y_loop
5733 add r1, 16
5734 cmp r1, bwq
5735 jl .bottom_x_loop
5736
5737 .top:
5738 ; top edge extension
5739 test topextq, topextq
5740 jz .end
5741 mov srcq, r9m
5742 mov dstq, dstm
5743 xor r1, r1
5744 .top_x_loop:
5745 mova m0, [srcq+r1*2]
5746 lea r3, [dstq+r1*2]
5747 mov r4, topextq
5748 .top_y_loop:
5749 mova [r3], m0
5750 add r3, dstrideq
5751 dec r4
5752 jg .top_y_loop
5753 add r1, 16
5754 cmp r1, bwq
5755 jl .top_x_loop
5756
5757 .end:
5758 RET
5759
5760 cglobal resize_16bpc, 6, 12, 16, dst, dst_stride, src, src_stride, \
5761 dst_w, h, src_w, dx, mx0, pxmax
5762 sub dword mx0m, 4<<14
5763 sub dword src_wm, 8
5764 vpbroadcastd m5, dxm
5765 vpbroadcastd m8, mx0m
5766 vpbroadcastd m6, src_wm
5767 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, _, _, pxmax
5768 LEA r7, $$
5769 %define base r7-$$
5770 vpbroadcastd m3, [base+pd_64]
5771 vpbroadcastw xm7, pxmaxm
5772 pmaddwd m2, m5, [base+rescale_mul] ; dx*[0,1,2,3,4,5,6,7]
5773 pslld m5, 3 ; dx*8
5774 pslld m6, 14
5775 paddd m8, m2 ; mx+[0..7]*dx
5776 .loop_y:
5777 xor xd, xd
5778 mova m4, m8 ; per-line working version of mx
5779 .loop_x:
5780 vpbroadcastd m10, [base+pd_63]
5781 pxor m2, m2
5782 pmaxsd m0, m4, m2
5783 psrad m9, m4, 8 ; filter offset (unmasked)
5784 pminsd m0, m6 ; iclip(mx, 0, src_w-8)
5785 psubd m1, m4, m0 ; pshufb offset
5786 psrad m0, 14 ; clipped src_x offset
5787 psrad m1, 14 ; pshufb edge_emu offset
5788 pand m9, m10 ; filter offset (masked)
5789 ; load source pixels
5790 movd r8d, xm0
5791 pextrd r9d, xm0, 1
5792 pextrd r10d, xm0, 2
5793 pextrd r11d, xm0, 3
5794 vextracti128 xm0, m0, 1
5795 movu xm10, [srcq+r8*2]
5796 movu xm11, [srcq+r9*2]
5797 movu xm12, [srcq+r10*2]
5798 movu xm13, [srcq+r11*2]
5799 movd r8d, xm0
5800 pextrd r9d, xm0, 1
5801 pextrd r10d, xm0, 2
5802 pextrd r11d, xm0, 3
5803 vinserti128 m10, [srcq+r8*2], 1
5804 vinserti128 m11, [srcq+r9*2], 1
5805 vinserti128 m12, [srcq+r10*2], 1
5806 vinserti128 m13, [srcq+r11*2], 1
5807 ptest m1, m1
5808 jz .filter
5809 movq r9, xm1
5810 pextrq r11, xm1, 1
5811 movsxd r8, r9d
5812 sar r9, 32
5813 movsxd r10, r11d
5814 sar r11, 32
5815 vextracti128 xm1, m1, 1
5816 movu xm14, [base+resize_shuf+8+r8*2]
5817 movu xm15, [base+resize_shuf+8+r9*2]
5818 movu xm0, [base+resize_shuf+8+r10*2]
5819 movu xm2, [base+resize_shuf+8+r11*2]
5820 movq r9, xm1
5821 pextrq r11, xm1, 1
5822 movsxd r8, r9d
5823 sar r9, 32
5824 movsxd r10, r11d
5825 sar r11, 32
5826 vinserti128 m14, [base+resize_shuf+8+r8*2], 1
5827 vinserti128 m15, [base+resize_shuf+8+r9*2], 1
5828 vinserti128 m0, [base+resize_shuf+8+r10*2], 1
5829 vinserti128 m2, [base+resize_shuf+8+r11*2], 1
5830 pshufb m10, m14
5831 pshufb m11, m15
5832 pshufb m12, m0
5833 pshufb m13, m2
5834 .filter:
5835 movd r8d, xm9
5836 pextrd r9d, xm9, 1
5837 pextrd r10d, xm9, 2
5838 pextrd r11d, xm9, 3
5839 vextracti128 xm9, m9, 1
5840 movq xm14, [base+resize_filter+r8*8]
5841 movq xm15, [base+resize_filter+r9*8]
5842 movq xm0, [base+resize_filter+r10*8]
5843 movq xm2, [base+resize_filter+r11*8]
5844 movd r8d, xm9
5845 pextrd r9d, xm9, 1
5846 pextrd r10d, xm9, 2
5847 pextrd r11d, xm9, 3
5848 movhps xm14, [base+resize_filter+r8*8]
5849 movhps xm15, [base+resize_filter+r9*8]
5850 movhps xm0, [base+resize_filter+r10*8]
5851 movhps xm2, [base+resize_filter+r11*8]
5852 pmovsxbw m14, xm14
5853 pmovsxbw m15, xm15
5854 pmovsxbw m0, xm0
5855 pmovsxbw m2, xm2
5856 pmaddwd m10, m14
5857 pmaddwd m11, m15
5858 pmaddwd m12, m0
5859 pmaddwd m13, m2
5860 phaddd m10, m11
5861 phaddd m12, m13
5862 phaddd m10, m12
5863 psubd m10, m3, m10
5864 psrad m10, 7
5865 vextracti128 xm0, m10, 1
5866 packusdw xm10, xm0
5867 pminsw xm10, xm7
5868 mova [dstq+xq*2], xm10
5869 paddd m4, m5
5870 add xd, 8
5871 cmp xd, dst_wd
5872 jl .loop_x
5873 add dstq, dst_strideq
5874 add srcq, src_strideq
5875 dec hd
5876 jg .loop_y
5877 RET
5878
5879 %endif ; ARCH_X86_64