Debian Codegraph @

.cargo/vendor/rav1e/src/x86/mc_avx2.asm

raw
1 ; Copyright © 2018-2021, VideoLAN and dav1d authors
2 ; Copyright © 2018-2021, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28
29 %if ARCH_X86_64
30
31 SECTION_RODATA 32
32
33 ; dav1d_obmc_masks[] with 64-x interleaved
34 obmc_masks: db 0, 0, 0, 0
35 ; 2
36 db 45, 19, 64, 0
37 ; 4
38 db 39, 25, 50, 14, 59, 5, 64, 0
39 ; 8
40 db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0
41 ; 16
42 db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10
43 db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0
44 ; 32
45 db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20
46 db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9
47 db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2
48 db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0
49
50 warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8
51 db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12
52 warp_8x8_shufB: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10
53 db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14
54 subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12
55 db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14
56 subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
57 subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
58 subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
59 subpel_v_shuf4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15
60 subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11
61 subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
62 bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12
63 bilin_h_shuf8: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
64 bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7
65 deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11
66 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
67 pb_8x0_8x8: db 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8
68 bdct_lb_dw: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12
69 wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
70 rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7
71 resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7
72
73 wm_420_sign: dd 0x01020102, 0x01010101
74 wm_422_sign: dd 0x80808080, 0x7f7f7f7f
75
76 pb_64: times 4 db 64
77 pw_m256: times 2 dw -256
78 pw_15: times 2 dw 15
79 pw_32: times 2 dw 32
80 pw_34: times 2 dw 34
81 pw_258: times 2 dw 258
82 pw_512: times 2 dw 512
83 pw_1024: times 2 dw 1024
84 pw_2048: times 2 dw 2048
85 pw_6903: times 2 dw 6903
86 pw_8192: times 2 dw 8192
87 pd_32: dd 32
88 pd_63: dd 63
89 pd_512: dd 512
90 pd_32768: dd 32768
91 pd_0x3ff: dd 0x3ff
92 pd_0x4000: dd 0x4000
93 pq_0x40000000: dq 0x40000000
94
95 cextern mc_subpel_filters
96 cextern mc_warp_filter2
97 cextern resize_filter
98
99 %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
100
101 %macro BASE_JMP_TABLE 3-*
102 %xdefine %1_%2_table (%%table - %3)
103 %xdefine %%base %1_%2
104 %%table:
105 %rep %0 - 2
106 dw %%base %+ _w%3 - %%base
107 %rotate 1
108 %endrep
109 %endmacro
110
111 %macro HV_JMP_TABLE 5-*
112 %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3)
113 %xdefine %%base %1_%3
114 %assign %%types %4
115 %if %%types & 1
116 %xdefine %1_%2_h_%3_table (%%h - %5)
117 %%h:
118 %rep %0 - 4
119 dw %%prefix %+ .h_w%5 - %%base
120 %rotate 1
121 %endrep
122 %rotate 4
123 %endif
124 %if %%types & 2
125 %xdefine %1_%2_v_%3_table (%%v - %5)
126 %%v:
127 %rep %0 - 4
128 dw %%prefix %+ .v_w%5 - %%base
129 %rotate 1
130 %endrep
131 %rotate 4
132 %endif
133 %if %%types & 4
134 %xdefine %1_%2_hv_%3_table (%%hv - %5)
135 %%hv:
136 %rep %0 - 4
137 dw %%prefix %+ .hv_w%5 - %%base
138 %rotate 1
139 %endrep
140 %endif
141 %endmacro
142
143 %macro BIDIR_JMP_TABLE 2-*
144 %xdefine %1_%2_table (%%table - 2*%3)
145 %xdefine %%base %1_%2_table
146 %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
147 %%table:
148 %rep %0 - 2
149 dd %%prefix %+ .w%3 - %%base
150 %rotate 1
151 %endrep
152 %endmacro
153
154 %macro SCALED_JMP_TABLE 2-*
155 %xdefine %1_%2_table (%%table - %3)
156 %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2)
157 %%table:
158 %rep %0 - 2
159 dw %%base %+ .w%3 - %%base
160 %rotate 1
161 %endrep
162 %rotate 2
163 %%dy_1024:
164 %xdefine %1_%2_dy1_table (%%dy_1024 - %3)
165 %rep %0 - 2
166 dw %%base %+ .dy1_w%3 - %%base
167 %rotate 1
168 %endrep
169 %rotate 2
170 %%dy_2048:
171 %xdefine %1_%2_dy2_table (%%dy_2048 - %3)
172 %rep %0 - 2
173 dw %%base %+ .dy2_w%3 - %%base
174 %rotate 1
175 %endrep
176 %endmacro
177
178 %xdefine put_avx2 mangle(private_prefix %+ _put_bilin_8bpc_avx2.put)
179 %xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_8bpc_avx2.prep)
180
181 %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
182
183 BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128
184 BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128
185 HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128
186 HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128
187 HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128
188 HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128
189 SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128
190 SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128
191 BIDIR_JMP_TABLE avg, avx2, 4, 8, 16, 32, 64, 128
192 BIDIR_JMP_TABLE w_avg, avx2, 4, 8, 16, 32, 64, 128
193 BIDIR_JMP_TABLE mask, avx2, 4, 8, 16, 32, 64, 128
194 BIDIR_JMP_TABLE w_mask_420, avx2, 4, 8, 16, 32, 64, 128
195 BIDIR_JMP_TABLE w_mask_422, avx2, 4, 8, 16, 32, 64, 128
196 BIDIR_JMP_TABLE w_mask_444, avx2, 4, 8, 16, 32, 64, 128
197 BIDIR_JMP_TABLE blend, avx2, 4, 8, 16, 32
198 BIDIR_JMP_TABLE blend_v, avx2, 2, 4, 8, 16, 32
199 BIDIR_JMP_TABLE blend_h, avx2, 2, 4, 8, 16, 32, 32, 32
200
201 SECTION .text
202
203 INIT_XMM avx2
204 cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
205 movifnidn mxyd, r6m ; mx
206 lea r7, [put_avx2]
207 tzcnt wd, wm
208 movifnidn hd, hm
209 test mxyd, mxyd
210 jnz .h
211 mov mxyd, r7m ; my
212 test mxyd, mxyd
213 jnz .v
214 .put:
215 movzx wd, word [r7+wq*2+table_offset(put,)]
216 add wq, r7
217 jmp wq
218 .put_w2:
219 movzx r6d, word [srcq+ssq*0]
220 movzx r7d, word [srcq+ssq*1]
221 lea srcq, [srcq+ssq*2]
222 mov [dstq+dsq*0], r6w
223 mov [dstq+dsq*1], r7w
224 lea dstq, [dstq+dsq*2]
225 sub hd, 2
226 jg .put_w2
227 RET
228 .put_w4:
229 mov r6d, [srcq+ssq*0]
230 mov r7d, [srcq+ssq*1]
231 lea srcq, [srcq+ssq*2]
232 mov [dstq+dsq*0], r6d
233 mov [dstq+dsq*1], r7d
234 lea dstq, [dstq+dsq*2]
235 sub hd, 2
236 jg .put_w4
237 RET
238 .put_w8:
239 mov r6, [srcq+ssq*0]
240 mov r7, [srcq+ssq*1]
241 lea srcq, [srcq+ssq*2]
242 mov [dstq+dsq*0], r6
243 mov [dstq+dsq*1], r7
244 lea dstq, [dstq+dsq*2]
245 sub hd, 2
246 jg .put_w8
247 RET
248 .put_w16:
249 movu m0, [srcq+ssq*0]
250 movu m1, [srcq+ssq*1]
251 lea srcq, [srcq+ssq*2]
252 mova [dstq+dsq*0], m0
253 mova [dstq+dsq*1], m1
254 lea dstq, [dstq+dsq*2]
255 sub hd, 2
256 jg .put_w16
257 RET
258 INIT_YMM avx2
259 .put_w32:
260 movu m0, [srcq+ssq*0]
261 movu m1, [srcq+ssq*1]
262 lea srcq, [srcq+ssq*2]
263 mova [dstq+dsq*0], m0
264 mova [dstq+dsq*1], m1
265 lea dstq, [dstq+dsq*2]
266 sub hd, 2
267 jg .put_w32
268 RET
269 .put_w64:
270 movu m0, [srcq+ssq*0+32*0]
271 movu m1, [srcq+ssq*0+32*1]
272 movu m2, [srcq+ssq*1+32*0]
273 movu m3, [srcq+ssq*1+32*1]
274 lea srcq, [srcq+ssq*2]
275 mova [dstq+dsq*0+32*0], m0
276 mova [dstq+dsq*0+32*1], m1
277 mova [dstq+dsq*1+32*0], m2
278 mova [dstq+dsq*1+32*1], m3
279 lea dstq, [dstq+dsq*2]
280 sub hd, 2
281 jg .put_w64
282 RET
283 .put_w128:
284 movu m0, [srcq+32*0]
285 movu m1, [srcq+32*1]
286 movu m2, [srcq+32*2]
287 movu m3, [srcq+32*3]
288 add srcq, ssq
289 mova [dstq+32*0], m0
290 mova [dstq+32*1], m1
291 mova [dstq+32*2], m2
292 mova [dstq+32*3], m3
293 add dstq, dsq
294 dec hd
295 jg .put_w128
296 RET
297 .h:
298 ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
299 ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
300 imul mxyd, 255
301 vbroadcasti128 m4, [bilin_h_shuf8]
302 add mxyd, 16
303 movd xm5, mxyd
304 mov mxyd, r7m ; my
305 vpbroadcastw m5, xm5
306 test mxyd, mxyd
307 jnz .hv
308 movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)]
309 vpbroadcastd m3, [pw_2048]
310 add wq, r7
311 jmp wq
312 .h_w2:
313 movd xm0, [srcq+ssq*0]
314 pinsrd xm0, [srcq+ssq*1], 1
315 lea srcq, [srcq+ssq*2]
316 pshufb xm0, xm4
317 pmaddubsw xm0, xm5
318 pmulhrsw xm0, xm3
319 packuswb xm0, xm0
320 pextrw [dstq+dsq*0], xm0, 0
321 pextrw [dstq+dsq*1], xm0, 2
322 lea dstq, [dstq+dsq*2]
323 sub hd, 2
324 jg .h_w2
325 RET
326 .h_w4:
327 mova xm4, [bilin_h_shuf4]
328 .h_w4_loop:
329 movq xm0, [srcq+ssq*0]
330 movhps xm0, [srcq+ssq*1]
331 lea srcq, [srcq+ssq*2]
332 pshufb xm0, xm4
333 pmaddubsw xm0, xm5
334 pmulhrsw xm0, xm3
335 packuswb xm0, xm0
336 movd [dstq+dsq*0], xm0
337 pextrd [dstq+dsq*1], xm0, 1
338 lea dstq, [dstq+dsq*2]
339 sub hd, 2
340 jg .h_w4_loop
341 RET
342 .h_w8:
343 movu xm0, [srcq+ssq*0]
344 movu xm1, [srcq+ssq*1]
345 lea srcq, [srcq+ssq*2]
346 pshufb xm0, xm4
347 pshufb xm1, xm4
348 pmaddubsw xm0, xm5
349 pmaddubsw xm1, xm5
350 pmulhrsw xm0, xm3
351 pmulhrsw xm1, xm3
352 packuswb xm0, xm1
353 movq [dstq+dsq*0], xm0
354 movhps [dstq+dsq*1], xm0
355 lea dstq, [dstq+dsq*2]
356 sub hd, 2
357 jg .h_w8
358 RET
359 .h_w16:
360 movu xm0, [srcq+ssq*0+8*0]
361 vinserti128 m0, [srcq+ssq*1+8*0], 1
362 movu xm1, [srcq+ssq*0+8*1]
363 vinserti128 m1, [srcq+ssq*1+8*1], 1
364 lea srcq, [srcq+ssq*2]
365 pshufb m0, m4
366 pshufb m1, m4
367 pmaddubsw m0, m5
368 pmaddubsw m1, m5
369 pmulhrsw m0, m3
370 pmulhrsw m1, m3
371 packuswb m0, m1
372 mova [dstq+dsq*0], xm0
373 vextracti128 [dstq+dsq*1], m0, 1
374 lea dstq, [dstq+dsq*2]
375 sub hd, 2
376 jg .h_w16
377 RET
378 .h_w32:
379 movu m0, [srcq+8*0]
380 movu m1, [srcq+8*1]
381 add srcq, ssq
382 pshufb m0, m4
383 pshufb m1, m4
384 pmaddubsw m0, m5
385 pmaddubsw m1, m5
386 pmulhrsw m0, m3
387 pmulhrsw m1, m3
388 packuswb m0, m1
389 mova [dstq], m0
390 add dstq, dsq
391 dec hd
392 jg .h_w32
393 RET
394 .h_w64:
395 movu m0, [srcq+8*0]
396 movu m1, [srcq+8*1]
397 pshufb m0, m4
398 pshufb m1, m4
399 pmaddubsw m0, m5
400 pmaddubsw m1, m5
401 pmulhrsw m0, m3
402 pmulhrsw m1, m3
403 packuswb m0, m1
404 movu m1, [srcq+8*4]
405 movu m2, [srcq+8*5]
406 add srcq, ssq
407 pshufb m1, m4
408 pshufb m2, m4
409 pmaddubsw m1, m5
410 pmaddubsw m2, m5
411 pmulhrsw m1, m3
412 pmulhrsw m2, m3
413 packuswb m1, m2
414 mova [dstq+32*0], m0
415 mova [dstq+32*1], m1
416 add dstq, dsq
417 dec hd
418 jg .h_w64
419 RET
420 .h_w128:
421 mov r6, -32*3
422 .h_w128_loop:
423 movu m0, [srcq+r6+32*3+8*0]
424 movu m1, [srcq+r6+32*3+8*1]
425 pshufb m0, m4
426 pshufb m1, m4
427 pmaddubsw m0, m5
428 pmaddubsw m1, m5
429 pmulhrsw m0, m3
430 pmulhrsw m1, m3
431 packuswb m0, m1
432 mova [dstq+r6+32*3], m0
433 add r6, 32
434 jle .h_w128_loop
435 add srcq, ssq
436 add dstq, dsq
437 dec hd
438 jg .h_w128
439 RET
440 .v:
441 movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)]
442 imul mxyd, 255
443 vpbroadcastd m5, [pw_2048]
444 add mxyd, 16
445 add wq, r7
446 movd xm4, mxyd
447 vpbroadcastw m4, xm4
448 jmp wq
449 .v_w2:
450 movd xm0, [srcq+ssq*0]
451 .v_w2_loop:
452 pinsrw xm1, xm0, [srcq+ssq*1], 1 ; 0 1
453 lea srcq, [srcq+ssq*2]
454 pinsrw xm0, xm1, [srcq+ssq*0], 0 ; 2 1
455 pshuflw xm1, xm1, q2301 ; 1 0
456 punpcklbw xm1, xm0
457 pmaddubsw xm1, xm4
458 pmulhrsw xm1, xm5
459 packuswb xm1, xm1
460 pextrw [dstq+dsq*0], xm1, 1
461 pextrw [dstq+dsq*1], xm1, 0
462 lea dstq, [dstq+dsq*2]
463 sub hd, 2
464 jg .v_w2_loop
465 RET
466 .v_w4:
467 movd xm0, [srcq+ssq*0]
468 .v_w4_loop:
469 vpbroadcastd xm2, [srcq+ssq*1]
470 lea srcq, [srcq+ssq*2]
471 vpblendd xm1, xm2, xm0, 0x01 ; 0 1
472 vpbroadcastd xm0, [srcq+ssq*0]
473 vpblendd xm2, xm0, 0x02 ; 1 2
474 punpcklbw xm1, xm2
475 pmaddubsw xm1, xm4
476 pmulhrsw xm1, xm5
477 packuswb xm1, xm1
478 movd [dstq+dsq*0], xm1
479 pextrd [dstq+dsq*1], xm1, 1
480 lea dstq, [dstq+dsq*2]
481 sub hd, 2
482 jg .v_w4_loop
483 RET
484 .v_w8:
485 movq xm0, [srcq+ssq*0]
486 .v_w8_loop:
487 movq xm2, [srcq+ssq*1]
488 lea srcq, [srcq+ssq*2]
489 punpcklbw xm1, xm0, xm2
490 movq xm0, [srcq+ssq*0]
491 punpcklbw xm2, xm0
492 pmaddubsw xm1, xm4
493 pmaddubsw xm2, xm4
494 pmulhrsw xm1, xm5
495 pmulhrsw xm2, xm5
496 packuswb xm1, xm2
497 movq [dstq+dsq*0], xm1
498 movhps [dstq+dsq*1], xm1
499 lea dstq, [dstq+dsq*2]
500 sub hd, 2
501 jg .v_w8_loop
502 RET
503 .v_w16:
504 movu xm0, [srcq+ssq*0]
505 .v_w16_loop:
506 vbroadcasti128 m3, [srcq+ssq*1]
507 lea srcq, [srcq+ssq*2]
508 vpblendd m2, m3, m0, 0x0f ; 0 1
509 vbroadcasti128 m0, [srcq+ssq*0]
510 vpblendd m3, m0, 0xf0 ; 1 2
511 punpcklbw m1, m2, m3
512 punpckhbw m2, m3
513 pmaddubsw m1, m4
514 pmaddubsw m2, m4
515 pmulhrsw m1, m5
516 pmulhrsw m2, m5
517 packuswb m1, m2
518 mova [dstq+dsq*0], xm1
519 vextracti128 [dstq+dsq*1], m1, 1
520 lea dstq, [dstq+dsq*2]
521 sub hd, 2
522 jg .v_w16_loop
523 RET
524 .v_w32:
525 %macro PUT_BILIN_V_W32 0
526 movu m0, [srcq+ssq*0]
527 %%loop:
528 movu m3, [srcq+ssq*1]
529 lea srcq, [srcq+ssq*2]
530 punpcklbw m1, m0, m3
531 punpckhbw m2, m0, m3
532 movu m0, [srcq+ssq*0]
533 pmaddubsw m1, m4
534 pmaddubsw m2, m4
535 pmulhrsw m1, m5
536 pmulhrsw m2, m5
537 packuswb m1, m2
538 punpcklbw m2, m3, m0
539 punpckhbw m3, m0
540 pmaddubsw m2, m4
541 pmaddubsw m3, m4
542 pmulhrsw m2, m5
543 pmulhrsw m3, m5
544 packuswb m2, m3
545 mova [dstq+dsq*0], m1
546 mova [dstq+dsq*1], m2
547 lea dstq, [dstq+dsq*2]
548 sub hd, 2
549 jg %%loop
550 %endmacro
551 PUT_BILIN_V_W32
552 RET
553 .v_w64:
554 movu m0, [srcq+32*0]
555 movu m1, [srcq+32*1]
556 .v_w64_loop:
557 add srcq, ssq
558 movu m3, [srcq+32*0]
559 punpcklbw m2, m0, m3
560 punpckhbw m0, m3
561 pmaddubsw m2, m4
562 pmaddubsw m0, m4
563 pmulhrsw m2, m5
564 pmulhrsw m0, m5
565 packuswb m2, m0
566 mova m0, m3
567 movu m3, [srcq+32*1]
568 mova [dstq+32*0], m2
569 punpcklbw m2, m1, m3
570 punpckhbw m1, m3
571 pmaddubsw m2, m4
572 pmaddubsw m1, m4
573 pmulhrsw m2, m5
574 pmulhrsw m1, m5
575 packuswb m2, m1
576 mova m1, m3
577 mova [dstq+32*1], m2
578 add dstq, dsq
579 dec hd
580 jg .v_w64_loop
581 RET
582 .v_w128:
583 lea r6d, [hq+(3<<8)]
584 mov r4, srcq
585 mov r7, dstq
586 .v_w128_loop:
587 PUT_BILIN_V_W32
588 add r4, 32
589 add r7, 32
590 movzx hd, r6b
591 mov srcq, r4
592 mov dstq, r7
593 sub r6d, 1<<8
594 jg .v_w128_loop
595 RET
596 .hv:
597 ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8
598 ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4
599 movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)]
600 WIN64_SPILL_XMM 8
601 shl mxyd, 11 ; can't shift by 12 due to signed overflow
602 vpbroadcastd m7, [pw_15]
603 movd xm6, mxyd
604 add wq, r7
605 paddb m5, m5
606 vpbroadcastw m6, xm6
607 jmp wq
608 .hv_w2:
609 vpbroadcastd xm0, [srcq+ssq*0]
610 pshufb xm0, xm4
611 pmaddubsw xm0, xm5
612 .hv_w2_loop:
613 movd xm1, [srcq+ssq*1]
614 lea srcq, [srcq+ssq*2]
615 pinsrd xm1, [srcq+ssq*0], 1
616 pshufb xm1, xm4
617 pmaddubsw xm1, xm5 ; 1 _ 2 _
618 shufps xm2, xm0, xm1, q1032 ; 0 _ 1 _
619 mova xm0, xm1
620 psubw xm1, xm2
621 pmulhw xm1, xm6
622 pavgw xm2, xm7
623 paddw xm1, xm2
624 psrlw xm1, 4
625 packuswb xm1, xm1
626 pextrw [dstq+dsq*0], xm1, 0
627 pextrw [dstq+dsq*1], xm1, 2
628 lea dstq, [dstq+dsq*2]
629 sub hd, 2
630 jg .hv_w2_loop
631 RET
632 .hv_w4:
633 mova xm4, [bilin_h_shuf4]
634 movddup xm0, [srcq+ssq*0]
635 pshufb xm0, xm4
636 pmaddubsw xm0, xm5
637 .hv_w4_loop:
638 movq xm1, [srcq+ssq*1]
639 lea srcq, [srcq+ssq*2]
640 movhps xm1, [srcq+ssq*0]
641 pshufb xm1, xm4
642 pmaddubsw xm1, xm5 ; 1 2
643 shufps xm2, xm0, xm1, q1032 ; 0 1
644 mova xm0, xm1
645 psubw xm1, xm2
646 pmulhw xm1, xm6
647 pavgw xm2, xm7
648 paddw xm1, xm2
649 psrlw xm1, 4
650 packuswb xm1, xm1
651 movd [dstq+dsq*0], xm1
652 pextrd [dstq+dsq*1], xm1, 1
653 lea dstq, [dstq+dsq*2]
654 sub hd, 2
655 jg .hv_w4_loop
656 RET
657 .hv_w8:
658 vbroadcasti128 m0, [srcq+ssq*0]
659 pshufb m0, m4
660 pmaddubsw m0, m5
661 .hv_w8_loop:
662 movu xm1, [srcq+ssq*1]
663 lea srcq, [srcq+ssq*2]
664 vinserti128 m1, [srcq+ssq*0], 1
665 pshufb m1, m4
666 pmaddubsw m1, m5 ; 1 2
667 vperm2i128 m2, m0, m1, 0x21 ; 0 1
668 mova m0, m1
669 psubw m1, m2
670 pmulhw m1, m6
671 pavgw m2, m7
672 paddw m1, m2
673 psrlw m1, 4
674 vextracti128 xm2, m1, 1
675 packuswb xm1, xm2
676 movq [dstq+dsq*0], xm1
677 movhps [dstq+dsq*1], xm1
678 lea dstq, [dstq+dsq*2]
679 sub hd, 2
680 jg .hv_w8_loop
681 RET
682 .hv_w16:
683 movu m0, [srcq+ssq*0+8*0]
684 vinserti128 m0, [srcq+ssq*0+8*1], 1
685 pshufb m0, m4
686 pmaddubsw m0, m5
687 .hv_w16_loop:
688 movu xm2, [srcq+ssq*1+8*0]
689 vinserti128 m2, [srcq+ssq*1+8*1], 1
690 lea srcq, [srcq+ssq*2]
691 movu xm3, [srcq+ssq*0+8*0]
692 vinserti128 m3, [srcq+ssq*0+8*1], 1
693 pshufb m2, m4
694 pshufb m3, m4
695 pmaddubsw m2, m5
696 psubw m1, m2, m0
697 pmulhw m1, m6
698 pavgw m0, m7
699 paddw m1, m0
700 pmaddubsw m0, m3, m5
701 psubw m3, m0, m2
702 pmulhw m3, m6
703 pavgw m2, m7
704 paddw m3, m2
705 psrlw m1, 4
706 psrlw m3, 4
707 packuswb m1, m3
708 vpermq m1, m1, q3120
709 mova [dstq+dsq*0], xm1
710 vextracti128 [dstq+dsq*1], m1, 1
711 lea dstq, [dstq+dsq*2]
712 sub hd, 2
713 jg .hv_w16_loop
714 RET
715 .hv_w128:
716 lea r6d, [hq+(3<<16)]
717 jmp .hv_w32_start
718 .hv_w64:
719 lea r6d, [hq+(1<<16)]
720 .hv_w32_start:
721 mov r4, srcq
722 mov r7, dstq
723 .hv_w32:
724 %if WIN64
725 movaps r4m, xmm8
726 %endif
727 .hv_w32_loop0:
728 movu m0, [srcq+8*0]
729 movu m1, [srcq+8*1]
730 pshufb m0, m4
731 pshufb m1, m4
732 pmaddubsw m0, m5
733 pmaddubsw m1, m5
734 .hv_w32_loop:
735 add srcq, ssq
736 movu m2, [srcq+8*0]
737 movu m3, [srcq+8*1]
738 pshufb m2, m4
739 pshufb m3, m4
740 pmaddubsw m2, m5
741 pmaddubsw m3, m5
742 psubw m8, m2, m0
743 pmulhw m8, m6
744 pavgw m0, m7
745 paddw m8, m0
746 mova m0, m2
747 psubw m2, m3, m1
748 pmulhw m2, m6
749 pavgw m1, m7
750 paddw m2, m1
751 mova m1, m3
752 psrlw m8, 4
753 psrlw m2, 4
754 packuswb m8, m2
755 mova [dstq], m8
756 add dstq, dsq
757 dec hd
758 jg .hv_w32_loop
759 add r4, 32
760 add r7, 32
761 movzx hd, r6b
762 mov srcq, r4
763 mov dstq, r7
764 sub r6d, 1<<16
765 jg .hv_w32_loop0
766 %if WIN64
767 movaps xmm8, r4m
768 %endif
769 RET
770
771 cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
772 movifnidn mxyd, r5m ; mx
773 lea r6, [prep%+SUFFIX]
774 tzcnt wd, wm
775 movifnidn hd, hm
776 test mxyd, mxyd
777 jnz .h
778 mov mxyd, r6m ; my
779 test mxyd, mxyd
780 jnz .v
781 .prep:
782 movzx wd, word [r6+wq*2+table_offset(prep,)]
783 add wq, r6
784 lea stride3q, [strideq*3]
785 jmp wq
786 .prep_w4:
787 movd xm0, [srcq+strideq*0]
788 pinsrd xm0, [srcq+strideq*1], 1
789 pinsrd xm0, [srcq+strideq*2], 2
790 pinsrd xm0, [srcq+stride3q ], 3
791 lea srcq, [srcq+strideq*4]
792 pmovzxbw m0, xm0
793 psllw m0, 4
794 mova [tmpq], m0
795 add tmpq, 32
796 sub hd, 4
797 jg .prep_w4
798 RET
799 .prep_w8:
800 movq xm0, [srcq+strideq*0]
801 movhps xm0, [srcq+strideq*1]
802 movq xm1, [srcq+strideq*2]
803 movhps xm1, [srcq+stride3q ]
804 lea srcq, [srcq+strideq*4]
805 pmovzxbw m0, xm0
806 pmovzxbw m1, xm1
807 psllw m0, 4
808 psllw m1, 4
809 mova [tmpq+32*0], m0
810 mova [tmpq+32*1], m1
811 add tmpq, 32*2
812 sub hd, 4
813 jg .prep_w8
814 RET
815 .prep_w16:
816 pmovzxbw m0, [srcq+strideq*0]
817 pmovzxbw m1, [srcq+strideq*1]
818 pmovzxbw m2, [srcq+strideq*2]
819 pmovzxbw m3, [srcq+stride3q ]
820 lea srcq, [srcq+strideq*4]
821 psllw m0, 4
822 psllw m1, 4
823 psllw m2, 4
824 psllw m3, 4
825 mova [tmpq+32*0], m0
826 mova [tmpq+32*1], m1
827 mova [tmpq+32*2], m2
828 mova [tmpq+32*3], m3
829 add tmpq, 32*4
830 sub hd, 4
831 jg .prep_w16
832 RET
833 .prep_w32:
834 pmovzxbw m0, [srcq+strideq*0+16*0]
835 pmovzxbw m1, [srcq+strideq*0+16*1]
836 pmovzxbw m2, [srcq+strideq*1+16*0]
837 pmovzxbw m3, [srcq+strideq*1+16*1]
838 lea srcq, [srcq+strideq*2]
839 psllw m0, 4
840 psllw m1, 4
841 psllw m2, 4
842 psllw m3, 4
843 mova [tmpq+32*0], m0
844 mova [tmpq+32*1], m1
845 mova [tmpq+32*2], m2
846 mova [tmpq+32*3], m3
847 add tmpq, 32*4
848 sub hd, 2
849 jg .prep_w32
850 RET
851 .prep_w64:
852 pmovzxbw m0, [srcq+16*0]
853 pmovzxbw m1, [srcq+16*1]
854 pmovzxbw m2, [srcq+16*2]
855 pmovzxbw m3, [srcq+16*3]
856 add srcq, strideq
857 psllw m0, 4
858 psllw m1, 4
859 psllw m2, 4
860 psllw m3, 4
861 mova [tmpq+32*0], m0
862 mova [tmpq+32*1], m1
863 mova [tmpq+32*2], m2
864 mova [tmpq+32*3], m3
865 add tmpq, 32*4
866 dec hd
867 jg .prep_w64
868 RET
869 .prep_w128:
870 pmovzxbw m0, [srcq+16*0]
871 pmovzxbw m1, [srcq+16*1]
872 pmovzxbw m2, [srcq+16*2]
873 pmovzxbw m3, [srcq+16*3]
874 psllw m0, 4
875 psllw m1, 4
876 psllw m2, 4
877 psllw m3, 4
878 mova [tmpq+32*0], m0
879 mova [tmpq+32*1], m1
880 mova [tmpq+32*2], m2
881 mova [tmpq+32*3], m3
882 pmovzxbw m0, [srcq+16*4]
883 pmovzxbw m1, [srcq+16*5]
884 pmovzxbw m2, [srcq+16*6]
885 pmovzxbw m3, [srcq+16*7]
886 add tmpq, 32*8
887 add srcq, strideq
888 psllw m0, 4
889 psllw m1, 4
890 psllw m2, 4
891 psllw m3, 4
892 mova [tmpq-32*4], m0
893 mova [tmpq-32*3], m1
894 mova [tmpq-32*2], m2
895 mova [tmpq-32*1], m3
896 dec hd
897 jg .prep_w128
898 RET
899 .h:
900 ; 16 * src[x] + (mx * (src[x + 1] - src[x]))
901 ; = (16 - mx) * src[x] + mx * src[x + 1]
902 imul mxyd, 255
903 vbroadcasti128 m4, [bilin_h_shuf8]
904 add mxyd, 16
905 movd xm5, mxyd
906 mov mxyd, r6m ; my
907 vpbroadcastw m5, xm5
908 test mxyd, mxyd
909 jnz .hv
910 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)]
911 add wq, r6
912 lea stride3q, [strideq*3]
913 jmp wq
914 .h_w4:
915 vbroadcasti128 m4, [bilin_h_shuf4]
916 .h_w4_loop:
917 movq xm0, [srcq+strideq*0]
918 movhps xm0, [srcq+strideq*1]
919 movq xm1, [srcq+strideq*2]
920 movhps xm1, [srcq+stride3q ]
921 lea srcq, [srcq+strideq*4]
922 vinserti128 m0, xm1, 1
923 pshufb m0, m4
924 pmaddubsw m0, m5
925 mova [tmpq], m0
926 add tmpq, 32
927 sub hd, 4
928 jg .h_w4_loop
929 RET
930 .h_w8:
931 .h_w8_loop:
932 movu xm0, [srcq+strideq*0]
933 vinserti128 m0, [srcq+strideq*1], 1
934 movu xm1, [srcq+strideq*2]
935 vinserti128 m1, [srcq+stride3q ], 1
936 lea srcq, [srcq+strideq*4]
937 pshufb m0, m4
938 pshufb m1, m4
939 pmaddubsw m0, m5
940 pmaddubsw m1, m5
941 mova [tmpq+32*0], m0
942 mova [tmpq+32*1], m1
943 add tmpq, 32*2
944 sub hd, 4
945 jg .h_w8_loop
946 RET
947 .h_w16:
948 .h_w16_loop:
949 movu xm0, [srcq+strideq*0+8*0]
950 vinserti128 m0, [srcq+strideq*0+8*1], 1
951 movu xm1, [srcq+strideq*1+8*0]
952 vinserti128 m1, [srcq+strideq*1+8*1], 1
953 movu xm2, [srcq+strideq*2+8*0]
954 vinserti128 m2, [srcq+strideq*2+8*1], 1
955 movu xm3, [srcq+stride3q +8*0]
956 vinserti128 m3, [srcq+stride3q +8*1], 1
957 lea srcq, [srcq+strideq*4]
958 pshufb m0, m4
959 pshufb m1, m4
960 pshufb m2, m4
961 pshufb m3, m4
962 pmaddubsw m0, m5
963 pmaddubsw m1, m5
964 pmaddubsw m2, m5
965 pmaddubsw m3, m5
966 mova [tmpq+32*0], m0
967 mova [tmpq+32*1], m1
968 mova [tmpq+32*2], m2
969 mova [tmpq+32*3], m3
970 add tmpq, 32*4
971 sub hd, 4
972 jg .h_w16_loop
973 RET
974 .h_w32:
975 .h_w32_loop:
976 movu xm0, [srcq+strideq*0+8*0]
977 vinserti128 m0, [srcq+strideq*0+8*1], 1
978 movu xm1, [srcq+strideq*0+8*2]
979 vinserti128 m1, [srcq+strideq*0+8*3], 1
980 movu xm2, [srcq+strideq*1+8*0]
981 vinserti128 m2, [srcq+strideq*1+8*1], 1
982 movu xm3, [srcq+strideq*1+8*2]
983 vinserti128 m3, [srcq+strideq*1+8*3], 1
984 lea srcq, [srcq+strideq*2]
985 pshufb m0, m4
986 pshufb m1, m4
987 pshufb m2, m4
988 pshufb m3, m4
989 pmaddubsw m0, m5
990 pmaddubsw m1, m5
991 pmaddubsw m2, m5
992 pmaddubsw m3, m5
993 mova [tmpq+32*0], m0
994 mova [tmpq+32*1], m1
995 mova [tmpq+32*2], m2
996 mova [tmpq+32*3], m3
997 add tmpq, 32*4
998 sub hd, 2
999 jg .h_w32_loop
1000 RET
1001 .h_w64:
1002 movu xm0, [srcq+8*0]
1003 vinserti128 m0, [srcq+8*1], 1
1004 movu xm1, [srcq+8*2]
1005 vinserti128 m1, [srcq+8*3], 1
1006 movu xm2, [srcq+8*4]
1007 vinserti128 m2, [srcq+8*5], 1
1008 movu xm3, [srcq+8*6]
1009 vinserti128 m3, [srcq+8*7], 1
1010 add srcq, strideq
1011 pshufb m0, m4
1012 pshufb m1, m4
1013 pshufb m2, m4
1014 pshufb m3, m4
1015 pmaddubsw m0, m5
1016 pmaddubsw m1, m5
1017 pmaddubsw m2, m5
1018 pmaddubsw m3, m5
1019 mova [tmpq+32*0], m0
1020 mova [tmpq+32*1], m1
1021 mova [tmpq+32*2], m2
1022 mova [tmpq+32*3], m3
1023 add tmpq, 32*4
1024 dec hd
1025 jg .h_w64
1026 RET
1027 .h_w128:
1028 movu xm0, [srcq+8*0]
1029 vinserti128 m0, [srcq+8*1], 1
1030 movu xm1, [srcq+8*2]
1031 vinserti128 m1, [srcq+8*3], 1
1032 movu xm2, [srcq+8*4]
1033 vinserti128 m2, [srcq+8*5], 1
1034 movu xm3, [srcq+8*6]
1035 vinserti128 m3, [srcq+8*7], 1
1036 pshufb m0, m4
1037 pshufb m1, m4
1038 pshufb m2, m4
1039 pshufb m3, m4
1040 pmaddubsw m0, m5
1041 pmaddubsw m1, m5
1042 pmaddubsw m2, m5
1043 pmaddubsw m3, m5
1044 mova [tmpq+32*0], m0
1045 mova [tmpq+32*1], m1
1046 mova [tmpq+32*2], m2
1047 mova [tmpq+32*3], m3
1048 movu xm0, [srcq+8* 8]
1049 vinserti128 m0, [srcq+8* 9], 1
1050 movu xm1, [srcq+8*10]
1051 vinserti128 m1, [srcq+8*11], 1
1052 movu xm2, [srcq+8*12]
1053 vinserti128 m2, [srcq+8*13], 1
1054 movu xm3, [srcq+8*14]
1055 vinserti128 m3, [srcq+8*15], 1
1056 add tmpq, 32*8
1057 add srcq, strideq
1058 pshufb m0, m4
1059 pshufb m1, m4
1060 pshufb m2, m4
1061 pshufb m3, m4
1062 pmaddubsw m0, m5
1063 pmaddubsw m1, m5
1064 pmaddubsw m2, m5
1065 pmaddubsw m3, m5
1066 mova [tmpq-32*4], m0
1067 mova [tmpq-32*3], m1
1068 mova [tmpq-32*2], m2
1069 mova [tmpq-32*1], m3
1070 dec hd
1071 jg .h_w128
1072 RET
1073 .v:
1074 WIN64_SPILL_XMM 7
1075 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)]
1076 imul mxyd, 255
1077 add mxyd, 16
1078 add wq, r6
1079 lea stride3q, [strideq*3]
1080 movd xm6, mxyd
1081 vpbroadcastw m6, xm6
1082 jmp wq
1083 .v_w4:
1084 movd xm0, [srcq+strideq*0]
1085 .v_w4_loop:
1086 vpbroadcastd m1, [srcq+strideq*2]
1087 vpbroadcastd xm2, [srcq+strideq*1]
1088 vpbroadcastd m3, [srcq+stride3q ]
1089 lea srcq, [srcq+strideq*4]
1090 vpblendd m1, m0, 0x05 ; 0 2 2 2
1091 vpbroadcastd m0, [srcq+strideq*0]
1092 vpblendd m3, m2, 0x0f ; 1 1 3 3
1093 vpblendd m2, m1, m0, 0xa0 ; 0 2 2 4
1094 vpblendd m1, m3, 0xaa ; 0 1 2 3
1095 vpblendd m2, m3, 0x55 ; 1 2 3 4
1096 punpcklbw m1, m2
1097 pmaddubsw m1, m6
1098 mova [tmpq], m1
1099 add tmpq, 32
1100 sub hd, 4
1101 jg .v_w4_loop
1102 RET
1103 .v_w8:
1104 movq xm0, [srcq+strideq*0]
1105 .v_w8_loop:
1106 vpbroadcastq m1, [srcq+strideq*2]
1107 vpbroadcastq m2, [srcq+strideq*1]
1108 vpbroadcastq m3, [srcq+stride3q ]
1109 lea srcq, [srcq+strideq*4]
1110 vpblendd m1, m0, 0x03 ; 0 2 2 2
1111 vpbroadcastq m0, [srcq+strideq*0]
1112 vpblendd m2, m3, 0xcc ; 1 3 1 3
1113 vpblendd m3, m2, m1, 0xf0 ; 1 3 2 2
1114 vpblendd m2, m1, 0x0f ; 0 2 1 3
1115 vpblendd m3, m0, 0xc0 ; 1 3 2 4
1116 punpcklbw m1, m2, m3
1117 punpckhbw m2, m3
1118 pmaddubsw m1, m6
1119 pmaddubsw m2, m6
1120 mova [tmpq+32*0], m1
1121 mova [tmpq+32*1], m2
1122 add tmpq, 32*2
1123 sub hd, 4
1124 jg .v_w8_loop
1125 RET
1126 .v_w16:
1127 vbroadcasti128 m0, [srcq+strideq*0]
1128 .v_w16_loop:
1129 vbroadcasti128 m1, [srcq+strideq*1]
1130 vbroadcasti128 m2, [srcq+strideq*2]
1131 vbroadcasti128 m3, [srcq+stride3q ]
1132 lea srcq, [srcq+strideq*4]
1133 shufpd m4, m0, m2, 0x0c ; 0 2
1134 vbroadcasti128 m0, [srcq+strideq*0]
1135 shufpd m1, m3, 0x0c ; 1 3
1136 shufpd m2, m0, 0x0c ; 2 4
1137 punpcklbw m3, m4, m1
1138 punpcklbw m5, m1, m2
1139 punpckhbw m4, m1
1140 punpckhbw m1, m2
1141 pmaddubsw m3, m6
1142 pmaddubsw m5, m6
1143 pmaddubsw m4, m6
1144 pmaddubsw m1, m6
1145 mova [tmpq+32*0], m3
1146 mova [tmpq+32*1], m5
1147 mova [tmpq+32*2], m4
1148 mova [tmpq+32*3], m1
1149 add tmpq, 32*4
1150 sub hd, 4
1151 jg .v_w16_loop
1152 RET
1153 .v_w32:
1154 vpermq m0, [srcq+strideq*0], q3120
1155 .v_w32_loop:
1156 vpermq m1, [srcq+strideq*1], q3120
1157 vpermq m2, [srcq+strideq*2], q3120
1158 vpermq m3, [srcq+stride3q ], q3120
1159 lea srcq, [srcq+strideq*4]
1160 punpcklbw m4, m0, m1
1161 punpckhbw m5, m0, m1
1162 vpermq m0, [srcq+strideq*0], q3120
1163 pmaddubsw m4, m6
1164 pmaddubsw m5, m6
1165 mova [tmpq+32*0], m4
1166 mova [tmpq+32*1], m5
1167 punpcklbw m4, m1, m2
1168 punpckhbw m1, m2
1169 pmaddubsw m4, m6
1170 pmaddubsw m1, m6
1171 punpcklbw m5, m2, m3
1172 punpckhbw m2, m3
1173 pmaddubsw m5, m6
1174 pmaddubsw m2, m6
1175 mova [tmpq+32*2], m4
1176 mova [tmpq+32*3], m1
1177 add tmpq, 32*8
1178 punpcklbw m1, m3, m0
1179 punpckhbw m3, m0
1180 pmaddubsw m1, m6
1181 pmaddubsw m3, m6
1182 mova [tmpq-32*4], m5
1183 mova [tmpq-32*3], m2
1184 mova [tmpq-32*2], m1
1185 mova [tmpq-32*1], m3
1186 sub hd, 4
1187 jg .v_w32_loop
1188 RET
1189 .v_w64:
1190 vpermq m0, [srcq+strideq*0+32*0], q3120
1191 vpermq m1, [srcq+strideq*0+32*1], q3120
1192 .v_w64_loop:
1193 vpermq m2, [srcq+strideq*1+32*0], q3120
1194 vpermq m3, [srcq+strideq*1+32*1], q3120
1195 lea srcq, [srcq+strideq*2]
1196 punpcklbw m4, m0, m2
1197 punpckhbw m0, m2
1198 pmaddubsw m4, m6
1199 pmaddubsw m0, m6
1200 mova [tmpq+32*0], m4
1201 mova [tmpq+32*1], m0
1202 punpcklbw m4, m1, m3
1203 punpckhbw m5, m1, m3
1204 vpermq m0, [srcq+strideq*0+32*0], q3120
1205 vpermq m1, [srcq+strideq*0+32*1], q3120
1206 pmaddubsw m4, m6
1207 pmaddubsw m5, m6
1208 mova [tmpq+32*2], m4
1209 mova [tmpq+32*3], m5
1210 add tmpq, 32*8
1211 punpcklbw m4, m2, m0
1212 punpckhbw m2, m0
1213 punpcklbw m5, m3, m1
1214 punpckhbw m3, m1
1215 pmaddubsw m4, m6
1216 pmaddubsw m2, m6
1217 pmaddubsw m5, m6
1218 pmaddubsw m3, m6
1219 mova [tmpq-32*4], m4
1220 mova [tmpq-32*3], m2
1221 mova [tmpq-32*2], m5
1222 mova [tmpq-32*1], m3
1223 sub hd, 2
1224 jg .v_w64_loop
1225 RET
1226 .v_w128:
1227 lea r6d, [hq+(3<<8)]
1228 mov r3, srcq
1229 mov r5, tmpq
1230 .v_w128_loop0:
1231 vpermq m0, [srcq+strideq*0], q3120
1232 .v_w128_loop:
1233 vpermq m1, [srcq+strideq*1], q3120
1234 lea srcq, [srcq+strideq*2]
1235 punpcklbw m2, m0, m1
1236 punpckhbw m3, m0, m1
1237 vpermq m0, [srcq+strideq*0], q3120
1238 pmaddubsw m2, m6
1239 pmaddubsw m3, m6
1240 punpcklbw m4, m1, m0
1241 punpckhbw m1, m0
1242 pmaddubsw m4, m6
1243 pmaddubsw m1, m6
1244 mova [tmpq+32*0], m2
1245 mova [tmpq+32*1], m3
1246 mova [tmpq+32*8], m4
1247 mova [tmpq+32*9], m1
1248 add tmpq, 32*16
1249 sub hd, 2
1250 jg .v_w128_loop
1251 add r3, 32
1252 add r5, 64
1253 movzx hd, r6b
1254 mov srcq, r3
1255 mov tmpq, r5
1256 sub r6d, 1<<8
1257 jg .v_w128_loop0
1258 RET
1259 .hv:
1260 ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4
1261 ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4)
1262 %assign stack_offset stack_offset - stack_size_padded
1263 WIN64_SPILL_XMM 7
1264 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)]
1265 shl mxyd, 11
1266 movd xm6, mxyd
1267 vpbroadcastw m6, xm6
1268 add wq, r6
1269 lea stride3q, [strideq*3]
1270 jmp wq
1271 .hv_w4:
1272 vbroadcasti128 m4, [bilin_h_shuf4]
1273 vpbroadcastq m0, [srcq+strideq*0]
1274 pshufb m0, m4
1275 pmaddubsw m0, m5
1276 .hv_w4_loop:
1277 movq xm1, [srcq+strideq*1]
1278 movhps xm1, [srcq+strideq*2]
1279 movq xm2, [srcq+stride3q ]
1280 lea srcq, [srcq+strideq*4]
1281 movhps xm2, [srcq+strideq*0]
1282 vinserti128 m1, xm2, 1
1283 pshufb m1, m4
1284 pmaddubsw m1, m5 ; 1 2 3 4
1285 vpblendd m2, m1, m0, 0xc0
1286 vpermq m2, m2, q2103 ; 0 1 2 3
1287 mova m0, m1
1288 psubw m1, m2
1289 pmulhrsw m1, m6
1290 paddw m1, m2
1291 mova [tmpq], m1
1292 add tmpq, 32
1293 sub hd, 4
1294 jg .hv_w4_loop
1295 RET
1296 .hv_w8:
1297 vbroadcasti128 m0, [srcq+strideq*0]
1298 pshufb m0, m4
1299 pmaddubsw m0, m5
1300 .hv_w8_loop:
1301 movu xm1, [srcq+strideq*1]
1302 vinserti128 m1, [srcq+strideq*2], 1
1303 movu xm2, [srcq+stride3q ]
1304 lea srcq, [srcq+strideq*4]
1305 vinserti128 m2, [srcq+strideq*0], 1
1306 pshufb m1, m4
1307 pshufb m2, m4
1308 pmaddubsw m1, m5 ; 1 2
1309 vperm2i128 m3, m0, m1, 0x21 ; 0 1
1310 pmaddubsw m0, m2, m5 ; 3 4
1311 vperm2i128 m2, m1, m0, 0x21 ; 2 3
1312 psubw m1, m3
1313 pmulhrsw m1, m6
1314 paddw m1, m3
1315 psubw m3, m0, m2
1316 pmulhrsw m3, m6
1317 paddw m3, m2
1318 mova [tmpq+32*0], m1
1319 mova [tmpq+32*1], m3
1320 add tmpq, 32*2
1321 sub hd, 4
1322 jg .hv_w8_loop
1323 RET
1324 .hv_w16:
1325 movu xm0, [srcq+strideq*0+8*0]
1326 vinserti128 m0, [srcq+strideq*0+8*1], 1
1327 pshufb m0, m4
1328 pmaddubsw m0, m5
1329 .hv_w16_loop:
1330 movu xm1, [srcq+strideq*1+8*0]
1331 vinserti128 m1, [srcq+strideq*1+8*1], 1
1332 lea srcq, [srcq+strideq*2]
1333 movu xm2, [srcq+strideq*0+8*0]
1334 vinserti128 m2, [srcq+strideq*0+8*1], 1
1335 pshufb m1, m4
1336 pshufb m2, m4
1337 pmaddubsw m1, m5
1338 psubw m3, m1, m0
1339 pmulhrsw m3, m6
1340 paddw m3, m0
1341 pmaddubsw m0, m2, m5
1342 psubw m2, m0, m1
1343 pmulhrsw m2, m6
1344 paddw m2, m1
1345 mova [tmpq+32*0], m3
1346 mova [tmpq+32*1], m2
1347 add tmpq, 32*2
1348 sub hd, 2
1349 jg .hv_w16_loop
1350 RET
1351 .hv_w32:
1352 movu xm0, [srcq+8*0]
1353 vinserti128 m0, [srcq+8*1], 1
1354 movu xm1, [srcq+8*2]
1355 vinserti128 m1, [srcq+8*3], 1
1356 pshufb m0, m4
1357 pshufb m1, m4
1358 pmaddubsw m0, m5
1359 pmaddubsw m1, m5
1360 .hv_w32_loop:
1361 add srcq, strideq
1362 movu xm2, [srcq+8*0]
1363 vinserti128 m2, [srcq+8*1], 1
1364 pshufb m2, m4
1365 pmaddubsw m2, m5
1366 psubw m3, m2, m0
1367 pmulhrsw m3, m6
1368 paddw m3, m0
1369 mova m0, m2
1370 movu xm2, [srcq+8*2]
1371 vinserti128 m2, [srcq+8*3], 1
1372 pshufb m2, m4
1373 pmaddubsw m2, m5
1374 mova [tmpq+32*0], m3
1375 psubw m3, m2, m1
1376 pmulhrsw m3, m6
1377 paddw m3, m1
1378 mova m1, m2
1379 mova [tmpq+32*1], m3
1380 add tmpq, 32*2
1381 dec hd
1382 jg .hv_w32_loop
1383 RET
1384 .hv_w128:
1385 lea r3d, [hq+(7<<8)]
1386 mov r6d, 256
1387 jmp .hv_w64_start
1388 .hv_w64:
1389 lea r3d, [hq+(3<<8)]
1390 mov r6d, 128
1391 .hv_w64_start:
1392 %if WIN64
1393 PUSH r7
1394 %endif
1395 mov r5, srcq
1396 mov r7, tmpq
1397 .hv_w64_loop0:
1398 movu xm0, [srcq+strideq*0+8*0]
1399 vinserti128 m0, [srcq+strideq*0+8*1], 1
1400 pshufb m0, m4
1401 pmaddubsw m0, m5
1402 .hv_w64_loop:
1403 movu xm1, [srcq+strideq*1+8*0]
1404 vinserti128 m1, [srcq+strideq*1+8*1], 1
1405 lea srcq, [srcq+strideq*2]
1406 movu xm2, [srcq+strideq*0+8*0]
1407 vinserti128 m2, [srcq+strideq*0+8*1], 1
1408 pshufb m1, m4
1409 pshufb m2, m4
1410 pmaddubsw m1, m5
1411 psubw m3, m1, m0
1412 pmulhrsw m3, m6
1413 paddw m3, m0
1414 pmaddubsw m0, m2, m5
1415 psubw m2, m0, m1
1416 pmulhrsw m2, m6
1417 paddw m2, m1
1418 mova [tmpq+r6*0], m3
1419 mova [tmpq+r6*1], m2
1420 lea tmpq, [tmpq+r6*2]
1421 sub hd, 2
1422 jg .hv_w64_loop
1423 add r5, 16
1424 add r7, 32
1425 movzx hd, r3b
1426 mov srcq, r5
1427 mov tmpq, r7
1428 sub r3d, 1<<8
1429 jg .hv_w64_loop0
1430 %if WIN64
1431 POP r7
1432 %endif
1433 RET
1434
1435 ; int8_t subpel_filters[5][15][8]
1436 %assign FILTER_REGULAR (0*15 << 16) | 3*15
1437 %assign FILTER_SMOOTH (1*15 << 16) | 4*15
1438 %assign FILTER_SHARP (2*15 << 16) | 3*15
1439
1440 %macro FN 4 ; fn, type, type_h, type_v
1441 cglobal %1_%2_8bpc
1442 mov t0d, FILTER_%3
1443 %ifidn %3, %4
1444 mov t1d, t0d
1445 %else
1446 mov t1d, FILTER_%4
1447 %endif
1448 %ifnidn %2, regular ; skip the jump in the last filter
1449 jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX)
1450 %endif
1451 %endmacro
1452
1453 %if WIN64
1454 DECLARE_REG_TMP 4, 5
1455 %else
1456 DECLARE_REG_TMP 7, 8
1457 %endif
1458
1459 %define PUT_8TAP_FN FN put_8tap,
1460 PUT_8TAP_FN sharp, SHARP, SHARP
1461 PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH
1462 PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP
1463 PUT_8TAP_FN smooth, SMOOTH, SMOOTH
1464 PUT_8TAP_FN sharp_regular, SHARP, REGULAR
1465 PUT_8TAP_FN regular_sharp, REGULAR, SHARP
1466 PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR
1467 PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH
1468 PUT_8TAP_FN regular, REGULAR, REGULAR
1469
1470 cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
1471 imul mxd, mxm, 0x010101
1472 add mxd, t0d ; 8tap_h, mx, 4tap_h
1473 imul myd, mym, 0x010101
1474 add myd, t1d ; 8tap_v, my, 4tap_v
1475 lea r8, [put_avx2]
1476 movsxd wq, wm
1477 movifnidn hd, hm
1478 test mxd, 0xf00
1479 jnz .h
1480 test myd, 0xf00
1481 jnz .v
1482 tzcnt wd, wd
1483 movzx wd, word [r8+wq*2+table_offset(put,)]
1484 add wq, r8
1485 lea r6, [ssq*3]
1486 lea r7, [dsq*3]
1487 %if WIN64
1488 pop r8
1489 %endif
1490 jmp wq
1491 .h:
1492 test myd, 0xf00
1493 jnz .hv
1494 vpbroadcastd m5, [pw_34] ; 2 + (8 << 2)
1495 WIN64_SPILL_XMM 11
1496 cmp wd, 4
1497 jl .h_w2
1498 vbroadcasti128 m6, [subpel_h_shufA]
1499 je .h_w4
1500 tzcnt wd, wd
1501 vbroadcasti128 m7, [subpel_h_shufB]
1502 vbroadcasti128 m8, [subpel_h_shufC]
1503 shr mxd, 16
1504 sub srcq, 3
1505 movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)]
1506 vpbroadcastd m9, [r8+mxq*8+subpel_filters-put_avx2+0]
1507 vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+4]
1508 add wq, r8
1509 jmp wq
1510 .h_w2:
1511 movzx mxd, mxb
1512 dec srcq
1513 mova xm4, [subpel_h_shuf4]
1514 vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2]
1515 .h_w2_loop:
1516 movq xm0, [srcq+ssq*0]
1517 movhps xm0, [srcq+ssq*1]
1518 lea srcq, [srcq+ssq*2]
1519 pshufb xm0, xm4
1520 pmaddubsw xm0, xm3
1521 phaddw xm0, xm0
1522 paddw xm0, xm5
1523 psraw xm0, 6
1524 packuswb xm0, xm0
1525 pextrw [dstq+dsq*0], xm0, 0
1526 pextrw [dstq+dsq*1], xm0, 1
1527 lea dstq, [dstq+dsq*2]
1528 sub hd, 2
1529 jg .h_w2_loop
1530 RET
1531 .h_w4:
1532 movzx mxd, mxb
1533 dec srcq
1534 vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2]
1535 .h_w4_loop:
1536 movq xm0, [srcq+ssq*0]
1537 movq xm1, [srcq+ssq*1]
1538 lea srcq, [srcq+ssq*2]
1539 pshufb xm0, xm6
1540 pshufb xm1, xm6
1541 pmaddubsw xm0, xm3
1542 pmaddubsw xm1, xm3
1543 phaddw xm0, xm1
1544 paddw xm0, xm5
1545 psraw xm0, 6
1546 packuswb xm0, xm0
1547 movd [dstq+dsq*0], xm0
1548 pextrd [dstq+dsq*1], xm0, 1
1549 lea dstq, [dstq+dsq*2]
1550 sub hd, 2
1551 jg .h_w4_loop
1552 RET
1553 .h_w8:
1554 %macro PUT_8TAP_H 4 ; dst/src, tmp[1-3]
1555 pshufb m%2, m%1, m7
1556 pshufb m%3, m%1, m8
1557 pshufb m%1, m6
1558 pmaddubsw m%4, m%2, m9
1559 pmaddubsw m%2, m10
1560 pmaddubsw m%3, m10
1561 pmaddubsw m%1, m9
1562 paddw m%3, m%4
1563 paddw m%1, m%2
1564 phaddw m%1, m%3
1565 paddw m%1, m5
1566 psraw m%1, 6
1567 %endmacro
1568 movu xm0, [srcq+ssq*0]
1569 vinserti128 m0, [srcq+ssq*1], 1
1570 lea srcq, [srcq+ssq*2]
1571 PUT_8TAP_H 0, 1, 2, 3
1572 vextracti128 xm1, m0, 1
1573 packuswb xm0, xm1
1574 movq [dstq+dsq*0], xm0
1575 movhps [dstq+dsq*1], xm0
1576 lea dstq, [dstq+dsq*2]
1577 sub hd, 2
1578 jg .h_w8
1579 RET
1580 .h_w16:
1581 movu xm0, [srcq+ssq*0+8*0]
1582 vinserti128 m0, [srcq+ssq*1+8*0], 1
1583 movu xm1, [srcq+ssq*0+8*1]
1584 vinserti128 m1, [srcq+ssq*1+8*1], 1
1585 PUT_8TAP_H 0, 2, 3, 4
1586 lea srcq, [srcq+ssq*2]
1587 PUT_8TAP_H 1, 2, 3, 4
1588 packuswb m0, m1
1589 mova [dstq+dsq*0], xm0
1590 vextracti128 [dstq+dsq*1], m0, 1
1591 lea dstq, [dstq+dsq*2]
1592 sub hd, 2
1593 jg .h_w16
1594 RET
1595 .h_w32:
1596 xor r6d, r6d
1597 jmp .h_start
1598 .h_w64:
1599 mov r6, -32*1
1600 jmp .h_start
1601 .h_w128:
1602 mov r6, -32*3
1603 .h_start:
1604 sub srcq, r6
1605 sub dstq, r6
1606 mov r4, r6
1607 .h_loop:
1608 movu m0, [srcq+r6+8*0]
1609 movu m1, [srcq+r6+8*1]
1610 PUT_8TAP_H 0, 2, 3, 4
1611 PUT_8TAP_H 1, 2, 3, 4
1612 packuswb m0, m1
1613 mova [dstq+r6], m0
1614 add r6, 32
1615 jle .h_loop
1616 add srcq, ssq
1617 add dstq, dsq
1618 mov r6, r4
1619 dec hd
1620 jg .h_loop
1621 RET
1622 .v:
1623 %assign stack_offset stack_offset - stack_size_padded
1624 WIN64_SPILL_XMM 16
1625 movzx mxd, myb
1626 shr myd, 16
1627 cmp hd, 6
1628 cmovs myd, mxd
1629 tzcnt r6d, wd
1630 movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)]
1631 vpbroadcastd m7, [pw_512]
1632 lea myq, [r8+myq*8+subpel_filters-put_avx2]
1633 vpbroadcastw m8, [myq+0]
1634 vpbroadcastw m9, [myq+2]
1635 vpbroadcastw m10, [myq+4]
1636 vpbroadcastw m11, [myq+6]
1637 add r6, r8
1638 lea ss3q, [ssq*3]
1639 sub srcq, ss3q
1640 jmp r6
1641 .v_w2:
1642 movd xm2, [srcq+ssq*0]
1643 pinsrw xm2, [srcq+ssq*1], 2
1644 pinsrw xm2, [srcq+ssq*2], 4
1645 add srcq, ss3q
1646 pinsrw xm2, [srcq+ssq*0], 6 ; 0 1 2 3
1647 movd xm3, [srcq+ssq*1]
1648 vpbroadcastd xm1, [srcq+ssq*2]
1649 add srcq, ss3q
1650 vpbroadcastd xm0, [srcq+ssq*0]
1651 vpblendd xm3, xm1, 0x02 ; 4 5
1652 vpblendd xm1, xm0, 0x02 ; 5 6
1653 palignr xm4, xm3, xm2, 4 ; 1 2 3 4
1654 punpcklbw xm3, xm1 ; 45 56
1655 punpcklbw xm1, xm2, xm4 ; 01 12
1656 punpckhbw xm2, xm4 ; 23 34
1657 .v_w2_loop:
1658 pmaddubsw xm5, xm1, xm8 ; a0 b0
1659 mova xm1, xm2
1660 pmaddubsw xm2, xm9 ; a1 b1
1661 paddw xm5, xm2
1662 mova xm2, xm3
1663 pmaddubsw xm3, xm10 ; a2 b2
1664 paddw xm5, xm3
1665 vpbroadcastd xm4, [srcq+ssq*1]
1666 lea srcq, [srcq+ssq*2]
1667 vpblendd xm3, xm0, xm4, 0x02 ; 6 7
1668 vpbroadcastd xm0, [srcq+ssq*0]
1669 vpblendd xm4, xm0, 0x02 ; 7 8
1670 punpcklbw xm3, xm4 ; 67 78
1671 pmaddubsw xm4, xm3, xm11 ; a3 b3
1672 paddw xm5, xm4
1673 pmulhrsw xm5, xm7
1674 packuswb xm5, xm5
1675 pextrw [dstq+dsq*0], xm5, 0
1676 pextrw [dstq+dsq*1], xm5, 2
1677 lea dstq, [dstq+dsq*2]
1678 sub hd, 2
1679 jg .v_w2_loop
1680 RET
1681 .v_w4:
1682 movd xm2, [srcq+ssq*0]
1683 pinsrd xm2, [srcq+ssq*1], 1
1684 pinsrd xm2, [srcq+ssq*2], 2
1685 add srcq, ss3q
1686 pinsrd xm2, [srcq+ssq*0], 3 ; 0 1 2 3
1687 movd xm3, [srcq+ssq*1]
1688 vpbroadcastd xm1, [srcq+ssq*2]
1689 add srcq, ss3q
1690 vpbroadcastd xm0, [srcq+ssq*0]
1691 vpblendd xm3, xm1, 0x02 ; 4 5
1692 vpblendd xm1, xm0, 0x02 ; 5 6
1693 palignr xm4, xm3, xm2, 4 ; 1 2 3 4
1694 punpcklbw xm3, xm1 ; 45 56
1695 punpcklbw xm1, xm2, xm4 ; 01 12
1696 punpckhbw xm2, xm4 ; 23 34
1697 .v_w4_loop:
1698 pmaddubsw xm5, xm1, xm8 ; a0 b0
1699 mova xm1, xm2
1700 pmaddubsw xm2, xm9 ; a1 b1
1701 paddw xm5, xm2
1702 mova xm2, xm3
1703 pmaddubsw xm3, xm10 ; a2 b2
1704 paddw xm5, xm3
1705 vpbroadcastd xm4, [srcq+ssq*1]
1706 lea srcq, [srcq+ssq*2]
1707 vpblendd xm3, xm0, xm4, 0x02 ; 6 7
1708 vpbroadcastd xm0, [srcq+ssq*0]
1709 vpblendd xm4, xm0, 0x02 ; 7 8
1710 punpcklbw xm3, xm4 ; 67 78
1711 pmaddubsw xm4, xm3, xm11 ; a3 b3
1712 paddw xm5, xm4
1713 pmulhrsw xm5, xm7
1714 packuswb xm5, xm5
1715 movd [dstq+dsq*0], xm5
1716 pextrd [dstq+dsq*1], xm5, 1
1717 lea dstq, [dstq+dsq*2]
1718 sub hd, 2
1719 jg .v_w4_loop
1720 RET
1721 .v_w8:
1722 movq xm1, [srcq+ssq*0]
1723 vpbroadcastq m4, [srcq+ssq*1]
1724 vpbroadcastq m2, [srcq+ssq*2]
1725 add srcq, ss3q
1726 vpbroadcastq m5, [srcq+ssq*0]
1727 vpbroadcastq m3, [srcq+ssq*1]
1728 vpbroadcastq m6, [srcq+ssq*2]
1729 add srcq, ss3q
1730 vpbroadcastq m0, [srcq+ssq*0]
1731 vpblendd m1, m4, 0x30
1732 vpblendd m4, m2, 0x30
1733 punpcklbw m1, m4 ; 01 12
1734 vpblendd m2, m5, 0x30
1735 vpblendd m5, m3, 0x30
1736 punpcklbw m2, m5 ; 23 34
1737 vpblendd m3, m6, 0x30
1738 vpblendd m6, m0, 0x30
1739 punpcklbw m3, m6 ; 45 56
1740 .v_w8_loop:
1741 vpbroadcastq m4, [srcq+ssq*1]
1742 lea srcq, [srcq+ssq*2]
1743 pmaddubsw m5, m1, m8 ; a0 b0
1744 mova m1, m2
1745 pmaddubsw m2, m9 ; a1 b1
1746 paddw m5, m2
1747 mova m2, m3
1748 pmaddubsw m3, m10 ; a2 b2
1749 paddw m5, m3
1750 vpblendd m3, m0, m4, 0x30
1751 vpbroadcastq m0, [srcq+ssq*0]
1752 vpblendd m4, m0, 0x30
1753 punpcklbw m3, m4 ; 67 78
1754 pmaddubsw m4, m3, m11 ; a3 b3
1755 paddw m5, m4
1756 pmulhrsw m5, m7
1757 vextracti128 xm4, m5, 1
1758 packuswb xm5, xm4
1759 movq [dstq+dsq*0], xm5
1760 movhps [dstq+dsq*1], xm5
1761 lea dstq, [dstq+dsq*2]
1762 sub hd, 2
1763 jg .v_w8_loop
1764 RET
1765 .v_w16:
1766 .v_w32:
1767 .v_w64:
1768 .v_w128:
1769 lea r6d, [wq*8-128]
1770 mov r4, srcq
1771 mov r7, dstq
1772 lea r6d, [hq+r6*2]
1773 .v_w16_loop0:
1774 vbroadcasti128 m4, [srcq+ssq*0]
1775 vbroadcasti128 m5, [srcq+ssq*1]
1776 vbroadcasti128 m6, [srcq+ssq*2]
1777 add srcq, ss3q
1778 vbroadcasti128 m0, [srcq+ssq*0]
1779 vbroadcasti128 m1, [srcq+ssq*1]
1780 vbroadcasti128 m2, [srcq+ssq*2]
1781 add srcq, ss3q
1782 vbroadcasti128 m3, [srcq+ssq*0]
1783 shufpd m4, m0, 0x0c
1784 shufpd m5, m1, 0x0c
1785 punpcklbw m1, m4, m5 ; 01
1786 punpckhbw m4, m5 ; 34
1787 shufpd m6, m2, 0x0c
1788 punpcklbw m2, m5, m6 ; 12
1789 punpckhbw m5, m6 ; 45
1790 shufpd m0, m3, 0x0c
1791 punpcklbw m3, m6, m0 ; 23
1792 punpckhbw m6, m0 ; 56
1793 .v_w16_loop:
1794 vbroadcasti128 m12, [srcq+ssq*1]
1795 lea srcq, [srcq+ssq*2]
1796 vbroadcasti128 m13, [srcq+ssq*0]
1797 pmaddubsw m14, m1, m8 ; a0
1798 pmaddubsw m15, m2, m8 ; b0
1799 mova m1, m3
1800 mova m2, m4
1801 pmaddubsw m3, m9 ; a1
1802 pmaddubsw m4, m9 ; b1
1803 paddw m14, m3
1804 paddw m15, m4
1805 mova m3, m5
1806 mova m4, m6
1807 pmaddubsw m5, m10 ; a2
1808 pmaddubsw m6, m10 ; b2
1809 paddw m14, m5
1810 paddw m15, m6
1811 shufpd m6, m0, m12, 0x0d
1812 shufpd m0, m12, m13, 0x0c
1813 punpcklbw m5, m6, m0 ; 67
1814 punpckhbw m6, m0 ; 78
1815 pmaddubsw m12, m5, m11 ; a3
1816 pmaddubsw m13, m6, m11 ; b3
1817 paddw m14, m12
1818 paddw m15, m13
1819 pmulhrsw m14, m7
1820 pmulhrsw m15, m7
1821 packuswb m14, m15
1822 vpermq m14, m14, q3120
1823 mova [dstq+dsq*0], xm14
1824 vextracti128 [dstq+dsq*1], m14, 1
1825 lea dstq, [dstq+dsq*2]
1826 sub hd, 2
1827 jg .v_w16_loop
1828 add r4, 16
1829 add r7, 16
1830 movzx hd, r6b
1831 mov srcq, r4
1832 mov dstq, r7
1833 sub r6d, 1<<8
1834 jg .v_w16_loop0
1835 RET
1836 .hv:
1837 %assign stack_offset stack_offset - stack_size_padded
1838 WIN64_SPILL_XMM 16
1839 cmp wd, 4
1840 jg .hv_w8
1841 movzx mxd, mxb
1842 dec srcq
1843 vpbroadcastd m7, [r8+mxq*8+subpel_filters-put_avx2+2]
1844 movzx mxd, myb
1845 shr myd, 16
1846 cmp hd, 6
1847 cmovs myd, mxd
1848 vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2]
1849 lea ss3q, [ssq*3]
1850 sub srcq, ss3q
1851 punpcklbw m0, m0
1852 psraw m0, 8 ; sign-extend
1853 vpbroadcastd m8, [pw_8192]
1854 vpbroadcastd m9, [pd_512]
1855 pshufd m10, m0, q0000
1856 pshufd m11, m0, q1111
1857 pshufd m12, m0, q2222
1858 pshufd m13, m0, q3333
1859 cmp wd, 4
1860 je .hv_w4
1861 vbroadcasti128 m6, [subpel_h_shuf4]
1862 movq xm2, [srcq+ssq*0]
1863 movhps xm2, [srcq+ssq*1]
1864 movq xm0, [srcq+ssq*2]
1865 add srcq, ss3q
1866 movhps xm0, [srcq+ssq*0]
1867 vpbroadcastq m3, [srcq+ssq*1]
1868 vpbroadcastq m4, [srcq+ssq*2]
1869 add srcq, ss3q
1870 vpbroadcastq m1, [srcq+ssq*0]
1871 vpblendd m2, m3, 0x30
1872 vpblendd m0, m1, 0x30
1873 vpblendd m2, m4, 0xc0
1874 pshufb m2, m6
1875 pshufb m0, m6
1876 pmaddubsw m2, m7
1877 pmaddubsw m0, m7
1878 phaddw m2, m0
1879 pmulhrsw m2, m8
1880 vextracti128 xm3, m2, 1
1881 palignr xm4, xm3, xm2, 4
1882 punpcklwd xm1, xm2, xm4 ; 01 12
1883 punpckhwd xm2, xm4 ; 23 34
1884 pshufd xm0, xm3, q2121
1885 punpcklwd xm3, xm0 ; 45 56
1886 .hv_w2_loop:
1887 movq xm4, [srcq+ssq*1]
1888 lea srcq, [srcq+ssq*2]
1889 movhps xm4, [srcq+ssq*0]
1890 pshufb xm4, xm6
1891 pmaddubsw xm4, xm7
1892 pmaddwd xm5, xm1, xm10 ; a0 b0
1893 mova xm1, xm2
1894 pmaddwd xm2, xm11 ; a1 b1
1895 paddd xm5, xm2
1896 mova xm2, xm3
1897 pmaddwd xm3, xm12 ; a2 b2
1898 phaddw xm4, xm4
1899 pmulhrsw xm4, xm8
1900 paddd xm5, xm3
1901 palignr xm3, xm4, xm0, 12
1902 mova xm0, xm4
1903 punpcklwd xm3, xm0 ; 67 78
1904 pmaddwd xm4, xm3, xm13 ; a3 b3
1905 paddd xm5, xm9
1906 paddd xm5, xm4
1907 psrad xm5, 10
1908 packssdw xm5, xm5
1909 packuswb xm5, xm5
1910 pextrw [dstq+dsq*0], xm5, 0
1911 pextrw [dstq+dsq*1], xm5, 1
1912 lea dstq, [dstq+dsq*2]
1913 sub hd, 2
1914 jg .hv_w2_loop
1915 RET
1916 .hv_w4:
1917 mova m6, [subpel_h_shuf4]
1918 vpbroadcastq m2, [srcq+ssq*0]
1919 vpbroadcastq m4, [srcq+ssq*1]
1920 vpbroadcastq m0, [srcq+ssq*2]
1921 add srcq, ss3q
1922 vpbroadcastq m5, [srcq+ssq*0]
1923 vpbroadcastq m3, [srcq+ssq*1]
1924 vpblendd m2, m4, 0xcc ; 0 1
1925 vpbroadcastq m4, [srcq+ssq*2]
1926 add srcq, ss3q
1927 vpbroadcastq m1, [srcq+ssq*0]
1928 vpblendd m0, m5, 0xcc ; 2 3
1929 vpblendd m3, m4, 0xcc ; 4 5
1930 pshufb m2, m6
1931 pshufb m0, m6
1932 pshufb m3, m6
1933 pshufb m1, m6
1934 pmaddubsw m2, m7
1935 pmaddubsw m0, m7
1936 pmaddubsw m3, m7
1937 pmaddubsw m1, m7
1938 phaddw m2, m0
1939 phaddw m3, m1
1940 pmulhrsw m2, m8
1941 pmulhrsw m3, m8
1942 palignr m4, m3, m2, 4
1943 punpcklwd m1, m2, m4 ; 01 12
1944 punpckhwd m2, m4 ; 23 34
1945 pshufd m0, m3, q2121
1946 punpcklwd m3, m0 ; 45 56
1947 .hv_w4_loop:
1948 vpbroadcastq m4, [srcq+ssq*1]
1949 lea srcq, [srcq+ssq*2]
1950 pmaddwd m5, m1, m10 ; a0 b0
1951 mova m1, m2
1952 pmaddwd m2, m11 ; a1 b1
1953 paddd m5, m2
1954 mova m2, m3
1955 pmaddwd m3, m12 ; a2 b2
1956 paddd m5, m3
1957 vpbroadcastq m3, [srcq+ssq*0]
1958 vpblendd m4, m3, 0xcc ; 7 8
1959 pshufb m4, m6
1960 pmaddubsw m4, m7
1961 phaddw m4, m4
1962 pmulhrsw m4, m8
1963 palignr m3, m4, m0, 12
1964 mova m0, m4
1965 punpcklwd m3, m0 ; 67 78
1966 pmaddwd m4, m3, m13 ; a3 b3
1967 paddd m5, m9
1968 paddd m5, m4
1969 psrad m5, 10
1970 vextracti128 xm4, m5, 1
1971 packssdw xm5, xm4
1972 packuswb xm5, xm5
1973 pshuflw xm5, xm5, q3120
1974 movd [dstq+dsq*0], xm5
1975 pextrd [dstq+dsq*1], xm5, 1
1976 lea dstq, [dstq+dsq*2]
1977 sub hd, 2
1978 jg .hv_w4_loop
1979 RET
1980 .hv_w8:
1981 shr mxd, 16
1982 sub srcq, 3
1983 vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+0]
1984 vpbroadcastd m11, [r8+mxq*8+subpel_filters-put_avx2+4]
1985 movzx mxd, myb
1986 shr myd, 16
1987 cmp hd, 6
1988 cmovs myd, mxd
1989 vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2]
1990 lea ss3q, [ssq*3]
1991 sub srcq, ss3q
1992 punpcklbw m0, m0
1993 psraw m0, 8 ; sign-extend
1994 pshufd m12, m0, q0000
1995 pshufd m13, m0, q1111
1996 pshufd m14, m0, q2222
1997 pshufd m15, m0, q3333
1998 lea r6d, [wq*8-64]
1999 mov r4, srcq
2000 mov r7, dstq
2001 lea r6d, [hq+r6*4]
2002 .hv_w8_loop0:
2003 vbroadcasti128 m7, [subpel_h_shufA]
2004 movu xm4, [srcq+ssq*0]
2005 vbroadcasti128 m8, [subpel_h_shufB]
2006 movu xm5, [srcq+ssq*1]
2007 vbroadcasti128 m9, [subpel_h_shufC]
2008 movu xm6, [srcq+ssq*2]
2009 add srcq, ss3q
2010 vbroadcasti128 m0, [srcq+ssq*0]
2011 vpblendd m4, m0, 0xf0 ; 0 3
2012 vinserti128 m5, [srcq+ssq*1], 1 ; 1 4
2013 vinserti128 m6, [srcq+ssq*2], 1 ; 2 5
2014 add srcq, ss3q
2015 vinserti128 m0, [srcq+ssq*0], 1 ; 3 6
2016 %macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3]
2017 pshufb %3, %1, %6
2018 pshufb %4, %1, %7
2019 pshufb %1, %5
2020 pmaddubsw %2, %3, m10
2021 pmaddubsw %4, m11
2022 pmaddubsw %3, m11
2023 pmaddubsw %1, m10
2024 paddw %2, %4
2025 paddw %1, %3
2026 phaddw %1, %2
2027 %endmacro
2028 HV_H_W8 m4, m1, m2, m3, m7, m8, m9
2029 HV_H_W8 m5, m1, m2, m3, m7, m8, m9
2030 HV_H_W8 m6, m1, m2, m3, m7, m8, m9
2031 HV_H_W8 m0, m1, m2, m3, m7, m8, m9
2032 vpbroadcastd m7, [pw_8192]
2033 vpermq m4, m4, q3120
2034 vpermq m5, m5, q3120
2035 vpermq m6, m6, q3120
2036 pmulhrsw m0, m7
2037 pmulhrsw m4, m7
2038 pmulhrsw m5, m7
2039 pmulhrsw m6, m7
2040 vpermq m7, m0, q3120
2041 punpcklwd m1, m4, m5 ; 01
2042 punpckhwd m4, m5 ; 34
2043 punpcklwd m2, m5, m6 ; 12
2044 punpckhwd m5, m6 ; 45
2045 punpcklwd m3, m6, m7 ; 23
2046 punpckhwd m6, m7 ; 56
2047 .hv_w8_loop:
2048 vextracti128 r6m, m0, 1 ; not enough registers
2049 movu xm0, [srcq+ssq*1]
2050 lea srcq, [srcq+ssq*2]
2051 vinserti128 m0, [srcq+ssq*0], 1 ; 7 8
2052 pmaddwd m8, m1, m12 ; a0
2053 pmaddwd m9, m2, m12 ; b0
2054 mova m1, m3
2055 mova m2, m4
2056 pmaddwd m3, m13 ; a1
2057 pmaddwd m4, m13 ; b1
2058 paddd m8, m3
2059 paddd m9, m4
2060 mova m3, m5
2061 mova m4, m6
2062 pmaddwd m5, m14 ; a2
2063 pmaddwd m6, m14 ; b2
2064 paddd m8, m5
2065 paddd m9, m6
2066 vbroadcasti128 m6, [subpel_h_shufB]
2067 vbroadcasti128 m7, [subpel_h_shufC]
2068 vbroadcasti128 m5, [subpel_h_shufA]
2069 HV_H_W8 m0, m5, m6, m7, m5, m6, m7
2070 vpbroadcastd m5, [pw_8192]
2071 vpbroadcastd m7, [pd_512]
2072 vbroadcasti128 m6, r6m
2073 pmulhrsw m0, m5
2074 paddd m8, m7
2075 paddd m9, m7
2076 vpermq m7, m0, q3120 ; 7 8
2077 shufpd m6, m6, m7, 0x04 ; 6 7
2078 punpcklwd m5, m6, m7 ; 67
2079 punpckhwd m6, m7 ; 78
2080 pmaddwd m7, m5, m15 ; a3
2081 paddd m8, m7
2082 pmaddwd m7, m6, m15 ; b3
2083 paddd m7, m9
2084 psrad m8, 10
2085 psrad m7, 10
2086 packssdw m8, m7
2087 vextracti128 xm7, m8, 1
2088 packuswb xm8, xm7
2089 pshufd xm7, xm8, q3120
2090 movq [dstq+dsq*0], xm7
2091 movhps [dstq+dsq*1], xm7
2092 lea dstq, [dstq+dsq*2]
2093 sub hd, 2
2094 jg .hv_w8_loop
2095 add r4, 8
2096 add r7, 8
2097 movzx hd, r6b
2098 mov srcq, r4
2099 mov dstq, r7
2100 sub r6d, 1<<8
2101 jg .hv_w8_loop0
2102 RET
2103
2104 %macro PREP_8TAP_H 0
2105 pshufb m1, m0, m5
2106 pshufb m2, m0, m6
2107 pshufb m3, m0, m7
2108 pmaddubsw m1, m8
2109 pmaddubsw m0, m2, m8
2110 pmaddubsw m2, m9
2111 pmaddubsw m3, m9
2112 paddw m1, m2
2113 paddw m0, m3
2114 phaddw m0, m1, m0
2115 pmulhrsw m0, m4
2116 %endmacro
2117
2118 %if WIN64
2119 DECLARE_REG_TMP 6, 4
2120 %else
2121 DECLARE_REG_TMP 6, 7
2122 %endif
2123
2124 %define PREP_8TAP_FN FN prep_8tap,
2125 PREP_8TAP_FN sharp, SHARP, SHARP
2126 PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH
2127 PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP
2128 PREP_8TAP_FN smooth, SMOOTH, SMOOTH
2129 PREP_8TAP_FN sharp_regular, SHARP, REGULAR
2130 PREP_8TAP_FN regular_sharp, REGULAR, SHARP
2131 PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR
2132 PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH
2133 PREP_8TAP_FN regular, REGULAR, REGULAR
2134
2135 cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
2136 imul mxd, mxm, 0x010101
2137 add mxd, t0d ; 8tap_h, mx, 4tap_h
2138 imul myd, mym, 0x010101
2139 add myd, t1d ; 8tap_v, my, 4tap_v
2140 lea r7, [prep%+SUFFIX]
2141 movsxd wq, wm
2142 movifnidn hd, hm
2143 test mxd, 0xf00
2144 jnz .h
2145 test myd, 0xf00
2146 jnz .v
2147 tzcnt wd, wd
2148 movzx wd, word [r7+wq*2+table_offset(prep,)]
2149 add wq, r7
2150 lea r6, [strideq*3]
2151 %if WIN64
2152 pop r7
2153 %endif
2154 jmp wq
2155 .h:
2156 test myd, 0xf00
2157 jnz .hv
2158 vpbroadcastd m4, [pw_8192]
2159 vbroadcasti128 m5, [subpel_h_shufA]
2160 WIN64_SPILL_XMM 10
2161 cmp wd, 4
2162 je .h_w4
2163 tzcnt wd, wd
2164 vbroadcasti128 m6, [subpel_h_shufB]
2165 vbroadcasti128 m7, [subpel_h_shufC]
2166 shr mxd, 16
2167 sub srcq, 3
2168 movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)]
2169 vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0]
2170 vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4]
2171 add wq, r7
2172 jmp wq
2173 .h_w4:
2174 movzx mxd, mxb
2175 dec srcq
2176 vpbroadcastd m6, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2]
2177 lea stride3q, [strideq*3]
2178 .h_w4_loop:
2179 movq xm0, [srcq+strideq*0]
2180 vpbroadcastq m2, [srcq+strideq*2]
2181 movq xm1, [srcq+strideq*1]
2182 vpblendd m0, m2, 0xf0
2183 vpbroadcastq m2, [srcq+stride3q ]
2184 lea srcq, [srcq+strideq*4]
2185 vpblendd m1, m2, 0xf0
2186 pshufb m0, m5
2187 pshufb m1, m5
2188 pmaddubsw m0, m6
2189 pmaddubsw m1, m6
2190 phaddw m0, m1
2191 pmulhrsw m0, m4
2192 mova [tmpq], m0
2193 add tmpq, 32
2194 sub hd, 4
2195 jg .h_w4_loop
2196 RET
2197 .h_w8:
2198 movu xm0, [srcq+strideq*0]
2199 vinserti128 m0, [srcq+strideq*1], 1
2200 lea srcq, [srcq+strideq*2]
2201 PREP_8TAP_H
2202 mova [tmpq], m0
2203 add tmpq, 32
2204 sub hd, 2
2205 jg .h_w8
2206 RET
2207 .h_w16:
2208 movu xm0, [srcq+strideq*0+8*0]
2209 vinserti128 m0, [srcq+strideq*0+8*1], 1
2210 PREP_8TAP_H
2211 mova [tmpq+32*0], m0
2212 movu xm0, [srcq+strideq*1+8*0]
2213 vinserti128 m0, [srcq+strideq*1+8*1], 1
2214 lea srcq, [srcq+strideq*2]
2215 PREP_8TAP_H
2216 mova [tmpq+32*1], m0
2217 add tmpq, 32*2
2218 sub hd, 2
2219 jg .h_w16
2220 RET
2221 .h_w32:
2222 xor r6d, r6d
2223 jmp .h_start
2224 .h_w64:
2225 mov r6, -32*1
2226 jmp .h_start
2227 .h_w128:
2228 mov r6, -32*3
2229 .h_start:
2230 sub srcq, r6
2231 mov r5, r6
2232 .h_loop:
2233 movu xm0, [srcq+r6+8*0]
2234 vinserti128 m0, [srcq+r6+8*1], 1
2235 PREP_8TAP_H
2236 mova [tmpq+32*0], m0
2237 movu xm0, [srcq+r6+8*2]
2238 vinserti128 m0, [srcq+r6+8*3], 1
2239 PREP_8TAP_H
2240 mova [tmpq+32*1], m0
2241 add tmpq, 32*2
2242 add r6, 32
2243 jle .h_loop
2244 add srcq, strideq
2245 mov r6, r5
2246 dec hd
2247 jg .h_loop
2248 RET
2249 .v:
2250 %assign stack_offset stack_offset - stack_size_padded
2251 WIN64_SPILL_XMM 16
2252 movzx mxd, myb ; Select 4-tap/8-tap filter multipliers.
2253 shr myd, 16 ; Note that the code is 8-tap only, having
2254 cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4
2255 cmove myd, mxd ; had a negligible effect on performance.
2256 ; TODO: Would a 6-tap code path be worth it?
2257 lea myq, [r7+myq*8+subpel_filters-prep%+SUFFIX]
2258 lea stride3q, [strideq*3]
2259 sub srcq, stride3q
2260 vpbroadcastd m7, [pw_8192]
2261 vpbroadcastw m8, [myq+0]
2262 vpbroadcastw m9, [myq+2]
2263 vpbroadcastw m10, [myq+4]
2264 vpbroadcastw m11, [myq+6]
2265 cmp wd, 8
2266 jg .v_w16
2267 je .v_w8
2268 .v_w4:
2269 movd xm0, [srcq+strideq*0]
2270 vpbroadcastd m1, [srcq+strideq*2]
2271 vpbroadcastd xm2, [srcq+strideq*1]
2272 add srcq, stride3q
2273 vpbroadcastd m3, [srcq+strideq*0]
2274 vpblendd m1, m0, 0x01 ; 0 2 2 _ 2 _ _ _
2275 vpblendd m3, m2, 0x03 ; 1 1 3 3 3 3 _ _
2276 vpbroadcastd m0, [srcq+strideq*1]
2277 vpbroadcastd m2, [srcq+strideq*2]
2278 vpblendd m1, m0, 0x68 ; 0 2 2 4 2 4 4 _
2279 vpbroadcastd m0, [srcq+stride3q ]
2280 vbroadcasti128 m5, [deint_shuf4]
2281 vpblendd m3, m2, 0xc0 ; 1 1 3 3 3 3 5 5
2282 vpblendd m2, m3, m1, 0x55 ; 0 1 2 3 2 3 4 5
2283 vpblendd m3, m1, 0xaa ; 1 2 3 4 3 4 5 _
2284 punpcklbw m1, m2, m3 ; 01 12 23 34
2285 vpblendd m3, m0, 0x80 ; 1 2 3 4 3 4 5 6
2286 punpckhbw m2, m3 ; 23 34 45 56
2287 .v_w4_loop:
2288 lea srcq, [srcq+strideq*4]
2289 pinsrd xm0, [srcq+strideq*0], 1
2290 vpbroadcastd m3, [srcq+strideq*1]
2291 vpbroadcastd m4, [srcq+strideq*2]
2292 vpblendd m3, m0, 0x03 ; 6 7 8 _ 8 _ _ _
2293 vpbroadcastd m0, [srcq+stride3q ]
2294 vpblendd m3, m4, 0x20 ; 6 7 8 _ 8 9 _ _
2295 vpblendd m3, m0, 0x40 ; 6 7 8 _ 8 9 a _
2296 pshufb m3, m5 ; 67 78 89 9a
2297 pmaddubsw m4, m1, m8
2298 vperm2i128 m1, m2, m3, 0x21 ; 45 56 67 78
2299 pmaddubsw m2, m9
2300 paddw m4, m2
2301 mova m2, m3
2302 pmaddubsw m3, m11
2303 paddw m3, m4
2304 pmaddubsw m4, m1, m10
2305 paddw m3, m4
2306 pmulhrsw m3, m7
2307 mova [tmpq], m3
2308 add tmpq, 32
2309 sub hd, 4
2310 jg .v_w4_loop
2311 RET
2312 .v_w8:
2313 movq xm1, [srcq+strideq*0]
2314 vpbroadcastq m4, [srcq+strideq*1]
2315 vpbroadcastq m2, [srcq+strideq*2]
2316 vpbroadcastq m5, [srcq+stride3q ]
2317 lea srcq, [srcq+strideq*4]
2318 vpbroadcastq m3, [srcq+strideq*0]
2319 vpbroadcastq m6, [srcq+strideq*1]
2320 vpbroadcastq m0, [srcq+strideq*2]
2321 vpblendd m1, m4, 0x30
2322 vpblendd m4, m2, 0x30
2323 punpcklbw m1, m4 ; 01 12
2324 vpblendd m2, m5, 0x30
2325 vpblendd m5, m3, 0x30
2326 punpcklbw m2, m5 ; 23 34
2327 vpblendd m3, m6, 0x30
2328 vpblendd m6, m0, 0x30
2329 punpcklbw m3, m6 ; 45 56
2330 .v_w8_loop:
2331 vpbroadcastq m4, [srcq+stride3q ]
2332 lea srcq, [srcq+strideq*4]
2333 pmaddubsw m5, m2, m9 ; a1
2334 pmaddubsw m6, m2, m8 ; b0
2335 vpblendd m2, m0, m4, 0x30
2336 vpbroadcastq m0, [srcq+strideq*0]
2337 vpblendd m4, m0, 0x30
2338 punpcklbw m2, m4 ; 67 78
2339 pmaddubsw m1, m8 ; a0
2340 pmaddubsw m4, m3, m9 ; b1
2341 paddw m5, m1
2342 mova m1, m3
2343 pmaddubsw m3, m10 ; a2
2344 paddw m6, m4
2345 paddw m5, m3
2346 vpbroadcastq m4, [srcq+strideq*1]
2347 vpblendd m3, m0, m4, 0x30
2348 vpbroadcastq m0, [srcq+strideq*2]
2349 vpblendd m4, m0, 0x30
2350 punpcklbw m3, m4 ; 89 9a
2351 pmaddubsw m4, m2, m11 ; a3
2352 paddw m5, m4
2353 pmaddubsw m4, m2, m10 ; b2
2354 paddw m6, m4
2355 pmaddubsw m4, m3, m11 ; b3
2356 paddw m6, m4
2357 pmulhrsw m5, m7
2358 pmulhrsw m6, m7
2359 mova [tmpq+32*0], m5
2360 mova [tmpq+32*1], m6
2361 add tmpq, 32*2
2362 sub hd, 4
2363 jg .v_w8_loop
2364 RET
2365 .v_w16:
2366 add wd, wd
2367 mov r5, srcq
2368 mov r7, tmpq
2369 lea r6d, [hq+wq*8-256]
2370 .v_w16_loop0:
2371 vbroadcasti128 m4, [srcq+strideq*0]
2372 vbroadcasti128 m5, [srcq+strideq*1]
2373 lea srcq, [srcq+strideq*2]
2374 vbroadcasti128 m0, [srcq+strideq*1]
2375 vbroadcasti128 m6, [srcq+strideq*0]
2376 lea srcq, [srcq+strideq*2]
2377 vbroadcasti128 m1, [srcq+strideq*0]
2378 vbroadcasti128 m2, [srcq+strideq*1]
2379 lea srcq, [srcq+strideq*2]
2380 vbroadcasti128 m3, [srcq+strideq*0]
2381 shufpd m4, m4, m0, 0x0c
2382 shufpd m5, m5, m1, 0x0c
2383 punpcklbw m1, m4, m5 ; 01
2384 punpckhbw m4, m5 ; 34
2385 shufpd m6, m6, m2, 0x0c
2386 punpcklbw m2, m5, m6 ; 12
2387 punpckhbw m5, m6 ; 45
2388 shufpd m0, m0, m3, 0x0c
2389 punpcklbw m3, m6, m0 ; 23
2390 punpckhbw m6, m0 ; 56
2391 .v_w16_loop:
2392 vbroadcasti128 m12, [srcq+strideq*1]
2393 lea srcq, [srcq+strideq*2]
2394 vbroadcasti128 m13, [srcq+strideq*0]
2395 pmaddubsw m14, m1, m8 ; a0
2396 pmaddubsw m15, m2, m8 ; b0
2397 mova m1, m3
2398 mova m2, m4
2399 pmaddubsw m3, m9 ; a1
2400 pmaddubsw m4, m9 ; b1
2401 paddw m14, m3
2402 paddw m15, m4
2403 mova m3, m5
2404 mova m4, m6
2405 pmaddubsw m5, m10 ; a2
2406 pmaddubsw m6, m10 ; b2
2407 paddw m14, m5
2408 paddw m15, m6
2409 shufpd m6, m0, m12, 0x0d
2410 shufpd m0, m12, m13, 0x0c
2411 punpcklbw m5, m6, m0 ; 67
2412 punpckhbw m6, m0 ; 78
2413 pmaddubsw m12, m5, m11 ; a3
2414 pmaddubsw m13, m6, m11 ; b3
2415 paddw m14, m12
2416 paddw m15, m13
2417 pmulhrsw m14, m7
2418 pmulhrsw m15, m7
2419 mova [tmpq+wq*0], m14
2420 mova [tmpq+wq*1], m15
2421 lea tmpq, [tmpq+wq*2]
2422 sub hd, 2
2423 jg .v_w16_loop
2424 add r5, 16
2425 add r7, 32
2426 movzx hd, r6b
2427 mov srcq, r5
2428 mov tmpq, r7
2429 sub r6d, 1<<8
2430 jg .v_w16_loop0
2431 RET
2432 .hv:
2433 %assign stack_offset stack_offset - stack_size_padded
2434 %assign stack_size_padded 0
2435 WIN64_SPILL_XMM 16
2436 cmp wd, 4
2437 je .hv_w4
2438 shr mxd, 16
2439 sub srcq, 3
2440 vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0]
2441 vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4]
2442 movzx mxd, myb
2443 shr myd, 16
2444 cmp hd, 4
2445 cmove myd, mxd
2446 vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX]
2447 lea stride3q, [strideq*3]
2448 sub srcq, stride3q
2449 punpcklbw m0, m0
2450 psraw m0, 8 ; sign-extend
2451 pshufd m12, m0, q0000
2452 pshufd m13, m0, q1111
2453 pshufd m14, m0, q2222
2454 pshufd m15, m0, q3333
2455 jmp .hv_w8
2456 .hv_w4:
2457 movzx mxd, mxb
2458 dec srcq
2459 vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2]
2460 movzx mxd, myb
2461 shr myd, 16
2462 cmp hd, 4
2463 cmove myd, mxd
2464 vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX]
2465 lea stride3q, [strideq*3]
2466 sub srcq, stride3q
2467 mova m7, [subpel_h_shuf4]
2468 pmovzxbd m9, [deint_shuf4]
2469 vpbroadcastd m10, [pw_8192]
2470 punpcklbw m0, m0
2471 psraw m0, 8 ; sign-extend
2472 vpbroadcastd m11, [pd_32]
2473 pshufd m12, m0, q0000
2474 pshufd m13, m0, q1111
2475 pshufd m14, m0, q2222
2476 pshufd m15, m0, q3333
2477 vpbroadcastq m2, [srcq+strideq*0]
2478 vpbroadcastq m4, [srcq+strideq*1]
2479 vpbroadcastq m0, [srcq+strideq*2]
2480 vpbroadcastq m5, [srcq+stride3q ]
2481 lea srcq, [srcq+strideq*4]
2482 vpbroadcastq m3, [srcq+strideq*0]
2483 vpbroadcastq m6, [srcq+strideq*1]
2484 vpbroadcastq m1, [srcq+strideq*2]
2485 vpblendd m2, m4, 0xcc ; 0 1
2486 vpblendd m0, m5, 0xcc ; 2 3
2487 vpblendd m3, m6, 0xcc ; 4 5
2488 pshufb m2, m7 ; 00 01 10 11 02 03 12 13
2489 pshufb m0, m7 ; 20 21 30 31 22 23 32 33
2490 pshufb m3, m7 ; 40 41 50 51 42 43 52 53
2491 pshufb m1, m7 ; 60 61 60 61 62 63 62 63
2492 pmaddubsw m2, m8
2493 pmaddubsw m0, m8
2494 pmaddubsw m3, m8
2495 pmaddubsw m1, m8
2496 phaddw m2, m0 ; 0a 1a 2a 3a 0b 1b 2b 3b
2497 phaddw m3, m1 ; 4a 5a 6a __ 4b 5b 6b __
2498 pmulhrsw m2, m10
2499 pmulhrsw m3, m10
2500 palignr m4, m3, m2, 4 ; 1a 2a 3a 4a 1b 2b 3b 4b
2501 punpcklwd m1, m2, m4 ; 01 12
2502 punpckhwd m2, m4 ; 23 34
2503 pshufd m0, m3, q2121
2504 punpcklwd m3, m0 ; 45 56
2505 .hv_w4_loop:
2506 pmaddwd m5, m1, m12 ; a0 b0
2507 pmaddwd m6, m2, m12 ; c0 d0
2508 pmaddwd m2, m13 ; a1 b1
2509 pmaddwd m4, m3, m13 ; c1 d1
2510 mova m1, m3
2511 pmaddwd m3, m14 ; a2 b2
2512 paddd m5, m2
2513 vpbroadcastq m2, [srcq+stride3q ]
2514 lea srcq, [srcq+strideq*4]
2515 paddd m6, m4
2516 vpbroadcastq m4, [srcq+strideq*0]
2517 paddd m5, m3
2518 vpbroadcastq m3, [srcq+strideq*1]
2519 vpblendd m2, m4, 0xcc
2520 vpbroadcastq m4, [srcq+strideq*2]
2521 vpblendd m3, m4, 0xcc
2522 pshufb m2, m7
2523 pshufb m3, m7
2524 pmaddubsw m2, m8
2525 pmaddubsw m3, m8
2526 phaddw m2, m3
2527 pmulhrsw m2, m10
2528 palignr m3, m2, m0, 12
2529 mova m0, m2
2530 punpcklwd m2, m3, m0 ; 67 78
2531 punpckhwd m3, m0 ; 89 9a
2532 pmaddwd m4, m2, m14 ; c2 d2
2533 paddd m6, m11
2534 paddd m5, m11
2535 paddd m6, m4
2536 pmaddwd m4, m2, m15 ; a3 b3
2537 paddd m5, m4
2538 pmaddwd m4, m3, m15 ; c3 d3
2539 paddd m6, m4
2540 psrad m5, 6
2541 psrad m6, 6
2542 packssdw m5, m6
2543 vpermd m5, m9, m5
2544 mova [tmpq], m5
2545 add tmpq, 32
2546 sub hd, 4
2547 jg .hv_w4_loop
2548 RET
2549 .hv_w8:
2550 lea r6d, [wq*8-64]
2551 mov r5, srcq
2552 mov r7, tmpq
2553 lea r6d, [hq+r6*4]
2554 .hv_w8_loop0:
2555 vbroadcasti128 m7, [subpel_h_shufA]
2556 movu xm4, [srcq+strideq*0]
2557 vbroadcasti128 m8, [subpel_h_shufB]
2558 movu xm5, [srcq+strideq*1]
2559 lea srcq, [srcq+strideq*2]
2560 vbroadcasti128 m9, [subpel_h_shufC]
2561 movu xm6, [srcq+strideq*0]
2562 vbroadcasti128 m0, [srcq+strideq*1]
2563 lea srcq, [srcq+strideq*2]
2564 vpblendd m4, m0, 0xf0 ; 0 3
2565 vinserti128 m5, [srcq+strideq*0], 1 ; 1 4
2566 vinserti128 m6, [srcq+strideq*1], 1 ; 2 5
2567 lea srcq, [srcq+strideq*2]
2568 vinserti128 m0, [srcq+strideq*0], 1 ; 3 6
2569 HV_H_W8 m4, m1, m2, m3, m7, m8, m9
2570 HV_H_W8 m5, m1, m2, m3, m7, m8, m9
2571 HV_H_W8 m6, m1, m2, m3, m7, m8, m9
2572 HV_H_W8 m0, m1, m2, m3, m7, m8, m9
2573 vpbroadcastd m7, [pw_8192]
2574 vpermq m4, m4, q3120
2575 vpermq m5, m5, q3120
2576 vpermq m6, m6, q3120
2577 pmulhrsw m0, m7
2578 pmulhrsw m4, m7
2579 pmulhrsw m5, m7
2580 pmulhrsw m6, m7
2581 vpermq m7, m0, q3120
2582 punpcklwd m1, m4, m5 ; 01
2583 punpckhwd m4, m5 ; 34
2584 punpcklwd m2, m5, m6 ; 12
2585 punpckhwd m5, m6 ; 45
2586 punpcklwd m3, m6, m7 ; 23
2587 punpckhwd m6, m7 ; 56
2588 .hv_w8_loop:
2589 vextracti128 [tmpq], m0, 1 ; not enough registers
2590 movu xm0, [srcq+strideq*1]
2591 lea srcq, [srcq+strideq*2]
2592 vinserti128 m0, [srcq+strideq*0], 1 ; 7 8
2593 pmaddwd m8, m1, m12 ; a0
2594 pmaddwd m9, m2, m12 ; b0
2595 mova m1, m3
2596 mova m2, m4
2597 pmaddwd m3, m13 ; a1
2598 pmaddwd m4, m13 ; b1
2599 paddd m8, m3
2600 paddd m9, m4
2601 mova m3, m5
2602 mova m4, m6
2603 pmaddwd m5, m14 ; a2
2604 pmaddwd m6, m14 ; b2
2605 paddd m8, m5
2606 paddd m9, m6
2607 vbroadcasti128 m6, [subpel_h_shufB]
2608 vbroadcasti128 m7, [subpel_h_shufC]
2609 vbroadcasti128 m5, [subpel_h_shufA]
2610 HV_H_W8 m0, m5, m6, m7, m5, m6, m7
2611 vpbroadcastd m5, [pw_8192]
2612 vpbroadcastd m7, [pd_32]
2613 vbroadcasti128 m6, [tmpq]
2614 pmulhrsw m0, m5
2615 paddd m8, m7
2616 paddd m9, m7
2617 vpermq m7, m0, q3120 ; 7 8
2618 shufpd m6, m6, m7, 0x04 ; 6 7
2619 punpcklwd m5, m6, m7 ; 67
2620 punpckhwd m6, m7 ; 78
2621 pmaddwd m7, m5, m15 ; a3
2622 paddd m8, m7
2623 pmaddwd m7, m6, m15 ; b3
2624 paddd m7, m9
2625 psrad m8, 6
2626 psrad m7, 6
2627 packssdw m8, m7
2628 vpermq m7, m8, q3120
2629 mova [tmpq+wq*0], xm7
2630 vextracti128 [tmpq+wq*2], m7, 1
2631 lea tmpq, [tmpq+wq*4]
2632 sub hd, 2
2633 jg .hv_w8_loop
2634 add r5, 8
2635 add r7, 16
2636 movzx hd, r6b
2637 mov srcq, r5
2638 mov tmpq, r7
2639 sub r6d, 1<<8
2640 jg .hv_w8_loop0
2641 RET
2642
2643 %macro movifprep 2
2644 %if isprep
2645 mov %1, %2
2646 %endif
2647 %endmacro
2648
2649 %macro REMAP_REG 2
2650 %xdefine r%1 r%2
2651 %xdefine r%1q r%2q
2652 %xdefine r%1d r%2d
2653 %endmacro
2654
2655 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0
2656 %if isprep
2657 %xdefine r14_save r14
2658 %assign %%i 14
2659 %rep 14
2660 %assign %%j %%i-1
2661 REMAP_REG %%i, %%j
2662 %assign %%i %%i-1
2663 %endrep
2664 %endif
2665 %endmacro
2666
2667 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0
2668 %if isprep
2669 %assign %%i 1
2670 %rep 13
2671 %assign %%j %%i+1
2672 REMAP_REG %%i, %%j
2673 %assign %%i %%i+1
2674 %endrep
2675 %xdefine r14 r14_save
2676 %undef r14_save
2677 %endif
2678 %endmacro
2679
2680 %macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged
2681 MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
2682 RET
2683 %if %1
2684 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2685 %endif
2686 %endmacro
2687
2688 %macro MC_8TAP_SCALED_H 8 ; dst, tmp[0-6]
2689 movq xm%1, [srcq+ r4]
2690 movq xm%2, [srcq+ r6]
2691 movhps xm%1, [srcq+ r7]
2692 movhps xm%2, [srcq+ r9]
2693 vinserti128 m%1, [srcq+r10], 1
2694 vinserti128 m%2, [srcq+r11], 1
2695 vpbroadcastq m%5, [srcq+r13]
2696 vpbroadcastq m%6, [srcq+ rX]
2697 add srcq, ssq
2698 movq xm%3, [srcq+ r4]
2699 movq xm%4, [srcq+ r6]
2700 movhps xm%3, [srcq+ r7]
2701 movhps xm%4, [srcq+ r9]
2702 vinserti128 m%3, [srcq+r10], 1
2703 vinserti128 m%4, [srcq+r11], 1
2704 vpbroadcastq m%7, [srcq+r13]
2705 vpbroadcastq m%8, [srcq+ rX]
2706 add srcq, ssq
2707 vpblendd m%1, m%5, 0xc0
2708 vpblendd m%2, m%6, 0xc0
2709 vpblendd m%3, m%7, 0xc0
2710 vpblendd m%4, m%8, 0xc0
2711 pmaddubsw m%1, m15
2712 pmaddubsw m%2, m10
2713 pmaddubsw m%3, m15
2714 pmaddubsw m%4, m10
2715 phaddw m%1, m%2
2716 phaddw m%3, m%4
2717 phaddw m%1, m%3
2718 pmulhrsw m%1, m12
2719 %endmacro
2720
2721 %macro MC_8TAP_SCALED 1
2722 %ifidn %1, put
2723 %assign isprep 0
2724 cglobal put_8tap_scaled_8bpc, 4, 14, 16, 128, dst, ds, src, ss, w, h, mx, my, dx, dy
2725 %xdefine base_reg r12
2726 %define rndshift 10
2727 %else
2728 %assign isprep 1
2729 cglobal prep_8tap_scaled_8bpc, 4, 14, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy
2730 %define tmp_stridem qword [rsp+120]
2731 %xdefine base_reg r11
2732 %define rndshift 6
2733 %endif
2734 lea base_reg, [%1_8tap_scaled_8bpc_avx2]
2735 %define base base_reg-%1_8tap_scaled_8bpc_avx2
2736 tzcnt wd, wm
2737 vpbroadcastd m8, dxm
2738 %if isprep && UNIX64
2739 movd xm14, mxd
2740 vpbroadcastd m14, xm14
2741 mov r5d, t0d
2742 DECLARE_REG_TMP 5, 7
2743 %else
2744 vpbroadcastd m14, mxm
2745 %endif
2746 mov dyd, dym
2747 %ifidn %1, put
2748 %if WIN64
2749 mov r8d, hm
2750 DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3
2751 %define hm r5m
2752 %define dxm r8m
2753 %else
2754 DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3
2755 %define hm r6m
2756 %endif
2757 %define dsm [rsp+112]
2758 %define rX r1
2759 %define rXd r1d
2760 %else ; prep
2761 %if WIN64
2762 mov r7d, hm
2763 DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3
2764 %define hm r4m
2765 %define dxm r7m
2766 %else
2767 DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3
2768 %define hm [rsp+112]
2769 %endif
2770 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2771 %define rX r14
2772 %define rXd r14d
2773 %endif
2774 vpbroadcastd m10, [base+pd_0x3ff]
2775 vpbroadcastd m12, [base+pw_8192]
2776 %ifidn %1, put
2777 vpbroadcastd m13, [base+pd_512]
2778 %else
2779 vpbroadcastd m13, [base+pd_32]
2780 %endif
2781 pxor m9, m9
2782 lea ss3q, [ssq*3]
2783 movzx r7d, t1b
2784 shr t1d, 16
2785 cmp hd, 6
2786 cmovs t1d, r7d
2787 sub srcq, ss3q
2788 cmp dyd, 1024
2789 je .dy1
2790 cmp dyd, 2048
2791 je .dy2
2792 movzx wd, word [base+%1_8tap_scaled_avx2_table+wq*2]
2793 add wq, base_reg
2794 jmp wq
2795 %ifidn %1, put
2796 .w2:
2797 mov myd, mym
2798 movzx t0d, t0b
2799 dec srcq
2800 movd xm15, t0d
2801 punpckldq m8, m9, m8
2802 paddd m14, m8 ; mx+dx*[0,1]
2803 vpbroadcastd m11, [base+pd_0x4000]
2804 vpbroadcastd xm15, xm15
2805 pand m8, m14, m10
2806 psrld m8, 6
2807 paddd xm15, xm8
2808 movd r4d, xm15
2809 pextrd r6d, xm15, 1
2810 vbroadcasti128 m5, [base+bdct_lb_dw]
2811 vbroadcasti128 m6, [base+subpel_s_shuf2]
2812 vpbroadcastd m15, [base+subpel_filters+r4*8+2]
2813 vpbroadcastd m7, [base+subpel_filters+r6*8+2]
2814 pcmpeqd m8, m9
2815 psrld m14, 10
2816 movq xm0, [srcq+ssq*0]
2817 movq xm1, [srcq+ssq*2]
2818 movhps xm0, [srcq+ssq*1]
2819 movhps xm1, [srcq+ss3q ]
2820 lea srcq, [srcq+ssq*4]
2821 pshufb m14, m5
2822 paddb m14, m6
2823 vinserti128 m0, [srcq+ssq*0], 1
2824 vinserti128 m1, [srcq+ssq*2], 1
2825 vpbroadcastq m2, [srcq+ssq*1]
2826 vpbroadcastq m3, [srcq+ss3q ]
2827 lea srcq, [srcq+ssq*4]
2828 vpblendd m15, m7, 0xaa
2829 vpblendd m0, m2, 0xc0 ; 0 1 4 5
2830 vpblendd m1, m3, 0xc0 ; 2 3 6 7
2831 pblendvb m15, m11, m8
2832 pshufb m0, m14
2833 pshufb m1, m14
2834 pmaddubsw m0, m15
2835 pmaddubsw m1, m15
2836 phaddw m0, m1
2837 pmulhrsw m0, m12 ; 0 1 2 3 4 5 6 7
2838 vextracti128 xm1, m0, 1 ; 4 5 6 7
2839 palignr xm2, xm1, xm0, 4 ; 1 2 3 4
2840 punpcklwd xm3, xm0, xm2 ; 01 12
2841 punpckhwd xm0, xm2 ; 23 34
2842 pshufd xm4, xm1, q0321 ; 5 6 7 _
2843 punpcklwd xm2, xm1, xm4 ; 45 56
2844 punpckhwd xm4, xm1, xm4 ; 67 __
2845 .w2_loop:
2846 and myd, 0x3ff
2847 mov r6d, 64 << 24
2848 mov r4d, myd
2849 shr r4d, 6
2850 lea r4d, [t1+r4]
2851 cmovnz r6q, [base+subpel_filters+r4*8]
2852 movq xm11, r6q
2853 pmovsxbw xm11, xm11
2854 pshufd xm8, xm11, q0000
2855 pshufd xm9, xm11, q1111
2856 pshufd xm10, xm11, q2222
2857 pshufd xm11, xm11, q3333
2858 pmaddwd xm5, xm3, xm8
2859 pmaddwd xm6, xm0, xm9
2860 pmaddwd xm7, xm2, xm10
2861 pmaddwd xm8, xm4, xm11
2862 paddd xm5, xm6
2863 paddd xm7, xm8
2864 paddd xm5, xm13
2865 paddd xm5, xm7
2866 psrad xm5, 10
2867 packssdw xm5, xm5
2868 packuswb xm5, xm5
2869 pextrw [dstq], xm5, 0
2870 add dstq, dsq
2871 dec hd
2872 jz .ret
2873 add myd, dyd
2874 test myd, ~0x3ff
2875 jz .w2_loop
2876 movq xm5, [srcq]
2877 test myd, 0x400
2878 jz .w2_skip_line
2879 add srcq, ssq
2880 shufps xm3, xm0, q1032 ; 01 12
2881 shufps xm0, xm2, q1032 ; 23 34
2882 shufps xm2, xm4, q1032 ; 45 56
2883 pshufb xm5, xm14
2884 pmaddubsw xm5, xm15
2885 phaddw xm5, xm5
2886 pmulhrsw xm5, xm12
2887 palignr xm1, xm5, xm1, 12
2888 punpcklqdq xm1, xm1 ; 6 7 6 7
2889 punpcklwd xm4, xm1, xm5 ; 67 __
2890 jmp .w2_loop
2891 .w2_skip_line:
2892 movhps xm5, [srcq+ssq*1]
2893 lea srcq, [srcq+ssq*2]
2894 mova xm3, xm0 ; 01 12
2895 mova xm0, xm2 ; 23 34
2896 pshufb xm5, xm14
2897 pmaddubsw xm5, xm15
2898 phaddw xm5, xm5
2899 pmulhrsw xm5, xm12 ; 6 7 6 7
2900 palignr xm1, xm5, xm1, 8 ; 4 5 6 7
2901 pshufd xm5, xm1, q0321 ; 5 6 7 _
2902 punpcklwd xm2, xm1, xm5 ; 45 56
2903 punpckhwd xm4, xm1, xm5 ; 67 __
2904 jmp .w2_loop
2905 %endif
2906 .w4:
2907 mov myd, mym
2908 vbroadcasti128 m7, [base+rescale_mul]
2909 movzx t0d, t0b
2910 dec srcq
2911 movd xm15, t0d
2912 pmaddwd m8, m7
2913 vpbroadcastd m11, [base+pd_0x4000]
2914 vpbroadcastd xm15, xm15
2915 paddd m14, m8 ; mx+dx*[0-3]
2916 pand m0, m14, m10
2917 psrld m0, 6
2918 paddd xm15, xm0
2919 movd r4d, xm15
2920 pextrd r6d, xm15, 1
2921 pextrd r11d, xm15, 2
2922 pextrd r13d, xm15, 3
2923 movd xm15, [base+subpel_filters+r4*8+2]
2924 vbroadcasti128 m5, [base+bdct_lb_dw]
2925 vpbroadcastq m6, [base+subpel_s_shuf2]
2926 pinsrd xm15, [base+subpel_filters+r6*8+2], 1
2927 pcmpeqd m0, m9
2928 psrld m14, 10
2929 movu xm7, [srcq+ssq*0]
2930 movu xm9, [srcq+ssq*1]
2931 pinsrd xm15, [base+subpel_filters+r11*8+2], 2
2932 movu xm8, [srcq+ssq*2]
2933 movu xm10, [srcq+ss3q ]
2934 pinsrd xm15, [base+subpel_filters+r13*8+2], 3
2935 lea srcq, [srcq+ssq*4]
2936 pshufb m14, m5
2937 paddb m14, m6
2938 vinserti128 m7, [srcq+ssq*0], 1
2939 vinserti128 m9, [srcq+ssq*1], 1
2940 vinserti128 m15, xm15, 1
2941 vinserti128 m8, [srcq+ssq*2], 1
2942 vinserti128 m10, [srcq+ss3q ], 1
2943 lea srcq, [srcq+ssq*4]
2944 pblendvb m15, m11, m0
2945 pshufb m7, m14
2946 pshufb m9, m14
2947 pshufb m8, m14
2948 pshufb m10, m14
2949 pmaddubsw m7, m15
2950 pmaddubsw m9, m15
2951 pmaddubsw m8, m15
2952 pmaddubsw m10, m15
2953 phaddw m7, m9
2954 phaddw m8, m10
2955 pmulhrsw m7, m12 ; 0 1 4 5
2956 pmulhrsw m8, m12 ; 2 3 6 7
2957 vextracti128 xm9, m7, 1 ; 4 5
2958 vextracti128 xm3, m8, 1 ; 6 7
2959 shufps xm4, xm7, xm8, q1032 ; 1 2
2960 shufps xm5, xm8, xm9, q1032 ; 3 4
2961 shufps xm6, xm9, xm3, q1032 ; 5 6
2962 psrldq xm11, xm3, 8 ; 7 _
2963 punpcklwd xm0, xm7, xm4 ; 01
2964 punpckhwd xm7, xm4 ; 12
2965 punpcklwd xm1, xm8, xm5 ; 23
2966 punpckhwd xm8, xm5 ; 34
2967 punpcklwd xm2, xm9, xm6 ; 45
2968 punpckhwd xm9, xm6 ; 56
2969 punpcklwd xm3, xm11 ; 67
2970 mova [rsp+0x00], xm7
2971 mova [rsp+0x10], xm8
2972 mova [rsp+0x20], xm9
2973 .w4_loop:
2974 and myd, 0x3ff
2975 mov r6d, 64 << 24
2976 mov r4d, myd
2977 shr r4d, 6
2978 lea r4d, [t1+r4]
2979 cmovnz r6q, [base+subpel_filters+r4*8]
2980 movq xm10, r6q
2981 pmovsxbw xm10, xm10
2982 pshufd xm7, xm10, q0000
2983 pshufd xm8, xm10, q1111
2984 pshufd xm9, xm10, q2222
2985 pshufd xm10, xm10, q3333
2986 pmaddwd xm4, xm0, xm7
2987 pmaddwd xm5, xm1, xm8
2988 pmaddwd xm6, xm2, xm9
2989 pmaddwd xm7, xm3, xm10
2990 paddd xm4, xm5
2991 paddd xm6, xm7
2992 paddd xm4, xm13
2993 paddd xm4, xm6
2994 psrad xm4, rndshift
2995 packssdw xm4, xm4
2996 %ifidn %1, put
2997 packuswb xm4, xm4
2998 movd [dstq], xm4
2999 add dstq, dsq
3000 %else
3001 movq [tmpq], xm4
3002 add tmpq, 8
3003 %endif
3004 dec hd
3005 jz .ret
3006 add myd, dyd
3007 test myd, ~0x3ff
3008 jz .w4_loop
3009 movu xm4, [srcq]
3010 test myd, 0x400
3011 jz .w4_skip_line
3012 mova xm0, [rsp+0x00]
3013 mova [rsp+0x00], xm1
3014 mova xm1, [rsp+0x10]
3015 mova [rsp+0x10], xm2
3016 mova xm2, [rsp+0x20]
3017 mova [rsp+0x20], xm3
3018 pshufb xm4, xm14
3019 pmaddubsw xm4, xm15
3020 phaddw xm4, xm4
3021 pmulhrsw xm4, xm12
3022 punpcklwd xm3, xm11, xm4
3023 mova xm11, xm4
3024 add srcq, ssq
3025 jmp .w4_loop
3026 .w4_skip_line:
3027 movu xm5, [srcq+ssq*1]
3028 movu m6, [rsp+0x10]
3029 pshufb xm4, xm14
3030 pshufb xm5, xm14
3031 pmaddubsw xm4, xm15
3032 pmaddubsw xm5, xm15
3033 movu [rsp+0x00], m6
3034 phaddw xm4, xm5
3035 pmulhrsw xm4, xm12
3036 punpcklwd xm9, xm11, xm4
3037 mova [rsp+0x20], xm9
3038 psrldq xm11, xm4, 8
3039 mova xm0, xm1
3040 mova xm1, xm2
3041 mova xm2, xm3
3042 punpcklwd xm3, xm4, xm11
3043 lea srcq, [srcq+ssq*2]
3044 jmp .w4_loop
3045 .w8:
3046 mov dword [rsp+48], 1
3047 movifprep tmp_stridem, 16
3048 jmp .w_start
3049 .w16:
3050 mov dword [rsp+48], 2
3051 movifprep tmp_stridem, 32
3052 jmp .w_start
3053 .w32:
3054 mov dword [rsp+48], 4
3055 movifprep tmp_stridem, 64
3056 jmp .w_start
3057 .w64:
3058 mov dword [rsp+48], 8
3059 movifprep tmp_stridem, 128
3060 jmp .w_start
3061 .w128:
3062 mov dword [rsp+48], 16
3063 movifprep tmp_stridem, 256
3064 .w_start:
3065 %ifidn %1, put
3066 movifnidn dsm, dsq
3067 %endif
3068 shr t0d, 16
3069 sub srcq, 3
3070 pmaddwd m8, [base+rescale_mul]
3071 movd xm15, t0d
3072 mov [rsp+72], t0d
3073 mov [rsp+56], srcq
3074 mov [rsp+64], r0q ; dstq / tmpq
3075 %if UNIX64
3076 mov hm, hd
3077 %endif
3078 shl dword dxm, 3 ; dx*8
3079 vpbroadcastd m15, xm15
3080 paddd m14, m8 ; mx+dx*[0-7]
3081 jmp .hloop
3082 .hloop_prep:
3083 dec dword [rsp+48]
3084 jz .ret
3085 add qword [rsp+64], 8*(isprep+1)
3086 mov hd, hm
3087 vpbroadcastd m8, dxm
3088 vpbroadcastd m10, [base+pd_0x3ff]
3089 paddd m14, m8, [rsp+16]
3090 vpbroadcastd m15, [rsp+72]
3091 pxor m9, m9
3092 mov srcq, [rsp+56]
3093 mov r0q, [rsp+64] ; dstq / tmpq
3094 .hloop:
3095 vpbroadcastq m11, [base+pq_0x40000000]
3096 pand m6, m14, m10
3097 psrld m6, 6
3098 paddd m15, m6
3099 pcmpeqd m6, m9
3100 vextracti128 xm7, m15, 1
3101 movd r4d, xm15
3102 pextrd r6d, xm15, 2
3103 pextrd r7d, xm15, 1
3104 pextrd r9d, xm15, 3
3105 movd r10d, xm7
3106 pextrd r11d, xm7, 2
3107 pextrd r13d, xm7, 1
3108 pextrd rXd, xm7, 3
3109 movu [rsp+16], m14
3110 movq xm15, [base+subpel_filters+ r4*8]
3111 movq xm10, [base+subpel_filters+ r6*8]
3112 movhps xm15, [base+subpel_filters+ r7*8]
3113 movhps xm10, [base+subpel_filters+ r9*8]
3114 vinserti128 m15, [base+subpel_filters+r10*8], 1
3115 vinserti128 m10, [base+subpel_filters+r11*8], 1
3116 vpbroadcastq m9, [base+subpel_filters+r13*8]
3117 vpbroadcastq m8, [base+subpel_filters+ rX*8]
3118 psrld m14, 10
3119 vextracti128 xm7, m14, 1
3120 mova [rsp], xm14
3121 movd r4d, xm14
3122 pextrd r6d, xm14, 2
3123 pextrd r7d, xm14, 1
3124 pextrd r9d, xm14, 3
3125 movd r10d, xm7
3126 pextrd r11d, xm7, 2
3127 pextrd r13d, xm7, 1
3128 pextrd rXd, xm7, 3
3129 pshufd m5, m6, q1100
3130 pshufd m6, m6, q3322
3131 vpblendd m15, m9, 0xc0
3132 vpblendd m10, m8, 0xc0
3133 pblendvb m15, m11, m5
3134 pblendvb m10, m11, m6
3135 vbroadcasti128 m14, [base+subpel_s_shuf8]
3136 MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b
3137 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b
3138 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b
3139 MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b
3140 mov myd, mym
3141 mov dyd, dym
3142 pshufb m0, m14 ; 01a 01b
3143 pshufb m1, m14 ; 23a 23b
3144 pshufb m2, m14 ; 45a 45b
3145 pshufb m3, m14 ; 67a 67b
3146 vbroadcasti128 m14, [base+wswap]
3147 .vloop:
3148 and myd, 0x3ff
3149 mov r6d, 64 << 24
3150 mov r4d, myd
3151 shr r4d, 6
3152 lea r4d, [t1+r4]
3153 cmovnz r6q, [base+subpel_filters+r4*8]
3154 movq xm11, r6q
3155 punpcklqdq xm11, xm11
3156 pmovsxbw m11, xm11
3157 pshufd m8, m11, q0000
3158 pshufd m9, m11, q1111
3159 pmaddwd m4, m0, m8
3160 pmaddwd m5, m1, m9
3161 pshufd m8, m11, q2222
3162 pshufd m11, m11, q3333
3163 pmaddwd m6, m2, m8
3164 pmaddwd m7, m3, m11
3165 paddd m4, m5
3166 paddd m6, m7
3167 paddd m4, m13
3168 paddd m4, m6
3169 psrad m4, rndshift
3170 vextracti128 xm5, m4, 1
3171 packssdw xm4, xm5
3172 %ifidn %1, put
3173 packuswb xm4, xm4
3174 movq [dstq], xm4
3175 add dstq, dsm
3176 %else
3177 mova [tmpq], xm4
3178 add tmpq, tmp_stridem
3179 %endif
3180 dec hd
3181 jz .hloop_prep
3182 add myd, dyd
3183 test myd, ~0x3ff
3184 jz .vloop
3185 test myd, 0x400
3186 mov [rsp+52], myd
3187 mov r4d, [rsp+ 0]
3188 mov r6d, [rsp+ 8]
3189 mov r7d, [rsp+ 4]
3190 mov r9d, [rsp+12]
3191 jz .skip_line
3192 vpbroadcastq m6, [srcq+r13]
3193 vpbroadcastq m7, [srcq+ rX]
3194 movq xm4, [srcq+ r4]
3195 movq xm5, [srcq+ r6]
3196 movhps xm4, [srcq+ r7]
3197 movhps xm5, [srcq+ r9]
3198 vinserti128 m4, [srcq+r10], 1
3199 vinserti128 m5, [srcq+r11], 1
3200 add srcq, ssq
3201 mov myd, [rsp+52]
3202 mov dyd, dym
3203 pshufb m0, m14
3204 pshufb m1, m14
3205 pshufb m2, m14
3206 pshufb m3, m14
3207 vpblendd m4, m6, 0xc0
3208 vpblendd m5, m7, 0xc0
3209 pmaddubsw m4, m15
3210 pmaddubsw m5, m10
3211 phaddw m4, m5
3212 pslld m5, m4, 16
3213 paddw m4, m5
3214 pmulhrsw m4, m12
3215 pblendw m0, m1, 0xaa
3216 pblendw m1, m2, 0xaa
3217 pblendw m2, m3, 0xaa
3218 pblendw m3, m4, 0xaa
3219 jmp .vloop
3220 .skip_line:
3221 mova m0, m1
3222 mova m1, m2
3223 mova m2, m3
3224 vpbroadcastq m7, [srcq+r13]
3225 vpbroadcastq m8, [srcq+ rX]
3226 movq xm3, [srcq+ r4]
3227 movq xm4, [srcq+ r6]
3228 movhps xm3, [srcq+ r7]
3229 movhps xm4, [srcq+ r9]
3230 vinserti128 m3, [srcq+r10], 1
3231 vinserti128 m4, [srcq+r11], 1
3232 add srcq, ssq
3233 movq xm5, [srcq+ r4]
3234 movq xm6, [srcq+ r6]
3235 movhps xm5, [srcq+ r7]
3236 movhps xm6, [srcq+ r9]
3237 vinserti128 m5, [srcq+r10], 1
3238 vinserti128 m6, [srcq+r11], 1
3239 vpbroadcastq m9, [srcq+r13]
3240 vpbroadcastq m11, [srcq+ rX]
3241 add srcq, ssq
3242 mov myd, [rsp+52]
3243 mov dyd, dym
3244 vpblendd m3, m7, 0xc0
3245 vpblendd m4, m8, 0xc0
3246 vpblendd m5, m9, 0xc0
3247 vpblendd m6, m11, 0xc0
3248 pmaddubsw m3, m15
3249 pmaddubsw m4, m10
3250 pmaddubsw m5, m15
3251 pmaddubsw m6, m10
3252 phaddw m3, m4
3253 phaddw m5, m6
3254 psrld m4, m3, 16
3255 pslld m6, m5, 16
3256 paddw m3, m4
3257 paddw m5, m6
3258 pblendw m3, m5, 0xaa
3259 pmulhrsw m3, m12
3260 jmp .vloop
3261 .dy1:
3262 movzx wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2]
3263 add wq, base_reg
3264 jmp wq
3265 %ifidn %1, put
3266 .dy1_w2:
3267 mov myd, mym
3268 movzx t0d, t0b
3269 dec srcq
3270 movd xm15, t0d
3271 punpckldq m8, m9, m8
3272 paddd m14, m8 ; mx+dx*[0-1]
3273 vpbroadcastd m11, [base+pd_0x4000]
3274 vpbroadcastd xm15, xm15
3275 pand m8, m14, m10
3276 psrld m8, 6
3277 paddd xm15, xm8
3278 movd r4d, xm15
3279 pextrd r6d, xm15, 1
3280 vbroadcasti128 m5, [base+bdct_lb_dw]
3281 vbroadcasti128 m6, [base+subpel_s_shuf2]
3282 vpbroadcastd m15, [base+subpel_filters+r4*8+2]
3283 vpbroadcastd m7, [base+subpel_filters+r6*8+2]
3284 pcmpeqd m8, m9
3285 psrld m14, 10
3286 movq xm0, [srcq+ssq*0]
3287 movq xm1, [srcq+ssq*2]
3288 movhps xm0, [srcq+ssq*1]
3289 movhps xm1, [srcq+ss3q ]
3290 lea srcq, [srcq+ssq*4]
3291 shr myd, 6
3292 mov r4d, 64 << 24
3293 lea myd, [t1+myq]
3294 cmovnz r4q, [base+subpel_filters+myq*8]
3295 pshufb m14, m5
3296 paddb m14, m6
3297 vinserti128 m0, [srcq+ssq*0], 1
3298 vinserti128 m1, [srcq+ssq*2], 1
3299 vpbroadcastq m2, [srcq+ssq*1]
3300 add srcq, ss3q
3301 movq xm10, r4q
3302 pmovsxbw xm10, xm10
3303 vpblendd m15, m7, 0xaa
3304 pblendvb m15, m11, m8
3305 pshufd xm8, xm10, q0000
3306 pshufd xm9, xm10, q1111
3307 pshufd xm11, xm10, q3333
3308 pshufd xm10, xm10, q2222
3309 vpblendd m0, m2, 0xc0
3310 pshufb m1, m14
3311 pshufb m0, m14
3312 pmaddubsw m1, m15
3313 pmaddubsw m0, m15
3314 phaddw m0, m1
3315 pmulhrsw m0, m12
3316 vextracti128 xm1, m0, 1
3317 palignr xm2, xm1, xm0, 4
3318 pshufd xm4, xm1, q2121
3319 punpcklwd xm3, xm0, xm2 ; 01 12
3320 punpckhwd xm0, xm2 ; 23 34
3321 punpcklwd xm2, xm1, xm4 ; 45 56
3322 .dy1_w2_loop:
3323 movq xm1, [srcq+ssq*0]
3324 movhps xm1, [srcq+ssq*1]
3325 lea srcq, [srcq+ssq*2]
3326 pmaddwd xm5, xm3, xm8
3327 pmaddwd xm6, xm0, xm9
3328 pmaddwd xm7, xm2, xm10
3329 mova xm3, xm0
3330 mova xm0, xm2
3331 paddd xm5, xm13
3332 paddd xm6, xm7
3333 pshufb xm1, xm14
3334 pmaddubsw xm1, xm15
3335 phaddw xm1, xm1
3336 pmulhrsw xm1, xm12
3337 palignr xm7, xm1, xm4, 12
3338 punpcklwd xm2, xm7, xm1 ; 67 78
3339 pmaddwd xm7, xm2, xm11
3340 mova xm4, xm1
3341 paddd xm5, xm6
3342 paddd xm5, xm7
3343 psrad xm5, rndshift
3344 packssdw xm5, xm5
3345 packuswb xm5, xm5
3346 pextrw [dstq+dsq*0], xm5, 0
3347 pextrw [dstq+dsq*1], xm5, 1
3348 lea dstq, [dstq+dsq*2]
3349 sub hd, 2
3350 jg .dy1_w2_loop
3351 RET
3352 %endif
3353 .dy1_w4:
3354 mov myd, mym
3355 vbroadcasti128 m7, [base+rescale_mul]
3356 movzx t0d, t0b
3357 dec srcq
3358 movd xm15, t0d
3359 pmaddwd m8, m7
3360 vpbroadcastd m11, [base+pd_0x4000]
3361 vpbroadcastd xm15, xm15
3362 paddd m14, m8 ; mx+dx*[0-3]
3363 pand m8, m14, m10
3364 psrld m8, 6
3365 paddd xm15, xm8
3366 vpermq m8, m8, q3120
3367 movd r4d, xm15
3368 pextrd r6d, xm15, 2
3369 pextrd r11d, xm15, 1
3370 pextrd r13d, xm15, 3
3371 movd xm15, [base+subpel_filters+r4*8+2]
3372 vpbroadcastd m7, [base+subpel_filters+r6*8+2]
3373 movu xm2, [srcq+ssq*0]
3374 movu xm3, [srcq+ssq*2]
3375 vbroadcasti128 m5, [base+bdct_lb_dw]
3376 vpbroadcastq m6, [base+subpel_s_shuf2]
3377 pcmpeqd m8, m9
3378 psrld m14, 10
3379 pinsrd xm15, [base+subpel_filters+r11*8+2], 1
3380 vpblendd m7, [base+subpel_filters+r13*8+2-20], 0x20
3381 vinserti128 m2, [srcq+ssq*1], 1
3382 vinserti128 m3, [srcq+ss3q ], 1
3383 lea srcq, [srcq+ssq*4]
3384 shr myd, 6
3385 mov r4d, 64 << 24
3386 lea myd, [t1+myq]
3387 cmovnz r4q, [base+subpel_filters+myq*8]
3388 pshufb m14, m5
3389 paddb m14, m6
3390 movu xm4, [srcq+ssq*0]
3391 movu xm5, [srcq+ssq*2]
3392 vinserti128 m4, [srcq+ssq*1], 1
3393 add srcq, ss3q
3394 vpblendd m15, m7, 0x30
3395 punpcklqdq m15, m15
3396 pblendvb m15, m11, m8
3397 movq xm10, r4q
3398 punpcklqdq xm10, xm10
3399 pmovsxbw m10, xm10
3400 pshufb m2, m14
3401 pshufb m3, m14
3402 pshufb m4, m14
3403 pshufb xm5, xm14
3404 vpermq m2, m2, q3120
3405 vpermq m3, m3, q3120
3406 vpermq m4, m4, q3120
3407 vpermq m5, m5, q3120
3408 pshufd m7, m10, q0000
3409 pshufd m8, m10, q1111
3410 pshufd m9, m10, q2222
3411 pshufd m10, m10, q3333
3412 pmaddubsw m2, m15
3413 pmaddubsw m3, m15
3414 pmaddubsw m4, m15
3415 pmaddubsw m5, m15
3416 phaddw m2, m3
3417 phaddw m4, m5
3418 pmulhrsw m2, m12
3419 pmulhrsw m4, m12
3420 palignr m5, m4, m2, 4
3421 pshufd m3, m4, q2121
3422 punpcklwd m0, m2, m5 ; 01 12
3423 punpckhwd m1, m2, m5 ; 23 34
3424 punpcklwd m2, m4, m3 ; 45 56
3425 .dy1_w4_loop:
3426 movu xm11, [srcq+ssq*0]
3427 vinserti128 m11, [srcq+ssq*1], 1
3428 lea srcq, [srcq+ssq*2]
3429 pmaddwd m4, m0, m7
3430 pmaddwd m5, m1, m8
3431 pmaddwd m6, m2, m9
3432 mova m0, m1
3433 mova m1, m2
3434 paddd m4, m13
3435 paddd m5, m6
3436 pshufb m11, m14
3437 vpermq m11, m11, q3120
3438 pmaddubsw m11, m15
3439 phaddw m11, m11
3440 pmulhrsw m11, m12
3441 palignr m6, m11, m3, 12
3442 punpcklwd m2, m6, m11 ; 67 78
3443 mova m3, m11
3444 pmaddwd m6, m2, m10
3445 paddd m4, m5
3446 paddd m4, m6
3447 psrad m4, rndshift
3448 vextracti128 xm5, m4, 1
3449 packssdw xm4, xm5
3450 %ifidn %1, put
3451 packuswb xm4, xm4
3452 pshuflw xm4, xm4, q3120
3453 movd [dstq+dsq*0], xm4
3454 pextrd [dstq+dsq*1], xm4, 1
3455 lea dstq, [dstq+dsq*2]
3456 %else
3457 pshufd xm4, xm4, q3120
3458 mova [tmpq], xm4
3459 add tmpq, 16
3460 %endif
3461 sub hd, 2
3462 jg .dy1_w4_loop
3463 MC_8TAP_SCALED_RET
3464 .dy1_w8:
3465 mov dword [rsp+72], 1
3466 movifprep tmp_stridem, 16
3467 jmp .dy1_w_start
3468 .dy1_w16:
3469 mov dword [rsp+72], 2
3470 movifprep tmp_stridem, 32
3471 jmp .dy1_w_start
3472 .dy1_w32:
3473 mov dword [rsp+72], 4
3474 movifprep tmp_stridem, 64
3475 jmp .dy1_w_start
3476 .dy1_w64:
3477 mov dword [rsp+72], 8
3478 movifprep tmp_stridem, 128
3479 jmp .dy1_w_start
3480 .dy1_w128:
3481 mov dword [rsp+72], 16
3482 movifprep tmp_stridem, 256
3483 .dy1_w_start:
3484 mov myd, mym
3485 %ifidn %1, put
3486 movifnidn dsm, dsq
3487 %endif
3488 shr t0d, 16
3489 sub srcq, 3
3490 shr myd, 6
3491 mov r4d, 64 << 24
3492 lea myd, [t1+myq]
3493 cmovnz r4q, [base+subpel_filters+myq*8]
3494 pmaddwd m8, [base+rescale_mul]
3495 movd xm15, t0d
3496 mov [rsp+76], t0d
3497 mov [rsp+80], srcq
3498 mov [rsp+88], r0q ; dstq / tmpq
3499 %if UNIX64
3500 mov hm, hd
3501 %endif
3502 shl dword dxm, 3 ; dx*8
3503 vpbroadcastd m15, xm15
3504 paddd m14, m8 ; mx+dx*[0-7]
3505 movq xm0, r4q
3506 pmovsxbw xm0, xm0
3507 mova [rsp+96], xm0
3508 jmp .dy1_hloop
3509 .dy1_hloop_prep:
3510 dec dword [rsp+72]
3511 jz .ret
3512 add qword [rsp+88], 8*(isprep+1)
3513 mov hd, hm
3514 vpbroadcastd m8, dxm
3515 vpbroadcastd m10, [base+pd_0x3ff]
3516 paddd m14, m8, [rsp+32]
3517 vpbroadcastd m15, [rsp+76]
3518 pxor m9, m9
3519 mov srcq, [rsp+80]
3520 mov r0q, [rsp+88] ; dstq / tmpq
3521 .dy1_hloop:
3522 vpbroadcastq m11, [base+pq_0x40000000]
3523 pand m6, m14, m10
3524 psrld m6, 6
3525 paddd m15, m6
3526 pcmpeqd m6, m9
3527 vextracti128 xm7, m15, 1
3528 movd r4d, xm15
3529 pextrd r6d, xm15, 2
3530 pextrd r7d, xm15, 1
3531 pextrd r9d, xm15, 3
3532 movd r10d, xm7
3533 pextrd r11d, xm7, 2
3534 pextrd r13d, xm7, 1
3535 pextrd rXd, xm7, 3
3536 movu [rsp+32], m14
3537 movq xm15, [base+subpel_filters+ r4*8]
3538 movq xm10, [base+subpel_filters+ r6*8]
3539 movhps xm15, [base+subpel_filters+ r7*8]
3540 movhps xm10, [base+subpel_filters+ r9*8]
3541 vinserti128 m15, [base+subpel_filters+r10*8], 1
3542 vinserti128 m10, [base+subpel_filters+r11*8], 1
3543 vpbroadcastq m9, [base+subpel_filters+r13*8]
3544 vpbroadcastq m8, [base+subpel_filters+ rX*8]
3545 psrld m14, 10
3546 vextracti128 xm7, m14, 1
3547 movq [rsp+64], xm14
3548 movd r4d, xm14
3549 pextrd r6d, xm14, 2
3550 pextrd r7d, xm14, 1
3551 pextrd r9d, xm14, 3
3552 movd r10d, xm7
3553 pextrd r11d, xm7, 2
3554 pextrd r13d, xm7, 1
3555 pextrd rXd, xm7, 3
3556 pshufd m5, m6, q1100
3557 pshufd m6, m6, q3322
3558 vpblendd m15, m9, 0xc0
3559 vpblendd m10, m8, 0xc0
3560 pblendvb m15, m11, m5
3561 pblendvb m10, m11, m6
3562 vbroadcasti128 m14, [base+subpel_s_shuf8]
3563 MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b
3564 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b
3565 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b
3566 MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b
3567 movu [rsp], m10
3568 vpbroadcastd m8, [rsp+0x60]
3569 vpbroadcastd m9, [rsp+0x64]
3570 vpbroadcastd m10, [rsp+0x68]
3571 vpbroadcastd m11, [rsp+0x6c]
3572 pshufb m0, m14 ; 01a 01b
3573 pshufb m1, m14 ; 23a 23b
3574 pshufb m2, m14 ; 45a 45b
3575 pshufb m3, m14 ; 67a 67b
3576 vbroadcasti128 m14, [base+wswap]
3577 .dy1_vloop:
3578 pmaddwd m4, m0, m8
3579 pmaddwd m5, m1, m9
3580 pmaddwd m6, m2, m10
3581 pmaddwd m7, m3, m11
3582 paddd m4, m5
3583 paddd m6, m7
3584 paddd m4, m13
3585 paddd m4, m6
3586 psrad m4, rndshift
3587 vextracti128 xm5, m4, 1
3588 packssdw xm4, xm5
3589 %ifidn %1, put
3590 packuswb xm4, xm4
3591 movq [dstq], xm4
3592 add dstq, dsm
3593 %else
3594 mova [tmpq], xm4
3595 add tmpq, tmp_stridem
3596 %endif
3597 dec hd
3598 jz .dy1_hloop_prep
3599 movq xm4, [srcq+ r4]
3600 movq xm5, [srcq+ r6]
3601 movhps xm4, [srcq+ r7]
3602 movhps xm5, [srcq+ r9]
3603 vinserti128 m4, [srcq+r10], 1
3604 vinserti128 m5, [srcq+r11], 1
3605 vpbroadcastq m6, [srcq+r13]
3606 vpbroadcastq m7, [srcq+ rX]
3607 add srcq, ssq
3608 pshufb m0, m14
3609 pshufb m1, m14
3610 pshufb m2, m14
3611 pshufb m3, m14
3612 vpblendd m4, m6, 0xc0
3613 vpblendd m5, m7, 0xc0
3614 pmaddubsw m4, m15
3615 pmaddubsw m5, [rsp]
3616 phaddw m4, m5
3617 pslld m5, m4, 16
3618 paddw m4, m5
3619 pmulhrsw m4, m12
3620 pblendw m0, m1, 0xaa
3621 pblendw m1, m2, 0xaa
3622 pblendw m2, m3, 0xaa
3623 pblendw m3, m4, 0xaa
3624 jmp .dy1_vloop
3625 .dy2:
3626 movzx wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2]
3627 add wq, base_reg
3628 jmp wq
3629 %ifidn %1, put
3630 .dy2_w2:
3631 mov myd, mym
3632 movzx t0d, t0b
3633 dec srcq
3634 movd xm15, t0d
3635 punpckldq m8, m9, m8
3636 paddd m14, m8 ; mx+dx*[0-1]
3637 vpbroadcastd m11, [base+pd_0x4000]
3638 vpbroadcastd xm15, xm15
3639 pand m8, m14, m10
3640 psrld m8, 6
3641 paddd xm15, xm8
3642 movd r4d, xm15
3643 pextrd r6d, xm15, 1
3644 vbroadcasti128 m5, [base+bdct_lb_dw]
3645 vbroadcasti128 m6, [base+subpel_s_shuf2]
3646 vpbroadcastd m15, [base+subpel_filters+r4*8+2]
3647 vpbroadcastd m7, [base+subpel_filters+r6*8+2]
3648 pcmpeqd m8, m9
3649 psrld m14, 10
3650 movq xm0, [srcq+ssq*0]
3651 vpbroadcastq m2, [srcq+ssq*1]
3652 movhps xm0, [srcq+ssq*2]
3653 vpbroadcastq m3, [srcq+ss3q ]
3654 lea srcq, [srcq+ssq*4]
3655 pshufb m14, m5
3656 paddb m14, m6
3657 vpblendd m15, m7, 0xaa
3658 pblendvb m15, m11, m8
3659 movhps xm1, [srcq+ssq*0]
3660 vpbroadcastq m4, [srcq+ssq*1]
3661 lea srcq, [srcq+ssq*2]
3662 shr myd, 6
3663 mov r4d, 64 << 24
3664 lea myd, [t1+myq]
3665 cmovnz r4q, [base+subpel_filters+myq*8]
3666 vpblendd m0, m2, 0x30
3667 vpblendd m1, m4, 0xc0
3668 vpblendd m0, m3, 0xc0
3669 pshufb m0, m14
3670 pshufb m1, m14
3671 pmaddubsw m0, m15
3672 pmaddubsw m1, m15
3673 movq xm11, r4q
3674 pmovsxbw xm11, xm11
3675 phaddw m0, m1
3676 pmulhrsw m0, m12 ; 0 2 _ 4 1 3 _ 5
3677 pshufd xm8, xm11, q0000
3678 pshufd xm9, xm11, q1111
3679 pshufd xm10, xm11, q2222
3680 pshufd xm11, xm11, q3333
3681 pshufd m2, m0, q3110 ; 0 2 2 4 1 3 3 5
3682 vextracti128 xm1, m2, 1
3683 punpcklwd xm3, xm2, xm1 ; 01 23
3684 punpckhwd xm2, xm1 ; 23 45
3685 .dy2_w2_loop:
3686 movq xm6, [srcq+ssq*0]
3687 vpbroadcastq m7, [srcq+ssq*1]
3688 movhps xm6, [srcq+ssq*2]
3689 vpbroadcastq m1, [srcq+ss3q ]
3690 lea srcq, [srcq+ssq*4]
3691 pmaddwd xm4, xm3, xm8
3692 pmaddwd xm5, xm2, xm9
3693 vpblendd m6, m7, 0x30
3694 vpblendd m6, m1, 0xc0
3695 pshufb m6, m14
3696 pmaddubsw m6, m15
3697 phaddw m6, m6
3698 pmulhrsw m6, m12
3699 palignr m0, m6, m0, 8
3700 pshufd m2, m0, q3221
3701 vextracti128 xm1, m2, 1
3702 punpcklwd xm3, xm2, xm1 ; 45 67
3703 punpckhwd xm2, xm1 ; 67 89
3704 pmaddwd xm6, xm3, xm10
3705 pmaddwd xm7, xm2, xm11
3706 paddd xm4, xm5
3707 paddd xm4, xm13
3708 paddd xm6, xm7
3709 paddd xm4, xm6
3710 psrad xm4, rndshift
3711 packssdw xm4, xm4
3712 packuswb xm4, xm4
3713 pextrw [dstq+dsq*0], xm4, 0
3714 pextrw [dstq+dsq*1], xm4, 1
3715 lea dstq, [dstq+dsq*2]
3716 sub hd, 2
3717 jg .dy2_w2_loop
3718 RET
3719 %endif
3720 .dy2_w4:
3721 mov myd, mym
3722 vbroadcasti128 m7, [base+rescale_mul]
3723 movzx t0d, t0b
3724 dec srcq
3725 movd xm15, t0d
3726 pmaddwd m8, m7
3727 vpbroadcastd m11, [base+pd_0x4000]
3728 vpbroadcastd xm15, xm15
3729 paddd m14, m8 ; mx+dx*[0-3]
3730 pand m8, m14, m10
3731 psrld m8, 6
3732 paddd xm15, xm8
3733 movd r4d, xm15
3734 pextrd r6d, xm15, 1
3735 pextrd r11d, xm15, 2
3736 pextrd r13d, xm15, 3
3737 movd xm15, [base+subpel_filters+r4*8+2]
3738 vbroadcasti128 m5, [base+bdct_lb_dw]
3739 vpbroadcastq m6, [base+subpel_s_shuf2]
3740 pinsrd xm15, [base+subpel_filters+r6*8+2], 1
3741 pcmpeqd m8, m9
3742 psrld m14, 10
3743 movu xm0, [srcq+ssq*0]
3744 movu xm2, [srcq+ssq*2]
3745 pinsrd xm15, [base+subpel_filters+r11*8+2], 2
3746 movu xm1, [srcq+ssq*1]
3747 movu xm3, [srcq+ss3q ]
3748 pinsrd xm15, [base+subpel_filters+r13*8+2], 3
3749 lea srcq, [srcq+ssq*4]
3750 shr myd, 6
3751 mov r4d, 64 << 24
3752 lea myd, [t1+myq]
3753 cmovnz r4q, [base+subpel_filters+myq*8]
3754 vinserti128 m15, xm15, 1
3755 pshufb m14, m5
3756 paddb m14, m6
3757 vinserti128 m2, [srcq+ssq*0], 1
3758 vinserti128 m3, [srcq+ssq*1], 1
3759 lea srcq, [srcq+ssq*2]
3760 pblendvb m15, m11, m8
3761 pshufb xm0, xm14
3762 pshufb m2, m14
3763 pshufb xm1, xm14
3764 pshufb m3, m14
3765 pmaddubsw xm0, xm15
3766 pmaddubsw m2, m15
3767 pmaddubsw xm1, xm15
3768 pmaddubsw m3, m15
3769 movq xm11, r4q
3770 punpcklqdq xm11, xm11
3771 pmovsxbw m11, xm11
3772 phaddw m0, m2
3773 phaddw m1, m3
3774 pmulhrsw m0, m12 ; 0 2 _ 4
3775 pmulhrsw m1, m12 ; 1 3 _ 5
3776 pshufd m8, m11, q0000
3777 pshufd m9, m11, q1111
3778 pshufd m10, m11, q2222
3779 pshufd m11, m11, q3333
3780 punpcklwd xm2, xm0, xm1
3781 punpckhwd m1, m0, m1 ; 23 45
3782 vinserti128 m0, m2, xm1, 1 ; 01 23
3783 .dy2_w4_loop:
3784 movu xm6, [srcq+ssq*0]
3785 movu xm7, [srcq+ssq*1]
3786 vinserti128 m6, [srcq+ssq*2], 1
3787 vinserti128 m7, [srcq+ss3q ], 1
3788 lea srcq, [srcq+ssq*4]
3789 pmaddwd m4, m0, m8
3790 pmaddwd m5, m1, m9
3791 pshufb m6, m14
3792 pshufb m7, m14
3793 pmaddubsw m6, m15
3794 pmaddubsw m7, m15
3795 psrld m2, m6, 16
3796 pslld m3, m7, 16
3797 paddw m6, m2
3798 paddw m7, m3
3799 pblendw m6, m7, 0xaa ; 67 89
3800 pmulhrsw m6, m12
3801 paddd m4, m5
3802 vperm2i128 m0, m1, m6, 0x21 ; 45 67
3803 mova m1, m6
3804 pmaddwd m6, m0, m10
3805 pmaddwd m7, m1, m11
3806 paddd m4, m13
3807 paddd m6, m7
3808 paddd m4, m6
3809 psrad m4, rndshift
3810 vextracti128 xm5, m4, 1
3811 packssdw xm4, xm5
3812 %ifidn %1, put
3813 packuswb xm4, xm4
3814 movd [dstq+dsq*0], xm4
3815 pextrd [dstq+dsq*1], xm4, 1
3816 lea dstq, [dstq+dsq*2]
3817 %else
3818 mova [tmpq], xm4
3819 add tmpq, 16
3820 %endif
3821 sub hd, 2
3822 jg .dy2_w4_loop
3823 MC_8TAP_SCALED_RET
3824 .dy2_w8:
3825 mov dword [rsp+40], 1
3826 movifprep tmp_stridem, 16
3827 jmp .dy2_w_start
3828 .dy2_w16:
3829 mov dword [rsp+40], 2
3830 movifprep tmp_stridem, 32
3831 jmp .dy2_w_start
3832 .dy2_w32:
3833 mov dword [rsp+40], 4
3834 movifprep tmp_stridem, 64
3835 jmp .dy2_w_start
3836 .dy2_w64:
3837 mov dword [rsp+40], 8
3838 movifprep tmp_stridem, 128
3839 jmp .dy2_w_start
3840 .dy2_w128:
3841 mov dword [rsp+40], 16
3842 movifprep tmp_stridem, 256
3843 .dy2_w_start:
3844 mov myd, mym
3845 %ifidn %1, put
3846 movifnidn dsm, dsq
3847 %endif
3848 shr t0d, 16
3849 sub srcq, 3
3850 shr myd, 6
3851 mov r4d, 64 << 24
3852 lea myd, [t1+myq]
3853 cmovnz r4q, [base+subpel_filters+myq*8]
3854 pmaddwd m8, [base+rescale_mul]
3855 movd xm15, t0d
3856 mov [rsp+64], t0d
3857 mov [rsp+48], srcq
3858 mov [rsp+56], r0q ; dstq / tmpq
3859 %if UNIX64
3860 mov hm, hd
3861 %endif
3862 shl dword dxm, 3 ; dx*8
3863 vpbroadcastd m15, xm15
3864 paddd m14, m8 ; mx+dx*[0-7]
3865 movq xm0, r4q
3866 pmovsxbw xm0, xm0
3867 mova [rsp+0x50], xm0
3868 jmp .dy2_hloop
3869 .dy2_hloop_prep:
3870 dec dword [rsp+40]
3871 jz .ret
3872 add qword [rsp+56], 8*(isprep+1)
3873 mov hd, hm
3874 vpbroadcastd m8, dxm
3875 vpbroadcastd m10, [base+pd_0x3ff]
3876 paddd m14, m8, [rsp]
3877 vpbroadcastd m15, [rsp+64]
3878 pxor m9, m9
3879 mov srcq, [rsp+48]
3880 mov r0q, [rsp+56] ; dstq / tmpq
3881 .dy2_hloop:
3882 vpbroadcastq m11, [base+pq_0x40000000]
3883 pand m6, m14, m10
3884 psrld m6, 6
3885 paddd m15, m6
3886 pcmpeqd m6, m9
3887 vextracti128 xm7, m15, 1
3888 movd r4d, xm15
3889 pextrd r6d, xm15, 2
3890 pextrd r7d, xm15, 1
3891 pextrd r9d, xm15, 3
3892 movd r10d, xm7
3893 pextrd r11d, xm7, 2
3894 pextrd r13d, xm7, 1
3895 pextrd rXd, xm7, 3
3896 movu [rsp], m14
3897 movq xm15, [base+subpel_filters+ r4*8]
3898 movq xm10, [base+subpel_filters+ r6*8]
3899 movhps xm15, [base+subpel_filters+ r7*8]
3900 movhps xm10, [base+subpel_filters+ r9*8]
3901 vinserti128 m15, [base+subpel_filters+r10*8], 1
3902 vinserti128 m10, [base+subpel_filters+r11*8], 1
3903 vpbroadcastq m9, [base+subpel_filters+r13*8]
3904 vpbroadcastq m8, [base+subpel_filters+ rX*8]
3905 psrld m14, 10
3906 vextracti128 xm7, m14, 1
3907 movd r4d, xm14
3908 pextrd r6d, xm14, 2
3909 pextrd r7d, xm14, 1
3910 pextrd r9d, xm14, 3
3911 movd r10d, xm7
3912 pextrd r11d, xm7, 2
3913 pextrd r13d, xm7, 1
3914 pextrd rXd, xm7, 3
3915 pshufd m5, m6, q1100
3916 pshufd m6, m6, q3322
3917 vpblendd m15, m9, 0xc0
3918 vpblendd m10, m8, 0xc0
3919 pblendvb m15, m11, m5
3920 pblendvb m10, m11, m6
3921 vbroadcasti128 m14, [base+subpel_s_shuf8]
3922 MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b
3923 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b
3924 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b
3925 MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b
3926 vpbroadcastd m8, [rsp+0x50]
3927 vpbroadcastd m9, [rsp+0x54]
3928 vpbroadcastd m11, [rsp+0x58]
3929 vpbroadcastd m4, [rsp+0x5c]
3930 pshufb m0, m14 ; 01a 01b
3931 pshufb m1, m14 ; 23a 23b
3932 pshufb m2, m14 ; 45a 45b
3933 pshufb m3, m14 ; 67a 67b
3934 SWAP m14, m4
3935 .dy2_vloop:
3936 pmaddwd m4, m0, m8
3937 pmaddwd m5, m1, m9
3938 pmaddwd m6, m2, m11
3939 pmaddwd m7, m3, m14
3940 paddd m4, m5
3941 paddd m6, m7
3942 paddd m4, m13
3943 paddd m4, m6
3944 psrad m4, rndshift
3945 vextracti128 xm5, m4, 1
3946 packssdw xm4, xm5
3947 %ifidn %1, put
3948 packuswb xm4, xm4
3949 movq [dstq], xm4
3950 add dstq, dsm
3951 %else
3952 mova [tmpq], xm4
3953 add tmpq, tmp_stridem
3954 %endif
3955 dec hd
3956 jz .dy2_hloop_prep
3957 mova m0, m1
3958 mova m1, m2
3959 mova m2, m3
3960 movq xm3, [srcq+ r4]
3961 movq xm4, [srcq+ r6]
3962 movhps xm3, [srcq+ r7]
3963 movhps xm4, [srcq+ r9]
3964 vinserti128 m3, [srcq+r10], 1
3965 vinserti128 m4, [srcq+r11], 1
3966 vpbroadcastq m5, [srcq+r13]
3967 vpbroadcastq m6, [srcq+ rX]
3968 add srcq, ssq
3969 vpblendd m3, m5, 0xc0
3970 vpblendd m4, m6, 0xc0
3971 pmaddubsw m3, m15
3972 pmaddubsw m4, m10
3973 phaddw m3, m4
3974 movq xm4, [srcq+ r4]
3975 movq xm5, [srcq+ r6]
3976 movhps xm4, [srcq+ r7]
3977 movhps xm5, [srcq+ r9]
3978 vinserti128 m4, [srcq+r10], 1
3979 vinserti128 m5, [srcq+r11], 1
3980 vpbroadcastq m6, [srcq+r13]
3981 vpbroadcastq m7, [srcq+ rX]
3982 add srcq, ssq
3983 vpblendd m4, m6, 0xc0
3984 vpblendd m5, m7, 0xc0
3985 pmaddubsw m4, m15
3986 pmaddubsw m5, m10
3987 phaddw m4, m5
3988 psrld m5, m3, 16
3989 pslld m6, m4, 16
3990 paddw m3, m5
3991 paddw m4, m6
3992 pblendw m3, m4, 0xaa
3993 pmulhrsw m3, m12
3994 jmp .dy2_vloop
3995 .ret:
3996 MC_8TAP_SCALED_RET 0
3997 %undef isprep
3998 %endmacro
3999
4000 %macro BILIN_SCALED_FN 1
4001 cglobal %1_bilin_scaled_8bpc
4002 mov t0d, (5*15 << 16) | 5*15
4003 mov t1d, t0d
4004 jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX)
4005 %endmacro
4006
4007 %if WIN64
4008 DECLARE_REG_TMP 6, 5
4009 %else
4010 DECLARE_REG_TMP 6, 8
4011 %endif
4012
4013 %define PUT_8TAP_SCALED_FN FN put_8tap_scaled,
4014 %define PREP_8TAP_SCALED_FN FN prep_8tap_scaled,
4015
4016 BILIN_SCALED_FN put
4017 PUT_8TAP_SCALED_FN sharp, SHARP, SHARP
4018 PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH
4019 PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP
4020 PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH
4021 PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR
4022 PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP
4023 PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR
4024 PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH
4025 PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR
4026 MC_8TAP_SCALED put
4027
4028 %if WIN64
4029 DECLARE_REG_TMP 5, 4
4030 %else
4031 DECLARE_REG_TMP 6, 7
4032 %endif
4033
4034 BILIN_SCALED_FN prep
4035 PREP_8TAP_SCALED_FN sharp, SHARP, SHARP
4036 PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH
4037 PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP
4038 PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH
4039 PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR
4040 PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP
4041 PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR
4042 PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH
4043 PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR
4044 MC_8TAP_SCALED prep
4045
4046 %macro WARP_V 5 ; dst, 02, 46, 13, 57
4047 ; Can be done using gathers, but that's terribly slow on many CPU:s
4048 lea tmp1d, [myq+deltaq*4]
4049 lea tmp2d, [myq+deltaq*1]
4050 shr myd, 10
4051 shr tmp1d, 10
4052 movq xm8, [filterq+myq *8]
4053 vinserti128 m8, [filterq+tmp1q*8], 1 ; a e
4054 lea tmp1d, [tmp2q+deltaq*4]
4055 lea myd, [tmp2q+deltaq*1]
4056 shr tmp2d, 10
4057 shr tmp1d, 10
4058 movq xm0, [filterq+tmp2q*8]
4059 vinserti128 m0, [filterq+tmp1q*8], 1 ; b f
4060 lea tmp1d, [myq+deltaq*4]
4061 lea tmp2d, [myq+deltaq*1]
4062 shr myd, 10
4063 shr tmp1d, 10
4064 movq xm9, [filterq+myq *8]
4065 vinserti128 m9, [filterq+tmp1q*8], 1 ; c g
4066 lea tmp1d, [tmp2q+deltaq*4]
4067 lea myd, [tmp2q+gammaq] ; my += gamma
4068 shr tmp2d, 10
4069 shr tmp1d, 10
4070 punpcklwd m8, m0
4071 movq xm0, [filterq+tmp2q*8]
4072 vinserti128 m0, [filterq+tmp1q*8], 1 ; d h
4073 punpcklwd m0, m9, m0
4074 punpckldq m9, m8, m0
4075 punpckhdq m0, m8, m0
4076 punpcklbw m8, m11, m9 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8
4077 punpckhbw m9, m11, m9 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8
4078 pmaddwd m%2, m8
4079 pmaddwd m9, m%3
4080 punpcklbw m8, m11, m0 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8
4081 punpckhbw m0, m11, m0 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8
4082 pmaddwd m8, m%4
4083 pmaddwd m0, m%5
4084 paddd m%2, m9
4085 paddd m0, m8
4086 paddd m%1, m0, m%2
4087 %endmacro
4088
4089 cglobal warp_affine_8x8t_8bpc, 0, 14, 0, tmp, ts
4090 %if WIN64
4091 sub rsp, 0xa0
4092 %endif
4093 call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main
4094 .loop:
4095 psrad m7, 13
4096 psrad m0, 13
4097 packssdw m7, m0
4098 pmulhrsw m7, m14 ; (x + (1 << 6)) >> 7
4099 vpermq m7, m7, q3120
4100 mova [tmpq+tsq*0], xm7
4101 vextracti128 [tmpq+tsq*2], m7, 1
4102 dec r4d
4103 jz mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).end
4104 call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main2
4105 lea tmpq, [tmpq+tsq*4]
4106 jmp .loop
4107
4108 cglobal warp_affine_8x8_8bpc, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \
4109 beta, filter, tmp1, delta, my, gamma
4110 %if WIN64
4111 sub rsp, 0xa0
4112 %assign xmm_regs_used 16
4113 %assign stack_size_padded 0xa0
4114 %assign stack_offset stack_offset+stack_size_padded
4115 %endif
4116 call .main
4117 jmp .start
4118 .loop:
4119 call .main2
4120 lea dstq, [dstq+dsq*2]
4121 .start:
4122 psrad m7, 18
4123 psrad m0, 18
4124 packusdw m7, m0
4125 pavgw m7, m11 ; (x + (1 << 10)) >> 11
4126 vextracti128 xm0, m7, 1
4127 packuswb xm7, xm0
4128 pshufd xm7, xm7, q3120
4129 movq [dstq+dsq*0], xm7
4130 movhps [dstq+dsq*1], xm7
4131 dec r4d
4132 jg .loop
4133 .end:
4134 RET
4135 ALIGN function_align
4136 .main:
4137 ; Stack args offset by one (r4m -> r5m etc.) due to call
4138 %if WIN64
4139 mov abcdq, r5m
4140 mov mxd, r6m
4141 movaps [rsp+stack_offset+0x10], xmm6
4142 movaps [rsp+stack_offset+0x20], xmm7
4143 movaps [rsp+0x28], xmm8
4144 movaps [rsp+0x38], xmm9
4145 movaps [rsp+0x48], xmm10
4146 movaps [rsp+0x58], xmm11
4147 movaps [rsp+0x68], xmm12
4148 movaps [rsp+0x78], xmm13
4149 movaps [rsp+0x88], xmm14
4150 movaps [rsp+0x98], xmm15
4151 %endif
4152 movsx alphad, word [abcdq+2*0]
4153 movsx betad, word [abcdq+2*1]
4154 mova m12, [warp_8x8_shufA]
4155 mova m13, [warp_8x8_shufB]
4156 vpbroadcastd m14, [pw_8192]
4157 vpbroadcastd m15, [pd_32768]
4158 pxor m11, m11
4159 lea filterq, [mc_warp_filter2]
4160 lea tmp1q, [ssq*3+3]
4161 add mxd, 512+(64<<10)
4162 lea tmp2d, [alphaq*3]
4163 sub srcq, tmp1q ; src -= src_stride*3 + 3
4164 sub betad, tmp2d ; beta -= alpha*3
4165 mov myd, r7m
4166 call .h
4167 psrld m1, m0, 16
4168 call .h
4169 psrld m4, m0, 16
4170 call .h
4171 pblendw m1, m0, 0xaa ; 02
4172 call .h
4173 pblendw m4, m0, 0xaa ; 13
4174 call .h
4175 psrld m2, m1, 16
4176 pblendw m2, m0, 0xaa ; 24
4177 call .h
4178 psrld m5, m4, 16
4179 pblendw m5, m0, 0xaa ; 35
4180 call .h
4181 psrld m3, m2, 16
4182 pblendw m3, m0, 0xaa ; 46
4183 movsx deltad, word [abcdq+2*2]
4184 movsx gammad, word [abcdq+2*3]
4185 add myd, 512+(64<<10)
4186 mov r4d, 4
4187 lea tmp1d, [deltaq*3]
4188 sub gammad, tmp1d ; gamma -= delta*3
4189 .main2:
4190 call .h
4191 psrld m6, m5, 16
4192 pblendw m6, m0, 0xaa ; 57
4193 WARP_V 7, 1, 3, 4, 6
4194 call .h
4195 mova m1, m2
4196 mova m2, m3
4197 psrld m3, 16
4198 pblendw m3, m0, 0xaa ; 68
4199 WARP_V 0, 4, 6, 1, 3
4200 mova m4, m5
4201 mova m5, m6
4202 ret
4203 ALIGN function_align
4204 .h:
4205 lea tmp1d, [mxq+alphaq*4]
4206 lea tmp2d, [mxq+alphaq*1]
4207 vbroadcasti128 m10, [srcq]
4208 shr mxd, 10
4209 shr tmp1d, 10
4210 movq xm8, [filterq+mxq *8]
4211 vinserti128 m8, [filterq+tmp1q*8], 1
4212 lea tmp1d, [tmp2q+alphaq*4]
4213 lea mxd, [tmp2q+alphaq*1]
4214 shr tmp2d, 10
4215 shr tmp1d, 10
4216 movq xm0, [filterq+tmp2q*8]
4217 vinserti128 m0, [filterq+tmp1q*8], 1
4218 lea tmp1d, [mxq+alphaq*4]
4219 lea tmp2d, [mxq+alphaq*1]
4220 shr mxd, 10
4221 shr tmp1d, 10
4222 movq xm9, [filterq+mxq *8]
4223 vinserti128 m9, [filterq+tmp1q*8], 1
4224 lea tmp1d, [tmp2q+alphaq*4]
4225 lea mxd, [tmp2q+betaq] ; mx += beta
4226 shr tmp2d, 10
4227 shr tmp1d, 10
4228 punpcklqdq m8, m0 ; 0 1 4 5
4229 movq xm0, [filterq+tmp2q*8]
4230 vinserti128 m0, [filterq+tmp1q*8], 1
4231 punpcklqdq m9, m0 ; 2 3 6 7
4232 pshufb m0, m10, m12
4233 pmaddubsw m0, m8
4234 pshufb m10, m13
4235 pmaddubsw m10, m9
4236 add srcq, ssq
4237 phaddw m0, m10
4238 pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13
4239 paddd m0, m15 ; rounded 14-bit result in upper 16 bits of dword
4240 ret
4241
4242 %macro BIDIR_FN 1 ; op
4243 %1 0
4244 lea stride3q, [strideq*3]
4245 jmp wq
4246 .w4:
4247 vextracti128 xm1, m0, 1
4248 movd [dstq ], xm0
4249 pextrd [dstq+strideq*1], xm0, 1
4250 movd [dstq+strideq*2], xm1
4251 pextrd [dstq+stride3q ], xm1, 1
4252 cmp hd, 4
4253 je .ret
4254 lea dstq, [dstq+strideq*4]
4255 pextrd [dstq ], xm0, 2
4256 pextrd [dstq+strideq*1], xm0, 3
4257 pextrd [dstq+strideq*2], xm1, 2
4258 pextrd [dstq+stride3q ], xm1, 3
4259 cmp hd, 8
4260 je .ret
4261 %1 2
4262 lea dstq, [dstq+strideq*4]
4263 vextracti128 xm1, m0, 1
4264 movd [dstq ], xm0
4265 pextrd [dstq+strideq*1], xm0, 1
4266 movd [dstq+strideq*2], xm1
4267 pextrd [dstq+stride3q ], xm1, 1
4268 lea dstq, [dstq+strideq*4]
4269 pextrd [dstq ], xm0, 2
4270 pextrd [dstq+strideq*1], xm0, 3
4271 pextrd [dstq+strideq*2], xm1, 2
4272 pextrd [dstq+stride3q ], xm1, 3
4273 .ret:
4274 RET
4275 .w8_loop:
4276 %1_INC_PTR 2
4277 %1 0
4278 lea dstq, [dstq+strideq*4]
4279 .w8:
4280 vextracti128 xm1, m0, 1
4281 movq [dstq ], xm0
4282 movq [dstq+strideq*1], xm1
4283 movhps [dstq+strideq*2], xm0
4284 movhps [dstq+stride3q ], xm1
4285 sub hd, 4
4286 jg .w8_loop
4287 RET
4288 .w16_loop:
4289 %1_INC_PTR 4
4290 %1 0
4291 lea dstq, [dstq+strideq*4]
4292 .w16:
4293 vpermq m0, m0, q3120
4294 mova [dstq ], xm0
4295 vextracti128 [dstq+strideq*1], m0, 1
4296 %1 2
4297 vpermq m0, m0, q3120
4298 mova [dstq+strideq*2], xm0
4299 vextracti128 [dstq+stride3q ], m0, 1
4300 sub hd, 4
4301 jg .w16_loop
4302 RET
4303 .w32_loop:
4304 %1_INC_PTR 4
4305 %1 0
4306 lea dstq, [dstq+strideq*2]
4307 .w32:
4308 vpermq m0, m0, q3120
4309 mova [dstq+strideq*0], m0
4310 %1 2
4311 vpermq m0, m0, q3120
4312 mova [dstq+strideq*1], m0
4313 sub hd, 2
4314 jg .w32_loop
4315 RET
4316 .w64_loop:
4317 %1_INC_PTR 4
4318 %1 0
4319 add dstq, strideq
4320 .w64:
4321 vpermq m0, m0, q3120
4322 mova [dstq], m0
4323 %1 2
4324 vpermq m0, m0, q3120
4325 mova [dstq+32], m0
4326 dec hd
4327 jg .w64_loop
4328 RET
4329 .w128_loop:
4330 %1 0
4331 add dstq, strideq
4332 .w128:
4333 vpermq m0, m0, q3120
4334 mova [dstq+0*32], m0
4335 %1 2
4336 vpermq m0, m0, q3120
4337 mova [dstq+1*32], m0
4338 %1_INC_PTR 8
4339 %1 -4
4340 vpermq m0, m0, q3120
4341 mova [dstq+2*32], m0
4342 %1 -2
4343 vpermq m0, m0, q3120
4344 mova [dstq+3*32], m0
4345 dec hd
4346 jg .w128_loop
4347 RET
4348 %endmacro
4349
4350 %macro AVG 1 ; src_offset
4351 mova m0, [tmp1q+(%1+0)*32]
4352 paddw m0, [tmp2q+(%1+0)*32]
4353 mova m1, [tmp1q+(%1+1)*32]
4354 paddw m1, [tmp2q+(%1+1)*32]
4355 pmulhrsw m0, m2
4356 pmulhrsw m1, m2
4357 packuswb m0, m1
4358 %endmacro
4359
4360 %macro AVG_INC_PTR 1
4361 add tmp1q, %1*32
4362 add tmp2q, %1*32
4363 %endmacro
4364
4365 cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3
4366 %define base r6-avg %+ SUFFIX %+ _table
4367 lea r6, [avg %+ SUFFIX %+ _table]
4368 tzcnt wd, wm
4369 movifnidn hd, hm
4370 movsxd wq, dword [r6+wq*4]
4371 vpbroadcastd m2, [base+pw_1024]
4372 add wq, r6
4373 BIDIR_FN AVG
4374
4375 %macro W_AVG 1 ; src_offset
4376 ; (a * weight + b * (16 - weight) + 128) >> 8
4377 ; = ((a - b) * weight + (b << 4) + 128) >> 8
4378 ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4
4379 ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4
4380 mova m0, [tmp1q+(%1+0)*32]
4381 psubw m2, m0, [tmp2q+(%1+0)*32]
4382 mova m1, [tmp1q+(%1+1)*32]
4383 psubw m3, m1, [tmp2q+(%1+1)*32]
4384 pmulhw m2, m4
4385 pmulhw m3, m4
4386 paddw m0, m2
4387 paddw m1, m3
4388 pmulhrsw m0, m5
4389 pmulhrsw m1, m5
4390 packuswb m0, m1
4391 %endmacro
4392
4393 %define W_AVG_INC_PTR AVG_INC_PTR
4394
4395 cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
4396 %define base r6-w_avg %+ SUFFIX %+ _table
4397 lea r6, [w_avg %+ SUFFIX %+ _table]
4398 tzcnt wd, wm
4399 movifnidn hd, hm
4400 vpbroadcastw m4, r6m ; weight
4401 movsxd wq, dword [r6+wq*4]
4402 vpbroadcastd m5, [base+pw_2048]
4403 psllw m4, 12 ; (weight-16) << 12 when interpreted as signed
4404 add wq, r6
4405 cmp dword r6m, 7
4406 jg .weight_gt7
4407 mov r6, tmp1q
4408 pxor m0, m0
4409 mov tmp1q, tmp2q
4410 psubw m4, m0, m4 ; -weight
4411 mov tmp2q, r6
4412 .weight_gt7:
4413 BIDIR_FN W_AVG
4414
4415 %macro MASK 1 ; src_offset
4416 ; (a * m + b * (64 - m) + 512) >> 10
4417 ; = ((a - b) * m + (b << 6) + 512) >> 10
4418 ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4
4419 vpermq m3, [maskq+%1*16], q3120
4420 mova m0, [tmp2q+(%1+0)*32]
4421 psubw m1, m0, [tmp1q+(%1+0)*32]
4422 psubb m3, m4, m3
4423 paddw m1, m1 ; (b - a) << 1
4424 paddb m3, m3
4425 punpcklbw m2, m4, m3 ; -m << 9
4426 pmulhw m1, m2
4427 paddw m0, m1
4428 mova m1, [tmp2q+(%1+1)*32]
4429 psubw m2, m1, [tmp1q+(%1+1)*32]
4430 paddw m2, m2
4431 punpckhbw m3, m4, m3
4432 pmulhw m2, m3
4433 paddw m1, m2
4434 pmulhrsw m0, m5
4435 pmulhrsw m1, m5
4436 packuswb m0, m1
4437 %endmacro
4438
4439 %macro MASK_INC_PTR 1
4440 add maskq, %1*16
4441 add tmp2q, %1*32
4442 add tmp1q, %1*32
4443 %endmacro
4444
4445 cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3
4446 %define base r7-mask %+ SUFFIX %+ _table
4447 lea r7, [mask %+ SUFFIX %+ _table]
4448 tzcnt wd, wm
4449 movifnidn hd, hm
4450 mov maskq, maskmp
4451 movsxd wq, dword [r7+wq*4]
4452 vpbroadcastd m5, [base+pw_2048]
4453 pxor m4, m4
4454 add wq, r7
4455 BIDIR_FN MASK
4456
4457 %macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4
4458 mova m%1, [tmp1q+32*%3]
4459 mova m1, [tmp2q+32*%3]
4460 psubw m1, m%1
4461 pabsw m%2, m1
4462 psubusw m%2, m6, m%2
4463 psrlw m%2, 8 ; 64 - m
4464 psllw m2, m%2, 10
4465 pmulhw m1, m2
4466 paddw m%1, m1
4467 mova m1, [tmp1q+32*%4]
4468 mova m2, [tmp2q+32*%4]
4469 psubw m2, m1
4470 pabsw m3, m2
4471 psubusw m3, m6, m3
4472 psrlw m3, 8
4473 %if %5
4474 packuswb m%2, m3
4475 psubb m%2, m5, m%2
4476 vpermq m%2, m%2, q3120
4477 %else
4478 phaddw m%2, m3
4479 %endif
4480 psllw m3, 10
4481 pmulhw m2, m3
4482 paddw m1, m2
4483 pmulhrsw m%1, m7
4484 pmulhrsw m1, m7
4485 packuswb m%1, m1
4486 %endmacro
4487
4488 cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask
4489 %define base r6-blend_avx2_table
4490 lea r6, [blend_avx2_table]
4491 tzcnt wd, wm
4492 movifnidn maskq, maskmp
4493 movifnidn hd, hm
4494 movsxd wq, dword [r6+wq*4]
4495 vpbroadcastd m4, [base+pb_64]
4496 vpbroadcastd m5, [base+pw_512]
4497 sub tmpq, maskq
4498 add wq, r6
4499 lea r6, [dsq*3]
4500 jmp wq
4501 .w4:
4502 movd xm0, [dstq+dsq*0]
4503 pinsrd xm0, [dstq+dsq*1], 1
4504 vpbroadcastd xm1, [dstq+dsq*2]
4505 pinsrd xm1, [dstq+r6 ], 3
4506 mova xm6, [maskq]
4507 psubb xm3, xm4, xm6
4508 punpcklbw xm2, xm3, xm6
4509 punpckhbw xm3, xm6
4510 mova xm6, [maskq+tmpq]
4511 add maskq, 4*4
4512 punpcklbw xm0, xm6
4513 punpckhbw xm1, xm6
4514 pmaddubsw xm0, xm2
4515 pmaddubsw xm1, xm3
4516 pmulhrsw xm0, xm5
4517 pmulhrsw xm1, xm5
4518 packuswb xm0, xm1
4519 movd [dstq+dsq*0], xm0
4520 pextrd [dstq+dsq*1], xm0, 1
4521 pextrd [dstq+dsq*2], xm0, 2
4522 pextrd [dstq+r6 ], xm0, 3
4523 lea dstq, [dstq+dsq*4]
4524 sub hd, 4
4525 jg .w4
4526 RET
4527 ALIGN function_align
4528 .w8:
4529 movq xm1, [dstq+dsq*0]
4530 movhps xm1, [dstq+dsq*1]
4531 vpbroadcastq m2, [dstq+dsq*2]
4532 vpbroadcastq m3, [dstq+r6 ]
4533 mova m0, [maskq]
4534 mova m6, [maskq+tmpq]
4535 add maskq, 8*4
4536 vpblendd m1, m2, 0x30
4537 vpblendd m1, m3, 0xc0
4538 psubb m3, m4, m0
4539 punpcklbw m2, m3, m0
4540 punpckhbw m3, m0
4541 punpcklbw m0, m1, m6
4542 punpckhbw m1, m6
4543 pmaddubsw m0, m2
4544 pmaddubsw m1, m3
4545 pmulhrsw m0, m5
4546 pmulhrsw m1, m5
4547 packuswb m0, m1
4548 vextracti128 xm1, m0, 1
4549 movq [dstq+dsq*0], xm0
4550 movhps [dstq+dsq*1], xm0
4551 movq [dstq+dsq*2], xm1
4552 movhps [dstq+r6 ], xm1
4553 lea dstq, [dstq+dsq*4]
4554 sub hd, 4
4555 jg .w8
4556 RET
4557 ALIGN function_align
4558 .w16:
4559 mova m0, [maskq]
4560 mova xm1, [dstq+dsq*0]
4561 vinserti128 m1, [dstq+dsq*1], 1
4562 psubb m3, m4, m0
4563 punpcklbw m2, m3, m0
4564 punpckhbw m3, m0
4565 mova m6, [maskq+tmpq]
4566 add maskq, 16*2
4567 punpcklbw m0, m1, m6
4568 punpckhbw m1, m6
4569 pmaddubsw m0, m2
4570 pmaddubsw m1, m3
4571 pmulhrsw m0, m5
4572 pmulhrsw m1, m5
4573 packuswb m0, m1
4574 mova [dstq+dsq*0], xm0
4575 vextracti128 [dstq+dsq*1], m0, 1
4576 lea dstq, [dstq+dsq*2]
4577 sub hd, 2
4578 jg .w16
4579 RET
4580 ALIGN function_align
4581 .w32:
4582 mova m0, [maskq]
4583 mova m1, [dstq]
4584 mova m6, [maskq+tmpq]
4585 add maskq, 32
4586 psubb m3, m4, m0
4587 punpcklbw m2, m3, m0
4588 punpckhbw m3, m0
4589 punpcklbw m0, m1, m6
4590 punpckhbw m1, m6
4591 pmaddubsw m0, m2
4592 pmaddubsw m1, m3
4593 pmulhrsw m0, m5
4594 pmulhrsw m1, m5
4595 packuswb m0, m1
4596 mova [dstq], m0
4597 add dstq, dsq
4598 dec hd
4599 jg .w32
4600 RET
4601
4602 cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask
4603 %define base r5-blend_v_avx2_table
4604 lea r5, [blend_v_avx2_table]
4605 tzcnt wd, wm
4606 movifnidn hd, hm
4607 movsxd wq, dword [r5+wq*4]
4608 vpbroadcastd m5, [base+pw_512]
4609 add wq, r5
4610 add maskq, obmc_masks-blend_v_avx2_table
4611 jmp wq
4612 .w2:
4613 vpbroadcastd xm2, [maskq+2*2]
4614 .w2_s0_loop:
4615 movd xm0, [dstq+dsq*0]
4616 pinsrw xm0, [dstq+dsq*1], 1
4617 movd xm1, [tmpq]
4618 add tmpq, 2*2
4619 punpcklbw xm0, xm1
4620 pmaddubsw xm0, xm2
4621 pmulhrsw xm0, xm5
4622 packuswb xm0, xm0
4623 pextrw [dstq+dsq*0], xm0, 0
4624 pextrw [dstq+dsq*1], xm0, 1
4625 lea dstq, [dstq+dsq*2]
4626 sub hd, 2
4627 jg .w2_s0_loop
4628 RET
4629 ALIGN function_align
4630 .w4:
4631 vpbroadcastq xm2, [maskq+4*2]
4632 .w4_loop:
4633 movd xm0, [dstq+dsq*0]
4634 pinsrd xm0, [dstq+dsq*1], 1
4635 movq xm1, [tmpq]
4636 add tmpq, 4*2
4637 punpcklbw xm0, xm1
4638 pmaddubsw xm0, xm2
4639 pmulhrsw xm0, xm5
4640 packuswb xm0, xm0
4641 movd [dstq+dsq*0], xm0
4642 pextrd [dstq+dsq*1], xm0, 1
4643 lea dstq, [dstq+dsq*2]
4644 sub hd, 2
4645 jg .w4_loop
4646 RET
4647 ALIGN function_align
4648 .w8:
4649 mova xm3, [maskq+8*2]
4650 .w8_loop:
4651 movq xm0, [dstq+dsq*0]
4652 vpbroadcastq xm1, [dstq+dsq*1]
4653 mova xm2, [tmpq]
4654 add tmpq, 8*2
4655 punpcklbw xm0, xm2
4656 punpckhbw xm1, xm2
4657 pmaddubsw xm0, xm3
4658 pmaddubsw xm1, xm3
4659 pmulhrsw xm0, xm5
4660 pmulhrsw xm1, xm5
4661 packuswb xm0, xm1
4662 movq [dstq+dsq*0], xm0
4663 movhps [dstq+dsq*1], xm0
4664 lea dstq, [dstq+dsq*2]
4665 sub hd, 2
4666 jg .w8_loop
4667 RET
4668 ALIGN function_align
4669 .w16:
4670 vbroadcasti128 m3, [maskq+16*2]
4671 vbroadcasti128 m4, [maskq+16*3]
4672 .w16_loop:
4673 mova xm1, [dstq+dsq*0]
4674 vinserti128 m1, [dstq+dsq*1], 1
4675 mova m2, [tmpq]
4676 add tmpq, 16*2
4677 punpcklbw m0, m1, m2
4678 punpckhbw m1, m2
4679 pmaddubsw m0, m3
4680 pmaddubsw m1, m4
4681 pmulhrsw m0, m5
4682 pmulhrsw m1, m5
4683 packuswb m0, m1
4684 mova [dstq+dsq*0], xm0
4685 vextracti128 [dstq+dsq*1], m0, 1
4686 lea dstq, [dstq+dsq*2]
4687 sub hd, 2
4688 jg .w16_loop
4689 RET
4690 ALIGN function_align
4691 .w32:
4692 mova xm3, [maskq+16*4]
4693 vinserti128 m3, [maskq+16*6], 1
4694 mova xm4, [maskq+16*5]
4695 vinserti128 m4, [maskq+16*7], 1
4696 .w32_loop:
4697 mova m1, [dstq]
4698 mova m2, [tmpq]
4699 add tmpq, 32
4700 punpcklbw m0, m1, m2
4701 punpckhbw m1, m2
4702 pmaddubsw m0, m3
4703 pmaddubsw m1, m4
4704 pmulhrsw m0, m5
4705 pmulhrsw m1, m5
4706 packuswb m0, m1
4707 mova [dstq], m0
4708 add dstq, dsq
4709 dec hd
4710 jg .w32_loop
4711 RET
4712
4713 cglobal blend_h_8bpc, 4, 7, 6, dst, ds, tmp, w, h, mask
4714 %define base r5-blend_h_avx2_table
4715 lea r5, [blend_h_avx2_table]
4716 mov r6d, wd
4717 tzcnt wd, wd
4718 mov hd, hm
4719 movsxd wq, dword [r5+wq*4]
4720 vpbroadcastd m5, [base+pw_512]
4721 add wq, r5
4722 lea maskq, [base+obmc_masks+hq*2]
4723 lea hd, [hq*3]
4724 shr hd, 2 ; h * 3/4
4725 lea maskq, [maskq+hq*2]
4726 neg hq
4727 jmp wq
4728 .w2:
4729 movd xm0, [dstq+dsq*0]
4730 pinsrw xm0, [dstq+dsq*1], 1
4731 movd xm2, [maskq+hq*2]
4732 movd xm1, [tmpq]
4733 add tmpq, 2*2
4734 punpcklwd xm2, xm2
4735 punpcklbw xm0, xm1
4736 pmaddubsw xm0, xm2
4737 pmulhrsw xm0, xm5
4738 packuswb xm0, xm0
4739 pextrw [dstq+dsq*0], xm0, 0
4740 pextrw [dstq+dsq*1], xm0, 1
4741 lea dstq, [dstq+dsq*2]
4742 add hq, 2
4743 jl .w2
4744 RET
4745 ALIGN function_align
4746 .w4:
4747 mova xm3, [blend_shuf]
4748 .w4_loop:
4749 movd xm0, [dstq+dsq*0]
4750 pinsrd xm0, [dstq+dsq*1], 1
4751 movd xm2, [maskq+hq*2]
4752 movq xm1, [tmpq]
4753 add tmpq, 4*2
4754 pshufb xm2, xm3
4755 punpcklbw xm0, xm1
4756 pmaddubsw xm0, xm2
4757 pmulhrsw xm0, xm5
4758 packuswb xm0, xm0
4759 movd [dstq+dsq*0], xm0
4760 pextrd [dstq+dsq*1], xm0, 1
4761 lea dstq, [dstq+dsq*2]
4762 add hq, 2
4763 jl .w4_loop
4764 RET
4765 ALIGN function_align
4766 .w8:
4767 vbroadcasti128 m4, [blend_shuf]
4768 shufpd m4, m4, 0x03
4769 .w8_loop:
4770 vpbroadcastq m1, [dstq+dsq*0]
4771 movq xm0, [dstq+dsq*1]
4772 vpblendd m0, m1, 0x30
4773 vpbroadcastd m3, [maskq+hq*2]
4774 movq xm1, [tmpq+8*1]
4775 vinserti128 m1, [tmpq+8*0], 1
4776 add tmpq, 8*2
4777 pshufb m3, m4
4778 punpcklbw m0, m1
4779 pmaddubsw m0, m3
4780 pmulhrsw m0, m5
4781 vextracti128 xm1, m0, 1
4782 packuswb xm0, xm1
4783 movhps [dstq+dsq*0], xm0
4784 movq [dstq+dsq*1], xm0
4785 lea dstq, [dstq+dsq*2]
4786 add hq, 2
4787 jl .w8_loop
4788 RET
4789 ALIGN function_align
4790 .w16:
4791 vbroadcasti128 m4, [blend_shuf]
4792 shufpd m4, m4, 0x0c
4793 .w16_loop:
4794 mova xm1, [dstq+dsq*0]
4795 vinserti128 m1, [dstq+dsq*1], 1
4796 vpbroadcastd m3, [maskq+hq*2]
4797 mova m2, [tmpq]
4798 add tmpq, 16*2
4799 pshufb m3, m4
4800 punpcklbw m0, m1, m2
4801 punpckhbw m1, m2
4802 pmaddubsw m0, m3
4803 pmaddubsw m1, m3
4804 pmulhrsw m0, m5
4805 pmulhrsw m1, m5
4806 packuswb m0, m1
4807 mova [dstq+dsq*0], xm0
4808 vextracti128 [dstq+dsq*1], m0, 1
4809 lea dstq, [dstq+dsq*2]
4810 add hq, 2
4811 jl .w16_loop
4812 RET
4813 ALIGN function_align
4814 .w32: ; w32/w64/w128
4815 sub dsq, r6
4816 .w32_loop0:
4817 vpbroadcastw m3, [maskq+hq*2]
4818 mov wd, r6d
4819 .w32_loop:
4820 mova m1, [dstq]
4821 mova m2, [tmpq]
4822 add tmpq, 32
4823 punpcklbw m0, m1, m2
4824 punpckhbw m1, m2
4825 pmaddubsw m0, m3
4826 pmaddubsw m1, m3
4827 pmulhrsw m0, m5
4828 pmulhrsw m1, m5
4829 packuswb m0, m1
4830 mova [dstq], m0
4831 add dstq, 32
4832 sub wd, 32
4833 jg .w32_loop
4834 add dstq, dsq
4835 inc hq
4836 jl .w32_loop0
4837 RET
4838
4839 cglobal emu_edge_8bpc, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \
4840 bottomext, rightext
4841 ; we assume that the buffer (stride) is larger than width, so we can
4842 ; safely overwrite by a few bytes
4843
4844 ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride)
4845 xor r12d, r12d
4846 lea r10, [ihq-1]
4847 cmp yq, ihq
4848 cmovs r10, yq
4849 test yq, yq
4850 cmovs r10, r12
4851 imul r10, sstrideq
4852 add srcq, r10
4853
4854 ; ref += iclip(x, 0, iw - 1)
4855 lea r10, [iwq-1]
4856 cmp xq, iwq
4857 cmovs r10, xq
4858 test xq, xq
4859 cmovs r10, r12
4860 add srcq, r10
4861
4862 ; bottom_ext = iclip(y + bh - ih, 0, bh - 1)
4863 lea bottomextq, [yq+bhq]
4864 sub bottomextq, ihq
4865 lea r3, [bhq-1]
4866 cmovs bottomextq, r12
4867
4868 DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \
4869 bottomext, rightext
4870
4871 ; top_ext = iclip(-y, 0, bh - 1)
4872 neg topextq
4873 cmovs topextq, r12
4874 cmp bottomextq, bhq
4875 cmovns bottomextq, r3
4876 cmp topextq, bhq
4877 cmovg topextq, r3
4878
4879 ; right_ext = iclip(x + bw - iw, 0, bw - 1)
4880 lea rightextq, [xq+bwq]
4881 sub rightextq, iwq
4882 lea r2, [bwq-1]
4883 cmovs rightextq, r12
4884
4885 DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \
4886 bottomext, rightext
4887
4888 ; left_ext = iclip(-x, 0, bw - 1)
4889 neg leftextq
4890 cmovs leftextq, r12
4891 cmp rightextq, bwq
4892 cmovns rightextq, r2
4893 cmp leftextq, bwq
4894 cmovns leftextq, r2
4895
4896 DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \
4897 dst, dstride, src, sstride, bottomext, rightext
4898
4899 ; center_h = bh - top_ext - bottom_ext
4900 lea r3, [bottomextq+topextq]
4901 sub centerhq, r3
4902
4903 ; blk += top_ext * PXSTRIDE(dst_stride)
4904 mov r2, topextq
4905 imul r2, dstrideq
4906 add dstq, r2
4907 mov r9m, dstq
4908
4909 ; center_w = bw - left_ext - right_ext
4910 mov centerwq, bwq
4911 lea r3, [rightextq+leftextq]
4912 sub centerwq, r3
4913
4914 %macro v_loop 3 ; need_left_ext, need_right_ext, suffix
4915 .v_loop_%3:
4916 %if %1
4917 ; left extension
4918 xor r3, r3
4919 vpbroadcastb m0, [srcq]
4920 .left_loop_%3:
4921 mova [dstq+r3], m0
4922 add r3, 32
4923 cmp r3, leftextq
4924 jl .left_loop_%3
4925
4926 ; body
4927 lea r12, [dstq+leftextq]
4928 %endif
4929 xor r3, r3
4930 .body_loop_%3:
4931 movu m0, [srcq+r3]
4932 %if %1
4933 movu [r12+r3], m0
4934 %else
4935 movu [dstq+r3], m0
4936 %endif
4937 add r3, 32
4938 cmp r3, centerwq
4939 jl .body_loop_%3
4940
4941 %if %2
4942 ; right extension
4943 %if %1
4944 add r12, centerwq
4945 %else
4946 lea r12, [dstq+centerwq]
4947 %endif
4948 xor r3, r3
4949 vpbroadcastb m0, [srcq+centerwq-1]
4950 .right_loop_%3:
4951 movu [r12+r3], m0
4952 add r3, 32
4953 cmp r3, rightextq
4954 jl .right_loop_%3
4955
4956 %endif
4957 add dstq, dstrideq
4958 add srcq, sstrideq
4959 dec centerhq
4960 jg .v_loop_%3
4961 %endmacro
4962
4963 test leftextq, leftextq
4964 jnz .need_left_ext
4965 test rightextq, rightextq
4966 jnz .need_right_ext
4967 v_loop 0, 0, 0
4968 jmp .body_done
4969
4970 .need_left_ext:
4971 test rightextq, rightextq
4972 jnz .need_left_right_ext
4973 v_loop 1, 0, 1
4974 jmp .body_done
4975
4976 .need_left_right_ext:
4977 v_loop 1, 1, 2
4978 jmp .body_done
4979
4980 .need_right_ext:
4981 v_loop 0, 1, 3
4982
4983 .body_done:
4984 ; bottom edge extension
4985 test bottomextq, bottomextq
4986 jz .top
4987 mov srcq, dstq
4988 sub srcq, dstrideq
4989 xor r1, r1
4990 .bottom_x_loop:
4991 mova m0, [srcq+r1]
4992 lea r3, [dstq+r1]
4993 mov r4, bottomextq
4994 .bottom_y_loop:
4995 mova [r3], m0
4996 add r3, dstrideq
4997 dec r4
4998 jg .bottom_y_loop
4999 add r1, 32
5000 cmp r1, bwq
5001 jl .bottom_x_loop
5002
5003 .top:
5004 ; top edge extension
5005 test topextq, topextq
5006 jz .end
5007 mov srcq, r9m
5008 mov dstq, dstm
5009 xor r1, r1
5010 .top_x_loop:
5011 mova m0, [srcq+r1]
5012 lea r3, [dstq+r1]
5013 mov r4, topextq
5014 .top_y_loop:
5015 mova [r3], m0
5016 add r3, dstrideq
5017 dec r4
5018 jg .top_y_loop
5019 add r1, 32
5020 cmp r1, bwq
5021 jl .top_x_loop
5022
5023 .end:
5024 RET
5025
5026 cglobal resize_8bpc, 6, 12, 16, dst, dst_stride, src, src_stride, \
5027 dst_w, h, src_w, dx, mx0
5028 sub dword mx0m, 4<<14
5029 sub dword src_wm, 8
5030 vpbroadcastd m5, dxm
5031 vpbroadcastd m8, mx0m
5032 vpbroadcastd m6, src_wm
5033
5034 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x
5035 LEA r7, $$
5036 %define base r7-$$
5037
5038 vpbroadcastd xm3, [base+pw_m256]
5039 vpbroadcastd m7, [base+pd_63]
5040 vbroadcasti128 m15, [base+pb_8x0_8x8]
5041 pmaddwd m2, m5, [base+rescale_mul] ; dx*[0,1,2,3,4,5,6,7]
5042 pslld m5, 3 ; dx*8
5043 pslld m6, 14
5044 paddd m8, m2 ; mx+[0..7]*dx
5045 pxor m2, m2
5046
5047 ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7
5048 ; m8 = mx+[0..7]*dx, m5 = dx*8, m6 = src_w, m7 = 0x3f, m15=0,8
5049
5050 .loop_y:
5051 xor xd, xd
5052 mova m4, m8 ; per-line working version of mx
5053
5054 .loop_x:
5055 pmaxsd m0, m4, m2
5056 psrad m9, m4, 8 ; filter offset (unmasked)
5057 pminsd m0, m6 ; iclip(mx, 0, src_w-8)
5058 psubd m1, m4, m0 ; pshufb offset
5059 psrad m0, 14 ; clipped src_x offset
5060 psrad m1, 14 ; pshufb edge_emu offset
5061 pand m9, m7 ; filter offset (masked)
5062
5063 ; load source pixels - this ugly code is vpgatherdq emulation since
5064 ; directly using vpgatherdq on Haswell is quite a bit slower :(
5065 movd r8d, xm0
5066 pextrd r9d, xm0, 1
5067 pextrd r10d, xm0, 2
5068 pextrd r11d, xm0, 3
5069 vextracti128 xm0, m0, 1
5070 movq xm12, [srcq+r8]
5071 movq xm13, [srcq+r10]
5072 movhps xm12, [srcq+r9]
5073 movhps xm13, [srcq+r11]
5074 movd r8d, xm0
5075 pextrd r9d, xm0, 1
5076 pextrd r10d, xm0, 2
5077 pextrd r11d, xm0, 3
5078 vinserti128 m12, [srcq+r8], 1
5079 vinserti128 m13, [srcq+r10], 1
5080 vpbroadcastq m10, [srcq+r9]
5081 vpbroadcastq m11, [srcq+r11]
5082 vpblendd m12, m10, 11000000b
5083 vpblendd m13, m11, 11000000b
5084
5085 ; if no emulation is required, we don't need to shuffle or emulate edges
5086 ; this also saves 2 quasi-vpgatherdqs
5087 vptest m1, m1
5088 jz .filter
5089
5090 movq r9, xm1
5091 pextrq r11, xm1, 1
5092 movsxd r8, r9d
5093 sar r9, 32
5094 movsxd r10, r11d
5095 sar r11, 32
5096 vextracti128 xm1, m1, 1
5097 movq xm14, [base+resize_shuf+4+r8]
5098 movq xm0, [base+resize_shuf+4+r10]
5099 movhps xm14, [base+resize_shuf+4+r9]
5100 movhps xm0, [base+resize_shuf+4+r11]
5101 movq r9, xm1
5102 pextrq r11, xm1, 1
5103 movsxd r8, r9d
5104 sar r9, 32
5105 movsxd r10, r11d
5106 sar r11, 32
5107 vinserti128 m14, [base+resize_shuf+4+r8], 1
5108 vinserti128 m0, [base+resize_shuf+4+r10], 1
5109 vpbroadcastq m10, [base+resize_shuf+4+r9]
5110 vpbroadcastq m11, [base+resize_shuf+4+r11]
5111 vpblendd m14, m10, 11000000b
5112 vpblendd m0, m11, 11000000b
5113
5114 paddb m14, m15
5115 paddb m0, m15
5116 pshufb m12, m14
5117 pshufb m13, m0
5118
5119 .filter:
5120 movd r8d, xm9
5121 pextrd r9d, xm9, 1
5122 pextrd r10d, xm9, 2
5123 pextrd r11d, xm9, 3
5124 vextracti128 xm9, m9, 1
5125 movq xm10, [base+resize_filter+r8*8]
5126 movq xm11, [base+resize_filter+r10*8]
5127 movhps xm10, [base+resize_filter+r9*8]
5128 movhps xm11, [base+resize_filter+r11*8]
5129 movd r8d, xm9
5130 pextrd r9d, xm9, 1
5131 pextrd r10d, xm9, 2
5132 pextrd r11d, xm9, 3
5133 vinserti128 m10, [base+resize_filter+r8*8], 1
5134 vinserti128 m11, [base+resize_filter+r10*8], 1
5135 vpbroadcastq m14, [base+resize_filter+r9*8]
5136 vpbroadcastq m1, [base+resize_filter+r11*8]
5137 vpblendd m10, m14, 11000000b
5138 vpblendd m11, m1, 11000000b
5139
5140 pmaddubsw m12, m10
5141 pmaddubsw m13, m11
5142 phaddw m12, m13
5143 vextracti128 xm13, m12, 1
5144 phaddsw xm12, xm13
5145 pmulhrsw xm12, xm3 ; x=(x+64)>>7
5146 packuswb xm12, xm12
5147 movq [dstq+xq], xm12
5148
5149 paddd m4, m5
5150 add xd, 8
5151 cmp xd, dst_wd
5152 jl .loop_x
5153
5154 add dstq, dst_strideq
5155 add srcq, src_strideq
5156 dec hd
5157 jg .loop_y
5158 RET
5159
5160 cglobal w_mask_420_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3
5161 %define base r7-w_mask_420_avx2_table
5162 lea r7, [w_mask_420_avx2_table]
5163 tzcnt wd, wm
5164 mov r6d, r7m ; sign
5165 movifnidn hd, hm
5166 movsxd wq, [r7+wq*4]
5167 vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
5168 vpbroadcastd m7, [base+pw_2048]
5169 pmovzxbd m9, [base+deint_shuf4]
5170 vpbroadcastd m8, [base+wm_420_sign+r6*4] ; 258 - sign
5171 add wq, r7
5172 W_MASK 0, 4, 0, 1
5173 mov maskq, maskmp
5174 lea stride3q, [strideq*3]
5175 jmp wq
5176 .w4:
5177 vextracti128 xm1, m0, 1
5178 movd [dstq+strideq*0], xm0
5179 pextrd [dstq+strideq*1], xm0, 1
5180 movd [dstq+strideq*2], xm1
5181 pextrd [dstq+stride3q ], xm1, 1
5182 cmp hd, 8
5183 jl .w4_end
5184 lea dstq, [dstq+strideq*4]
5185 pextrd [dstq+strideq*0], xm0, 2
5186 pextrd [dstq+strideq*1], xm0, 3
5187 pextrd [dstq+strideq*2], xm1, 2
5188 pextrd [dstq+stride3q ], xm1, 3
5189 jg .w4_h16
5190 .w4_end:
5191 vextracti128 xm0, m4, 1
5192 vpblendd xm1, xm4, xm0, 0x05
5193 vpblendd xm4, xm0, 0x0a
5194 pshufd xm1, xm1, q2301
5195 psubw xm4, xm8, xm4
5196 psubw xm4, xm1
5197 psrlw xm4, 2
5198 packuswb xm4, xm4
5199 movq [maskq], xm4
5200 RET
5201 .w4_h16:
5202 W_MASK 0, 5, 2, 3
5203 lea dstq, [dstq+strideq*4]
5204 phaddd m4, m5
5205 vextracti128 xm1, m0, 1
5206 psubw m4, m8, m4
5207 psrlw m4, 2
5208 vpermd m4, m9, m4
5209 vextracti128 xm5, m4, 1
5210 packuswb xm4, xm5
5211 movd [dstq+strideq*0], xm0
5212 pextrd [dstq+strideq*1], xm0, 1
5213 movd [dstq+strideq*2], xm1
5214 pextrd [dstq+stride3q], xm1, 1
5215 lea dstq, [dstq+strideq*4]
5216 pextrd [dstq+strideq*0], xm0, 2
5217 pextrd [dstq+strideq*1], xm0, 3
5218 pextrd [dstq+strideq*2], xm1, 2
5219 pextrd [dstq+stride3q ], xm1, 3
5220 mova [maskq], xm4
5221 RET
5222 .w8_loop:
5223 add tmp1q, 2*32
5224 add tmp2q, 2*32
5225 W_MASK 0, 4, 0, 1
5226 lea dstq, [dstq+strideq*4]
5227 add maskq, 8
5228 .w8:
5229 vextracti128 xm2, m4, 1
5230 vextracti128 xm1, m0, 1
5231 psubw xm4, xm8, xm4
5232 psubw xm4, xm2
5233 psrlw xm4, 2
5234 packuswb xm4, xm4
5235 movq [dstq+strideq*0], xm0
5236 movq [dstq+strideq*1], xm1
5237 movhps [dstq+strideq*2], xm0
5238 movhps [dstq+stride3q ], xm1
5239 movq [maskq], xm4
5240 sub hd, 4
5241 jg .w8_loop
5242 RET
5243 .w16_loop:
5244 add tmp1q, 4*32
5245 add tmp2q, 4*32
5246 W_MASK 0, 4, 0, 1
5247 lea dstq, [dstq+strideq*4]
5248 add maskq, 16
5249 .w16:
5250 vpermq m0, m0, q3120
5251 mova [dstq+strideq*0], xm0
5252 vextracti128 [dstq+strideq*1], m0, 1
5253 W_MASK 0, 5, 2, 3
5254 punpckhqdq m1, m4, m5
5255 punpcklqdq m4, m5
5256 psubw m1, m8, m1
5257 psubw m1, m4
5258 psrlw m1, 2
5259 vpermq m0, m0, q3120
5260 packuswb m1, m1
5261 vpermd m1, m9, m1
5262 mova [dstq+strideq*2], xm0
5263 vextracti128 [dstq+stride3q ], m0, 1
5264 mova [maskq], xm1
5265 sub hd, 4
5266 jg .w16_loop
5267 RET
5268 .w32_loop:
5269 add tmp1q, 4*32
5270 add tmp2q, 4*32
5271 W_MASK 0, 4, 0, 1
5272 lea dstq, [dstq+strideq*2]
5273 add maskq, 16
5274 .w32:
5275 vpermq m0, m0, q3120
5276 mova [dstq+strideq*0], m0
5277 W_MASK 0, 5, 2, 3
5278 psubw m4, m8, m4
5279 psubw m4, m5
5280 psrlw m4, 2
5281 vpermq m0, m0, q3120
5282 packuswb m4, m4
5283 vpermd m4, m9, m4
5284 mova [dstq+strideq*1], m0
5285 mova [maskq], xm4
5286 sub hd, 2
5287 jg .w32_loop
5288 RET
5289 .w64_loop_even:
5290 psubw m10, m8, m4
5291 psubw m11, m8, m5
5292 dec hd
5293 .w64_loop:
5294 add tmp1q, 4*32
5295 add tmp2q, 4*32
5296 W_MASK 0, 4, 0, 1
5297 add dstq, strideq
5298 .w64:
5299 vpermq m0, m0, q3120
5300 mova [dstq+32*0], m0
5301 W_MASK 0, 5, 2, 3
5302 vpermq m0, m0, q3120
5303 mova [dstq+32*1], m0
5304 test hd, 1
5305 jz .w64_loop_even
5306 psubw m4, m10, m4
5307 psubw m5, m11, m5
5308 psrlw m4, 2
5309 psrlw m5, 2
5310 packuswb m4, m5
5311 vpermd m4, m9, m4
5312 mova [maskq], m4
5313 add maskq, 32
5314 dec hd
5315 jg .w64_loop
5316 RET
5317 .w128_loop_even:
5318 psubw m12, m8, m4
5319 psubw m13, m8, m5
5320 dec hd
5321 .w128_loop:
5322 W_MASK 0, 4, 0, 1
5323 add dstq, strideq
5324 .w128:
5325 vpermq m0, m0, q3120
5326 mova [dstq+32*0], m0
5327 W_MASK 0, 5, 2, 3
5328 vpermq m0, m0, q3120
5329 mova [dstq+32*1], m0
5330 add tmp1q, 8*32
5331 add tmp2q, 8*32
5332 test hd, 1
5333 jz .w128_even
5334 psubw m4, m10, m4
5335 psubw m5, m11, m5
5336 psrlw m4, 2
5337 psrlw m5, 2
5338 packuswb m4, m5
5339 vpermd m4, m9, m4
5340 mova [maskq+32*0], m4
5341 jmp .w128_odd
5342 .w128_even:
5343 psubw m10, m8, m4
5344 psubw m11, m8, m5
5345 .w128_odd:
5346 W_MASK 0, 4, -4, -3
5347 vpermq m0, m0, q3120
5348 mova [dstq+32*2], m0
5349 W_MASK 0, 5, -2, -1
5350 vpermq m0, m0, q3120
5351 mova [dstq+32*3], m0
5352 test hd, 1
5353 jz .w128_loop_even
5354 psubw m4, m12, m4
5355 psubw m5, m13, m5
5356 psrlw m4, 2
5357 psrlw m5, 2
5358 packuswb m4, m5
5359 vpermd m4, m9, m4
5360 mova [maskq+32*1], m4
5361 add maskq, 64
5362 dec hd
5363 jg .w128_loop
5364 RET
5365
5366 cglobal w_mask_422_8bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3
5367 %define base r7-w_mask_422_avx2_table
5368 lea r7, [w_mask_422_avx2_table]
5369 tzcnt wd, wm
5370 mov r6d, r7m ; sign
5371 movifnidn hd, hm
5372 pxor m9, m9
5373 movsxd wq, dword [r7+wq*4]
5374 vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
5375 vpbroadcastd m7, [base+pw_2048]
5376 pmovzxbd m10, [base+deint_shuf4]
5377 vpbroadcastd m8, [base+wm_422_sign+r6*4] ; 128 - sign
5378 add wq, r7
5379 mov maskq, maskmp
5380 W_MASK 0, 4, 0, 1
5381 lea stride3q, [strideq*3]
5382 jmp wq
5383 .w4:
5384 vextracti128 xm1, m0, 1
5385 movd [dstq+strideq*0], xm0
5386 pextrd [dstq+strideq*1], xm0, 1
5387 movd [dstq+strideq*2], xm1
5388 pextrd [dstq+stride3q ], xm1, 1
5389 cmp hd, 8
5390 jl .w4_end
5391 lea dstq, [dstq+strideq*4]
5392 pextrd [dstq+strideq*0], xm0, 2
5393 pextrd [dstq+strideq*1], xm0, 3
5394 pextrd [dstq+strideq*2], xm1, 2
5395 pextrd [dstq+stride3q ], xm1, 3
5396 jg .w4_h16
5397 .w4_end:
5398 vextracti128 xm5, m4, 1
5399 packuswb xm4, xm5
5400 psubb xm5, xm8, xm4
5401 pavgb xm5, xm9
5402 pshufd xm5, xm5, q3120
5403 mova [maskq], xm5
5404 RET
5405 .w4_h16:
5406 W_MASK 0, 5, 2, 3
5407 lea dstq, [dstq+strideq*4]
5408 packuswb m4, m5
5409 psubb m5, m8, m4
5410 pavgb m5, m9
5411 vpermd m5, m10, m5
5412 vextracti128 xm1, m0, 1
5413 movd [dstq+strideq*0], xm0
5414 pextrd [dstq+strideq*1], xm0, 1
5415 movd [dstq+strideq*2], xm1
5416 pextrd [dstq+stride3q ], xm1, 1
5417 lea dstq, [dstq+strideq*4]
5418 pextrd [dstq+strideq*0], xm0, 2
5419 pextrd [dstq+strideq*1], xm0, 3
5420 pextrd [dstq+strideq*2], xm1, 2
5421 pextrd [dstq+stride3q ], xm1, 3
5422 mova [maskq], m5
5423 RET
5424 .w8_loop:
5425 add tmp1q, 32*2
5426 add tmp2q, 32*2
5427 W_MASK 0, 4, 0, 1
5428 lea dstq, [dstq+strideq*4]
5429 add maskq, 16
5430 .w8:
5431 vextracti128 xm5, m4, 1
5432 vextracti128 xm1, m0, 1
5433 packuswb xm4, xm5
5434 psubb xm5, xm8, xm4
5435 pavgb xm5, xm9
5436 pshufd xm5, xm5, q3120
5437 movq [dstq+strideq*0], xm0
5438 movq [dstq+strideq*1], xm1
5439 movhps [dstq+strideq*2], xm0
5440 movhps [dstq+stride3q ], xm1
5441 mova [maskq], xm5
5442 sub hd, 4
5443 jg .w8_loop
5444 RET
5445 .w16_loop:
5446 add tmp1q, 32*4
5447 add tmp2q, 32*4
5448 W_MASK 0, 4, 0, 1
5449 lea dstq, [dstq+strideq*4]
5450 add maskq, 32
5451 .w16:
5452 vpermq m0, m0, q3120
5453 mova [dstq+strideq*0], xm0
5454 vextracti128 [dstq+strideq*1], m0, 1
5455 W_MASK 0, 5, 2, 3
5456 packuswb m4, m5
5457 psubb m5, m8, m4
5458 pavgb m5, m9
5459 vpermq m0, m0, q3120
5460 vpermd m5, m10, m5
5461 mova [dstq+strideq*2], xm0
5462 vextracti128 [dstq+stride3q ], m0, 1
5463 mova [maskq], m5
5464 sub hd, 4
5465 jg .w16_loop
5466 RET
5467 .w32_loop:
5468 add tmp1q, 32*4
5469 add tmp2q, 32*4
5470 W_MASK 0, 4, 0, 1
5471 lea dstq, [dstq+strideq*2]
5472 add maskq, 32
5473 .w32:
5474 vpermq m0, m0, q3120
5475 mova [dstq+strideq*0], m0
5476 W_MASK 0, 5, 2, 3
5477 packuswb m4, m5
5478 psubb m5, m8, m4
5479 pavgb m5, m9
5480 vpermq m0, m0, q3120
5481 vpermd m5, m10, m5
5482 mova [dstq+strideq*1], m0
5483 mova [maskq], m5
5484 sub hd, 2
5485 jg .w32_loop
5486 RET
5487 .w64_loop:
5488 add tmp1q, 32*4
5489 add tmp2q, 32*4
5490 W_MASK 0, 4, 0, 1
5491 add dstq, strideq
5492 add maskq, 32
5493 .w64:
5494 vpermq m0, m0, q3120
5495 mova [dstq+32*0], m0
5496 W_MASK 0, 5, 2, 3
5497 packuswb m4, m5
5498 psubb m5, m8, m4
5499 pavgb m5, m9
5500 vpermq m0, m0, q3120
5501 vpermd m5, m10, m5
5502 mova [dstq+32*1], m0
5503 mova [maskq], m5
5504 dec hd
5505 jg .w64_loop
5506 RET
5507 .w128_loop:
5508 add tmp1q, 32*8
5509 add tmp2q, 32*8
5510 W_MASK 0, 4, 0, 1
5511 add dstq, strideq
5512 add maskq, 32*2
5513 .w128:
5514 vpermq m0, m0, q3120
5515 mova [dstq+32*0], m0
5516 W_MASK 0, 5, 2, 3
5517 packuswb m4, m5
5518 psubb m5, m8, m4
5519 pavgb m5, m9
5520 vpermq m0, m0, q3120
5521 vpermd m5, m10, m5
5522 mova [dstq+32*1], m0
5523 mova [maskq+32*0], m5
5524 W_MASK 0, 4, 4, 5
5525 vpermq m0, m0, q3120
5526 mova [dstq+32*2], m0
5527 W_MASK 0, 5, 6, 7
5528 packuswb m4, m5
5529 psubb m5, m8, m4
5530 pavgb m5, m9
5531 vpermq m0, m0, q3120
5532 vpermd m5, m10, m5
5533 mova [dstq+32*3], m0
5534 mova [maskq+32*1], m5
5535 dec hd
5536 jg .w128_loop
5537 RET
5538
5539 cglobal w_mask_444_8bpc, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3
5540 %define base r7-w_mask_444_avx2_table
5541 lea r7, [w_mask_444_avx2_table]
5542 tzcnt wd, wm
5543 movifnidn hd, hm
5544 mov maskq, maskmp
5545 movsxd wq, dword [r7+wq*4]
5546 vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
5547 vpbroadcastd m5, [base+pb_64]
5548 vpbroadcastd m7, [base+pw_2048]
5549 add wq, r7
5550 W_MASK 0, 4, 0, 1, 1
5551 lea stride3q, [strideq*3]
5552 jmp wq
5553 .w4:
5554 vextracti128 xm1, m0, 1
5555 movd [dstq+strideq*0], xm0
5556 pextrd [dstq+strideq*1], xm0, 1
5557 movd [dstq+strideq*2], xm1
5558 pextrd [dstq+stride3q ], xm1, 1
5559 mova [maskq+32*0], m4
5560 cmp hd, 8
5561 jl .w4_end
5562 lea dstq, [dstq+strideq*4]
5563 pextrd [dstq+strideq*0], xm0, 2
5564 pextrd [dstq+strideq*1], xm0, 3
5565 pextrd [dstq+strideq*2], xm1, 2
5566 pextrd [dstq+stride3q ], xm1, 3
5567 je .w4_end
5568 W_MASK 0, 4, 2, 3, 1
5569 lea dstq, [dstq+strideq*4]
5570 vextracti128 xm1, m0, 1
5571 movd [dstq+strideq*0], xm0
5572 pextrd [dstq+strideq*1], xm0, 1
5573 movd [dstq+strideq*2], xm1
5574 pextrd [dstq+stride3q ], xm1, 1
5575 lea dstq, [dstq+strideq*4]
5576 pextrd [dstq+strideq*0], xm0, 2
5577 pextrd [dstq+strideq*1], xm0, 3
5578 pextrd [dstq+strideq*2], xm1, 2
5579 pextrd [dstq+stride3q ], xm1, 3
5580 mova [maskq+32*1], m4
5581 .w4_end:
5582 RET
5583 .w8_loop:
5584 add tmp1q, 32*2
5585 add tmp2q, 32*2
5586 W_MASK 0, 4, 0, 1, 1
5587 lea dstq, [dstq+strideq*4]
5588 add maskq, 32
5589 .w8:
5590 vextracti128 xm1, m0, 1
5591 movq [dstq+strideq*0], xm0
5592 movq [dstq+strideq*1], xm1
5593 movhps [dstq+strideq*2], xm0
5594 movhps [dstq+stride3q ], xm1
5595 mova [maskq], m4
5596 sub hd, 4
5597 jg .w8_loop
5598 RET
5599 .w16_loop:
5600 add tmp1q, 32*2
5601 add tmp2q, 32*2
5602 W_MASK 0, 4, 0, 1, 1
5603 lea dstq, [dstq+strideq*2]
5604 add maskq, 32
5605 .w16:
5606 vpermq m0, m0, q3120
5607 mova [dstq+strideq*0], xm0
5608 vextracti128 [dstq+strideq*1], m0, 1
5609 mova [maskq], m4
5610 sub hd, 2
5611 jg .w16_loop
5612 RET
5613 .w32_loop:
5614 add tmp1q, 32*2
5615 add tmp2q, 32*2
5616 W_MASK 0, 4, 0, 1, 1
5617 add dstq, strideq
5618 add maskq, 32
5619 .w32:
5620 vpermq m0, m0, q3120
5621 mova [dstq], m0
5622 mova [maskq], m4
5623 dec hd
5624 jg .w32_loop
5625 RET
5626 .w64_loop:
5627 add tmp1q, 32*4
5628 add tmp2q, 32*4
5629 W_MASK 0, 4, 0, 1, 1
5630 add dstq, strideq
5631 add maskq, 32*2
5632 .w64:
5633 vpermq m0, m0, q3120
5634 mova [dstq+32*0], m0
5635 mova [maskq+32*0], m4
5636 W_MASK 0, 4, 2, 3, 1
5637 vpermq m0, m0, q3120
5638 mova [dstq+32*1], m0
5639 mova [maskq+32*1], m4
5640 dec hd
5641 jg .w64_loop
5642 RET
5643 .w128_loop:
5644 add tmp1q, 32*8
5645 add tmp2q, 32*8
5646 W_MASK 0, 4, 0, 1, 1
5647 add dstq, strideq
5648 add maskq, 32*4
5649 .w128:
5650 vpermq m0, m0, q3120
5651 mova [dstq+32*0], m0
5652 mova [maskq+32*0], m4
5653 W_MASK 0, 4, 2, 3, 1
5654 vpermq m0, m0, q3120
5655 mova [dstq+32*1], m0
5656 mova [maskq+32*1], m4
5657 W_MASK 0, 4, 4, 5, 1
5658 vpermq m0, m0, q3120
5659 mova [dstq+32*2], m0
5660 mova [maskq+32*2], m4
5661 W_MASK 0, 4, 6, 7, 1
5662 vpermq m0, m0, q3120
5663 mova [dstq+32*3], m0
5664 mova [maskq+32*3], m4
5665 dec hd
5666 jg .w128_loop
5667 RET
5668
5669 %endif ; ARCH_X86_64