Debian Codegraph @

.cargo/vendor/rav1e/src/x86/mc16_sse.asm

raw
1 ; Copyright © 2021, VideoLAN and dav1d authors
2 ; Copyright © 2021, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28
29 SECTION_RODATA
30
31 ; dav1d_obmc_masks[] << 9
32 obmc_masks: dw 0, 0, 9728, 0, 12800, 7168, 2560, 0
33 dw 14336, 11264, 8192, 5632, 3584, 1536, 0, 0
34 dw 15360, 13824, 12288, 10752, 9216, 7680, 6144, 5120
35 dw 4096, 3072, 2048, 1536, 0, 0, 0, 0
36 dw 15872, 14848, 14336, 13312, 12288, 11776, 10752, 10240
37 dw 9728, 8704, 8192, 7168, 6656, 6144, 5632, 4608
38 dw 4096, 3584, 3072, 2560, 2048, 2048, 1536, 1024
39
40 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
41 spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9
42 spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13
43 spel_h_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9
44 spel_s_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7
45 spel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
46 unpckw: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
47 rescale_mul: dd 0, 1, 2, 3
48 resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7
49 db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15
50 bdct_lb_q: times 8 db 0
51 times 8 db 4
52 times 8 db 8
53 times 8 db 12
54
55 pw_2: times 8 dw 2
56 pw_16: times 4 dw 16
57 prep_mul: times 4 dw 16
58 times 8 dw 4
59 pw_64: times 8 dw 64
60 pw_256: times 8 dw 256
61 pw_2048: times 4 dw 2048
62 bidir_mul: times 4 dw 2048
63 pw_8192: times 8 dw 8192
64 pw_27615: times 8 dw 27615
65 pw_32766: times 8 dw 32766
66 pw_m512: times 8 dw -512
67 pd_63: times 4 dd 63
68 pd_64: times 4 dd 64
69 pd_512: times 4 dd 512
70 pd_m524256: times 4 dd -524256 ; -8192 << 6 + 32
71 pd_0x3ff: times 4 dd 0x3ff
72 pd_0x4000: times 4 dd 0x4000
73 pq_0x400000: times 2 dq 0x400000
74 pq_0x40000000: times 2 dq 0x40000000
75 pd_65538: times 2 dd 65538
76
77 put_bilin_h_rnd: times 4 dw 8
78 times 4 dw 10
79 s_8tap_h_rnd: times 2 dd 2
80 times 2 dd 8
81 put_s_8tap_v_rnd: times 2 dd 512
82 times 2 dd 128
83 s_8tap_h_sh: dd 2, 4
84 put_s_8tap_v_sh: dd 10, 8
85 bidir_rnd: times 4 dw -16400
86 times 4 dw -16388
87 put_8tap_h_rnd: dd 34, 34, 40, 40
88 prep_8tap_1d_rnd: times 2 dd 8 - (8192 << 4)
89 prep_8tap_2d_rnd: times 4 dd 32 - (8192 << 5)
90
91 warp8x8_shift: dd 11, 13
92 warp8x8_rnd1: dd 1024, 1024, 4096, 4096
93 warp8x8_rnd2: times 4 dw 4096
94 times 4 dw 16384
95 warp8x8t_rnd: times 2 dd 16384 - (8192 << 15)
96
97 %macro BIDIR_JMP_TABLE 2-*
98 %xdefine %1_%2_table (%%table - 2*%3)
99 %xdefine %%base %1_%2_table
100 %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2)
101 %%table:
102 %rep %0 - 2
103 dd %%prefix %+ .w%3 - %%base
104 %rotate 1
105 %endrep
106 %endmacro
107
108 BIDIR_JMP_TABLE avg, ssse3, 4, 8, 16, 32, 64, 128
109 BIDIR_JMP_TABLE w_avg, ssse3, 4, 8, 16, 32, 64, 128
110 BIDIR_JMP_TABLE mask, ssse3, 4, 8, 16, 32, 64, 128
111 BIDIR_JMP_TABLE w_mask_420, ssse3, 4, 8, 16, 32, 64, 128
112 BIDIR_JMP_TABLE w_mask_422, ssse3, 4, 8, 16, 32, 64, 128
113 BIDIR_JMP_TABLE w_mask_444, ssse3, 4, 8, 16, 32, 64, 128
114 BIDIR_JMP_TABLE blend, ssse3, 4, 8, 16, 32
115 BIDIR_JMP_TABLE blend_v, ssse3, 2, 4, 8, 16, 32
116 BIDIR_JMP_TABLE blend_h, ssse3, 2, 4, 8, 16, 32, 64, 128
117
118 %macro BASE_JMP_TABLE 3-*
119 %xdefine %1_%2_table (%%table - %3)
120 %xdefine %%base %1_%2
121 %%table:
122 %rep %0 - 2
123 dw %%base %+ _w%3 - %%base
124 %rotate 1
125 %endrep
126 %endmacro
127
128 %xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_16bpc_ssse3.put)
129 %xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_16bpc_ssse3.prep)
130
131 BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128
132 BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128
133
134 %macro SCALED_JMP_TABLE 2-*
135 %xdefine %1_%2_table (%%table - %3)
136 %xdefine %%base mangle(private_prefix %+ _%1_16bpc_%2)
137 %%table:
138 %rep %0 - 2
139 dw %%base %+ .w%3 - %%base
140 %rotate 1
141 %endrep
142 %rotate 2
143 %%dy_1024:
144 %xdefine %1_%2_dy1_table (%%dy_1024 - %3)
145 %rep %0 - 2
146 dw %%base %+ .dy1_w%3 - %%base
147 %rotate 1
148 %endrep
149 %rotate 2
150 %%dy_2048:
151 %xdefine %1_%2_dy2_table (%%dy_2048 - %3)
152 %rep %0 - 2
153 dw %%base %+ .dy2_w%3 - %%base
154 %rotate 1
155 %endrep
156 %endmacro
157
158 SCALED_JMP_TABLE put_8tap_scaled, ssse3, 2, 4, 8, 16, 32, 64, 128
159 SCALED_JMP_TABLE prep_8tap_scaled, ssse3, 4, 8, 16, 32, 64, 128
160
161 cextern mc_subpel_filters
162 %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
163
164 cextern mc_warp_filter
165 cextern resize_filter
166
167 SECTION .text
168
169 %if UNIX64
170 DECLARE_REG_TMP 7
171 %else
172 DECLARE_REG_TMP 5
173 %endif
174
175 INIT_XMM ssse3
176 cglobal put_bilin_16bpc, 4, 7, 0, dst, ds, src, ss, w, h, mxy
177 %define base t0-put_ssse3
178 mov mxyd, r6m ; mx
179 LEA t0, put_ssse3
180 movifnidn wd, wm
181 test mxyd, mxyd
182 jnz .h
183 mov mxyd, r7m ; my
184 test mxyd, mxyd
185 jnz .v
186 .put:
187 tzcnt wd, wd
188 movzx wd, word [base+put_ssse3_table+wq*2]
189 add wq, t0
190 movifnidn hd, hm
191 jmp wq
192 .put_w2:
193 mov r4d, [srcq+ssq*0]
194 mov r6d, [srcq+ssq*1]
195 lea srcq, [srcq+ssq*2]
196 mov [dstq+dsq*0], r4d
197 mov [dstq+dsq*1], r6d
198 lea dstq, [dstq+dsq*2]
199 sub hd, 2
200 jg .put_w2
201 RET
202 .put_w4:
203 movq m0, [srcq+ssq*0]
204 movq m1, [srcq+ssq*1]
205 lea srcq, [srcq+ssq*2]
206 movq [dstq+dsq*0], m0
207 movq [dstq+dsq*1], m1
208 lea dstq, [dstq+dsq*2]
209 sub hd, 2
210 jg .put_w4
211 RET
212 .put_w8:
213 movu m0, [srcq+ssq*0]
214 movu m1, [srcq+ssq*1]
215 lea srcq, [srcq+ssq*2]
216 mova [dstq+dsq*0], m0
217 mova [dstq+dsq*1], m1
218 lea dstq, [dstq+dsq*2]
219 sub hd, 2
220 jg .put_w8
221 RET
222 .put_w16:
223 movu m0, [srcq+ssq*0+16*0]
224 movu m1, [srcq+ssq*0+16*1]
225 movu m2, [srcq+ssq*1+16*0]
226 movu m3, [srcq+ssq*1+16*1]
227 lea srcq, [srcq+ssq*2]
228 mova [dstq+dsq*0+16*0], m0
229 mova [dstq+dsq*0+16*1], m1
230 mova [dstq+dsq*1+16*0], m2
231 mova [dstq+dsq*1+16*1], m3
232 lea dstq, [dstq+dsq*2]
233 sub hd, 2
234 jg .put_w16
235 RET
236 .put_w32:
237 movu m0, [srcq+16*0]
238 movu m1, [srcq+16*1]
239 movu m2, [srcq+16*2]
240 movu m3, [srcq+16*3]
241 add srcq, ssq
242 mova [dstq+16*0], m0
243 mova [dstq+16*1], m1
244 mova [dstq+16*2], m2
245 mova [dstq+16*3], m3
246 add dstq, dsq
247 dec hd
248 jg .put_w32
249 RET
250 .put_w64:
251 movu m0, [srcq+16*0]
252 movu m1, [srcq+16*1]
253 movu m2, [srcq+16*2]
254 movu m3, [srcq+16*3]
255 mova [dstq+16*0], m0
256 mova [dstq+16*1], m1
257 mova [dstq+16*2], m2
258 mova [dstq+16*3], m3
259 movu m0, [srcq+16*4]
260 movu m1, [srcq+16*5]
261 movu m2, [srcq+16*6]
262 movu m3, [srcq+16*7]
263 add srcq, ssq
264 mova [dstq+16*4], m0
265 mova [dstq+16*5], m1
266 mova [dstq+16*6], m2
267 mova [dstq+16*7], m3
268 add dstq, dsq
269 dec hd
270 jg .put_w64
271 RET
272 .put_w128:
273 add srcq, 16*8
274 add dstq, 16*8
275 .put_w128_loop:
276 movu m0, [srcq-16*8]
277 movu m1, [srcq-16*7]
278 movu m2, [srcq-16*6]
279 movu m3, [srcq-16*5]
280 mova [dstq-16*8], m0
281 mova [dstq-16*7], m1
282 mova [dstq-16*6], m2
283 mova [dstq-16*5], m3
284 movu m0, [srcq-16*4]
285 movu m1, [srcq-16*3]
286 movu m2, [srcq-16*2]
287 movu m3, [srcq-16*1]
288 mova [dstq-16*4], m0
289 mova [dstq-16*3], m1
290 mova [dstq-16*2], m2
291 mova [dstq-16*1], m3
292 movu m0, [srcq+16*0]
293 movu m1, [srcq+16*1]
294 movu m2, [srcq+16*2]
295 movu m3, [srcq+16*3]
296 mova [dstq+16*0], m0
297 mova [dstq+16*1], m1
298 mova [dstq+16*2], m2
299 mova [dstq+16*3], m3
300 movu m0, [srcq+16*4]
301 movu m1, [srcq+16*5]
302 movu m2, [srcq+16*6]
303 movu m3, [srcq+16*7]
304 add srcq, ssq
305 mova [dstq+16*4], m0
306 mova [dstq+16*5], m1
307 mova [dstq+16*6], m2
308 mova [dstq+16*7], m3
309 add dstq, dsq
310 dec hd
311 jg .put_w128_loop
312 RET
313 .h:
314 movd m5, mxyd
315 mov mxyd, r7m ; my
316 mova m4, [base+pw_16]
317 pshufb m5, [base+pw_256]
318 psubw m4, m5
319 test mxyd, mxyd
320 jnz .hv
321 ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v
322 mov r6d, r8m ; bitdepth_max
323 shr r6d, 11
324 movddup m3, [base+put_bilin_h_rnd+r6*8]
325 movifnidn hd, hm
326 sub wd, 8
327 jg .h_w16
328 je .h_w8
329 cmp wd, -4
330 je .h_w4
331 .h_w2:
332 movq m1, [srcq+ssq*0]
333 movhps m1, [srcq+ssq*1]
334 lea srcq, [srcq+ssq*2]
335 pmullw m0, m4, m1
336 psrlq m1, 16
337 pmullw m1, m5
338 paddw m0, m3
339 paddw m0, m1
340 psrlw m0, 4
341 movd [dstq+dsq*0], m0
342 punpckhqdq m0, m0
343 movd [dstq+dsq*1], m0
344 lea dstq, [dstq+dsq*2]
345 sub hd, 2
346 jg .h_w2
347 RET
348 .h_w4:
349 movq m0, [srcq+ssq*0]
350 movhps m0, [srcq+ssq*1]
351 movq m1, [srcq+ssq*0+2]
352 movhps m1, [srcq+ssq*1+2]
353 lea srcq, [srcq+ssq*2]
354 pmullw m0, m4
355 pmullw m1, m5
356 paddw m0, m3
357 paddw m0, m1
358 psrlw m0, 4
359 movq [dstq+dsq*0], m0
360 movhps [dstq+dsq*1], m0
361 lea dstq, [dstq+dsq*2]
362 sub hd, 2
363 jg .h_w4
364 RET
365 .h_w8:
366 movu m0, [srcq+ssq*0]
367 movu m1, [srcq+ssq*0+2]
368 pmullw m0, m4
369 pmullw m1, m5
370 paddw m0, m3
371 paddw m0, m1
372 movu m1, [srcq+ssq*1]
373 movu m2, [srcq+ssq*1+2]
374 lea srcq, [srcq+ssq*2]
375 pmullw m1, m4
376 pmullw m2, m5
377 paddw m1, m3
378 paddw m1, m2
379 psrlw m0, 4
380 psrlw m1, 4
381 mova [dstq+dsq*0], m0
382 mova [dstq+dsq*1], m1
383 lea dstq, [dstq+dsq*2]
384 sub hd, 2
385 jg .h_w8
386 RET
387 .h_w16:
388 lea srcq, [srcq+wq*2]
389 lea dstq, [dstq+wq*2]
390 neg wq
391 .h_w16_loop0:
392 mov r6, wq
393 .h_w16_loop:
394 movu m0, [srcq+r6*2+ 0]
395 movu m1, [srcq+r6*2+ 2]
396 pmullw m0, m4
397 pmullw m1, m5
398 paddw m0, m3
399 paddw m0, m1
400 movu m1, [srcq+r6*2+16]
401 movu m2, [srcq+r6*2+18]
402 pmullw m1, m4
403 pmullw m2, m5
404 paddw m1, m3
405 paddw m1, m2
406 psrlw m0, 4
407 psrlw m1, 4
408 mova [dstq+r6*2+16*0], m0
409 mova [dstq+r6*2+16*1], m1
410 add r6, 16
411 jl .h_w16_loop
412 add srcq, ssq
413 add dstq, dsq
414 dec hd
415 jg .h_w16_loop0
416 RET
417 .v:
418 shl mxyd, 11
419 movd m5, mxyd
420 pshufb m5, [base+pw_256]
421 movifnidn hd, hm
422 cmp wd, 4
423 jg .v_w8
424 je .v_w4
425 .v_w2:
426 movd m0, [srcq+ssq*0]
427 .v_w2_loop:
428 movd m1, [srcq+ssq*1]
429 lea srcq, [srcq+ssq*2]
430 punpcklqdq m2, m0, m1
431 movd m0, [srcq+ssq*0]
432 punpcklqdq m1, m0
433 psubw m1, m2
434 pmulhrsw m1, m5
435 paddw m1, m2
436 movd [dstq+dsq*0], m1
437 punpckhqdq m1, m1
438 movd [dstq+dsq*1], m1
439 lea dstq, [dstq+dsq*2]
440 sub hd, 2
441 jg .v_w2_loop
442 RET
443 .v_w4:
444 movq m0, [srcq+ssq*0]
445 .v_w4_loop:
446 movq m1, [srcq+ssq*1]
447 lea srcq, [srcq+ssq*2]
448 punpcklqdq m2, m0, m1
449 movq m0, [srcq+ssq*0]
450 punpcklqdq m1, m0
451 psubw m1, m2
452 pmulhrsw m1, m5
453 paddw m1, m2
454 movq [dstq+dsq*0], m1
455 movhps [dstq+dsq*1], m1
456 lea dstq, [dstq+dsq*2]
457 sub hd, 2
458 jg .v_w4_loop
459 RET
460 .v_w8:
461 %if ARCH_X86_64
462 %if WIN64
463 push r7
464 %endif
465 shl wd, 5
466 mov r7, srcq
467 lea r6d, [wq+hq-256]
468 mov r4, dstq
469 %else
470 mov r6, srcq
471 %endif
472 .v_w8_loop0:
473 movu m0, [srcq+ssq*0]
474 .v_w8_loop:
475 movu m3, [srcq+ssq*1]
476 lea srcq, [srcq+ssq*2]
477 psubw m1, m3, m0
478 pmulhrsw m1, m5
479 paddw m1, m0
480 movu m0, [srcq+ssq*0]
481 psubw m2, m0, m3
482 pmulhrsw m2, m5
483 paddw m2, m3
484 mova [dstq+dsq*0], m1
485 mova [dstq+dsq*1], m2
486 lea dstq, [dstq+dsq*2]
487 sub hd, 2
488 jg .v_w8_loop
489 %if ARCH_X86_64
490 add r7, 16
491 add r4, 16
492 movzx hd, r6b
493 mov srcq, r7
494 mov dstq, r4
495 sub r6d, 1<<8
496 %else
497 mov dstq, dstmp
498 add r6, 16
499 mov hd, hm
500 add dstq, 16
501 mov srcq, r6
502 mov dstmp, dstq
503 sub wd, 8
504 %endif
505 jg .v_w8_loop0
506 %if WIN64
507 pop r7
508 %endif
509 RET
510 .hv:
511 WIN64_SPILL_XMM 8
512 shl mxyd, 11
513 mova m3, [base+pw_2]
514 movd m6, mxyd
515 mova m7, [base+pw_8192]
516 pshufb m6, [base+pw_256]
517 test dword r8m, 0x800
518 jnz .hv_12bpc
519 psllw m4, 2
520 psllw m5, 2
521 mova m7, [base+pw_2048]
522 .hv_12bpc:
523 movifnidn hd, hm
524 cmp wd, 4
525 jg .hv_w8
526 je .hv_w4
527 .hv_w2:
528 movddup m0, [srcq+ssq*0]
529 pshufhw m1, m0, q0321
530 pmullw m0, m4
531 pmullw m1, m5
532 paddw m0, m3
533 paddw m0, m1
534 psrlw m0, 2
535 .hv_w2_loop:
536 movq m2, [srcq+ssq*1]
537 lea srcq, [srcq+ssq*2]
538 movhps m2, [srcq+ssq*0]
539 pmullw m1, m4, m2
540 psrlq m2, 16
541 pmullw m2, m5
542 paddw m1, m3
543 paddw m1, m2
544 psrlw m1, 2 ; 1 _ 2 _
545 shufpd m2, m0, m1, 0x01 ; 0 _ 1 _
546 mova m0, m1
547 psubw m1, m2
548 paddw m1, m1
549 pmulhw m1, m6
550 paddw m1, m2
551 pmulhrsw m1, m7
552 movd [dstq+dsq*0], m1
553 punpckhqdq m1, m1
554 movd [dstq+dsq*1], m1
555 lea dstq, [dstq+dsq*2]
556 sub hd, 2
557 jg .hv_w2_loop
558 RET
559 .hv_w4:
560 movddup m0, [srcq+ssq*0]
561 movddup m1, [srcq+ssq*0+2]
562 pmullw m0, m4
563 pmullw m1, m5
564 paddw m0, m3
565 paddw m0, m1
566 psrlw m0, 2
567 .hv_w4_loop:
568 movq m1, [srcq+ssq*1]
569 movq m2, [srcq+ssq*1+2]
570 lea srcq, [srcq+ssq*2]
571 movhps m1, [srcq+ssq*0]
572 movhps m2, [srcq+ssq*0+2]
573 pmullw m1, m4
574 pmullw m2, m5
575 paddw m1, m3
576 paddw m1, m2
577 psrlw m1, 2 ; 1 2
578 shufpd m2, m0, m1, 0x01 ; 0 1
579 mova m0, m1
580 psubw m1, m2
581 paddw m1, m1
582 pmulhw m1, m6
583 paddw m1, m2
584 pmulhrsw m1, m7
585 movq [dstq+dsq*0], m1
586 movhps [dstq+dsq*1], m1
587 lea dstq, [dstq+dsq*2]
588 sub hd, 2
589 jg .hv_w4_loop
590 RET
591 .hv_w8:
592 %if ARCH_X86_64
593 %if WIN64
594 push r7
595 %endif
596 shl wd, 5
597 lea r6d, [wq+hq-256]
598 mov r4, srcq
599 mov r7, dstq
600 %else
601 mov r6, srcq
602 %endif
603 .hv_w8_loop0:
604 movu m0, [srcq+ssq*0]
605 movu m1, [srcq+ssq*0+2]
606 pmullw m0, m4
607 pmullw m1, m5
608 paddw m0, m3
609 paddw m0, m1
610 psrlw m0, 2
611 .hv_w8_loop:
612 movu m1, [srcq+ssq*1]
613 movu m2, [srcq+ssq*1+2]
614 lea srcq, [srcq+ssq*2]
615 pmullw m1, m4
616 pmullw m2, m5
617 paddw m1, m3
618 paddw m1, m2
619 psrlw m1, 2
620 psubw m2, m1, m0
621 paddw m2, m2
622 pmulhw m2, m6
623 paddw m2, m0
624 pmulhrsw m2, m7
625 mova [dstq+dsq*0], m2
626 movu m0, [srcq+ssq*0]
627 movu m2, [srcq+ssq*0+2]
628 pmullw m0, m4
629 pmullw m2, m5
630 paddw m0, m3
631 paddw m0, m2
632 psrlw m0, 2
633 psubw m2, m0, m1
634 paddw m2, m2
635 pmulhw m2, m6
636 paddw m2, m1
637 pmulhrsw m2, m7
638 mova [dstq+dsq*1], m2
639 lea dstq, [dstq+dsq*2]
640 sub hd, 2
641 jg .hv_w8_loop
642 %if ARCH_X86_64
643 add r4, 16
644 add r7, 16
645 movzx hd, r6b
646 mov srcq, r4
647 mov dstq, r7
648 sub r6d, 1<<8
649 %else
650 mov dstq, dstmp
651 add r6, 16
652 mov hd, hm
653 add dstq, 16
654 mov srcq, r6
655 mov dstmp, dstq
656 sub wd, 8
657 %endif
658 jg .hv_w8_loop0
659 %if WIN64
660 pop r7
661 %endif
662 RET
663
664 cglobal prep_bilin_16bpc, 4, 7, 0, tmp, src, stride, w, h, mxy, stride3
665 %define base r6-prep_ssse3
666 movifnidn mxyd, r5m ; mx
667 LEA r6, prep_ssse3
668 movifnidn hd, hm
669 test mxyd, mxyd
670 jnz .h
671 mov mxyd, r6m ; my
672 test mxyd, mxyd
673 jnz .v
674 .prep:
675 tzcnt wd, wd
676 movzx wd, word [base+prep_ssse3_table+wq*2]
677 mov r5d, r7m ; bitdepth_max
678 mova m5, [base+pw_8192]
679 add wq, r6
680 shr r5d, 11
681 movddup m4, [base+prep_mul+r5*8]
682 lea stride3q, [strideq*3]
683 jmp wq
684 .prep_w4:
685 movq m0, [srcq+strideq*0]
686 movhps m0, [srcq+strideq*1]
687 movq m1, [srcq+strideq*2]
688 movhps m1, [srcq+stride3q ]
689 lea srcq, [srcq+strideq*4]
690 pmullw m0, m4
691 pmullw m1, m4
692 psubw m0, m5
693 psubw m1, m5
694 mova [tmpq+16*0], m0
695 mova [tmpq+16*1], m1
696 add tmpq, 16*2
697 sub hd, 4
698 jg .prep_w4
699 RET
700 .prep_w8:
701 movu m0, [srcq+strideq*0]
702 movu m1, [srcq+strideq*1]
703 movu m2, [srcq+strideq*2]
704 movu m3, [srcq+stride3q ]
705 lea srcq, [srcq+strideq*4]
706 REPX {pmullw x, m4}, m0, m1, m2, m3
707 REPX {psubw x, m5}, m0, m1, m2, m3
708 mova [tmpq+16*0], m0
709 mova [tmpq+16*1], m1
710 mova [tmpq+16*2], m2
711 mova [tmpq+16*3], m3
712 add tmpq, 16*4
713 sub hd, 4
714 jg .prep_w8
715 RET
716 .prep_w16:
717 movu m0, [srcq+strideq*0+16*0]
718 movu m1, [srcq+strideq*0+16*1]
719 movu m2, [srcq+strideq*1+16*0]
720 movu m3, [srcq+strideq*1+16*1]
721 lea srcq, [srcq+strideq*2]
722 REPX {pmullw x, m4}, m0, m1, m2, m3
723 REPX {psubw x, m5}, m0, m1, m2, m3
724 mova [tmpq+16*0], m0
725 mova [tmpq+16*1], m1
726 mova [tmpq+16*2], m2
727 mova [tmpq+16*3], m3
728 add tmpq, 16*4
729 sub hd, 2
730 jg .prep_w16
731 RET
732 .prep_w32:
733 movu m0, [srcq+16*0]
734 movu m1, [srcq+16*1]
735 movu m2, [srcq+16*2]
736 movu m3, [srcq+16*3]
737 add srcq, strideq
738 REPX {pmullw x, m4}, m0, m1, m2, m3
739 REPX {psubw x, m5}, m0, m1, m2, m3
740 mova [tmpq+16*0], m0
741 mova [tmpq+16*1], m1
742 mova [tmpq+16*2], m2
743 mova [tmpq+16*3], m3
744 add tmpq, 16*4
745 dec hd
746 jg .prep_w32
747 RET
748 .prep_w64:
749 movu m0, [srcq+16*0]
750 movu m1, [srcq+16*1]
751 movu m2, [srcq+16*2]
752 movu m3, [srcq+16*3]
753 REPX {pmullw x, m4}, m0, m1, m2, m3
754 REPX {psubw x, m5}, m0, m1, m2, m3
755 mova [tmpq+16*0], m0
756 mova [tmpq+16*1], m1
757 mova [tmpq+16*2], m2
758 mova [tmpq+16*3], m3
759 movu m0, [srcq+16*4]
760 movu m1, [srcq+16*5]
761 movu m2, [srcq+16*6]
762 movu m3, [srcq+16*7]
763 add srcq, strideq
764 REPX {pmullw x, m4}, m0, m1, m2, m3
765 REPX {psubw x, m5}, m0, m1, m2, m3
766 mova [tmpq+16*4], m0
767 mova [tmpq+16*5], m1
768 mova [tmpq+16*6], m2
769 mova [tmpq+16*7], m3
770 add tmpq, 16*8
771 dec hd
772 jg .prep_w64
773 RET
774 .prep_w128:
775 movu m0, [srcq+16* 0]
776 movu m1, [srcq+16* 1]
777 movu m2, [srcq+16* 2]
778 movu m3, [srcq+16* 3]
779 REPX {pmullw x, m4}, m0, m1, m2, m3
780 REPX {psubw x, m5}, m0, m1, m2, m3
781 mova [tmpq+16*0], m0
782 mova [tmpq+16*1], m1
783 mova [tmpq+16*2], m2
784 mova [tmpq+16*3], m3
785 movu m0, [srcq+16* 4]
786 movu m1, [srcq+16* 5]
787 movu m2, [srcq+16* 6]
788 movu m3, [srcq+16* 7]
789 REPX {pmullw x, m4}, m0, m1, m2, m3
790 REPX {psubw x, m5}, m0, m1, m2, m3
791 mova [tmpq+16*4], m0
792 mova [tmpq+16*5], m1
793 mova [tmpq+16*6], m2
794 mova [tmpq+16*7], m3
795 movu m0, [srcq+16* 8]
796 movu m1, [srcq+16* 9]
797 movu m2, [srcq+16*10]
798 movu m3, [srcq+16*11]
799 add tmpq, 16*16
800 REPX {pmullw x, m4}, m0, m1, m2, m3
801 REPX {psubw x, m5}, m0, m1, m2, m3
802 mova [tmpq-16*8], m0
803 mova [tmpq-16*7], m1
804 mova [tmpq-16*6], m2
805 mova [tmpq-16*5], m3
806 movu m0, [srcq+16*12]
807 movu m1, [srcq+16*13]
808 movu m2, [srcq+16*14]
809 movu m3, [srcq+16*15]
810 add srcq, strideq
811 REPX {pmullw x, m4}, m0, m1, m2, m3
812 REPX {psubw x, m5}, m0, m1, m2, m3
813 mova [tmpq-16*4], m0
814 mova [tmpq-16*3], m1
815 mova [tmpq-16*2], m2
816 mova [tmpq-16*1], m3
817 dec hd
818 jg .prep_w128
819 RET
820 .h:
821 movd m4, mxyd
822 mov mxyd, r6m ; my
823 mova m3, [base+pw_16]
824 pshufb m4, [base+pw_256]
825 mova m5, [base+pw_32766]
826 psubw m3, m4
827 test dword r7m, 0x800
828 jnz .h_12bpc
829 psllw m3, 2
830 psllw m4, 2
831 .h_12bpc:
832 test mxyd, mxyd
833 jnz .hv
834 sub wd, 8
835 je .h_w8
836 jg .h_w16
837 .h_w4:
838 movq m0, [srcq+strideq*0]
839 movhps m0, [srcq+strideq*1]
840 movq m1, [srcq+strideq*0+2]
841 movhps m1, [srcq+strideq*1+2]
842 lea srcq, [srcq+strideq*2]
843 pmullw m0, m3
844 pmullw m1, m4
845 psubw m0, m5
846 paddw m0, m1
847 psraw m0, 2
848 mova [tmpq], m0
849 add tmpq, 16
850 sub hd, 2
851 jg .h_w4
852 RET
853 .h_w8:
854 movu m0, [srcq+strideq*0]
855 movu m1, [srcq+strideq*0+2]
856 pmullw m0, m3
857 pmullw m1, m4
858 psubw m0, m5
859 paddw m0, m1
860 movu m1, [srcq+strideq*1]
861 movu m2, [srcq+strideq*1+2]
862 lea srcq, [srcq+strideq*2]
863 pmullw m1, m3
864 pmullw m2, m4
865 psubw m1, m5
866 paddw m1, m2
867 psraw m0, 2
868 psraw m1, 2
869 mova [tmpq+16*0], m0
870 mova [tmpq+16*1], m1
871 add tmpq, 16*2
872 sub hd, 2
873 jg .h_w8
874 RET
875 .h_w16:
876 lea srcq, [srcq+wq*2]
877 neg wq
878 .h_w16_loop0:
879 mov r6, wq
880 .h_w16_loop:
881 movu m0, [srcq+r6*2+ 0]
882 movu m1, [srcq+r6*2+ 2]
883 pmullw m0, m3
884 pmullw m1, m4
885 psubw m0, m5
886 paddw m0, m1
887 movu m1, [srcq+r6*2+16]
888 movu m2, [srcq+r6*2+18]
889 pmullw m1, m3
890 pmullw m2, m4
891 psubw m1, m5
892 paddw m1, m2
893 psraw m0, 2
894 psraw m1, 2
895 mova [tmpq+16*0], m0
896 mova [tmpq+16*1], m1
897 add tmpq, 16*2
898 add r6, 16
899 jl .h_w16_loop
900 add srcq, strideq
901 dec hd
902 jg .h_w16_loop0
903 RET
904 .v:
905 movd m4, mxyd
906 mova m3, [base+pw_16]
907 pshufb m4, [base+pw_256]
908 mova m5, [base+pw_32766]
909 psubw m3, m4
910 test dword r7m, 0x800
911 jnz .v_12bpc
912 psllw m3, 2
913 psllw m4, 2
914 .v_12bpc:
915 cmp wd, 8
916 je .v_w8
917 jg .v_w16
918 .v_w4:
919 movq m0, [srcq+strideq*0]
920 .v_w4_loop:
921 movq m2, [srcq+strideq*1]
922 lea srcq, [srcq+strideq*2]
923 punpcklqdq m1, m0, m2 ; 0 1
924 movq m0, [srcq+strideq*0]
925 punpcklqdq m2, m0 ; 1 2
926 pmullw m1, m3
927 pmullw m2, m4
928 psubw m1, m5
929 paddw m1, m2
930 psraw m1, 2
931 mova [tmpq], m1
932 add tmpq, 16
933 sub hd, 2
934 jg .v_w4_loop
935 RET
936 .v_w8:
937 movu m0, [srcq+strideq*0]
938 .v_w8_loop:
939 movu m2, [srcq+strideq*1]
940 lea srcq, [srcq+strideq*2]
941 pmullw m0, m3
942 pmullw m1, m4, m2
943 psubw m0, m5
944 paddw m1, m0
945 movu m0, [srcq+strideq*0]
946 psraw m1, 2
947 pmullw m2, m3
948 mova [tmpq+16*0], m1
949 pmullw m1, m4, m0
950 psubw m2, m5
951 paddw m1, m2
952 psraw m1, 2
953 mova [tmpq+16*1], m1
954 add tmpq, 16*2
955 sub hd, 2
956 jg .v_w8_loop
957 RET
958 .v_w16:
959 %if WIN64
960 push r7
961 %endif
962 mov r5, srcq
963 %if ARCH_X86_64
964 lea r6d, [wq*4-32]
965 mov wd, wd
966 lea r6d, [hq+r6*8]
967 mov r7, tmpq
968 %else
969 mov r6d, wd
970 %endif
971 .v_w16_loop0:
972 movu m0, [srcq+strideq*0]
973 .v_w16_loop:
974 movu m2, [srcq+strideq*1]
975 lea srcq, [srcq+strideq*2]
976 pmullw m0, m3
977 pmullw m1, m4, m2
978 psubw m0, m5
979 paddw m1, m0
980 movu m0, [srcq+strideq*0]
981 psraw m1, 2
982 pmullw m2, m3
983 mova [tmpq+wq*0], m1
984 pmullw m1, m4, m0
985 psubw m2, m5
986 paddw m1, m2
987 psraw m1, 2
988 mova [tmpq+wq*2], m1
989 lea tmpq, [tmpq+wq*4]
990 sub hd, 2
991 jg .v_w16_loop
992 %if ARCH_X86_64
993 add r5, 16
994 add r7, 16
995 movzx hd, r6b
996 mov srcq, r5
997 mov tmpq, r7
998 sub r6d, 1<<8
999 %else
1000 mov tmpq, tmpmp
1001 add r5, 16
1002 mov hd, hm
1003 add tmpq, 16
1004 mov srcq, r5
1005 mov tmpmp, tmpq
1006 sub r6d, 8
1007 %endif
1008 jg .v_w16_loop0
1009 %if WIN64
1010 pop r7
1011 %endif
1012 RET
1013 .hv:
1014 WIN64_SPILL_XMM 7
1015 shl mxyd, 11
1016 movd m6, mxyd
1017 pshufb m6, [base+pw_256]
1018 cmp wd, 8
1019 je .hv_w8
1020 jg .hv_w16
1021 .hv_w4:
1022 movddup m0, [srcq+strideq*0]
1023 movddup m1, [srcq+strideq*0+2]
1024 pmullw m0, m3
1025 pmullw m1, m4
1026 psubw m0, m5
1027 paddw m0, m1
1028 psraw m0, 2
1029 .hv_w4_loop:
1030 movq m1, [srcq+strideq*1]
1031 movq m2, [srcq+strideq*1+2]
1032 lea srcq, [srcq+strideq*2]
1033 movhps m1, [srcq+strideq*0]
1034 movhps m2, [srcq+strideq*0+2]
1035 pmullw m1, m3
1036 pmullw m2, m4
1037 psubw m1, m5
1038 paddw m1, m2
1039 psraw m1, 2 ; 1 2
1040 shufpd m2, m0, m1, 0x01 ; 0 1
1041 mova m0, m1
1042 psubw m1, m2
1043 pmulhrsw m1, m6
1044 paddw m1, m2
1045 mova [tmpq], m1
1046 add tmpq, 16
1047 sub hd, 2
1048 jg .hv_w4_loop
1049 RET
1050 .hv_w8:
1051 movu m0, [srcq+strideq*0]
1052 movu m1, [srcq+strideq*0+2]
1053 pmullw m0, m3
1054 pmullw m1, m4
1055 psubw m0, m5
1056 paddw m0, m1
1057 psraw m0, 2
1058 .hv_w8_loop:
1059 movu m1, [srcq+strideq*1]
1060 movu m2, [srcq+strideq*1+2]
1061 lea srcq, [srcq+strideq*2]
1062 pmullw m1, m3
1063 pmullw m2, m4
1064 psubw m1, m5
1065 paddw m1, m2
1066 psraw m1, 2
1067 psubw m2, m1, m0
1068 pmulhrsw m2, m6
1069 paddw m2, m0
1070 mova [tmpq+16*0], m2
1071 movu m0, [srcq+strideq*0]
1072 movu m2, [srcq+strideq*0+2]
1073 pmullw m0, m3
1074 pmullw m2, m4
1075 psubw m0, m5
1076 paddw m0, m2
1077 psraw m0, 2
1078 psubw m2, m0, m1
1079 pmulhrsw m2, m6
1080 paddw m2, m1
1081 mova [tmpq+16*1], m2
1082 add tmpq, 16*2
1083 sub hd, 2
1084 jg .hv_w8_loop
1085 RET
1086 .hv_w16:
1087 %if WIN64
1088 push r7
1089 %endif
1090 mov r5, srcq
1091 %if ARCH_X86_64
1092 lea r6d, [wq*4-32]
1093 mov wd, wd
1094 lea r6d, [hq+r6*8]
1095 mov r7, tmpq
1096 %else
1097 mov r6d, wd
1098 %endif
1099 .hv_w16_loop0:
1100 movu m0, [srcq+strideq*0]
1101 movu m1, [srcq+strideq*0+2]
1102 pmullw m0, m3
1103 pmullw m1, m4
1104 psubw m0, m5
1105 paddw m0, m1
1106 psraw m0, 2
1107 .hv_w16_loop:
1108 movu m1, [srcq+strideq*1]
1109 movu m2, [srcq+strideq*1+2]
1110 lea srcq, [srcq+strideq*2]
1111 pmullw m1, m3
1112 pmullw m2, m4
1113 psubw m1, m5
1114 paddw m1, m2
1115 psraw m1, 2
1116 psubw m2, m1, m0
1117 pmulhrsw m2, m6
1118 paddw m2, m0
1119 mova [tmpq+wq*0], m2
1120 movu m0, [srcq+strideq*0]
1121 movu m2, [srcq+strideq*0+2]
1122 pmullw m0, m3
1123 pmullw m2, m4
1124 psubw m0, m5
1125 paddw m0, m2
1126 psraw m0, 2
1127 psubw m2, m0, m1
1128 pmulhrsw m2, m6
1129 paddw m2, m1
1130 mova [tmpq+wq*2], m2
1131 lea tmpq, [tmpq+wq*4]
1132 sub hd, 2
1133 jg .hv_w16_loop
1134 %if ARCH_X86_64
1135 add r5, 16
1136 add r7, 16
1137 movzx hd, r6b
1138 mov srcq, r5
1139 mov tmpq, r7
1140 sub r6d, 1<<8
1141 %else
1142 mov tmpq, tmpmp
1143 add r5, 16
1144 mov hd, hm
1145 add tmpq, 16
1146 mov srcq, r5
1147 mov tmpmp, tmpq
1148 sub r6d, 8
1149 %endif
1150 jg .hv_w16_loop0
1151 %if WIN64
1152 pop r7
1153 %endif
1154 RET
1155
1156 ; int8_t subpel_filters[5][15][8]
1157 %assign FILTER_REGULAR (0*15 << 16) | 3*15
1158 %assign FILTER_SMOOTH (1*15 << 16) | 4*15
1159 %assign FILTER_SHARP (2*15 << 16) | 3*15
1160
1161 %macro FN 4 ; prefix, type, type_h, type_v
1162 cglobal %1_%2_16bpc
1163 mov t0d, FILTER_%3
1164 %ifidn %3, %4
1165 mov t1d, t0d
1166 %else
1167 mov t1d, FILTER_%4
1168 %endif
1169 %ifnidn %2, regular ; skip the jump in the last filter
1170 jmp mangle(private_prefix %+ _%1_16bpc %+ SUFFIX)
1171 %endif
1172 %endmacro
1173
1174 %if ARCH_X86_32
1175 DECLARE_REG_TMP 1, 2, 6
1176 %elif WIN64
1177 DECLARE_REG_TMP 4, 5, 8
1178 %else
1179 DECLARE_REG_TMP 7, 8, 8
1180 %endif
1181
1182 %define PUT_8TAP_FN FN put_8tap,
1183 PUT_8TAP_FN sharp, SHARP, SHARP
1184 PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH
1185 PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP
1186 PUT_8TAP_FN smooth, SMOOTH, SMOOTH
1187 PUT_8TAP_FN sharp_regular, SHARP, REGULAR
1188 PUT_8TAP_FN regular_sharp, REGULAR, SHARP
1189 PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR
1190 PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH
1191 PUT_8TAP_FN regular, REGULAR, REGULAR
1192
1193 %if ARCH_X86_32
1194 cglobal put_8tap_16bpc, 0, 7, 8, dst, ds, src, ss, w, h, mx, my
1195 %define mxb r0b
1196 %define mxd r0
1197 %define mxq r0
1198 %define myb r1b
1199 %define myd r1
1200 %define myq r1
1201 %define m8 [esp+16*0]
1202 %define m9 [esp+16*1]
1203 %define m10 [esp+16*2]
1204 %define m11 [esp+16*3]
1205 %define m12 [esp+16*4]
1206 %define m13 [esp+16*5]
1207 %define m14 [esp+16*6]
1208 %define m15 [esp+16*7]
1209 %else
1210 cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my
1211 %endif
1212 %define base t2-put_ssse3
1213 imul mxd, mxm, 0x010101
1214 add mxd, t0d ; 8tap_h, mx, 4tap_h
1215 imul myd, mym, 0x010101
1216 add myd, t1d ; 8tap_v, my, 4tap_v
1217 LEA t2, put_ssse3
1218 movifnidn wd, wm
1219 movifnidn srcq, srcmp
1220 movifnidn ssq, ssmp
1221 movifnidn hd, hm
1222 test mxd, 0xf00
1223 jnz .h
1224 test myd, 0xf00
1225 jnz .v
1226 tzcnt wd, wd
1227 movzx wd, word [base+put_ssse3_table+wq*2]
1228 movifnidn dstq, dstmp
1229 movifnidn dsq, dsmp
1230 add wq, t2
1231 %if WIN64
1232 pop r8
1233 pop r7
1234 %endif
1235 jmp wq
1236 .h:
1237 test myd, 0xf00
1238 jnz .hv
1239 mov myd, r8m
1240 movd m5, r8m
1241 shr myd, 11
1242 movddup m4, [base+put_8tap_h_rnd+myq*8]
1243 movifnidn dsq, dsmp
1244 pshufb m5, [base+pw_256]
1245 cmp wd, 4
1246 jg .h_w8
1247 movzx mxd, mxb
1248 lea srcq, [srcq-2]
1249 movq m3, [base+subpel_filters+mxq*8]
1250 movifnidn dstq, dstmp
1251 punpcklbw m3, m3
1252 psraw m3, 8 ; sign-extend
1253 je .h_w4
1254 .h_w2:
1255 mova m2, [base+spel_h_shuf2]
1256 pshufd m3, m3, q2121
1257 .h_w2_loop:
1258 movu m0, [srcq+ssq*0]
1259 movu m1, [srcq+ssq*1]
1260 lea srcq, [srcq+ssq*2]
1261 pshufb m0, m2
1262 pshufb m1, m2
1263 pmaddwd m0, m3
1264 pmaddwd m1, m3
1265 phaddd m0, m1
1266 paddd m0, m4
1267 psrad m0, 6
1268 packssdw m0, m0
1269 pxor m1, m1
1270 pminsw m0, m5
1271 pmaxsw m0, m1
1272 movd [dstq+dsq*0], m0
1273 pshuflw m0, m0, q3232
1274 movd [dstq+dsq*1], m0
1275 lea dstq, [dstq+dsq*2]
1276 sub hd, 2
1277 jg .h_w2_loop
1278 RET
1279 .h_w4:
1280 WIN64_SPILL_XMM 8
1281 mova m6, [base+spel_h_shufA]
1282 mova m7, [base+spel_h_shufB]
1283 pshufd m2, m3, q1111
1284 pshufd m3, m3, q2222
1285 .h_w4_loop:
1286 movu m1, [srcq]
1287 add srcq, ssq
1288 pshufb m0, m1, m6 ; 0 1 1 2 2 3 3 4
1289 pshufb m1, m7 ; 2 3 3 4 4 5 5 6
1290 pmaddwd m0, m2
1291 pmaddwd m1, m3
1292 paddd m0, m4
1293 paddd m0, m1
1294 psrad m0, 6
1295 packssdw m0, m0
1296 pxor m1, m1
1297 pminsw m0, m5
1298 pmaxsw m0, m1
1299 movq [dstq], m0
1300 add dstq, dsq
1301 dec hd
1302 jg .h_w4_loop
1303 RET
1304 .h_w8:
1305 %if WIN64
1306 %assign stack_offset stack_offset - stack_size_padded
1307 WIN64_SPILL_XMM 12
1308 %endif
1309 shr mxd, 16
1310 movq m3, [base+subpel_filters+mxq*8]
1311 movifnidn dstq, dstmp
1312 mova m6, [base+spel_h_shufA]
1313 mova m7, [base+spel_h_shufB]
1314 %if UNIX64
1315 mov wd, wd
1316 %endif
1317 lea srcq, [srcq+wq*2]
1318 punpcklbw m3, m3
1319 lea dstq, [dstq+wq*2]
1320 psraw m3, 8
1321 neg wq
1322 %if ARCH_X86_32
1323 ALLOC_STACK -16*4
1324 pshufd m0, m3, q0000
1325 pshufd m1, m3, q1111
1326 pshufd m2, m3, q2222
1327 pshufd m3, m3, q3333
1328 mova m8, m0
1329 mova m9, m1
1330 mova m10, m2
1331 mova m11, m3
1332 %else
1333 pshufd m8, m3, q0000
1334 pshufd m9, m3, q1111
1335 pshufd m10, m3, q2222
1336 pshufd m11, m3, q3333
1337 %endif
1338 .h_w8_loop0:
1339 mov r6, wq
1340 .h_w8_loop:
1341 movu m0, [srcq+r6*2- 6]
1342 movu m1, [srcq+r6*2+ 2]
1343 pshufb m2, m0, m6 ; 0 1 1 2 2 3 3 4
1344 pshufb m0, m7 ; 2 3 3 4 4 5 5 6
1345 pmaddwd m2, m8 ; abcd0
1346 pmaddwd m0, m9 ; abcd1
1347 pshufb m3, m1, m6 ; 4 5 5 6 6 7 7 8
1348 pshufb m1, m7 ; 6 7 7 8 8 9 9 a
1349 paddd m2, m4
1350 paddd m0, m2
1351 pmaddwd m2, m10, m3 ; abcd2
1352 pmaddwd m3, m8 ; efgh0
1353 paddd m0, m2
1354 pmaddwd m2, m11, m1 ; abcd3
1355 pmaddwd m1, m9 ; efgh1
1356 paddd m0, m2
1357 movu m2, [srcq+r6*2+10]
1358 paddd m3, m4
1359 paddd m1, m3
1360 pshufb m3, m2, m6 ; 8 9 9 a a b b c
1361 pshufb m2, m7 ; a b b c c d d e
1362 pmaddwd m3, m10 ; efgh2
1363 pmaddwd m2, m11 ; efgh3
1364 paddd m1, m3
1365 paddd m1, m2
1366 psrad m0, 6
1367 psrad m1, 6
1368 packssdw m0, m1
1369 pxor m1, m1
1370 pminsw m0, m5
1371 pmaxsw m0, m1
1372 mova [dstq+r6*2], m0
1373 add r6, 8
1374 jl .h_w8_loop
1375 add srcq, ssq
1376 add dstq, dsq
1377 dec hd
1378 jg .h_w8_loop0
1379 RET
1380 .v:
1381 movzx mxd, myb
1382 shr myd, 16
1383 cmp hd, 6
1384 cmovb myd, mxd
1385 movq m3, [base+subpel_filters+myq*8]
1386 %if STACK_ALIGNMENT < 16
1387 %xdefine rstk rsp
1388 %else
1389 %assign stack_offset stack_offset - stack_size_padded
1390 %endif
1391 %if WIN64
1392 WIN64_SPILL_XMM 15
1393 %endif
1394 movd m7, r8m
1395 movifnidn dstq, dstmp
1396 movifnidn dsq, dsmp
1397 punpcklbw m3, m3
1398 pshufb m7, [base+pw_256]
1399 psraw m3, 8 ; sign-extend
1400 %if ARCH_X86_32
1401 ALLOC_STACK -16*7
1402 pshufd m0, m3, q0000
1403 pshufd m1, m3, q1111
1404 pshufd m2, m3, q2222
1405 pshufd m3, m3, q3333
1406 mova m8, m0
1407 mova m9, m1
1408 mova m10, m2
1409 mova m11, m3
1410 %else
1411 pshufd m8, m3, q0000
1412 pshufd m9, m3, q1111
1413 pshufd m10, m3, q2222
1414 pshufd m11, m3, q3333
1415 %endif
1416 lea r6, [ssq*3]
1417 sub srcq, r6
1418 cmp wd, 2
1419 jne .v_w4
1420 .v_w2:
1421 movd m1, [srcq+ssq*0]
1422 movd m4, [srcq+ssq*1]
1423 movd m2, [srcq+ssq*2]
1424 add srcq, r6
1425 movd m5, [srcq+ssq*0]
1426 movd m3, [srcq+ssq*1]
1427 movd m6, [srcq+ssq*2]
1428 add srcq, r6
1429 movd m0, [srcq+ssq*0]
1430 punpckldq m1, m4 ; 0 1
1431 punpckldq m4, m2 ; 1 2
1432 punpckldq m2, m5 ; 2 3
1433 punpckldq m5, m3 ; 3 4
1434 punpckldq m3, m6 ; 4 5
1435 punpckldq m6, m0 ; 5 6
1436 punpcklwd m1, m4 ; 01 12
1437 punpcklwd m2, m5 ; 23 34
1438 punpcklwd m3, m6 ; 45 56
1439 pxor m6, m6
1440 .v_w2_loop:
1441 movd m4, [srcq+ssq*1]
1442 lea srcq, [srcq+ssq*2]
1443 pmaddwd m5, m8, m1 ; a0 b0
1444 mova m1, m2
1445 pmaddwd m2, m9 ; a1 b1
1446 paddd m5, m2
1447 mova m2, m3
1448 pmaddwd m3, m10 ; a2 b2
1449 paddd m5, m3
1450 punpckldq m3, m0, m4 ; 6 7
1451 movd m0, [srcq+ssq*0]
1452 punpckldq m4, m0 ; 7 8
1453 punpcklwd m3, m4 ; 67 78
1454 pmaddwd m4, m11, m3 ; a3 b3
1455 paddd m5, m4
1456 psrad m5, 5
1457 packssdw m5, m5
1458 pmaxsw m5, m6
1459 pavgw m5, m6
1460 pminsw m5, m7
1461 movd [dstq+dsq*0], m5
1462 pshuflw m5, m5, q3232
1463 movd [dstq+dsq*1], m5
1464 lea dstq, [dstq+dsq*2]
1465 sub hd, 2
1466 jg .v_w2_loop
1467 RET
1468 .v_w4:
1469 %if ARCH_X86_32
1470 shl wd, 14
1471 %if STACK_ALIGNMENT < 16
1472 mov [esp+4*29], srcq
1473 mov [esp+4*30], dstq
1474 %else
1475 mov srcmp, srcq
1476 %endif
1477 lea wd, [wq+hq-(1<<16)]
1478 %else
1479 shl wd, 6
1480 mov r7, srcq
1481 mov r8, dstq
1482 lea wd, [wq+hq-(1<<8)]
1483 %endif
1484 .v_w4_loop0:
1485 movq m1, [srcq+ssq*0]
1486 movq m2, [srcq+ssq*1]
1487 movq m3, [srcq+ssq*2]
1488 add srcq, r6
1489 movq m4, [srcq+ssq*0]
1490 movq m5, [srcq+ssq*1]
1491 movq m6, [srcq+ssq*2]
1492 add srcq, r6
1493 movq m0, [srcq+ssq*0]
1494 punpcklwd m1, m2 ; 01
1495 punpcklwd m2, m3 ; 12
1496 punpcklwd m3, m4 ; 23
1497 punpcklwd m4, m5 ; 34
1498 punpcklwd m5, m6 ; 45
1499 punpcklwd m6, m0 ; 56
1500 %if ARCH_X86_32
1501 jmp .v_w4_loop_start
1502 .v_w4_loop:
1503 mova m1, m12
1504 mova m2, m13
1505 mova m3, m14
1506 .v_w4_loop_start:
1507 pmaddwd m1, m8 ; a0
1508 pmaddwd m2, m8 ; b0
1509 mova m12, m3
1510 mova m13, m4
1511 pmaddwd m3, m9 ; a1
1512 pmaddwd m4, m9 ; b1
1513 paddd m1, m3
1514 paddd m2, m4
1515 mova m14, m5
1516 mova m4, m6
1517 pmaddwd m5, m10 ; a2
1518 pmaddwd m6, m10 ; b2
1519 paddd m1, m5
1520 paddd m2, m6
1521 movq m6, [srcq+ssq*1]
1522 lea srcq, [srcq+ssq*2]
1523 punpcklwd m5, m0, m6 ; 67
1524 movq m0, [srcq+ssq*0]
1525 pmaddwd m3, m11, m5 ; a3
1526 punpcklwd m6, m0 ; 78
1527 paddd m1, m3
1528 pmaddwd m3, m11, m6 ; b3
1529 paddd m2, m3
1530 psrad m1, 5
1531 psrad m2, 5
1532 packssdw m1, m2
1533 pxor m2, m2
1534 pmaxsw m1, m2
1535 pavgw m1, m2
1536 pminsw m1, m7
1537 movq [dstq+dsq*0], m1
1538 movhps [dstq+dsq*1], m1
1539 lea dstq, [dstq+dsq*2]
1540 sub hd, 2
1541 jg .v_w4_loop
1542 %if STACK_ALIGNMENT < 16
1543 mov srcq, [esp+4*29]
1544 mov dstq, [esp+4*30]
1545 movzx hd, ww
1546 add srcq, 8
1547 add dstq, 8
1548 mov [esp+4*29], srcq
1549 mov [esp+4*30], dstq
1550 %else
1551 mov srcq, srcmp
1552 mov dstq, dstmp
1553 movzx hd, ww
1554 add srcq, 8
1555 add dstq, 8
1556 mov srcmp, srcq
1557 mov dstmp, dstq
1558 %endif
1559 sub wd, 1<<16
1560 %else
1561 .v_w4_loop:
1562 pmaddwd m12, m8, m1 ; a0
1563 pmaddwd m13, m8, m2 ; b0
1564 mova m1, m3
1565 mova m2, m4
1566 pmaddwd m3, m9 ; a1
1567 pmaddwd m4, m9 ; b1
1568 paddd m12, m3
1569 paddd m13, m4
1570 mova m3, m5
1571 mova m4, m6
1572 pmaddwd m5, m10 ; a2
1573 pmaddwd m6, m10 ; b2
1574 paddd m12, m5
1575 paddd m13, m6
1576 movq m6, [srcq+ssq*1]
1577 lea srcq, [srcq+ssq*2]
1578 punpcklwd m5, m0, m6 ; 67
1579 movq m0, [srcq+ssq*0]
1580 pmaddwd m14, m11, m5 ; a3
1581 punpcklwd m6, m0 ; 78
1582 paddd m12, m14
1583 pmaddwd m14, m11, m6 ; b3
1584 paddd m13, m14
1585 psrad m12, 5
1586 psrad m13, 5
1587 packssdw m12, m13
1588 pxor m13, m13
1589 pmaxsw m12, m13
1590 pavgw m12, m13
1591 pminsw m12, m7
1592 movq [dstq+dsq*0], m12
1593 movhps [dstq+dsq*1], m12
1594 lea dstq, [dstq+dsq*2]
1595 sub hd, 2
1596 jg .v_w4_loop
1597 add r7, 8
1598 add r8, 8
1599 movzx hd, wb
1600 mov srcq, r7
1601 mov dstq, r8
1602 sub wd, 1<<8
1603 %endif
1604 jg .v_w4_loop0
1605 RET
1606 .hv:
1607 %if STACK_ALIGNMENT < 16
1608 %xdefine rstk rsp
1609 %else
1610 %assign stack_offset stack_offset - stack_size_padded
1611 %endif
1612 %if ARCH_X86_32
1613 movd m4, r8m
1614 mova m6, [base+pd_512]
1615 pshufb m4, [base+pw_256]
1616 %else
1617 %if WIN64
1618 ALLOC_STACK 16*6, 16
1619 %endif
1620 movd m15, r8m
1621 pshufb m15, [base+pw_256]
1622 %endif
1623 cmp wd, 4
1624 jg .hv_w8
1625 movzx mxd, mxb
1626 je .hv_w4
1627 movq m0, [base+subpel_filters+mxq*8]
1628 movzx mxd, myb
1629 shr myd, 16
1630 cmp hd, 6
1631 cmovb myd, mxd
1632 movq m3, [base+subpel_filters+myq*8]
1633 %if ARCH_X86_32
1634 mov dstq, dstmp
1635 mov dsq, dsmp
1636 mova m5, [base+spel_h_shuf2]
1637 ALLOC_STACK -16*8
1638 %else
1639 mova m6, [base+pd_512]
1640 mova m9, [base+spel_h_shuf2]
1641 %endif
1642 pshuflw m0, m0, q2121
1643 pxor m7, m7
1644 punpcklbw m7, m0
1645 punpcklbw m3, m3
1646 psraw m3, 8 ; sign-extend
1647 test dword r8m, 0x800
1648 jz .hv_w2_10bpc
1649 psraw m7, 2
1650 psllw m3, 2
1651 .hv_w2_10bpc:
1652 lea r6, [ssq*3]
1653 sub srcq, 2
1654 sub srcq, r6
1655 %if ARCH_X86_32
1656 pshufd m0, m3, q0000
1657 pshufd m1, m3, q1111
1658 pshufd m2, m3, q2222
1659 pshufd m3, m3, q3333
1660 mova m9, m5
1661 mova m11, m0
1662 mova m12, m1
1663 mova m13, m2
1664 mova m14, m3
1665 mova m15, m4
1666 %else
1667 pshufd m11, m3, q0000
1668 pshufd m12, m3, q1111
1669 pshufd m13, m3, q2222
1670 pshufd m14, m3, q3333
1671 %endif
1672 movu m2, [srcq+ssq*0]
1673 movu m3, [srcq+ssq*1]
1674 movu m1, [srcq+ssq*2]
1675 add srcq, r6
1676 movu m4, [srcq+ssq*0]
1677 %if ARCH_X86_32
1678 REPX {pshufb x, m5}, m2, m3, m1, m4
1679 %else
1680 REPX {pshufb x, m9}, m2, m3, m1, m4
1681 %endif
1682 REPX {pmaddwd x, m7}, m2, m3, m1, m4
1683 phaddd m2, m3 ; 0 1
1684 phaddd m1, m4 ; 2 3
1685 movu m3, [srcq+ssq*1]
1686 movu m4, [srcq+ssq*2]
1687 add srcq, r6
1688 movu m0, [srcq+ssq*0]
1689 %if ARCH_X86_32
1690 REPX {pshufb x, m5}, m3, m4, m0
1691 %else
1692 REPX {pshufb x, m9}, m3, m4, m0
1693 %endif
1694 REPX {pmaddwd x, m7}, m3, m4, m0
1695 phaddd m3, m4 ; 4 5
1696 phaddd m0, m0 ; 6 6
1697 REPX {paddd x, m6}, m2, m1, m3, m0
1698 REPX {psrad x, 10}, m2, m1, m3, m0
1699 packssdw m2, m1 ; 0 1 2 3
1700 packssdw m3, m0 ; 4 5 6 _
1701 palignr m4, m3, m2, 4 ; 1 2 3 4
1702 pshufd m5, m3, q0321 ; 5 6 _ _
1703 punpcklwd m1, m2, m4 ; 01 12
1704 punpckhwd m2, m4 ; 23 34
1705 punpcklwd m3, m5 ; 45 56
1706 .hv_w2_loop:
1707 movu m4, [srcq+ssq*1]
1708 lea srcq, [srcq+ssq*2]
1709 movu m5, [srcq+ssq*0]
1710 pshufb m4, m9
1711 pshufb m5, m9
1712 pmaddwd m4, m7
1713 pmaddwd m5, m7
1714 phaddd m4, m5
1715 pmaddwd m5, m11, m1 ; a0 b0
1716 mova m1, m2
1717 pmaddwd m2, m12 ; a1 b1
1718 paddd m5, m2
1719 mova m2, m3
1720 pmaddwd m3, m13 ; a2 b2
1721 paddd m5, m3
1722 paddd m4, m6
1723 psrad m4, 10 ; 7 8
1724 packssdw m0, m4
1725 pshufd m3, m0, q2103
1726 punpckhwd m3, m0 ; 67 78
1727 mova m0, m4
1728 pmaddwd m4, m14, m3 ; a3 b3
1729 paddd m5, m6
1730 paddd m5, m4
1731 psrad m5, 10
1732 packssdw m5, m5
1733 pxor m4, m4
1734 pminsw m5, m15
1735 pmaxsw m5, m4
1736 movd [dstq+dsq*0], m5
1737 pshuflw m5, m5, q3232
1738 movd [dstq+dsq*1], m5
1739 lea dstq, [dstq+dsq*2]
1740 sub hd, 2
1741 jg .hv_w2_loop
1742 RET
1743 .hv_w8:
1744 shr mxd, 16
1745 .hv_w4:
1746 movq m2, [base+subpel_filters+mxq*8]
1747 movzx mxd, myb
1748 shr myd, 16
1749 cmp hd, 6
1750 cmovb myd, mxd
1751 movq m3, [base+subpel_filters+myq*8]
1752 %if ARCH_X86_32
1753 %if STACK_ALIGNMENT < 16
1754 %xdefine rstk rsp
1755 %else
1756 %assign stack_offset stack_offset - stack_size_padded
1757 %endif
1758 mov dstq, dstmp
1759 mov dsq, dsmp
1760 mova m0, [base+spel_h_shufA]
1761 mova m1, [base+spel_h_shufB]
1762 ALLOC_STACK -16*15
1763 mova m8, m0
1764 mova m9, m1
1765 mova m14, m6
1766 %else
1767 mova m8, [base+spel_h_shufA]
1768 mova m9, [base+spel_h_shufB]
1769 %endif
1770 pxor m0, m0
1771 punpcklbw m0, m2
1772 punpcklbw m3, m3
1773 psraw m3, 8
1774 test dword r8m, 0x800
1775 jz .hv_w4_10bpc
1776 psraw m0, 2
1777 psllw m3, 2
1778 .hv_w4_10bpc:
1779 lea r6, [ssq*3]
1780 sub srcq, 6
1781 sub srcq, r6
1782 %if ARCH_X86_32
1783 %define tmp esp+16*8
1784 shl wd, 14
1785 %if STACK_ALIGNMENT < 16
1786 mov [esp+4*61], srcq
1787 mov [esp+4*62], dstq
1788 %else
1789 mov srcmp, srcq
1790 %endif
1791 mova [tmp+16*5], m4
1792 lea wd, [wq+hq-(1<<16)]
1793 pshufd m1, m0, q0000
1794 pshufd m2, m0, q1111
1795 pshufd m5, m0, q2222
1796 pshufd m0, m0, q3333
1797 mova m10, m1
1798 mova m11, m2
1799 mova m12, m5
1800 mova m13, m0
1801 %else
1802 %if WIN64
1803 %define tmp rsp
1804 %else
1805 %define tmp rsp-104 ; red zone
1806 %endif
1807 shl wd, 6
1808 mov r7, srcq
1809 mov r8, dstq
1810 lea wd, [wq+hq-(1<<8)]
1811 pshufd m10, m0, q0000
1812 pshufd m11, m0, q1111
1813 pshufd m12, m0, q2222
1814 pshufd m13, m0, q3333
1815 mova [tmp+16*5], m15
1816 %endif
1817 pshufd m0, m3, q0000
1818 pshufd m1, m3, q1111
1819 pshufd m2, m3, q2222
1820 pshufd m3, m3, q3333
1821 mova [tmp+16*1], m0
1822 mova [tmp+16*2], m1
1823 mova [tmp+16*3], m2
1824 mova [tmp+16*4], m3
1825 %macro PUT_8TAP_HV_H 4-5 m14 ; dst/src+0, src+8, tmp, shift, [pd_512]
1826 pshufb m%3, m%1, m8 ; 0 1 1 2 2 3 3 4
1827 pshufb m%1, m9 ; 2 3 3 4 4 5 5 6
1828 pmaddwd m%3, m10
1829 pmaddwd m%1, m11
1830 paddd m%3, %5
1831 paddd m%1, m%3
1832 pshufb m%3, m%2, m8 ; 4 5 5 6 6 7 7 8
1833 pshufb m%2, m9 ; 6 7 7 8 8 9 9 a
1834 pmaddwd m%3, m12
1835 pmaddwd m%2, m13
1836 paddd m%1, m%3
1837 paddd m%1, m%2
1838 psrad m%1, %4
1839 %endmacro
1840 .hv_w4_loop0:
1841 %if ARCH_X86_64
1842 mova m14, [pd_512]
1843 %endif
1844 movu m4, [srcq+ssq*0+0]
1845 movu m1, [srcq+ssq*0+8]
1846 movu m5, [srcq+ssq*1+0]
1847 movu m2, [srcq+ssq*1+8]
1848 movu m6, [srcq+ssq*2+0]
1849 movu m3, [srcq+ssq*2+8]
1850 add srcq, r6
1851 PUT_8TAP_HV_H 4, 1, 0, 10
1852 PUT_8TAP_HV_H 5, 2, 0, 10
1853 PUT_8TAP_HV_H 6, 3, 0, 10
1854 movu m7, [srcq+ssq*0+0]
1855 movu m2, [srcq+ssq*0+8]
1856 movu m1, [srcq+ssq*1+0]
1857 movu m3, [srcq+ssq*1+8]
1858 PUT_8TAP_HV_H 7, 2, 0, 10
1859 PUT_8TAP_HV_H 1, 3, 0, 10
1860 movu m2, [srcq+ssq*2+0]
1861 movu m3, [srcq+ssq*2+8]
1862 add srcq, r6
1863 PUT_8TAP_HV_H 2, 3, 0, 10
1864 packssdw m4, m7 ; 0 3
1865 packssdw m5, m1 ; 1 4
1866 movu m0, [srcq+ssq*0+0]
1867 movu m1, [srcq+ssq*0+8]
1868 PUT_8TAP_HV_H 0, 1, 3, 10
1869 packssdw m6, m2 ; 2 5
1870 packssdw m7, m0 ; 3 6
1871 punpcklwd m1, m4, m5 ; 01
1872 punpckhwd m4, m5 ; 34
1873 punpcklwd m2, m5, m6 ; 12
1874 punpckhwd m5, m6 ; 45
1875 punpcklwd m3, m6, m7 ; 23
1876 punpckhwd m6, m7 ; 56
1877 %if ARCH_X86_32
1878 jmp .hv_w4_loop_start
1879 .hv_w4_loop:
1880 mova m1, [tmp+16*6]
1881 mova m2, m15
1882 .hv_w4_loop_start:
1883 mova m7, [tmp+16*1]
1884 pmaddwd m1, m7 ; a0
1885 pmaddwd m2, m7 ; b0
1886 mova m7, [tmp+16*2]
1887 mova [tmp+16*6], m3
1888 pmaddwd m3, m7 ; a1
1889 mova m15, m4
1890 pmaddwd m4, m7 ; b1
1891 mova m7, [tmp+16*3]
1892 paddd m1, m3
1893 paddd m2, m4
1894 mova m3, m5
1895 pmaddwd m5, m7 ; a2
1896 mova m4, m6
1897 pmaddwd m6, m7 ; b2
1898 paddd m1, m5
1899 paddd m2, m6
1900 movu m7, [srcq+ssq*1+0]
1901 movu m5, [srcq+ssq*1+8]
1902 lea srcq, [srcq+ssq*2]
1903 PUT_8TAP_HV_H 7, 5, 6, 10
1904 packssdw m0, m7 ; 6 7
1905 mova [tmp+16*0], m0
1906 movu m0, [srcq+ssq*0+0]
1907 movu m5, [srcq+ssq*0+8]
1908 PUT_8TAP_HV_H 0, 5, 6, 10
1909 mova m6, [tmp+16*0]
1910 packssdw m7, m0 ; 7 8
1911 punpcklwd m5, m6, m7 ; 67
1912 punpckhwd m6, m7 ; 78
1913 pmaddwd m7, m5, [tmp+16*4]
1914 paddd m1, m7 ; a3
1915 pmaddwd m7, m6, [tmp+16*4]
1916 paddd m2, m7 ; b3
1917 psrad m1, 9
1918 psrad m2, 9
1919 packssdw m1, m2
1920 pxor m7, m7
1921 pmaxsw m1, m7
1922 pavgw m7, m1
1923 pminsw m7, [tmp+16*5]
1924 movq [dstq+dsq*0], m7
1925 movhps [dstq+dsq*1], m7
1926 lea dstq, [dstq+dsq*2]
1927 sub hd, 2
1928 jg .hv_w4_loop
1929 %if STACK_ALIGNMENT < 16
1930 mov srcq, [esp+4*61]
1931 mov dstq, [esp+4*62]
1932 add srcq, 8
1933 add dstq, 8
1934 mov [esp+4*61], srcq
1935 mov [esp+4*62], dstq
1936 %else
1937 mov srcq, srcmp
1938 mov dstq, dstmp
1939 add srcq, 8
1940 add dstq, 8
1941 mov srcmp, srcq
1942 mov dstmp, dstq
1943 %endif
1944 movzx hd, ww
1945 sub wd, 1<<16
1946 %else
1947 .hv_w4_loop:
1948 mova m15, [tmp+16*1]
1949 pmaddwd m14, m15, m1 ; a0
1950 pmaddwd m15, m2 ; b0
1951 mova m7, [tmp+16*2]
1952 mova m1, m3
1953 pmaddwd m3, m7 ; a1
1954 mova m2, m4
1955 pmaddwd m4, m7 ; b1
1956 mova m7, [tmp+16*3]
1957 paddd m14, m3
1958 paddd m15, m4
1959 mova m3, m5
1960 pmaddwd m5, m7 ; a2
1961 mova m4, m6
1962 pmaddwd m6, m7 ; b2
1963 paddd m14, m5
1964 paddd m15, m6
1965 movu m7, [srcq+ssq*1+0]
1966 movu m5, [srcq+ssq*1+8]
1967 lea srcq, [srcq+ssq*2]
1968 PUT_8TAP_HV_H 7, 5, 6, 10, [pd_512]
1969 packssdw m0, m7 ; 6 7
1970 mova [tmp+16*0], m0
1971 movu m0, [srcq+ssq*0+0]
1972 movu m5, [srcq+ssq*0+8]
1973 PUT_8TAP_HV_H 0, 5, 6, 10, [pd_512]
1974 mova m6, [tmp+16*0]
1975 packssdw m7, m0 ; 7 8
1976 punpcklwd m5, m6, m7 ; 67
1977 punpckhwd m6, m7 ; 78
1978 pmaddwd m7, m5, [tmp+16*4]
1979 paddd m14, m7 ; a3
1980 pmaddwd m7, m6, [tmp+16*4]
1981 paddd m15, m7 ; b3
1982 psrad m14, 9
1983 psrad m15, 9
1984 packssdw m14, m15
1985 pxor m7, m7
1986 pmaxsw m14, m7
1987 pavgw m7, m14
1988 pminsw m7, [tmp+16*5]
1989 movq [dstq+dsq*0], m7
1990 movhps [dstq+dsq*1], m7
1991 lea dstq, [dstq+dsq*2]
1992 sub hd, 2
1993 jg .hv_w4_loop
1994 add r7, 8
1995 add r8, 8
1996 movzx hd, wb
1997 mov srcq, r7
1998 mov dstq, r8
1999 sub wd, 1<<8
2000 %endif
2001 jg .hv_w4_loop0
2002 RET
2003 %undef tmp
2004
2005 %if ARCH_X86_32
2006 DECLARE_REG_TMP 2, 1, 6, 4
2007 %elif WIN64
2008 DECLARE_REG_TMP 6, 4, 7, 4
2009 %else
2010 DECLARE_REG_TMP 6, 7, 7, 8
2011 %endif
2012
2013 %define PREP_8TAP_FN FN prep_8tap,
2014 PREP_8TAP_FN sharp, SHARP, SHARP
2015 PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH
2016 PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP
2017 PREP_8TAP_FN smooth, SMOOTH, SMOOTH
2018 PREP_8TAP_FN sharp_regular, SHARP, REGULAR
2019 PREP_8TAP_FN regular_sharp, REGULAR, SHARP
2020 PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR
2021 PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH
2022 PREP_8TAP_FN regular, REGULAR, REGULAR
2023
2024 %if ARCH_X86_32
2025 cglobal prep_8tap_16bpc, 0, 7, 8, tmp, src, ss, w, h, mx, my
2026 %define mxb r0b
2027 %define mxd r0
2028 %define mxq r0
2029 %define myb r2b
2030 %define myd r2
2031 %define myq r2
2032 %else
2033 cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, ss, w, h, mx, my
2034 %endif
2035 %define base t2-prep_ssse3
2036 imul mxd, mxm, 0x010101
2037 add mxd, t0d ; 8tap_h, mx, 4tap_h
2038 imul myd, mym, 0x010101
2039 add myd, t1d ; 8tap_v, my, 4tap_v
2040 LEA t2, prep_ssse3
2041 movifnidn wd, wm
2042 movifnidn srcq, srcmp
2043 test mxd, 0xf00
2044 jnz .h
2045 movifnidn hd, hm
2046 test myd, 0xf00
2047 jnz .v
2048 tzcnt wd, wd
2049 mov myd, r7m ; bitdepth_max
2050 movzx wd, word [base+prep_ssse3_table+wq*2]
2051 mova m5, [base+pw_8192]
2052 shr myd, 11
2053 add wq, t2
2054 movddup m4, [base+prep_mul+myq*8]
2055 movifnidn ssq, ssmp
2056 movifnidn tmpq, tmpmp
2057 lea r6, [ssq*3]
2058 %if WIN64
2059 pop r7
2060 %endif
2061 jmp wq
2062 .h:
2063 test myd, 0xf00
2064 jnz .hv
2065 movifnidn ssq, r2mp
2066 movifnidn hd, r4m
2067 movddup m5, [base+prep_8tap_1d_rnd]
2068 cmp wd, 4
2069 jne .h_w8
2070 movzx mxd, mxb
2071 movq m0, [base+subpel_filters+mxq*8]
2072 mova m3, [base+spel_h_shufA]
2073 mova m4, [base+spel_h_shufB]
2074 movifnidn tmpq, tmpmp
2075 sub srcq, 2
2076 WIN64_SPILL_XMM 8
2077 punpcklbw m0, m0
2078 psraw m0, 8
2079 test dword r7m, 0x800
2080 jnz .h_w4_12bpc
2081 psllw m0, 2
2082 .h_w4_12bpc:
2083 pshufd m6, m0, q1111
2084 pshufd m7, m0, q2222
2085 .h_w4_loop:
2086 movu m1, [srcq+ssq*0]
2087 movu m2, [srcq+ssq*1]
2088 lea srcq, [srcq+ssq*2]
2089 pshufb m0, m1, m3 ; 0 1 1 2 2 3 3 4
2090 pshufb m1, m4 ; 2 3 3 4 4 5 5 6
2091 pmaddwd m0, m6
2092 pmaddwd m1, m7
2093 paddd m0, m5
2094 paddd m0, m1
2095 pshufb m1, m2, m3
2096 pshufb m2, m4
2097 pmaddwd m1, m6
2098 pmaddwd m2, m7
2099 paddd m1, m5
2100 paddd m1, m2
2101 psrad m0, 4
2102 psrad m1, 4
2103 packssdw m0, m1
2104 mova [tmpq], m0
2105 add tmpq, 16
2106 sub hd, 2
2107 jg .h_w4_loop
2108 RET
2109 .h_w8:
2110 WIN64_SPILL_XMM 11
2111 shr mxd, 16
2112 movq m2, [base+subpel_filters+mxq*8]
2113 mova m4, [base+spel_h_shufA]
2114 mova m6, [base+spel_h_shufB]
2115 movifnidn tmpq, r0mp
2116 add wd, wd
2117 punpcklbw m2, m2
2118 add srcq, wq
2119 psraw m2, 8
2120 add tmpq, wq
2121 neg wq
2122 test dword r7m, 0x800
2123 jnz .h_w8_12bpc
2124 psllw m2, 2
2125 .h_w8_12bpc:
2126 pshufd m7, m2, q0000
2127 %if ARCH_X86_32
2128 ALLOC_STACK -16*3
2129 pshufd m0, m2, q1111
2130 pshufd m1, m2, q2222
2131 pshufd m2, m2, q3333
2132 mova m8, m0
2133 mova m9, m1
2134 mova m10, m2
2135 %else
2136 pshufd m8, m2, q1111
2137 pshufd m9, m2, q2222
2138 pshufd m10, m2, q3333
2139 %endif
2140 .h_w8_loop0:
2141 mov r6, wq
2142 .h_w8_loop:
2143 movu m0, [srcq+r6- 6]
2144 movu m1, [srcq+r6+ 2]
2145 pshufb m2, m0, m4 ; 0 1 1 2 2 3 3 4
2146 pshufb m0, m6 ; 2 3 3 4 4 5 5 6
2147 pmaddwd m2, m7 ; abcd0
2148 pmaddwd m0, m8 ; abcd1
2149 pshufb m3, m1, m4 ; 4 5 5 6 6 7 7 8
2150 pshufb m1, m6 ; 6 7 7 8 8 9 9 a
2151 paddd m2, m5
2152 paddd m0, m2
2153 pmaddwd m2, m9, m3 ; abcd2
2154 pmaddwd m3, m7 ; efgh0
2155 paddd m0, m2
2156 pmaddwd m2, m10, m1 ; abcd3
2157 pmaddwd m1, m8 ; efgh1
2158 paddd m0, m2
2159 movu m2, [srcq+r6+10]
2160 paddd m3, m5
2161 paddd m1, m3
2162 pshufb m3, m2, m4 ; a b b c c d d e
2163 pshufb m2, m6 ; 8 9 9 a a b b c
2164 pmaddwd m3, m9 ; efgh2
2165 pmaddwd m2, m10 ; efgh3
2166 paddd m1, m3
2167 paddd m1, m2
2168 psrad m0, 4
2169 psrad m1, 4
2170 packssdw m0, m1
2171 mova [tmpq+r6], m0
2172 add r6, 16
2173 jl .h_w8_loop
2174 add srcq, ssq
2175 sub tmpq, wq
2176 dec hd
2177 jg .h_w8_loop0
2178 RET
2179 .v:
2180 movzx mxd, myb
2181 shr myd, 16
2182 cmp hd, 4
2183 cmove myd, mxd
2184 movq m3, [base+subpel_filters+myq*8]
2185 %if STACK_ALIGNMENT < 16
2186 %xdefine rstk rsp
2187 %else
2188 %assign stack_offset stack_offset - stack_size_padded
2189 %endif
2190 WIN64_SPILL_XMM 15
2191 movddup m7, [base+prep_8tap_1d_rnd]
2192 movifnidn ssq, r2mp
2193 movifnidn tmpq, r0mp
2194 punpcklbw m3, m3
2195 psraw m3, 8 ; sign-extend
2196 test dword r7m, 0x800
2197 jnz .v_12bpc
2198 psllw m3, 2
2199 .v_12bpc:
2200 %if ARCH_X86_32
2201 ALLOC_STACK -16*7
2202 pshufd m0, m3, q0000
2203 pshufd m1, m3, q1111
2204 pshufd m2, m3, q2222
2205 pshufd m3, m3, q3333
2206 mova m8, m0
2207 mova m9, m1
2208 mova m10, m2
2209 mova m11, m3
2210 %else
2211 pshufd m8, m3, q0000
2212 pshufd m9, m3, q1111
2213 pshufd m10, m3, q2222
2214 pshufd m11, m3, q3333
2215 %endif
2216 lea r6, [ssq*3]
2217 sub srcq, r6
2218 mov r6d, wd
2219 shl wd, 6
2220 mov r5, srcq
2221 %if ARCH_X86_64
2222 mov r7, tmpq
2223 %elif STACK_ALIGNMENT < 16
2224 mov [esp+4*29], tmpq
2225 %endif
2226 lea wd, [wq+hq-(1<<8)]
2227 .v_loop0:
2228 movq m1, [srcq+ssq*0]
2229 movq m2, [srcq+ssq*1]
2230 lea srcq, [srcq+ssq*2]
2231 movq m3, [srcq+ssq*0]
2232 movq m4, [srcq+ssq*1]
2233 lea srcq, [srcq+ssq*2]
2234 movq m5, [srcq+ssq*0]
2235 movq m6, [srcq+ssq*1]
2236 lea srcq, [srcq+ssq*2]
2237 movq m0, [srcq+ssq*0]
2238 punpcklwd m1, m2 ; 01
2239 punpcklwd m2, m3 ; 12
2240 punpcklwd m3, m4 ; 23
2241 punpcklwd m4, m5 ; 34
2242 punpcklwd m5, m6 ; 45
2243 punpcklwd m6, m0 ; 56
2244 %if ARCH_X86_32
2245 jmp .v_loop_start
2246 .v_loop:
2247 mova m1, m12
2248 mova m2, m13
2249 mova m3, m14
2250 .v_loop_start:
2251 pmaddwd m1, m8 ; a0
2252 pmaddwd m2, m8 ; b0
2253 mova m12, m3
2254 mova m13, m4
2255 pmaddwd m3, m9 ; a1
2256 pmaddwd m4, m9 ; b1
2257 paddd m1, m3
2258 paddd m2, m4
2259 mova m14, m5
2260 mova m4, m6
2261 pmaddwd m5, m10 ; a2
2262 pmaddwd m6, m10 ; b2
2263 paddd m1, m5
2264 paddd m2, m6
2265 movq m6, [srcq+ssq*1]
2266 lea srcq, [srcq+ssq*2]
2267 punpcklwd m5, m0, m6 ; 67
2268 movq m0, [srcq+ssq*0]
2269 pmaddwd m3, m11, m5 ; a3
2270 punpcklwd m6, m0 ; 78
2271 paddd m1, m7
2272 paddd m1, m3
2273 pmaddwd m3, m11, m6 ; b3
2274 paddd m2, m7
2275 paddd m2, m3
2276 psrad m1, 4
2277 psrad m2, 4
2278 packssdw m1, m2
2279 movq [tmpq+r6*0], m1
2280 movhps [tmpq+r6*2], m1
2281 lea tmpq, [tmpq+r6*4]
2282 sub hd, 2
2283 jg .v_loop
2284 %if STACK_ALIGNMENT < 16
2285 mov tmpq, [esp+4*29]
2286 add r5, 8
2287 add tmpq, 8
2288 mov srcq, r5
2289 mov [esp+4*29], tmpq
2290 %else
2291 mov tmpq, tmpmp
2292 add r5, 8
2293 add tmpq, 8
2294 mov srcq, r5
2295 mov tmpmp, tmpq
2296 %endif
2297 %else
2298 .v_loop:
2299 pmaddwd m12, m8, m1 ; a0
2300 pmaddwd m13, m8, m2 ; b0
2301 mova m1, m3
2302 mova m2, m4
2303 pmaddwd m3, m9 ; a1
2304 pmaddwd m4, m9 ; b1
2305 paddd m12, m3
2306 paddd m13, m4
2307 mova m3, m5
2308 mova m4, m6
2309 pmaddwd m5, m10 ; a2
2310 pmaddwd m6, m10 ; b2
2311 paddd m12, m5
2312 paddd m13, m6
2313 movq m6, [srcq+ssq*1]
2314 lea srcq, [srcq+ssq*2]
2315 punpcklwd m5, m0, m6 ; 67
2316 movq m0, [srcq+ssq*0]
2317 pmaddwd m14, m11, m5 ; a3
2318 punpcklwd m6, m0 ; 78
2319 paddd m12, m7
2320 paddd m12, m14
2321 pmaddwd m14, m11, m6 ; b3
2322 paddd m13, m7
2323 paddd m13, m14
2324 psrad m12, 4
2325 psrad m13, 4
2326 packssdw m12, m13
2327 movq [tmpq+r6*0], m12
2328 movhps [tmpq+r6*2], m12
2329 lea tmpq, [tmpq+r6*4]
2330 sub hd, 2
2331 jg .v_loop
2332 add r5, 8
2333 add r7, 8
2334 mov srcq, r5
2335 mov tmpq, r7
2336 %endif
2337 movzx hd, wb
2338 sub wd, 1<<8
2339 jg .v_loop0
2340 RET
2341 .hv:
2342 %if STACK_ALIGNMENT < 16
2343 %xdefine rstk rsp
2344 %else
2345 %assign stack_offset stack_offset - stack_size_padded
2346 %endif
2347 movzx t3d, mxb
2348 shr mxd, 16
2349 cmp wd, 4
2350 cmove mxd, t3d
2351 movifnidn hd, r4m
2352 movq m2, [base+subpel_filters+mxq*8]
2353 movzx mxd, myb
2354 shr myd, 16
2355 cmp hd, 4
2356 cmove myd, mxd
2357 movq m3, [base+subpel_filters+myq*8]
2358 %if ARCH_X86_32
2359 mov ssq, r2mp
2360 mov tmpq, r0mp
2361 mova m0, [base+spel_h_shufA]
2362 mova m1, [base+spel_h_shufB]
2363 mova m4, [base+prep_8tap_2d_rnd]
2364 ALLOC_STACK -16*14
2365 mova m8, m0
2366 mova m9, m1
2367 mova m14, m4
2368 %else
2369 %if WIN64
2370 ALLOC_STACK 16*6, 16
2371 %endif
2372 mova m8, [base+spel_h_shufA]
2373 mova m9, [base+spel_h_shufB]
2374 %endif
2375 pxor m0, m0
2376 punpcklbw m0, m2
2377 punpcklbw m3, m3
2378 psraw m0, 4
2379 psraw m3, 8
2380 test dword r7m, 0x800
2381 jz .hv_10bpc
2382 psraw m0, 2
2383 .hv_10bpc:
2384 lea r6, [ssq*3]
2385 sub srcq, 6
2386 sub srcq, r6
2387 mov r6d, wd
2388 shl wd, 6
2389 mov r5, srcq
2390 %if ARCH_X86_32
2391 %define tmp esp+16*8
2392 %if STACK_ALIGNMENT < 16
2393 mov [esp+4*61], tmpq
2394 %endif
2395 pshufd m1, m0, q0000
2396 pshufd m2, m0, q1111
2397 pshufd m5, m0, q2222
2398 pshufd m0, m0, q3333
2399 mova m10, m1
2400 mova m11, m2
2401 mova m12, m5
2402 mova m13, m0
2403 %else
2404 %if WIN64
2405 %define tmp rsp
2406 %else
2407 %define tmp rsp-88 ; red zone
2408 %endif
2409 mov r7, tmpq
2410 pshufd m10, m0, q0000
2411 pshufd m11, m0, q1111
2412 pshufd m12, m0, q2222
2413 pshufd m13, m0, q3333
2414 %endif
2415 lea wd, [wq+hq-(1<<8)]
2416 pshufd m0, m3, q0000
2417 pshufd m1, m3, q1111
2418 pshufd m2, m3, q2222
2419 pshufd m3, m3, q3333
2420 mova [tmp+16*1], m0
2421 mova [tmp+16*2], m1
2422 mova [tmp+16*3], m2
2423 mova [tmp+16*4], m3
2424 .hv_loop0:
2425 %if ARCH_X86_64
2426 mova m14, [prep_8tap_2d_rnd]
2427 %endif
2428 movu m4, [srcq+ssq*0+0]
2429 movu m1, [srcq+ssq*0+8]
2430 movu m5, [srcq+ssq*1+0]
2431 movu m2, [srcq+ssq*1+8]
2432 lea srcq, [srcq+ssq*2]
2433 movu m6, [srcq+ssq*0+0]
2434 movu m3, [srcq+ssq*0+8]
2435 PUT_8TAP_HV_H 4, 1, 0, 6
2436 PUT_8TAP_HV_H 5, 2, 0, 6
2437 PUT_8TAP_HV_H 6, 3, 0, 6
2438 movu m7, [srcq+ssq*1+0]
2439 movu m2, [srcq+ssq*1+8]
2440 lea srcq, [srcq+ssq*2]
2441 movu m1, [srcq+ssq*0+0]
2442 movu m3, [srcq+ssq*0+8]
2443 PUT_8TAP_HV_H 7, 2, 0, 6
2444 PUT_8TAP_HV_H 1, 3, 0, 6
2445 movu m2, [srcq+ssq*1+0]
2446 movu m3, [srcq+ssq*1+8]
2447 lea srcq, [srcq+ssq*2]
2448 PUT_8TAP_HV_H 2, 3, 0, 6
2449 packssdw m4, m7 ; 0 3
2450 packssdw m5, m1 ; 1 4
2451 movu m0, [srcq+ssq*0+0]
2452 movu m1, [srcq+ssq*0+8]
2453 PUT_8TAP_HV_H 0, 1, 3, 6
2454 packssdw m6, m2 ; 2 5
2455 packssdw m7, m0 ; 3 6
2456 punpcklwd m1, m4, m5 ; 01
2457 punpckhwd m4, m5 ; 34
2458 punpcklwd m2, m5, m6 ; 12
2459 punpckhwd m5, m6 ; 45
2460 punpcklwd m3, m6, m7 ; 23
2461 punpckhwd m6, m7 ; 56
2462 %if ARCH_X86_32
2463 jmp .hv_loop_start
2464 .hv_loop:
2465 mova m1, [tmp+16*5]
2466 mova m2, m15
2467 .hv_loop_start:
2468 mova m7, [tmp+16*1]
2469 pmaddwd m1, m7 ; a0
2470 pmaddwd m2, m7 ; b0
2471 mova m7, [tmp+16*2]
2472 mova [tmp+16*5], m3
2473 pmaddwd m3, m7 ; a1
2474 mova m15, m4
2475 pmaddwd m4, m7 ; b1
2476 mova m7, [tmp+16*3]
2477 paddd m1, m14
2478 paddd m2, m14
2479 paddd m1, m3
2480 paddd m2, m4
2481 mova m3, m5
2482 pmaddwd m5, m7 ; a2
2483 mova m4, m6
2484 pmaddwd m6, m7 ; b2
2485 paddd m1, m5
2486 paddd m2, m6
2487 movu m7, [srcq+ssq*1+0]
2488 movu m5, [srcq+ssq*1+8]
2489 lea srcq, [srcq+ssq*2]
2490 PUT_8TAP_HV_H 7, 5, 6, 6
2491 packssdw m0, m7 ; 6 7
2492 mova [tmp+16*0], m0
2493 movu m0, [srcq+ssq*0+0]
2494 movu m5, [srcq+ssq*0+8]
2495 PUT_8TAP_HV_H 0, 5, 6, 6
2496 mova m6, [tmp+16*0]
2497 packssdw m7, m0 ; 7 8
2498 punpcklwd m5, m6, m7 ; 67
2499 punpckhwd m6, m7 ; 78
2500 pmaddwd m7, m5, [tmp+16*4]
2501 paddd m1, m7 ; a3
2502 pmaddwd m7, m6, [tmp+16*4]
2503 paddd m2, m7 ; b3
2504 psrad m1, 6
2505 psrad m2, 6
2506 packssdw m1, m2
2507 movq [tmpq+r6*0], m1
2508 movhps [tmpq+r6*2], m1
2509 lea tmpq, [tmpq+r6*4]
2510 sub hd, 2
2511 jg .hv_loop
2512 %if STACK_ALIGNMENT < 16
2513 mov tmpq, [esp+4*61]
2514 add r5, 8
2515 add tmpq, 8
2516 mov srcq, r5
2517 mov [esp+4*61], tmpq
2518 %else
2519 mov tmpq, tmpmp
2520 add r5, 8
2521 add tmpq, 8
2522 mov srcq, r5
2523 mov tmpmp, tmpq
2524 %endif
2525 %else
2526 .hv_loop:
2527 mova m15, [tmp+16*1]
2528 mova m7, [prep_8tap_2d_rnd]
2529 pmaddwd m14, m15, m1 ; a0
2530 pmaddwd m15, m2 ; b0
2531 paddd m14, m7
2532 paddd m15, m7
2533 mova m7, [tmp+16*2]
2534 mova m1, m3
2535 pmaddwd m3, m7 ; a1
2536 mova m2, m4
2537 pmaddwd m4, m7 ; b1
2538 mova m7, [tmp+16*3]
2539 paddd m14, m3
2540 paddd m15, m4
2541 mova m3, m5
2542 pmaddwd m5, m7 ; a2
2543 mova m4, m6
2544 pmaddwd m6, m7 ; b2
2545 paddd m14, m5
2546 paddd m15, m6
2547 movu m7, [srcq+ssq*1+0]
2548 movu m5, [srcq+ssq*1+8]
2549 lea srcq, [srcq+ssq*2]
2550 PUT_8TAP_HV_H 7, 5, 6, 6, [prep_8tap_2d_rnd]
2551 packssdw m0, m7 ; 6 7
2552 mova [tmp+16*0], m0
2553 movu m0, [srcq+ssq*0+0]
2554 movu m5, [srcq+ssq*0+8]
2555 PUT_8TAP_HV_H 0, 5, 6, 6, [prep_8tap_2d_rnd]
2556 mova m6, [tmp+16*0]
2557 packssdw m7, m0 ; 7 8
2558 punpcklwd m5, m6, m7 ; 67
2559 punpckhwd m6, m7 ; 78
2560 pmaddwd m7, m5, [tmp+16*4]
2561 paddd m14, m7 ; a3
2562 pmaddwd m7, m6, [tmp+16*4]
2563 paddd m15, m7 ; b3
2564 psrad m14, 6
2565 psrad m15, 6
2566 packssdw m14, m15
2567 movq [tmpq+r6*0], m14
2568 movhps [tmpq+r6*2], m14
2569 lea tmpq, [tmpq+r6*4]
2570 sub hd, 2
2571 jg .hv_loop
2572 add r5, 8
2573 add r7, 8
2574 mov srcq, r5
2575 mov tmpq, r7
2576 %endif
2577 movzx hd, wb
2578 sub wd, 1<<8
2579 jg .hv_loop0
2580 RET
2581 %undef tmp
2582
2583 %macro movifprep 2
2584 %if isprep
2585 mov %1, %2
2586 %endif
2587 %endmacro
2588
2589 %macro SAVE_REG 1
2590 %xdefine r%1_save r%1
2591 %xdefine r%1q_save r%1q
2592 %xdefine r%1d_save r%1d
2593 %if ARCH_X86_32
2594 %define r%1m_save [rstk+stack_offset+(%1+1)*4]
2595 %endif
2596 %endmacro
2597
2598 %macro LOAD_REG 1
2599 %xdefine r%1 r%1_save
2600 %xdefine r%1q r%1q_save
2601 %xdefine r%1d r%1d_save
2602 %if ARCH_X86_32
2603 %define r%1m r%1m_save
2604 %endif
2605 %undef r%1d_save
2606 %undef r%1q_save
2607 %undef r%1_save
2608 %endmacro
2609
2610 %macro REMAP_REG 2-3
2611 %xdefine r%1 r%2
2612 %xdefine r%1q r%2q
2613 %xdefine r%1d r%2d
2614 %if ARCH_X86_32
2615 %if %3 == 0
2616 %xdefine r%1m r%2m
2617 %else
2618 %define r%1m [rstk+stack_offset+(%1+1)*4]
2619 %endif
2620 %endif
2621 %endmacro
2622
2623 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0
2624 %if isprep
2625 %if ARCH_X86_64
2626 SAVE_REG 14
2627 %assign %%i 14
2628 %rep 14
2629 %assign %%j %%i-1
2630 REMAP_REG %%i, %%j
2631 %assign %%i %%i-1
2632 %endrep
2633 %else
2634 SAVE_REG 5
2635 %assign %%i 5
2636 %rep 5
2637 %assign %%j %%i-1
2638 REMAP_REG %%i, %%j, 0
2639 %assign %%i %%i-1
2640 %endrep
2641 %endif
2642 %endif
2643 %endmacro
2644
2645 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0
2646 %if isprep
2647 %assign %%i 1
2648 %if ARCH_X86_64
2649 %rep 13
2650 %assign %%j %%i+1
2651 REMAP_REG %%i, %%j
2652 %assign %%i %%i+1
2653 %endrep
2654 LOAD_REG 14
2655 %else
2656 %rep 4
2657 %assign %%j %%i+1
2658 REMAP_REG %%i, %%j, 1
2659 %assign %%i %%i+1
2660 %endrep
2661 LOAD_REG 5
2662 %endif
2663 %endif
2664 %endmacro
2665
2666 %macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged
2667 MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
2668 RET
2669 %if %1
2670 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2671 %endif
2672 %endmacro
2673
2674 %if ARCH_X86_32
2675 %macro MC_4TAP_SCALED_H 1 ; dst_mem
2676 movu m7, [srcq+ssq*0]
2677 movu m2, [srcq+ssq*1]
2678 movu m5, [r4 +ssq*0]
2679 movu m6, [r4 +ssq*1]
2680 lea srcq, [srcq+ssq*2]
2681 lea r4, [r4 +ssq*2]
2682 REPX {pshufb x, m12}, m7, m2
2683 REPX {pmaddwd x, m13}, m7, m2
2684 REPX {pshufb x, m14}, m5, m6
2685 REPX {pmaddwd x, m15}, m5, m6
2686 phaddd m7, m5
2687 phaddd m2, m6
2688 mova m5, [esp+0x00]
2689 movd m6, [esp+0x10]
2690 paddd m7, m5
2691 paddd m2, m5
2692 psrad m7, m6
2693 psrad m2, m6
2694 packssdw m7, m2
2695 mova [stk+%1], m7
2696 %endmacro
2697 %endif
2698
2699 %if ARCH_X86_64
2700 %macro MC_8TAP_SCALED_H 8 ; dst, tmp[0-6]
2701 movu m%1, [srcq+ r4*2]
2702 movu m%2, [srcq+ r6*2]
2703 movu m%3, [srcq+ r7*2]
2704 movu m%4, [srcq+ r9*2]
2705 movu m%5, [srcq+r10*2]
2706 movu m%6, [srcq+r11*2]
2707 movu m%7, [srcq+r13*2]
2708 movu m%8, [srcq+ rX*2]
2709 add srcq, ssq
2710 pmaddwd m%1, [stk+0x10]
2711 pmaddwd m%2, [stk+0x20]
2712 pmaddwd m%3, [stk+0x30]
2713 pmaddwd m%4, [stk+0x40]
2714 pmaddwd m%5, [stk+0x50]
2715 pmaddwd m%6, [stk+0x60]
2716 pmaddwd m%7, [stk+0x70]
2717 pmaddwd m%8, [stk+0x80]
2718 phaddd m%1, m%2
2719 phaddd m%3, m%4
2720 phaddd m%5, m%6
2721 phaddd m%7, m%8
2722 phaddd m%1, m%3
2723 phaddd m%5, m%7
2724 paddd m%1, hround
2725 paddd m%5, hround
2726 psrad m%1, m12
2727 psrad m%5, m12
2728 packssdw m%1, m%5
2729 %endmacro
2730 %else
2731 %macro MC_8TAP_SCALED_H 2-3 1 ; weights_mem_start, h_mem, load_fh_offsets
2732 %if %3 == 1
2733 mov r0, [stk+ 0]
2734 mov rX, [stk+ 4]
2735 mov r4, [stk+ 8]
2736 mov r5, [stk+12]
2737 %endif
2738 movu m0, [srcq+r0*2]
2739 movu m1, [srcq+rX*2]
2740 movu m2, [srcq+r4*2]
2741 movu m3, [srcq+r5*2]
2742 mov r0, [stk+16]
2743 mov rX, [stk+20]
2744 mov r4, [stk+24]
2745 mov r5, [stk+28]
2746 pmaddwd m0, [stk+%1+0x00]
2747 pmaddwd m1, [stk+%1+0x10]
2748 pmaddwd m2, [stk+%1+0x20]
2749 pmaddwd m3, [stk+%1+0x30]
2750 phaddd m0, m1
2751 phaddd m2, m3
2752 movu m4, [srcq+r0*2]
2753 movu m5, [srcq+rX*2]
2754 movu m6, [srcq+r4*2]
2755 movu m7, [srcq+r5*2]
2756 add srcq, ssq
2757 pmaddwd m4, [stk+%1+0xa0]
2758 pmaddwd m5, [stk+%1+0xb0]
2759 pmaddwd m6, [stk+%1+0xc0]
2760 pmaddwd m7, [stk+%1+0xd0]
2761 phaddd m4, m5
2762 phaddd m6, m7
2763 phaddd m0, m2
2764 phaddd m4, m6
2765 paddd m0, hround
2766 paddd m4, hround
2767 psrad m0, m12
2768 psrad m4, m12
2769 packssdw m0, m4
2770 %if %2 != 0
2771 mova [stk+%2], m0
2772 %endif
2773 %endmacro
2774 %endif
2775
2776 %macro MC_8TAP_SCALED 1
2777 %ifidn %1, put
2778 %assign isput 1
2779 %assign isprep 0
2780 %if ARCH_X86_64
2781 %if required_stack_alignment <= STACK_ALIGNMENT
2782 cglobal put_8tap_scaled_16bpc, 2, 15, 16, 0x1c0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax
2783 %else
2784 cglobal put_8tap_scaled_16bpc, 2, 14, 16, 0x1c0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax
2785 %endif
2786 %else ; ARCH_X86_32
2787 %if required_stack_alignment <= STACK_ALIGNMENT
2788 cglobal put_8tap_scaled_16bpc, 0, 7, 8, 0x200, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax
2789 %else
2790 cglobal put_8tap_scaled_16bpc, 0, 7, 8, -0x200-0x30, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax
2791 %endif
2792 %endif
2793 %xdefine base_reg r12
2794 %else ; prep
2795 %assign isput 0
2796 %assign isprep 1
2797 %if ARCH_X86_64
2798 %if required_stack_alignment <= STACK_ALIGNMENT
2799 cglobal prep_8tap_scaled_16bpc, 2, 15, 16, 0x1c0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax
2800 %xdefine tmp_stridem r14q
2801 %else
2802 cglobal prep_8tap_scaled_16bpc, 2, 14, 16, 0x1c0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax
2803 %define tmp_stridem qword [stk+0x138]
2804 %endif
2805 %xdefine base_reg r11
2806 %else ; ARCH_X86_32
2807 %if required_stack_alignment <= STACK_ALIGNMENT
2808 cglobal prep_8tap_scaled_16bpc, 0, 7, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy, pxmax
2809 %else
2810 cglobal prep_8tap_scaled_16bpc, 0, 6, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy, pxmax
2811 %endif
2812 %define tmp_stridem dword [stk+0x138]
2813 %endif
2814 %endif
2815 %if ARCH_X86_32
2816 mov [esp+0x1f0], t0d
2817 mov [esp+0x1f4], t1d
2818 %if isput && required_stack_alignment > STACK_ALIGNMENT
2819 mov dstd, dstm
2820 mov dsd, dsm
2821 mov srcd, srcm
2822 mov ssd, ssm
2823 mov hd, hm
2824 mov r4, mxm
2825 %define r0m [esp+0x200]
2826 %define dsm [esp+0x204]
2827 %define dsmp dsm
2828 %define r1m dsm
2829 %define r2m [esp+0x208]
2830 %define ssm [esp+0x20c]
2831 %define r3m ssm
2832 %define hm [esp+0x210]
2833 %define mxm [esp+0x214]
2834 mov r0m, dstd
2835 mov dsm, dsd
2836 mov r2m, srcd
2837 mov ssm, ssd
2838 mov hm, hd
2839 mov r0, mym
2840 mov r1, dxm
2841 mov r2, dym
2842 %define mym [esp+0x218]
2843 %define dxm [esp+0x21c]
2844 %define dym [esp+0x220]
2845 mov mxm, r4
2846 mov mym, r0
2847 mov dxm, r1
2848 mov dym, r2
2849 tzcnt wd, wm
2850 %endif
2851 %if isput
2852 mov r3, pxmaxm
2853 %define pxmaxm r3
2854 %else
2855 mov r2, pxmaxm
2856 %endif
2857 %if isprep && required_stack_alignment > STACK_ALIGNMENT
2858 %xdefine base_reg r5
2859 %else
2860 %xdefine base_reg r6
2861 %endif
2862 %endif
2863 LEA base_reg, %1_8tap_scaled_16bpc_ssse3
2864 %xdefine base base_reg-%1_8tap_scaled_16bpc_ssse3
2865 %if ARCH_X86_64 || isprep || required_stack_alignment <= STACK_ALIGNMENT
2866 tzcnt wd, wm
2867 %endif
2868 %if ARCH_X86_64
2869 %if isput
2870 mov r7d, pxmaxm
2871 %endif
2872 %else
2873 %define m8 m0
2874 %define m9 m1
2875 %define m14 m4
2876 %define m15 m3
2877 %endif
2878 movd m8, dxm
2879 movd m14, mxm
2880 %if isput
2881 movd m15, pxmaxm
2882 %endif
2883 pshufd m8, m8, q0000
2884 pshufd m14, m14, q0000
2885 %if isput
2886 pshuflw m15, m15, q0000
2887 punpcklqdq m15, m15
2888 %endif
2889 %if isprep
2890 %if UNIX64
2891 mov r5d, t0d
2892 DECLARE_REG_TMP 5, 7
2893 %endif
2894 %if ARCH_X86_64
2895 mov r6d, pxmaxm
2896 %endif
2897 %endif
2898 %if ARCH_X86_64
2899 mov dyd, dym
2900 %endif
2901 %if isput
2902 %if WIN64
2903 mov r8d, hm
2904 DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3
2905 %define hm r5m
2906 %define dxm r8m
2907 %elif ARCH_X86_64
2908 DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3
2909 %define hm r6m
2910 %else
2911 %endif
2912 %if ARCH_X86_64
2913 %if required_stack_alignment > STACK_ALIGNMENT
2914 %define dsm [rsp+0x138]
2915 %define rX r1
2916 %define rXd r1d
2917 %else
2918 %define dsm dsq
2919 %define rX r14
2920 %define rXd r14d
2921 %endif
2922 %else
2923 %define rX r1
2924 %endif
2925 %else ; prep
2926 %if WIN64
2927 mov r7d, hm
2928 DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3
2929 %define hm r4m
2930 %define dxm r7m
2931 %elif ARCH_X86_64
2932 DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3
2933 %xdefine hm r7m
2934 %endif
2935 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2936 %if ARCH_X86_64
2937 %define rX r14
2938 %define rXd r14d
2939 %else
2940 %define rX r3
2941 %endif
2942 %endif
2943 %if ARCH_X86_64
2944 shr r7d, 11
2945 mova m10, [base+pd_0x3ff]
2946 movddup m11, [base+s_8tap_h_rnd+r7*8]
2947 movd m12, [base+s_8tap_h_sh+r7*4]
2948 %if isput
2949 movddup m13, [base+put_s_8tap_v_rnd+r7*8]
2950 movd m7, [base+put_s_8tap_v_sh+r7*4]
2951 %define pxmaxm [rsp]
2952 mova pxmaxm, m15
2953 punpcklqdq m12, m7
2954 %endif
2955 lea ss3q, [ssq*3]
2956 movzx r7d, t1b
2957 shr t1d, 16
2958 cmp hd, 6
2959 cmovs t1d, r7d
2960 sub srcq, ss3q
2961 %else
2962 %define m10 [base+pd_0x3ff]
2963 %define m11 [esp+0x00]
2964 %define m12 [esp+0x10]
2965 shr r3, 11
2966 movddup m1, [base+s_8tap_h_rnd+r3*8]
2967 movd m2, [base+s_8tap_h_sh+r3*4]
2968 %if isput
2969 %define m13 [esp+0x20]
2970 %define pxmaxm [esp+0x30]
2971 %define stk esp+0x40
2972 movddup m5, [base+put_s_8tap_v_rnd+r3*8]
2973 movd m6, [base+put_s_8tap_v_sh+r3*4]
2974 mova pxmaxm, m15
2975 punpcklqdq m2, m6
2976 mova m13, m5
2977 %else
2978 %define m13 [base+pd_m524256]
2979 %endif
2980 mov ssd, ssm
2981 mova m11, m1
2982 mova m12, m2
2983 MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
2984 mov r1, [esp+0x1f4]
2985 lea r0, [ssd*3]
2986 movzx r2, r1b
2987 shr r1, 16
2988 cmp dword hm, 6
2989 cmovs r1, r2
2990 mov [esp+0x1f4], r1
2991 %if isprep
2992 mov r1, r1m
2993 %endif
2994 mov r2, r2m
2995 sub srcq, r0
2996 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
2997 %define ss3q r0
2998 %define myd r4
2999 %define dyd dword dym
3000 %define hd dword hm
3001 %endif
3002 cmp dyd, 1024
3003 je .dy1
3004 cmp dyd, 2048
3005 je .dy2
3006 movzx wd, word [base+%1_8tap_scaled_ssse3_table+wq*2]
3007 add wq, base_reg
3008 jmp wq
3009 %if isput
3010 .w2:
3011 %if ARCH_X86_64
3012 mov myd, mym
3013 movzx t0d, t0b
3014 sub srcq, 2
3015 movd m15, t0d
3016 %else
3017 movzx r4, byte [esp+0x1f0]
3018 sub srcq, 2
3019 movd m15, r4
3020 %endif
3021 pxor m9, m9
3022 punpckldq m9, m8
3023 paddd m14, m9 ; mx+dx*[0-1]
3024 %if ARCH_X86_64
3025 mova m9, [base+pd_0x4000]
3026 %endif
3027 pshufd m15, m15, q0000
3028 pand m8, m14, m10
3029 psrld m8, 6
3030 paddd m15, m8
3031 movd r4d, m15
3032 pshufd m15, m15, q0321
3033 %if ARCH_X86_64
3034 movd r6d, m15
3035 %else
3036 movd r3d, m15
3037 %endif
3038 mova m5, [base+bdct_lb_q]
3039 mova m6, [base+spel_s_shuf2]
3040 movd m15, [base+subpel_filters+r4*8+2]
3041 %if ARCH_X86_64
3042 movd m7, [base+subpel_filters+r6*8+2]
3043 %else
3044 movd m7, [base+subpel_filters+r3*8+2]
3045 %endif
3046 pxor m2, m2
3047 pcmpeqd m8, m2
3048 psrld m14, 10
3049 paddd m14, m14
3050 %if ARCH_X86_32
3051 mov r3, r3m
3052 pshufb m14, m5
3053 paddb m14, m6
3054 mova [stk], m14
3055 SWAP m5, m0
3056 SWAP m6, m3
3057 %define m15 m6
3058 %endif
3059 movu m0, [srcq+ssq*0]
3060 movu m1, [srcq+ssq*1]
3061 movu m2, [srcq+ssq*2]
3062 movu m3, [srcq+ss3q ]
3063 lea srcq, [srcq+ssq*4]
3064 punpckldq m15, m7
3065 %if ARCH_X86_64
3066 pshufb m14, m5
3067 paddb m14, m6
3068 pand m9, m8
3069 pandn m8, m15
3070 SWAP m15, m8
3071 por m15, m9
3072 movu m4, [srcq+ssq*0]
3073 movu m5, [srcq+ssq*1]
3074 movu m6, [srcq+ssq*2]
3075 movu m7, [srcq+ss3q ]
3076 lea srcq, [srcq+ssq*4]
3077 %else
3078 pand m7, m5, [base+pd_0x4000]
3079 pandn m5, m15
3080 por m5, m7
3081 %define m15 m5
3082 %endif
3083 punpcklbw m15, m15
3084 psraw m15, 8
3085 REPX {pshufb x, m14}, m0, m1, m2, m3
3086 REPX {pmaddwd x, m15}, m0, m1, m2, m3
3087 %if ARCH_X86_64
3088 REPX {pshufb x, m14}, m4, m5, m6, m7
3089 REPX {pmaddwd x, m15}, m4, m5, m6, m7
3090 phaddd m0, m1
3091 phaddd m2, m3
3092 phaddd m4, m5
3093 phaddd m6, m7
3094 REPX {paddd x, m11}, m0, m2, m4, m6
3095 REPX {psrad x, m12}, m0, m2, m4, m6
3096 packssdw m0, m2 ; 0 1 2 3
3097 packssdw m4, m6 ; 4 5 6 7
3098 SWAP m1, m4
3099 %else
3100 mova [stk+0x10], m15
3101 phaddd m0, m1
3102 phaddd m2, m3
3103 movu m1, [srcq+ssq*0]
3104 movu m7, [srcq+ssq*1]
3105 movu m6, [srcq+ssq*2]
3106 movu m3, [srcq+ss3q ]
3107 lea srcq, [srcq+ssq*4]
3108 REPX {pshufb x, m14}, m1, m7, m6, m3
3109 REPX {pmaddwd x, m15}, m1, m7, m6, m3
3110 phaddd m1, m7
3111 phaddd m6, m3
3112 REPX {paddd x, m11}, m0, m2, m1, m6
3113 REPX {psrad x, m12}, m0, m2, m1, m6
3114 packssdw m0, m2
3115 packssdw m1, m6
3116 %define m14 [stk+0x00]
3117 %define m15 [stk+0x10]
3118 %endif
3119 palignr m2, m1, m0, 4 ; 1 2 3 4
3120 punpcklwd m3, m0, m2 ; 01 12
3121 punpckhwd m0, m2 ; 23 34
3122 pshufd m5, m1, q0321 ; 5 6 7 _
3123 punpcklwd m2, m1, m5 ; 45 56
3124 punpckhwd m4, m1, m5 ; 67 __
3125 %if ARCH_X86_32
3126 mov myd, mym
3127 mov r0, r0m
3128 mova [stk+0x20], m3
3129 mova [stk+0x30], m0
3130 mova [stk+0x40], m2
3131 mova [stk+0x50], m4
3132 %endif
3133 .w2_loop:
3134 and myd, 0x3ff
3135 %if ARCH_X86_64
3136 mov r6d, 64 << 24
3137 mov r4d, myd
3138 shr r4d, 6
3139 lea r4d, [t1+r4]
3140 cmovnz r6q, [base+subpel_filters+r4*8]
3141 movq m10, r6q
3142 punpcklbw m10, m10
3143 psraw m10, 8
3144 pshufd m7, m10, q0000
3145 pshufd m8, m10, q1111
3146 pmaddwd m5, m3, m7
3147 pmaddwd m6, m0, m8
3148 pshufd m9, m10, q2222
3149 pshufd m10, m10, q3333
3150 pmaddwd m7, m2, m9
3151 pmaddwd m8, m4, m10
3152 paddd m5, m6
3153 paddd m7, m8
3154 %else
3155 mov r1, [esp+0x1f4]
3156 xor r3, r3
3157 mov r5, myd
3158 shr r5, 6
3159 lea r1, [r1+r5]
3160 mov r5, 64 << 24
3161 cmovnz r3, [base+subpel_filters+r1*8+4]
3162 cmovnz r5, [base+subpel_filters+r1*8+0]
3163 movd m6, r3
3164 movd m7, r5
3165 punpckldq m7, m6
3166 punpcklbw m7, m7
3167 psraw m7, 8
3168 pshufd m5, m7, q0000
3169 pshufd m6, m7, q1111
3170 pmaddwd m3, m5
3171 pmaddwd m0, m6
3172 pshufd m5, m7, q2222
3173 pshufd m7, m7, q3333
3174 pmaddwd m2, m5
3175 pmaddwd m4, m7
3176 paddd m3, m0
3177 paddd m2, m4
3178 SWAP m5, m3
3179 SWAP m7, m2
3180 %define m8 m3
3181 %endif
3182 paddd m5, m13
3183 pshufd m6, m12, q1032
3184 pxor m8, m8
3185 paddd m5, m7
3186 psrad m5, m6
3187 packssdw m5, m5
3188 pmaxsw m5, m8
3189 pminsw m5, pxmaxm
3190 movd [dstq], m5
3191 add dstq, dsmp
3192 dec hd
3193 jz .ret
3194 %if ARCH_X86_64
3195 add myd, dyd
3196 %else
3197 add myd, dym
3198 %endif
3199 test myd, ~0x3ff
3200 %if ARCH_X86_32
3201 SWAP m3, m5
3202 SWAP m2, m7
3203 mova m3, [stk+0x20]
3204 mova m0, [stk+0x30]
3205 mova m2, [stk+0x40]
3206 mova m4, [stk+0x50]
3207 %endif
3208 jz .w2_loop
3209 %if ARCH_X86_32
3210 mov r3, r3m
3211 %endif
3212 movu m5, [srcq]
3213 test myd, 0x400
3214 jz .w2_skip_line
3215 add srcq, ssq
3216 shufps m3, m0, q1032 ; 01 12
3217 shufps m0, m2, q1032 ; 23 34
3218 shufps m2, m4, q1032 ; 45 56
3219 pshufb m5, m14
3220 pmaddwd m5, m15
3221 phaddd m5, m5
3222 paddd m5, m11
3223 psrad m5, m12
3224 packssdw m5, m5
3225 palignr m4, m5, m1, 12
3226 punpcklqdq m1, m4, m4 ; 6 7 6 7
3227 punpcklwd m4, m1, m5 ; 67 __
3228 %if ARCH_X86_32
3229 mova [stk+0x20], m3
3230 mova [stk+0x30], m0
3231 mova [stk+0x40], m2
3232 mova [stk+0x50], m4
3233 %endif
3234 jmp .w2_loop
3235 .w2_skip_line:
3236 movu m6, [srcq+ssq*1]
3237 lea srcq, [srcq+ssq*2]
3238 mova m3, m0 ; 01 12
3239 mova m0, m2 ; 23 34
3240 pshufb m5, m14
3241 pshufb m6, m14
3242 pmaddwd m5, m15
3243 pmaddwd m6, m15
3244 phaddd m5, m6
3245 paddd m5, m11
3246 psrad m5, m12
3247 packssdw m5, m5 ; 6 7 6 7
3248 punpckhqdq m1, m5 ; 4 5 6 7
3249 pshufd m5, m1, q0321 ; 5 6 7 _
3250 punpcklwd m2, m1, m5 ; 45 56
3251 punpckhwd m4, m1, m5 ; 67 __
3252 %if ARCH_X86_32
3253 mova [stk+0x20], m3
3254 mova [stk+0x30], m0
3255 mova [stk+0x40], m2
3256 mova [stk+0x50], m4
3257 %endif
3258 jmp .w2_loop
3259 %endif
3260 INIT_XMM ssse3
3261 .w4:
3262 %if ARCH_X86_64
3263 mov myd, mym
3264 mova [rsp+0x10], m11
3265 mova [rsp+0x20], m12
3266 %if isput
3267 mova [rsp+0x30], m13
3268 %endif
3269 movzx t0d, t0b
3270 sub srcq, 2
3271 movd m15, t0d
3272 %else
3273 %define m8 m0
3274 %xdefine m14 m4
3275 %define m15 m3
3276 movzx r4, byte [esp+0x1f0]
3277 sub srcq, 2
3278 movd m15, r4
3279 %endif
3280 pmaddwd m8, [base+rescale_mul]
3281 %if ARCH_X86_64
3282 mova m9, [base+pd_0x4000]
3283 %else
3284 %define m9 [base+pd_0x4000]
3285 %endif
3286 pshufd m15, m15, q0000
3287 paddd m14, m8 ; mx+dx*[0-3]
3288 pand m0, m14, m10
3289 psrld m0, 6
3290 paddd m15, m0
3291 pshufd m7, m15, q1032
3292 %if ARCH_X86_64
3293 movd r4d, m15
3294 movd r11d, m7
3295 pshufd m15, m15, q0321
3296 pshufd m7, m7, q0321
3297 movd r6d, m15
3298 movd r13d, m7
3299 mova m10, [base+bdct_lb_q+ 0]
3300 mova m11, [base+bdct_lb_q+16]
3301 movd m13, [base+subpel_filters+ r4*8+2]
3302 movd m2, [base+subpel_filters+ r6*8+2]
3303 movd m15, [base+subpel_filters+r11*8+2]
3304 movd m4, [base+subpel_filters+r13*8+2]
3305 %else
3306 movd r0, m15
3307 movd r4, m7
3308 pshufd m15, m15, q0321
3309 pshufd m7, m7, q0321
3310 movd rX, m15
3311 movd r5, m7
3312 mova m5, [base+bdct_lb_q+ 0]
3313 mova m6, [base+bdct_lb_q+16]
3314 movd m1, [base+subpel_filters+r0*8+2]
3315 movd m2, [base+subpel_filters+rX*8+2]
3316 movd m3, [base+subpel_filters+r4*8+2]
3317 movd m7, [base+subpel_filters+r5*8+2]
3318 movifprep r3, r3m
3319 SWAP m4, m7
3320 %define m10 m5
3321 %define m11 m6
3322 %define m12 m1
3323 %define m13 m1
3324 %endif
3325 psrld m14, 10
3326 paddd m14, m14
3327 punpckldq m13, m2
3328 punpckldq m15, m4
3329 punpcklqdq m13, m15
3330 pxor m2, m2
3331 pcmpeqd m0, m2
3332 %if ARCH_X86_64
3333 pand m9, m0
3334 %else
3335 pand m2, m9, m0
3336 %define m9 m2
3337 SWAP m7, m4
3338 %endif
3339 pandn m0, m13
3340 %if ARCH_X86_64
3341 SWAP m13, m0
3342 %else
3343 %define m13 m0
3344 %endif
3345 por m13, m9
3346 punpckhbw m15, m13, m13
3347 punpcklbw m13, m13
3348 psraw m15, 8
3349 psraw m13, 8
3350 pshufb m12, m14, m10
3351 pshufb m14, m11
3352 mova m10, [base+spel_s_shuf2]
3353 movd r4d, m14
3354 shr r4d, 24
3355 %if ARCH_X86_32
3356 mova [stk+0x20], m13
3357 mova [stk+0x30], m15
3358 pxor m2, m2
3359 %endif
3360 pshufb m7, m14, m2
3361 psubb m14, m7
3362 paddb m12, m10
3363 paddb m14, m10
3364 %if ARCH_X86_64
3365 lea r6, [r4+ssq*1]
3366 lea r11, [r4+ssq*2]
3367 lea r13, [r4+ss3q ]
3368 movu m7, [srcq+ssq*0]
3369 movu m9, [srcq+ssq*1]
3370 movu m8, [srcq+ssq*2]
3371 movu m10, [srcq+ss3q ]
3372 movu m1, [srcq+r4 ]
3373 movu m3, [srcq+r6 ]
3374 movu m2, [srcq+r11 ]
3375 movu m4, [srcq+r13 ]
3376 lea srcq, [srcq+ssq*4]
3377 REPX {pshufb x, m12}, m7, m9, m8, m10
3378 REPX {pmaddwd x, m13}, m7, m9, m8, m10
3379 REPX {pshufb x, m14}, m1, m2, m3, m4
3380 REPX {pmaddwd x, m15}, m1, m2, m3, m4
3381 mova m5, [rsp+0x10]
3382 movd xm6, [rsp+0x20]
3383 phaddd m7, m1
3384 phaddd m9, m3
3385 phaddd m8, m2
3386 phaddd m10, m4
3387 movu m1, [srcq+ssq*0]
3388 movu m2, [srcq+ssq*1]
3389 movu m3, [srcq+ssq*2]
3390 movu m4, [srcq+ss3q ]
3391 REPX {paddd x, m5}, m7, m9, m8, m10
3392 REPX {psrad x, xm6}, m7, m9, m8, m10
3393 packssdw m7, m9 ; 0 1
3394 packssdw m8, m10 ; 2 3
3395 movu m0, [srcq+r4 ]
3396 movu m9, [srcq+r6 ]
3397 movu m10, [srcq+r11 ]
3398 movu m11, [srcq+r13 ]
3399 lea srcq, [srcq+ssq*4]
3400 REPX {pshufb x, m12}, m1, m2, m3, m4
3401 REPX {pmaddwd x, m13}, m1, m2, m3, m4
3402 REPX {pshufb x, m14}, m0, m9, m10, m11
3403 REPX {pmaddwd x, m15}, m0, m9, m10, m11
3404 phaddd m1, m0
3405 phaddd m2, m9
3406 phaddd m3, m10
3407 phaddd m4, m11
3408 REPX {paddd x, m5}, m1, m2, m3, m4
3409 REPX {psrad x, xm6}, m1, m2, m3, m4
3410 packssdw m1, m2 ; 4 5
3411 packssdw m3, m4 ; 6 7
3412 SWAP m9, m1
3413 shufps m4, m7, m8, q1032 ; 1 2
3414 shufps m5, m8, m9, q1032 ; 3 4
3415 shufps m6, m9, m3, q1032 ; 5 6
3416 pshufd m10, m3, q1032 ; 7 _
3417 punpcklwd m0, m7, m4 ; 01
3418 punpckhwd m7, m4 ; 12
3419 punpcklwd m1, m8, m5 ; 23
3420 punpckhwd m8, m5 ; 34
3421 punpcklwd m2, m9, m6 ; 45
3422 punpckhwd m9, m6 ; 56
3423 punpcklwd m3, m10 ; 67
3424 mova [rsp+0x40], m7
3425 mova [rsp+0x50], m8
3426 mova [rsp+0x60], m9
3427 %else
3428 mova [stk+0x00], m12
3429 mova [stk+0x10], m14
3430 add r4, srcq
3431 MC_4TAP_SCALED_H 0x40 ; 0 1
3432 MC_4TAP_SCALED_H 0x50 ; 2 3
3433 MC_4TAP_SCALED_H 0x60 ; 4 5
3434 MC_4TAP_SCALED_H 0x70 ; 6 7
3435 mova m4, [stk+0x40]
3436 mova m5, [stk+0x50]
3437 mova m6, [stk+0x60]
3438 mova m7, [stk+0x70]
3439 mov [stk+0xc0], r4
3440 shufps m1, m4, m5, q1032 ; 1 2
3441 shufps m2, m5, m6, q1032 ; 3 4
3442 shufps m3, m6, m7, q1032 ; 5 6
3443 pshufd m0, m7, q1032 ; 7 _
3444 mova [stk+0xb0], m0
3445 punpcklwd m0, m4, m1 ; 01
3446 punpckhwd m4, m1 ; 12
3447 punpcklwd m1, m5, m2 ; 23
3448 punpckhwd m5, m2 ; 34
3449 punpcklwd m2, m6, m3 ; 45
3450 punpckhwd m6, m3 ; 56
3451 punpcklwd m3, m7, [stk+0xb0] ; 67
3452 mov myd, mym
3453 mov r0, r0m
3454 mova [stk+0x40], m0 ; 01
3455 mova [stk+0x50], m1 ; 23
3456 mova [stk+0x60], m2 ; 45
3457 mova [stk+0x70], m3 ; 67
3458 mova [stk+0x80], m4 ; 12
3459 mova [stk+0x90], m5 ; 34
3460 mova [stk+0xa0], m6 ; 56
3461 %define m12 [stk+0x00]
3462 %define m14 [stk+0x10]
3463 %define m13 [stk+0x20]
3464 %define m15 [stk+0x30]
3465 %define hrnd_mem [esp+0x00]
3466 %define hsh_mem [esp+0x10]
3467 %if isput
3468 %define vrnd_mem [esp+0x20]
3469 %else
3470 %define vrnd_mem [base+pd_m524256]
3471 %endif
3472 %endif
3473 .w4_loop:
3474 and myd, 0x3ff
3475 %if ARCH_X86_64
3476 mov r11d, 64 << 24
3477 mov r13d, myd
3478 shr r13d, 6
3479 lea r13d, [t1+r13]
3480 cmovnz r11q, [base+subpel_filters+r13*8]
3481 movq m9, r11q
3482 punpcklbw m9, m9
3483 psraw m9, 8
3484 pshufd m7, m9, q0000
3485 pshufd m8, m9, q1111
3486 pmaddwd m4, m0, m7
3487 pmaddwd m5, m1, m8
3488 pshufd m7, m9, q2222
3489 pshufd m9, m9, q3333
3490 pmaddwd m6, m2, m7
3491 pmaddwd m8, m3, m9
3492 %if isput
3493 movd m9, [rsp+0x28]
3494 %define vrnd_mem [rsp+0x30]
3495 %else
3496 %define vrnd_mem [base+pd_m524256]
3497 %endif
3498 paddd m4, m5
3499 paddd m6, m8
3500 paddd m4, m6
3501 paddd m4, vrnd_mem
3502 %else
3503 mov mym, myd
3504 mov r5, [esp+0x1f4]
3505 xor r3, r3
3506 shr r4, 6
3507 lea r5, [r5+r4]
3508 mov r4, 64 << 24
3509 cmovnz r4, [base+subpel_filters+r5*8+0]
3510 cmovnz r3, [base+subpel_filters+r5*8+4]
3511 movd m7, r4
3512 movd m6, r3
3513 punpckldq m7, m6
3514 punpcklbw m7, m7
3515 psraw m7, 8
3516 pshufd m4, m7, q0000
3517 pshufd m5, m7, q1111
3518 pshufd m6, m7, q2222
3519 pshufd m7, m7, q3333
3520 pmaddwd m0, m4
3521 pmaddwd m1, m5
3522 pmaddwd m2, m6
3523 pmaddwd m3, m7
3524 %if isput
3525 movd m4, [esp+0x18]
3526 %endif
3527 paddd m0, m1
3528 paddd m2, m3
3529 paddd m0, vrnd_mem
3530 paddd m0, m2
3531 SWAP m4, m0
3532 %define m9 m0
3533 %endif
3534 %if isput
3535 pxor m5, m5
3536 psrad m4, m9
3537 packssdw m4, m4
3538 pmaxsw m4, m5
3539 pminsw m4, pxmaxm
3540 movq [dstq], m4
3541 add dstq, dsmp
3542 %else
3543 psrad m4, 6
3544 packssdw m4, m4
3545 movq [tmpq], m4
3546 add tmpq, 8
3547 %endif
3548 dec hd
3549 jz .ret
3550 %if ARCH_X86_64
3551 add myd, dyd
3552 test myd, ~0x3ff
3553 jz .w4_loop
3554 mova m8, [rsp+0x10]
3555 movd m9, [rsp+0x20]
3556 movu m4, [srcq]
3557 movu m5, [srcq+r4]
3558 test myd, 0x400
3559 jz .w4_skip_line
3560 mova m0, [rsp+0x40]
3561 mova [rsp+0x40], m1
3562 mova m1, [rsp+0x50]
3563 mova [rsp+0x50], m2
3564 mova m2, [rsp+0x60]
3565 mova [rsp+0x60], m3
3566 pshufb m4, m12
3567 pshufb m5, m14
3568 pmaddwd m4, m13
3569 pmaddwd m5, m15
3570 phaddd m4, m5
3571 paddd m4, m8
3572 psrad m4, m9
3573 packssdw m4, m4
3574 punpcklwd m3, m10, m4
3575 mova m10, m4
3576 add srcq, ssq
3577 jmp .w4_loop
3578 .w4_skip_line:
3579 movu m6, [srcq+ssq*1]
3580 movu m7, [srcq+r6]
3581 mova m0, [rsp+0x50]
3582 mova m11, [rsp+0x60]
3583 pshufb m4, m12
3584 pshufb m6, m12
3585 pshufb m5, m14
3586 pshufb m7, m14
3587 pmaddwd m4, m13
3588 pmaddwd m6, m13
3589 pmaddwd m5, m15
3590 pmaddwd m7, m15
3591 mova [rsp+0x40], m0
3592 mova [rsp+0x50], m11
3593 phaddd m4, m5
3594 phaddd m6, m7
3595 paddd m4, m8
3596 paddd m6, m8
3597 psrad m4, m9
3598 psrad m6, m9
3599 packssdw m4, m6
3600 punpcklwd m9, m10, m4
3601 mova [rsp+0x60], m9
3602 pshufd m10, m4, q1032
3603 mova m0, m1
3604 mova m1, m2
3605 mova m2, m3
3606 punpcklwd m3, m4, m10
3607 lea srcq, [srcq+ssq*2]
3608 jmp .w4_loop
3609 %else
3610 SWAP m0, m4
3611 mov myd, mym
3612 mov r3, r3m
3613 add myd, dym
3614 test myd, ~0x3ff
3615 jnz .w4_next_line
3616 mova m0, [stk+0x40]
3617 mova m1, [stk+0x50]
3618 mova m2, [stk+0x60]
3619 mova m3, [stk+0x70]
3620 jmp .w4_loop
3621 .w4_next_line:
3622 mov r5, [stk+0xc0]
3623 movu m4, [srcq]
3624 movu m5, [r5]
3625 test myd, 0x400
3626 jz .w4_skip_line
3627 add [stk+0xc0], ssq
3628 mova m0, [stk+0x80]
3629 mova m3, [stk+0x50]
3630 mova [stk+0x40], m0
3631 mova [stk+0x80], m3
3632 mova m1, [stk+0x90]
3633 mova m6, [stk+0x60]
3634 mova [stk+0x50], m1
3635 mova [stk+0x90], m6
3636 mova m2, [stk+0xa0]
3637 mova m7, [stk+0x70]
3638 mova [stk+0x60], m2
3639 mova [stk+0xa0], m7
3640 pshufb m4, m12
3641 pshufb m5, m14
3642 pmaddwd m4, m13
3643 pmaddwd m5, m15
3644 phaddd m4, m5
3645 paddd m4, hrnd_mem
3646 psrad m4, hsh_mem
3647 packssdw m4, m4
3648 punpcklwd m3, [stk+0xb0], m4
3649 mova [stk+0xb0], m4
3650 mova [stk+0x70], m3
3651 add srcq, ssq
3652 jmp .w4_loop
3653 .w4_skip_line:
3654 movu m6, [srcq+ssq*1]
3655 movu m7, [r5 +ssq*1]
3656 lea r5, [r5 +ssq*2]
3657 mov [stk+0xc0], r5
3658 mova m0, [stk+0x50]
3659 mova m1, [stk+0x60]
3660 mova m2, [stk+0x70]
3661 mova m3, [stk+0x90]
3662 pshufb m4, m12
3663 pshufb m6, m12
3664 pshufb m5, m14
3665 pshufb m7, m14
3666 pmaddwd m4, m13
3667 pmaddwd m6, m13
3668 pmaddwd m5, m15
3669 pmaddwd m7, m15
3670 mova [stk+0x40], m0
3671 mova [stk+0x50], m1
3672 mova [stk+0x60], m2
3673 mova [stk+0x80], m3
3674 phaddd m4, m5
3675 phaddd m6, m7
3676 mova m5, [stk+0xa0]
3677 mova m7, [stk+0xb0]
3678 paddd m4, hrnd_mem
3679 paddd m6, hrnd_mem
3680 psrad m4, hsh_mem
3681 psrad m6, hsh_mem
3682 packssdw m4, m6
3683 punpcklwd m7, m4
3684 pshufd m6, m4, q1032
3685 mova [stk+0x90], m5
3686 mova [stk+0xa0], m7
3687 mova [stk+0xb0], m6
3688 punpcklwd m3, m4, m6
3689 mova [stk+0x70], m3
3690 lea srcq, [srcq+ssq*2]
3691 jmp .w4_loop
3692 %endif
3693 INIT_XMM ssse3
3694 %if ARCH_X86_64
3695 %define stk rsp+0x20
3696 %endif
3697 .w8:
3698 mov dword [stk+0xf0], 1
3699 movifprep tmp_stridem, 16
3700 jmp .w_start
3701 .w16:
3702 mov dword [stk+0xf0], 2
3703 movifprep tmp_stridem, 32
3704 jmp .w_start
3705 .w32:
3706 mov dword [stk+0xf0], 4
3707 movifprep tmp_stridem, 64
3708 jmp .w_start
3709 .w64:
3710 mov dword [stk+0xf0], 8
3711 movifprep tmp_stridem, 128
3712 jmp .w_start
3713 .w128:
3714 mov dword [stk+0xf0], 16
3715 movifprep tmp_stridem, 256
3716 .w_start:
3717 %if ARCH_X86_64
3718 %ifidn %1, put
3719 movifnidn dsm, dsq
3720 %endif
3721 mova [rsp+0x10], m11
3722 %define hround m11
3723 shr t0d, 16
3724 movd m15, t0d
3725 %if isprep
3726 mova m13, [base+pd_m524256]
3727 %endif
3728 %else
3729 %define hround [esp+0x00]
3730 %define m12 [esp+0x10]
3731 %define m10 [base+pd_0x3ff]
3732 %define m8 m0
3733 %xdefine m14 m4
3734 %define m15 m3
3735 %if isprep
3736 %define ssq ssm
3737 %endif
3738 mov r4, [esp+0x1f0]
3739 shr r4, 16
3740 movd m15, r4
3741 mov r0, r0m
3742 mov myd, mym
3743 %endif
3744 sub srcq, 6
3745 pslld m7, m8, 2 ; dx*4
3746 pmaddwd m8, [base+rescale_mul] ; dx*[0-3]
3747 pshufd m15, m15, q0000
3748 paddd m14, m8 ; mx+dx*[0-3]
3749 mova [stk+0x100], m7
3750 mova [stk+0x120], m15
3751 mov [stk+0x0f8], srcq
3752 mov [stk+0x130], r0q ; dstq / tmpq
3753 %if ARCH_X86_64 && UNIX64
3754 mov hm, hd
3755 %elif ARCH_X86_32
3756 mov r5, hm
3757 mov [stk+0x0f4], myd
3758 mov [stk+0x134], r5
3759 %endif
3760 jmp .hloop
3761 .hloop_prep:
3762 dec dword [stk+0x0f0]
3763 jz .ret
3764 %if ARCH_X86_64
3765 add qword [stk+0x130], 16
3766 mov hd, hm
3767 %else
3768 add dword [stk+0x130], 16
3769 mov myd, [stk+0x0f4]
3770 mov r5, [stk+0x134]
3771 mov r0, [stk+0x130]
3772 %endif
3773 mova m7, [stk+0x100]
3774 mova m14, [stk+0x110]
3775 %if ARCH_X86_64
3776 mova m10, [base+pd_0x3ff]
3777 mova m11, [rsp+0x10]
3778 %endif
3779 mova m15, [stk+0x120]
3780 mov srcq, [stk+0x0f8]
3781 %if ARCH_X86_64
3782 mov r0q, [stk+0x130] ; dstq / tmpq
3783 %else
3784 mov mym, myd
3785 mov hm, r5
3786 mov r0m, r0
3787 mov r3, r3m
3788 %endif
3789 paddd m14, m7
3790 .hloop:
3791 %if ARCH_X86_64
3792 mova m9, [base+pq_0x40000000]
3793 %else
3794 %define m9 [base+pq_0x40000000]
3795 %endif
3796 pxor m1, m1
3797 psrld m2, m14, 10
3798 mova [stk], m2
3799 pand m6, m14, m10
3800 psrld m6, 6
3801 paddd m5, m15, m6
3802 pcmpeqd m6, m1
3803 pshufd m2, m5, q1032
3804 %if ARCH_X86_64
3805 movd r4d, m5
3806 movd r6d, m2
3807 pshufd m5, m5, q0321
3808 pshufd m2, m2, q0321
3809 movd r7d, m5
3810 movd r9d, m2
3811 movq m0, [base+subpel_filters+r4*8]
3812 movq m1, [base+subpel_filters+r6*8]
3813 movhps m0, [base+subpel_filters+r7*8]
3814 movhps m1, [base+subpel_filters+r9*8]
3815 %else
3816 movd r0, m5
3817 movd rX, m2
3818 pshufd m5, m5, q0321
3819 pshufd m2, m2, q0321
3820 movd r4, m5
3821 movd r5, m2
3822 movq m0, [base+subpel_filters+r0*8]
3823 movq m1, [base+subpel_filters+rX*8]
3824 movhps m0, [base+subpel_filters+r4*8]
3825 movhps m1, [base+subpel_filters+r5*8]
3826 %endif
3827 paddd m14, m7 ; mx+dx*[4-7]
3828 pand m5, m14, m10
3829 psrld m5, 6
3830 paddd m15, m5
3831 pxor m2, m2
3832 pcmpeqd m5, m2
3833 mova [stk+0x110], m14
3834 pshufd m4, m15, q1032
3835 %if ARCH_X86_64
3836 movd r10d, m15
3837 movd r11d, m4
3838 pshufd m15, m15, q0321
3839 pshufd m4, m4, q0321
3840 movd r13d, m15
3841 movd rXd, m4
3842 movq m2, [base+subpel_filters+r10*8]
3843 movq m3, [base+subpel_filters+r11*8]
3844 movhps m2, [base+subpel_filters+r13*8]
3845 movhps m3, [base+subpel_filters+ rX*8]
3846 psrld m14, 10
3847 movq r11, m14
3848 punpckhqdq m14, m14
3849 movq rX, m14
3850 mov r10d, r11d
3851 shr r11, 32
3852 mov r13d, rXd
3853 shr rX, 32
3854 mov r4d, [stk+ 0]
3855 mov r6d, [stk+ 4]
3856 mov r7d, [stk+ 8]
3857 mov r9d, [stk+12]
3858 pshufd m4, m6, q1100
3859 pshufd m6, m6, q3322
3860 pshufd m14, m5, q1100
3861 pshufd m5, m5, q3322
3862 pand m7, m9, m4
3863 pand m8, m9, m6
3864 pand m15, m9, m14
3865 pand m9, m9, m5
3866 pandn m4, m0
3867 pandn m6, m1
3868 pandn m14, m2
3869 pandn m5, m3
3870 por m7, m4
3871 por m8, m6
3872 por m15, m14
3873 por m9, m5
3874 punpcklbw m0, m7, m7
3875 punpckhbw m7, m7
3876 punpcklbw m1, m8, m8
3877 punpckhbw m8, m8
3878 psraw m0, 8
3879 psraw m7, 8
3880 psraw m1, 8
3881 psraw m8, 8
3882 punpcklbw m2, m15, m15
3883 punpckhbw m15, m15
3884 punpcklbw m3, m9, m9
3885 punpckhbw m9, m9
3886 psraw m2, 8
3887 psraw m15, 8
3888 psraw m3, 8
3889 psraw m9, 8
3890 mova [stk+0x10], m0
3891 mova [stk+0x20], m7
3892 mova [stk+0x30], m1
3893 mova [stk+0x40], m8
3894 mova [stk+0x50], m2
3895 mova [stk+0x60], m15
3896 mova [stk+0x70], m3
3897 mova [stk+0x80], m9
3898 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10 ; 0
3899 mova [stk+0x90], m1
3900 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 1, 9, 10 ; 1
3901 mova [stk+0xa0], m2
3902 MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10 ; 2
3903 mova [stk+0xb0], m3
3904 MC_8TAP_SCALED_H 4, 5, 6, 1, 2, 3, 9, 10 ; 3
3905 mova [stk+0xc0], m4
3906 MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10 ; 4
3907 mova [stk+0xd0], m5
3908 MC_8TAP_SCALED_H 6, 1, 2, 3, 4, 5, 9, 10 ; 5
3909 MC_8TAP_SCALED_H 7, 1, 2, 3, 4, 5, 9, 10 ; 6
3910 MC_8TAP_SCALED_H 8, 1, 2, 3, 4, 5, 9, 10 ; 7
3911 mova m5, [stk+0xd0]
3912 mova m1, [stk+0x90]
3913 mova m2, [stk+0xa0]
3914 mova m3, [stk+0xb0]
3915 mova m9, [stk+0xc0]
3916 mov myd, mym
3917 mov dyd, dym
3918 punpcklwd m4, m5, m6 ; 45a
3919 punpckhwd m5, m6 ; 45b
3920 punpcklwd m6, m7, m8 ; 67a
3921 punpckhwd m7, m8 ; 67b
3922 punpcklwd m0, m1, m2 ; 01a
3923 punpckhwd m1, m2 ; 01b
3924 punpcklwd m2, m3, m9 ; 23a
3925 punpckhwd m3, m9 ; 23b
3926 mova [stk+0x90], m4
3927 mova [stk+0xa0], m5
3928 mova [stk+0xb0], m6
3929 mova [stk+0xc0], m7
3930 %define hround [rsp+0x10]
3931 .vloop:
3932 and myd, 0x3ff
3933 mov r6d, 64 << 24
3934 mov r4d, myd
3935 shr r4d, 6
3936 lea r4d, [t1+r4]
3937 cmovnz r6q, [base+subpel_filters+r4*8]
3938 movq m11, r6q
3939 punpcklbw m11, m11
3940 psraw m11, 8
3941 pshufd m5, m11, q0000
3942 pshufd m7, m11, q1111
3943 pshufd m10, m11, q2222
3944 pshufd m11, m11, q3333
3945 pmaddwd m4, m5, m0
3946 pmaddwd m5, m5, m1
3947 pmaddwd m6, m7, m2
3948 pmaddwd m7, m7, m3
3949 paddd m4, m13
3950 paddd m5, m13
3951 paddd m4, m6
3952 paddd m5, m7
3953 pmaddwd m6, [stk+0x90], m10
3954 pmaddwd m7, [stk+0xa0], m10
3955 pmaddwd m8, [stk+0xb0], m11
3956 pmaddwd m9, [stk+0xc0], m11
3957 paddd m4, m6
3958 paddd m5, m7
3959 %if isput
3960 pshufd m6, m12, q1032
3961 %endif
3962 paddd m4, m8
3963 paddd m5, m9
3964 %else
3965 movd r0, m15
3966 movd rX, m4
3967 pshufd m15, m15, q0321
3968 pshufd m4, m4, q0321
3969 movd r4, m15
3970 movd r5, m4
3971 mova m14, [stk+0x110]
3972 movq m2, [base+subpel_filters+r0*8]
3973 movq m3, [base+subpel_filters+rX*8]
3974 movhps m2, [base+subpel_filters+r4*8]
3975 movhps m3, [base+subpel_filters+r5*8]
3976 psrld m14, 10
3977 mova [stk+16], m14
3978 mov r0, [stk+ 0]
3979 mov rX, [stk+ 4]
3980 mov r4, [stk+ 8]
3981 mov r5, [stk+12]
3982 mova [stk+0x20], m0
3983 mova [stk+0x30], m1
3984 mova [stk+0x40], m2
3985 mova [stk+0x50], m3
3986 pshufd m4, m6, q1100
3987 pshufd m6, m6, q3322
3988 pshufd m7, m5, q1100
3989 pshufd m5, m5, q3322
3990 pand m0, m9, m4
3991 pand m1, m9, m6
3992 pand m2, m9, m7
3993 pand m3, m9, m5
3994 pandn m4, [stk+0x20]
3995 pandn m6, [stk+0x30]
3996 pandn m7, [stk+0x40]
3997 pandn m5, [stk+0x50]
3998 por m0, m4
3999 por m1, m6
4000 por m2, m7
4001 por m3, m5
4002 punpcklbw m4, m0, m0
4003 punpckhbw m0, m0
4004 punpcklbw m5, m1, m1
4005 punpckhbw m1, m1
4006 psraw m4, 8
4007 psraw m0, 8
4008 psraw m5, 8
4009 psraw m1, 8
4010 punpcklbw m6, m2, m2
4011 punpckhbw m2, m2
4012 punpcklbw m7, m3, m3
4013 punpckhbw m3, m3
4014 psraw m6, 8
4015 psraw m2, 8
4016 psraw m7, 8
4017 psraw m3, 8
4018 mova [stk+0x0a0], m4
4019 mova [stk+0x0b0], m0
4020 mova [stk+0x0c0], m5
4021 mova [stk+0x0d0], m1
4022 mova [stk+0x140], m6
4023 mova [stk+0x150], m2
4024 mova [stk+0x160], m7
4025 mova [stk+0x170], m3
4026 MC_8TAP_SCALED_H 0xa0, 0x20, 0 ; 0
4027 MC_8TAP_SCALED_H 0xa0, 0x30 ; 1
4028 MC_8TAP_SCALED_H 0xa0, 0x40 ; 2
4029 MC_8TAP_SCALED_H 0xa0, 0x50 ; 3
4030 MC_8TAP_SCALED_H 0xa0, 0x60 ; 4
4031 MC_8TAP_SCALED_H 0xa0, 0x70 ; 5
4032 MC_8TAP_SCALED_H 0xa0, 0x80 ; 6
4033 MC_8TAP_SCALED_H 0xa0, 0x90 ; 7
4034 mova m5, [stk+0x60]
4035 mova m6, [stk+0x70]
4036 mova m7, [stk+0x80]
4037 mova m0, [stk+0x90]
4038 mov myd, mym
4039 punpcklwd m4, m5, m6 ; 45a
4040 punpckhwd m5, m6 ; 45b
4041 punpcklwd m6, m7, m0 ; 67a
4042 punpckhwd m7, m0 ; 67b
4043 mova [stk+0x60], m4
4044 mova [stk+0x70], m5
4045 mova [stk+0x80], m6
4046 mova [stk+0x90], m7
4047 mova m1, [stk+0x20]
4048 mova m2, [stk+0x30]
4049 mova m3, [stk+0x40]
4050 mova m4, [stk+0x50]
4051 punpcklwd m0, m1, m2 ; 01a
4052 punpckhwd m1, m2 ; 01b
4053 punpcklwd m2, m3, m4 ; 23a
4054 punpckhwd m3, m4 ; 23b
4055 mova [stk+0x20], m0
4056 mova [stk+0x30], m1
4057 mova [stk+0x40], m2
4058 mova [stk+0x50], m3
4059 .vloop:
4060 mov r0, r0m
4061 mov r5, [esp+0x1f4]
4062 and myd, 0x3ff
4063 mov mym, myd
4064 xor r3, r3
4065 shr r4, 6
4066 lea r5, [r5+r4]
4067 mov r4, 64 << 24
4068 cmovnz r4, [base+subpel_filters+r5*8+0]
4069 cmovnz r3, [base+subpel_filters+r5*8+4]
4070 movd m7, r4
4071 movd m6, r3
4072 punpckldq m7, m6
4073 punpcklbw m7, m7
4074 psraw m7, 8
4075 pshufd m4, m7, q0000
4076 pshufd m5, m7, q1111
4077 pmaddwd m0, m4
4078 pmaddwd m1, m4
4079 pmaddwd m2, m5
4080 pmaddwd m3, m5
4081 pshufd m6, m7, q2222
4082 pshufd m7, m7, q3333
4083 paddd m0, m2
4084 paddd m1, m3
4085 pmaddwd m2, [stk+0x60], m6
4086 pmaddwd m3, [stk+0x70], m6
4087 pmaddwd m4, [stk+0x80], m7
4088 pmaddwd m5, [stk+0x90], m7
4089 %if isput
4090 movd m6, [esp+0x18]
4091 %endif
4092 paddd m0, m2
4093 paddd m1, m3
4094 paddd m0, vrnd_mem
4095 paddd m1, vrnd_mem
4096 paddd m4, m0
4097 paddd m5, m1
4098 %endif
4099 %ifidn %1, put
4100 psrad m4, m6
4101 psrad m5, m6
4102 packssdw m4, m5
4103 pxor m7, m7
4104 pmaxsw m4, m7
4105 pminsw m4, pxmaxm
4106 mova [dstq], m4
4107 add dstq, dsm
4108 %else
4109 psrad m4, 6
4110 psrad m5, 6
4111 packssdw m4, m5
4112 mova [tmpq], m4
4113 add tmpq, tmp_stridem
4114 %endif
4115 dec hd
4116 jz .hloop_prep
4117 %if ARCH_X86_64
4118 add myd, dyd
4119 test myd, ~0x3ff
4120 jz .vloop
4121 test myd, 0x400
4122 mov [stk+0x140], myd
4123 mov r4d, [stk+ 0]
4124 mov r6d, [stk+ 4]
4125 mov r7d, [stk+ 8]
4126 mov r9d, [stk+12]
4127 jz .skip_line
4128 mova m14, [base+unpckw]
4129 movu m8, [srcq+r10*2]
4130 movu m9, [srcq+r11*2]
4131 movu m10, [srcq+r13*2]
4132 movu m11, [srcq+ rX*2]
4133 movu m4, [srcq+ r4*2]
4134 movu m5, [srcq+ r6*2]
4135 movu m6, [srcq+ r7*2]
4136 movu m7, [srcq+ r9*2]
4137 add srcq, ssq
4138 mov myd, [stk+0x140]
4139 mov dyd, dym
4140 pshufd m15, m14, q1032
4141 pshufb m0, m14 ; 0a 1a
4142 pshufb m1, m14 ; 0b 1b
4143 pshufb m2, m15 ; 3a 2a
4144 pshufb m3, m15 ; 3b 2b
4145 pmaddwd m8, [stk+0x50]
4146 pmaddwd m9, [stk+0x60]
4147 pmaddwd m10, [stk+0x70]
4148 pmaddwd m11, [stk+0x80]
4149 pmaddwd m4, [stk+0x10]
4150 pmaddwd m5, [stk+0x20]
4151 pmaddwd m6, [stk+0x30]
4152 pmaddwd m7, [stk+0x40]
4153 phaddd m8, m9
4154 phaddd m10, m11
4155 mova m11, hround
4156 phaddd m4, m5
4157 phaddd m6, m7
4158 phaddd m8, m10
4159 phaddd m4, m6
4160 paddd m4, m11
4161 paddd m8, m11
4162 psrad m4, m12
4163 psrad m8, m12
4164 packssdw m4, m8
4165 pshufb m5, [stk+0x90], m14 ; 4a 5a
4166 pshufb m6, [stk+0xa0], m14 ; 4b 5b
4167 pshufb m7, [stk+0xb0], m15 ; 7a 6a
4168 pshufb m8, [stk+0xc0], m15 ; 7b 6b
4169 punpckhwd m0, m2 ; 12a
4170 punpckhwd m1, m3 ; 12b
4171 punpcklwd m2, m5 ; 34a
4172 punpcklwd m3, m6 ; 34b
4173 punpckhwd m5, m7 ; 56a
4174 punpckhwd m6, m8 ; 56b
4175 punpcklwd m7, m4 ; 78a
4176 punpckhqdq m4, m4
4177 punpcklwd m8, m4 ; 78b
4178 mova [stk+0x90], m5
4179 mova [stk+0xa0], m6
4180 mova [stk+0xb0], m7
4181 mova [stk+0xc0], m8
4182 jmp .vloop
4183 .skip_line:
4184 MC_8TAP_SCALED_H 4, 8, 5, 6, 7, 9, 10, 11
4185 MC_8TAP_SCALED_H 8, 5, 6, 7, 9, 0, 10, 11
4186 mov myd, [stk+0x140]
4187 mov dyd, dym
4188 mova m0, m2 ; 01a
4189 mova m1, m3 ; 01b
4190 mova m2, [stk+0x90] ; 23a
4191 mova m3, [stk+0xa0] ; 23b
4192 mova m5, [stk+0xb0] ; 45a
4193 mova m6, [stk+0xc0] ; 45b
4194 punpcklwd m7, m4, m8 ; 67a
4195 punpckhwd m4, m8 ; 67b
4196 mova [stk+0x90], m5
4197 mova [stk+0xa0], m6
4198 mova [stk+0xb0], m7
4199 mova [stk+0xc0], m4
4200 %else
4201 mov r0m, r0
4202 mov myd, mym
4203 mov r3, r3m
4204 add myd, dym
4205 test myd, ~0x3ff
4206 mov mym, myd
4207 jnz .next_line
4208 mova m0, [stk+0x20]
4209 mova m1, [stk+0x30]
4210 mova m2, [stk+0x40]
4211 mova m3, [stk+0x50]
4212 jmp .vloop
4213 .next_line:
4214 test myd, 0x400
4215 mov r0, [stk+ 0]
4216 mov rX, [stk+ 4]
4217 mov r4, [stk+ 8]
4218 mov r5, [stk+12]
4219 jz .skip_line
4220 MC_8TAP_SCALED_H 0xa0, 0xe0, 0 ; 8
4221 mova m7, [base+unpckw]
4222 pshufd m4, m7, q1032
4223 pshufb m0, [stk+0x20], m7 ; 0a 1a
4224 pshufb m1, [stk+0x30], m7 ; 0b 1b
4225 pshufb m2, [stk+0x40], m4 ; 3a 2a
4226 pshufb m3, [stk+0x50], m4 ; 3b 2b
4227 pshufb m5, [stk+0x60], m7 ; 4a 5a
4228 pshufb m6, [stk+0x70], m7 ; 4b 5b
4229 pshufb m7, [stk+0x80], m4 ; 7a 6a
4230 punpckhwd m0, m2 ; 12a
4231 punpckhwd m1, m3 ; 12b
4232 punpcklwd m2, m5 ; 34a
4233 punpcklwd m3, m6 ; 34b
4234 mova [stk+0x20], m0
4235 mova [stk+0x30], m1
4236 mova [stk+0x40], m2
4237 mova [stk+0x50], m3
4238 punpckhwd m5, m7 ; 56a
4239 mova [stk+0x60], m5
4240 pshufb m5, [stk+0x90], m4 ; 7b 6b
4241 punpcklwd m7, [stk+0xe0] ; 78a
4242 punpckhwd m6, m5 ; 56b
4243 mova [stk+0x70], m6
4244 movq m6, [stk+0xe8]
4245 mova [stk+0x80], m7
4246 punpcklwd m5, m6
4247 mov myd, mym
4248 mova [stk+0x90], m5
4249 jmp .vloop
4250 .skip_line:
4251 MC_8TAP_SCALED_H 0xa0, 0xe0, 0 ; 8
4252 MC_8TAP_SCALED_H 0xa0, 0 ; 9
4253 mova m7, [stk+0xe0]
4254 mova m2, [stk+0x60] ; 23a
4255 mova m3, [stk+0x70] ; 23b
4256 mova m4, [stk+0x80] ; 45a
4257 mova m5, [stk+0x90] ; 45b
4258 punpcklwd m6, m7, m0 ; 67a
4259 punpckhwd m7, m0 ; 67b
4260 mova m0, [stk+0x40] ; 01a
4261 mova m1, [stk+0x50] ; 01b
4262 mov myd, mym
4263 mova [stk+0x40], m2
4264 mova [stk+0x50], m3
4265 mova [stk+0x60], m4
4266 mova [stk+0x70], m5
4267 mova [stk+0x80], m6
4268 mova [stk+0x90], m7
4269 mova [stk+0x20], m0
4270 mova [stk+0x30], m1
4271 %endif
4272 jmp .vloop
4273 INIT_XMM ssse3
4274 .dy1:
4275 movzx wd, word [base+%1_8tap_scaled_ssse3_dy1_table+wq*2]
4276 add wq, base_reg
4277 jmp wq
4278 %if isput
4279 .dy1_w2:
4280 %if ARCH_X86_64
4281 mov myd, mym
4282 movzx t0d, t0b
4283 sub srcq, 2
4284 movd m15, t0d
4285 %else
4286 %define m8 m0
4287 %define m9 m1
4288 %define m14 m4
4289 %define m15 m3
4290 %define m11 [esp+0x00]
4291 %define m12 [esp+0x10]
4292 %define m13 [esp+0x20]
4293 movzx r5, byte [esp+0x1f0]
4294 sub srcq, 2
4295 movd m15, r5
4296 mov r1, r1m
4297 %endif
4298 pxor m9, m9
4299 punpckldq m9, m8
4300 paddd m14, m9 ; mx+dx*[0-1]
4301 %if ARCH_X86_64
4302 mova m9, [base+pd_0x4000]
4303 %endif
4304 pshufd m15, m15, q0000
4305 pand m8, m14, m10
4306 psrld m8, 6
4307 paddd m15, m8
4308 movd r4d, m15
4309 pshufd m15, m15, q0321
4310 %if ARCH_X86_64
4311 movd r6d, m15
4312 %else
4313 movd r3d, m15
4314 %endif
4315 mova m5, [base+bdct_lb_q]
4316 mova m6, [base+spel_s_shuf2]
4317 movd m15, [base+subpel_filters+r4*8+2]
4318 %if ARCH_X86_64
4319 movd m7, [base+subpel_filters+r6*8+2]
4320 %else
4321 movd m7, [base+subpel_filters+r3*8+2]
4322 %endif
4323 pxor m2, m2
4324 pcmpeqd m8, m2
4325 psrld m14, 10
4326 paddd m14, m14
4327 %if ARCH_X86_32
4328 mov r3, r3m
4329 pshufb m14, m5
4330 paddb m14, m6
4331 mova [stk], m14
4332 SWAP m5, m0
4333 SWAP m6, m3
4334 %define m15 m6
4335 %endif
4336 movu m0, [srcq+ssq*0]
4337 movu m1, [srcq+ssq*1]
4338 movu m2, [srcq+ssq*2]
4339 movu m3, [srcq+ss3q ]
4340 lea srcq, [srcq+ssq*4]
4341 punpckldq m15, m7
4342 %if ARCH_X86_64
4343 pshufb m14, m5
4344 paddb m14, m6
4345 pand m9, m8
4346 pandn m8, m15
4347 SWAP m15, m8
4348 por m15, m9
4349 movu m4, [srcq+ssq*0]
4350 movu m5, [srcq+ssq*1]
4351 movu m6, [srcq+ssq*2]
4352 add srcq, ss3q
4353 shr myd, 6
4354 mov r4d, 64 << 24
4355 lea myd, [t1+myq]
4356 cmovnz r4q, [base+subpel_filters+myq*8]
4357 %else
4358 pand m7, m5, [base+pd_0x4000]
4359 pandn m5, m15
4360 por m5, m7
4361 %define m15 m5
4362 mov myd, mym
4363 mov r5, [esp+0x1f4]
4364 xor r3, r3
4365 shr myd, 6
4366 lea r5, [r5+myd]
4367 mov r4, 64 << 24
4368 cmovnz r4, [base+subpel_filters+r5*8+0]
4369 cmovnz r3, [base+subpel_filters+r5*8+4]
4370 mov [stk+0x20], r3
4371 mov r3, r3m
4372 %endif
4373 punpcklbw m15, m15
4374 psraw m15, 8
4375 REPX {pshufb x, m14}, m0, m1, m2, m3
4376 REPX {pmaddwd x, m15}, m0, m1, m2, m3
4377 %if ARCH_X86_64
4378 REPX {pshufb x, m14}, m4, m5, m6
4379 REPX {pmaddwd x, m15}, m4, m5, m6
4380 phaddd m0, m1
4381 phaddd m2, m3
4382 phaddd m4, m5
4383 phaddd m6, m6
4384 REPX {paddd x, m11}, m0, m2, m4, m6
4385 REPX {psrad x, m12}, m0, m2, m4, m6
4386 packssdw m0, m2 ; 0 1 2 3
4387 packssdw m4, m6 ; 4 5 6
4388 SWAP m1, m4
4389 movq m10, r4
4390 %else
4391 mova [stk+0x10], m15
4392 phaddd m0, m1
4393 phaddd m2, m3
4394 movu m1, [srcq+ssq*0]
4395 movu m7, [srcq+ssq*1]
4396 movu m6, [srcq+ssq*2]
4397 add srcq, ss3q
4398 REPX {pshufb x, m14}, m1, m7, m6
4399 REPX {pmaddwd x, m15}, m1, m7, m6
4400 %define m14 [stk+0x00]
4401 %define m15 [stk+0x10]
4402 phaddd m1, m7
4403 phaddd m6, m6
4404 REPX {paddd x, m11}, m0, m2, m1, m6
4405 REPX {psrad x, m12}, m0, m2, m1, m6
4406 packssdw m0, m2
4407 packssdw m1, m6
4408 %define m8 m6
4409 %define m9 m4
4410 %define m10 m5
4411 movd m10, r4
4412 movd m9, [stk+0x20]
4413 punpckldq m10, m9
4414 %endif
4415 punpcklbw m10, m10
4416 psraw m10, 8
4417 pshufd m7, m10, q0000
4418 pshufd m8, m10, q1111
4419 pshufd m9, m10, q2222
4420 pshufd m10, m10, q3333
4421 %if ARCH_X86_32
4422 mova [stk+0x50], m7
4423 mova [stk+0x60], m8
4424 mova [stk+0x70], m9
4425 mova [stk+0x80], m10
4426 %define m7 [stk+0x50]
4427 %define m8 [stk+0x60]
4428 %define m9 [stk+0x70]
4429 %define m10 [stk+0x80]
4430 %endif
4431 palignr m2, m1, m0, 4 ; 1 2 3 4
4432 punpcklwd m3, m0, m2 ; 01 12
4433 punpckhwd m0, m2 ; 23 34
4434 pshufd m4, m1, q2121 ; 5 6 5 6
4435 punpcklwd m2, m1, m4 ; 45 56
4436 %if ARCH_X86_32
4437 mov r0, r0m
4438 %endif
4439 .dy1_w2_loop:
4440 movu m1, [srcq+ssq*0]
4441 movu m6, [srcq+ssq*1]
4442 lea srcq, [srcq+ssq*2]
4443 pmaddwd m5, m3, m7
4444 mova m3, m0
4445 pmaddwd m0, m8
4446 pshufb m1, m14
4447 pshufb m6, m14
4448 pmaddwd m1, m15
4449 pmaddwd m6, m15
4450 phaddd m1, m6
4451 paddd m1, m11
4452 psrad m1, m12
4453 packssdw m1, m1
4454 paddd m5, m0
4455 mova m0, m2
4456 pmaddwd m2, m9
4457 paddd m5, m2
4458 palignr m2, m1, m4, 12
4459 punpcklwd m2, m1 ; 67 78
4460 pmaddwd m4, m2, m10
4461 paddd m5, m13
4462 paddd m5, m4
4463 pxor m6, m6
4464 mova m4, m1
4465 pshufd m1, m12, q1032
4466 psrad m5, m1
4467 packssdw m5, m5
4468 pmaxsw m5, m6
4469 pminsw m5, pxmaxm
4470 movd [dstq+dsq*0], m5
4471 pshuflw m5, m5, q1032
4472 movd [dstq+dsq*1], m5
4473 lea dstq, [dstq+dsq*2]
4474 sub hd, 2
4475 jg .dy1_w2_loop
4476 RET
4477 %endif
4478 INIT_XMM ssse3
4479 .dy1_w4:
4480 %if ARCH_X86_64
4481 mov myd, mym
4482 mova [rsp+0x10], m11
4483 mova [rsp+0x20], m12
4484 %if isput
4485 mova [rsp+0x30], m13
4486 %define vrnd_mem [rsp+0x30]
4487 %define stk rsp+0x40
4488 %else
4489 %define vrnd_mem [base+pd_m524256]
4490 %define stk rsp+0x30
4491 %endif
4492 movzx t0d, t0b
4493 sub srcq, 2
4494 movd m15, t0d
4495 %else
4496 %define m10 [base+pd_0x3ff]
4497 %define m9 [base+pd_0x4000]
4498 %define m8 m0
4499 %xdefine m14 m4
4500 %define m15 m3
4501 %if isprep
4502 %define ssq r3
4503 %endif
4504 movzx r5, byte [esp+0x1f0]
4505 sub srcq, 2
4506 movd m15, r5
4507 %endif
4508 pmaddwd m8, [base+rescale_mul]
4509 %if ARCH_X86_64
4510 mova m9, [base+pd_0x4000]
4511 %endif
4512 pshufd m15, m15, q0000
4513 paddd m14, m8 ; mx+dx*[0-3]
4514 pand m0, m14, m10
4515 psrld m0, 6
4516 paddd m15, m0
4517 pshufd m7, m15, q1032
4518 %if ARCH_X86_64
4519 movd r4d, m15
4520 movd r11d, m7
4521 pshufd m15, m15, q0321
4522 pshufd m7, m7, q0321
4523 movd r6d, m15
4524 movd r13d, m7
4525 mova m10, [base+bdct_lb_q+ 0]
4526 mova m11, [base+bdct_lb_q+16]
4527 movd m13, [base+subpel_filters+ r4*8+2]
4528 movd m2, [base+subpel_filters+ r6*8+2]
4529 movd m15, [base+subpel_filters+r11*8+2]
4530 movd m4, [base+subpel_filters+r13*8+2]
4531 %else
4532 movd r0, m15
4533 movd r4, m7
4534 pshufd m15, m15, q0321
4535 pshufd m7, m7, q0321
4536 movd rX, m15
4537 movd r5, m7
4538 mova m5, [base+bdct_lb_q+ 0]
4539 mova m6, [base+bdct_lb_q+16]
4540 movd m1, [base+subpel_filters+r0*8+2]
4541 movd m2, [base+subpel_filters+rX*8+2]
4542 movd m3, [base+subpel_filters+r4*8+2]
4543 movd m7, [base+subpel_filters+r5*8+2]
4544 SWAP m4, m7
4545 %if isprep
4546 mov r3, r3m
4547 %endif
4548 %define m10 m5
4549 %define m11 m6
4550 %define m12 m1
4551 %define m13 m1
4552 %endif
4553 psrld m14, 10
4554 paddd m14, m14
4555 punpckldq m13, m2
4556 punpckldq m15, m4
4557 punpcklqdq m13, m15
4558 pxor m2, m2
4559 pcmpeqd m0, m2
4560 %if ARCH_X86_64
4561 pand m9, m0
4562 %else
4563 pand m2, m9, m0
4564 %define m9 m2
4565 SWAP m7, m4
4566 %endif
4567 pandn m0, m13
4568 %if ARCH_X86_64
4569 SWAP m13, m0
4570 %else
4571 %define m13 m0
4572 %endif
4573 por m13, m9
4574 punpckhbw m15, m13, m13
4575 punpcklbw m13, m13
4576 psraw m15, 8
4577 psraw m13, 8
4578 pshufb m12, m14, m10
4579 pshufb m14, m11
4580 mova m10, [base+spel_s_shuf2]
4581 movd r4d, m14
4582 shr r4d, 24
4583 %if ARCH_X86_32
4584 mova [stk+0x40], m13
4585 mova [stk+0x50], m15
4586 pxor m2, m2
4587 %endif
4588 pshufb m7, m14, m2
4589 psubb m14, m7
4590 paddb m12, m10
4591 paddb m14, m10
4592 %if ARCH_X86_64
4593 lea r6, [r4+ssq*1]
4594 lea r11, [r4+ssq*2]
4595 lea r13, [r4+ss3q ]
4596 movu m7, [srcq+ssq*0]
4597 movu m9, [srcq+ssq*1]
4598 movu m8, [srcq+ssq*2]
4599 movu m10, [srcq+ss3q ]
4600 movu m1, [srcq+r4 ]
4601 movu m3, [srcq+r6 ]
4602 movu m2, [srcq+r11 ]
4603 movu m4, [srcq+r13 ]
4604 lea srcq, [srcq+ssq*4]
4605 REPX {pshufb x, m12}, m7, m9, m8, m10
4606 REPX {pmaddwd x, m13}, m7, m9, m8, m10
4607 REPX {pshufb x, m14}, m1, m3, m2, m4
4608 REPX {pmaddwd x, m15}, m1, m3, m2, m4
4609 mova m5, [rsp+0x10]
4610 movd xm6, [rsp+0x20]
4611 phaddd m7, m1
4612 phaddd m9, m3
4613 phaddd m8, m2
4614 phaddd m10, m4
4615 movu m1, [srcq+ssq*0]
4616 movu m2, [srcq+ssq*1]
4617 movu m3, [srcq+ssq*2]
4618 REPX {paddd x, m5}, m7, m9, m8, m10
4619 REPX {psrad x, xm6}, m7, m9, m8, m10
4620 packssdw m7, m9 ; 0 1
4621 packssdw m8, m10 ; 2 3
4622 movu m0, [srcq+r4 ]
4623 movu m9, [srcq+r6 ]
4624 movu m10, [srcq+r11 ]
4625 add srcq, ss3q
4626 REPX {pshufb x, m12}, m1, m2, m3
4627 REPX {pmaddwd x, m13}, m1, m2, m3
4628 REPX {pshufb x, m14}, m0, m9, m10
4629 REPX {pmaddwd x, m15}, m0, m9, m10
4630 phaddd m1, m0
4631 phaddd m2, m9
4632 phaddd m3, m10
4633 shr myd, 6
4634 mov r13d, 64 << 24
4635 lea myd, [t1+myq]
4636 cmovnz r13q, [base+subpel_filters+myq*8]
4637 REPX {paddd x, m5}, m1, m2, m3
4638 REPX {psrad x, xm6}, m1, m2, m3
4639 packssdw m1, m2 ; 4 5
4640 packssdw m3, m3 ; 6 6
4641 SWAP m9, m1
4642 shufps m4, m7, m8, q1032 ; 1 2
4643 shufps m5, m8, m9, q1032 ; 3 4
4644 shufps m6, m9, m3, q1032 ; 5 6
4645 punpcklwd m0, m7, m4 ; 01
4646 punpckhwd m7, m4 ; 12
4647 punpcklwd m1, m8, m5 ; 23
4648 punpckhwd m8, m5 ; 34
4649 punpcklwd m2, m9, m6 ; 45
4650 punpckhwd m9, m6 ; 56
4651 movq m10, r13
4652 mova [stk+0x00], m1
4653 mova [stk+0x10], m8
4654 mova [stk+0x20], m2
4655 mova [stk+0x30], m9
4656 mova [stk+0x40], m3
4657 %define hrnd_mem [rsp+0x10]
4658 %define hsh_mem [rsp+0x20]
4659 %define vsh_mem [rsp+0x28]
4660 %if isput
4661 %define vrnd_mem [rsp+0x30]
4662 %else
4663 %define vrnd_mem [base+pd_m524256]
4664 %endif
4665 %else
4666 mova [stk+0x20], m12
4667 mova [stk+0x30], m14
4668 add r4, srcq
4669 MC_4TAP_SCALED_H 0x60 ; 0 1
4670 MC_4TAP_SCALED_H 0x70 ; 2 3
4671 MC_4TAP_SCALED_H 0x80 ; 4 5
4672 movu m7, [srcq]
4673 movu m2, [r4]
4674 add srcq, ssq
4675 add r4, ssq
4676 mov [stk+0xb0], r4
4677 pshufb m7, m12
4678 pshufb m2, m14
4679 pmaddwd m7, m13
4680 pmaddwd m2, m15
4681 phaddd m7, m2
4682 paddd m7, [esp+0x00]
4683 psrad m7, [esp+0x10]
4684 packssdw m7, m7 ; 6 6
4685 mova m4, [stk+0x60]
4686 mova m5, [stk+0x70]
4687 mova m6, [stk+0x80]
4688 mov myd, mym
4689 mov rX, [esp+0x1f4]
4690 xor r5, r5
4691 shr myd, 6
4692 lea rX, [rX+myd]
4693 mov r4, 64 << 24
4694 cmovnz r4, [base+subpel_filters+rX*8+0]
4695 cmovnz r5, [base+subpel_filters+rX*8+4]
4696 mov r3, r3m
4697 shufps m1, m4, m5, q1032 ; 1 2
4698 shufps m2, m5, m6, q1032 ; 3 4
4699 shufps m3, m6, m7, q1032 ; 5 6
4700 mova [stk+0xa0], m7
4701 punpcklwd m0, m4, m1 ; 01
4702 punpckhwd m4, m1 ; 12
4703 punpcklwd m1, m5, m2 ; 23
4704 punpckhwd m5, m2 ; 34
4705 punpcklwd m2, m6, m3 ; 45
4706 punpckhwd m6, m3 ; 56
4707 movd m7, r4
4708 movd m3, r5
4709 mov r0, r0m
4710 %if isput
4711 mov r1, r1m
4712 %endif
4713 mov r4, [stk+0xb0]
4714 mova [stk+0xc0], m4 ; 12
4715 mova [stk+0x60], m1 ; 23
4716 mova [stk+0x70], m2 ; 45
4717 mova [stk+0x80], m5 ; 34
4718 mova [stk+0x90], m6 ; 56
4719 %define m12 [stk+0x20]
4720 %define m14 [stk+0x30]
4721 %define m13 [stk+0x40]
4722 %define m15 [stk+0x50]
4723 %define hrnd_mem [esp+0x00]
4724 %define hsh_mem [esp+0x10]
4725 %define vsh_mem [esp+0x18]
4726 %if isput
4727 %define vrnd_mem [esp+0x20]
4728 %else
4729 %define vrnd_mem [base+pd_m524256]
4730 %endif
4731 %define m10 m7
4732 punpckldq m10, m3
4733 %endif
4734 punpcklbw m10, m10
4735 psraw m10, 8
4736 pshufd m3, m10, q0000
4737 pshufd m4, m10, q1111
4738 pshufd m5, m10, q2222
4739 pshufd m10, m10, q3333
4740 %if ARCH_X86_32
4741 %xdefine m8 m3
4742 %xdefine m9 m6
4743 %xdefine m11 m5
4744 %xdefine m6 m4
4745 mova [stk+0x100], m3
4746 mova [stk+0x110], m4
4747 mova [stk+0x120], m5
4748 mova [stk+0x130], m10
4749 %define m3 [stk+0x100]
4750 %define m4 [stk+0x110]
4751 %define m5 [stk+0x120]
4752 %define m10 [stk+0x130]
4753 mova m7, [stk+0xc0]
4754 mova m8, [stk+0x80]
4755 %endif
4756 .dy1_w4_loop:
4757 movu m11, [srcq+ssq*0]
4758 movu m6, [srcq+ssq*1]
4759 pmaddwd m0, m3
4760 pmaddwd m7, m3
4761 pmaddwd m1, m4
4762 pmaddwd m8, m4
4763 pmaddwd m2, m5
4764 pmaddwd m9, m5
4765 paddd m1, m0
4766 paddd m8, m7
4767 %if ARCH_X86_64
4768 movu m0, [srcq+r4]
4769 movu m7, [srcq+r6]
4770 %else
4771 movu m0, [r4+ssq*0]
4772 movu m7, [r4+ssq*1]
4773 lea r4, [r4+ssq*2]
4774 %endif
4775 lea srcq, [srcq+ssq*2]
4776 paddd m1, m2
4777 paddd m8, m9
4778 pshufb m11, m12
4779 pshufb m6, m12
4780 pmaddwd m11, m13
4781 pmaddwd m6, m13
4782 pshufb m0, m14
4783 pshufb m7, m14
4784 pmaddwd m0, m15
4785 pmaddwd m7, m15
4786 phaddd m11, m0
4787 phaddd m6, m7
4788 paddd m11, hrnd_mem
4789 paddd m6, hrnd_mem
4790 psrad m11, hsh_mem
4791 psrad m6, hsh_mem
4792 packssdw m11, m6 ; 7 8
4793 %if ARCH_X86_64
4794 shufps m9, [stk+0x40], m11, q1032 ; 6 7
4795 mova m0, [stk+0x00]
4796 mova [stk+0x40], m11
4797 %else
4798 shufps m9, [stk+0xa0], m11, q1032 ; 6 7
4799 mova m0, [stk+0x60]
4800 mova [stk+0xa0], m11
4801 %endif
4802 punpcklwd m2, m9, m11 ; 67
4803 punpckhwd m9, m11 ; 78
4804 pmaddwd m6, m2, m10
4805 pmaddwd m7, m9, m10
4806 %if isput
4807 movd m11, vsh_mem
4808 %endif
4809 paddd m1, vrnd_mem
4810 paddd m8, vrnd_mem
4811 paddd m1, m6
4812 paddd m8, m7
4813 %if ARCH_X86_64
4814 mova m7, [stk+0x10]
4815 %else
4816 mova m7, [stk+0x80]
4817 %endif
4818 %if isput
4819 psrad m1, m11
4820 psrad m8, m11
4821 %else
4822 psrad m1, 6
4823 psrad m8, 6
4824 %endif
4825 packssdw m1, m8
4826 %if ARCH_X86_64
4827 mova m8, [stk+0x30]
4828 %else
4829 mova m8, [stk+0x90]
4830 %endif
4831 %if isput
4832 pxor m6, m6
4833 pmaxsw m1, m6
4834 pminsw m1, pxmaxm
4835 movq [dstq+dsq*0], m1
4836 movhps [dstq+dsq*1], m1
4837 lea dstq, [dstq+dsq*2]
4838 %else
4839 mova [tmpq], m1
4840 add tmpq, 16
4841 %endif
4842 %if ARCH_X86_64
4843 mova m1, [stk+0x20]
4844 mova [stk+0x10], m8
4845 mova [stk+0x00], m1
4846 mova [stk+0x20], m2
4847 mova [stk+0x30], m9
4848 %else
4849 mova m1, [stk+0x70]
4850 mova [stk+0x80], m8
4851 mova [stk+0x60], m1
4852 mova [stk+0x70], m2
4853 mova [stk+0x90], m9
4854 %endif
4855 sub hd, 2
4856 jg .dy1_w4_loop
4857 MC_8TAP_SCALED_RET ; why not jz .ret?
4858 INIT_XMM ssse3
4859 .dy1_w8:
4860 mov dword [stk+0xf0], 1
4861 movifprep tmp_stridem, 16
4862 jmp .dy1_w_start
4863 .dy1_w16:
4864 mov dword [stk+0xf0], 2
4865 movifprep tmp_stridem, 32
4866 jmp .dy1_w_start
4867 .dy1_w32:
4868 mov dword [stk+0xf0], 4
4869 movifprep tmp_stridem, 64
4870 jmp .dy1_w_start
4871 .dy1_w64:
4872 mov dword [stk+0xf0], 8
4873 movifprep tmp_stridem, 128
4874 jmp .dy1_w_start
4875 .dy1_w128:
4876 mov dword [stk+0xf0], 16
4877 movifprep tmp_stridem, 256
4878 .dy1_w_start:
4879 mov myd, mym
4880 %if ARCH_X86_64
4881 %ifidn %1, put
4882 movifnidn dsm, dsq
4883 %endif
4884 mova [rsp+0x10], m11
4885 mova [rsp+0x20], m12
4886 %define hround m11
4887 %if isput
4888 mova [rsp+0x30], m13
4889 %else
4890 mova m13, [base+pd_m524256]
4891 %endif
4892 shr t0d, 16
4893 shr myd, 6
4894 mov r4d, 64 << 24
4895 lea myd, [t1+myq]
4896 cmovnz r4q, [base+subpel_filters+myq*8]
4897 movd m15, t0d
4898 %else
4899 %define hround [esp+0x00]
4900 %define m12 [esp+0x10]
4901 %define m10 [base+pd_0x3ff]
4902 %define m8 m0
4903 %xdefine m14 m4
4904 %xdefine m15 m3
4905 %if isprep
4906 %define ssq ssm
4907 %endif
4908 mov r5, [esp+0x1f0]
4909 mov r3, [esp+0x1f4]
4910 shr r5, 16
4911 movd m15, r5
4912 xor r5, r5
4913 shr myd, 6
4914 lea r3, [r3+myd]
4915 mov r4, 64 << 24
4916 cmovnz r4, [base+subpel_filters+r3*8+0]
4917 cmovnz r5, [base+subpel_filters+r3*8+4]
4918 mov r0, r0m
4919 mov r3, r3m
4920 %endif
4921 sub srcq, 6
4922 pslld m7, m8, 2 ; dx*4
4923 pmaddwd m8, [base+rescale_mul] ; dx*[0-3]
4924 pshufd m15, m15, q0000
4925 paddd m14, m8 ; mx+dx*[0-3]
4926 %if ARCH_X86_64
4927 movq m3, r4q
4928 %else
4929 movd m5, r4
4930 movd m6, r5
4931 punpckldq m5, m6
4932 SWAP m3, m5
4933 %endif
4934 punpcklbw m3, m3
4935 psraw m3, 8
4936 mova [stk+0x100], m7
4937 mova [stk+0x120], m15
4938 mov [stk+0x0f8], srcq
4939 mov [stk+0x130], r0q ; dstq / tmpq
4940 pshufd m0, m3, q0000
4941 pshufd m1, m3, q1111
4942 pshufd m2, m3, q2222
4943 pshufd m3, m3, q3333
4944 %if ARCH_X86_64
4945 mova [stk+0x140], m0
4946 mova [stk+0x150], m1
4947 mova [stk+0x160], m2
4948 mova [stk+0x170], m3
4949 %if UNIX64
4950 mov hm, hd
4951 %endif
4952 %else
4953 mova [stk+0x180], m0
4954 mova [stk+0x190], m1
4955 mova [stk+0x1a0], m2
4956 mova [stk+0x1b0], m3
4957 SWAP m5, m3
4958 mov r5, hm
4959 mov [stk+0x134], r5
4960 %endif
4961 jmp .dy1_hloop
4962 .dy1_hloop_prep:
4963 dec dword [stk+0x0f0]
4964 jz .ret
4965 %if ARCH_X86_64
4966 add qword [stk+0x130], 16
4967 mov hd, hm
4968 %else
4969 add dword [stk+0x130], 16
4970 mov r5, [stk+0x134]
4971 mov r0, [stk+0x130]
4972 %endif
4973 mova m7, [stk+0x100]
4974 mova m14, [stk+0x110]
4975 %if ARCH_X86_64
4976 mova m10, [base+pd_0x3ff]
4977 mova m11, [rsp+0x10]
4978 %endif
4979 mova m15, [stk+0x120]
4980 mov srcq, [stk+0x0f8]
4981 %if ARCH_X86_64
4982 mov r0q, [stk+0x130] ; dstq / tmpq
4983 %else
4984 mov hm, r5
4985 mov r0m, r0
4986 mov r3, r3m
4987 %endif
4988 paddd m14, m7
4989 .dy1_hloop:
4990 %if ARCH_X86_64
4991 mova m9, [base+pq_0x40000000]
4992 %else
4993 %define m9 [base+pq_0x40000000]
4994 %endif
4995 pxor m1, m1
4996 psrld m2, m14, 10
4997 mova [stk], m2
4998 pand m6, m14, m10
4999 psrld m6, 6
5000 paddd m5, m15, m6
5001 pcmpeqd m6, m1
5002 pshufd m2, m5, q1032
5003 %if ARCH_X86_64
5004 movd r4d, m5
5005 movd r6d, m2
5006 pshufd m5, m5, q0321
5007 pshufd m2, m2, q0321
5008 movd r7d, m5
5009 movd r9d, m2
5010 movq m0, [base+subpel_filters+r4*8]
5011 movq m1, [base+subpel_filters+r6*8]
5012 movhps m0, [base+subpel_filters+r7*8]
5013 movhps m1, [base+subpel_filters+r9*8]
5014 %else
5015 movd r0, m5
5016 movd rX, m2
5017 pshufd m5, m5, q0321
5018 pshufd m2, m2, q0321
5019 movd r4, m5
5020 movd r5, m2
5021 movq m0, [base+subpel_filters+r0*8]
5022 movq m1, [base+subpel_filters+rX*8]
5023 movhps m0, [base+subpel_filters+r4*8]
5024 movhps m1, [base+subpel_filters+r5*8]
5025 %endif
5026 paddd m14, m7 ; mx+dx*[4-7]
5027 pand m5, m14, m10
5028 psrld m5, 6
5029 paddd m15, m5
5030 pxor m2, m2
5031 pcmpeqd m5, m2
5032 mova [stk+0x110], m14
5033 pshufd m4, m15, q1032
5034 %if ARCH_X86_64
5035 movd r10d, m15
5036 movd r11d, m4
5037 pshufd m15, m15, q0321
5038 pshufd m4, m4, q0321
5039 movd r13d, m15
5040 movd rXd, m4
5041 movq m2, [base+subpel_filters+r10*8]
5042 movq m3, [base+subpel_filters+r11*8]
5043 movhps m2, [base+subpel_filters+r13*8]
5044 movhps m3, [base+subpel_filters+ rX*8]
5045 psrld m14, 10
5046 movq r11, m14
5047 punpckhqdq m14, m14
5048 movq rX, m14
5049 mov r10d, r11d
5050 shr r11, 32
5051 mov r13d, rXd
5052 shr rX, 32
5053 mov r4d, [stk+ 0]
5054 mov r6d, [stk+ 4]
5055 mov r7d, [stk+ 8]
5056 mov r9d, [stk+12]
5057 pshufd m4, m6, q1100
5058 pshufd m6, m6, q3322
5059 pshufd m14, m5, q1100
5060 pshufd m5, m5, q3322
5061 pand m7, m9, m4
5062 pand m8, m9, m6
5063 pand m15, m9, m14
5064 pand m9, m9, m5
5065 pandn m4, m0
5066 pandn m6, m1
5067 pandn m14, m2
5068 pandn m5, m3
5069 por m7, m4
5070 por m8, m6
5071 por m15, m14
5072 por m9, m5
5073 punpcklbw m0, m7, m7
5074 punpckhbw m7, m7
5075 punpcklbw m1, m8, m8
5076 punpckhbw m8, m8
5077 psraw m0, 8
5078 psraw m7, 8
5079 psraw m1, 8
5080 psraw m8, 8
5081 punpcklbw m2, m15, m15
5082 punpckhbw m15, m15
5083 punpcklbw m3, m9, m9
5084 punpckhbw m9, m9
5085 psraw m2, 8
5086 psraw m15, 8
5087 psraw m3, 8
5088 psraw m9, 8
5089 mova [stk+0x10], m0
5090 mova [stk+0x20], m7
5091 mova [stk+0x30], m1
5092 mova [stk+0x40], m8
5093 mova [stk+0x50], m2
5094 mova [stk+0x60], m15
5095 mova [stk+0x70], m3
5096 mova [stk+0x80], m9
5097 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10 ; 0
5098 mova [stk+0x90], m1
5099 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 1, 9, 10 ; 1
5100 mova [stk+0xa0], m2
5101 MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10 ; 2
5102 mova [stk+0xb0], m3
5103 MC_8TAP_SCALED_H 4, 5, 6, 1, 2, 3, 9, 10 ; 3
5104 mova [stk+0xc0], m4
5105 MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10 ; 4
5106 mova [stk+0xd0], m5
5107 MC_8TAP_SCALED_H 6, 1, 2, 3, 4, 5, 9, 10 ; 5
5108 MC_8TAP_SCALED_H 7, 1, 2, 3, 4, 5, 9, 10 ; 6
5109 MC_8TAP_SCALED_H 8, 1, 2, 3, 4, 5, 9, 10 ; 7
5110 mova m5, [stk+0xd0]
5111 mova m1, [stk+0x90]
5112 mova m2, [stk+0xa0]
5113 mova m3, [stk+0xb0]
5114 mova m9, [stk+0xc0]
5115 punpcklwd m4, m5, m6 ; 45a
5116 punpckhwd m5, m6 ; 45b
5117 punpcklwd m6, m7, m8 ; 67a
5118 punpckhwd m7, m8 ; 67b
5119 punpcklwd m0, m1, m2 ; 01a
5120 punpckhwd m1, m2 ; 01b
5121 punpcklwd m2, m3, m9 ; 23a
5122 punpckhwd m3, m9 ; 23b
5123 mova m10, [stk+0x140]
5124 mova m11, [stk+0x150]
5125 mova m14, [stk+0x160]
5126 mova m15, [stk+0x170]
5127 mova [stk+0x90], m4
5128 mova [stk+0xa0], m5
5129 mova [stk+0xb0], m6
5130 mova [stk+0xc0], m7
5131 %define hround [rsp+0x10]
5132 %define shift [rsp+0x20]
5133 %if isput
5134 %define vround [rsp+0x30]
5135 %else
5136 %define vround [base+pd_m524256]
5137 %endif
5138 .dy1_vloop:
5139 pmaddwd m4, m0, m10
5140 pmaddwd m5, m1, m10
5141 pmaddwd m6, m2, m11
5142 pmaddwd m7, m3, m11
5143 paddd m4, m13
5144 paddd m5, m13
5145 paddd m4, m6
5146 paddd m5, m7
5147 pmaddwd m6, [stk+0x90], m14
5148 pmaddwd m7, [stk+0xa0], m14
5149 pmaddwd m8, [stk+0xb0], m15
5150 pmaddwd m9, [stk+0xc0], m15
5151 paddd m4, m6
5152 paddd m5, m7
5153 %if isput
5154 pshufd m6, m12, q1032
5155 %endif
5156 paddd m4, m8
5157 paddd m5, m9
5158 %else
5159 movd r0, m15
5160 movd rX, m4
5161 pshufd m15, m15, q0321
5162 pshufd m4, m4, q0321
5163 movd r4, m15
5164 movd r5, m4
5165 mova m14, [stk+0x110]
5166 movq m2, [base+subpel_filters+r0*8]
5167 movq m3, [base+subpel_filters+rX*8]
5168 movhps m2, [base+subpel_filters+r4*8]
5169 movhps m3, [base+subpel_filters+r5*8]
5170 psrld m14, 10
5171 mova [stk+16], m14
5172 mov r0, [stk+ 0]
5173 mov rX, [stk+ 4]
5174 mov r4, [stk+ 8]
5175 mov r5, [stk+12]
5176 mova [stk+0x20], m0
5177 mova [stk+0x30], m1
5178 mova [stk+0x40], m2
5179 mova [stk+0x50], m3
5180 pshufd m4, m6, q1100
5181 pshufd m6, m6, q3322
5182 pshufd m7, m5, q1100
5183 pshufd m5, m5, q3322
5184 pand m0, m9, m4
5185 pand m1, m9, m6
5186 pand m2, m9, m7
5187 pand m3, m9, m5
5188 pandn m4, [stk+0x20]
5189 pandn m6, [stk+0x30]
5190 pandn m7, [stk+0x40]
5191 pandn m5, [stk+0x50]
5192 por m0, m4
5193 por m1, m6
5194 por m2, m7
5195 por m3, m5
5196 punpcklbw m4, m0, m0
5197 punpckhbw m0, m0
5198 punpcklbw m5, m1, m1
5199 punpckhbw m1, m1
5200 psraw m4, 8
5201 psraw m0, 8
5202 psraw m5, 8
5203 psraw m1, 8
5204 punpcklbw m6, m2, m2
5205 punpckhbw m2, m2
5206 punpcklbw m7, m3, m3
5207 punpckhbw m3, m3
5208 psraw m6, 8
5209 psraw m2, 8
5210 psraw m7, 8
5211 psraw m3, 8
5212 mova [stk+0x0a0], m4
5213 mova [stk+0x0b0], m0
5214 mova [stk+0x0c0], m5
5215 mova [stk+0x0d0], m1
5216 mova [stk+0x140], m6
5217 mova [stk+0x150], m2
5218 mova [stk+0x160], m7
5219 mova [stk+0x170], m3
5220 MC_8TAP_SCALED_H 0xa0, 0x20, 0 ; 0
5221 MC_8TAP_SCALED_H 0xa0, 0x30 ; 1
5222 MC_8TAP_SCALED_H 0xa0, 0x40 ; 2
5223 MC_8TAP_SCALED_H 0xa0, 0x50 ; 3
5224 MC_8TAP_SCALED_H 0xa0, 0x60 ; 4
5225 MC_8TAP_SCALED_H 0xa0, 0x70 ; 5
5226 MC_8TAP_SCALED_H 0xa0, 0x80 ; 6
5227 MC_8TAP_SCALED_H 0xa0, 0x90 ; 7
5228 mova m5, [stk+0x60]
5229 mova m6, [stk+0x70]
5230 mova m7, [stk+0x80]
5231 mova m0, [stk+0x90]
5232 mov r0, r0m
5233 punpcklwd m4, m5, m6 ; 45a
5234 punpckhwd m5, m6 ; 45b
5235 punpcklwd m6, m7, m0 ; 67a
5236 punpckhwd m7, m0 ; 67b
5237 mova [stk+0x60], m4
5238 mova [stk+0x70], m5
5239 mova [stk+0x80], m6
5240 mova [stk+0x90], m7
5241 mova m1, [stk+0x20]
5242 mova m2, [stk+0x30]
5243 mova m3, [stk+0x40]
5244 mova m4, [stk+0x50]
5245 punpcklwd m0, m1, m2 ; 01a
5246 punpckhwd m1, m2 ; 01b
5247 punpcklwd m2, m3, m4 ; 23a
5248 punpckhwd m3, m4 ; 23b
5249 mova m4, [stk+0x180]
5250 mova m5, [stk+0x190]
5251 mova m6, [stk+0x1a0]
5252 mova m7, [stk+0x1b0]
5253 mova [stk+0x20], m0
5254 mova [stk+0x30], m1
5255 mova [stk+0x40], m2
5256 mova [stk+0x50], m3
5257 .dy1_vloop:
5258 pmaddwd m0, m4
5259 pmaddwd m1, m4
5260 pmaddwd m2, m5
5261 pmaddwd m3, m5
5262 paddd m0, m2
5263 paddd m1, m3
5264 pmaddwd m2, [stk+0x60], m6
5265 pmaddwd m3, [stk+0x70], m6
5266 pmaddwd m4, [stk+0x80], m7
5267 pmaddwd m5, [stk+0x90], m7
5268 %if isput
5269 movd m6, [esp+0x18]
5270 %endif
5271 paddd m0, m2
5272 paddd m1, m3
5273 paddd m0, vrnd_mem
5274 paddd m1, vrnd_mem
5275 paddd m4, m0
5276 paddd m5, m1
5277 %endif
5278 %ifidn %1, put
5279 psrad m4, m6
5280 psrad m5, m6
5281 packssdw m4, m5
5282 pxor m7, m7
5283 pmaxsw m4, m7
5284 pminsw m4, pxmaxm
5285 mova [dstq], m4
5286 add dstq, dsm
5287 %else
5288 psrad m4, 6
5289 psrad m5, 6
5290 packssdw m4, m5
5291 mova [tmpq], m4
5292 add tmpq, tmp_stridem
5293 %endif
5294 dec hd
5295 jz .dy1_hloop_prep
5296 %if ARCH_X86_64
5297 movu m8, [srcq+r10*2]
5298 movu m9, [srcq+r11*2]
5299 movu m12, [srcq+r13*2]
5300 movu m13, [srcq+ rX*2]
5301 movu m4, [srcq+ r4*2]
5302 movu m5, [srcq+ r6*2]
5303 movu m6, [srcq+ r7*2]
5304 movu m7, [srcq+ r9*2]
5305 add srcq, ssq
5306 pmaddwd m8, [stk+0x50]
5307 pmaddwd m9, [stk+0x60]
5308 pmaddwd m12, [stk+0x70]
5309 pmaddwd m13, [stk+0x80]
5310 pmaddwd m4, [stk+0x10]
5311 pmaddwd m5, [stk+0x20]
5312 pmaddwd m6, [stk+0x30]
5313 pmaddwd m7, [stk+0x40]
5314 phaddd m8, m9
5315 phaddd m12, m13
5316 mova m9, [base+unpckw]
5317 mova m13, hround
5318 phaddd m4, m5
5319 phaddd m6, m7
5320 phaddd m8, m12
5321 phaddd m4, m6
5322 pshufd m5, m9, q1032
5323 pshufb m0, m9 ; 0a 1a
5324 pshufb m1, m9 ; 0b 1b
5325 pshufb m2, m5 ; 3a 2a
5326 pshufb m3, m5 ; 3b 2b
5327 mova m12, shift
5328 paddd m4, m13
5329 paddd m8, m13
5330 psrad m4, m12
5331 psrad m8, m12
5332 packssdw m4, m8
5333 pshufb m6, [stk+0x90], m9 ; 4a 5a
5334 pshufb m7, [stk+0xa0], m9 ; 4b 5b
5335 pshufb m8, [stk+0xb0], m5 ; 7a 6a
5336 pshufb m13, [stk+0xc0], m5 ; 7b 6b
5337 punpckhwd m0, m2 ; 12a
5338 punpckhwd m1, m3 ; 12b
5339 punpcklwd m2, m6 ; 34a
5340 punpcklwd m3, m7 ; 34b
5341 punpckhwd m6, m8 ; 56a
5342 punpckhwd m7, m13 ; 56b
5343 punpcklwd m8, m4 ; 78a
5344 punpckhqdq m4, m4
5345 punpcklwd m13, m4 ; 78b
5346 mova [stk+0x90], m6
5347 mova [stk+0xa0], m7
5348 mova [stk+0xb0], m8
5349 mova [stk+0xc0], m13
5350 mova m13, vround
5351 %else
5352 mov r0m, r0
5353 mov r3, r3m
5354 mov r0, [stk+ 0]
5355 mov rX, [stk+ 4]
5356 mov r4, [stk+ 8]
5357 mov r5, [stk+12]
5358 MC_8TAP_SCALED_H 0xa0, 0xe0, 0 ; 8
5359 mova m7, [base+unpckw]
5360 pshufd m4, m7, q1032
5361 pshufb m0, [stk+0x20], m7 ; 0a 1a
5362 pshufb m1, [stk+0x30], m7 ; 0b 1b
5363 pshufb m2, [stk+0x40], m4 ; 3a 2a
5364 pshufb m3, [stk+0x50], m4 ; 3b 2b
5365 pshufb m5, [stk+0x60], m7 ; 4a 5a
5366 pshufb m6, [stk+0x70], m7 ; 4b 5b
5367 pshufb m7, [stk+0x80], m4 ; 7a 6a
5368 punpckhwd m0, m2 ; 12a
5369 punpckhwd m1, m3 ; 12b
5370 punpcklwd m2, m5 ; 34a
5371 punpcklwd m3, m6 ; 34b
5372 mova [stk+0x20], m0
5373 mova [stk+0x30], m1
5374 mova [stk+0x40], m2
5375 mova [stk+0x50], m3
5376 punpckhwd m5, m7 ; 56a
5377 mova [stk+0x60], m5
5378 pshufb m5, [stk+0x90], m4 ; 7b 6b
5379 punpcklwd m7, [stk+0xe0] ; 78a
5380 mova m4, [stk+0x180]
5381 punpckhwd m6, m5 ; 56b
5382 mova [stk+0x70], m6
5383 movq m6, [stk+0xe8]
5384 mova [stk+0x80], m7
5385 mova m7, [stk+0x1b0]
5386 punpcklwd m5, m6
5387 mova m6, [stk+0x1a0]
5388 mova [stk+0x90], m5
5389 mova m5, [stk+0x190]
5390 mov r0, r0m
5391 %endif
5392 jmp .dy1_vloop
5393 INIT_XMM ssse3
5394 %if ARCH_X86_64
5395 %define stk rsp+0x20
5396 %endif
5397 .dy2:
5398 movzx wd, word [base+%1_8tap_scaled_ssse3_dy2_table+wq*2]
5399 add wq, base_reg
5400 jmp wq
5401 %if isput
5402 .dy2_w2:
5403 %if ARCH_X86_64
5404 mov myd, mym
5405 mova [rsp+0x10], m13
5406 %define vrnd_mem [rsp+0x10]
5407 movzx t0d, t0b
5408 sub srcq, 2
5409 movd m15, t0d
5410 %else
5411 %define m8 m0
5412 %define m9 m1
5413 %define m14 m4
5414 %define m15 m3
5415 %define m11 [esp+0x00]
5416 %define m12 [esp+0x10]
5417 %define vrnd_mem [esp+0x20]
5418 mov r1, r1m
5419 movzx r5, byte [esp+0x1f0]
5420 sub srcq, 2
5421 movd m15, r5
5422 %endif
5423 pxor m9, m9
5424 punpckldq m9, m8
5425 paddd m14, m9 ; mx+dx*[0-1]
5426 %if ARCH_X86_64
5427 mova m9, [base+pd_0x4000]
5428 %endif
5429 pshufd m15, m15, q0000
5430 pand m8, m14, m10
5431 psrld m8, 6
5432 paddd m15, m8
5433 movd r4d, m15
5434 pshufd m15, m15, q0321
5435 %if ARCH_X86_64
5436 movd r6d, m15
5437 %else
5438 movd r3d, m15
5439 %endif
5440 mova m5, [base+bdct_lb_q]
5441 mova m6, [base+spel_s_shuf2]
5442 movd m15, [base+subpel_filters+r4*8+2]
5443 %if ARCH_X86_64
5444 movd m7, [base+subpel_filters+r6*8+2]
5445 %else
5446 movd m7, [base+subpel_filters+r3*8+2]
5447 %endif
5448 pxor m2, m2
5449 pcmpeqd m8, m2
5450 psrld m14, 10
5451 paddd m14, m14
5452 %if ARCH_X86_32
5453 mov r3, r3m
5454 pshufb m14, m5
5455 paddb m14, m6
5456 mova [stk], m14
5457 SWAP m5, m0
5458 SWAP m6, m3
5459 %define m15 m6
5460 %endif
5461 movu m0, [srcq+ssq*0]
5462 movu m1, [srcq+ssq*2]
5463 movu m2, [srcq+ssq*4]
5464 punpckldq m15, m7
5465 %if ARCH_X86_64
5466 pshufb m14, m5
5467 paddb m14, m6
5468 pand m9, m8
5469 pandn m8, m15
5470 SWAP m15, m8
5471 por m15, m9
5472 movu m4, [srcq+ssq*1]
5473 movu m5, [srcq+ss3q ]
5474 lea srcq, [srcq+ssq*4]
5475 movu m6, [srcq+ssq*1]
5476 lea srcq, [srcq+ssq*2]
5477 shr myd, 6
5478 mov r4d, 64 << 24
5479 lea myd, [t1+myq]
5480 cmovnz r4q, [base+subpel_filters+myq*8]
5481 %else
5482 pand m7, m5, [base+pd_0x4000]
5483 pandn m5, m15
5484 por m5, m7
5485 %define m15 m5
5486 mov myd, mym
5487 mov r5, [esp+0x1f4]
5488 xor r3, r3
5489 shr myd, 6
5490 lea r5, [r5+myd]
5491 mov r4, 64 << 24
5492 cmovnz r4, [base+subpel_filters+r5*8+0]
5493 cmovnz r3, [base+subpel_filters+r5*8+4]
5494 mov [stk+0x20], r3
5495 mov r3, r3m
5496 %endif
5497 punpcklbw m15, m15
5498 psraw m15, 8
5499 REPX {pshufb x, m14}, m0, m1, m2
5500 REPX {pmaddwd x, m15}, m0, m1, m2
5501 %if ARCH_X86_64
5502 REPX {pshufb x, m14}, m4, m5, m6
5503 REPX {pmaddwd x, m15}, m4, m5, m6
5504 phaddd m0, m1
5505 phaddd m1, m2
5506 phaddd m4, m5
5507 phaddd m5, m6
5508 REPX {paddd x, m11}, m0, m1, m4, m5
5509 REPX {psrad x, m12}, m0, m1, m4, m5
5510 packssdw m0, m1 ; 0 2 2 4
5511 packssdw m4, m5 ; 1 3 3 5
5512 SWAP m2, m4
5513 movq m10, r4
5514 %else
5515 mova [stk+0x10], m15
5516 phaddd m0, m1
5517 phaddd m1, m2
5518 movu m2, [srcq+ssq*1]
5519 movu m7, [srcq+ss3q ]
5520 lea srcq, [srcq+ssq*4]
5521 movu m6, [srcq+ssq*1]
5522 lea srcq, [srcq+ssq*2]
5523 REPX {pshufb x, m14}, m2, m7, m6
5524 REPX {pmaddwd x, m15}, m2, m7, m6
5525 %define m14 [stk+0x00]
5526 %define m15 [stk+0x10]
5527 phaddd m2, m7
5528 phaddd m7, m6
5529 REPX {paddd x, m11}, m0, m1, m2, m7
5530 REPX {psrad x, m12}, m0, m1, m2, m7
5531 packssdw m0, m1
5532 packssdw m2, m7
5533 %define m8 m6
5534 %define m9 m4
5535 %define m10 m5
5536 movd m10, r4
5537 movd m9, [stk+0x20]
5538 punpckldq m10, m9
5539 %endif
5540 punpcklbw m10, m10
5541 psraw m10, 8
5542 pshufd m7, m10, q0000
5543 pshufd m8, m10, q1111
5544 pshufd m9, m10, q2222
5545 pshufd m10, m10, q3333
5546 %if ARCH_X86_32
5547 mova [stk+0x50], m7
5548 mova [stk+0x60], m8
5549 mova [stk+0x70], m9
5550 mova [stk+0x80], m10
5551 %xdefine m13 m7
5552 %define m7 [stk+0x50]
5553 %define m8 [stk+0x60]
5554 %define m9 [stk+0x70]
5555 %define m10 [stk+0x80]
5556 %endif
5557 punpcklwd m1, m0, m2 ; 01 23
5558 punpckhwd m3, m0, m2 ; 23 45
5559 %if ARCH_X86_32
5560 mov r4, r0m
5561 %define dstq r4
5562 mova [stk+0x20], m3
5563 mova [stk+0x30], m0
5564 %endif
5565 .dy2_w2_loop:
5566 movu m4, [srcq+ssq*0]
5567 movu m5, [srcq+ssq*1]
5568 movu m6, [srcq+ssq*2]
5569 movu m13, [srcq+ss3q ]
5570 lea srcq, [srcq+ssq*4]
5571 pmaddwd m3, m8
5572 REPX {pshufb x, m14}, m4, m5, m6, m13
5573 REPX {pmaddwd x, m15}, m4, m5, m6, m13
5574 phaddd m4, m5
5575 phaddd m6, m13
5576 pmaddwd m5, m1, m7
5577 paddd m4, m11
5578 paddd m6, m11
5579 psrad m4, m12
5580 psrad m6, m12
5581 packssdw m4, m6 ; 6 7 8 9
5582 paddd m5, m3
5583 pshufd m3, m4, q2200
5584 pshufd m4, m4, q3311
5585 palignr m3, m0, 12 ; 4 6 6 8
5586 palignr m4, m2, 12 ; 5 7 7 9
5587 mova m0, m3
5588 mova m2, m4
5589 punpcklwd m1, m3, m4
5590 punpckhwd m3, m4
5591 pmaddwd m6, m1, m9
5592 pmaddwd m4, m3, m10
5593 paddd m5, vrnd_mem
5594 paddd m6, m4
5595 paddd m5, m6
5596 pshufd m4, m12, q1032
5597 pxor m6, m6
5598 psrad m5, m4
5599 packssdw m5, m5
5600 pmaxsw m5, m6
5601 pminsw m5, pxmaxm
5602 movd [dstq+dsq*0], m5
5603 pshuflw m5, m5, q1032
5604 movd [dstq+dsq*1], m5
5605 lea dstq, [dstq+dsq*2]
5606 sub hd, 2
5607 jg .dy2_w2_loop
5608 RET
5609 %endif
5610 INIT_XMM ssse3
5611 .dy2_w4:
5612 %if ARCH_X86_64
5613 mov myd, mym
5614 mova [rsp+0x10], m11
5615 mova [rsp+0x20], m12
5616 %if isput
5617 mova [rsp+0x30], m13
5618 %define vrnd_mem [rsp+0x30]
5619 %define stk rsp+0x40
5620 %else
5621 %define vrnd_mem [base+pd_m524256]
5622 %define stk rsp+0x30
5623 %endif
5624 movzx t0d, t0b
5625 sub srcq, 2
5626 movd m15, t0d
5627 %else
5628 %define m10 [base+pd_0x3ff]
5629 %define m9 [base+pd_0x4000]
5630 %define m8 m0
5631 %xdefine m14 m4
5632 %define m15 m3
5633 %if isprep
5634 %define ssq r3
5635 %endif
5636 movzx r5, byte [esp+0x1f0]
5637 sub srcq, 2
5638 movd m15, r5
5639 %endif
5640 pmaddwd m8, [base+rescale_mul]
5641 %if ARCH_X86_64
5642 mova m9, [base+pd_0x4000]
5643 %endif
5644 pshufd m15, m15, q0000
5645 paddd m14, m8 ; mx+dx*[0-3]
5646 pand m0, m14, m10
5647 psrld m0, 6
5648 paddd m15, m0
5649 pshufd m7, m15, q1032
5650 %if ARCH_X86_64
5651 movd r4d, m15
5652 movd r11d, m7
5653 pshufd m15, m15, q0321
5654 pshufd m7, m7, q0321
5655 movd r6d, m15
5656 movd r13d, m7
5657 mova m10, [base+bdct_lb_q+ 0]
5658 mova m11, [base+bdct_lb_q+16]
5659 movd m13, [base+subpel_filters+ r4*8+2]
5660 movd m2, [base+subpel_filters+ r6*8+2]
5661 movd m15, [base+subpel_filters+r11*8+2]
5662 movd m4, [base+subpel_filters+r13*8+2]
5663 %else
5664 movd r1, m15
5665 movd r4, m7
5666 pshufd m15, m15, q0321
5667 pshufd m7, m7, q0321
5668 movd r3, m15
5669 movd r5, m7
5670 mova m5, [base+bdct_lb_q+ 0]
5671 mova m6, [base+bdct_lb_q+16]
5672 movd m1, [base+subpel_filters+r1*8+2]
5673 movd m2, [base+subpel_filters+r3*8+2]
5674 movd m3, [base+subpel_filters+r4*8+2]
5675 movd m7, [base+subpel_filters+r5*8+2]
5676 SWAP m4, m7
5677 mov r3, r3m
5678 %if isprep
5679 lea ss3q, [ssq*3]
5680 %endif
5681 %define m10 m5
5682 %define m11 m6
5683 %define m12 m1
5684 %define m13 m1
5685 %endif
5686 psrld m14, 10
5687 paddd m14, m14
5688 punpckldq m13, m2
5689 punpckldq m15, m4
5690 punpcklqdq m13, m15
5691 pxor m2, m2
5692 pcmpeqd m0, m2
5693 %if ARCH_X86_64
5694 pand m9, m0
5695 %else
5696 pand m2, m9, m0
5697 %define m9 m2
5698 SWAP m7, m4
5699 %endif
5700 pandn m0, m13
5701 %if ARCH_X86_64
5702 SWAP m13, m0
5703 %else
5704 %define m13 m0
5705 %endif
5706 por m13, m9
5707 punpckhbw m15, m13, m13
5708 punpcklbw m13, m13
5709 psraw m15, 8
5710 psraw m13, 8
5711 pshufb m12, m14, m10
5712 pshufb m14, m11
5713 mova m10, [base+spel_s_shuf2]
5714 movd r4d, m14
5715 shr r4d, 24
5716 %if ARCH_X86_32
5717 mova [stk+0x40], m13
5718 mova [stk+0x50], m15
5719 pxor m2, m2
5720 %endif
5721 pshufb m7, m14, m2
5722 psubb m14, m7
5723 paddb m12, m10
5724 paddb m14, m10
5725 %if ARCH_X86_64
5726 lea r6, [r4+ssq*1]
5727 lea r11, [r4+ssq*2]
5728 lea r13, [r4+ss3q ]
5729 movu m1, [srcq+ssq*0]
5730 movu m8, [srcq+ssq*2]
5731 movu m9, [srcq+ssq*1]
5732 movu m10, [srcq+ss3q ]
5733 movu m7, [srcq+r4 ]
5734 movu m2, [srcq+r11 ]
5735 movu m3, [srcq+r6 ]
5736 movu m4, [srcq+r13 ]
5737 lea srcq, [srcq+ssq*4]
5738 REPX {pshufb x, m12}, m1, m9, m8, m10
5739 REPX {pmaddwd x, m13}, m1, m9, m8, m10
5740 REPX {pshufb x, m14}, m7, m3, m2, m4
5741 REPX {pmaddwd x, m15}, m7, m3, m2, m4
5742 mova m5, [rsp+0x10]
5743 movd xm6, [rsp+0x20]
5744 phaddd m1, m7
5745 phaddd m8, m2
5746 phaddd m9, m3
5747 phaddd m10, m4
5748 movu m2, [srcq+ssq*0]
5749 movu m3, [srcq+ssq*1]
5750 REPX {paddd x, m5}, m1, m9, m8, m10
5751 REPX {psrad x, xm6}, m1, m9, m8, m10
5752 packssdw m1, m8 ; 0 2
5753 packssdw m9, m10 ; 1 3
5754 movu m0, [srcq+r4 ]
5755 movu m8, [srcq+r6 ]
5756 lea srcq, [srcq+ssq*2]
5757 REPX {pshufb x, m12}, m2, m3
5758 REPX {pmaddwd x, m13}, m2, m3
5759 REPX {pshufb x, m14}, m0, m8
5760 REPX {pmaddwd x, m15}, m0, m8
5761 phaddd m2, m0
5762 phaddd m3, m8
5763 shr myd, 6
5764 mov r9d, 64 << 24
5765 lea myd, [t1+myq]
5766 cmovnz r9q, [base+subpel_filters+myq*8]
5767 REPX {paddd x, m5}, m2, m3
5768 REPX {psrad x, xm6}, m2, m3
5769 packssdw m2, m3 ; 4 5
5770 pshufd m3, m2, q1032 ; 5 _
5771 punpcklwd m0, m1, m9 ; 01
5772 punpckhwd m1, m9 ; 23
5773 punpcklwd m2, m3 ; 45
5774 movq m10, r9
5775 %define hrnd_mem [rsp+0x10]
5776 %define hsh_mem [rsp+0x20]
5777 %define vsh_mem [rsp+0x28]
5778 %if isput
5779 %define vrnd_mem [rsp+0x30]
5780 %else
5781 %define vrnd_mem [base+pd_m524256]
5782 %endif
5783 %else
5784 mova [stk+0x20], m12
5785 mova [stk+0x30], m14
5786 add r4, srcq
5787 MC_4TAP_SCALED_H 0x60 ; 0 1
5788 MC_4TAP_SCALED_H 0x70 ; 2 3
5789 MC_4TAP_SCALED_H 0x80 ; 4 5
5790 mov [stk+0xe0], r4
5791 mova m3, [base+spel_s_shuf8]
5792 mova m0, [stk+0x60]
5793 mova m1, [stk+0x70]
5794 mova m2, [stk+0x80]
5795 mov myd, mym
5796 mov rX, [esp+0x1f4]
5797 xor r5, r5
5798 shr myd, 6
5799 lea rX, [rX+myd]
5800 mov r4, 64 << 24
5801 cmovnz r4, [base+subpel_filters+rX*8+0]
5802 cmovnz r5, [base+subpel_filters+rX*8+4]
5803 mov r3, r3m
5804 pshufb m0, m3 ; 01
5805 pshufb m1, m3 ; 23
5806 pshufb m2, m3 ; 45
5807 movd m7, r4
5808 movd m4, r5
5809 mov r5, r0m
5810 %if isput
5811 mov r1, r1m
5812 %endif
5813 mov r4, [stk+0xe0]
5814 %define dstq r5
5815 %define tmpq r5
5816 %define m12 [stk+0x20]
5817 %define m14 [stk+0x30]
5818 %define m13 [stk+0x40]
5819 %define m15 [stk+0x50]
5820 %define hrnd_mem [esp+0x00]
5821 %define hsh_mem [esp+0x10]
5822 %define vsh_mem [esp+0x18]
5823 %if isput
5824 %define vrnd_mem [esp+0x20]
5825 %else
5826 %define vrnd_mem [base+pd_m524256]
5827 %endif
5828 %define m10 m7
5829 punpckldq m10, m4
5830 %endif
5831 punpcklbw m10, m10
5832 psraw m10, 8
5833 pshufd m3, m10, q0000
5834 pshufd m4, m10, q1111
5835 pshufd m5, m10, q2222
5836 pshufd m10, m10, q3333
5837 %if ARCH_X86_32
5838 %xdefine m8 m3
5839 %xdefine m9 m6
5840 %xdefine m11 m5
5841 %xdefine m6 m4
5842 mova [stk+0x100], m3
5843 mova [stk+0x110], m4
5844 mova [stk+0x120], m5
5845 mova [stk+0x130], m10
5846 %define m3 [stk+0x100]
5847 %define m4 [stk+0x110]
5848 %define m5 [stk+0x120]
5849 %define m10 [stk+0x130]
5850 %endif
5851 .dy2_w4_loop:
5852 pmaddwd m8, m0, m3
5853 pmaddwd m9, m1, m3
5854 mova m0, m2
5855 pmaddwd m1, m4
5856 pmaddwd m11, m2, m4
5857 paddd m8, vrnd_mem
5858 paddd m9, vrnd_mem
5859 pmaddwd m2, m5
5860 paddd m8, m1
5861 paddd m9, m11
5862 paddd m8, m2
5863 movu m6, [srcq+ssq*0]
5864 movu m1, [srcq+ssq*2]
5865 %if ARCH_X86_64
5866 movu m11, [srcq+r4 ]
5867 movu m2, [srcq+r11]
5868 %else
5869 movu m11, [r4+ssq*0]
5870 movu m2, [r4+ssq*2]
5871 %endif
5872 pshufb m6, m12
5873 pshufb m1, m12
5874 pmaddwd m6, m13
5875 pmaddwd m1, m13
5876 pshufb m11, m14
5877 pshufb m2, m14
5878 pmaddwd m11, m15
5879 pmaddwd m2, m15
5880 phaddd m6, m11
5881 phaddd m1, m2
5882 paddd m6, hrnd_mem
5883 paddd m1, hrnd_mem
5884 psrad m6, hsh_mem
5885 psrad m1, hsh_mem
5886 movu m7, [srcq+ssq*1]
5887 movu m11, [srcq+ss3q ]
5888 packssdw m6, m1 ; 6 8
5889 %if ARCH_X86_64
5890 movu m2, [srcq+r6 ]
5891 movu m1, [srcq+r13]
5892 %else
5893 movu m2, [r4+ssq*1]
5894 movu m1, [r4+ss3q ]
5895 %endif
5896 pshufb m7, m12
5897 pshufb m11, m12
5898 pmaddwd m7, m13
5899 pmaddwd m11, m13
5900 pshufb m2, m14
5901 pshufb m1, m14
5902 pmaddwd m2, m15
5903 pmaddwd m1, m15
5904 phaddd m7, m2
5905 phaddd m11, m1
5906 paddd m7, hrnd_mem
5907 paddd m11, hrnd_mem
5908 psrad m7, hsh_mem
5909 psrad m11, hsh_mem
5910 packssdw m7, m11 ; 7 9
5911 %if ARCH_X86_32
5912 lea r4, [r4+ssq*4]
5913 %endif
5914 lea srcq, [srcq+ssq*4]
5915 punpcklwd m1, m6, m7 ; 67
5916 punpckhwd m6, m7 ; 89
5917 mova m2, m6
5918 pmaddwd m11, m1, m5
5919 pmaddwd m7, m1, m10
5920 pmaddwd m6, m10
5921 paddd m9, m11
5922 %if isput
5923 movd m11, vsh_mem
5924 %endif
5925 paddd m8, m7
5926 paddd m9, m6
5927 %if isput
5928 psrad m8, m11
5929 psrad m9, m11
5930 packssdw m8, m9
5931 pxor m7, m7
5932 pmaxsw m8, m7
5933 pminsw m8, pxmaxm
5934 movq [dstq+dsq*0], m8
5935 movhps [dstq+dsq*1], m8
5936 lea dstq, [dstq+dsq*2]
5937 %else
5938 psrad m8, 6
5939 psrad m9, 6
5940 packssdw m8, m9
5941 mova [tmpq], m8
5942 add tmpq, 16
5943 %endif
5944 sub hd, 2
5945 jg .dy2_w4_loop
5946 MC_8TAP_SCALED_RET ; why not jz .ret?
5947 INIT_XMM ssse3
5948 .dy2_w8:
5949 mov dword [stk+0xf0], 1
5950 movifprep tmp_stridem, 16
5951 jmp .dy2_w_start
5952 .dy2_w16:
5953 mov dword [stk+0xf0], 2
5954 movifprep tmp_stridem, 32
5955 jmp .dy2_w_start
5956 .dy2_w32:
5957 mov dword [stk+0xf0], 4
5958 movifprep tmp_stridem, 64
5959 jmp .dy2_w_start
5960 .dy2_w64:
5961 mov dword [stk+0xf0], 8
5962 movifprep tmp_stridem, 128
5963 jmp .dy2_w_start
5964 .dy2_w128:
5965 mov dword [stk+0xf0], 16
5966 movifprep tmp_stridem, 256
5967 .dy2_w_start:
5968 mov myd, mym
5969 %if ARCH_X86_64
5970 %ifidn %1, put
5971 movifnidn dsm, dsq
5972 %endif
5973 mova [rsp+0x10], m11
5974 mova [rsp+0x20], m12
5975 %define hround m11
5976 %if isput
5977 mova [rsp+0x30], m13
5978 %else
5979 mova m13, [base+pd_m524256]
5980 %endif
5981 shr t0d, 16
5982 shr myd, 6
5983 mov r4d, 64 << 24
5984 lea myd, [t1+myq]
5985 cmovnz r4q, [base+subpel_filters+myq*8]
5986 movd m15, t0d
5987 %else
5988 %define hround [esp+0x00]
5989 %define m12 [esp+0x10]
5990 %define m10 [base+pd_0x3ff]
5991 %define m8 m0
5992 %xdefine m14 m4
5993 %xdefine m15 m3
5994 %if isput
5995 %define dstq r0
5996 %else
5997 %define tmpq r0
5998 %define ssq ssm
5999 %endif
6000 mov r5, [esp+0x1f0]
6001 mov r3, [esp+0x1f4]
6002 shr r5, 16
6003 movd m15, r5
6004 xor r5, r5
6005 shr myd, 6
6006 lea r3, [r3+myd]
6007 mov r4, 64 << 24
6008 cmovnz r4, [base+subpel_filters+r3*8+0]
6009 cmovnz r5, [base+subpel_filters+r3*8+4]
6010 mov r0, r0m
6011 mov r3, r3m
6012 %endif
6013 sub srcq, 6
6014 pslld m7, m8, 2 ; dx*4
6015 pmaddwd m8, [base+rescale_mul] ; dx*[0-3]
6016 pshufd m15, m15, q0000
6017 paddd m14, m8 ; mx+dx*[0-3]
6018 %if ARCH_X86_64
6019 movq m3, r4q
6020 %else
6021 movd m5, r4
6022 movd m6, r5
6023 punpckldq m5, m6
6024 SWAP m3, m5
6025 %endif
6026 punpcklbw m3, m3
6027 psraw m3, 8
6028 mova [stk+0x100], m7
6029 mova [stk+0x120], m15
6030 mov [stk+0x0f8], srcq
6031 mov [stk+0x130], r0q ; dstq / tmpq
6032 pshufd m0, m3, q0000
6033 pshufd m1, m3, q1111
6034 pshufd m2, m3, q2222
6035 pshufd m3, m3, q3333
6036 %if ARCH_X86_64
6037 mova [stk+0x140], m0
6038 mova [stk+0x150], m1
6039 mova [stk+0x160], m2
6040 mova [stk+0x170], m3
6041 %if UNIX64
6042 mov hm, hd
6043 %endif
6044 %else
6045 mova [stk+0x180], m0
6046 mova [stk+0x190], m1
6047 mova [stk+0x1a0], m2
6048 mova [stk+0x1b0], m3
6049 SWAP m5, m3
6050 mov r5, hm
6051 mov [stk+0x134], r5
6052 %endif
6053 jmp .dy2_hloop
6054 .dy2_hloop_prep:
6055 dec dword [stk+0x0f0]
6056 jz .ret
6057 %if ARCH_X86_64
6058 add qword [stk+0x130], 16
6059 mov hd, hm
6060 %else
6061 add dword [stk+0x130], 16
6062 mov r5, [stk+0x134]
6063 mov r0, [stk+0x130]
6064 %endif
6065 mova m7, [stk+0x100]
6066 mova m14, [stk+0x110]
6067 %if ARCH_X86_64
6068 mova m10, [base+pd_0x3ff]
6069 mova m11, [rsp+0x10]
6070 %endif
6071 mova m15, [stk+0x120]
6072 mov srcq, [stk+0x0f8]
6073 %if ARCH_X86_64
6074 mov r0q, [stk+0x130] ; dstq / tmpq
6075 %else
6076 mov hm, r5
6077 mov r0m, r0
6078 mov r3, r3m
6079 %endif
6080 paddd m14, m7
6081 .dy2_hloop:
6082 %if ARCH_X86_64
6083 mova m9, [base+pq_0x40000000]
6084 %else
6085 %define m9 [base+pq_0x40000000]
6086 %endif
6087 pxor m1, m1
6088 psrld m2, m14, 10
6089 mova [stk], m2
6090 pand m6, m14, m10
6091 psrld m6, 6
6092 paddd m5, m15, m6
6093 pcmpeqd m6, m1
6094 pshufd m2, m5, q1032
6095 %if ARCH_X86_64
6096 movd r4d, m5
6097 movd r6d, m2
6098 pshufd m5, m5, q0321
6099 pshufd m2, m2, q0321
6100 movd r7d, m5
6101 movd r9d, m2
6102 movq m0, [base+subpel_filters+r4*8]
6103 movq m1, [base+subpel_filters+r6*8]
6104 movhps m0, [base+subpel_filters+r7*8]
6105 movhps m1, [base+subpel_filters+r9*8]
6106 %else
6107 movd r0, m5
6108 movd rX, m2
6109 pshufd m5, m5, q0321
6110 pshufd m2, m2, q0321
6111 movd r4, m5
6112 movd r5, m2
6113 movq m0, [base+subpel_filters+r0*8]
6114 movq m1, [base+subpel_filters+rX*8]
6115 movhps m0, [base+subpel_filters+r4*8]
6116 movhps m1, [base+subpel_filters+r5*8]
6117 %endif
6118 paddd m14, m7 ; mx+dx*[4-7]
6119 pand m5, m14, m10
6120 psrld m5, 6
6121 paddd m15, m5
6122 pxor m2, m2
6123 pcmpeqd m5, m2
6124 mova [stk+0x110], m14
6125 pshufd m4, m15, q1032
6126 %if ARCH_X86_64
6127 movd r10d, m15
6128 movd r11d, m4
6129 pshufd m15, m15, q0321
6130 pshufd m4, m4, q0321
6131 movd r13d, m15
6132 movd rXd, m4
6133 movq m2, [base+subpel_filters+r10*8]
6134 movq m3, [base+subpel_filters+r11*8]
6135 movhps m2, [base+subpel_filters+r13*8]
6136 movhps m3, [base+subpel_filters+ rX*8]
6137 psrld m14, 10
6138 movq r11, m14
6139 punpckhqdq m14, m14
6140 movq rX, m14
6141 mov r10d, r11d
6142 shr r11, 32
6143 mov r13d, rXd
6144 shr rX, 32
6145 mov r4d, [stk+ 0]
6146 mov r6d, [stk+ 4]
6147 mov r7d, [stk+ 8]
6148 mov r9d, [stk+12]
6149 pshufd m4, m6, q1100
6150 pshufd m6, m6, q3322
6151 pshufd m14, m5, q1100
6152 pshufd m5, m5, q3322
6153 pand m7, m9, m4
6154 pand m8, m9, m6
6155 pand m15, m9, m14
6156 pand m9, m9, m5
6157 pandn m4, m0
6158 pandn m6, m1
6159 pandn m14, m2
6160 pandn m5, m3
6161 por m7, m4
6162 por m8, m6
6163 por m15, m14
6164 por m9, m5
6165 punpcklbw m0, m7, m7
6166 punpckhbw m7, m7
6167 punpcklbw m1, m8, m8
6168 punpckhbw m8, m8
6169 psraw m0, 8
6170 psraw m7, 8
6171 psraw m1, 8
6172 psraw m8, 8
6173 punpcklbw m2, m15, m15
6174 punpckhbw m15, m15
6175 punpcklbw m3, m9, m9
6176 punpckhbw m9, m9
6177 psraw m2, 8
6178 psraw m15, 8
6179 psraw m3, 8
6180 psraw m9, 8
6181 mova [stk+0x10], m0
6182 mova [stk+0x20], m7
6183 mova [stk+0x30], m1
6184 mova [stk+0x40], m8
6185 mova [stk+0x50], m2
6186 mova [stk+0x60], m15
6187 mova [stk+0x70], m3
6188 mova [stk+0x80], m9
6189 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10 ; 0
6190 mova [stk+0x90], m1
6191 MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 1, 9, 10 ; 1
6192 mova [stk+0xa0], m2
6193 MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10 ; 2
6194 mova [stk+0xb0], m3
6195 MC_8TAP_SCALED_H 4, 5, 6, 1, 2, 3, 9, 10 ; 3
6196 mova [stk+0xc0], m4
6197 MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10 ; 4
6198 mova [stk+0xd0], m5
6199 MC_8TAP_SCALED_H 6, 1, 2, 3, 4, 5, 9, 10 ; 5
6200 MC_8TAP_SCALED_H 7, 1, 2, 3, 4, 5, 9, 10 ; 6
6201 MC_8TAP_SCALED_H 8, 1, 2, 3, 4, 5, 9, 10 ; 7
6202 mova m5, [stk+0xd0]
6203 mova m1, [stk+0x90]
6204 mova m2, [stk+0xa0]
6205 mova m3, [stk+0xb0]
6206 mova m9, [stk+0xc0]
6207 punpcklwd m4, m5, m6 ; 45a
6208 punpckhwd m5, m6 ; 45b
6209 punpcklwd m6, m7, m8 ; 67a
6210 punpckhwd m7, m8 ; 67b
6211 punpcklwd m0, m1, m2 ; 01a
6212 punpckhwd m1, m2 ; 01b
6213 punpcklwd m2, m3, m9 ; 23a
6214 punpckhwd m3, m9 ; 23b
6215 mova m10, [stk+0x140]
6216 mova m11, [stk+0x150]
6217 mova m14, [stk+0x160]
6218 mova m15, [stk+0x170]
6219 mova [stk+0x90], m4
6220 mova [stk+0xa0], m5
6221 mova [stk+0xb0], m6
6222 mova [stk+0xc0], m7
6223 %define hround [rsp+0x10]
6224 %define shift [rsp+0x20]
6225 %if isput
6226 %define vround [rsp+0x30]
6227 %else
6228 %define vround [base+pd_m524256]
6229 %endif
6230 .dy2_vloop:
6231 pmaddwd m4, m0, m10
6232 pmaddwd m5, m1, m10
6233 pmaddwd m6, m2, m11
6234 pmaddwd m7, m3, m11
6235 paddd m4, m13
6236 paddd m5, m13
6237 paddd m4, m6
6238 paddd m5, m7
6239 pmaddwd m6, [stk+0x90], m14
6240 pmaddwd m7, [stk+0xa0], m14
6241 pmaddwd m8, [stk+0xb0], m15
6242 pmaddwd m9, [stk+0xc0], m15
6243 paddd m4, m6
6244 paddd m5, m7
6245 %if isput
6246 pshufd m6, m12, q1032
6247 %endif
6248 paddd m4, m8
6249 paddd m5, m9
6250 %else
6251 movd r0, m15
6252 movd rX, m4
6253 pshufd m15, m15, q0321
6254 pshufd m4, m4, q0321
6255 movd r4, m15
6256 movd r5, m4
6257 mova m14, [stk+0x110]
6258 movq m2, [base+subpel_filters+r0*8]
6259 movq m3, [base+subpel_filters+rX*8]
6260 movhps m2, [base+subpel_filters+r4*8]
6261 movhps m3, [base+subpel_filters+r5*8]
6262 psrld m14, 10
6263 mova [stk+16], m14
6264 mov r0, [stk+ 0]
6265 mov rX, [stk+ 4]
6266 mov r4, [stk+ 8]
6267 mov r5, [stk+12]
6268 mova [stk+0x20], m0
6269 mova [stk+0x30], m1
6270 mova [stk+0x40], m2
6271 mova [stk+0x50], m3
6272 pshufd m4, m6, q1100
6273 pshufd m6, m6, q3322
6274 pshufd m7, m5, q1100
6275 pshufd m5, m5, q3322
6276 pand m0, m9, m4
6277 pand m1, m9, m6
6278 pand m2, m9, m7
6279 pand m3, m9, m5
6280 pandn m4, [stk+0x20]
6281 pandn m6, [stk+0x30]
6282 pandn m7, [stk+0x40]
6283 pandn m5, [stk+0x50]
6284 por m0, m4
6285 por m1, m6
6286 por m2, m7
6287 por m3, m5
6288 punpcklbw m4, m0, m0
6289 punpckhbw m0, m0
6290 punpcklbw m5, m1, m1
6291 punpckhbw m1, m1
6292 psraw m4, 8
6293 psraw m0, 8
6294 psraw m5, 8
6295 psraw m1, 8
6296 punpcklbw m6, m2, m2
6297 punpckhbw m2, m2
6298 punpcklbw m7, m3, m3
6299 punpckhbw m3, m3
6300 psraw m6, 8
6301 psraw m2, 8
6302 psraw m7, 8
6303 psraw m3, 8
6304 mova [stk+0x0a0], m4
6305 mova [stk+0x0b0], m0
6306 mova [stk+0x0c0], m5
6307 mova [stk+0x0d0], m1
6308 mova [stk+0x140], m6
6309 mova [stk+0x150], m2
6310 mova [stk+0x160], m7
6311 mova [stk+0x170], m3
6312 MC_8TAP_SCALED_H 0xa0, 0x20, 0 ; 0
6313 MC_8TAP_SCALED_H 0xa0, 0x30 ; 1
6314 MC_8TAP_SCALED_H 0xa0, 0x40 ; 2
6315 MC_8TAP_SCALED_H 0xa0, 0x50 ; 3
6316 MC_8TAP_SCALED_H 0xa0, 0x60 ; 4
6317 MC_8TAP_SCALED_H 0xa0, 0x70 ; 5
6318 MC_8TAP_SCALED_H 0xa0, 0x80 ; 6
6319 MC_8TAP_SCALED_H 0xa0, 0x90 ; 7
6320 mova m5, [stk+0x60]
6321 mova m6, [stk+0x70]
6322 mova m7, [stk+0x80]
6323 mova m0, [stk+0x90]
6324 mov r0, r0m
6325 punpcklwd m4, m5, m6 ; 45a
6326 punpckhwd m5, m6 ; 45b
6327 punpcklwd m6, m7, m0 ; 67a
6328 punpckhwd m7, m0 ; 67b
6329 mova [stk+0x60], m4
6330 mova [stk+0x70], m5
6331 mova [stk+0x80], m6
6332 mova [stk+0x90], m7
6333 mova m1, [stk+0x20]
6334 mova m2, [stk+0x30]
6335 mova m3, [stk+0x40]
6336 mova m4, [stk+0x50]
6337 punpcklwd m0, m1, m2 ; 01a
6338 punpckhwd m1, m2 ; 01b
6339 punpcklwd m2, m3, m4 ; 23a
6340 punpckhwd m3, m4 ; 23b
6341 mova m4, [stk+0x180]
6342 mova m5, [stk+0x190]
6343 mova m6, [stk+0x1a0]
6344 mova m7, [stk+0x1b0]
6345 mova [stk+0x40], m2
6346 mova [stk+0x50], m3
6347 .dy2_vloop:
6348 pmaddwd m0, m4
6349 pmaddwd m1, m4
6350 pmaddwd m2, m5
6351 pmaddwd m3, m5
6352 paddd m0, m2
6353 paddd m1, m3
6354 pmaddwd m2, [stk+0x60], m6
6355 pmaddwd m3, [stk+0x70], m6
6356 pmaddwd m4, [stk+0x80], m7
6357 pmaddwd m5, [stk+0x90], m7
6358 %if isput
6359 movd m6, [esp+0x18]
6360 %endif
6361 paddd m0, m2
6362 paddd m1, m3
6363 paddd m0, vrnd_mem
6364 paddd m1, vrnd_mem
6365 paddd m4, m0
6366 paddd m5, m1
6367 %endif
6368 %ifidn %1, put
6369 psrad m4, m6
6370 psrad m5, m6
6371 packssdw m4, m5
6372 pxor m7, m7
6373 pmaxsw m4, m7
6374 pminsw m4, pxmaxm
6375 mova [dstq], m4
6376 add dstq, dsm
6377 %else
6378 psrad m4, 6
6379 psrad m5, 6
6380 packssdw m4, m5
6381 mova [tmpq], m4
6382 add tmpq, tmp_stridem
6383 %endif
6384 dec hd
6385 jz .dy2_hloop_prep
6386 %if ARCH_X86_64
6387 MC_8TAP_SCALED_H 4, 8, 5, 6, 7, 9, 0, 1
6388 mova [stk+0xd0], m4
6389 MC_8TAP_SCALED_H 8, 5, 6, 7, 9, 4, 0, 1
6390 mova m4, [stk+0xd0]
6391 mova m0, m2 ; 01a
6392 mova m1, m3 ; 01b
6393 mova m2, [stk+0x90] ; 23a
6394 mova m3, [stk+0xa0] ; 23b
6395 mova m5, [stk+0xb0] ; 45a
6396 mova m6, [stk+0xc0] ; 45b
6397 punpcklwd m7, m4, m8 ; 67a
6398 punpckhwd m4, m8 ; 67b
6399 mova [stk+0x90], m5
6400 mova [stk+0xa0], m6
6401 mova [stk+0xb0], m7
6402 mova [stk+0xc0], m4
6403 %else
6404 mov r0m, r0
6405 mov r3, r3m
6406 MC_8TAP_SCALED_H 0xa0, 0xe0 ; 8
6407 MC_8TAP_SCALED_H 0xa0, 0 ; 9
6408 mova m7, [stk+0xe0]
6409 mova m2, [stk+0x60] ; 23a
6410 mova m3, [stk+0x70] ; 23b
6411 mova m4, [stk+0x80] ; 45a
6412 mova m5, [stk+0x90] ; 45b
6413 punpcklwd m6, m7, m0 ; 67a
6414 punpckhwd m7, m0 ; 67b
6415 mova m0, [stk+0x40] ; 01a
6416 mova m1, [stk+0x50] ; 01b
6417 mova [stk+0x40], m2
6418 mova [stk+0x50], m3
6419 mova [stk+0x60], m4
6420 mova [stk+0x70], m5
6421 mova m4, [stk+0x180]
6422 mova m5, [stk+0x190]
6423 mova [stk+0x80], m6
6424 mova [stk+0x90], m7
6425 mova m6, [stk+0x1a0]
6426 mova m7, [stk+0x1b0]
6427 mov r0, r0m
6428 %endif
6429 jmp .dy2_vloop
6430 INIT_XMM ssse3
6431 .ret:
6432 MC_8TAP_SCALED_RET 0
6433 %if ARCH_X86_32 && !isprep && required_stack_alignment > STACK_ALIGNMENT
6434 %define r0m [rstk+stack_offset+ 4]
6435 %define r1m [rstk+stack_offset+ 8]
6436 %define r2m [rstk+stack_offset+12]
6437 %define r3m [rstk+stack_offset+16]
6438 %endif
6439 %undef isput
6440 %undef isprep
6441 %endmacro
6442
6443 %macro BILIN_SCALED_FN 1
6444 cglobal %1_bilin_scaled_16bpc
6445 mov t0d, (5*15 << 16) | 5*15
6446 mov t1d, (5*15 << 16) | 5*15
6447 jmp mangle(private_prefix %+ _%1_8tap_scaled_16bpc %+ SUFFIX)
6448 %endmacro
6449
6450 %if WIN64
6451 DECLARE_REG_TMP 6, 5
6452 %elif ARCH_X86_64
6453 DECLARE_REG_TMP 6, 8
6454 %else
6455 DECLARE_REG_TMP 1, 2
6456 %endif
6457 BILIN_SCALED_FN put
6458 FN put_8tap_scaled, sharp, SHARP, SHARP
6459 FN put_8tap_scaled, sharp_smooth, SHARP, SMOOTH
6460 FN put_8tap_scaled, smooth_sharp, SMOOTH, SHARP
6461 FN put_8tap_scaled, smooth, SMOOTH, SMOOTH
6462 FN put_8tap_scaled, sharp_regular, SHARP, REGULAR
6463 FN put_8tap_scaled, regular_sharp, REGULAR, SHARP
6464 FN put_8tap_scaled, smooth_regular, SMOOTH, REGULAR
6465 FN put_8tap_scaled, regular_smooth, REGULAR, SMOOTH
6466 FN put_8tap_scaled, regular, REGULAR, REGULAR
6467 MC_8TAP_SCALED put
6468
6469 %if WIN64
6470 DECLARE_REG_TMP 5, 4
6471 %elif ARCH_X86_64
6472 DECLARE_REG_TMP 6, 7
6473 %else
6474 DECLARE_REG_TMP 1, 2
6475 %endif
6476 BILIN_SCALED_FN prep
6477 FN prep_8tap_scaled, sharp, SHARP, SHARP
6478 FN prep_8tap_scaled, sharp_smooth, SHARP, SMOOTH
6479 FN prep_8tap_scaled, smooth_sharp, SMOOTH, SHARP
6480 FN prep_8tap_scaled, smooth, SMOOTH, SMOOTH
6481 FN prep_8tap_scaled, sharp_regular, SHARP, REGULAR
6482 FN prep_8tap_scaled, regular_sharp, REGULAR, SHARP
6483 FN prep_8tap_scaled, smooth_regular, SMOOTH, REGULAR
6484 FN prep_8tap_scaled, regular_smooth, REGULAR, SMOOTH
6485 FN prep_8tap_scaled, regular, REGULAR, REGULAR
6486 MC_8TAP_SCALED prep
6487
6488 %if ARCH_X86_64
6489 DECLARE_REG_TMP 6
6490 %else
6491 DECLARE_REG_TMP 2
6492 %endif
6493
6494 %if ARCH_X86_64
6495 ; warp8x8t spills one less xmm register than warp8x8 on WIN64, compensate that
6496 ; by allocating 16 bytes more stack space so that stack offsets match up.
6497 %if WIN64 && STACK_ALIGNMENT == 16
6498 %assign stksz 16*14
6499 %else
6500 %assign stksz 16*13
6501 %endif
6502 cglobal warp_affine_8x8t_16bpc, 4, 13, 9, stksz, dst, ds, src, ss, delta, \
6503 mx, tmp, alpha, beta, \
6504 filter, my, gamma, cnt
6505 %assign stack_size_padded_8x8t stack_size_padded
6506 %else
6507 cglobal warp_affine_8x8t_16bpc, 0, 7, 8, -16*17, alpha, gamma, src, tmp, \
6508 filter, mx, my
6509 %define m8 [esp+16*13]
6510 %define m9 [esp+16*14]
6511 %define cntd dword [esp+4*63]
6512 %define dstq tmpq
6513 %define dsq 0
6514 %if STACK_ALIGNMENT < 16
6515 %define dstm [esp+4*65]
6516 %define dsm [esp+4*66]
6517 %else
6518 %define dstm r0m
6519 %define dsm r1m
6520 %endif
6521 %endif
6522 %define base filterq-$$
6523 mov t0d, r7m
6524 LEA filterq, $$
6525 shr t0d, 11
6526 %if ARCH_X86_64
6527 movddup m8, [base+warp8x8t_rnd]
6528 %else
6529 movddup m1, [base+warp8x8t_rnd]
6530 mov r1, r1m
6531 add r1, r1
6532 mova m8, m1
6533 mov r1m, r1 ; ds *= 2
6534 %endif
6535 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main
6536 jmp .start
6537 .loop:
6538 %if ARCH_X86_64
6539 lea dstq, [dstq+dsq*4]
6540 %else
6541 add dstq, dsm
6542 mov dstm, dstq
6543 %endif
6544 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main2
6545 .start:
6546 %if ARCH_X86_32
6547 mov dstq, dstm
6548 %endif
6549 paddd m1, m8
6550 paddd m2, m8
6551 psrad m1, 15
6552 psrad m2, 15
6553 packssdw m1, m2
6554 mova [dstq+dsq*0], m1
6555 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main3
6556 %if ARCH_X86_32
6557 mov dstq, dstm
6558 add dstq, dsm
6559 %endif
6560 paddd m1, m8
6561 paddd m2, m8
6562 psrad m1, 15
6563 psrad m2, 15
6564 packssdw m1, m2
6565 mova [dstq+dsq*2], m1
6566 dec cntd
6567 jg .loop
6568 RET
6569
6570 %if ARCH_X86_64
6571 cglobal warp_affine_8x8_16bpc, 4, 13, 10, 16*13, dst, ds, src, ss, delta, \
6572 mx, tmp, alpha, beta, \
6573 filter, my, gamma, cnt
6574 ASSERT stack_size_padded == stack_size_padded_8x8t
6575 %else
6576 cglobal warp_affine_8x8_16bpc, 0, 7, 8, -16*17, alpha, gamma, src, tmp, \
6577 filter, mx, my
6578 %endif
6579 mov t0d, r7m
6580 LEA filterq, $$
6581 shr t0d, 11
6582 %if ARCH_X86_64
6583 movddup m8, [base+warp8x8_rnd2+t0*8]
6584 movd m9, r7m ; pixel_max
6585 pshufb m9, [base+pw_256]
6586 %else
6587 movddup m1, [base+warp8x8_rnd2+t0*8]
6588 movd m2, r7m ; pixel_max
6589 pshufb m2, [base+pw_256]
6590 mova m8, m1
6591 mova m9, m2
6592 %endif
6593 call .main
6594 jmp .start
6595 .loop:
6596 %if ARCH_X86_64
6597 lea dstq, [dstq+dsq*2]
6598 %else
6599 add dstq, dsm
6600 mov dstm, dstq
6601 %endif
6602 call .main2
6603 .start:
6604 %if ARCH_X86_32
6605 mov dstq, dstm
6606 %endif
6607 psrad m1, 16
6608 psrad m2, 16
6609 packssdw m1, m2
6610 pmaxsw m1, m6
6611 pmulhrsw m1, m8
6612 pminsw m1, m9
6613 mova [dstq+dsq*0], m1
6614 call .main3
6615 %if ARCH_X86_32
6616 mov dstq, dstm
6617 add dstq, dsm
6618 %endif
6619 psrad m1, 16
6620 psrad m2, 16
6621 packssdw m1, m2
6622 pmaxsw m1, m6
6623 pmulhrsw m1, m8
6624 pminsw m1, m9
6625 mova [dstq+dsq*1], m1
6626 dec cntd
6627 jg .loop
6628 RET
6629 ALIGN function_align
6630 .main:
6631 ; Stack args offset by one (r4m -> r5m etc.) due to call
6632 %if WIN64
6633 mov deltaq, r5m
6634 mov mxd, r6m
6635 %endif
6636 movd m0, [base+warp8x8_shift+t0*4]
6637 movddup m7, [base+warp8x8_rnd1+t0*8]
6638 add filterq, mc_warp_filter-$$
6639 %if ARCH_X86_64
6640 movsx alphad, word [deltaq+2*0]
6641 movsx betad, word [deltaq+2*1]
6642 movsx gammad, word [deltaq+2*2]
6643 movsx deltad, word [deltaq+2*3]
6644 lea tmpq, [ssq*3]
6645 add mxd, 512+(64<<10)
6646 sub srcq, tmpq ; src -= ss*3
6647 imul tmpd, alphad, -7
6648 mov myd, r7m
6649 add betad, tmpd ; beta -= alpha*7
6650 imul tmpd, gammad, -7
6651 add myd, 512+(64<<10)
6652 mov cntd, 4
6653 add deltad, tmpd ; delta -= gamma*7
6654 %else
6655 %if STACK_ALIGNMENT < 16
6656 %assign stack_offset stack_offset - gprsize
6657 %endif
6658 mov r3d, r5m ; abcd
6659 %if STACK_ALIGNMENT < 16
6660 mov r0, r1m ; dst
6661 mov r1, r2m ; ds
6662 mov [esp+gprsize+4*65], r0
6663 mov [esp+gprsize+4*66], r1
6664 %endif
6665 movsx alphad, word [r3+2*0]
6666 movsx r2d, word [r3+2*1]
6667 movsx gammad, word [r3+2*2]
6668 movsx r3d, word [r3+2*3]
6669 imul r5d, alphad, -7
6670 add r2d, r5d ; beta -= alpha*7
6671 imul r5d, gammad, -7
6672 mov [esp+gprsize+4*60], r2d
6673 add r3d, r5d ; delta -= gamma*7
6674 mov [esp+gprsize+4*61], r3d
6675 mov r3d, r4m ; ss
6676 mov srcq, r3m
6677 mov mxd, r6m
6678 mov myd, r7m
6679 mov dword [esp+gprsize+4*63], 4 ; cnt
6680 mov [esp+gprsize+4*62], r3
6681 lea r3, [r3*3]
6682 add mxd, 512+(64<<10)
6683 add myd, 512+(64<<10)
6684 sub srcq, r3 ; src -= ss*3
6685 %if STACK_ALIGNMENT < 16
6686 %assign stack_offset stack_offset + gprsize
6687 %endif
6688 %endif
6689 mova [rsp+gprsize], m0
6690 pxor m6, m6
6691 call .h
6692 mova m5, m0
6693 call .h
6694 punpcklwd m1, m5, m0 ; 01
6695 punpckhwd m5, m0
6696 mova [rsp+gprsize+16* 1], m1
6697 mova [rsp+gprsize+16* 4], m5
6698 mova m5, m0
6699 call .h
6700 punpcklwd m1, m5, m0 ; 12
6701 punpckhwd m5, m0
6702 mova [rsp+gprsize+16* 7], m1
6703 mova [rsp+gprsize+16*10], m5
6704 mova m5, m0
6705 call .h
6706 punpcklwd m1, m5, m0 ; 23
6707 punpckhwd m5, m0
6708 mova [rsp+gprsize+16* 2], m1
6709 mova [rsp+gprsize+16* 5], m5
6710 mova m5, m0
6711 call .h
6712 punpcklwd m1, m5, m0 ; 34
6713 punpckhwd m5, m0
6714 mova [rsp+gprsize+16* 8], m1
6715 mova [rsp+gprsize+16*11], m5
6716 mova m5, m0
6717 call .h
6718 punpcklwd m1, m5, m0 ; 45
6719 punpckhwd m5, m0
6720 mova [rsp+gprsize+16* 3], m1
6721 mova [rsp+gprsize+16* 6], m5
6722 mova m5, m0
6723 call .h
6724 punpcklwd m1, m5, m0 ; 56
6725 punpckhwd m5, m0
6726 mova [rsp+gprsize+16* 9], m1
6727 mova [rsp+gprsize+16*12], m5
6728 mova m5, m0
6729 .main2:
6730 call .h
6731 %macro WARP_V 6 ; 01l, 23l, 45l, 01h, 23h, 45h
6732 lea tmpd, [myq+gammaq]
6733 shr myd, 10
6734 movq m4, [filterq+myq*8] ; a
6735 lea myd, [tmpq+gammaq]
6736 shr tmpd, 10
6737 movq m2, [filterq+tmpq*8] ; b
6738 lea tmpd, [myq+gammaq]
6739 shr myd, 10
6740 movq m3, [filterq+myq*8] ; c
6741 lea myd, [tmpq+gammaq]
6742 shr tmpd, 10
6743 movq m1, [filterq+tmpq*8] ; d
6744 lea tmpd, [myq+gammaq]
6745 shr myd, 10
6746 punpcklwd m4, m2
6747 punpcklwd m3, m1
6748 punpckldq m2, m4, m3
6749 punpckhdq m4, m3
6750 punpcklbw m1, m6, m2 ; a0 a1 b0 b1 c0 c1 d0 d1 << 8
6751 pmaddwd m1, [rsp+gprsize+16*%1]
6752 punpckhbw m3, m6, m2 ; a2 a3 b2 b3 c2 c3 d2 d3 << 8
6753 mova m2, [rsp+gprsize+16*%2]
6754 pmaddwd m3, m2
6755 mova [rsp+gprsize+16*%1], m2
6756 paddd m1, m3
6757 punpcklbw m3, m6, m4 ; a4 a5 b4 b5 c4 c5 d4 d5 << 8
6758 mova m2, [rsp+gprsize+16*%3]
6759 pmaddwd m3, m2
6760 mova [rsp+gprsize+16*%2], m2
6761 paddd m1, m3
6762 punpcklwd m3, m5, m0 ; 67
6763 punpckhbw m2, m6, m4 ; a6 a7 b6 b7 c6 c7 d6 d7 << 8
6764 pmaddwd m2, m3
6765 mova [rsp+gprsize+16*%3], m3
6766 paddd m1, m2
6767 movq m4, [filterq+myq*8] ; e
6768 lea myd, [tmpq+gammaq]
6769 shr tmpd, 10
6770 movq m3, [filterq+tmpq*8] ; f
6771 lea tmpd, [myq+gammaq]
6772 shr myd, 10
6773 movq m2, [filterq+myq*8] ; g
6774 %if ARCH_X86_64
6775 lea myd, [tmpq+deltaq] ; my += delta
6776 %else
6777 mov myd, [esp+gprsize+4*61]
6778 add myd, tmpd
6779 %endif
6780 shr tmpd, 10
6781 punpcklwd m4, m3
6782 movq m3, [filterq+tmpq*8] ; h
6783 punpcklwd m2, m3
6784 punpckldq m3, m4, m2
6785 punpckhdq m4, m2
6786 punpcklbw m2, m6, m3 ; e0 e1 f0 f1 g0 g1 h0 h1 << 8
6787 pmaddwd m2, [rsp+gprsize+16*%4]
6788 punpckhbw m6, m3 ; e2 e3 f2 f3 g2 g3 h2 h3 << 8
6789 mova m3, [rsp+gprsize+16*%5]
6790 pmaddwd m6, m3
6791 mova [rsp+gprsize+16*%4], m3
6792 pxor m3, m3
6793 paddd m2, m6
6794 punpcklbw m3, m4 ; e4 e5 f4 f5 g4 g5 h4 h5 << 8
6795 mova m6, [rsp+gprsize+16*%6]
6796 pmaddwd m3, m6
6797 mova [rsp+gprsize+16*%5], m6
6798 punpckhwd m5, m0
6799 pxor m6, m6
6800 paddd m2, m3
6801 punpckhbw m3, m6, m4 ; e6 e7 f6 f7 g6 g7 h6 h7 << 8
6802 pmaddwd m3, m5
6803 mova [rsp+gprsize+16*%6], m5
6804 mova m5, m0
6805 paddd m2, m3
6806 %endmacro
6807 WARP_V 1, 2, 3, 4, 5, 6
6808 ret
6809 .main3:
6810 call .h
6811 WARP_V 7, 8, 9, 10, 11, 12
6812 ret
6813 ALIGN function_align
6814 .h:
6815 lea tmpd, [mxq+alphaq]
6816 shr mxd, 10
6817 movq m3, [filterq+mxq*8]
6818 punpcklbw m0, m6, m3
6819 movu m3, [srcq-6]
6820 pmaddwd m0, m3 ; 0
6821 lea mxd, [tmpq+alphaq]
6822 shr tmpd, 10
6823 movq m3, [filterq+tmpq*8]
6824 punpcklbw m2, m6, m3
6825 movu m3, [srcq-4]
6826 pmaddwd m2, m3 ; 1
6827 lea tmpd, [mxq+alphaq]
6828 shr mxd, 10
6829 movq m3, [filterq+mxq*8]
6830 phaddd m0, m2 ; 0 1
6831 punpcklbw m2, m6, m3
6832 movu m3, [srcq-2]
6833 pmaddwd m2, m3 ; 2
6834 lea mxd, [tmpq+alphaq]
6835 shr tmpd, 10
6836 movq m3, [filterq+tmpq*8]
6837 punpcklbw m1, m6, m3
6838 movu m3, [srcq+0]
6839 pmaddwd m1, m3 ; 3
6840 lea tmpd, [mxq+alphaq]
6841 shr mxd, 10
6842 movq m3, [filterq+mxq*8]
6843 phaddd m2, m1 ; 2 3
6844 punpcklbw m1, m6, m3
6845 movu m3, [srcq+2]
6846 pmaddwd m1, m3 ; 4
6847 lea mxd, [tmpq+alphaq]
6848 shr tmpd, 10
6849 movq m3, [filterq+tmpq*8]
6850 phaddd m0, m2 ; 0 1 2 3
6851 punpcklbw m2, m6, m3
6852 movu m3, [srcq+4]
6853 pmaddwd m2, m3 ; 5
6854 lea tmpd, [mxq+alphaq]
6855 shr mxd, 10
6856 movq m3, [filterq+mxq*8]
6857 phaddd m1, m2 ; 4 5
6858 punpcklbw m2, m6, m3
6859 movu m3, [srcq+6]
6860 pmaddwd m2, m3 ; 6
6861 %if ARCH_X86_64
6862 lea mxd, [tmpq+betaq] ; mx += beta
6863 %else
6864 mov mxd, [esp+gprsize*2+4*60]
6865 add mxd, tmpd
6866 %endif
6867 shr tmpd, 10
6868 movq m3, [filterq+tmpq*8]
6869 punpcklbw m4, m6, m3
6870 movu m3, [srcq+8]
6871 %if ARCH_X86_64
6872 add srcq, ssq
6873 %else
6874 add srcq, [esp+gprsize*2+4*62]
6875 %endif
6876 pmaddwd m3, m4 ; 7
6877 phaddd m2, m3 ; 6 7
6878 phaddd m1, m2 ; 4 5 6 7
6879 paddd m0, m7
6880 paddd m1, m7
6881 psrad m0, [rsp+gprsize*2]
6882 psrad m1, [rsp+gprsize*2]
6883 packssdw m0, m1
6884 ret
6885
6886 %macro BIDIR_FN 0
6887 call .main
6888 jmp wq
6889 .w4_loop:
6890 call .main
6891 lea dstq, [dstq+strideq*2]
6892 .w4:
6893 movq [dstq+strideq*0], m0
6894 movhps [dstq+strideq*1], m0
6895 lea dstq, [dstq+strideq*2]
6896 movq [dstq+strideq*0], m1
6897 movhps [dstq+strideq*1], m1
6898 sub hd, 4
6899 jg .w4_loop
6900 .ret:
6901 RET
6902 .w8_loop:
6903 call .main
6904 lea dstq, [dstq+strideq*2]
6905 .w8:
6906 mova [dstq+strideq*0], m0
6907 mova [dstq+strideq*1], m1
6908 sub hd, 2
6909 jne .w8_loop
6910 RET
6911 .w16_loop:
6912 call .main
6913 add dstq, strideq
6914 .w16:
6915 mova [dstq+16*0], m0
6916 mova [dstq+16*1], m1
6917 dec hd
6918 jg .w16_loop
6919 RET
6920 .w32_loop:
6921 call .main
6922 add dstq, strideq
6923 .w32:
6924 mova [dstq+16*0], m0
6925 mova [dstq+16*1], m1
6926 call .main
6927 mova [dstq+16*2], m0
6928 mova [dstq+16*3], m1
6929 dec hd
6930 jg .w32_loop
6931 RET
6932 .w64_loop:
6933 call .main
6934 add dstq, strideq
6935 .w64:
6936 mova [dstq+16*0], m0
6937 mova [dstq+16*1], m1
6938 call .main
6939 mova [dstq+16*2], m0
6940 mova [dstq+16*3], m1
6941 call .main
6942 mova [dstq+16*4], m0
6943 mova [dstq+16*5], m1
6944 call .main
6945 mova [dstq+16*6], m0
6946 mova [dstq+16*7], m1
6947 dec hd
6948 jg .w64_loop
6949 RET
6950 .w128_loop:
6951 call .main
6952 add dstq, strideq
6953 .w128:
6954 mova [dstq+16* 0], m0
6955 mova [dstq+16* 1], m1
6956 call .main
6957 mova [dstq+16* 2], m0
6958 mova [dstq+16* 3], m1
6959 call .main
6960 mova [dstq+16* 4], m0
6961 mova [dstq+16* 5], m1
6962 call .main
6963 mova [dstq+16* 6], m0
6964 mova [dstq+16* 7], m1
6965 call .main
6966 mova [dstq+16* 8], m0
6967 mova [dstq+16* 9], m1
6968 call .main
6969 mova [dstq+16*10], m0
6970 mova [dstq+16*11], m1
6971 call .main
6972 mova [dstq+16*12], m0
6973 mova [dstq+16*13], m1
6974 call .main
6975 mova [dstq+16*14], m0
6976 mova [dstq+16*15], m1
6977 dec hd
6978 jg .w128_loop
6979 RET
6980 %endmacro
6981
6982 %if UNIX64
6983 DECLARE_REG_TMP 7
6984 %else
6985 DECLARE_REG_TMP 5
6986 %endif
6987
6988 cglobal avg_16bpc, 4, 7, 4, dst, stride, tmp1, tmp2, w, h
6989 %define base r6-avg_ssse3_table
6990 LEA r6, avg_ssse3_table
6991 tzcnt wd, wm
6992 mov t0d, r6m ; pixel_max
6993 movsxd wq, [r6+wq*4]
6994 shr t0d, 11
6995 movddup m2, [base+bidir_rnd+t0*8]
6996 movddup m3, [base+bidir_mul+t0*8]
6997 movifnidn hd, hm
6998 add wq, r6
6999 BIDIR_FN
7000 ALIGN function_align
7001 .main:
7002 mova m0, [tmp1q+16*0]
7003 paddsw m0, [tmp2q+16*0]
7004 mova m1, [tmp1q+16*1]
7005 paddsw m1, [tmp2q+16*1]
7006 add tmp1q, 16*2
7007 add tmp2q, 16*2
7008 pmaxsw m0, m2
7009 pmaxsw m1, m2
7010 psubsw m0, m2
7011 psubsw m1, m2
7012 pmulhw m0, m3
7013 pmulhw m1, m3
7014 ret
7015
7016 cglobal w_avg_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, h
7017 %define base r6-w_avg_ssse3_table
7018 LEA r6, w_avg_ssse3_table
7019 tzcnt wd, wm
7020 mov t0d, r6m ; weight
7021 movd m6, r7m ; pixel_max
7022 movddup m5, [base+pd_65538]
7023 movsxd wq, [r6+wq*4]
7024 pshufb m6, [base+pw_256]
7025 add wq, r6
7026 lea r6d, [t0-16]
7027 shl t0d, 16
7028 sub t0d, r6d ; 16-weight, weight
7029 paddw m5, m6
7030 mov r6d, t0d
7031 shl t0d, 2
7032 test dword r7m, 0x800
7033 cmovnz r6d, t0d
7034 movifnidn hd, hm
7035 movd m4, r6d
7036 pslld m5, 7
7037 pxor m7, m7
7038 pshufd m4, m4, q0000
7039 BIDIR_FN
7040 ALIGN function_align
7041 .main:
7042 mova m2, [tmp1q+16*0]
7043 mova m0, [tmp2q+16*0]
7044 punpckhwd m3, m0, m2
7045 punpcklwd m0, m2
7046 mova m2, [tmp1q+16*1]
7047 mova m1, [tmp2q+16*1]
7048 add tmp1q, 16*2
7049 add tmp2q, 16*2
7050 pmaddwd m3, m4
7051 pmaddwd m0, m4
7052 paddd m3, m5
7053 paddd m0, m5
7054 psrad m3, 8
7055 psrad m0, 8
7056 packssdw m0, m3
7057 punpckhwd m3, m1, m2
7058 punpcklwd m1, m2
7059 pmaddwd m3, m4
7060 pmaddwd m1, m4
7061 paddd m3, m5
7062 paddd m1, m5
7063 psrad m3, 8
7064 psrad m1, 8
7065 packssdw m1, m3
7066 pminsw m0, m6
7067 pminsw m1, m6
7068 pmaxsw m0, m7
7069 pmaxsw m1, m7
7070 ret
7071
7072 %if ARCH_X86_64
7073 cglobal mask_16bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask
7074 %else
7075 cglobal mask_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, mask
7076 %define hd dword r5m
7077 %define m8 [base+pw_64]
7078 %endif
7079 %define base r6-mask_ssse3_table
7080 LEA r6, mask_ssse3_table
7081 tzcnt wd, wm
7082 mov t0d, r7m ; pixel_max
7083 shr t0d, 11
7084 movsxd wq, [r6+wq*4]
7085 movddup m6, [base+bidir_rnd+t0*8]
7086 movddup m7, [base+bidir_mul+t0*8]
7087 %if ARCH_X86_64
7088 mova m8, [base+pw_64]
7089 movifnidn hd, hm
7090 %endif
7091 add wq, r6
7092 mov maskq, r6mp
7093 BIDIR_FN
7094 ALIGN function_align
7095 .main:
7096 movq m3, [maskq+8*0]
7097 mova m0, [tmp1q+16*0]
7098 mova m4, [tmp2q+16*0]
7099 pxor m5, m5
7100 punpcklbw m3, m5
7101 punpckhwd m2, m0, m4
7102 punpcklwd m0, m4
7103 psubw m1, m8, m3
7104 punpckhwd m4, m3, m1 ; m, 64-m
7105 punpcklwd m3, m1
7106 pmaddwd m2, m4 ; tmp1 * m + tmp2 * (64-m)
7107 pmaddwd m0, m3
7108 movq m3, [maskq+8*1]
7109 mova m1, [tmp1q+16*1]
7110 mova m4, [tmp2q+16*1]
7111 add maskq, 8*2
7112 add tmp1q, 16*2
7113 add tmp2q, 16*2
7114 psrad m2, 5
7115 psrad m0, 5
7116 packssdw m0, m2
7117 punpcklbw m3, m5
7118 punpckhwd m2, m1, m4
7119 punpcklwd m1, m4
7120 psubw m5, m8, m3
7121 punpckhwd m4, m3, m5 ; m, 64-m
7122 punpcklwd m3, m5
7123 pmaddwd m2, m4 ; tmp1 * m + tmp2 * (64-m)
7124 pmaddwd m1, m3
7125 psrad m2, 5
7126 psrad m1, 5
7127 packssdw m1, m2
7128 pmaxsw m0, m6
7129 pmaxsw m1, m6
7130 psubsw m0, m6
7131 psubsw m1, m6
7132 pmulhw m0, m7
7133 pmulhw m1, m7
7134 ret
7135
7136 cglobal w_mask_420_16bpc, 4, 7, 12, dst, stride, tmp1, tmp2, w, h, mask
7137 %define base t0-w_mask_420_ssse3_table
7138 LEA t0, w_mask_420_ssse3_table
7139 tzcnt wd, wm
7140 mov r6d, r8m ; pixel_max
7141 movd m0, r7m ; sign
7142 shr r6d, 11
7143 movsxd wq, [t0+wq*4]
7144 %if ARCH_X86_64
7145 mova m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
7146 mova m9, [base+pw_64]
7147 movddup m10, [base+bidir_rnd+r6*8]
7148 movddup m11, [base+bidir_mul+r6*8]
7149 %else
7150 mova m1, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
7151 mova m2, [base+pw_64]
7152 movddup m3, [base+bidir_rnd+r6*8]
7153 movddup m4, [base+bidir_mul+r6*8]
7154 ALLOC_STACK -16*4
7155 mova [rsp+16*0], m1
7156 mova [rsp+16*1], m2
7157 mova [rsp+16*2], m3
7158 mova [rsp+16*3], m4
7159 %define m8 [rsp+gprsize+16*0]
7160 %define m9 [rsp+gprsize+16*1]
7161 %define m10 [rsp+gprsize+16*2]
7162 %define m11 [rsp+gprsize+16*3]
7163 %endif
7164 movd m7, [base+pw_2]
7165 psubw m7, m0
7166 pshufb m7, [base+pw_256]
7167 add wq, t0
7168 movifnidn hd, r5m
7169 mov maskq, r6mp
7170 call .main
7171 jmp wq
7172 .w4_loop:
7173 call .main
7174 lea dstq, [dstq+strideq*2]
7175 add maskq, 4
7176 .w4:
7177 movq [dstq+strideq*0], m0
7178 phaddw m2, m3
7179 movhps [dstq+strideq*1], m0
7180 phaddd m2, m2
7181 lea dstq, [dstq+strideq*2]
7182 paddw m2, m7
7183 movq [dstq+strideq*0], m1
7184 psrlw m2, 2
7185 movhps [dstq+strideq*1], m1
7186 packuswb m2, m2
7187 movd [maskq], m2
7188 sub hd, 4
7189 jg .w4_loop
7190 RET
7191 .w8_loop:
7192 call .main
7193 lea dstq, [dstq+strideq*2]
7194 add maskq, 4
7195 .w8:
7196 mova [dstq+strideq*0], m0
7197 paddw m2, m3
7198 phaddw m2, m2
7199 mova [dstq+strideq*1], m1
7200 paddw m2, m7
7201 psrlw m2, 2
7202 packuswb m2, m2
7203 movd [maskq], m2
7204 sub hd, 2
7205 jg .w8_loop
7206 RET
7207 .w16_loop:
7208 call .main
7209 lea dstq, [dstq+strideq*2]
7210 add maskq, 8
7211 .w16:
7212 mova [dstq+strideq*1+16*0], m2
7213 mova [dstq+strideq*0+16*0], m0
7214 mova [dstq+strideq*1+16*1], m3
7215 mova [dstq+strideq*0+16*1], m1
7216 call .main
7217 paddw m2, [dstq+strideq*1+16*0]
7218 paddw m3, [dstq+strideq*1+16*1]
7219 mova [dstq+strideq*1+16*0], m0
7220 phaddw m2, m3
7221 mova [dstq+strideq*1+16*1], m1
7222 paddw m2, m7
7223 psrlw m2, 2
7224 packuswb m2, m2
7225 movq [maskq], m2
7226 sub hd, 2
7227 jg .w16_loop
7228 RET
7229 .w32_loop:
7230 call .main
7231 lea dstq, [dstq+strideq*2]
7232 add maskq, 16
7233 .w32:
7234 mova [dstq+strideq*1+16*0], m2
7235 mova [dstq+strideq*0+16*0], m0
7236 mova [dstq+strideq*1+16*1], m3
7237 mova [dstq+strideq*0+16*1], m1
7238 call .main
7239 mova [dstq+strideq*0+16*2], m0
7240 phaddw m2, m3
7241 mova [dstq+strideq*1+16*3], m2
7242 mova [dstq+strideq*0+16*3], m1
7243 call .main
7244 paddw m2, [dstq+strideq*1+16*0]
7245 paddw m3, [dstq+strideq*1+16*1]
7246 mova [dstq+strideq*1+16*0], m0
7247 phaddw m2, m3
7248 mova [dstq+strideq*1+16*2], m2
7249 mova [dstq+strideq*1+16*1], m1
7250 call .main
7251 phaddw m2, m3
7252 paddw m3, m7, [dstq+strideq*1+16*2]
7253 paddw m2, [dstq+strideq*1+16*3]
7254 mova [dstq+strideq*1+16*2], m0
7255 paddw m2, m7
7256 psrlw m3, 2
7257 psrlw m2, 2
7258 mova [dstq+strideq*1+16*3], m1
7259 packuswb m3, m2
7260 mova [maskq], m3
7261 sub hd, 2
7262 jg .w32_loop
7263 RET
7264 .w64_loop:
7265 call .main
7266 lea dstq, [dstq+strideq*2]
7267 add maskq, 16*2
7268 .w64:
7269 mova [dstq+strideq*1+16*1], m2
7270 mova [dstq+strideq*0+16*0], m0
7271 mova [dstq+strideq*1+16*2], m3
7272 mova [dstq+strideq*0+16*1], m1
7273 call .main
7274 mova [dstq+strideq*1+16*3], m2
7275 mova [dstq+strideq*0+16*2], m0
7276 mova [dstq+strideq*1+16*4], m3
7277 mova [dstq+strideq*0+16*3], m1
7278 call .main
7279 mova [dstq+strideq*1+16*5], m2
7280 mova [dstq+strideq*0+16*4], m0
7281 mova [dstq+strideq*1+16*6], m3
7282 mova [dstq+strideq*0+16*5], m1
7283 call .main
7284 mova [dstq+strideq*0+16*6], m0
7285 phaddw m2, m3
7286 mova [dstq+strideq*1+16*7], m2
7287 mova [dstq+strideq*0+16*7], m1
7288 call .main
7289 paddw m2, [dstq+strideq*1+16*1]
7290 paddw m3, [dstq+strideq*1+16*2]
7291 mova [dstq+strideq*1+16*0], m0
7292 phaddw m2, m3
7293 mova [dstq+strideq*1+16*2], m2
7294 mova [dstq+strideq*1+16*1], m1
7295 call .main
7296 paddw m2, [dstq+strideq*1+16*3]
7297 paddw m3, [dstq+strideq*1+16*4]
7298 phaddw m2, m3
7299 paddw m3, m7, [dstq+strideq*1+16*2]
7300 mova [dstq+strideq*1+16*2], m0
7301 paddw m2, m7
7302 psrlw m3, 2
7303 psrlw m2, 2
7304 mova [dstq+strideq*1+16*3], m1
7305 packuswb m3, m2
7306 mova [maskq+16*0], m3
7307 call .main
7308 paddw m2, [dstq+strideq*1+16*5]
7309 paddw m3, [dstq+strideq*1+16*6]
7310 mova [dstq+strideq*1+16*4], m0
7311 phaddw m2, m3
7312 mova [dstq+strideq*1+16*6], m2
7313 mova [dstq+strideq*1+16*5], m1
7314 call .main
7315 phaddw m2, m3
7316 paddw m3, m7, [dstq+strideq*1+16*6]
7317 paddw m2, [dstq+strideq*1+16*7]
7318 mova [dstq+strideq*1+16*6], m0
7319 paddw m2, m7
7320 psrlw m3, 2
7321 psrlw m2, 2
7322 mova [dstq+strideq*1+16*7], m1
7323 packuswb m3, m2
7324 mova [maskq+16*1], m3
7325 sub hd, 2
7326 jg .w64_loop
7327 RET
7328 .w128_loop:
7329 call .main
7330 lea dstq, [dstq+strideq*2]
7331 add maskq, 16*4
7332 .w128:
7333 mova [dstq+strideq*1+16* 1], m2
7334 mova [dstq+strideq*0+16* 0], m0
7335 mova [dstq+strideq*1+16* 2], m3
7336 mova [dstq+strideq*0+16* 1], m1
7337 call .main
7338 mova [dstq+strideq*1+16* 3], m2
7339 mova [dstq+strideq*0+16* 2], m0
7340 mova [dstq+strideq*1+16* 4], m3
7341 mova [dstq+strideq*0+16* 3], m1
7342 call .main
7343 mova [dstq+strideq*1+16* 5], m2
7344 mova [dstq+strideq*0+16* 4], m0
7345 mova [dstq+strideq*1+16* 6], m3
7346 mova [dstq+strideq*0+16* 5], m1
7347 call .main
7348 mova [dstq+strideq*1+16* 7], m2
7349 mova [dstq+strideq*0+16* 6], m0
7350 mova [dstq+strideq*1+16* 8], m3
7351 mova [dstq+strideq*0+16* 7], m1
7352 call .main
7353 mova [dstq+strideq*1+16* 9], m2
7354 mova [dstq+strideq*0+16* 8], m0
7355 mova [dstq+strideq*1+16*10], m3
7356 mova [dstq+strideq*0+16* 9], m1
7357 call .main
7358 mova [dstq+strideq*1+16*11], m2
7359 mova [dstq+strideq*0+16*10], m0
7360 mova [dstq+strideq*1+16*12], m3
7361 mova [dstq+strideq*0+16*11], m1
7362 call .main
7363 mova [dstq+strideq*1+16*13], m2
7364 mova [dstq+strideq*0+16*12], m0
7365 mova [dstq+strideq*1+16*14], m3
7366 mova [dstq+strideq*0+16*13], m1
7367 call .main
7368 mova [dstq+strideq*0+16*14], m0
7369 phaddw m2, m3
7370 mova [dstq+strideq*1+16*15], m2
7371 mova [dstq+strideq*0+16*15], m1
7372 call .main
7373 paddw m2, [dstq+strideq*1+16* 1]
7374 paddw m3, [dstq+strideq*1+16* 2]
7375 mova [dstq+strideq*1+16* 0], m0
7376 phaddw m2, m3
7377 mova [dstq+strideq*1+16* 2], m2
7378 mova [dstq+strideq*1+16* 1], m1
7379 call .main
7380 paddw m2, [dstq+strideq*1+16* 3]
7381 paddw m3, [dstq+strideq*1+16* 4]
7382 phaddw m2, m3
7383 paddw m3, m7, [dstq+strideq*1+16* 2]
7384 mova [dstq+strideq*1+16* 2], m0
7385 paddw m2, m7
7386 psrlw m3, 2
7387 psrlw m2, 2
7388 mova [dstq+strideq*1+16* 3], m1
7389 packuswb m3, m2
7390 mova [maskq+16*0], m3
7391 call .main
7392 paddw m2, [dstq+strideq*1+16* 5]
7393 paddw m3, [dstq+strideq*1+16* 6]
7394 mova [dstq+strideq*1+16* 4], m0
7395 phaddw m2, m3
7396 mova [dstq+strideq*1+16* 6], m2
7397 mova [dstq+strideq*1+16* 5], m1
7398 call .main
7399 paddw m2, [dstq+strideq*1+16* 7]
7400 paddw m3, [dstq+strideq*1+16* 8]
7401 phaddw m2, m3
7402 paddw m3, m7, [dstq+strideq*1+16* 6]
7403 mova [dstq+strideq*1+16* 6], m0
7404 paddw m2, m7
7405 psrlw m3, 2
7406 psrlw m2, 2
7407 mova [dstq+strideq*1+16* 7], m1
7408 packuswb m3, m2
7409 mova [maskq+16*1], m3
7410 call .main
7411 paddw m2, [dstq+strideq*1+16* 9]
7412 paddw m3, [dstq+strideq*1+16*10]
7413 mova [dstq+strideq*1+16* 8], m0
7414 phaddw m2, m3
7415 mova [dstq+strideq*1+16*10], m2
7416 mova [dstq+strideq*1+16* 9], m1
7417 call .main
7418 paddw m2, [dstq+strideq*1+16*11]
7419 paddw m3, [dstq+strideq*1+16*12]
7420 phaddw m2, m3
7421 paddw m3, m7, [dstq+strideq*1+16*10]
7422 mova [dstq+strideq*1+16*10], m0
7423 paddw m2, m7
7424 psrlw m3, 2
7425 psrlw m2, 2
7426 mova [dstq+strideq*1+16*11], m1
7427 packuswb m3, m2
7428 mova [maskq+16*2], m3
7429 call .main
7430 paddw m2, [dstq+strideq*1+16*13]
7431 paddw m3, [dstq+strideq*1+16*14]
7432 mova [dstq+strideq*1+16*12], m0
7433 phaddw m2, m3
7434 mova [dstq+strideq*1+16*14], m2
7435 mova [dstq+strideq*1+16*13], m1
7436 call .main
7437 phaddw m2, m3
7438 paddw m3, m7, [dstq+strideq*1+16*14]
7439 paddw m2, [dstq+strideq*1+16*15]
7440 mova [dstq+strideq*1+16*14], m0
7441 paddw m2, m7
7442 psrlw m3, 2
7443 psrlw m2, 2
7444 mova [dstq+strideq*1+16*15], m1
7445 packuswb m3, m2
7446 mova [maskq+16*3], m3
7447 sub hd, 2
7448 jg .w128_loop
7449 RET
7450 ALIGN function_align
7451 .main:
7452 %macro W_MASK 2 ; dst/tmp_offset, mask
7453 mova m%1, [tmp1q+16*%1]
7454 mova m%2, [tmp2q+16*%1]
7455 punpcklwd m4, m%2, m%1
7456 punpckhwd m5, m%2, m%1
7457 psubsw m%1, m%2
7458 pabsw m%1, m%1
7459 psubusw m6, m8, m%1
7460 psrlw m6, 10 ; 64-m
7461 psubw m%2, m9, m6 ; m
7462 punpcklwd m%1, m6, m%2
7463 punpckhwd m6, m%2
7464 pmaddwd m%1, m4
7465 pmaddwd m6, m5
7466 psrad m%1, 5
7467 psrad m6, 5
7468 packssdw m%1, m6
7469 pmaxsw m%1, m10
7470 psubsw m%1, m10
7471 pmulhw m%1, m11
7472 %endmacro
7473 W_MASK 0, 2
7474 W_MASK 1, 3
7475 add tmp1q, 16*2
7476 add tmp2q, 16*2
7477 ret
7478
7479 cglobal w_mask_422_16bpc, 4, 7, 12, dst, stride, tmp1, tmp2, w, h, mask
7480 %define base t0-w_mask_422_ssse3_table
7481 LEA t0, w_mask_422_ssse3_table
7482 tzcnt wd, wm
7483 mov r6d, r8m ; pixel_max
7484 movd m7, r7m ; sign
7485 shr r6d, 11
7486 movsxd wq, [t0+wq*4]
7487 %if ARCH_X86_64
7488 mova m8, [base+pw_27615]
7489 mova m9, [base+pw_64]
7490 movddup m10, [base+bidir_rnd+r6*8]
7491 movddup m11, [base+bidir_mul+r6*8]
7492 %else
7493 mova m1, [base+pw_27615]
7494 mova m2, [base+pw_64]
7495 movddup m3, [base+bidir_rnd+r6*8]
7496 movddup m4, [base+bidir_mul+r6*8]
7497 ALLOC_STACK -16*4
7498 mova [rsp+16*0], m1
7499 mova [rsp+16*1], m2
7500 mova [rsp+16*2], m3
7501 mova [rsp+16*3], m4
7502 %endif
7503 pxor m0, m0
7504 add wq, t0
7505 pshufb m7, m0
7506 movifnidn hd, r5m
7507 mov maskq, r6mp
7508 call .main
7509 jmp wq
7510 .w4_loop:
7511 call .main
7512 lea dstq, [dstq+strideq*2]
7513 .w4:
7514 movq [dstq+strideq*0], m0
7515 movhps [dstq+strideq*1], m0
7516 lea dstq, [dstq+strideq*2]
7517 movq [dstq+strideq*0], m1
7518 movhps [dstq+strideq*1], m1
7519 sub hd, 4
7520 jg .w4_loop
7521 .end:
7522 RET
7523 .w8_loop:
7524 call .main
7525 lea dstq, [dstq+strideq*2]
7526 .w8:
7527 mova [dstq+strideq*0], m0
7528 mova [dstq+strideq*1], m1
7529 sub hd, 2
7530 jg .w8_loop
7531 .w8_end:
7532 RET
7533 .w16_loop:
7534 call .main
7535 lea dstq, [dstq+strideq*2]
7536 .w16:
7537 mova [dstq+strideq*0+16*0], m0
7538 mova [dstq+strideq*0+16*1], m1
7539 call .main
7540 mova [dstq+strideq*1+16*0], m0
7541 mova [dstq+strideq*1+16*1], m1
7542 sub hd, 2
7543 jg .w16_loop
7544 RET
7545 .w32_loop:
7546 call .main
7547 add dstq, strideq
7548 .w32:
7549 mova [dstq+16*0], m0
7550 mova [dstq+16*1], m1
7551 call .main
7552 mova [dstq+16*2], m0
7553 mova [dstq+16*3], m1
7554 dec hd
7555 jg .w32_loop
7556 RET
7557 .w64_loop:
7558 call .main
7559 add dstq, strideq
7560 .w64:
7561 mova [dstq+16*0], m0
7562 mova [dstq+16*1], m1
7563 call .main
7564 mova [dstq+16*2], m0
7565 mova [dstq+16*3], m1
7566 call .main
7567 mova [dstq+16*4], m0
7568 mova [dstq+16*5], m1
7569 call .main
7570 mova [dstq+16*6], m0
7571 mova [dstq+16*7], m1
7572 dec hd
7573 jg .w64_loop
7574 RET
7575 .w128_loop:
7576 call .main
7577 add dstq, strideq
7578 .w128:
7579 mova [dstq+16* 0], m0
7580 mova [dstq+16* 1], m1
7581 call .main
7582 mova [dstq+16* 2], m0
7583 mova [dstq+16* 3], m1
7584 call .main
7585 mova [dstq+16* 4], m0
7586 mova [dstq+16* 5], m1
7587 call .main
7588 mova [dstq+16* 6], m0
7589 mova [dstq+16* 7], m1
7590 call .main
7591 mova [dstq+16* 8], m0
7592 mova [dstq+16* 9], m1
7593 call .main
7594 mova [dstq+16*10], m0
7595 mova [dstq+16*11], m1
7596 call .main
7597 mova [dstq+16*12], m0
7598 mova [dstq+16*13], m1
7599 call .main
7600 mova [dstq+16*14], m0
7601 mova [dstq+16*15], m1
7602 dec hd
7603 jg .w128_loop
7604 RET
7605 ALIGN function_align
7606 .main:
7607 W_MASK 0, 2
7608 W_MASK 1, 3
7609 phaddw m2, m3
7610 add tmp1q, 16*2
7611 add tmp2q, 16*2
7612 packuswb m2, m2
7613 pxor m3, m3
7614 psubb m2, m7
7615 pavgb m2, m3
7616 movq [maskq], m2
7617 add maskq, 8
7618 ret
7619
7620 cglobal w_mask_444_16bpc, 4, 7, 12, dst, stride, tmp1, tmp2, w, h, mask
7621 %define base t0-w_mask_444_ssse3_table
7622 LEA t0, w_mask_444_ssse3_table
7623 tzcnt wd, wm
7624 mov r6d, r8m ; pixel_max
7625 shr r6d, 11
7626 movsxd wq, [t0+wq*4]
7627 %if ARCH_X86_64
7628 mova m8, [base+pw_27615]
7629 mova m9, [base+pw_64]
7630 movddup m10, [base+bidir_rnd+r6*8]
7631 movddup m11, [base+bidir_mul+r6*8]
7632 %else
7633 mova m1, [base+pw_27615]
7634 mova m2, [base+pw_64]
7635 movddup m3, [base+bidir_rnd+r6*8]
7636 movddup m7, [base+bidir_mul+r6*8]
7637 ALLOC_STACK -16*3
7638 mova [rsp+16*0], m1
7639 mova [rsp+16*1], m2
7640 mova [rsp+16*2], m3
7641 %define m11 m7
7642 %endif
7643 add wq, t0
7644 movifnidn hd, r5m
7645 mov maskq, r6mp
7646 call .main
7647 jmp wq
7648 .w4_loop:
7649 call .main
7650 lea dstq, [dstq+strideq*2]
7651 .w4:
7652 movq [dstq+strideq*0], m0
7653 movhps [dstq+strideq*1], m0
7654 lea dstq, [dstq+strideq*2]
7655 movq [dstq+strideq*0], m1
7656 movhps [dstq+strideq*1], m1
7657 sub hd, 4
7658 jg .w4_loop
7659 .end:
7660 RET
7661 .w8_loop:
7662 call .main
7663 lea dstq, [dstq+strideq*2]
7664 .w8:
7665 mova [dstq+strideq*0], m0
7666 mova [dstq+strideq*1], m1
7667 sub hd, 2
7668 jg .w8_loop
7669 .w8_end:
7670 RET
7671 .w16_loop:
7672 call .main
7673 lea dstq, [dstq+strideq*2]
7674 .w16:
7675 mova [dstq+strideq*0+16*0], m0
7676 mova [dstq+strideq*0+16*1], m1
7677 call .main
7678 mova [dstq+strideq*1+16*0], m0
7679 mova [dstq+strideq*1+16*1], m1
7680 sub hd, 2
7681 jg .w16_loop
7682 RET
7683 .w32_loop:
7684 call .main
7685 add dstq, strideq
7686 .w32:
7687 mova [dstq+16*0], m0
7688 mova [dstq+16*1], m1
7689 call .main
7690 mova [dstq+16*2], m0
7691 mova [dstq+16*3], m1
7692 dec hd
7693 jg .w32_loop
7694 RET
7695 .w64_loop:
7696 call .main
7697 add dstq, strideq
7698 .w64:
7699 mova [dstq+16*0], m0
7700 mova [dstq+16*1], m1
7701 call .main
7702 mova [dstq+16*2], m0
7703 mova [dstq+16*3], m1
7704 call .main
7705 mova [dstq+16*4], m0
7706 mova [dstq+16*5], m1
7707 call .main
7708 mova [dstq+16*6], m0
7709 mova [dstq+16*7], m1
7710 dec hd
7711 jg .w64_loop
7712 RET
7713 .w128_loop:
7714 call .main
7715 add dstq, strideq
7716 .w128:
7717 mova [dstq+16* 0], m0
7718 mova [dstq+16* 1], m1
7719 call .main
7720 mova [dstq+16* 2], m0
7721 mova [dstq+16* 3], m1
7722 call .main
7723 mova [dstq+16* 4], m0
7724 mova [dstq+16* 5], m1
7725 call .main
7726 mova [dstq+16* 6], m0
7727 mova [dstq+16* 7], m1
7728 call .main
7729 mova [dstq+16* 8], m0
7730 mova [dstq+16* 9], m1
7731 call .main
7732 mova [dstq+16*10], m0
7733 mova [dstq+16*11], m1
7734 call .main
7735 mova [dstq+16*12], m0
7736 mova [dstq+16*13], m1
7737 call .main
7738 mova [dstq+16*14], m0
7739 mova [dstq+16*15], m1
7740 dec hd
7741 jg .w128_loop
7742 RET
7743 ALIGN function_align
7744 .main:
7745 W_MASK 0, 2
7746 W_MASK 1, 3
7747 packuswb m2, m3
7748 add tmp1q, 16*2
7749 add tmp2q, 16*2
7750 mova [maskq], m2
7751 add maskq, 16
7752 ret
7753
7754 ; (a * (64 - m) + b * m + 32) >> 6
7755 ; = (((b - a) * m + 32) >> 6) + a
7756 ; = (((b - a) * (m << 9) + 16384) >> 15) + a
7757 ; except m << 9 overflows int16_t when m == 64 (which is possible),
7758 ; but if we negate m it works out (-64 << 9 == -32768).
7759 ; = (((a - b) * (m * -512) + 16384) >> 15) + a
7760 cglobal blend_16bpc, 3, 7, 8, dst, stride, tmp, w, h, mask, stride3
7761 %define base r6-blend_ssse3_table
7762 LEA r6, blend_ssse3_table
7763 tzcnt wd, wm
7764 movifnidn hd, hm
7765 movsxd wq, [r6+wq*4]
7766 movifnidn maskq, maskmp
7767 mova m7, [base+pw_m512]
7768 add wq, r6
7769 lea stride3q, [strideq*3]
7770 pxor m6, m6
7771 jmp wq
7772 .w4:
7773 mova m5, [maskq]
7774 movq m0, [dstq+strideq*0]
7775 movhps m0, [dstq+strideq*1]
7776 movq m1, [dstq+strideq*2]
7777 movhps m1, [dstq+stride3q ]
7778 psubw m2, m0, [tmpq+16*0]
7779 psubw m3, m1, [tmpq+16*1]
7780 add maskq, 16
7781 add tmpq, 32
7782 punpcklbw m4, m5, m6
7783 punpckhbw m5, m6
7784 pmullw m4, m7
7785 pmullw m5, m7
7786 pmulhrsw m2, m4
7787 pmulhrsw m3, m5
7788 paddw m0, m2
7789 paddw m1, m3
7790 movq [dstq+strideq*0], m0
7791 movhps [dstq+strideq*1], m0
7792 movq [dstq+strideq*2], m1
7793 movhps [dstq+stride3q ], m1
7794 lea dstq, [dstq+strideq*4]
7795 sub hd, 4
7796 jg .w4
7797 RET
7798 .w8:
7799 mova m5, [maskq]
7800 mova m0, [dstq+strideq*0]
7801 mova m1, [dstq+strideq*1]
7802 psubw m2, m0, [tmpq+16*0]
7803 psubw m3, m1, [tmpq+16*1]
7804 add maskq, 16
7805 add tmpq, 32
7806 punpcklbw m4, m5, m6
7807 punpckhbw m5, m6
7808 pmullw m4, m7
7809 pmullw m5, m7
7810 pmulhrsw m2, m4
7811 pmulhrsw m3, m5
7812 paddw m0, m2
7813 paddw m1, m3
7814 mova [dstq+strideq*0], m0
7815 mova [dstq+strideq*1], m1
7816 lea dstq, [dstq+strideq*2]
7817 sub hd, 2
7818 jg .w8
7819 RET
7820 .w16:
7821 mova m5, [maskq]
7822 mova m0, [dstq+16*0]
7823 mova m1, [dstq+16*1]
7824 psubw m2, m0, [tmpq+16*0]
7825 psubw m3, m1, [tmpq+16*1]
7826 add maskq, 16
7827 add tmpq, 32
7828 punpcklbw m4, m5, m6
7829 punpckhbw m5, m6
7830 pmullw m4, m7
7831 pmullw m5, m7
7832 pmulhrsw m2, m4
7833 pmulhrsw m3, m5
7834 paddw m0, m2
7835 paddw m1, m3
7836 mova [dstq+16*0], m0
7837 mova [dstq+16*1], m1
7838 add dstq, strideq
7839 dec hd
7840 jg .w16
7841 RET
7842 .w32:
7843 mova m5, [maskq+16*0]
7844 mova m0, [dstq+16*0]
7845 mova m1, [dstq+16*1]
7846 psubw m2, m0, [tmpq+16*0]
7847 psubw m3, m1, [tmpq+16*1]
7848 punpcklbw m4, m5, m6
7849 punpckhbw m5, m6
7850 pmullw m4, m7
7851 pmullw m5, m7
7852 pmulhrsw m2, m4
7853 pmulhrsw m3, m5
7854 paddw m0, m2
7855 paddw m1, m3
7856 mova [dstq+16*0], m0
7857 mova [dstq+16*1], m1
7858 mova m5, [maskq+16*1]
7859 mova m0, [dstq+16*2]
7860 mova m1, [dstq+16*3]
7861 psubw m2, m0, [tmpq+16*2]
7862 psubw m3, m1, [tmpq+16*3]
7863 add maskq, 32
7864 add tmpq, 64
7865 punpcklbw m4, m5, m6
7866 punpckhbw m5, m6
7867 pmullw m4, m7
7868 pmullw m5, m7
7869 pmulhrsw m2, m4
7870 pmulhrsw m3, m5
7871 paddw m0, m2
7872 paddw m1, m3
7873 mova [dstq+16*2], m0
7874 mova [dstq+16*3], m1
7875 add dstq, strideq
7876 dec hd
7877 jg .w32
7878 RET
7879
7880 cglobal blend_v_16bpc, 3, 6, 6, dst, stride, tmp, w, h
7881 %define base r5-blend_v_ssse3_table
7882 LEA r5, blend_v_ssse3_table
7883 tzcnt wd, wm
7884 movifnidn hd, hm
7885 movsxd wq, [r5+wq*4]
7886 add wq, r5
7887 jmp wq
7888 .w2:
7889 movd m4, [base+obmc_masks+2*2]
7890 .w2_loop:
7891 movd m0, [dstq+strideq*0]
7892 movd m2, [tmpq+4*0]
7893 movd m1, [dstq+strideq*1]
7894 movd m3, [tmpq+4*1]
7895 add tmpq, 4*2
7896 psubw m2, m0
7897 psubw m3, m1
7898 pmulhrsw m2, m4
7899 pmulhrsw m3, m4
7900 paddw m0, m2
7901 paddw m1, m3
7902 movd [dstq+strideq*0], m0
7903 movd [dstq+strideq*1], m1
7904 lea dstq, [dstq+strideq*2]
7905 sub hd, 2
7906 jg .w2_loop
7907 RET
7908 .w4:
7909 movddup m2, [base+obmc_masks+4*2]
7910 .w4_loop:
7911 movq m0, [dstq+strideq*0]
7912 movhps m0, [dstq+strideq*1]
7913 mova m1, [tmpq]
7914 add tmpq, 8*2
7915 psubw m1, m0
7916 pmulhrsw m1, m2
7917 paddw m0, m1
7918 movq [dstq+strideq*0], m0
7919 movhps [dstq+strideq*1], m0
7920 lea dstq, [dstq+strideq*2]
7921 sub hd, 2
7922 jg .w4_loop
7923 RET
7924 .w8:
7925 mova m4, [base+obmc_masks+8*2]
7926 .w8_loop:
7927 mova m0, [dstq+strideq*0]
7928 mova m2, [tmpq+16*0]
7929 mova m1, [dstq+strideq*1]
7930 mova m3, [tmpq+16*1]
7931 add tmpq, 16*2
7932 psubw m2, m0
7933 psubw m3, m1
7934 pmulhrsw m2, m4
7935 pmulhrsw m3, m4
7936 paddw m0, m2
7937 paddw m1, m3
7938 mova [dstq+strideq*0], m0
7939 mova [dstq+strideq*1], m1
7940 lea dstq, [dstq+strideq*2]
7941 sub hd, 2
7942 jg .w8_loop
7943 RET
7944 .w16:
7945 mova m4, [base+obmc_masks+16*2]
7946 movq m5, [base+obmc_masks+16*3]
7947 .w16_loop:
7948 mova m0, [dstq+16*0]
7949 mova m2, [tmpq+16*0]
7950 mova m1, [dstq+16*1]
7951 mova m3, [tmpq+16*1]
7952 add tmpq, 16*2
7953 psubw m2, m0
7954 psubw m3, m1
7955 pmulhrsw m2, m4
7956 pmulhrsw m3, m5
7957 paddw m0, m2
7958 paddw m1, m3
7959 mova [dstq+16*0], m0
7960 mova [dstq+16*1], m1
7961 add dstq, strideq
7962 dec hd
7963 jg .w16_loop
7964 RET
7965 .w32:
7966 %if WIN64
7967 movaps [rsp+8], m6
7968 %endif
7969 mova m4, [base+obmc_masks+16*4]
7970 mova m5, [base+obmc_masks+16*5]
7971 mova m6, [base+obmc_masks+16*6]
7972 .w32_loop:
7973 mova m0, [dstq+16*0]
7974 mova m2, [tmpq+16*0]
7975 mova m1, [dstq+16*1]
7976 mova m3, [tmpq+16*1]
7977 psubw m2, m0
7978 psubw m3, m1
7979 pmulhrsw m2, m4
7980 pmulhrsw m3, m5
7981 paddw m0, m2
7982 mova m2, [dstq+16*2]
7983 paddw m1, m3
7984 mova m3, [tmpq+16*2]
7985 add tmpq, 16*4
7986 psubw m3, m2
7987 pmulhrsw m3, m6
7988 paddw m2, m3
7989 mova [dstq+16*0], m0
7990 mova [dstq+16*1], m1
7991 mova [dstq+16*2], m2
7992 add dstq, strideq
7993 dec hd
7994 jg .w32_loop
7995 %if WIN64
7996 movaps m6, [rsp+8]
7997 %endif
7998 RET
7999
8000 %macro BLEND_H_ROW 2-3 0; dst_off, tmp_off, inc_tmp
8001 mova m0, [dstq+16*(%1+0)]
8002 mova m2, [tmpq+16*(%2+0)]
8003 mova m1, [dstq+16*(%1+1)]
8004 mova m3, [tmpq+16*(%2+1)]
8005 %if %3
8006 add tmpq, 16*%3
8007 %endif
8008 psubw m2, m0
8009 psubw m3, m1
8010 pmulhrsw m2, m5
8011 pmulhrsw m3, m5
8012 paddw m0, m2
8013 paddw m1, m3
8014 mova [dstq+16*(%1+0)], m0
8015 mova [dstq+16*(%1+1)], m1
8016 %endmacro
8017
8018 cglobal blend_h_16bpc, 3, 7, 6, dst, ds, tmp, w, h, mask
8019 %define base r6-blend_h_ssse3_table
8020 LEA r6, blend_h_ssse3_table
8021 tzcnt wd, wm
8022 mov hd, hm
8023 movsxd wq, [r6+wq*4]
8024 movddup m4, [base+blend_shuf]
8025 lea maskq, [base+obmc_masks+hq*2]
8026 lea hd, [hq*3]
8027 add wq, r6
8028 shr hd, 2 ; h * 3/4
8029 lea maskq, [maskq+hq*2]
8030 neg hq
8031 jmp wq
8032 .w2:
8033 movd m0, [dstq+dsq*0]
8034 movd m2, [dstq+dsq*1]
8035 movd m3, [maskq+hq*2]
8036 movq m1, [tmpq]
8037 add tmpq, 4*2
8038 punpckldq m0, m2
8039 punpcklwd m3, m3
8040 psubw m1, m0
8041 pmulhrsw m1, m3
8042 paddw m0, m1
8043 movd [dstq+dsq*0], m0
8044 psrlq m0, 32
8045 movd [dstq+dsq*1], m0
8046 lea dstq, [dstq+dsq*2]
8047 add hq, 2
8048 jl .w2
8049 RET
8050 .w4:
8051 mova m3, [base+blend_shuf]
8052 .w4_loop:
8053 movq m0, [dstq+dsq*0]
8054 movhps m0, [dstq+dsq*1]
8055 movd m2, [maskq+hq*2]
8056 mova m1, [tmpq]
8057 add tmpq, 8*2
8058 psubw m1, m0
8059 pshufb m2, m3
8060 pmulhrsw m1, m2
8061 paddw m0, m1
8062 movq [dstq+dsq*0], m0
8063 movhps [dstq+dsq*1], m0
8064 lea dstq, [dstq+dsq*2]
8065 add hq, 2
8066 jl .w4_loop
8067 RET
8068 .w8:
8069 movddup m5, [base+blend_shuf+8]
8070 %if WIN64
8071 movaps [rsp+ 8], m6
8072 movaps [rsp+24], m7
8073 %endif
8074 .w8_loop:
8075 movd m7, [maskq+hq*2]
8076 mova m0, [dstq+dsq*0]
8077 mova m2, [tmpq+16*0]
8078 mova m1, [dstq+dsq*1]
8079 mova m3, [tmpq+16*1]
8080 add tmpq, 16*2
8081 pshufb m6, m7, m4
8082 psubw m2, m0
8083 pshufb m7, m5
8084 psubw m3, m1
8085 pmulhrsw m2, m6
8086 pmulhrsw m3, m7
8087 paddw m0, m2
8088 paddw m1, m3
8089 mova [dstq+dsq*0], m0
8090 mova [dstq+dsq*1], m1
8091 lea dstq, [dstq+dsq*2]
8092 add hq, 2
8093 jl .w8_loop
8094 %if WIN64
8095 movaps m6, [rsp+ 8]
8096 movaps m7, [rsp+24]
8097 %endif
8098 RET
8099 .w16:
8100 movd m5, [maskq+hq*2]
8101 pshufb m5, m4
8102 BLEND_H_ROW 0, 0, 2
8103 add dstq, dsq
8104 inc hq
8105 jl .w16
8106 RET
8107 .w32:
8108 movd m5, [maskq+hq*2]
8109 pshufb m5, m4
8110 BLEND_H_ROW 0, 0
8111 BLEND_H_ROW 2, 2, 4
8112 add dstq, dsq
8113 inc hq
8114 jl .w32
8115 RET
8116 .w64:
8117 movd m5, [maskq+hq*2]
8118 pshufb m5, m4
8119 BLEND_H_ROW 0, 0
8120 BLEND_H_ROW 2, 2
8121 BLEND_H_ROW 4, 4
8122 BLEND_H_ROW 6, 6, 8
8123 add dstq, dsq
8124 inc hq
8125 jl .w64
8126 RET
8127 .w128:
8128 movd m5, [maskq+hq*2]
8129 pshufb m5, m4
8130 BLEND_H_ROW 0, 0
8131 BLEND_H_ROW 2, 2
8132 BLEND_H_ROW 4, 4
8133 BLEND_H_ROW 6, 6, 16
8134 BLEND_H_ROW 8, -8
8135 BLEND_H_ROW 10, -6
8136 BLEND_H_ROW 12, -4
8137 BLEND_H_ROW 14, -2
8138 add dstq, dsq
8139 inc hq
8140 jl .w128
8141 RET
8142
8143 ; emu_edge args:
8144 ; const intptr_t bw, const intptr_t bh, const intptr_t iw, const intptr_t ih,
8145 ; const intptr_t x, const intptr_t y, pixel *dst, const ptrdiff_t dst_stride,
8146 ; const pixel *ref, const ptrdiff_t ref_stride
8147 ;
8148 ; bw, bh total filled size
8149 ; iw, ih, copied block -> fill bottom, right
8150 ; x, y, offset in bw/bh -> fill top, left
8151 cglobal emu_edge_16bpc, 10, 13, 1, bw, bh, iw, ih, x, \
8152 y, dst, dstride, src, sstride, \
8153 bottomext, rightext, blk
8154 ; we assume that the buffer (stride) is larger than width, so we can
8155 ; safely overwrite by a few bytes
8156
8157 %if ARCH_X86_64
8158 %define reg_zero r12q
8159 %define reg_tmp r10
8160 %define reg_src srcq
8161 %define reg_bottomext bottomextq
8162 %define reg_rightext rightextq
8163 %define reg_blkm r9m
8164 %else
8165 %define reg_zero r6
8166 %define reg_tmp r0
8167 %define reg_src r1
8168 %define reg_bottomext r0
8169 %define reg_rightext r1
8170 %define reg_blkm r2m
8171 %endif
8172 ;
8173 ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride)
8174 xor reg_zero, reg_zero
8175 lea reg_tmp, [ihq-1]
8176 cmp yq, ihq
8177 cmovs reg_tmp, yq
8178 test yq, yq
8179 cmovs reg_tmp, reg_zero
8180 %if ARCH_X86_64
8181 imul reg_tmp, sstrideq
8182 add srcq, reg_tmp
8183 %else
8184 imul reg_tmp, sstridem
8185 mov reg_src, srcm
8186 add reg_src, reg_tmp
8187 %endif
8188 ;
8189 ; ref += iclip(x, 0, iw - 1)
8190 lea reg_tmp, [iwq-1]
8191 cmp xq, iwq
8192 cmovs reg_tmp, xq
8193 test xq, xq
8194 cmovs reg_tmp, reg_zero
8195 lea reg_src, [reg_src+reg_tmp*2]
8196 %if ARCH_X86_32
8197 mov srcm, reg_src
8198 %endif
8199 ;
8200 ; bottom_ext = iclip(y + bh - ih, 0, bh - 1)
8201 %if ARCH_X86_32
8202 mov r1, r1m ; restore bh
8203 %endif
8204 lea reg_bottomext, [yq+bhq]
8205 sub reg_bottomext, ihq
8206 lea r3, [bhq-1]
8207 cmovs reg_bottomext, reg_zero
8208 ;
8209
8210 DEFINE_ARGS bw, bh, iw, ih, x, \
8211 topext, dst, dstride, src, sstride, \
8212 bottomext, rightext, blk
8213
8214 ; top_ext = iclip(-y, 0, bh - 1)
8215 neg topextq
8216 cmovs topextq, reg_zero
8217 cmp reg_bottomext, bhq
8218 cmovns reg_bottomext, r3
8219 cmp topextq, bhq
8220 cmovg topextq, r3
8221 %if ARCH_X86_32
8222 mov r4m, reg_bottomext
8223 ;
8224 ; right_ext = iclip(x + bw - iw, 0, bw - 1)
8225 mov r0, r0m ; restore bw
8226 %endif
8227 lea reg_rightext, [xq+bwq]
8228 sub reg_rightext, iwq
8229 lea r2, [bwq-1]
8230 cmovs reg_rightext, reg_zero
8231
8232 DEFINE_ARGS bw, bh, iw, ih, leftext, \
8233 topext, dst, dstride, src, sstride, \
8234 bottomext, rightext, blk
8235
8236 ; left_ext = iclip(-x, 0, bw - 1)
8237 neg leftextq
8238 cmovs leftextq, reg_zero
8239 cmp reg_rightext, bwq
8240 cmovns reg_rightext, r2
8241 %if ARCH_X86_32
8242 mov r3m, r1
8243 %endif
8244 cmp leftextq, bwq
8245 cmovns leftextq, r2
8246
8247 %undef reg_zero
8248 %undef reg_tmp
8249 %undef reg_src
8250 %undef reg_bottomext
8251 %undef reg_rightext
8252
8253 DEFINE_ARGS bw, centerh, centerw, dummy, leftext, \
8254 topext, dst, dstride, src, sstride, \
8255 bottomext, rightext, blk
8256
8257 ; center_h = bh - top_ext - bottom_ext
8258 %if ARCH_X86_64
8259 lea r3, [bottomextq+topextq]
8260 sub centerhq, r3
8261 %else
8262 mov r1, centerhm ; restore r1
8263 sub centerhq, topextq
8264 sub centerhq, r4m
8265 mov r1m, centerhq
8266 %endif
8267 ;
8268 ; blk += top_ext * PXSTRIDE(dst_stride)
8269 mov r2, topextq
8270 %if ARCH_X86_64
8271 imul r2, dstrideq
8272 %else
8273 mov r6, r6m ; restore dstq
8274 imul r2, dstridem
8275 %endif
8276 add dstq, r2
8277 mov reg_blkm, dstq ; save pointer for ext
8278 ;
8279 ; center_w = bw - left_ext - right_ext
8280 mov centerwq, bwq
8281 %if ARCH_X86_64
8282 lea r3, [rightextq+leftextq]
8283 sub centerwq, r3
8284 %else
8285 sub centerwq, r3m
8286 sub centerwq, leftextq
8287 %endif
8288
8289 ; vloop Macro
8290 %macro v_loop 3 ; need_left_ext, need_right_ext, suffix
8291 %if ARCH_X86_64
8292 %define reg_tmp r12
8293 %else
8294 %define reg_tmp r0
8295 %endif
8296 .v_loop_%3:
8297 %if ARCH_X86_32
8298 mov r0, r0m
8299 mov r1, r1m
8300 %endif
8301 %if %1
8302 ; left extension
8303 %if ARCH_X86_64
8304 movd m0, [srcq]
8305 %else
8306 mov r3, srcm
8307 movd m0, [r3]
8308 %endif
8309 pshuflw m0, m0, q0000
8310 punpcklqdq m0, m0
8311 xor r3, r3
8312 .left_loop_%3:
8313 mova [dstq+r3*2], m0
8314 add r3, mmsize/2
8315 cmp r3, leftextq
8316 jl .left_loop_%3
8317 ; body
8318 lea reg_tmp, [dstq+leftextq*2]
8319 %endif
8320 xor r3, r3
8321 .body_loop_%3:
8322 %if ARCH_X86_64
8323 movu m0, [srcq+r3*2]
8324 %else
8325 mov r1, srcm
8326 movu m0, [r1+r3*2]
8327 %endif
8328 %if %1
8329 movu [reg_tmp+r3*2], m0
8330 %else
8331 movu [dstq+r3*2], m0
8332 %endif
8333 add r3, mmsize/2
8334 cmp r3, centerwq
8335 jl .body_loop_%3
8336 %if %2
8337 ; right extension
8338 %if %1
8339 lea reg_tmp, [reg_tmp+centerwq*2]
8340 %else
8341 lea reg_tmp, [dstq+centerwq*2]
8342 %endif
8343 %if ARCH_X86_64
8344 movd m0, [srcq+centerwq*2-2]
8345 %else
8346 mov r3, srcm
8347 movd m0, [r3+centerwq*2-2]
8348 %endif
8349 pshuflw m0, m0, q0000
8350 punpcklqdq m0, m0
8351 xor r3, r3
8352 .right_loop_%3:
8353 movu [reg_tmp+r3*2], m0
8354 add r3, mmsize/2
8355 %if ARCH_X86_64
8356 cmp r3, rightextq
8357 %else
8358 cmp r3, r3m
8359 %endif
8360 jl .right_loop_%3
8361 %endif
8362 %if ARCH_X86_64
8363 add dstq, dstrideq
8364 add srcq, sstrideq
8365 dec centerhq
8366 jg .v_loop_%3
8367 %else
8368 add dstq, dstridem
8369 mov r0, sstridem
8370 add srcm, r0
8371 sub dword centerhm, 1
8372 jg .v_loop_%3
8373 mov r0, r0m ; restore r0
8374 %endif
8375 %endmacro ; vloop MACRO
8376
8377 test leftextq, leftextq
8378 jnz .need_left_ext
8379 %if ARCH_X86_64
8380 test rightextq, rightextq
8381 jnz .need_right_ext
8382 %else
8383 cmp leftextq, r3m ; leftextq == 0
8384 jne .need_right_ext
8385 %endif
8386 v_loop 0, 0, 0
8387 jmp .body_done
8388
8389 ;left right extensions
8390 .need_left_ext:
8391 %if ARCH_X86_64
8392 test rightextq, rightextq
8393 %else
8394 mov r3, r3m
8395 test r3, r3
8396 %endif
8397 jnz .need_left_right_ext
8398 v_loop 1, 0, 1
8399 jmp .body_done
8400
8401 .need_left_right_ext:
8402 v_loop 1, 1, 2
8403 jmp .body_done
8404
8405 .need_right_ext:
8406 v_loop 0, 1, 3
8407
8408 .body_done:
8409 ; r0 ; bw
8410 ; r1 ;; x loop
8411 ; r4 ;; y loop
8412 ; r5 ; topextq
8413 ; r6 ;dstq
8414 ; r7 ;dstrideq
8415 ; r8 ; srcq
8416 %if ARCH_X86_64
8417 %define reg_dstride dstrideq
8418 %else
8419 %define reg_dstride r2
8420 %endif
8421 ;
8422 ; bottom edge extension
8423 %if ARCH_X86_64
8424 test bottomextq, bottomextq
8425 jz .top
8426 %else
8427 xor r1, r1
8428 cmp r1, r4m
8429 je .top
8430 %endif
8431 ;
8432 %if ARCH_X86_64
8433 mov srcq, dstq
8434 sub srcq, dstrideq
8435 xor r1, r1
8436 %else
8437 mov r3, dstq
8438 mov reg_dstride, dstridem
8439 sub r3, reg_dstride
8440 mov srcm, r3
8441 %endif
8442 ;
8443 .bottom_x_loop:
8444 %if ARCH_X86_64
8445 mova m0, [srcq+r1*2]
8446 lea r3, [dstq+r1*2]
8447 mov r4, bottomextq
8448 %else
8449 mov r3, srcm
8450 mova m0, [r3+r1*2]
8451 lea r3, [dstq+r1*2]
8452 mov r4, r4m
8453 %endif
8454 ;
8455 .bottom_y_loop:
8456 mova [r3], m0
8457 add r3, reg_dstride
8458 dec r4
8459 jg .bottom_y_loop
8460 add r1, mmsize/2
8461 cmp r1, bwq
8462 jl .bottom_x_loop
8463
8464 .top:
8465 ; top edge extension
8466 test topextq, topextq
8467 jz .end
8468 %if ARCH_X86_64
8469 mov srcq, reg_blkm
8470 %else
8471 mov r3, reg_blkm
8472 mov reg_dstride, dstridem
8473 %endif
8474 mov dstq, dstm
8475 xor r1, r1
8476 ;
8477 .top_x_loop:
8478 %if ARCH_X86_64
8479 mova m0, [srcq+r1*2]
8480 %else
8481 mov r3, reg_blkm
8482 mova m0, [r3+r1*2]
8483 %endif
8484 lea r3, [dstq+r1*2]
8485 mov r4, topextq
8486 ;
8487 .top_y_loop:
8488 mova [r3], m0
8489 add r3, reg_dstride
8490 dec r4
8491 jg .top_y_loop
8492 add r1, mmsize/2
8493 cmp r1, bwq
8494 jl .top_x_loop
8495
8496 .end:
8497 RET
8498
8499 %undef reg_dstride
8500 %undef reg_blkm
8501 %undef reg_tmp
8502
8503 %macro SCRATCH 3
8504 %if ARCH_X86_32
8505 mova [rsp+%3*mmsize], m%1
8506 %define m%2 [rsp+%3*mmsize]
8507 %else
8508 SWAP %1, %2
8509 %endif
8510 %endmacro
8511
8512 %if ARCH_X86_64
8513 cglobal resize_16bpc, 0, 12, 16, 1*16, dst, dst_stride, src, src_stride, \
8514 dst_w, h, src_w, dx, mx0, pxmax
8515 %elif STACK_ALIGNMENT >= 16
8516 cglobal resize_16bpc, 0, 7, 8, 6*16, dst, dst_stride, src, src_stride, \
8517 dst_w, h, src_w, dx, mx0, pxmax
8518 %else
8519 cglobal resize_16bpc, 0, 6, 8, 6*16, dst, dst_stride, src, src_stride, \
8520 dst_w, h, src_w, dx, mx0, pxmax
8521 %endif
8522 movifnidn dstq, dstmp
8523 movifnidn srcq, srcmp
8524 %if STACK_ALIGNMENT >= 16
8525 movifnidn dst_wd, dst_wm
8526 %endif
8527 %if ARCH_X86_64
8528 movifnidn hd, hm
8529 %endif
8530 sub dword mx0m, 4<<14
8531 sub dword src_wm, 8
8532 movd m4, pxmaxm
8533 movd m7, dxm
8534 movd m6, mx0m
8535 movd m5, src_wm
8536 punpcklwd m4, m4
8537 pshufd m4, m4, q0000
8538 pshufd m7, m7, q0000
8539 pshufd m6, m6, q0000
8540 pshufd m5, m5, q0000
8541 mova [rsp+16*3*ARCH_X86_32], m4
8542 %if ARCH_X86_64
8543 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x
8544 LEA r7, $$
8545 %define base r7-$$
8546 %else
8547 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x
8548 %define hd dword r5m
8549 %if STACK_ALIGNMENT >= 16
8550 LEA r6, $$
8551 %define base r6-$$
8552 %else
8553 LEA r4, $$
8554 %define base r4-$$
8555 %endif
8556 %endif
8557 %if ARCH_X86_64
8558 mova m12, [base+pd_64]
8559 mova m11, [base+pd_63]
8560 %else
8561 %define m12 [base+pd_64]
8562 %define m11 [base+pd_63]
8563 %endif
8564 pmaddwd m4, m7, [base+rescale_mul] ; dx*[0,1,2,3]
8565 pslld m7, 2 ; dx*4
8566 pslld m5, 14
8567 paddd m6, m4 ; mx+[0..3]*dx
8568 SCRATCH 7, 15, 0
8569 SCRATCH 6, 14, 1
8570 SCRATCH 5, 13, 2
8571 pxor m1, m1
8572 .loop_y:
8573 xor xd, xd
8574 mova m0, m14 ; per-line working version of mx
8575 .loop_x:
8576 pcmpgtd m1, m0
8577 pandn m1, m0
8578 psrad m2, m0, 8 ; filter offset (unmasked)
8579 pcmpgtd m3, m13, m1
8580 pand m1, m3
8581 pandn m3, m13
8582 por m1, m3
8583 psubd m3, m0, m1 ; pshufb offset
8584 psrad m1, 14 ; clipped src_x offset
8585 psrad m3, 14 ; pshufb edge_emu offset
8586 pand m2, m11 ; filter offset (masked)
8587 ; load source pixels
8588 %if ARCH_X86_64
8589 movd r8d, m1
8590 pshuflw m1, m1, q3232
8591 movd r9d, m1
8592 punpckhqdq m1, m1
8593 movd r10d, m1
8594 psrlq m1, 32
8595 movd r11d, m1
8596 movu m4, [srcq+r8*2]
8597 movu m5, [srcq+r9*2]
8598 movu m6, [srcq+r10*2]
8599 movu m7, [srcq+r11*2]
8600 ; if no emulation is required, we don't need to shuffle or emulate edges
8601 packssdw m3, m3
8602 movq r11, m3
8603 test r11, r11
8604 jz .filter
8605 movsx r8, r11w
8606 sar r11, 16
8607 movsx r9, r11w
8608 sar r11, 16
8609 movsx r10, r11w
8610 sar r11, 16
8611 movu m1, [base+resize_shuf+8+r8*2]
8612 movu m3, [base+resize_shuf+8+r9*2]
8613 movu m8, [base+resize_shuf+8+r10*2]
8614 movu m9, [base+resize_shuf+8+r11*2]
8615 pshufb m4, m1
8616 pshufb m5, m3
8617 pshufb m6, m8
8618 pshufb m7, m9
8619 .filter:
8620 movd r8d, m2
8621 pshuflw m2, m2, q3232
8622 movd r9d, m2
8623 punpckhqdq m2, m2
8624 movd r10d, m2
8625 psrlq m2, 32
8626 movd r11d, m2
8627 movq m8, [base+resize_filter+r8*8]
8628 movq m2, [base+resize_filter+r9*8]
8629 pxor m9, m9
8630 punpcklbw m1, m9, m8
8631 punpcklbw m3, m9, m2
8632 psraw m1, 8
8633 psraw m3, 8
8634 movq m10, [base+resize_filter+r10*8]
8635 movq m2, [base+resize_filter+r11*8]
8636 punpcklbw m8, m9, m10
8637 punpcklbw m9, m2
8638 psraw m8, 8
8639 psraw m9, 8
8640 pmaddwd m4, m1
8641 pmaddwd m5, m3
8642 pmaddwd m6, m8
8643 pmaddwd m7, m9
8644 phaddd m4, m5
8645 %else
8646 movd r3, m1
8647 pshuflw m1, m1, q3232
8648 movd r1, m1
8649 punpckhqdq m1, m1
8650 movu m4, [srcq+r3*2]
8651 movu m5, [srcq+r1*2]
8652 movd r3, m1
8653 psrlq m1, 32
8654 movd r1, m1
8655 movu m6, [srcq+r3*2]
8656 movu m7, [srcq+r1*2]
8657 ; if no emulation is required, we don't need to shuffle or emulate edges
8658 pxor m1, m1
8659 pcmpeqb m1, m3
8660 pmovmskb r3d, m1
8661 cmp r3d, 0xffff
8662 je .filter
8663 movd r3, m3
8664 movu m1, [base+resize_shuf+8+r3*2]
8665 pshuflw m3, m3, q3232
8666 movd r1, m3
8667 pshufb m4, m1
8668 movu m1, [base+resize_shuf+8+r1*2]
8669 punpckhqdq m3, m3
8670 movd r3, m3
8671 pshufb m5, m1
8672 movu m1, [base+resize_shuf+8+r3*2]
8673 psrlq m3, 32
8674 movd r1, m3
8675 pshufb m6, m1
8676 movu m1, [base+resize_shuf+8+r1*2]
8677 pshufb m7, m1
8678 .filter:
8679 mova [esp+4*16], m6
8680 mova [esp+5*16], m7
8681 movd r3, m2
8682 pshuflw m2, m2, q3232
8683 movd r1, m2
8684 movq m6, [base+resize_filter+r3*8]
8685 movq m7, [base+resize_filter+r1*8]
8686 pxor m3, m3
8687 punpcklbw m1, m3, m6
8688 punpcklbw m3, m7
8689 psraw m1, 8
8690 psraw m3, 8
8691 pmaddwd m4, m1
8692 pmaddwd m5, m3
8693 punpckhqdq m2, m2
8694 movd r3, m2
8695 psrlq m2, 32
8696 movd r1, m2
8697 phaddd m4, m5
8698 movq m2, [base+resize_filter+r3*8]
8699 movq m5, [base+resize_filter+r1*8]
8700 mova m6, [esp+4*16]
8701 mova m7, [esp+5*16]
8702 pxor m3, m3
8703 punpcklbw m1, m3, m2
8704 punpcklbw m3, m5
8705 psraw m1, 8
8706 psraw m3, 8
8707 pmaddwd m6, m1
8708 pmaddwd m7, m3
8709 %endif
8710 phaddd m6, m7
8711 phaddd m4, m6
8712 pxor m1, m1
8713 psubd m2, m12, m4
8714 psrad m2, 7
8715 packssdw m2, m2
8716 pmaxsw m2, m1
8717 pminsw m2, [rsp+16*3*ARCH_X86_32]
8718 movq [dstq+xq*2], m2
8719 paddd m0, m15
8720 add xd, 4
8721 %if STACK_ALIGNMENT >= 16
8722 cmp xd, dst_wd
8723 %else
8724 cmp xd, dst_wm
8725 %endif
8726 jl .loop_x
8727 add dstq, dst_stridemp
8728 add srcq, src_stridemp
8729 dec hd
8730 jg .loop_y
8731 RET