Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/x86/mc16_avx512.asm

raw
1 ; Copyright © 2020, VideoLAN and dav1d authors
2 ; Copyright © 2020, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "src/asm/x86/x86inc.asm"
28
29 %if ARCH_X86_64
30
31 SECTION_RODATA 64
32
33 spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9
34 db 32, 33, 34, 35, 34, 35, 36, 37, 36, 37, 38, 39, 38, 39, 40, 41
35 spel_h_shufC: db 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15, 16, 17
36 db 40, 41, 42, 43, 42, 43, 44, 45, 44, 45, 46, 47, 46, 47, 48, 49
37 db 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23, 24, 25
38 db 48, 49, 50, 51, 50, 51, 52, 53, 52, 53, 54, 55, 54, 55, 56, 57
39 spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13
40 db 36, 37, 38, 39, 38, 39, 40, 41, 40, 41, 42, 43, 42, 43, 44, 45
41 spel_h_shufD: db 12, 13, 14, 15, 14, 15, 16, 17, 16, 17, 18, 19, 18, 19, 20, 21
42 db 44, 45, 46, 47, 46, 47, 48, 49, 48, 49, 50, 51, 50, 51, 52, 53
43 db 20, 21, 22, 23, 22, 23, 24, 25, 24, 25, 26, 27, 26, 27, 28, 29
44 db 52, 53, 54, 55, 54, 55, 56, 57, 56, 57, 58, 59, 58, 59, 60, 61
45 spel_v_shuf8: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
46 db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39
47 db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31
48 db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47
49 spel_v_shuf16: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39
50 db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47
51 db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55
52 db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63
53 prep_endA: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30
54 db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62
55 db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94
56 db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126
57 prep_endB: db 1, 2, 5, 6, 9, 10, 13, 14, 33, 34, 37, 38, 41, 42, 45, 46
58 db 17, 18, 21, 22, 25, 26, 29, 30, 49, 50, 53, 54, 57, 58, 61, 62
59 db 65, 66, 69, 70, 73, 74, 77, 78, 97, 98,101,102,105,106,109,110
60 db 81, 82, 85, 86, 89, 90, 93, 94,113,114,117,118,121,122,125,126
61 prep_endC: db 1, 2, 5, 6, 9, 10, 13, 14, 65, 66, 69, 70, 73, 74, 77, 78
62 db 17, 18, 21, 22, 25, 26, 29, 30, 81, 82, 85, 86, 89, 90, 93, 94
63 db 33, 34, 37, 38, 41, 42, 45, 46, 97, 98,101,102,105,106,109,110
64 db 49, 50, 53, 54, 57, 58, 61, 62,113,114,117,118,121,122,125,126
65 spel_shuf4a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30
66 db 17, 18, 33, 34, 21, 22, 37, 38, 25, 26, 41, 42, 29, 30, 45, 46
67 spel_shuf4b: db 18, 19, 33, 34, 22, 23, 37, 38, 26, 27, 41, 42, 30, 31, 45, 46
68 db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62
69 spel_shuf8a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30
70 db 17, 18, 65, 66, 21, 22, 69, 70, 25, 26, 73, 74, 29, 30, 77, 78
71 db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62
72 db 49, 50, 97, 98, 53, 54,101,102, 57, 58,105,106, 61, 62,109,110
73 spel_shuf8b: db 18, 19, 65, 66, 22, 23, 69, 70, 26, 27, 73, 74, 30, 31, 77, 78
74 db 65, 66, 81, 82, 69, 70, 85, 86, 73, 74, 89, 90, 77, 78, 93, 94
75 db 50, 51, 97, 98, 54, 55,101,102, 58, 59,105,106, 62, 63,109,110
76 db 97, 98,113,114,101,102,117,118,105,106,121,122,109,110,125,126
77 spel_shuf16: db 1, 2, 33, 34, 5, 6, 37, 38, 9, 10, 41, 42, 13, 14, 45, 46
78 db 17, 18, 49, 50, 21, 22, 53, 54, 25, 26, 57, 58, 29, 30, 61, 62
79 db 65, 66, 97, 98, 69, 70,101,102, 73, 74,105,106, 77, 78,109,110
80 db 81, 82,113,114, 85, 86,117,118, 89, 90,121,122, 93, 94,125,126
81 spel_shuf32: db 1, 2, 65, 66, 5, 6, 69, 70, 9, 10, 73, 74, 13, 14, 77, 78
82 db 17, 18, 81, 82, 21, 22, 85, 86, 25, 26, 89, 90, 29, 30, 93, 94
83 db 33, 34, 97, 98, 37, 38,101,102, 41, 42,105,106, 45, 46,109,110
84 db 49, 50,113,114, 53, 54,117,118, 57, 58,121,122, 61, 62,125,126
85 spel_h_shuf2b: db 1, 2, 17, 18, 5, 6, 21, 22, 17, 18, 33, 34, 21, 22, 37, 38
86 db 33, 34, 49, 50, 37, 38, 53, 54, 49, 50, 9, 10, 53, 54, 13, 14
87 db 9, 10, 25, 26, 13, 14, 29, 30, 25, 26, 41, 42, 29, 30, 45, 46
88 spel_shuf2: db 10, 11, 17, 18, 14, 15, 21, 22, 17, 18, 25, 26, 21, 22, 29, 30
89 spel_h_shuf2a: db 0, 1, 2, 3, 2, 3, 4, 5, 16, 17, 18, 19, 18, 19, 20, 21
90 db 4, 5, 6, 7, 6, 7, 8, 9, 20, 21, 22, 23, 22, 23, 24, 25
91 w_mask_end42x: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
92 db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
93 w_mask_end444: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
94 db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
95 db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94
96 db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126
97 w_mask_shuf4: db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30
98 db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62
99 db 64, 66, 72, 74, 68, 70, 76, 78, 80, 82, 88, 90, 84, 86, 92, 94
100 db 96, 98,104,106,100,102,108,110,112,114,120,122,116,118,124,126
101 w_mask_shuf8: db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30
102 db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62
103 db 64, 66, 80, 82, 68, 70, 84, 86, 72, 74, 88, 90, 76, 78, 92, 94
104 db 96, 98,112,114,100,102,116,118,104,106,120,122,108,110,124,126
105 w_mask_shuf16: db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46
106 db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62
107 db 64, 66, 96, 98, 68, 70,100,102, 72, 74,104,106, 76, 78,108,110
108 db 80, 82,112,114, 84, 86,116,118, 88, 90,120,122, 92, 94,124,126
109 warp8x8_permA: db 0, 1, 2, 3, 32, 33, 34, 35, 2, 3, 4, 5, 34, 35, 36, 37
110 db 4, 5, 6, 7, 36, 37, 38, 39, 6, 7, 8, 9, 38, 39, 40, 41
111 db 8, 9, 10, 11, 40, 41, 42, 43, 10, 11, 12, 13, 42, 43, 44, 45
112 db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49
113 warp8x8_permB: db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49
114 db 16, 17, 18, 19, 48, 49, 50, 51, 18, 19, 20, 21, 50, 51, 52, 53
115 db 20, 21, 22, 23, 52, 53, 54, 55, 22, 23, 24, 25, 54, 55, 56, 57
116 db 24, 25, 26, 27, 56, 57, 58, 59, 26, 27, 28, 29, 58, 59, 60, 61
117 warp8x8_end: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53
118 db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55
119 db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61
120 db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63
121 deint_q_shuf: ;dq 0, 2, 4, 6, 1, 3, 5, 7
122 pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7
123 dd 1
124 pw_2048: times 2 dw 2048
125 dd 3
126 pw_8192: times 2 dw 8192
127 avg_shift: dw 5, 5, 3, 3
128 pw_27615: times 2 dw 27615
129 pw_32766: times 2 dw 32766
130 warp8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13
131 warp8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15
132 warp_shift_h: db 11, 19, 11, 19, 43, 51, 43, 51, 13, 21, 13, 21, 45, 53, 45, 53
133 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
134 resize_permA: dd 0, 4, 8, 12, 1, 5, 9, 13, 16, 20, 24, 28, 17, 21, 25, 29
135 resize_permB: dd 2, 6, 10, 14, 3, 7, 11, 15, 18, 22, 26, 30, 19, 23, 27, 31
136 resize_permC: dq 0, 1, 4, 5, 8, 9, 12, 13
137 resize_permD: dq 2, 3, 6, 7, 10, 11, 14, 15
138 resize_permE: dq 0, 2, 4, 6
139 resize_shufA: db -1, 0, -1, 1, -1, 4, -1, 5, -1, 8, -1, 9, -1, 12, -1, 13
140 resize_shufB: db -1, 2, -1, 3, -1, 6, -1, 7, -1, 10, -1, 11, -1, 14, -1, 15
141 rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
142 resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7
143 db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15
144
145 prep_hv_shift: dq 6, 4
146 put_bilin_h_rnd: dw 8, 8, 10, 10
147 prep_mul: dw 16, 16, 4, 4
148 put_8tap_h_rnd: dd 34, 40
149 prep_8tap_rnd: dd 128 - (8192 << 8)
150 warp_8x8_rnd_h: dd 512, 2048
151 warp_8x8_rnd_v: dd 262144, 65536
152 warp_8x8t_rnd_v: dd 16384 - (8192 << 15)
153 avg_round: dw -16400, -16400, -16388, -16388
154 w_avg_round: dd 128 + (8192 << 4), 32 + (8192 << 4)
155 mask_round: dd 512 + (8192 << 6), 128 + (8192 << 6)
156 w_mask_round: dd 128, 64
157 bidir_shift: dw 6, 6, 4, 4
158
159 pb_64: times 4 db 64
160 pw_m512: times 2 dw -512
161 pw_2: times 2 dw 2
162 pw_64: times 2 dw 64
163 pd_32: dd 32
164 pd_63: dd 63
165 pd_128: dd 128
166 pd_640: dd 640
167 pd_2176: dd 2176
168 pd_16384: dd 16384
169 pd_0_4: dd 0, 4
170
171 %define pw_16 prep_mul
172 %define pd_512 warp_8x8_rnd_h
173
174 %macro BASE_JMP_TABLE 3-*
175 %xdefine %1_%2_table (%%table - %3)
176 %xdefine %%base %1_%2
177 %%table:
178 %rep %0 - 2
179 dw %%base %+ _w%3 - %%base
180 %rotate 1
181 %endrep
182 %endmacro
183
184 %macro HV_JMP_TABLE 5-*
185 %xdefine %%prefix mangle(private_prefix %+ _%1_%2_16bpc_%3)
186 %xdefine %%base %1_%3
187 %assign %%types %4
188 %if %%types & 1
189 %xdefine %1_%2_h_%3_table (%%h - %5)
190 %%h:
191 %rep %0 - 4
192 dw %%prefix %+ .h_w%5 - %%base
193 %rotate 1
194 %endrep
195 %rotate 4
196 %endif
197 %if %%types & 2
198 %xdefine %1_%2_v_%3_table (%%v - %5)
199 %%v:
200 %rep %0 - 4
201 dw %%prefix %+ .v_w%5 - %%base
202 %rotate 1
203 %endrep
204 %rotate 4
205 %endif
206 %if %%types & 4
207 %xdefine %1_%2_hv_%3_table (%%hv - %5)
208 %%hv:
209 %rep %0 - 4
210 dw %%prefix %+ .hv_w%5 - %%base
211 %rotate 1
212 %endrep
213 %endif
214 %endmacro
215
216 %macro BIDIR_JMP_TABLE 2-*
217 %xdefine %1_%2_table (%%table - 2*%3)
218 %xdefine %%base %1_%2_table
219 %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2)
220 %%table:
221 %rep %0 - 2
222 dd %%prefix %+ .w%3 - %%base
223 %rotate 1
224 %endrep
225 %endmacro
226
227 %xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_16bpc_avx512icl.put)
228 %xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_16bpc_avx512icl.prep)
229
230 BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128
231 BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128
232 BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128
233 BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128
234 BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128
235 BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128
236 BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32
237 BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32
238 BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128
239 BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128
240 BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128
241 HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128
242 HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128
243 HV_JMP_TABLE put, 8tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128
244 HV_JMP_TABLE prep, 8tap, avx512icl, 2, 4, 8, 16, 32, 64, 128
245
246 %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
247
248 cextern mc_subpel_filters
249 %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
250
251 cextern mc_warp_filter
252 cextern obmc_masks_avx2
253 cextern resize_filter
254
255 SECTION .text
256
257 %if WIN64
258 DECLARE_REG_TMP 4
259 %else
260 DECLARE_REG_TMP 8
261 %endif
262
263 INIT_ZMM avx512icl
264 cglobal put_bilin_16bpc, 4, 8, 13, dst, ds, src, ss, w, h, mxy
265 mov mxyd, r6m ; mx
266 lea r7, [put_avx512icl]
267 tzcnt t0d, wm
268 movifnidn hd, hm
269 test mxyd, mxyd
270 jnz .h
271 mov mxyd, r7m ; my
272 test mxyd, mxyd
273 jnz .v
274 .put:
275 movzx t0d, word [r7+t0*2+table_offset(put,)]
276 add t0, r7
277 jmp t0
278 .put_w2:
279 mov r6d, [srcq+ssq*0]
280 mov r7d, [srcq+ssq*1]
281 lea srcq, [srcq+ssq*2]
282 mov [dstq+dsq*0], r6d
283 mov [dstq+dsq*1], r7d
284 lea dstq, [dstq+dsq*2]
285 sub hd, 2
286 jg .put_w2
287 RET
288 .put_w4:
289 mov r6, [srcq+ssq*0]
290 mov r7, [srcq+ssq*1]
291 lea srcq, [srcq+ssq*2]
292 mov [dstq+dsq*0], r6
293 mov [dstq+dsq*1], r7
294 lea dstq, [dstq+dsq*2]
295 sub hd, 2
296 jg .put_w4
297 RET
298 .put_w8:
299 movu xmm0, [srcq+ssq*0]
300 movu xmm1, [srcq+ssq*1]
301 lea srcq, [srcq+ssq*2]
302 mova [dstq+dsq*0], xmm0
303 mova [dstq+dsq*1], xmm1
304 lea dstq, [dstq+dsq*2]
305 sub hd, 2
306 jg .put_w8
307 RET
308 .put_w16:
309 movu ym0, [srcq+ssq*0]
310 movu ym1, [srcq+ssq*1]
311 lea srcq, [srcq+ssq*2]
312 mova [dstq+dsq*0], ym0
313 mova [dstq+dsq*1], ym1
314 lea dstq, [dstq+dsq*2]
315 sub hd, 2
316 jg .put_w16
317 RET
318 .put_w32:
319 movu m0, [srcq+ssq*0]
320 movu m1, [srcq+ssq*1]
321 lea srcq, [srcq+ssq*2]
322 mova [dstq+dsq*0], m0
323 mova [dstq+dsq*1], m1
324 lea dstq, [dstq+dsq*2]
325 sub hd, 2
326 jg .put_w32
327 RET
328 .put_w64:
329 movu m0, [srcq+ssq*0+64*0]
330 movu m1, [srcq+ssq*0+64*1]
331 movu m2, [srcq+ssq*1+64*0]
332 movu m3, [srcq+ssq*1+64*1]
333 lea srcq, [srcq+ssq*2]
334 mova [dstq+dsq*0+64*0], m0
335 mova [dstq+dsq*0+64*1], m1
336 mova [dstq+dsq*1+64*0], m2
337 mova [dstq+dsq*1+64*1], m3
338 lea dstq, [dstq+dsq*2]
339 sub hd, 2
340 jg .put_w64
341 RET
342 .put_w128:
343 movu m0, [srcq+64*0]
344 movu m1, [srcq+64*1]
345 movu m2, [srcq+64*2]
346 movu m3, [srcq+64*3]
347 add srcq, ssq
348 mova [dstq+64*0], m0
349 mova [dstq+64*1], m1
350 mova [dstq+64*2], m2
351 mova [dstq+64*3], m3
352 add dstq, dsq
353 dec hd
354 jg .put_w128
355 RET
356 .h:
357 vpbroadcastw m5, mxyd
358 mov mxyd, r7m ; my
359 vpbroadcastd m4, [pw_16]
360 psubw m4, m5
361 test mxyd, mxyd
362 jnz .hv
363 ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v
364 movzx t0d, word [r7+t0*2+table_offset(put, _bilin_h)]
365 mov r6d, r8m ; bitdepth_max
366 add t0, r7
367 shr r6d, 11
368 vpbroadcastd m6, [r7-put_avx512icl+put_bilin_h_rnd+r6*4]
369 jmp t0
370 .h_w2:
371 movq xmm1, [srcq+ssq*0]
372 movhps xmm1, [srcq+ssq*1]
373 lea srcq, [srcq+ssq*2]
374 pmullw xmm0, xmm1, xm4
375 psrlq xmm1, 16
376 pmullw xmm1, xm5
377 paddw xmm0, xm6
378 paddw xmm0, xmm1
379 psrlw xmm0, 4
380 movd [dstq+dsq*0], xmm0
381 pextrd [dstq+dsq*1], xmm0, 2
382 lea dstq, [dstq+dsq*2]
383 sub hd, 2
384 jg .h_w2
385 RET
386 .h_w4:
387 movq xmm0, [srcq+ssq*0+0]
388 movhps xmm0, [srcq+ssq*1+0]
389 movq xmm1, [srcq+ssq*0+2]
390 movhps xmm1, [srcq+ssq*1+2]
391 lea srcq, [srcq+ssq*2]
392 pmullw xmm0, xm4
393 pmullw xmm1, xm5
394 paddw xmm0, xm6
395 paddw xmm0, xmm1
396 psrlw xmm0, 4
397 movq [dstq+dsq*0], xmm0
398 movhps [dstq+dsq*1], xmm0
399 lea dstq, [dstq+dsq*2]
400 sub hd, 2
401 jg .h_w4
402 RET
403 .h_w8:
404 movu xm0, [srcq+ssq*0+0]
405 vinserti32x4 ym0, [srcq+ssq*1+0], 1
406 movu xm1, [srcq+ssq*0+2]
407 vinserti32x4 ym1, [srcq+ssq*1+2], 1
408 lea srcq, [srcq+ssq*2]
409 pmullw ym0, ym4
410 pmullw ym1, ym5
411 paddw ym0, ym6
412 paddw ym0, ym1
413 psrlw ym0, 4
414 mova [dstq+dsq*0], xm0
415 vextracti32x4 [dstq+dsq*1], ym0, 1
416 lea dstq, [dstq+dsq*2]
417 sub hd, 2
418 jg .h_w8
419 RET
420 .h_w16:
421 movu ym0, [srcq+ssq*0+0]
422 vinserti32x8 m0, [srcq+ssq*1+0], 1
423 movu ym1, [srcq+ssq*0+2]
424 vinserti32x8 m1, [srcq+ssq*1+2], 1
425 lea srcq, [srcq+ssq*2]
426 pmullw m0, m4
427 pmullw m1, m5
428 paddw m0, m6
429 paddw m0, m1
430 psrlw m0, 4
431 mova [dstq+dsq*0], ym0
432 vextracti32x8 [dstq+dsq*1], m0, 1
433 lea dstq, [dstq+dsq*2]
434 sub hd, 2
435 jg .h_w16
436 RET
437 .h_w32:
438 pmullw m0, m4, [srcq+ssq*0+0]
439 pmullw m2, m5, [srcq+ssq*0+2]
440 pmullw m1, m4, [srcq+ssq*1+0]
441 pmullw m3, m5, [srcq+ssq*1+2]
442 lea srcq, [srcq+ssq*2]
443 paddw m0, m6
444 paddw m1, m6
445 paddw m0, m2
446 paddw m1, m3
447 psrlw m0, 4
448 psrlw m1, 4
449 mova [dstq+dsq*0], m0
450 mova [dstq+dsq*1], m1
451 lea dstq, [dstq+dsq*2]
452 sub hd, 2
453 jg .h_w32
454 RET
455 .h_w64:
456 pmullw m0, m4, [srcq+64*0+0]
457 pmullw m2, m5, [srcq+64*0+2]
458 pmullw m1, m4, [srcq+64*1+0]
459 pmullw m3, m5, [srcq+64*1+2]
460 add srcq, ssq
461 paddw m0, m6
462 paddw m1, m6
463 paddw m0, m2
464 paddw m1, m3
465 psrlw m0, 4
466 psrlw m1, 4
467 mova [dstq+64*0], m0
468 mova [dstq+64*1], m1
469 add dstq, dsq
470 dec hd
471 jg .h_w64
472 RET
473 .h_w128:
474 pmullw m0, m4, [srcq+64*0+0]
475 pmullw m7, m5, [srcq+64*0+2]
476 pmullw m1, m4, [srcq+64*1+0]
477 pmullw m8, m5, [srcq+64*1+2]
478 pmullw m2, m4, [srcq+64*2+0]
479 pmullw m9, m5, [srcq+64*2+2]
480 pmullw m3, m4, [srcq+64*3+0]
481 pmullw m10, m5, [srcq+64*3+2]
482 add srcq, ssq
483 REPX {paddw x, m6}, m0, m1, m2, m3
484 paddw m0, m7
485 paddw m1, m8
486 paddw m2, m9
487 paddw m3, m10
488 REPX {psrlw x, 4}, m0, m1, m2, m3
489 mova [dstq+64*0], m0
490 mova [dstq+64*1], m1
491 mova [dstq+64*2], m2
492 mova [dstq+64*3], m3
493 add dstq, dsq
494 dec hd
495 jg .h_w128
496 RET
497 .v:
498 movzx t0d, word [r7+t0*2+table_offset(put, _bilin_v)]
499 shl mxyd, 11
500 vpbroadcastw m8, mxyd
501 add t0, r7
502 jmp t0
503 .v_w2:
504 movd xmm0, [srcq+ssq*0]
505 .v_w2_loop:
506 movd xmm1, [srcq+ssq*1]
507 lea srcq, [srcq+ssq*2]
508 punpckldq xmm2, xmm0, xmm1
509 movd xmm0, [srcq+ssq*0]
510 punpckldq xmm1, xmm0
511 psubw xmm1, xmm2
512 pmulhrsw xmm1, xm8
513 paddw xmm1, xmm2
514 movd [dstq+dsq*0], xmm1
515 pextrd [dstq+dsq*1], xmm1, 1
516 lea dstq, [dstq+dsq*2]
517 sub hd, 2
518 jg .v_w2_loop
519 RET
520 .v_w4:
521 movq xmm0, [srcq+ssq*0]
522 .v_w4_loop:
523 movq xmm1, [srcq+ssq*1]
524 lea srcq, [srcq+ssq*2]
525 punpcklqdq xmm2, xmm0, xmm1
526 movq xmm0, [srcq+ssq*0]
527 punpcklqdq xmm1, xmm0
528 psubw xmm1, xmm2
529 pmulhrsw xmm1, xm8
530 paddw xmm1, xmm2
531 movq [dstq+dsq*0], xmm1
532 movhps [dstq+dsq*1], xmm1
533 lea dstq, [dstq+dsq*2]
534 sub hd, 2
535 jg .v_w4_loop
536 RET
537 .v_w8:
538 movu xmm0, [srcq+ssq*0]
539 .v_w8_loop:
540 vbroadcasti128 ymm1, [srcq+ssq*1]
541 lea srcq, [srcq+ssq*2]
542 vpblendd ymm2, ymm0, ymm1, 0xf0
543 vbroadcasti128 ymm0, [srcq+ssq*0]
544 vpblendd ymm1, ymm0, 0xf0
545 psubw ymm1, ymm2
546 pmulhrsw ymm1, ym8
547 paddw ymm1, ymm2
548 mova [dstq+dsq*0], xmm1
549 vextracti128 [dstq+dsq*1], ymm1, 1
550 lea dstq, [dstq+dsq*2]
551 sub hd, 2
552 jg .v_w8_loop
553 vzeroupper
554 RET
555 .v_w16:
556 movu ym0, [srcq+ssq*0]
557 .v_w16_loop:
558 movu ym3, [srcq+ssq*1]
559 lea srcq, [srcq+ssq*2]
560 psubw ym1, ym3, ym0
561 pmulhrsw ym1, ym8
562 paddw ym1, ym0
563 movu ym0, [srcq+ssq*0]
564 psubw ym2, ym0, ym3
565 pmulhrsw ym2, ym8
566 paddw ym2, ym3
567 mova [dstq+dsq*0], ym1
568 mova [dstq+dsq*1], ym2
569 lea dstq, [dstq+dsq*2]
570 sub hd, 2
571 jg .v_w16_loop
572 RET
573 .v_w32:
574 movu m0, [srcq+ssq*0]
575 .v_w32_loop:
576 movu m3, [srcq+ssq*1]
577 lea srcq, [srcq+ssq*2]
578 psubw m1, m3, m0
579 pmulhrsw m1, m8
580 paddw m1, m0
581 movu m0, [srcq+ssq*0]
582 psubw m2, m0, m3
583 pmulhrsw m2, m8
584 paddw m2, m3
585 mova [dstq+dsq*0], m1
586 mova [dstq+dsq*1], m2
587 lea dstq, [dstq+dsq*2]
588 sub hd, 2
589 jg .v_w32_loop
590 RET
591 .v_w64:
592 movu m0, [srcq+ssq*0+64*0]
593 movu m1, [srcq+ssq*0+64*1]
594 .v_w64_loop:
595 movu m2, [srcq+ssq*1+64*0]
596 movu m3, [srcq+ssq*1+64*1]
597 lea srcq, [srcq+ssq*2]
598 psubw m4, m2, m0
599 pmulhrsw m4, m8
600 paddw m4, m0
601 movu m0, [srcq+ssq*0+64*0]
602 psubw m5, m3, m1
603 pmulhrsw m5, m8
604 paddw m5, m1
605 movu m1, [srcq+ssq*0+64*1]
606 psubw m6, m0, m2
607 pmulhrsw m6, m8
608 psubw m7, m1, m3
609 pmulhrsw m7, m8
610 mova [dstq+dsq*0+64*0], m4
611 mova [dstq+dsq*0+64*1], m5
612 paddw m6, m2
613 paddw m7, m3
614 mova [dstq+dsq*1+64*0], m6
615 mova [dstq+dsq*1+64*1], m7
616 lea dstq, [dstq+dsq*2]
617 sub hd, 2
618 jg .v_w64_loop
619 RET
620 .v_w128:
621 movu m0, [srcq+ssq*0+64*0]
622 movu m1, [srcq+ssq*0+64*1]
623 movu m2, [srcq+ssq*0+64*2]
624 movu m3, [srcq+ssq*0+64*3]
625 .v_w128_loop:
626 movu m4, [srcq+ssq*1+64*0]
627 movu m5, [srcq+ssq*1+64*1]
628 movu m6, [srcq+ssq*1+64*2]
629 movu m7, [srcq+ssq*1+64*3]
630 lea srcq, [srcq+ssq*2]
631 psubw m9, m4, m0
632 pmulhrsw m9, m8
633 paddw m9, m0
634 movu m0, [srcq+ssq*0+64*0]
635 psubw m10, m5, m1
636 pmulhrsw m10, m8
637 paddw m10, m1
638 movu m1, [srcq+ssq*0+64*1]
639 psubw m11, m6, m2
640 pmulhrsw m11, m8
641 paddw m11, m2
642 movu m2, [srcq+ssq*0+64*2]
643 psubw m12, m7, m3
644 pmulhrsw m12, m8
645 paddw m12, m3
646 movu m3, [srcq+ssq*0+64*3]
647 mova [dstq+dsq*0+64*0], m9
648 psubw m9, m0, m4
649 pmulhrsw m9, m8
650 mova [dstq+dsq*0+64*1], m10
651 psubw m10, m1, m5
652 pmulhrsw m10, m8
653 mova [dstq+dsq*0+64*2], m11
654 psubw m11, m2, m6
655 pmulhrsw m11, m8
656 mova [dstq+dsq*0+64*3], m12
657 psubw m12, m3, m7
658 pmulhrsw m12, m8
659 paddw m9, m4
660 paddw m10, m5
661 mova [dstq+dsq*1+64*0], m9
662 mova [dstq+dsq*1+64*1], m10
663 paddw m11, m6
664 paddw m12, m7
665 mova [dstq+dsq*1+64*2], m11
666 mova [dstq+dsq*1+64*3], m12
667 lea dstq, [dstq+dsq*2]
668 sub hd, 2
669 jg .v_w128_loop
670 RET
671 .hv:
672 movzx t0d, word [r7+t0*2+table_offset(put, _bilin_hv)]
673 shl mxyd, 11
674 vpbroadcastd m6, [pw_2]
675 vpbroadcastw m7, mxyd
676 vpbroadcastd m8, [pw_8192]
677 add t0, r7
678 test dword r8m, 0x800
679 jnz .hv_12bpc
680 psllw m4, 2
681 psllw m5, 2
682 vpbroadcastd m8, [pw_2048]
683 .hv_12bpc:
684 jmp t0
685 .hv_w2:
686 vpbroadcastq xmm1, [srcq+ssq*0]
687 pmullw xmm0, xmm1, xm4
688 psrlq xmm1, 16
689 pmullw xmm1, xm5
690 paddw xmm0, xm6
691 paddw xmm0, xmm1
692 psrlw xmm0, 2
693 .hv_w2_loop:
694 movq xmm2, [srcq+ssq*1]
695 lea srcq, [srcq+ssq*2]
696 movhps xmm2, [srcq+ssq*0]
697 pmullw xmm1, xmm2, xm4
698 psrlq xmm2, 16
699 pmullw xmm2, xm5
700 paddw xmm1, xm6
701 paddw xmm1, xmm2
702 psrlw xmm1, 2 ; 1 _ 2 _
703 shufpd xmm2, xmm0, xmm1, 0x01 ; 0 _ 1 _
704 mova xmm0, xmm1
705 psubw xmm1, xmm2
706 paddw xmm1, xmm1
707 pmulhw xmm1, xm7
708 paddw xmm1, xmm2
709 pmulhrsw xmm1, xm8
710 movd [dstq+dsq*0], xmm1
711 pextrd [dstq+dsq*1], xmm1, 2
712 lea dstq, [dstq+dsq*2]
713 sub hd, 2
714 jg .hv_w2_loop
715 RET
716 .hv_w4:
717 pmullw xmm0, xm4, [srcq+ssq*0-8]
718 pmullw xmm1, xm5, [srcq+ssq*0-6]
719 paddw xmm0, xm6
720 paddw xmm0, xmm1
721 psrlw xmm0, 2
722 .hv_w4_loop:
723 movq xmm1, [srcq+ssq*1+0]
724 movq xmm2, [srcq+ssq*1+2]
725 lea srcq, [srcq+ssq*2]
726 movhps xmm1, [srcq+ssq*0+0]
727 movhps xmm2, [srcq+ssq*0+2]
728 pmullw xmm1, xm4
729 pmullw xmm2, xm5
730 paddw xmm1, xm6
731 paddw xmm1, xmm2
732 psrlw xmm1, 2 ; 1 2
733 shufpd xmm2, xmm0, xmm1, 0x01 ; 0 1
734 mova xmm0, xmm1
735 psubw xmm1, xmm2
736 paddw xmm1, xmm1
737 pmulhw xmm1, xm7
738 paddw xmm1, xmm2
739 pmulhrsw xmm1, xm8
740 movq [dstq+dsq*0], xmm1
741 movhps [dstq+dsq*1], xmm1
742 lea dstq, [dstq+dsq*2]
743 sub hd, 2
744 jg .hv_w4_loop
745 RET
746 .hv_w8:
747 pmullw xmm0, xm4, [srcq+ssq*0+0]
748 pmullw xmm1, xm5, [srcq+ssq*0+2]
749 paddw xmm0, xm6
750 paddw xmm0, xmm1
751 psrlw xmm0, 2
752 vinserti32x4 ym0, xmm0, 1
753 .hv_w8_loop:
754 movu xm1, [srcq+ssq*1+0]
755 movu xm2, [srcq+ssq*1+2]
756 lea srcq, [srcq+ssq*2]
757 vinserti32x4 ym1, [srcq+ssq*0+0], 1
758 vinserti32x4 ym2, [srcq+ssq*0+2], 1
759 pmullw ym1, ym4
760 pmullw ym2, ym5
761 paddw ym1, ym6
762 paddw ym1, ym2
763 psrlw ym1, 2 ; 1 2
764 vshufi32x4 ym2, ym0, ym1, 0x01 ; 0 1
765 mova ym0, ym1
766 psubw ym1, ym2
767 paddw ym1, ym1
768 pmulhw ym1, ym7
769 paddw ym1, ym2
770 pmulhrsw ym1, ym8
771 mova [dstq+dsq*0], xm1
772 vextracti32x4 [dstq+dsq*1], ym1, 1
773 lea dstq, [dstq+dsq*2]
774 sub hd, 2
775 jg .hv_w8_loop
776 RET
777 .hv_w16:
778 pmullw ym0, ym4, [srcq+ssq*0+0]
779 pmullw ym1, ym5, [srcq+ssq*0+2]
780 paddw ym0, ym6
781 paddw ym0, ym1
782 psrlw ym0, 2
783 vinserti32x8 m0, ym0, 1
784 .hv_w16_loop:
785 movu ym1, [srcq+ssq*1+0]
786 movu ym2, [srcq+ssq*1+2]
787 lea srcq, [srcq+ssq*2]
788 vinserti32x8 m1, [srcq+ssq*0+0], 1
789 vinserti32x8 m2, [srcq+ssq*0+2], 1
790 pmullw m1, m4
791 pmullw m2, m5
792 paddw m1, m6
793 paddw m1, m2
794 psrlw m1, 2 ; 1 2
795 vshufi32x4 m2, m0, m1, q1032 ; 0 1
796 mova m0, m1
797 psubw m1, m2
798 paddw m1, m1
799 pmulhw m1, m7
800 paddw m1, m2
801 pmulhrsw m1, m8
802 mova [dstq+dsq*0], ym1
803 vextracti32x8 [dstq+dsq*1], m1, 1
804 lea dstq, [dstq+dsq*2]
805 sub hd, 2
806 jg .hv_w16_loop
807 RET
808 .hv_w32:
809 .hv_w64:
810 .hv_w128:
811 movifnidn wd, wm
812 lea r6d, [hq+wq*8-256]
813 mov r4, srcq
814 mov r7, dstq
815 .hv_w32_loop0:
816 pmullw m0, m4, [srcq+ssq*0+0]
817 pmullw m1, m5, [srcq+ssq*0+2]
818 paddw m0, m6
819 paddw m0, m1
820 psrlw m0, 2
821 .hv_w32_loop:
822 pmullw m3, m4, [srcq+ssq*1+0]
823 pmullw m1, m5, [srcq+ssq*1+2]
824 lea srcq, [srcq+ssq*2]
825 paddw m3, m6
826 paddw m3, m1
827 psrlw m3, 2
828 psubw m1, m3, m0
829 paddw m1, m1
830 pmulhw m1, m7
831 paddw m1, m0
832 pmullw m0, m4, [srcq+ssq*0+0]
833 pmullw m2, m5, [srcq+ssq*0+2]
834 paddw m0, m6
835 paddw m0, m2
836 psrlw m0, 2
837 psubw m2, m0, m3
838 paddw m2, m2
839 pmulhw m2, m7
840 paddw m2, m3
841 pmulhrsw m1, m8
842 pmulhrsw m2, m8
843 mova [dstq+dsq*0], m1
844 mova [dstq+dsq*1], m2
845 lea dstq, [dstq+dsq*2]
846 sub hd, 2
847 jg .hv_w32_loop
848 add r4, 64
849 add r7, 64
850 movzx hd, r6b
851 mov srcq, r4
852 mov dstq, r7
853 sub r6d, 1<<8
854 jg .hv_w32_loop0
855 RET
856
857 cglobal prep_bilin_16bpc, 3, 7, 16, tmp, src, stride, w, h, mxy, stride3
858 movifnidn mxyd, r5m ; mx
859 lea r6, [prep_avx512icl]
860 tzcnt wd, wm
861 movifnidn hd, hm
862 test mxyd, mxyd
863 jnz .h
864 mov mxyd, r6m ; my
865 test mxyd, mxyd
866 jnz .v
867 .prep:
868 movzx wd, word [r6+wq*2+table_offset(prep,)]
869 mov r5d, r7m ; bitdepth_max
870 vpbroadcastd m5, [r6-prep_avx512icl+pw_8192]
871 add wq, r6
872 shr r5d, 11
873 vpbroadcastd m4, [r6-prep_avx512icl+prep_mul+r5*4]
874 lea stride3q, [strideq*3]
875 jmp wq
876 .prep_w4:
877 movq xmm0, [srcq+strideq*0]
878 movhps xmm0, [srcq+strideq*1]
879 vpbroadcastq ymm1, [srcq+strideq*2]
880 vpbroadcastq ymm2, [srcq+stride3q ]
881 lea srcq, [srcq+strideq*4]
882 vpblendd ymm0, ymm1, 0x30
883 vpblendd ymm0, ymm2, 0xc0
884 pmullw ymm0, ym4
885 psubw ymm0, ym5
886 mova [tmpq], ymm0
887 add tmpq, 32
888 sub hd, 4
889 jg .prep_w4
890 vzeroupper
891 RET
892 .prep_w8:
893 movu xm0, [srcq+strideq*0]
894 vinserti32x4 ym0, [srcq+strideq*1], 1
895 vinserti32x4 m0, [srcq+strideq*2], 2
896 vinserti32x4 m0, [srcq+stride3q ], 3
897 lea srcq, [srcq+strideq*4]
898 pmullw m0, m4
899 psubw m0, m5
900 mova [tmpq], m0
901 add tmpq, 64
902 sub hd, 4
903 jg .prep_w8
904 RET
905 .prep_w16:
906 movu ym0, [srcq+strideq*0]
907 vinserti32x8 m0, [srcq+strideq*1], 1
908 movu ym1, [srcq+strideq*2]
909 vinserti32x8 m1, [srcq+stride3q ], 1
910 lea srcq, [srcq+strideq*4]
911 pmullw m0, m4
912 pmullw m1, m4
913 psubw m0, m5
914 psubw m1, m5
915 mova [tmpq+64*0], m0
916 mova [tmpq+64*1], m1
917 add tmpq, 64*2
918 sub hd, 4
919 jg .prep_w16
920 RET
921 .prep_w32:
922 pmullw m0, m4, [srcq+strideq*0]
923 pmullw m1, m4, [srcq+strideq*1]
924 pmullw m2, m4, [srcq+strideq*2]
925 pmullw m3, m4, [srcq+stride3q ]
926 lea srcq, [srcq+strideq*4]
927 REPX {psubw x, m5}, m0, m1, m2, m3
928 mova [tmpq+64*0], m0
929 mova [tmpq+64*1], m1
930 mova [tmpq+64*2], m2
931 mova [tmpq+64*3], m3
932 add tmpq, 64*4
933 sub hd, 4
934 jg .prep_w32
935 RET
936 .prep_w64:
937 pmullw m0, m4, [srcq+strideq*0+64*0]
938 pmullw m1, m4, [srcq+strideq*0+64*1]
939 pmullw m2, m4, [srcq+strideq*1+64*0]
940 pmullw m3, m4, [srcq+strideq*1+64*1]
941 lea srcq, [srcq+strideq*2]
942 REPX {psubw x, m5}, m0, m1, m2, m3
943 mova [tmpq+64*0], m0
944 mova [tmpq+64*1], m1
945 mova [tmpq+64*2], m2
946 mova [tmpq+64*3], m3
947 add tmpq, 64*4
948 sub hd, 2
949 jg .prep_w64
950 RET
951 .prep_w128:
952 pmullw m0, m4, [srcq+64*0]
953 pmullw m1, m4, [srcq+64*1]
954 pmullw m2, m4, [srcq+64*2]
955 pmullw m3, m4, [srcq+64*3]
956 add srcq, strideq
957 REPX {psubw x, m5}, m0, m1, m2, m3
958 mova [tmpq+64*0], m0
959 mova [tmpq+64*1], m1
960 mova [tmpq+64*2], m2
961 mova [tmpq+64*3], m3
962 add tmpq, 64*4
963 dec hd
964 jg .prep_w128
965 RET
966 .h:
967 vpbroadcastw m5, mxyd
968 mov mxyd, r6m ; my
969 vpbroadcastd m4, [pw_16]
970 vpbroadcastd m6, [pw_32766]
971 psubw m4, m5
972 test dword r7m, 0x800
973 jnz .h_12bpc
974 psllw m4, 2
975 psllw m5, 2
976 .h_12bpc:
977 test mxyd, mxyd
978 jnz .hv
979 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)]
980 add wq, r6
981 lea stride3q, [strideq*3]
982 jmp wq
983 .h_w4:
984 movu xm1, [srcq+strideq*0]
985 vinserti32x4 ym1, [srcq+strideq*2], 1
986 movu xm2, [srcq+strideq*1]
987 vinserti32x4 ym2, [srcq+stride3q ], 1
988 lea srcq, [srcq+strideq*4]
989 punpcklqdq ym0, ym1, ym2
990 psrldq ym1, 2
991 psrldq ym2, 2
992 pmullw ym0, ym4
993 punpcklqdq ym1, ym2
994 pmullw ym1, ym5
995 psubw ym0, ym6
996 paddw ym0, ym1
997 psraw ym0, 2
998 mova [tmpq], ym0
999 add tmpq, 32
1000 sub hd, 4
1001 jg .h_w4
1002 RET
1003 .h_w8:
1004 movu xm0, [srcq+strideq*0+0]
1005 movu xm1, [srcq+strideq*0+2]
1006 vinserti32x4 ym0, [srcq+strideq*1+0], 1
1007 vinserti32x4 ym1, [srcq+strideq*1+2], 1
1008 vinserti32x4 m0, [srcq+strideq*2+0], 2
1009 vinserti32x4 m1, [srcq+strideq*2+2], 2
1010 vinserti32x4 m0, [srcq+stride3q +0], 3
1011 vinserti32x4 m1, [srcq+stride3q +2], 3
1012 lea srcq, [srcq+strideq*4]
1013 pmullw m0, m4
1014 pmullw m1, m5
1015 psubw m0, m6
1016 paddw m0, m1
1017 psraw m0, 2
1018 mova [tmpq], m0
1019 add tmpq, 64
1020 sub hd, 4
1021 jg .h_w8
1022 RET
1023 .h_w16:
1024 movu ym0, [srcq+strideq*0+0]
1025 vinserti32x8 m0, [srcq+strideq*1+0], 1
1026 movu ym1, [srcq+strideq*0+2]
1027 vinserti32x8 m1, [srcq+strideq*1+2], 1
1028 lea srcq, [srcq+strideq*2]
1029 pmullw m0, m4
1030 pmullw m1, m5
1031 psubw m0, m6
1032 paddw m0, m1
1033 psraw m0, 2
1034 mova [tmpq], m0
1035 add tmpq, 64
1036 sub hd, 2
1037 jg .h_w16
1038 RET
1039 .h_w32:
1040 pmullw m0, m4, [srcq+strideq*0+0]
1041 pmullw m2, m5, [srcq+strideq*0+2]
1042 pmullw m1, m4, [srcq+strideq*1+0]
1043 pmullw m3, m5, [srcq+strideq*1+2]
1044 lea srcq, [srcq+strideq*2]
1045 psubw m0, m6
1046 psubw m1, m6
1047 paddw m0, m2
1048 paddw m1, m3
1049 psraw m0, 2
1050 psraw m1, 2
1051 mova [tmpq+64*0], m0
1052 mova [tmpq+64*1], m1
1053 add tmpq, 64*2
1054 sub hd, 2
1055 jg .h_w32
1056 RET
1057 .h_w64:
1058 pmullw m0, m4, [srcq+ 0]
1059 pmullw m2, m5, [srcq+ 2]
1060 pmullw m1, m4, [srcq+64]
1061 pmullw m3, m5, [srcq+66]
1062 add srcq, strideq
1063 psubw m0, m6
1064 psubw m1, m6
1065 paddw m0, m2
1066 paddw m1, m3
1067 psraw m0, 2
1068 psraw m1, 2
1069 mova [tmpq+64*0], m0
1070 mova [tmpq+64*1], m1
1071 add tmpq, 64*2
1072 dec hd
1073 jg .h_w64
1074 RET
1075 .h_w128:
1076 pmullw m0, m4, [srcq+ 0]
1077 pmullw m7, m5, [srcq+ 2]
1078 pmullw m1, m4, [srcq+ 64]
1079 pmullw m8, m5, [srcq+ 66]
1080 pmullw m2, m4, [srcq+128]
1081 pmullw m9, m5, [srcq+130]
1082 pmullw m3, m4, [srcq+192]
1083 pmullw m10, m5, [srcq+194]
1084 add srcq, strideq
1085 REPX {psubw x, m6}, m0, m1, m2, m3
1086 paddw m0, m7
1087 paddw m1, m8
1088 paddw m2, m9
1089 paddw m3, m10
1090 REPX {psraw x, 2}, m0, m1, m2, m3
1091 mova [tmpq+64*0], m0
1092 mova [tmpq+64*1], m1
1093 mova [tmpq+64*2], m2
1094 mova [tmpq+64*3], m3
1095 add tmpq, 64*4
1096 dec hd
1097 jg .h_w128
1098 RET
1099 .v:
1100 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)]
1101 vpbroadcastw m9, mxyd
1102 vpbroadcastd m8, [pw_16]
1103 vpbroadcastd m10, [pw_32766]
1104 add wq, r6
1105 lea stride3q, [strideq*3]
1106 psubw m8, m9
1107 test dword r7m, 0x800
1108 jnz .v_12bpc
1109 psllw m8, 2
1110 psllw m9, 2
1111 .v_12bpc:
1112 jmp wq
1113 .v_w4:
1114 movq xmm0, [srcq+strideq*0]
1115 .v_w4_loop:
1116 vpbroadcastq xmm2, [srcq+strideq*1]
1117 vpbroadcastq ymm1, [srcq+strideq*2]
1118 vpbroadcastq ymm3, [srcq+stride3q ]
1119 lea srcq, [srcq+strideq*4]
1120 vpblendd ymm2, ymm1, 0x30
1121 vpblendd ymm2, ymm3, 0xc0
1122 vpblendd ymm1, ymm2, ymm0, 0x03 ; 0 1 2 3
1123 movq xmm0, [srcq+strideq*0]
1124 valignq ymm2, ymm0, ymm2, 1 ; 1 2 3 4
1125 pmullw ymm1, ym8
1126 pmullw ymm2, ym9
1127 psubw ymm1, ym10
1128 paddw ymm1, ymm2
1129 psraw ymm1, 2
1130 mova [tmpq], ymm1
1131 add tmpq, 32
1132 sub hd, 4
1133 jg .v_w4_loop
1134 vzeroupper
1135 RET
1136 .v_w8:
1137 movu xm0, [srcq+strideq*0]
1138 .v_w8_loop:
1139 vinserti32x4 ym1, ym0, [srcq+strideq*1], 1
1140 vinserti32x4 m1, [srcq+strideq*2], 2
1141 vinserti32x4 m1, [srcq+stride3q ], 3 ; 0 1 2 3
1142 lea srcq, [srcq+strideq*4]
1143 movu xm0, [srcq+strideq*0]
1144 valignq m2, m0, m1, 2 ; 1 2 3 4
1145 pmullw m1, m8
1146 pmullw m2, m9
1147 psubw m1, m10
1148 paddw m1, m2
1149 psraw m1, 2
1150 mova [tmpq], m1
1151 add tmpq, 64
1152 sub hd, 4
1153 jg .v_w8_loop
1154 RET
1155 .v_w16:
1156 movu ym0, [srcq+strideq*0]
1157 .v_w16_loop:
1158 vinserti32x8 m1, m0, [srcq+strideq*1], 1 ; 0 1
1159 movu ym3, [srcq+strideq*2]
1160 vinserti32x8 m2, m3, [srcq+stride3q ], 1 ; 2 3
1161 lea srcq, [srcq+strideq*4]
1162 movu ym0, [srcq+strideq*0]
1163 vshufi32x4 m3, m1, m3, q1032 ; 1 2
1164 vshufi32x4 m4, m2, m0, q1032 ; 3 4
1165 pmullw m1, m8
1166 pmullw m2, m8
1167 pmullw m3, m9
1168 pmullw m4, m9
1169 psubw m1, m10
1170 psubw m2, m10
1171 paddw m1, m3
1172 paddw m2, m4
1173 psraw m1, 2
1174 psraw m2, 2
1175 mova [tmpq+64*0], m1
1176 mova [tmpq+64*1], m2
1177 add tmpq, 64*2
1178 sub hd, 4
1179 jg .v_w16_loop
1180 RET
1181 .v_w32:
1182 movu m0, [srcq+strideq*0]
1183 .v_w32_loop:
1184 movu m3, [srcq+strideq*1]
1185 lea srcq, [srcq+strideq*2]
1186 pmullw m1, m8, m0
1187 movu m0, [srcq+strideq*0]
1188 pmullw m2, m8, m3
1189 pmullw m3, m9
1190 pmullw m4, m9, m0
1191 psubw m1, m10
1192 psubw m2, m10
1193 paddw m1, m3
1194 paddw m2, m4
1195 psraw m1, 2
1196 psraw m2, 2
1197 mova [tmpq+64*0], m1
1198 mova [tmpq+64*1], m2
1199 add tmpq, 64*2
1200 sub hd, 2
1201 jg .v_w32_loop
1202 RET
1203 .v_w64:
1204 movu m0, [srcq+64*0]
1205 movu m1, [srcq+64*1]
1206 .v_w64_loop:
1207 add srcq, strideq
1208 pmullw m2, m8, m0
1209 movu m0, [srcq+64*0]
1210 pmullw m3, m8, m1
1211 movu m1, [srcq+64*1]
1212 pmullw m4, m9, m0
1213 pmullw m5, m9, m1
1214 psubw m2, m10
1215 psubw m3, m10
1216 paddw m2, m4
1217 paddw m3, m5
1218 psraw m2, 2
1219 psraw m3, 2
1220 mova [tmpq+64*0], m2
1221 mova [tmpq+64*1], m3
1222 add tmpq, 64*2
1223 dec hd
1224 jg .v_w64_loop
1225 RET
1226 .v_w128:
1227 movu m0, [srcq+64*0]
1228 movu m1, [srcq+64*1]
1229 movu m2, [srcq+64*2]
1230 movu m3, [srcq+64*3]
1231 .v_w128_loop:
1232 add srcq, strideq
1233 pmullw m4, m8, m0
1234 movu m0, [srcq+64*0]
1235 pmullw m5, m8, m1
1236 movu m1, [srcq+64*1]
1237 pmullw m6, m8, m2
1238 movu m2, [srcq+64*2]
1239 pmullw m7, m8, m3
1240 movu m3, [srcq+64*3]
1241 pmullw m11, m9, m0
1242 pmullw m12, m9, m1
1243 pmullw m13, m9, m2
1244 pmullw m14, m9, m3
1245 REPX {psubw x, m10}, m4, m5, m6, m7
1246 paddw m4, m11
1247 paddw m5, m12
1248 paddw m6, m13
1249 paddw m7, m14
1250 REPX {psraw x, 2}, m4, m5, m6, m7
1251 mova [tmpq+64*0], m4
1252 mova [tmpq+64*1], m5
1253 mova [tmpq+64*2], m6
1254 mova [tmpq+64*3], m7
1255 add tmpq, 64*4
1256 dec hd
1257 jg .v_w128_loop
1258 RET
1259 .hv:
1260 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)]
1261 shl mxyd, 11
1262 vpbroadcastw m7, mxyd
1263 add wq, r6
1264 lea stride3q, [strideq*3]
1265 jmp wq
1266 .hv_w4:
1267 movq xmm0, [srcq+strideq*0+0]
1268 movq xmm1, [srcq+strideq*0+2]
1269 pmullw xmm0, xm4
1270 pmullw xmm1, xm5
1271 psubw xmm0, xm6
1272 paddw xmm0, xmm1
1273 psraw xmm0, 2
1274 vpbroadcastq ym0, xmm0
1275 .hv_w4_loop:
1276 movu xm1, [srcq+strideq*1]
1277 vinserti128 ym1, [srcq+stride3q ], 1
1278 movu xm2, [srcq+strideq*2]
1279 lea srcq, [srcq+strideq*4]
1280 vinserti128 ym2, [srcq+strideq*0], 1
1281 punpcklqdq ym3, ym1, ym2
1282 psrldq ym1, 2
1283 psrldq ym2, 2
1284 pmullw ym3, ym4
1285 punpcklqdq ym1, ym2
1286 pmullw ym1, ym5
1287 psubw ym3, ym6
1288 paddw ym1, ym3
1289 psraw ym1, 2 ; 1 2 3 4
1290 valignq ym2, ym1, ym0, 3 ; 0 1 2 3
1291 mova ym0, ym1
1292 psubw ym1, ym2
1293 pmulhrsw ym1, ym7
1294 paddw ym1, ym2
1295 mova [tmpq], ym1
1296 add tmpq, 32
1297 sub hd, 4
1298 jg .hv_w4_loop
1299 RET
1300 .hv_w8:
1301 pmullw xm0, xm4, [srcq+strideq*0+0]
1302 pmullw xm1, xm5, [srcq+strideq*0+2]
1303 psubw xm0, xm6
1304 paddw xm0, xm1
1305 psraw xm0, 2
1306 vinserti32x4 m0, xm0, 3
1307 .hv_w8_loop:
1308 movu xm1, [srcq+strideq*1+0]
1309 movu xm2, [srcq+strideq*1+2]
1310 vinserti32x4 ym1, [srcq+strideq*2+0], 1
1311 vinserti32x4 ym2, [srcq+strideq*2+2], 1
1312 vinserti32x4 m1, [srcq+stride3q +0], 2
1313 vinserti32x4 m2, [srcq+stride3q +2], 2
1314 lea srcq, [srcq+strideq*4]
1315 vinserti32x4 m1, [srcq+strideq*0+0], 3
1316 vinserti32x4 m2, [srcq+strideq*0+2], 3
1317 pmullw m1, m4
1318 pmullw m2, m5
1319 psubw m1, m6
1320 paddw m1, m2
1321 psraw m1, 2 ; 1 2 3 4
1322 valignq m2, m1, m0, 6 ; 0 1 2 3
1323 mova m0, m1
1324 psubw m1, m2
1325 pmulhrsw m1, m7
1326 paddw m1, m2
1327 mova [tmpq], m1
1328 add tmpq, 64
1329 sub hd, 4
1330 jg .hv_w8_loop
1331 RET
1332 .hv_w16:
1333 pmullw ym0, ym4, [srcq+strideq*0+0]
1334 pmullw ym1, ym5, [srcq+strideq*0+2]
1335 psubw ym0, ym6
1336 paddw ym0, ym1
1337 psraw ym0, 2
1338 vinserti32x8 m0, ym0, 1
1339 .hv_w16_loop:
1340 movu ym1, [srcq+strideq*1+0]
1341 movu ym2, [srcq+strideq*1+2]
1342 lea srcq, [srcq+strideq*2]
1343 vinserti32x8 m1, [srcq+strideq*0+0], 1
1344 vinserti32x8 m2, [srcq+strideq*0+2], 1
1345 pmullw m1, m4
1346 pmullw m2, m5
1347 psubw m1, m6
1348 paddw m1, m2
1349 psraw m1, 2 ; 1 2
1350 vshufi32x4 m2, m0, m1, q1032 ; 0 1
1351 mova m0, m1
1352 psubw m1, m2
1353 pmulhrsw m1, m7
1354 paddw m1, m2
1355 mova [tmpq], m1
1356 add tmpq, 64
1357 sub hd, 2
1358 jg .hv_w16_loop
1359 RET
1360 .hv_w32:
1361 pmullw m0, m4, [srcq+strideq*0+0]
1362 pmullw m1, m5, [srcq+strideq*0+2]
1363 psubw m0, m6
1364 paddw m0, m1
1365 psraw m0, 2
1366 .hv_w32_loop:
1367 pmullw m3, m4, [srcq+strideq*1+0]
1368 pmullw m1, m5, [srcq+strideq*1+2]
1369 lea srcq, [srcq+strideq*2]
1370 psubw m3, m6
1371 paddw m3, m1
1372 psraw m3, 2
1373 psubw m1, m3, m0
1374 pmulhrsw m1, m7
1375 paddw m1, m0
1376 pmullw m0, m4, [srcq+strideq*0+0]
1377 pmullw m2, m5, [srcq+strideq*0+2]
1378 psubw m0, m6
1379 paddw m0, m2
1380 psraw m0, 2
1381 psubw m2, m0, m3
1382 pmulhrsw m2, m7
1383 paddw m2, m3
1384 mova [tmpq+64*0], m1
1385 mova [tmpq+64*1], m2
1386 add tmpq, 64*2
1387 sub hd, 2
1388 jg .hv_w32_loop
1389 RET
1390 .hv_w64:
1391 pmullw m0, m4, [srcq+ 0]
1392 pmullw m2, m5, [srcq+ 2]
1393 pmullw m1, m4, [srcq+64]
1394 pmullw m3, m5, [srcq+66]
1395 psubw m0, m6
1396 psubw m1, m6
1397 paddw m0, m2
1398 paddw m1, m3
1399 psraw m0, 2
1400 psraw m1, 2
1401 .hv_w64_loop:
1402 add srcq, strideq
1403 pmullw m2, m4, [srcq+ 0]
1404 pmullw m8, m5, [srcq+ 2]
1405 pmullw m3, m4, [srcq+64]
1406 pmullw m9, m5, [srcq+66]
1407 psubw m2, m6
1408 psubw m3, m6
1409 paddw m2, m8
1410 paddw m3, m9
1411 psraw m2, 2
1412 psraw m3, 2
1413 psubw m8, m2, m0
1414 psubw m9, m3, m1
1415 pmulhrsw m8, m7
1416 pmulhrsw m9, m7
1417 paddw m8, m0
1418 mova m0, m2
1419 paddw m9, m1
1420 mova m1, m3
1421 mova [tmpq+64*0], m8
1422 mova [tmpq+64*1], m9
1423 add tmpq, 64*2
1424 dec hd
1425 jg .hv_w64_loop
1426 RET
1427 .hv_w128:
1428 pmullw m0, m4, [srcq+ 0]
1429 pmullw m8, m5, [srcq+ 2]
1430 pmullw m1, m4, [srcq+ 64]
1431 pmullw m9, m5, [srcq+ 66]
1432 pmullw m2, m4, [srcq+128]
1433 pmullw m10, m5, [srcq+130]
1434 pmullw m3, m4, [srcq+192]
1435 pmullw m11, m5, [srcq+194]
1436 REPX {psubw x, m6}, m0, m1, m2, m3
1437 paddw m0, m8
1438 paddw m1, m9
1439 paddw m2, m10
1440 paddw m3, m11
1441 REPX {psraw x, 2}, m0, m1, m2, m3
1442 .hv_w128_loop:
1443 add srcq, strideq
1444 pmullw m8, m4, [srcq+ 0]
1445 pmullw m12, m5, [srcq+ 2]
1446 pmullw m9, m4, [srcq+ 64]
1447 pmullw m13, m5, [srcq+ 66]
1448 pmullw m10, m4, [srcq+128]
1449 pmullw m14, m5, [srcq+130]
1450 pmullw m11, m4, [srcq+192]
1451 pmullw m15, m5, [srcq+194]
1452 REPX {psubw x, m6}, m8, m9, m10, m11
1453 paddw m8, m12
1454 paddw m9, m13
1455 paddw m10, m14
1456 paddw m11, m15
1457 REPX {psraw x, 2}, m8, m9, m10, m11
1458 psubw m12, m8, m0
1459 psubw m13, m9, m1
1460 psubw m14, m10, m2
1461 psubw m15, m11, m3
1462 REPX {pmulhrsw x, m7}, m12, m13, m14, m15
1463 paddw m12, m0
1464 mova m0, m8
1465 paddw m13, m1
1466 mova m1, m9
1467 mova [tmpq+64*0], m12
1468 mova [tmpq+64*1], m13
1469 paddw m14, m2
1470 mova m2, m10
1471 paddw m15, m3
1472 mova m3, m11
1473 mova [tmpq+64*2], m14
1474 mova [tmpq+64*3], m15
1475 add tmpq, 64*4
1476 dec hd
1477 jg .hv_w128_loop
1478 RET
1479
1480 ; int8_t subpel_filters[5][15][8]
1481 %assign FILTER_REGULAR (0*15 << 16) | 3*15
1482 %assign FILTER_SMOOTH (1*15 << 16) | 4*15
1483 %assign FILTER_SHARP (2*15 << 16) | 3*15
1484
1485 %macro MC_8TAP_FN 4 ; prefix, type, type_h, type_v
1486 cglobal %1_8tap_%2_16bpc
1487 mov t0d, FILTER_%3
1488 %ifidn %3, %4
1489 mov t1d, t0d
1490 %else
1491 mov t1d, FILTER_%4
1492 %endif
1493 %ifnidn %2, regular ; skip the jump in the last filter
1494 jmp mangle(private_prefix %+ _%1_8tap_16bpc %+ SUFFIX)
1495 %endif
1496 %endmacro
1497
1498 %if WIN64
1499 DECLARE_REG_TMP 4, 5
1500 %define buf rsp+stack_offset+8 ; shadow space
1501 %else
1502 DECLARE_REG_TMP 7, 8
1503 %define buf rsp-40 ; red zone
1504 %endif
1505
1506 MC_8TAP_FN put, sharp, SHARP, SHARP
1507 MC_8TAP_FN put, sharp_smooth, SHARP, SMOOTH
1508 MC_8TAP_FN put, smooth_sharp, SMOOTH, SHARP
1509 MC_8TAP_FN put, smooth, SMOOTH, SMOOTH
1510 MC_8TAP_FN put, sharp_regular, SHARP, REGULAR
1511 MC_8TAP_FN put, regular_sharp, REGULAR, SHARP
1512 MC_8TAP_FN put, smooth_regular, SMOOTH, REGULAR
1513 MC_8TAP_FN put, regular_smooth, REGULAR, SMOOTH
1514 MC_8TAP_FN put, regular, REGULAR, REGULAR
1515
1516 cglobal put_8tap_16bpc, 4, 9, 16, dst, ds, src, ss, w, h, mx, my
1517 %define base r8-put_avx512icl
1518 imul mxd, mxm, 0x010101
1519 add mxd, t0d ; 8tap_h, mx, 4tap_h
1520 imul myd, mym, 0x010101
1521 add myd, t1d ; 8tap_v, my, 4tap_v
1522 lea r8, [put_avx512icl]
1523 movifnidn wd, wm
1524 movifnidn hd, hm
1525 test mxd, 0xf00
1526 jnz .h
1527 test myd, 0xf00
1528 jnz .v
1529 tzcnt wd, wd
1530 movzx wd, word [r8+wq*2+table_offset(put,)]
1531 add wq, r8
1532 %if WIN64
1533 pop r8
1534 %endif
1535 jmp wq
1536 .h_w2:
1537 movzx mxd, mxb
1538 sub srcq, 2
1539 mova ym2, [spel_h_shuf2a]
1540 pmovsxbw xmm4, [base+subpel_filters+mxq*8]
1541 pshufd xmm3, xmm4, q1111
1542 pshufd xmm4, xmm4, q2222
1543 .h_w2_loop:
1544 movu xm1, [srcq+ssq*0]
1545 vinserti32x4 ym1, [srcq+ssq*1], 1
1546 lea srcq, [srcq+ssq*2]
1547 mova xmm0, xm8
1548 vpermb ym1, ym2, ym1
1549 vpdpwssd xmm0, xmm3, xm1
1550 vextracti32x4 xm1, ym1, 1
1551 vpdpwssd xmm0, xmm4, xm1
1552 psrad xmm0, 6
1553 packusdw xmm0, xmm0
1554 pminsw xmm0, xm9
1555 movd [dstq+dsq*0], xmm0
1556 pextrd [dstq+dsq*1], xmm0, 1
1557 lea dstq, [dstq+dsq*2]
1558 sub hd, 2
1559 jg .h_w2_loop
1560 RET
1561 .h_w4:
1562 movzx mxd, mxb
1563 sub srcq, 2
1564 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
1565 vbroadcasti32x4 ym4, [spel_h_shufA]
1566 vbroadcasti32x4 ym5, [spel_h_shufB]
1567 pshufd xmm0, xmm0, q2211
1568 vpbroadcastq ym6, xmm0
1569 vpermq ym7, ymm0, q1111
1570 .h_w4_loop:
1571 movu xm2, [srcq+ssq*0]
1572 vinserti32x4 ym2, [srcq+ssq*1], 1
1573 lea srcq, [srcq+ssq*2]
1574 mova ym0, ym8
1575 pshufb ym1, ym2, ym4
1576 vpdpwssd ym0, ym6, ym1
1577 pshufb ym2, ym5
1578 vpdpwssd ym0, ym7, ym2
1579 psrad ym0, 6
1580 vextracti32x4 xm1, ym0, 1
1581 packusdw xm0, xm1
1582 pminsw xmm0, xm0, xm9
1583 movq [dstq+dsq*0], xmm0
1584 movhps [dstq+dsq*1], xmm0
1585 lea dstq, [dstq+dsq*2]
1586 sub hd, 2
1587 jg .h_w4_loop
1588 RET
1589 .h:
1590 test myd, 0xf00
1591 jnz .hv
1592 mov r7d, r8m
1593 vpbroadcastw m9, r8m
1594 shr r7d, 11
1595 vpbroadcastd m8, [base+put_8tap_h_rnd+r7*4]
1596 cmp wd, 4
1597 je .h_w4
1598 jl .h_w2
1599 shr mxd, 16
1600 sub srcq, 6
1601 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
1602 mova [buf], xmm0
1603 vpbroadcastd m10, xmm0
1604 vpbroadcastd m11, [buf+ 4]
1605 vpbroadcastd m12, [buf+ 8]
1606 vpbroadcastd m13, [buf+12]
1607 sub wd, 16
1608 je .h_w16
1609 jg .h_w32
1610 .h_w8:
1611 mova m4, [spel_h_shufA]
1612 movu m5, [spel_h_shufB]
1613 movu m6, [spel_h_shufC]
1614 mova m7, [spel_h_shufD]
1615 .h_w8_loop:
1616 movu ym2, [srcq+ssq*0]
1617 vinserti32x8 m2, [srcq+ssq*1], 1
1618 lea srcq, [srcq+ssq*2]
1619 mova m0, m8
1620 vpermb m1, m4, m2
1621 vpdpwssd m0, m10, m1
1622 vpermb m1, m5, m2
1623 vpdpwssd m0, m11, m1
1624 vpermb m1, m6, m2
1625 vpdpwssd m0, m12, m1
1626 vpermb m1, m7, m2
1627 vpdpwssd m0, m13, m1
1628 psrad m0, 6
1629 vextracti32x8 ym1, m0, 1
1630 packusdw ym0, ym1
1631 pminsw ym0, ym9
1632 mova [dstq+dsq*0], xm0
1633 vextracti32x4 [dstq+dsq*1], ym0, 1
1634 lea dstq, [dstq+dsq*2]
1635 sub hd, 2
1636 jg .h_w8_loop
1637 RET
1638 .h_w16:
1639 vbroadcasti32x4 m6, [spel_h_shufA]
1640 vbroadcasti32x4 m7, [spel_h_shufB]
1641 .h_w16_loop:
1642 movu ym2, [srcq+ssq*0+ 0]
1643 vinserti32x8 m2, [srcq+ssq*1+ 0], 1
1644 movu ym3, [srcq+ssq*0+16]
1645 vinserti32x8 m3, [srcq+ssq*1+16], 1
1646 lea srcq, [srcq+ssq*2]
1647 mova m0, m8
1648 mova m1, m8
1649 pshufb m4, m2, m6
1650 vpdpwssd m0, m10, m4 ; a0
1651 pshufb m4, m3, m6
1652 vpdpwssd m1, m12, m4 ; b2
1653 pshufb m4, m2, m7
1654 vpdpwssd m0, m11, m4 ; a1
1655 pshufb m4, m3, m7
1656 vpdpwssd m1, m13, m4 ; b3
1657 shufpd m2, m3, 0x55
1658 pshufb m4, m2, m6
1659 vpdpwssd m0, m12, m4 ; a2
1660 vpdpwssd m1, m10, m4 ; b0
1661 pshufb m2, m7
1662 vpdpwssd m0, m13, m2 ; a3
1663 vpdpwssd m1, m11, m2 ; b1
1664 psrad m0, 6
1665 psrad m1, 6
1666 packusdw m0, m1
1667 pminsw m0, m9
1668 mova [dstq+dsq*0], ym0
1669 vextracti32x8 [dstq+dsq*1], m0, 1
1670 lea dstq, [dstq+dsq*2]
1671 sub hd, 2
1672 jg .h_w16_loop
1673 RET
1674 .h_w32:
1675 lea srcq, [srcq+wq*2]
1676 vbroadcasti32x4 m6, [spel_h_shufA]
1677 lea dstq, [dstq+wq*2]
1678 vbroadcasti32x4 m7, [spel_h_shufB]
1679 neg wq
1680 .h_w32_loop0:
1681 mov r6, wq
1682 .h_w32_loop:
1683 movu m2, [srcq+r6*2+ 0]
1684 movu m3, [srcq+r6*2+ 8]
1685 mova m0, m8
1686 mova m1, m8
1687 pshufb m4, m2, m6
1688 vpdpwssd m0, m10, m4 ; a0
1689 pshufb m4, m3, m6
1690 vpdpwssd m1, m10, m4 ; b0
1691 vpdpwssd m0, m12, m4 ; a2
1692 movu m4, [srcq+r6*2+16]
1693 pshufb m3, m7
1694 vpdpwssd m1, m11, m3 ; b1
1695 vpdpwssd m0, m13, m3 ; a3
1696 pshufb m3, m4, m6
1697 vpdpwssd m1, m12, m3 ; b2
1698 pshufb m2, m7
1699 vpdpwssd m0, m11, m2 ; a1
1700 pshufb m4, m7
1701 vpdpwssd m1, m13, m4 ; b3
1702 psrad m0, 6
1703 psrad m1, 6
1704 packusdw m0, m1
1705 pminsw m0, m9
1706 mova [dstq+r6*2], m0
1707 add r6, 32
1708 jl .h_w32_loop
1709 add srcq, ssq
1710 add dstq, dsq
1711 dec hd
1712 jg .h_w32_loop0
1713 RET
1714 .v:
1715 movzx mxd, myb
1716 shr myd, 16
1717 cmp hd, 6
1718 cmovs myd, mxd
1719 vpbroadcastd m10, [pd_32]
1720 pmovsxbw xmm0, [base+subpel_filters+myq*8]
1721 tzcnt r7d, wd
1722 vpbroadcastw m11, r8m
1723 lea r6, [ssq*3]
1724 movzx r7d, word [r8+r7*2+table_offset(put, _8tap_v)]
1725 sub srcq, r6
1726 mova [rsp+stack_offset+8], xmm0
1727 vpbroadcastd m12, xmm0
1728 add r7, r8
1729 vpbroadcastd m13, [rsp+stack_offset+12]
1730 vpbroadcastd m14, [rsp+stack_offset+16]
1731 vpbroadcastd m15, [rsp+stack_offset+20]
1732 jmp r7
1733 .v_w2:
1734 movd xmm2, [srcq+ssq*0]
1735 pinsrd xmm2, [srcq+ssq*1], 1
1736 pinsrd xmm2, [srcq+ssq*2], 2
1737 add srcq, r6
1738 pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3
1739 movd xmm3, [srcq+ssq*1]
1740 vpbroadcastd xmm1, [srcq+ssq*2]
1741 add srcq, r6
1742 vpbroadcastd xmm0, [srcq+ssq*0]
1743 vpblendd xmm3, xmm1, 0x02 ; 4 5
1744 vpblendd xmm1, xmm0, 0x02 ; 5 6
1745 palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4
1746 punpcklwd xmm3, xmm1 ; 45 56
1747 punpcklwd xmm1, xmm2, xmm4 ; 01 12
1748 punpckhwd xmm2, xmm4 ; 23 34
1749 .v_w2_loop:
1750 vpbroadcastd xmm4, [srcq+ssq*1]
1751 lea srcq, [srcq+ssq*2]
1752 mova xmm5, xm10
1753 vpdpwssd xmm5, xm12, xmm1 ; a0 b0
1754 mova xmm1, xmm2
1755 vpdpwssd xmm5, xm13, xmm2 ; a1 b1
1756 mova xmm2, xmm3
1757 vpdpwssd xmm5, xm14, xmm3 ; a2 b2
1758 vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7
1759 vpbroadcastd xmm0, [srcq+ssq*0]
1760 vpblendd xmm4, xmm0, 0x02 ; 7 8
1761 punpcklwd xmm3, xmm4 ; 67 78
1762 vpdpwssd xmm5, xm15, xmm3 ; a3 b3
1763 psrad xmm5, 6
1764 packusdw xmm5, xmm5
1765 pminsw xmm5, xm11
1766 movd [dstq+dsq*0], xmm5
1767 pextrd [dstq+dsq*1], xmm5, 1
1768 lea dstq, [dstq+dsq*2]
1769 sub hd, 2
1770 jg .v_w2_loop
1771 RET
1772 .v_w4:
1773 movq xmm1, [srcq+ssq*0]
1774 vpbroadcastq ymm0, [srcq+ssq*1]
1775 vpbroadcastq ymm2, [srcq+ssq*2]
1776 add srcq, r6
1777 vpbroadcastq ymm4, [srcq+ssq*0]
1778 vpbroadcastq ymm3, [srcq+ssq*1]
1779 vpbroadcastq ymm5, [srcq+ssq*2]
1780 add srcq, r6
1781 vpblendd ymm1, ymm0, 0x30
1782 vpblendd ymm0, ymm2, 0x30
1783 punpcklwd ymm1, ymm0 ; 01 12
1784 vpbroadcastq ymm0, [srcq+ssq*0]
1785 vpblendd ymm2, ymm4, 0x30
1786 vpblendd ymm4, ymm3, 0x30
1787 punpcklwd ymm2, ymm4 ; 23 34
1788 vpblendd ymm3, ymm5, 0x30
1789 vpblendd ymm5, ymm0, 0x30
1790 punpcklwd ymm3, ymm5 ; 45 56
1791 .v_w4_loop:
1792 vpbroadcastq ymm5, [srcq+ssq*1]
1793 lea srcq, [srcq+ssq*2]
1794 mova ymm4, ym10
1795 vpdpwssd ymm4, ym12, ymm1 ; a0 b0
1796 mova ymm1, ymm2
1797 vpdpwssd ymm4, ym13, ymm2 ; a1 b1
1798 mova ymm2, ymm3
1799 vpdpwssd ymm4, ym14, ymm3 ; a2 b2
1800 vpblendd ymm3, ymm0, ymm5, 0x30
1801 vpbroadcastq ymm0, [srcq+ssq*0]
1802 vpblendd ymm5, ymm0, 0x30
1803 punpcklwd ymm3, ymm5 ; 67 78
1804 vpdpwssd ymm4, ym15, ymm3 ; a3 b3
1805 psrad ymm4, 6
1806 vextracti128 xmm5, ymm4, 1
1807 packusdw xmm4, xmm5
1808 pminsw xmm4, xm11
1809 movq [dstq+dsq*0], xmm4
1810 movhps [dstq+dsq*1], xmm4
1811 lea dstq, [dstq+dsq*2]
1812 sub hd, 2
1813 jg .v_w4_loop
1814 vzeroupper
1815 RET
1816 .v_w8:
1817 vbroadcasti32x4 m2, [srcq+ssq*2]
1818 vinserti32x4 m1, m2, [srcq+ssq*0], 0
1819 vinserti32x4 m1, [srcq+ssq*1], 1 ; 0 1 2
1820 add srcq, r6
1821 vinserti32x4 ym2, [srcq+ssq*0], 1
1822 vinserti32x4 m2, [srcq+ssq*1], 2 ; 2 3 4
1823 mova m6, [spel_v_shuf8]
1824 movu xm0, [srcq+ssq*1]
1825 vinserti32x4 ym0, [srcq+ssq*2], 1
1826 add srcq, r6
1827 vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6
1828 vpermb m1, m6, m1 ; 01 12
1829 vpermb m2, m6, m2 ; 23 34
1830 vpermb m3, m6, m0 ; 45 56
1831 .v_w8_loop:
1832 vinserti32x4 m0, [srcq+ssq*1], 3
1833 lea srcq, [srcq+ssq*2]
1834 movu xm5, [srcq+ssq*0]
1835 mova m4, m10
1836 vpdpwssd m4, m12, m1 ; a0 b0
1837 mova m1, m2
1838 vshufi32x4 m0, m5, q1032 ; 6 7 8
1839 vpdpwssd m4, m13, m2 ; a1 b1
1840 mova m2, m3
1841 vpdpwssd m4, m14, m3 ; a2 b2
1842 vpermb m3, m6, m0 ; 67 78
1843 vpdpwssd m4, m15, m3 ; a3 b3
1844 psrad m4, 6
1845 vextracti32x8 ym5, m4, 1
1846 packusdw ym4, ym5
1847 pminsw ym4, ym11
1848 mova [dstq+dsq*0], xm4
1849 vextracti32x4 [dstq+dsq*1], ym4, 1
1850 lea dstq, [dstq+dsq*2]
1851 sub hd, 2
1852 jg .v_w8_loop
1853 RET
1854 .v_w16:
1855 vbroadcasti32x8 m1, [srcq+ssq*1]
1856 vinserti32x8 m0, m1, [srcq+ssq*0], 0
1857 vinserti32x8 m1, [srcq+ssq*2], 1
1858 mova m8, [spel_v_shuf16]
1859 add srcq, r6
1860 movu ym3, [srcq+ssq*0]
1861 vinserti32x8 m3, [srcq+ssq*1], 1
1862 movu ym5, [srcq+ssq*2]
1863 add srcq, r6
1864 vinserti32x8 m5, [srcq+ssq*0], 1
1865 vpermb m0, m8, m0 ; 01
1866 vpermb m1, m8, m1 ; 12
1867 vpermb m3, m8, m3 ; 34
1868 vpermb m5, m8, m5 ; 56
1869 mova m9, [deint_q_shuf]
1870 vpshrdd m2, m1, m3, 16 ; 23
1871 vpshrdd m4, m3, m5, 16 ; 45
1872 .v_w16_loop:
1873 mova m6, m10
1874 mova m7, m10
1875 vpdpwssd m6, m12, m0 ; a0
1876 mova m0, m2
1877 vpdpwssd m7, m12, m1 ; b0
1878 mova m1, m3
1879 vpdpwssd m6, m13, m2 ; a1
1880 mova m2, m4
1881 vpdpwssd m7, m13, m3 ; b1
1882 mova m3, m5
1883 vpdpwssd m6, m14, m4 ; a2
1884 mova m4, m5
1885 vpdpwssd m7, m14, m5 ; b2
1886 movu ym5, [srcq+ssq*1]
1887 lea srcq, [srcq+ssq*2]
1888 vinserti32x8 m5, [srcq+ssq*0], 1
1889 vpermb m5, m8, m5 ; 78
1890 vpshrdd m4, m5, 16 ; 67
1891 vpdpwssd m6, m15, m4 ; a3
1892 vpdpwssd m7, m15, m5 ; b3
1893 psrad m6, 6
1894 psrad m7, 6
1895 packusdw m6, m7
1896 pminsw m6, m11
1897 vpermq m6, m9, m6
1898 mova [dstq+dsq*0], ym6
1899 vextracti32x8 [dstq+dsq*1], m6, 1
1900 lea dstq, [dstq+dsq*2]
1901 sub hd, 2
1902 jg .v_w16_loop
1903 RET
1904 .v_w32:
1905 .v_w64:
1906 .v_w128:
1907 %if WIN64
1908 movaps [rsp+stack_offset+8], xmm6
1909 %endif
1910 lea wd, [hq+wq*8-256]
1911 mov r7, srcq
1912 mov r8, dstq
1913 .v_w32_loop0:
1914 movu m16, [srcq+ssq*0]
1915 movu m17, [srcq+ssq*1]
1916 movu m18, [srcq+ssq*2]
1917 add srcq, r6
1918 movu m19, [srcq+ssq*0]
1919 movu m20, [srcq+ssq*1]
1920 movu m21, [srcq+ssq*2]
1921 add srcq, r6
1922 movu m22, [srcq+ssq*0]
1923 punpcklwd m0, m16, m17 ; 01l
1924 punpckhwd m16, m17 ; 01h
1925 punpcklwd m1, m17, m18 ; 12l
1926 punpckhwd m17, m18 ; 12h
1927 punpcklwd m2, m18, m19 ; 23l
1928 punpckhwd m18, m19 ; 23h
1929 punpcklwd m3, m19, m20 ; 34l
1930 punpckhwd m19, m20 ; 34h
1931 punpcklwd m4, m20, m21 ; 45l
1932 punpckhwd m20, m21 ; 45h
1933 punpcklwd m5, m21, m22 ; 56l
1934 punpckhwd m21, m22 ; 56h
1935 .v_w32_loop:
1936 mova m6, m10
1937 vpdpwssd m6, m12, m0 ; a0l
1938 mova m8, m10
1939 vpdpwssd m8, m12, m16 ; a0h
1940 mova m7, m10
1941 vpdpwssd m7, m12, m1 ; b0l
1942 mova m9, m10
1943 vpdpwssd m9, m12, m17 ; b0h
1944 mova m0, m2
1945 vpdpwssd m6, m13, m2 ; a1l
1946 mova m16, m18
1947 vpdpwssd m8, m13, m18 ; a1h
1948 mova m1, m3
1949 vpdpwssd m7, m13, m3 ; b1l
1950 mova m17, m19
1951 vpdpwssd m9, m13, m19 ; b1h
1952 mova m2, m4
1953 vpdpwssd m6, m14, m4 ; a2l
1954 mova m18, m20
1955 vpdpwssd m8, m14, m20 ; a2h
1956 mova m3, m5
1957 vpdpwssd m7, m14, m5 ; b2l
1958 mova m19, m21
1959 vpdpwssd m9, m14, m21 ; b2h
1960 movu m21, [srcq+ssq*1]
1961 lea srcq, [srcq+ssq*2]
1962 punpcklwd m4, m22, m21 ; 67l
1963 punpckhwd m20, m22, m21 ; 67h
1964 movu m22, [srcq+ssq*0]
1965 vpdpwssd m6, m15, m4 ; a3l
1966 vpdpwssd m8, m15, m20 ; a3h
1967 punpcklwd m5, m21, m22 ; 78l
1968 punpckhwd m21, m22 ; 78h
1969 vpdpwssd m7, m15, m5 ; b3l
1970 vpdpwssd m9, m15, m21 ; b3h
1971 REPX {psrad x, 6}, m6, m8, m7, m9
1972 packusdw m6, m8
1973 packusdw m7, m9
1974 pminsw m6, m11
1975 pminsw m7, m11
1976 mova [dstq+dsq*0], m6
1977 mova [dstq+dsq*1], m7
1978 lea dstq, [dstq+dsq*2]
1979 sub hd, 2
1980 jg .v_w32_loop
1981 add r7, 64
1982 add r8, 64
1983 movzx hd, wb
1984 mov srcq, r7
1985 mov dstq, r8
1986 sub wd, 1<<8
1987 jg .v_w32_loop0
1988 %if WIN64
1989 movaps xmm6, [rsp+stack_offset+8]
1990 %endif
1991 vzeroupper
1992 RET
1993 .hv:
1994 vpbroadcastw m11, r8m
1995 cmp wd, 4
1996 jg .hv_w8
1997 movzx mxd, mxb
1998 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
1999 movzx mxd, myb
2000 shr myd, 16
2001 cmp hd, 6
2002 cmovs myd, mxd
2003 pmovsxbw xmm1, [base+subpel_filters+myq*8]
2004 lea r6, [ssq*3]
2005 sub srcq, 2
2006 sub srcq, r6
2007 test dword r8m, 0x800
2008 jnz .hv_12bit
2009 vpbroadcastd m10, [pd_2176]
2010 psllw xmm0, 6
2011 jmp .hv_main
2012 .hv_12bit:
2013 vpbroadcastd m10, [pd_640]
2014 psllw xmm0, 4
2015 psllw xmm1, 2
2016 .hv_main:
2017 mova [buf+ 0], xmm0
2018 mova [buf+16], xmm1
2019 vpbroadcastd m8, [buf+ 4]
2020 vpbroadcastd m9, [buf+ 8]
2021 vpbroadcastd ym12, xmm1
2022 vpbroadcastd ym13, [buf+20]
2023 vpbroadcastd ym14, [buf+24]
2024 vpbroadcastd ym15, [buf+28]
2025 movu xm4, [srcq+ssq*0]
2026 vinserti32x4 ym4, [srcq+ssq*1], 1
2027 vinserti32x4 m4, [srcq+ssq*2], 2
2028 add srcq, r6
2029 vinserti32x4 m4, [srcq+ssq*0], 3 ; 0 1 2 3
2030 movu xm0, [srcq+ssq*1]
2031 vinserti32x4 ym0, [srcq+ssq*2], 1
2032 add srcq, r6
2033 vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6
2034 cmp wd, 4
2035 je .hv_w4
2036 vbroadcasti32x4 m2, [spel_h_shufA]
2037 mova m3, [spel_h_shuf2b]
2038 mova ym6, [spel_h_shuf2a]
2039 mova xm7, [spel_shuf2]
2040 mova m1, m10
2041 pshufb m4, m2
2042 pshufb m0, m2
2043 punpcklqdq m2, m4, m0
2044 vpdpwssd m1, m8, m2 ; 04 15 26 3_
2045 punpckhqdq m4, m0
2046 vpdpwssd m1, m9, m4
2047 vpermb m1, m3, m1 ; 01 12
2048 vextracti32x4 xm2, ym1, 1 ; 23 34
2049 vextracti32x4 xm3, m1, 2 ; 45 56
2050 .hv_w2_loop:
2051 movu xm5, [srcq+ssq*1]
2052 lea srcq, [srcq+ssq*2]
2053 vinserti32x4 ym5, [srcq+ssq*0], 1
2054 mova xm4, xm10
2055 vpermb ym5, ym6, ym5
2056 pmaddwd xmm0, xm12, xm1 ; a0 b0
2057 vpdpwssd xm4, xm8, xm5
2058 vextracti32x4 xm5, ym5, 1
2059 mova xm1, xm2
2060 vpdpwssd xmm0, xm13, xm2 ; a1 b1
2061 vpdpwssd xm4, xm9, xm5 ; 7 8
2062 mova xm2, xm3
2063 vpdpwssd xmm0, xm14, xm3 ; a2 b2
2064 vpermt2b xm3, xm7, xm4 ; 67 78
2065 vpdpwssd xmm0, xm15, xm3 ; a3 b3
2066 psrad xmm0, 10
2067 packusdw xmm0, xmm0
2068 pminsw xmm0, xm11
2069 movd [dstq+dsq*0], xmm0
2070 pextrd [dstq+dsq*1], xmm0, 1
2071 lea dstq, [dstq+dsq*2]
2072 sub hd, 2
2073 jg .hv_w2_loop
2074 RET
2075 .hv_w4:
2076 vbroadcasti32x4 m19, [spel_h_shufA]
2077 vbroadcasti32x4 m20, [spel_h_shufB]
2078 mova ym6, [spel_shuf4a]
2079 mova ym7, [spel_shuf4b]
2080 mova m2, m10
2081 mova m3, m10
2082 pshufb m1, m4, m19
2083 vpdpwssd m2, m8, m1
2084 pshufb m1, m0, m19
2085 vpdpwssd m3, m8, m1
2086 pshufb m4, m20
2087 vpdpwssd m2, m9, m4
2088 pshufb m0, m20
2089 vpdpwssd m3, m9, m0
2090 vpermb m1, m6, m2 ; 01 12
2091 vshufi32x4 m2, m3, q1032
2092 vpermb m3, m6, m3 ; 45 56
2093 vpermb m2, m6, m2 ; 23 34
2094 .hv_w4_loop:
2095 movu xm18, [srcq+ssq*1]
2096 lea srcq, [srcq+ssq*2]
2097 vinserti128 ym18, [srcq+ssq*0], 1
2098 mova ym4, ym10
2099 pshufb ym17, ym18, ym19
2100 pmaddwd ym16, ym12, ym1 ; a0 b0
2101 vpdpwssd ym4, ym8, ym17
2102 pshufb ym18, ym20
2103 mova ym1, ym2
2104 vpdpwssd ym16, ym13, ym2 ; a1 b1
2105 vpdpwssd ym4, ym9, ym18 ; 7 8
2106 mova ym2, ym3
2107 vpdpwssd ym16, ym14, ym3 ; a2 b2
2108 vpermt2b ym3, ym7, ym4 ; 67 78
2109 vpdpwssd ym16, ym15, ym3 ; a3 b3
2110 psrad ym16, 10
2111 vextracti128 xm17, ym16, 1
2112 packusdw xm16, xm17
2113 pminsw xm16, xm11
2114 movq [dstq+dsq*0], xm16
2115 movhps [dstq+dsq*1], xm16
2116 lea dstq, [dstq+dsq*2]
2117 sub hd, 2
2118 jg .hv_w4_loop
2119 vzeroupper
2120 RET
2121 .hv_w8:
2122 shr mxd, 16
2123 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
2124 movzx mxd, myb
2125 shr myd, 16
2126 cmp hd, 6
2127 cmovs myd, mxd
2128 pmovsxbw xmm1, [base+subpel_filters+myq*8]
2129 lea r6, [ssq*3]
2130 sub srcq, 6
2131 sub srcq, r6
2132 test dword r8m, 0x800
2133 jnz .hv_w8_12bit
2134 vpbroadcastd m10, [pd_2176]
2135 psllw xmm0, 6
2136 jmp .hv_w8_main
2137 .hv_w8_12bit:
2138 vpbroadcastd m10, [pd_640]
2139 psllw xmm0, 4
2140 psllw xmm1, 2
2141 .hv_w8_main:
2142 mova [buf+ 0], xmm0
2143 mova [buf+16], xmm1
2144 vpbroadcastd m12, xmm0
2145 vpbroadcastd m13, [buf+ 4]
2146 vpbroadcastd m14, [buf+ 8]
2147 vpbroadcastd m15, [buf+12]
2148 vpbroadcastd m16, xmm1
2149 vpbroadcastd m17, [buf+20]
2150 vpbroadcastd m18, [buf+24]
2151 vpbroadcastd m19, [buf+28]
2152 cmp wd, 16
2153 je .hv_w16
2154 jg .hv_w32
2155 mova m5, [spel_h_shufA]
2156 movu ym0, [srcq+ssq*0]
2157 vinserti32x8 m0, [srcq+ssq*1], 1 ; 0 1
2158 movu ym9, [srcq+ssq*2]
2159 add srcq, r6
2160 vinserti32x8 m9, [srcq+ssq*0], 1 ; 2 3
2161 movu ym20, [srcq+ssq*1]
2162 vinserti32x8 m20, [srcq+ssq*2], 1 ; 4 5
2163 add srcq, r6
2164 movu ym21, [srcq+ssq*0] ; 6
2165 movu m6, [spel_h_shufB]
2166 movu m7, [spel_h_shufC]
2167 vpermb m8, m5, m0
2168 mova m1, m10
2169 vpdpwssd m1, m12, m8 ; a0 b0
2170 vpermb m8, m5, m9
2171 mova m2, m10
2172 vpdpwssd m2, m12, m8 ; c0 d0
2173 vpermb m8, m5, m20
2174 mova m3, m10
2175 vpdpwssd m3, m12, m8 ; e0 f0
2176 vpermb m8, m5, m21
2177 mova m4, m10
2178 vpdpwssd m4, m12, m8 ; g0
2179 vpermb m8, m6, m0
2180 vpdpwssd m1, m13, m8 ; a1 b1
2181 vpermb m8, m6, m9
2182 vpdpwssd m2, m13, m8 ; c1 d1
2183 vpermb m8, m6, m20
2184 vpdpwssd m3, m13, m8 ; e1 f1
2185 vpermb m8, m6, m21
2186 vpdpwssd m4, m13, m8 ; g1
2187 vpermb m8, m7, m0
2188 vpdpwssd m1, m14, m8 ; a2 b2
2189 vpermb m8, m7, m9
2190 vpdpwssd m2, m14, m8 ; c2 d2
2191 vpermb m8, m7, m20
2192 vpdpwssd m3, m14, m8 ; e2 f2
2193 vpermb m8, m7, m21
2194 vpdpwssd m4, m14, m8 ; g2
2195 mova m8, [spel_h_shufD]
2196 vpermb m0, m8, m0
2197 vpdpwssd m1, m15, m0 ; a3 b3
2198 mova m0, [spel_shuf8a]
2199 vpermb m9, m8, m9
2200 vpdpwssd m2, m15, m9 ; c3 d3
2201 mova m9, [spel_shuf8b]
2202 vpermb m20, m8, m20
2203 vpdpwssd m3, m15, m20 ; e3 f3
2204 vpermb m21, m8, m21
2205 vpdpwssd m4, m15, m21 ; g3
2206 vpermt2b m1, m0, m2 ; 01 12
2207 vpermt2b m2, m0, m3 ; 23 34
2208 vpermt2b m3, m0, m4 ; 45 56
2209 .hv_w8_loop:
2210 movu ym0, [srcq+ssq*1]
2211 lea srcq, [srcq+ssq*2]
2212 vinserti32x8 m0, [srcq+ssq*0], 1
2213 mova m4, m10
2214 vpermb m21, m5, m0
2215 vpdpwssd m4, m12, m21 ; h0 i0
2216 vpermb m21, m6, m0
2217 pmaddwd m20, m16, m1 ; A0 B0
2218 vpdpwssd m4, m13, m21 ; h1 i1
2219 vpermb m21, m7, m0
2220 mova m1, m2
2221 vpdpwssd m20, m17, m2 ; A1 B1
2222 vpdpwssd m4, m14, m21 ; h2 i2
2223 vpermb m21, m8, m0
2224 mova m2, m3
2225 vpdpwssd m20, m18, m3 ; A2 B2
2226 vpdpwssd m4, m15, m21 ; h3 i3
2227 vpermt2b m3, m9, m4 ; 67 78
2228 vpdpwssd m20, m19, m3 ; A3 B3
2229 psrad m20, 10
2230 vextracti32x8 ym21, m20, 1
2231 packusdw ym20, ym21
2232 pminsw ym20, ym11
2233 mova [dstq+dsq*0], xm20
2234 vextracti128 [dstq+dsq*1], ym20, 1
2235 lea dstq, [dstq+dsq*2]
2236 sub hd, 2
2237 jg .hv_w8_loop
2238 vzeroupper
2239 RET
2240 .hv_w16:
2241 WIN64_SPILL_XMM 26
2242 vbroadcasti32x8 m5, [srcq+ssq*0+ 8]
2243 vinserti32x8 m4, m5, [srcq+ssq*0+ 0], 0
2244 vinserti32x8 m5, [srcq+ssq*0+16], 1 ; 0
2245 movu ym6, [srcq+ssq*1+ 0]
2246 movu ym7, [srcq+ssq*1+16]
2247 vinserti32x8 m6, [srcq+ssq*2+ 0], 1
2248 vinserti32x8 m7, [srcq+ssq*2+16], 1 ; 1 2
2249 add srcq, r6
2250 movu ym22, [srcq+ssq*0+ 0]
2251 movu ym23, [srcq+ssq*0+16]
2252 vinserti32x8 m22, [srcq+ssq*1+ 0], 1
2253 vinserti32x8 m23, [srcq+ssq*1+16], 1 ; 3 4
2254 movu ym24, [srcq+ssq*2+ 0]
2255 movu ym25, [srcq+ssq*2+16]
2256 add srcq, r6
2257 vinserti32x8 m24, [srcq+ssq*0+ 0], 1
2258 vinserti32x8 m25, [srcq+ssq*0+16], 1 ; 5 6
2259 vbroadcasti32x4 m20, [spel_h_shufA]
2260 vbroadcasti32x4 m21, [spel_h_shufB]
2261 mova m9, [spel_shuf16]
2262 pshufb m0, m4, m20
2263 mova m1, m10
2264 vpdpwssd m1, m12, m0 ; a0
2265 pshufb m0, m6, m20
2266 mova m2, m10
2267 vpdpwssd m2, m12, m0 ; b0
2268 pshufb m0, m7, m20
2269 mova m3, m10
2270 vpdpwssd m3, m14, m0 ; c2
2271 pshufb m0, m4, m21
2272 vpdpwssd m1, m13, m0 ; a1
2273 pshufb m0, m6, m21
2274 vpdpwssd m2, m13, m0 ; b1
2275 pshufb m0, m7, m21
2276 vpdpwssd m3, m15, m0 ; c3
2277 pshufb m0, m5, m20
2278 vpdpwssd m1, m14, m0 ; a2
2279 shufpd m6, m7, 0x55
2280 pshufb m7, m6, m20
2281 vpdpwssd m2, m14, m7 ; b2
2282 vpdpwssd m3, m12, m7 ; c0
2283 pshufb m5, m21
2284 vpdpwssd m1, m15, m5 ; a3
2285 pshufb m6, m21
2286 vpdpwssd m2, m15, m6 ; b3
2287 vpdpwssd m3, m13, m6 ; c1
2288 pshufb m0, m22, m20
2289 mova m4, m10
2290 vpdpwssd m4, m12, m0 ; d0
2291 pshufb m0, m23, m20
2292 mova m5, m10
2293 vpdpwssd m5, m14, m0 ; e2
2294 pshufb m0, m24, m20
2295 mova m6, m10
2296 vpdpwssd m6, m12, m0 ; f0
2297 pshufb m0, m25, m20
2298 mova m7, m10
2299 vpdpwssd m7, m14, m0 ; g2
2300 pshufb m0, m22, m21
2301 vpdpwssd m4, m13, m0 ; d1
2302 pshufb m0, m23, m21
2303 vpdpwssd m5, m15, m0 ; e3
2304 pshufb m0, m24, m21
2305 vpdpwssd m6, m13, m0 ; f1
2306 pshufb m0, m25, m21
2307 vpdpwssd m7, m15, m0 ; g3
2308 shufpd m22, m23, 0x55
2309 pshufb m23, m22, m20
2310 vpdpwssd m4, m14, m23 ; d2
2311 vpdpwssd m5, m12, m23 ; e0
2312 shufpd m24, m25, 0x55
2313 pshufb m25, m24, m20
2314 vpdpwssd m6, m14, m25 ; f2
2315 vpdpwssd m7, m12, m25 ; g0
2316 pshufb m22, m21
2317 vpdpwssd m4, m15, m22 ; d3
2318 vpdpwssd m5, m13, m22 ; e1
2319 pshufb m24, m21
2320 vpdpwssd m6, m15, m24 ; f3
2321 vpdpwssd m7, m13, m24 ; g1
2322 pslldq m1, 1
2323 vpermt2b m2, m9, m3 ; 12
2324 vpermt2b m4, m9, m5 ; 34
2325 vpermt2b m6, m9, m7 ; 56
2326 vpshrdd m1, m2, 16 ; 01
2327 vpshrdd m3, m2, m4, 16 ; 23
2328 vpshrdd m5, m4, m6, 16 ; 45
2329 .hv_w16_loop:
2330 movu ym24, [srcq+ssq*1+ 0]
2331 movu ym25, [srcq+ssq*1+16]
2332 lea srcq, [srcq+ssq*2]
2333 vinserti32x8 m24, [srcq+ssq*0+ 0], 1
2334 vinserti32x8 m25, [srcq+ssq*0+16], 1
2335 mova m7, m10
2336 mova m8, m10
2337 pshufb m0, m24, m20
2338 vpdpwssd m7, m12, m0 ; h0
2339 pshufb m0, m25, m20
2340 vpdpwssd m8, m14, m0 ; i2
2341 pmaddwd m22, m16, m1 ; A0
2342 mova m1, m3
2343 pmaddwd m23, m16, m2 ; B0
2344 mova m2, m4
2345 pshufb m0, m24, m21
2346 vpdpwssd m7, m13, m0 ; h1
2347 pshufb m0, m25, m21
2348 vpdpwssd m8, m15, m0 ; i3
2349 vpdpwssd m22, m17, m3 ; A1
2350 mova m3, m5
2351 vpdpwssd m23, m17, m4 ; B1
2352 mova m4, m6
2353 shufpd m24, m25, 0x55
2354 pshufb m25, m24, m20
2355 vpdpwssd m7, m14, m25 ; h2
2356 vpdpwssd m8, m12, m25 ; i0
2357 vpdpwssd m22, m18, m5 ; A2
2358 vpdpwssd m23, m18, m6 ; B2
2359 pshufb m24, m21
2360 vpdpwssd m7, m15, m24 ; h3
2361 vpdpwssd m8, m13, m24 ; i1
2362 vpermt2b m7, m9, m8 ; 78
2363 vpshrdd m5, m6, m7, 16 ; 67
2364 vpdpwssd m22, m19, m5 ; A3
2365 vpdpwssd m23, m19, m7 ; B3
2366 mova m6, m7
2367 psrad m22, 10
2368 psrad m23, 10
2369 vshufi32x4 m0, m22, m23, q3232
2370 vinserti32x8 m22, ym23, 1
2371 packusdw m22, m0
2372 pminsw m22, m11
2373 mova [dstq+dsq*0], ym22
2374 vextracti32x8 [dstq+dsq*1], m22, 1
2375 lea dstq, [dstq+dsq*2]
2376 sub hd, 2
2377 jg .hv_w16_loop
2378 RET
2379 .hv_w32:
2380 %assign stack_offset stack_offset - stack_size_padded
2381 WIN64_SPILL_XMM 32
2382 vbroadcasti32x4 m20, [spel_h_shufA]
2383 vbroadcasti32x4 m21, [spel_h_shufB]
2384 mova m22, [spel_shuf32]
2385 lea wd, [hq+wq*8-256]
2386 mov r7, srcq
2387 mov r8, dstq
2388 .hv_w32_loop0:
2389 movu m6, [srcq+ssq*0+ 0]
2390 movu m7, [srcq+ssq*0+ 8]
2391 movu m8, [srcq+ssq*0+16]
2392 mova m0, m10
2393 mova m23, m10
2394 pshufb m9, m6, m20
2395 vpdpwssd m0, m12, m9 ; a0l
2396 pshufb m9, m7, m20
2397 vpdpwssd m23, m12, m9 ; a0h
2398 vpdpwssd m0, m14, m9 ; a2l
2399 pshufb m7, m21
2400 vpdpwssd m23, m13, m7 ; a1h
2401 vpdpwssd m0, m15, m7 ; a3l
2402 pshufb m7, m8, m20
2403 vpdpwssd m23, m14, m7 ; a2h
2404 pshufb m6, m21
2405 vpdpwssd m0, m13, m6 ; a1l
2406 pshufb m8, m21
2407 vpdpwssd m23, m15, m8 ; a3h
2408 %macro PUT_8TAP_HV_W32 5 ; dst_lo, dst_hi, stride_name, stride[1-2]
2409 movu m6, [srcq+%3*%4+ 0]
2410 movu m7, [srcq+%3*%4+ 8]
2411 movu m8, [srcq+%3*%4+16]
2412 %if %4 == 2
2413 add srcq, r6
2414 %endif
2415 movu m29, [srcq+%3*%5+ 0]
2416 movu m30, [srcq+%3*%5+ 8]
2417 movu m31, [srcq+%3*%5+16]
2418 %if %5 == 2
2419 add srcq, r6
2420 %endif
2421 mova m%1, m10
2422 mova m9, m10
2423 pshufb m%2, m6, m20
2424 vpdpwssd m%1, m12, m%2 ; x0l
2425 pshufb m%2, m29, m20
2426 vpdpwssd m9, m12, m%2 ; y0l
2427 pshufb m6, m21
2428 vpdpwssd m%1, m13, m6 ; x1l
2429 pshufb m29, m21
2430 vpdpwssd m9, m13, m29 ; y1l
2431 pshufb m6, m7, m20
2432 mova m%2, m10
2433 vpdpwssd m%2, m12, m6 ; x0h
2434 pshufb m29, m30, m20
2435 vpdpwssd m%1, m14, m6 ; y2l
2436 mova m6, m10
2437 vpdpwssd m6, m12, m29 ; x0h
2438 pshufb m7, m21
2439 vpdpwssd m9, m14, m29 ; y2l
2440 pshufb m30, m21
2441 vpdpwssd m%2, m13, m7 ; x1h
2442 vpdpwssd m%1, m15, m7 ; x3l
2443 pshufb m7, m8, m20
2444 vpdpwssd m6, m13, m30 ; y1h
2445 vpdpwssd m9, m15, m30 ; y3l
2446 pshufb m30, m31, m20
2447 vpdpwssd m%2, m14, m7 ; x2h
2448 pshufb m8, m21
2449 vpdpwssd m6, m14, m30 ; y2h
2450 pshufb m31, m21
2451 vpdpwssd m%2, m15, m8 ; x3h
2452 vpdpwssd m6, m15, m31 ; y3h
2453 %if %1 == 1
2454 vpermt2b m0, m22, m%1 ; 01l
2455 vpermt2b m23, m22, m%2 ; 01h
2456 %endif
2457 vpermt2b m%1, m22, m9 ; xyl
2458 vpermt2b m%2, m22, m6 ; xyh
2459 %endmacro
2460 PUT_8TAP_HV_W32 1, 24, ssq, 1, 2 ; 12
2461 PUT_8TAP_HV_W32 3, 26, ssq, 0, 1 ; 34
2462 PUT_8TAP_HV_W32 5, 28, ssq, 2, 0 ; 56
2463 vpshrdd m2, m1, m3, 16 ; 23l
2464 vpshrdd m25, m24, m26, 16 ; 23h
2465 vpshrdd m4, m3, m5, 16 ; 45l
2466 vpshrdd m27, m26, m28, 16 ; 45h
2467 .hv_w32_loop:
2468 movu m7, [srcq+ssq*1+ 0]
2469 movu m9, [srcq+ssq*2+ 0]
2470 movu m6, [srcq+ssq*1+ 8]
2471 movu m8, [srcq+ssq*2+ 8]
2472 mova m29, m10
2473 mova m31, m10
2474 pshufb m30, m7, m20
2475 vpdpwssd m29, m12, m30 ; h0l
2476 pshufb m30, m9, m20
2477 vpdpwssd m31, m12, m30 ; i0l
2478 pshufb m7, m21
2479 vpdpwssd m29, m13, m7 ; h1l
2480 pshufb m9, m21
2481 vpdpwssd m31, m13, m9 ; i1l
2482 pshufb m7, m6, m20
2483 vpdpwssd m29, m14, m7 ; h2l
2484 pshufb m9, m8, m20
2485 vpdpwssd m31, m14, m9 ; i2l
2486 pshufb m6, m21
2487 vpdpwssd m29, m15, m6 ; h3l
2488 pshufb m8, m21
2489 vpdpwssd m31, m15, m8 ; i3l
2490 mova m30, m10
2491 vpdpwssd m30, m12, m7 ; h0h
2492 movu m7, [srcq+ssq*1+16]
2493 lea srcq, [srcq+ssq*2]
2494 vpermt2b m29, m22, m31 ; 78l
2495 mova m31, m10
2496 vpdpwssd m31, m12, m9 ; i0h
2497 movu m9, [srcq+ssq*0+16]
2498 vpdpwssd m30, m13, m6 ; h1h
2499 pshufb m6, m7, m20
2500 vpdpwssd m31, m13, m8 ; i1h
2501 pshufb m8, m9, m20
2502 vpdpwssd m30, m14, m6 ; h2h
2503 pmaddwd m6, m16, m0 ; A0l
2504 pshufb m7, m21
2505 vpdpwssd m31, m14, m8 ; i2h
2506 pmaddwd m8, m16, m23 ; A0h
2507 pshufb m9, m21
2508 vpdpwssd m30, m15, m7 ; h3h
2509 pmaddwd m7, m16, m1 ; B0l
2510 vpdpwssd m31, m15, m9 ; i3h
2511 pmaddwd m9, m16, m24 ; B0h
2512 mova m0, m2
2513 vpdpwssd m6, m17, m2 ; A1l
2514 mova m23, m25
2515 vpdpwssd m8, m17, m25 ; A1h
2516 mova m1, m3
2517 vpdpwssd m7, m17, m3 ; B1l
2518 mova m24, m26
2519 vpdpwssd m9, m17, m26 ; B1h
2520 vpermt2b m30, m22, m31 ; 78h
2521 vpdpwssd m6, m18, m4 ; A2l
2522 mova m2, m4
2523 vpdpwssd m8, m18, m27 ; A2h
2524 mova m25, m27
2525 vpdpwssd m7, m18, m5 ; B2l
2526 mova m3, m5
2527 vpdpwssd m9, m18, m28 ; B2h
2528 mova m26, m28
2529 vpshrdd m4, m5, m29, 16 ; 67l
2530 vpdpwssd m6, m19, m4 ; A3l
2531 vpshrdd m27, m28, m30, 16 ; 67h
2532 vpdpwssd m8, m19, m27 ; A3h
2533 mova m5, m29
2534 vpdpwssd m7, m19, m29 ; B3l
2535 mova m28, m30
2536 vpdpwssd m9, m19, m30 ; B3h
2537 REPX {psrad x, 10}, m6, m8, m7, m9
2538 packusdw m6, m8
2539 packusdw m7, m9
2540 pminsw m6, m11
2541 pminsw m7, m11
2542 mova [dstq+dsq*0], m6
2543 mova [dstq+dsq*1], m7
2544 lea dstq, [dstq+dsq*2]
2545 sub hd, 2
2546 jg .hv_w32_loop
2547 add r7, 64
2548 add r8, 64
2549 movzx hd, wb
2550 mov srcq, r7
2551 mov dstq, r8
2552 sub wd, 1<<8
2553 jg .hv_w32_loop0
2554 RET
2555
2556 %if WIN64
2557 DECLARE_REG_TMP 6, 4
2558 %else
2559 DECLARE_REG_TMP 6, 7
2560 %endif
2561
2562 MC_8TAP_FN prep, sharp, SHARP, SHARP
2563 MC_8TAP_FN prep, sharp_smooth, SHARP, SMOOTH
2564 MC_8TAP_FN prep, smooth_sharp, SMOOTH, SHARP
2565 MC_8TAP_FN prep, smooth, SMOOTH, SMOOTH
2566 MC_8TAP_FN prep, sharp_regular, SHARP, REGULAR
2567 MC_8TAP_FN prep, regular_sharp, REGULAR, SHARP
2568 MC_8TAP_FN prep, smooth_regular, SMOOTH, REGULAR
2569 MC_8TAP_FN prep, regular_smooth, REGULAR, SMOOTH
2570 MC_8TAP_FN prep, regular, REGULAR, REGULAR
2571
2572 cglobal prep_8tap_16bpc, 3, 8, 16, tmp, src, stride, w, h, mx, my, stride3
2573 %define base r7-prep_avx512icl
2574 imul mxd, mxm, 0x010101
2575 add mxd, t0d ; 8tap_h, mx, 4tap_h
2576 imul myd, mym, 0x010101
2577 add myd, t1d ; 8tap_v, my, 4tap_v
2578 lea r7, [prep_avx512icl]
2579 mov wd, wm
2580 movifnidn hd, hm
2581 test mxd, 0xf00
2582 jnz .h
2583 test myd, 0xf00
2584 jnz .v
2585 tzcnt wd, wd
2586 mov r5d, r7m ; bitdepth_max
2587 vpbroadcastd m5, [pw_8192]
2588 movzx wd, word [r7+wq*2+table_offset(prep,)]
2589 shr r5d, 11
2590 vpbroadcastd m4, [r7-prep_avx512icl+prep_mul+r5*4]
2591 add wq, r7
2592 lea r6, [strideq*3]
2593 %if WIN64
2594 pop r7
2595 %endif
2596 jmp wq
2597 .h_w4:
2598 movzx mxd, mxb
2599 sub srcq, 2
2600 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
2601 mov r5d, r7m
2602 vbroadcasti32x4 m4, [spel_h_shufA]
2603 vbroadcasti32x4 m5, [spel_h_shufB]
2604 shr r5d, 11
2605 mova ym9, [prep_endA]
2606 psllw xmm0, [base+prep_hv_shift+r5*8]
2607 mova [tmpq], xmm0
2608 vpbroadcastd m6, [tmpq+4]
2609 vpbroadcastd m7, [tmpq+8]
2610 .h_w4_loop:
2611 movu xm2, [srcq+strideq*0]
2612 vinserti32x4 ym2, [srcq+strideq*1], 1
2613 vinserti32x4 m2, [srcq+strideq*2], 2
2614 vinserti32x4 m2, [srcq+r6 ], 3
2615 lea srcq, [srcq+strideq*4]
2616 mova m0, m10
2617 pshufb m1, m2, m4
2618 vpdpwssd m0, m6, m1
2619 pshufb m2, m5
2620 vpdpwssd m0, m7, m2
2621 vpermb m0, m9, m0
2622 mova [tmpq], ym0
2623 add tmpq, 32
2624 sub hd, 4
2625 jg .h_w4_loop
2626 RET
2627 .h:
2628 test myd, 0xf00
2629 jnz .hv
2630 vpbroadcastd m10, [prep_8tap_rnd]
2631 lea r6, [strideq*3]
2632 cmp wd, 4
2633 je .h_w4
2634 shr mxd, 16
2635 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
2636 mov r5d, r7m
2637 sub srcq, 6
2638 shr r5d, 11
2639 psllw xmm0, [base+prep_hv_shift+r5*8]
2640 mova [tmpq], xmm0
2641 vpbroadcastd m12, xmm0
2642 vpbroadcastd m13, [tmpq+ 4]
2643 vpbroadcastd m14, [tmpq+ 8]
2644 vpbroadcastd m15, [tmpq+12]
2645 cmp wd, 16
2646 je .h_w16
2647 jg .h_w32
2648 .h_w8:
2649 mova m6, [spel_h_shufA]
2650 movu m7, [spel_h_shufB]
2651 movu m8, [spel_h_shufC]
2652 mova m9, [spel_h_shufD]
2653 mova m11, [prep_endB]
2654 .h_w8_loop:
2655 movu ym4, [srcq+strideq*0]
2656 vinserti32x8 m4, [srcq+strideq*1], 1
2657 movu ym5, [srcq+strideq*2]
2658 vinserti32x8 m5, [srcq+r6 ], 1
2659 lea srcq, [srcq+strideq*4]
2660 mova m0, m10
2661 mova m1, m10
2662 vpermb m2, m6, m4
2663 vpermb m3, m6, m5
2664 vpdpwssd m0, m12, m2
2665 vpdpwssd m1, m12, m3
2666 vpermb m2, m7, m4
2667 vpermb m3, m7, m5
2668 vpdpwssd m0, m13, m2
2669 vpdpwssd m1, m13, m3
2670 vpermb m2, m8, m4
2671 vpermb m3, m8, m5
2672 vpdpwssd m0, m14, m2
2673 vpdpwssd m1, m14, m3
2674 vpermb m2, m9, m4
2675 vpermb m3, m9, m5
2676 vpdpwssd m0, m15, m2
2677 vpdpwssd m1, m15, m3
2678 vpermt2b m0, m11, m1
2679 mova [tmpq], m0
2680 add tmpq, 64
2681 sub hd, 4
2682 jg .h_w8_loop
2683 RET
2684 .h_w16:
2685 vbroadcasti32x4 m6, [spel_h_shufA]
2686 vbroadcasti32x4 m7, [spel_h_shufB]
2687 mova m11, [prep_endC]
2688 .h_w16_loop:
2689 movu ym2, [srcq+strideq*0+ 0]
2690 vinserti32x8 m2, [srcq+strideq*1+ 0], 1
2691 movu ym3, [srcq+strideq*0+16]
2692 vinserti32x8 m3, [srcq+strideq*1+16], 1
2693 lea srcq, [srcq+strideq*2]
2694 mova m0, m10
2695 mova m1, m10
2696 pshufb m4, m2, m6
2697 vpdpwssd m0, m12, m4 ; a0
2698 pshufb m4, m3, m6
2699 vpdpwssd m1, m14, m4 ; b2
2700 pshufb m4, m2, m7
2701 vpdpwssd m0, m13, m4 ; a1
2702 pshufb m4, m3, m7
2703 vpdpwssd m1, m15, m4 ; b3
2704 shufpd m2, m3, 0x55
2705 pshufb m4, m2, m6
2706 vpdpwssd m0, m14, m4 ; a2
2707 vpdpwssd m1, m12, m4 ; b0
2708 pshufb m2, m7
2709 vpdpwssd m0, m15, m2 ; a3
2710 vpdpwssd m1, m13, m2 ; b1
2711 vpermt2b m0, m11, m1
2712 mova [tmpq], m0
2713 add tmpq, 64
2714 sub hd, 2
2715 jg .h_w16_loop
2716 RET
2717 .h_w32:
2718 vbroadcasti32x4 m6, [spel_h_shufA]
2719 lea srcq, [srcq+wq*2]
2720 vbroadcasti32x4 m7, [spel_h_shufB]
2721 neg wq
2722 mova m11, [prep_endC]
2723 .h_w32_loop0:
2724 mov r6, wq
2725 .h_w32_loop:
2726 movu m2, [srcq+r6*2+ 0]
2727 movu m3, [srcq+r6*2+ 8]
2728 mova m0, m10
2729 mova m1, m10
2730 pshufb m4, m2, m6
2731 vpdpwssd m0, m12, m4 ; a0
2732 pshufb m4, m3, m6
2733 vpdpwssd m1, m12, m4 ; b0
2734 vpdpwssd m0, m14, m4 ; a2
2735 movu m4, [srcq+r6*2+16]
2736 pshufb m3, m7
2737 vpdpwssd m1, m13, m3 ; b1
2738 vpdpwssd m0, m15, m3 ; a3
2739 pshufb m3, m4, m6
2740 vpdpwssd m1, m14, m3 ; b2
2741 pshufb m2, m7
2742 vpdpwssd m0, m13, m2 ; a1
2743 pshufb m4, m7
2744 vpdpwssd m1, m15, m4 ; b3
2745 vpermt2b m0, m11, m1
2746 mova [tmpq], m0
2747 add tmpq, 64
2748 add r6, 32
2749 jl .h_w32_loop
2750 add srcq, strideq
2751 dec hd
2752 jg .h_w32_loop0
2753 RET
2754 .v:
2755 movzx mxd, myb
2756 shr myd, 16
2757 cmp hd, 4
2758 cmove myd, mxd
2759 mov r5d, r7m
2760 vpbroadcastd m10, [prep_8tap_rnd]
2761 pmovsxbw xmm0, [base+subpel_filters+myq*8]
2762 tzcnt r6d, wd
2763 shr r5d, 11
2764 movzx r6d, word [r7+r6*2+table_offset(prep, _8tap_v)]
2765 psllw xmm0, [base+prep_hv_shift+r5*8]
2766 add r7, r6
2767 lea r6, [strideq*3]
2768 sub srcq, r6
2769 mova [tmpq], xmm0
2770 vpbroadcastd m12, xmm0
2771 vpbroadcastd m13, [tmpq+ 4]
2772 vpbroadcastd m14, [tmpq+ 8]
2773 vpbroadcastd m15, [tmpq+12]
2774 jmp r7
2775 .v_w4:
2776 movq xmm1, [srcq+strideq*0]
2777 vpbroadcastq ymm0, [srcq+strideq*1]
2778 vpbroadcastq ymm2, [srcq+strideq*2]
2779 add srcq, r6
2780 vpbroadcastq ymm4, [srcq+strideq*0]
2781 vpbroadcastq ymm3, [srcq+strideq*1]
2782 vpbroadcastq ymm5, [srcq+strideq*2]
2783 mova xm11, [prep_endA]
2784 add srcq, r6
2785 vpblendd ymm1, ymm0, 0x30
2786 vpblendd ymm0, ymm2, 0x30
2787 punpcklwd ymm1, ymm0 ; 01 12
2788 vpbroadcastq ymm0, [srcq+strideq*0]
2789 vpblendd ymm2, ymm4, 0x30
2790 vpblendd ymm4, ymm3, 0x30
2791 punpcklwd ymm2, ymm4 ; 23 34
2792 vpblendd ymm3, ymm5, 0x30
2793 vpblendd ymm5, ymm0, 0x30
2794 punpcklwd ymm3, ymm5 ; 45 56
2795 .v_w4_loop:
2796 vpbroadcastq ymm5, [srcq+strideq*1]
2797 lea srcq, [srcq+strideq*2]
2798 mova ymm4, ym10
2799 vpdpwssd ymm4, ym12, ymm1 ; a0 b0
2800 mova ymm1, ymm2
2801 vpdpwssd ymm4, ym13, ymm2 ; a1 b1
2802 mova ymm2, ymm3
2803 vpdpwssd ymm4, ym14, ymm3 ; a2 b2
2804 vpblendd ymm3, ymm0, ymm5, 0x30
2805 vpbroadcastq ymm0, [srcq+strideq*0]
2806 vpblendd ymm5, ymm0, 0x30
2807 punpcklwd ymm3, ymm5 ; 67 78
2808 vpdpwssd ymm4, ym15, ymm3 ; a3 b3
2809 vpermb ymm4, ym11, ymm4
2810 mova [tmpq], xmm4
2811 add tmpq, 16
2812 sub hd, 2
2813 jg .v_w4_loop
2814 vzeroupper
2815 RET
2816 .v_w8:
2817 vbroadcasti32x4 m2, [srcq+strideq*2]
2818 vinserti32x4 m1, m2, [srcq+strideq*0], 0
2819 vinserti32x4 m1, [srcq+strideq*1], 1 ; 0 1 2
2820 add srcq, r6
2821 vinserti32x4 ym2, [srcq+strideq*0], 1
2822 vinserti32x4 m2, [srcq+strideq*1], 2 ; 2 3 4
2823 mova m6, [spel_v_shuf8]
2824 movu xm0, [srcq+strideq*1]
2825 vinserti32x4 ym0, [srcq+strideq*2], 1
2826 add srcq, r6
2827 vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6
2828 mova ym11, [prep_endB]
2829 vpermb m1, m6, m1 ; 01 12
2830 vpermb m2, m6, m2 ; 23 34
2831 vpermb m3, m6, m0 ; 45 56
2832 .v_w8_loop:
2833 vinserti32x4 m0, [srcq+strideq*1], 3
2834 lea srcq, [srcq+strideq*2]
2835 movu xm5, [srcq+strideq*0]
2836 mova m4, m10
2837 vpdpwssd m4, m12, m1 ; a0 b0
2838 mova m1, m2
2839 vshufi32x4 m0, m5, q1032 ; 6 7 8
2840 vpdpwssd m4, m13, m2 ; a1 b1
2841 mova m2, m3
2842 vpdpwssd m4, m14, m3 ; a2 b2
2843 vpermb m3, m6, m0 ; 67 78
2844 vpdpwssd m4, m15, m3 ; a3 b3
2845 vpermb m4, m11, m4
2846 mova [tmpq], ym4
2847 add tmpq, 32
2848 sub hd, 2
2849 jg .v_w8_loop
2850 RET
2851 .v_w16:
2852 vbroadcasti32x8 m1, [srcq+strideq*1]
2853 vinserti32x8 m0, m1, [srcq+strideq*0], 0
2854 vinserti32x8 m1, [srcq+strideq*2], 1
2855 mova m8, [spel_v_shuf16]
2856 add srcq, r6
2857 movu ym3, [srcq+strideq*0]
2858 vinserti32x8 m3, [srcq+strideq*1], 1
2859 movu ym5, [srcq+strideq*2]
2860 add srcq, r6
2861 vinserti32x8 m5, [srcq+strideq*0], 1
2862 mova m11, [prep_endA]
2863 vpermb m0, m8, m0 ; 01
2864 vpermb m1, m8, m1 ; 12
2865 vpermb m3, m8, m3 ; 34
2866 vpermb m5, m8, m5 ; 56
2867 vpshrdd m2, m1, m3, 16 ; 23
2868 vpshrdd m4, m3, m5, 16 ; 45
2869 .v_w16_loop:
2870 mova m6, m10
2871 mova m7, m10
2872 vpdpwssd m6, m12, m0 ; a0
2873 mova m0, m2
2874 vpdpwssd m7, m12, m1 ; b0
2875 mova m1, m3
2876 vpdpwssd m6, m13, m2 ; a1
2877 mova m2, m4
2878 vpdpwssd m7, m13, m3 ; b1
2879 mova m3, m5
2880 vpdpwssd m6, m14, m4 ; a2
2881 mova m4, m5
2882 vpdpwssd m7, m14, m5 ; b2
2883 movu ym5, [srcq+strideq*1]
2884 lea srcq, [srcq+strideq*2]
2885 vinserti32x8 m5, [srcq+strideq*0], 1
2886 vpermb m5, m8, m5 ; 78
2887 vpshrdd m4, m5, 16 ; 67
2888 vpdpwssd m6, m15, m4 ; a3
2889 vpdpwssd m7, m15, m5 ; b3
2890 vpermt2b m6, m11, m7
2891 mova [tmpq], m6
2892 add tmpq, 64
2893 sub hd, 2
2894 jg .v_w16_loop
2895 RET
2896 .v_w32:
2897 .v_w64:
2898 .v_w128:
2899 %if WIN64
2900 PUSH r8
2901 movaps [rsp+stack_offset+8], xmm6
2902 %endif
2903 lea r5, [hq+wq*8-256]
2904 mov r7, srcq
2905 mov r8, tmpq
2906 .v_w32_loop0:
2907 movu m16, [srcq+strideq*0]
2908 movu m17, [srcq+strideq*1]
2909 movu m18, [srcq+strideq*2]
2910 add srcq, r6
2911 movu m19, [srcq+strideq*0]
2912 movu m20, [srcq+strideq*1]
2913 movu m21, [srcq+strideq*2]
2914 add srcq, r6
2915 movu m22, [srcq+strideq*0]
2916 mova m11, [prep_endC]
2917 punpcklwd m0, m16, m17 ; 01l
2918 punpckhwd m16, m17 ; 01h
2919 punpcklwd m1, m17, m18 ; 12l
2920 punpckhwd m17, m18 ; 12h
2921 punpcklwd m2, m18, m19 ; 23l
2922 punpckhwd m18, m19 ; 23h
2923 punpcklwd m3, m19, m20 ; 34l
2924 punpckhwd m19, m20 ; 34h
2925 punpcklwd m4, m20, m21 ; 45l
2926 punpckhwd m20, m21 ; 45h
2927 punpcklwd m5, m21, m22 ; 56l
2928 punpckhwd m21, m22 ; 56h
2929 .v_w32_loop:
2930 mova m6, m10
2931 vpdpwssd m6, m12, m0 ; a0l
2932 mova m8, m10
2933 vpdpwssd m8, m12, m16 ; a0h
2934 mova m7, m10
2935 vpdpwssd m7, m12, m1 ; b0l
2936 mova m9, m10
2937 vpdpwssd m9, m12, m17 ; b0h
2938 mova m0, m2
2939 vpdpwssd m6, m13, m2 ; a1l
2940 mova m16, m18
2941 vpdpwssd m8, m13, m18 ; a1h
2942 mova m1, m3
2943 vpdpwssd m7, m13, m3 ; b1l
2944 mova m17, m19
2945 vpdpwssd m9, m13, m19 ; b1h
2946 mova m2, m4
2947 vpdpwssd m6, m14, m4 ; a2l
2948 mova m18, m20
2949 vpdpwssd m8, m14, m20 ; a2h
2950 mova m3, m5
2951 vpdpwssd m7, m14, m5 ; b2l
2952 mova m19, m21
2953 vpdpwssd m9, m14, m21 ; b2h
2954 movu m21, [srcq+strideq*1]
2955 lea srcq, [srcq+strideq*2]
2956 punpcklwd m4, m22, m21 ; 67l
2957 punpckhwd m20, m22, m21 ; 67h
2958 movu m22, [srcq+strideq*0]
2959 vpdpwssd m6, m15, m4 ; a3l
2960 vpdpwssd m8, m15, m20 ; a3h
2961 punpcklwd m5, m21, m22 ; 78l
2962 punpckhwd m21, m22 ; 78h
2963 vpdpwssd m7, m15, m5 ; b3l
2964 vpdpwssd m9, m15, m21 ; b3h
2965 vpermt2b m6, m11, m8
2966 vpermt2b m7, m11, m9
2967 mova [tmpq+wq*0], m6
2968 mova [tmpq+wq*2], m7
2969 lea tmpq, [tmpq+wq*4]
2970 sub hd, 2
2971 jg .v_w32_loop
2972 add r7, 64
2973 add r8, 64
2974 movzx hd, r5b
2975 mov srcq, r7
2976 mov tmpq, r8
2977 sub r5d, 1<<8
2978 jg .v_w32_loop0
2979 %if WIN64
2980 movaps xmm6, [rsp+stack_offset+8]
2981 POP r8
2982 %endif
2983 vzeroupper
2984 RET
2985 .hv:
2986 cmp wd, 4
2987 jg .hv_w8
2988 movzx mxd, mxb
2989 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
2990 movzx mxd, myb
2991 shr myd, 16
2992 cmp hd, 4
2993 cmove myd, mxd
2994 mov r5d, r7m
2995 pmovsxbw xmm1, [base+subpel_filters+myq*8]
2996 lea r6, [strideq*3]
2997 sub srcq, 2
2998 shr r5d, 11
2999 sub srcq, r6
3000 psllw xmm0, [base+prep_hv_shift+r5*8]
3001 psllw xmm1, 2
3002 vpbroadcastd m10, [prep_8tap_rnd]
3003 vpbroadcastd ym11, [pd_128]
3004 mova xm21, [prep_endA]
3005 mova [tmpq+ 0], xmm0
3006 mova [tmpq+16], xmm1
3007 vpbroadcastd m8, [tmpq+ 4]
3008 vpbroadcastd m9, [tmpq+ 8]
3009 vpbroadcastd ym12, xmm1
3010 vpbroadcastd ym13, [tmpq+20]
3011 vpbroadcastd ym14, [tmpq+24]
3012 vpbroadcastd ym15, [tmpq+28]
3013 movu xm4, [srcq+strideq*0]
3014 vinserti32x4 ym4, [srcq+strideq*1], 1
3015 vinserti32x4 m4, [srcq+strideq*2], 2
3016 add srcq, r6
3017 vinserti32x4 m4, [srcq+strideq*0], 3 ; 0 1 2 3
3018 movu xm0, [srcq+strideq*1]
3019 vinserti32x4 ym0, [srcq+strideq*2], 1
3020 add srcq, r6
3021 vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6
3022 vbroadcasti32x4 m19, [spel_h_shufA]
3023 vbroadcasti32x4 m20, [spel_h_shufB]
3024 mova ym6, [spel_shuf4a]
3025 mova ym7, [spel_shuf4b]
3026 mova m2, m10
3027 mova m3, m10
3028 pshufb m1, m4, m19
3029 vpdpwssd m2, m8, m1
3030 pshufb m1, m0, m19
3031 vpdpwssd m3, m8, m1
3032 pshufb m4, m20
3033 vpdpwssd m2, m9, m4
3034 pshufb m0, m20
3035 vpdpwssd m3, m9, m0
3036 vpermb m1, m6, m2 ; 01 12
3037 vshufi32x4 m2, m3, q1032
3038 vpermb m3, m6, m3 ; 45 56
3039 vpermb m2, m6, m2 ; 23 34
3040 .hv_w4_loop:
3041 movu xm18, [srcq+strideq*1]
3042 lea srcq, [srcq+strideq*2]
3043 vinserti128 ym18, [srcq+strideq*0], 1
3044 mova ym16, ym11
3045 mova ym4, ym10
3046 pshufb ym17, ym18, ym19
3047 vpdpwssd ym16, ym12, ym1 ; a0 b0
3048 vpdpwssd ym4, ym8, ym17
3049 pshufb ym18, ym20
3050 mova ym1, ym2
3051 vpdpwssd ym16, ym13, ym2 ; a1 b1
3052 vpdpwssd ym4, ym9, ym18 ; 7 8
3053 mova ym2, ym3
3054 vpdpwssd ym16, ym14, ym3 ; a2 b2
3055 vpermt2b ym3, ym7, ym4 ; 67 78
3056 vpdpwssd ym16, ym15, ym3 ; a3 b3
3057 vpermb ym16, ym21, ym16
3058 mova [tmpq], xm16
3059 add tmpq, 16
3060 sub hd, 2
3061 jg .hv_w4_loop
3062 vzeroupper
3063 RET
3064 .hv_w8:
3065 shr mxd, 16
3066 pmovsxbw xmm0, [base+subpel_filters+mxq*8]
3067 movzx mxd, myb
3068 shr myd, 16
3069 cmp hd, 6
3070 cmovs myd, mxd
3071 mov r5d, r7m
3072 pmovsxbw xmm1, [base+subpel_filters+myq*8]
3073 lea r6, [strideq*3]
3074 sub srcq, 6
3075 shr r5d, 11
3076 sub srcq, r6
3077 vpbroadcastd m10, [prep_8tap_rnd]
3078 vpbroadcastd m11, [pd_128]
3079 psllw xmm0, [base+prep_hv_shift+r5*8]
3080 psllw xmm1, 2
3081 mova [tmpq+ 0], xmm0
3082 mova [tmpq+16], xmm1
3083 vpbroadcastd m12, xmm0
3084 vpbroadcastd m13, [tmpq+ 4]
3085 vpbroadcastd m14, [tmpq+ 8]
3086 vpbroadcastd m15, [tmpq+12]
3087 vpbroadcastd m16, xmm1
3088 vpbroadcastd m17, [tmpq+20]
3089 vpbroadcastd m18, [tmpq+24]
3090 vpbroadcastd m19, [tmpq+28]
3091 cmp wd, 16
3092 je .hv_w16
3093 jg .hv_w32
3094 WIN64_SPILL_XMM 23
3095 mova m5, [spel_h_shufA]
3096 movu ym0, [srcq+strideq*0]
3097 vinserti32x8 m0, [srcq+strideq*1], 1 ; 0 1
3098 movu ym9, [srcq+strideq*2]
3099 add srcq, r6
3100 vinserti32x8 m9, [srcq+strideq*0], 1 ; 2 3
3101 movu ym20, [srcq+strideq*1]
3102 vinserti32x8 m20, [srcq+strideq*2], 1 ; 4 5
3103 add srcq, r6
3104 movu ym21, [srcq+strideq*0] ; 6
3105 movu m6, [spel_h_shufB]
3106 movu m7, [spel_h_shufC]
3107 mova ym22, [prep_endB]
3108 vpermb m8, m5, m0
3109 mova m1, m10
3110 vpdpwssd m1, m12, m8 ; a0 b0
3111 vpermb m8, m5, m9
3112 mova m2, m10
3113 vpdpwssd m2, m12, m8 ; c0 d0
3114 vpermb m8, m5, m20
3115 mova m3, m10
3116 vpdpwssd m3, m12, m8 ; e0 f0
3117 vpermb m8, m5, m21
3118 mova m4, m10
3119 vpdpwssd m4, m12, m8 ; g0
3120 vpermb m8, m6, m0
3121 vpdpwssd m1, m13, m8 ; a1 b1
3122 vpermb m8, m6, m9
3123 vpdpwssd m2, m13, m8 ; c1 d1
3124 vpermb m8, m6, m20
3125 vpdpwssd m3, m13, m8 ; e1 f1
3126 vpermb m8, m6, m21
3127 vpdpwssd m4, m13, m8 ; g1
3128 vpermb m8, m7, m0
3129 vpdpwssd m1, m14, m8 ; a2 b2
3130 vpermb m8, m7, m9
3131 vpdpwssd m2, m14, m8 ; c2 d2
3132 vpermb m8, m7, m20
3133 vpdpwssd m3, m14, m8 ; e2 f2
3134 vpermb m8, m7, m21
3135 vpdpwssd m4, m14, m8 ; g2
3136 mova m8, [spel_h_shufD]
3137 vpermb m0, m8, m0
3138 vpdpwssd m1, m15, m0 ; a3 b3
3139 mova m0, [spel_shuf8a]
3140 vpermb m9, m8, m9
3141 vpdpwssd m2, m15, m9 ; c3 d3
3142 mova m9, [spel_shuf8b]
3143 vpermb m20, m8, m20
3144 vpdpwssd m3, m15, m20 ; e3 f3
3145 vpermb m21, m8, m21
3146 vpdpwssd m4, m15, m21 ; g3
3147 vpermt2b m1, m0, m2 ; 01 12
3148 vpermt2b m2, m0, m3 ; 23 34
3149 vpermt2b m3, m0, m4 ; 45 56
3150 .hv_w8_loop:
3151 movu ym0, [srcq+strideq*1]
3152 lea srcq, [srcq+strideq*2]
3153 vinserti32x8 m0, [srcq+strideq*0], 1
3154 mova m4, m10
3155 mova m20, m11
3156 vpermb m21, m5, m0
3157 vpdpwssd m4, m12, m21 ; h0 i0
3158 vpermb m21, m6, m0
3159 vpdpwssd m20, m16, m1 ; A0 B0
3160 vpdpwssd m4, m13, m21 ; h1 i1
3161 vpermb m21, m7, m0
3162 mova m1, m2
3163 vpdpwssd m20, m17, m2 ; A1 B1
3164 vpdpwssd m4, m14, m21 ; h2 i2
3165 vpermb m21, m8, m0
3166 mova m2, m3
3167 vpdpwssd m20, m18, m3 ; A2 B2
3168 vpdpwssd m4, m15, m21 ; h3 i3
3169 vpermt2b m3, m9, m4 ; 67 78
3170 vpdpwssd m20, m19, m3 ; A3 B3
3171 vpermb m20, m22, m20
3172 mova [tmpq], ym20
3173 add tmpq, 32
3174 sub hd, 2
3175 jg .hv_w8_loop
3176 RET
3177 .hv_w16:
3178 %assign stack_offset stack_offset - stack_size_padded
3179 WIN64_SPILL_XMM 27
3180 vbroadcasti32x8 m5, [srcq+strideq*0+ 8]
3181 vinserti32x8 m4, m5, [srcq+strideq*0+ 0], 0
3182 vinserti32x8 m5, [srcq+strideq*0+16], 1 ; 0
3183 movu ym6, [srcq+strideq*1+ 0]
3184 movu ym7, [srcq+strideq*1+16]
3185 vinserti32x8 m6, [srcq+strideq*2+ 0], 1
3186 vinserti32x8 m7, [srcq+strideq*2+16], 1 ; 1 2
3187 add srcq, r6
3188 movu ym22, [srcq+strideq*0+ 0]
3189 movu ym23, [srcq+strideq*0+16]
3190 vinserti32x8 m22, [srcq+strideq*1+ 0], 1
3191 vinserti32x8 m23, [srcq+strideq*1+16], 1 ; 3 4
3192 movu ym24, [srcq+strideq*2+ 0]
3193 movu ym25, [srcq+strideq*2+16]
3194 add srcq, r6
3195 vinserti32x8 m24, [srcq+strideq*0+ 0], 1
3196 vinserti32x8 m25, [srcq+strideq*0+16], 1 ; 5 6
3197 vbroadcasti32x4 m20, [spel_h_shufA]
3198 vbroadcasti32x4 m21, [spel_h_shufB]
3199 mova m9, [spel_shuf16]
3200 mova m26, [prep_endB]
3201 pshufb m0, m4, m20
3202 mova m1, m10
3203 vpdpwssd m1, m12, m0 ; a0
3204 pshufb m0, m6, m20
3205 mova m2, m10
3206 vpdpwssd m2, m12, m0 ; b0
3207 pshufb m0, m7, m20
3208 mova m3, m10
3209 vpdpwssd m3, m14, m0 ; c2
3210 pshufb m0, m4, m21
3211 vpdpwssd m1, m13, m0 ; a1
3212 pshufb m0, m6, m21
3213 vpdpwssd m2, m13, m0 ; b1
3214 pshufb m0, m7, m21
3215 vpdpwssd m3, m15, m0 ; c3
3216 pshufb m0, m5, m20
3217 vpdpwssd m1, m14, m0 ; a2
3218 shufpd m6, m7, 0x55
3219 pshufb m7, m6, m20
3220 vpdpwssd m2, m14, m7 ; b2
3221 vpdpwssd m3, m12, m7 ; c0
3222 pshufb m5, m21
3223 vpdpwssd m1, m15, m5 ; a3
3224 pshufb m6, m21
3225 vpdpwssd m2, m15, m6 ; b3
3226 vpdpwssd m3, m13, m6 ; c1
3227 pshufb m0, m22, m20
3228 mova m4, m10
3229 vpdpwssd m4, m12, m0 ; d0
3230 pshufb m0, m23, m20
3231 mova m5, m10
3232 vpdpwssd m5, m14, m0 ; e2
3233 pshufb m0, m24, m20
3234 mova m6, m10
3235 vpdpwssd m6, m12, m0 ; f0
3236 pshufb m0, m25, m20
3237 mova m7, m10
3238 vpdpwssd m7, m14, m0 ; g2
3239 pshufb m0, m22, m21
3240 vpdpwssd m4, m13, m0 ; d1
3241 pshufb m0, m23, m21
3242 vpdpwssd m5, m15, m0 ; e3
3243 pshufb m0, m24, m21
3244 vpdpwssd m6, m13, m0 ; f1
3245 pshufb m0, m25, m21
3246 vpdpwssd m7, m15, m0 ; g3
3247 shufpd m22, m23, 0x55
3248 pshufb m23, m22, m20
3249 vpdpwssd m4, m14, m23 ; d2
3250 vpdpwssd m5, m12, m23 ; e0
3251 shufpd m24, m25, 0x55
3252 pshufb m25, m24, m20
3253 vpdpwssd m6, m14, m25 ; f2
3254 vpdpwssd m7, m12, m25 ; g0
3255 pshufb m22, m21
3256 vpdpwssd m4, m15, m22 ; d3
3257 vpdpwssd m5, m13, m22 ; e1
3258 pshufb m24, m21
3259 vpdpwssd m6, m15, m24 ; f3
3260 vpdpwssd m7, m13, m24 ; g1
3261 pslldq m1, 1
3262 vpermt2b m2, m9, m3 ; 12
3263 vpermt2b m4, m9, m5 ; 34
3264 vpermt2b m6, m9, m7 ; 56
3265 vpshrdd m1, m2, 16 ; 01
3266 vpshrdd m3, m2, m4, 16 ; 23
3267 vpshrdd m5, m4, m6, 16 ; 45
3268 .hv_w16_loop:
3269 movu ym24, [srcq+strideq*1+ 0]
3270 movu ym25, [srcq+strideq*1+16]
3271 lea srcq, [srcq+strideq*2]
3272 vinserti32x8 m24, [srcq+strideq*0+ 0], 1
3273 vinserti32x8 m25, [srcq+strideq*0+16], 1
3274 mova m7, m10
3275 mova m8, m10
3276 pshufb m0, m24, m20
3277 vpdpwssd m7, m12, m0 ; h0
3278 mova m22, m11
3279 pshufb m0, m25, m20
3280 vpdpwssd m8, m14, m0 ; i2
3281 mova m23, m11
3282 vpdpwssd m22, m16, m1 ; A0
3283 mova m1, m3
3284 vpdpwssd m23, m16, m2 ; B0
3285 mova m2, m4
3286 pshufb m0, m24, m21
3287 vpdpwssd m7, m13, m0 ; h1
3288 pshufb m0, m25, m21
3289 vpdpwssd m8, m15, m0 ; i3
3290 vpdpwssd m22, m17, m3 ; A1
3291 mova m3, m5
3292 vpdpwssd m23, m17, m4 ; B1
3293 mova m4, m6
3294 shufpd m24, m25, 0x55
3295 pshufb m25, m24, m20
3296 vpdpwssd m7, m14, m25 ; h2
3297 vpdpwssd m8, m12, m25 ; i0
3298 vpdpwssd m22, m18, m5 ; A2
3299 vpdpwssd m23, m18, m6 ; B2
3300 pshufb m24, m21
3301 vpdpwssd m7, m15, m24 ; h3
3302 vpdpwssd m8, m13, m24 ; i1
3303 vpermt2b m7, m9, m8 ; 78
3304 vpshrdd m5, m6, m7, 16 ; 67
3305 vpdpwssd m22, m19, m5 ; A3
3306 vpdpwssd m23, m19, m7 ; B3
3307 mova m6, m7
3308 vpermt2b m22, m26, m23
3309 mova [tmpq], m22
3310 add tmpq, 64
3311 sub hd, 2
3312 jg .hv_w16_loop
3313 RET
3314 .hv_w32:
3315 %if WIN64
3316 %assign stack_offset stack_offset - stack_size_padded
3317 PUSH r8
3318 %assign regs_used regs_used + 1
3319 WIN64_SPILL_XMM 32
3320 %endif
3321 vbroadcasti32x4 m20, [spel_h_shufA]
3322 vbroadcasti32x4 m21, [spel_h_shufB]
3323 mova m22, [spel_shuf32]
3324 lea r5d, [hq+wq*8-256]
3325 mov r7, srcq
3326 mov r8, tmpq
3327 .hv_w32_loop0:
3328 movu m6, [srcq+strideq*0+ 0]
3329 movu m7, [srcq+strideq*0+ 8]
3330 movu m8, [srcq+strideq*0+16]
3331 mova m0, m10
3332 mova m23, m10
3333 pshufb m9, m6, m20
3334 vpdpwssd m0, m12, m9 ; a0l
3335 pshufb m9, m7, m20
3336 vpdpwssd m23, m12, m9 ; a0h
3337 vpdpwssd m0, m14, m9 ; a2l
3338 pshufb m7, m21
3339 vpdpwssd m23, m13, m7 ; a1h
3340 vpdpwssd m0, m15, m7 ; a3l
3341 pshufb m7, m8, m20
3342 vpdpwssd m23, m14, m7 ; a2h
3343 pshufb m6, m21
3344 vpdpwssd m0, m13, m6 ; a1l
3345 pshufb m8, m21
3346 vpdpwssd m23, m15, m8 ; a3h
3347 PUT_8TAP_HV_W32 1, 24, strideq, 1, 2 ; 12
3348 PUT_8TAP_HV_W32 3, 26, strideq, 0, 1 ; 34
3349 PUT_8TAP_HV_W32 5, 28, strideq, 2, 0 ; 56
3350 vpshrdd m2, m1, m3, 16 ; 23l
3351 vpshrdd m25, m24, m26, 16 ; 23h
3352 vpshrdd m4, m3, m5, 16 ; 45l
3353 vpshrdd m27, m26, m28, 16 ; 45h
3354 .hv_w32_loop:
3355 movu m7, [srcq+strideq*1+ 0]
3356 movu m9, [srcq+strideq*2+ 0]
3357 movu m6, [srcq+strideq*1+ 8]
3358 movu m8, [srcq+strideq*2+ 8]
3359 mova m29, m10
3360 mova m31, m10
3361 pshufb m30, m7, m20
3362 vpdpwssd m29, m12, m30 ; h0l
3363 pshufb m30, m9, m20
3364 vpdpwssd m31, m12, m30 ; i0l
3365 pshufb m7, m21
3366 vpdpwssd m29, m13, m7 ; h1l
3367 pshufb m9, m21
3368 vpdpwssd m31, m13, m9 ; i1l
3369 pshufb m7, m6, m20
3370 vpdpwssd m29, m14, m7 ; h2l
3371 pshufb m9, m8, m20
3372 vpdpwssd m31, m14, m9 ; i2l
3373 pshufb m6, m21
3374 vpdpwssd m29, m15, m6 ; h3l
3375 pshufb m8, m21
3376 vpdpwssd m31, m15, m8 ; i3l
3377 mova m30, m10
3378 vpdpwssd m30, m12, m7 ; h0h
3379 movu m7, [srcq+strideq*1+16]
3380 lea srcq, [srcq+strideq*2]
3381 vpermt2b m29, m22, m31 ; 78l
3382 mova m31, m10
3383 vpdpwssd m31, m12, m9 ; i0h
3384 movu m9, [srcq+strideq*0+16]
3385 vpdpwssd m30, m13, m6 ; h1h
3386 pshufb m6, m7, m20
3387 vpdpwssd m31, m13, m8 ; i1h
3388 pshufb m8, m9, m20
3389 vpdpwssd m30, m14, m6 ; h2h
3390 mova m6, m11
3391 vpdpwssd m6, m16, m0 ; A0l
3392 pshufb m7, m21
3393 vpdpwssd m31, m14, m8 ; i2h
3394 mova m8, m11
3395 vpdpwssd m8, m16, m23 ; A0h
3396 pshufb m9, m21
3397 vpdpwssd m30, m15, m7 ; h3h
3398 mova m7, m11
3399 vpdpwssd m7, m16, m1 ; B0l
3400 vpdpwssd m31, m15, m9 ; i3h
3401 mova m9, m11
3402 vpdpwssd m9, m16, m24 ; B0h
3403 mova m0, m2
3404 vpdpwssd m6, m17, m2 ; A1l
3405 mova m23, m25
3406 vpdpwssd m8, m17, m25 ; A1h
3407 mova m1, m3
3408 vpdpwssd m7, m17, m3 ; B1l
3409 mova m24, m26
3410 vpdpwssd m9, m17, m26 ; B1h
3411 vpermt2b m30, m22, m31 ; 78h
3412 mova m31, [prep_endC]
3413 vpdpwssd m6, m18, m4 ; A2l
3414 mova m2, m4
3415 vpdpwssd m8, m18, m27 ; A2h
3416 mova m25, m27
3417 vpdpwssd m7, m18, m5 ; B2l
3418 mova m3, m5
3419 vpdpwssd m9, m18, m28 ; B2h
3420 mova m26, m28
3421 vpshrdd m4, m5, m29, 16 ; 67l
3422 vpdpwssd m6, m19, m4 ; A3l
3423 vpshrdd m27, m28, m30, 16 ; 67h
3424 vpdpwssd m8, m19, m27 ; A3h
3425 mova m5, m29
3426 vpdpwssd m7, m19, m29 ; B3l
3427 mova m28, m30
3428 vpdpwssd m9, m19, m30 ; B3h
3429 vpermt2b m6, m31, m8
3430 vpermt2b m7, m31, m9
3431 mova [tmpq+wq*0], m6
3432 mova [tmpq+wq*2], m7
3433 lea tmpq, [tmpq+wq*4]
3434 sub hd, 2
3435 jg .hv_w32_loop
3436 add r7, 64
3437 add r8, 64
3438 movzx hd, r5b
3439 mov srcq, r7
3440 mov tmpq, r8
3441 sub r5d, 1<<8
3442 jg .hv_w32_loop0
3443 RET
3444
3445 %if WIN64
3446 DECLARE_REG_TMP 5
3447 %else
3448 DECLARE_REG_TMP 7
3449 %endif
3450
3451 cglobal warp_affine_8x8t_16bpc, 4, 7, 22, tmp, ts
3452 %define base r6-pd_0to7
3453 mov t0d, r7m
3454 lea r6, [pd_0to7]
3455 shr t0d, 11
3456 vpbroadcastd m8, [base+warp_8x8t_rnd_v]
3457 vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4]
3458 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main
3459 psrad m14, m16, 15
3460 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2
3461 psrad m16, 15
3462 packssdw m14, m16
3463 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2
3464 psrad m15, m16, 15
3465 call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2
3466 add tsq, tsq
3467 psrad m16, 15
3468 packssdw m15, m16
3469 jmp mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).end
3470
3471 cglobal warp_affine_8x8_16bpc, 4, 7, 22, dst, ds, src, ss, abcd
3472 mov t0d, r7m ; pixel_max
3473 lea r6, [pd_0to7]
3474 shr t0d, 11
3475 vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4]
3476 vpbroadcastd m8, [base+warp_8x8_rnd_v+t0*4]
3477 call .main
3478 psrad m14, m16, 13
3479 call .main2
3480 psrad m16, 13
3481 packusdw m14, m16
3482 call .main2
3483 psrad m15, m16, 13
3484 call .main2
3485 vpbroadcastd m0, [base+bidir_shift+t0*4]
3486 vpsrlvw m14, m0
3487 psrad m16, 13
3488 packusdw m15, m16
3489 vpsrlvw m15, m0
3490 .end:
3491 mova m0, [base+warp8x8_end]
3492 vpermb m16, m0, m14
3493 lea r2, [dsq*3]
3494 mova [dstq+dsq*0], xm16
3495 vextracti128 [dstq+dsq*1], ym16, 1
3496 vextracti32x4 [dstq+dsq*2], m16, 2
3497 vextracti32x4 [dstq+r2 ], m16, 3
3498 vpermb m16, m0, m15
3499 lea dstq, [dstq+dsq*4]
3500 mova [dstq+dsq*0], xm16
3501 vextracti128 [dstq+dsq*1], ym16, 1
3502 vextracti32x4 [dstq+dsq*2], m16, 2
3503 vextracti32x4 [dstq+r2 ], m16, 3
3504 RET
3505 .main:
3506 vpbroadcastd ym3, [base+pd_512]
3507 %if WIN64
3508 mov abcdq, r5mp
3509 vpaddd ym18, ym3, r6m {1to8} ; mx
3510 %else
3511 add r5d, 512
3512 vpbroadcastd ym18, r5d
3513 %endif
3514 vpaddd ym20, ym3, r7m {1to8} ; my
3515 mova ym16, [base+pd_0to7]
3516 vpbroadcastd ym19, [abcdq+4*0] ; alpha
3517 vpbroadcastd ym21, [abcdq+4*1] ; gamma
3518 lea r4, [ssq*3+6]
3519 vpdpwssd ym18, ym19, ym16 ; tmx
3520 vpdpwssd ym20, ym21, ym16 ; tmy
3521 sub srcq, r4
3522 mova m10, [base+warp8x8_permA]
3523 lea r4, [mc_warp_filter+64*8]
3524 vbroadcasti32x4 m12, [base+warp8x8_permC]
3525 kxnorb k1, k1, k1
3526 vbroadcasti32x4 m13, [base+warp8x8_permD]
3527 movu ym5, [srcq+0]
3528 vinserti32x8 m5, [srcq+8], 1
3529 psrad ym17, ym18, 10
3530 mova m11, [base+warp8x8_permB]
3531 kmovb k2, k1
3532 vpgatherdq m3{k1}, [r4+ym17*8] ; filter_x0
3533 psrad ym19, 16 ; beta
3534 psrad ym21, 16 ; delta
3535 paddd ym18, ym19
3536 vpermb m4, m10, m5
3537 vpbroadcastq m9, [base+warp_shift_h+t0*8]
3538 pshufd m3, m3, q3120
3539 paddd m7, m1, m1
3540 pshufb m2, m3, m12
3541 vpdpwssd m1, m4, m2
3542 vpermb m5, m11, m5
3543 vshufi32x4 m4, m5, q1021
3544 pshufb m3, m13
3545 vpdpwssd m1, m4, m3
3546 call .h
3547 psllq m2, m1, 32
3548 paddd m1, m2
3549 vpmultishiftqb m1, m9, m1
3550 vpshrdq m1, m0, 48 ; 01 12
3551 call .h
3552 vpshrdq m2, m1, m0, 48 ; 23 34
3553 call .h
3554 vpshrdq m3, m2, m0, 48 ; 45 56
3555 .main2:
3556 call .h
3557 psrad ym6, ym20, 10
3558 kmovb k1, k2
3559 paddd ym17, ym20, ym21 ; my += delta
3560 vpgatherdq m20{k2}, [r4+ym6*8] ; filter_y0
3561 psrad ym16, ym17, 10
3562 kmovb k2, k1
3563 vpgatherdq m6{k1}, [r4+ym16*8] ; filter_y1
3564 shufps m5, m20, m6, q2020
3565 mova m16, m8
3566 pshufb m4, m5, m12
3567 vpdpwssd m16, m1, m4 ; a0 b0
3568 pshufb m5, m13
3569 mova m1, m2
3570 vpdpwssd m16, m2, m5 ; a1 b1
3571 shufps m6, m20, m6, q3131
3572 paddd ym20, ym17, ym21
3573 pshufb m4, m6, m12
3574 mova m2, m3
3575 vpdpwssd m16, m3, m4 ; a2 b2
3576 vpshrdq m3, m0, 48 ; 67 78
3577 pshufb m6, m13
3578 vpdpwssd m16, m3, m6 ; a3 b3
3579 ret
3580 ALIGN function_align
3581 .h:
3582 movu ym16, [srcq+ssq*1]
3583 psrad ym6, ym18, 10
3584 lea srcq, [srcq+ssq*2]
3585 vinserti32x8 m5, m16, [srcq+ssq*0], 1
3586 kmovb k1, k2
3587 paddd ym17, ym18, ym19 ; mx += beta
3588 vpgatherdq m18{k2}, [r4+ym6*8] ; filter_x1
3589 psrad ym16, ym17, 10
3590 kmovb k2, k1
3591 vpgatherdq m6{k1}, [r4+ym16*8] ; filter_x2
3592 vpermb m4, m10, m5
3593 shufps m16, m18, m6, q2020
3594 shufps m6, m18, m6, q3131
3595 mova m0, m7
3596 pshufb m18, m16, m12
3597 vpdpwssd m0, m4, m18 ; a0 b0
3598 vpermb m5, m11, m5
3599 pshufb m18, m6, m13
3600 vpdpwssd m0, m5, m18 ; a3 b3
3601 paddd ym18, ym17, ym19
3602 vshufi32x4 m17, m4, m5, q1021
3603 pshufb m16, m13
3604 vpdpwssd m0, m17, m16 ; a1 b1
3605 vshufi32x4 m4, m5, q2132
3606 pshufb m6, m12
3607 vpdpwssd m0, m4, m6 ; a2 b2
3608 vpmultishiftqb m0, m9, m0 ; a a b b
3609 ret
3610
3611 %macro BIDIR_FN 0
3612 call .main
3613 lea stride3q, [strideq*3]
3614 jmp wq
3615 .w4:
3616 movq [dstq ], xm0
3617 movhps [dstq+strideq*1], xm0
3618 vextracti32x4 xm2, ym0, 1
3619 movq [dstq+strideq*2], xm2
3620 movhps [dstq+stride3q ], xm2
3621 cmp hd, 8
3622 jl .w4_end
3623 vextracti32x4 xm2, m0, 2
3624 lea dstq, [dstq+strideq*4]
3625 movq [dstq ], xm2
3626 movhps [dstq+strideq*1], xm2
3627 vextracti32x4 xm0, m0, 3
3628 movq [dstq+strideq*2], xm0
3629 movhps [dstq+stride3q ], xm0
3630 je .w4_end
3631 lea dstq, [dstq+strideq*4]
3632 movq [dstq ], xm1
3633 movhps [dstq+strideq*1], xm1
3634 vextracti32x4 xm0, ym1, 1
3635 movq [dstq+strideq*2], xm0
3636 movhps [dstq+stride3q ], xm0
3637 vextracti32x4 xm0, m1, 2
3638 lea dstq, [dstq+strideq*4]
3639 movq [dstq ], xm0
3640 movhps [dstq+strideq*1], xm0
3641 vextracti32x4 xm1, m1, 3
3642 movq [dstq+strideq*2], xm1
3643 movhps [dstq+stride3q ], xm1
3644 .w4_end:
3645 RET
3646 .w8_loop:
3647 call .main
3648 lea dstq, [dstq+strideq*4]
3649 .w8:
3650 mova [dstq+strideq*0], xm0
3651 vextracti32x4 [dstq+strideq*1], ym0, 1
3652 vextracti32x4 [dstq+strideq*2], m0, 2
3653 vextracti32x4 [dstq+stride3q ], m0, 3
3654 sub hd, 8
3655 jl .w8_end
3656 lea dstq, [dstq+strideq*4]
3657 mova [dstq+strideq*0], xm1
3658 vextracti32x4 [dstq+strideq*1], ym1, 1
3659 vextracti32x4 [dstq+strideq*2], m1, 2
3660 vextracti32x4 [dstq+stride3q ], m1, 3
3661 jg .w8_loop
3662 .w8_end:
3663 RET
3664 .w16_loop:
3665 call .main
3666 lea dstq, [dstq+strideq*4]
3667 .w16:
3668 mova [dstq+strideq*0], ym0
3669 vextracti32x8 [dstq+strideq*1], m0, 1
3670 mova [dstq+strideq*2], ym1
3671 vextracti32x8 [dstq+stride3q ], m1, 1
3672 sub hd, 4
3673 jg .w16_loop
3674 RET
3675 .w32_loop:
3676 call .main
3677 lea dstq, [dstq+strideq*2]
3678 .w32:
3679 mova [dstq+strideq*0], m0
3680 mova [dstq+strideq*1], m1
3681 sub hd, 2
3682 jg .w32_loop
3683 RET
3684 .w64_loop:
3685 call .main
3686 add dstq, strideq
3687 .w64:
3688 mova [dstq+64*0], m0
3689 mova [dstq+64*1], m1
3690 dec hd
3691 jg .w64_loop
3692 RET
3693 .w128_loop:
3694 call .main
3695 add dstq, strideq
3696 .w128:
3697 mova [dstq+64*0], m0
3698 mova [dstq+64*1], m1
3699 call .main
3700 mova [dstq+64*2], m0
3701 mova [dstq+64*3], m1
3702 dec hd
3703 jg .w128_loop
3704 RET
3705 %endmacro
3706
3707 %if WIN64
3708 DECLARE_REG_TMP 5
3709 %else
3710 DECLARE_REG_TMP 7
3711 %endif
3712
3713 cglobal avg_16bpc, 4, 7, 4, dst, stride, tmp1, tmp2, w, h, stride3
3714 %define base r6-avg_avx512icl_table
3715 lea r6, [avg_avx512icl_table]
3716 tzcnt wd, wm
3717 mov t0d, r6m ; pixel_max
3718 movsxd wq, [r6+wq*4]
3719 shr t0d, 11
3720 vpbroadcastd m2, [base+avg_round+t0*4]
3721 vpbroadcastd m3, [base+avg_shift+t0*4]
3722 movifnidn hd, hm
3723 add wq, r6
3724 BIDIR_FN
3725 ALIGN function_align
3726 .main:
3727 mova m0, [tmp1q+64*0]
3728 paddsw m0, [tmp2q+64*0]
3729 mova m1, [tmp1q+64*1]
3730 paddsw m1, [tmp2q+64*1]
3731 add tmp1q, 64*2
3732 add tmp2q, 64*2
3733 pmaxsw m0, m2
3734 pmaxsw m1, m2
3735 psubsw m0, m2
3736 psubsw m1, m2
3737 vpsrlvw m0, m3
3738 vpsrlvw m1, m3
3739 ret
3740
3741 cglobal w_avg_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, h, stride3
3742 %define base r6-w_avg_avx512icl_table
3743 lea r6, [w_avg_avx512icl_table]
3744 tzcnt wd, wm
3745 mov t0d, r7m ; pixel_max
3746 shr t0d, 11
3747 movsxd wq, [r6+wq*4]
3748 vpbroadcastd m5, [base+w_avg_round+t0*4]
3749 vpbroadcastd m7, [base+bidir_shift+t0*4]
3750 add wq, r6
3751 mov r6d, r6m ; weight
3752 lea t0d, [r6-16]
3753 shl r6d, 16
3754 sub r6d, t0d ; 16-weight, weight
3755 movifnidn hd, hm
3756 vpbroadcastd m6, r6d
3757 BIDIR_FN
3758 ALIGN function_align
3759 .main:
3760 mova m3, [tmp1q+64*0]
3761 mova m1, [tmp2q+64*0]
3762 mova m0, [tmp1q+64*1]
3763 mova m4, [tmp2q+64*1]
3764 add tmp1q, 64*2
3765 add tmp2q, 64*2
3766 punpcklwd m2, m1, m3
3767 punpckhwd m1, m3
3768 punpcklwd m3, m4, m0
3769 punpckhwd m4, m0
3770 mova m0, m5
3771 vpdpwssd m0, m6, m2
3772 mova m2, m5
3773 vpdpwssd m2, m6, m1
3774 mova m1, m5
3775 vpdpwssd m1, m6, m3
3776 mova m3, m5
3777 vpdpwssd m3, m6, m4
3778 REPX {psrad x, 2}, m0, m2, m1, m3
3779 packusdw m0, m2
3780 packusdw m1, m3
3781 vpsrlvw m0, m7
3782 vpsrlvw m1, m7
3783 ret
3784
3785 cglobal mask_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3
3786 %define base r7-mask_avx512icl_table
3787 lea r7, [mask_avx512icl_table]
3788 tzcnt wd, wm
3789 mov r6d, r7m ; pixel_max
3790 movifnidn hd, hm
3791 shr r6d, 11
3792 movsxd wq, [r7+wq*4]
3793 vpbroadcastd m8, [base+pw_64]
3794 vpbroadcastd m9, [base+mask_round+r6*4]
3795 vpbroadcastd m10, [base+bidir_shift+r6*4]
3796 mov maskq, maskmp
3797 add wq, r7
3798 BIDIR_FN
3799 ALIGN function_align
3800 .main:
3801 pmovzxbw m1, [maskq+32*0]
3802 mova m4, [tmp1q+64*0]
3803 mova m2, [tmp2q+64*0]
3804 pmovzxbw m6, [maskq+32*1]
3805 mova m5, [tmp1q+64*1]
3806 mova m3, [tmp2q+64*1]
3807 add maskq, 32*2
3808 add tmp1q, 64*2
3809 add tmp2q, 64*2
3810 punpcklwd m7, m4, m2
3811 punpckhwd m4, m2
3812 psubw m0, m8, m1
3813 punpcklwd m2, m1, m0 ; m, 64-m
3814 punpckhwd m1, m0
3815 mova m0, m9
3816 vpdpwssd m0, m7, m2
3817 mova m2, m9
3818 vpdpwssd m2, m4, m1 ; tmp1 * m + tmp2 * (64-m)
3819 punpcklwd m7, m5, m3
3820 punpckhwd m5, m3
3821 psubw m1, m8, m6
3822 punpcklwd m3, m6, m1
3823 punpckhwd m6, m1
3824 mova m1, m9
3825 vpdpwssd m1, m7, m3
3826 mova m3, m9
3827 vpdpwssd m3, m5, m6
3828 REPX {psrad x, 4}, m0, m2, m1, m3
3829 packusdw m0, m2
3830 packusdw m1, m3
3831 vpsrlvw m0, m10
3832 vpsrlvw m1, m10
3833 ret
3834
3835 cglobal w_mask_420_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
3836 %define base r7-w_mask_420_avx512icl_table
3837 lea r7, [w_mask_420_avx512icl_table]
3838 tzcnt wd, wm
3839 mov r6d, r8m ; pixel_max
3840 movifnidn hd, hm
3841 shr r6d, 11
3842 movsxd wq, [r7+wq*4]
3843 vpbroadcastd m10, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
3844 vpbroadcastd m11, [base+pw_64]
3845 vpbroadcastd m12, [base+mask_round+r6*4]
3846 vpbroadcastd m13, [base+bidir_shift+r6*4]
3847 mov r6d, r7m ; sign
3848 vpbroadcastd m14, [base+w_mask_round+r6*4]
3849 mova ym15, [w_mask_end42x]
3850 mov maskq, maskmp
3851 add wq, r7
3852 call .main
3853 lea stride3q, [strideq*3]
3854 jmp wq
3855 .w4:
3856 mova m4, [w_mask_shuf4]
3857 vpermt2b m2, m4, m3
3858 mova m3, m14
3859 vpdpbusd m3, m2, [pb_64] {1to16}
3860 vpermb m3, m15, m3
3861 movq [dstq+strideq*0], xm0
3862 movhps [dstq+strideq*1], xm0
3863 vextracti32x4 xm2, ym0, 1
3864 movq [dstq+strideq*2], xm2
3865 movhps [dstq+stride3q ], xm2
3866 mova [maskq], xm3
3867 cmp hd, 8
3868 jl .w4_end
3869 vextracti32x4 xm2, m0, 2
3870 lea dstq, [dstq+strideq*4]
3871 movq [dstq+strideq*0], xm2
3872 movhps [dstq+strideq*1], xm2
3873 vextracti32x4 xm0, m0, 3
3874 movq [dstq+strideq*2], xm0
3875 movhps [dstq+stride3q ], xm0
3876 je .w4_end
3877 lea dstq, [dstq+strideq*4]
3878 movq [dstq+strideq*0], xm1
3879 movhps [dstq+strideq*1], xm1
3880 vextracti32x4 xm2, ym1, 1
3881 movq [dstq+strideq*2], xm2
3882 movhps [dstq+stride3q ], xm2
3883 vextracti32x4 xm2, m1, 2
3884 lea dstq, [dstq+strideq*4]
3885 movq [dstq+strideq*0], xm2
3886 movhps [dstq+strideq*1], xm2
3887 vextracti32x4 xm1, m1, 3
3888 movq [dstq+strideq*2], xm1
3889 movhps [dstq+stride3q ], xm1
3890 .w4_end:
3891 RET
3892 .w8:
3893 mova m8, [w_mask_shuf8]
3894 vpbroadcastd m9, [pb_64]
3895 jmp .w8_start
3896 .w8_loop:
3897 call .main
3898 lea dstq, [dstq+strideq*4]
3899 add maskq, 16
3900 .w8_start:
3901 vpermt2b m2, m8, m3
3902 mova m3, m14
3903 vpdpbusd m3, m2, m9
3904 vpermb m3, m15, m3
3905 mova [dstq+strideq*0], xm0
3906 vextracti32x4 [dstq+strideq*1], ym0, 1
3907 vextracti32x4 [dstq+strideq*2], m0, 2
3908 vextracti32x4 [dstq+stride3q ], m0, 3
3909 mova [maskq], xm3
3910 sub hd, 8
3911 jl .w8_end
3912 lea dstq, [dstq+strideq*4]
3913 mova [dstq+strideq*0], xm1
3914 vextracti32x4 [dstq+strideq*1], ym1, 1
3915 vextracti32x4 [dstq+strideq*2], m1, 2
3916 vextracti32x4 [dstq+stride3q ], m1, 3
3917 jg .w8_loop
3918 .w8_end:
3919 RET
3920 .w16:
3921 mova m8, [w_mask_shuf16]
3922 vpbroadcastd m9, [pb_64]
3923 jmp .w16_start
3924 .w16_loop:
3925 call .main
3926 lea dstq, [dstq+strideq*4]
3927 add maskq, 16
3928 .w16_start:
3929 vpermt2b m2, m8, m3
3930 mova m3, m14
3931 vpdpbusd m3, m2, m9
3932 vpermb m3, m15, m3
3933 mova [dstq+strideq*0], ym0
3934 vextracti32x8 [dstq+strideq*1], m0, 1
3935 mova [dstq+strideq*2], ym1
3936 vextracti32x8 [dstq+stride3q ], m1, 1
3937 mova [maskq], xm3
3938 sub hd, 4
3939 jg .w16_loop
3940 RET
3941 .w32_loop:
3942 call .main
3943 lea dstq, [dstq+strideq*4]
3944 add maskq, 32
3945 .w32:
3946 paddw m2, m3
3947 mova m8, m14
3948 vpdpwssd m8, m11, m2
3949 mova [dstq+strideq*0], m0
3950 mova [dstq+strideq*1], m1
3951 call .main
3952 paddw m2, m3
3953 mova m3, m14
3954 vpdpwssd m3, m11, m2
3955 vpermt2b m8, m15, m3
3956 mova [dstq+strideq*2], m0
3957 mova [dstq+stride3q ], m1
3958 mova [maskq], ym8
3959 sub hd, 4
3960 jg .w32_loop
3961 RET
3962 .w64_loop:
3963 call .main
3964 lea dstq, [dstq+strideq*2]
3965 add maskq, 32
3966 .w64:
3967 mova m8, m2
3968 mova m9, m3
3969 mova [dstq+strideq*0+64*0], m0
3970 mova [dstq+strideq*0+64*1], m1
3971 call .main
3972 paddw m8, m2
3973 paddw m9, m3
3974 mova m2, m14
3975 vpdpwssd m2, m11, m8
3976 mova m3, m14
3977 vpdpwssd m3, m11, m9
3978 vpermt2b m2, m15, m3
3979 mova [dstq+strideq*1+64*0], m0
3980 mova [dstq+strideq*1+64*1], m1
3981 mova [maskq], ym2
3982 sub hd, 2
3983 jg .w64_loop
3984 RET
3985 .w128_loop:
3986 call .main
3987 lea dstq, [dstq+strideq*2]
3988 add maskq, 64
3989 .w128:
3990 mova m16, m2
3991 mova m8, m3
3992 mova [dstq+strideq*0+64*0], m0
3993 mova [dstq+strideq*0+64*1], m1
3994 call .main
3995 mova m17, m2
3996 mova m9, m3
3997 mova [dstq+strideq*0+64*2], m0
3998 mova [dstq+strideq*0+64*3], m1
3999 call .main
4000 paddw m2, m16
4001 paddw m3, m8
4002 mova m16, m14
4003 vpdpwssd m16, m11, m2
4004 mova m8, m14
4005 vpdpwssd m8, m11, m3
4006 mova [dstq+strideq*1+64*0], m0
4007 mova [dstq+strideq*1+64*1], m1
4008 call .main
4009 paddw m2, m17
4010 paddw m3, m9
4011 mova m17, m14
4012 vpdpwssd m17, m11, m2
4013 mova m9, m14
4014 vpdpwssd m9, m11, m3
4015 vpermt2b m16, m15, m8
4016 vpermt2b m17, m15, m9
4017 mova [dstq+strideq*1+64*2], m0
4018 mova [dstq+strideq*1+64*3], m1
4019 mova [maskq+32*0], ym16
4020 mova [maskq+32*1], ym17
4021 sub hd, 2
4022 jg .w128_loop
4023 vzeroupper
4024 RET
4025 ALIGN function_align
4026 .main:
4027 mova m1, [tmp1q+64*0]
4028 mova m3, [tmp2q+64*0]
4029 mova m4, [tmp1q+64*1]
4030 mova m7, [tmp2q+64*1]
4031 add tmp1q, 64*2
4032 add tmp2q, 64*2
4033 psubsw m6, m1, m3
4034 punpcklwd m5, m3, m1
4035 pabsw m6, m6
4036 punpckhwd m3, m1
4037 psubusw m6, m10, m6
4038 psrlw m6, 10 ; 64-m
4039 psubw m2, m11, m6 ; m
4040 punpcklwd m1, m6, m2
4041 punpckhwd m6, m2
4042 mova m0, m12
4043 vpdpwssd m0, m5, m1
4044 mova m1, m12
4045 vpdpwssd m1, m3, m6
4046 psubsw m5, m4, m7
4047 punpcklwd m6, m7, m4
4048 pabsw m5, m5
4049 punpckhwd m7, m4
4050 psubusw m5, m10, m5
4051 psrlw m5, 10
4052 psubw m3, m11, m5
4053 punpcklwd m4, m5, m3
4054 psrad m0, 4
4055 punpckhwd m5, m3
4056 psrad m1, 4
4057 packusdw m0, m1
4058 mova m1, m12
4059 vpdpwssd m1, m6, m4
4060 mova m4, m12
4061 vpdpwssd m4, m7, m5
4062 psrad m1, 4
4063 psrad m4, 4
4064 packusdw m1, m4
4065 vpsrlvw m0, m13
4066 vpsrlvw m1, m13
4067 ret
4068
4069 cglobal w_mask_422_16bpc, 4, 8, 15, dst, stride, tmp1, tmp2, w, h, mask, stride3
4070 %define base r7-w_mask_422_avx512icl_table
4071 lea r7, [w_mask_422_avx512icl_table]
4072 tzcnt wd, wm
4073 mov r6d, r8m ; pixel_max
4074 movifnidn hd, hm
4075 shr r6d, 11
4076 movsxd wq, [r7+wq*4]
4077 vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
4078 vpbroadcastd m9, [base+pw_64]
4079 vpbroadcastd m10, [base+mask_round+r6*4]
4080 vpbroadcastd m11, [base+bidir_shift+r6*4]
4081 mov r6d, r7m ; sign
4082 vpbroadcastd m12, [base+w_mask_round+r6*4]
4083 mova ym13, [w_mask_end42x]
4084 mov maskq, maskmp
4085 add wq, r7
4086 paddw m14, m9, m9 ; pw_128
4087 call .main
4088 lea stride3q, [strideq*3]
4089 jmp wq
4090 .w4:
4091 movq [dstq+strideq*0], xm0
4092 movhps [dstq+strideq*1], xm0
4093 vextracti32x4 xm2, ym0, 1
4094 movq [dstq+strideq*2], xm2
4095 movhps [dstq+stride3q ], xm2
4096 cmp hd, 8
4097 jl .w4_end
4098 vextracti32x4 xm2, m0, 2
4099 lea dstq, [dstq+strideq*4]
4100 movq [dstq+strideq*0], xm2
4101 movhps [dstq+strideq*1], xm2
4102 vextracti32x4 xm0, m0, 3
4103 movq [dstq+strideq*2], xm0
4104 movhps [dstq+stride3q ], xm0
4105 je .w4_end
4106 lea dstq, [dstq+strideq*4]
4107 movq [dstq+strideq*0], xm1
4108 movhps [dstq+strideq*1], xm1
4109 vextracti32x4 xm2, ym1, 1
4110 movq [dstq+strideq*2], xm2
4111 movhps [dstq+stride3q ], xm2
4112 vextracti32x4 xm2, m1, 2
4113 lea dstq, [dstq+strideq*4]
4114 movq [dstq+strideq*0], xm2
4115 movhps [dstq+strideq*1], xm2
4116 vextracti32x4 xm1, m1, 3
4117 movq [dstq+strideq*2], xm1
4118 movhps [dstq+stride3q ], xm1
4119 .w4_end:
4120 RET
4121 .w8_loop:
4122 call .main
4123 lea dstq, [dstq+strideq*4]
4124 .w8:
4125 mova [dstq+strideq*0], xm0
4126 vextracti32x4 [dstq+strideq*1], ym0, 1
4127 vextracti32x4 [dstq+strideq*2], m0, 2
4128 vextracti32x4 [dstq+stride3q ], m0, 3
4129 sub hd, 8
4130 jl .w8_end
4131 lea dstq, [dstq+strideq*4]
4132 mova [dstq+strideq*0], xm1
4133 vextracti32x4 [dstq+strideq*1], ym1, 1
4134 vextracti32x4 [dstq+strideq*2], m1, 2
4135 vextracti32x4 [dstq+stride3q ], m1, 3
4136 jg .w8_loop
4137 .w8_end:
4138 RET
4139 .w16_loop:
4140 call .main
4141 lea dstq, [dstq+strideq*4]
4142 .w16:
4143 mova [dstq+strideq*0], ym0
4144 vextracti32x8 [dstq+strideq*1], m0, 1
4145 mova [dstq+strideq*2], ym1
4146 vextracti32x8 [dstq+stride3q ], m1, 1
4147 sub hd, 4
4148 jg .w16_loop
4149 RET
4150 .w32_loop:
4151 call .main
4152 lea dstq, [dstq+strideq*2]
4153 .w32:
4154 mova [dstq+strideq*0], m0
4155 mova [dstq+strideq*1], m1
4156 sub hd, 2
4157 jg .w32_loop
4158 RET
4159 .w64_loop:
4160 call .main
4161 add dstq, strideq
4162 .w64:
4163 mova [dstq+64*0], m0
4164 mova [dstq+64*1], m1
4165 dec hd
4166 jg .w64_loop
4167 RET
4168 .w128_loop:
4169 call .main
4170 add dstq, strideq
4171 .w128:
4172 mova [dstq+64*0], m0
4173 mova [dstq+64*1], m1
4174 call .main
4175 mova [dstq+64*2], m0
4176 mova [dstq+64*3], m1
4177 dec hd
4178 jg .w128_loop
4179 RET
4180 ALIGN function_align
4181 .main:
4182 mova m1, [tmp1q+64*0]
4183 mova m3, [tmp2q+64*0]
4184 mova m4, [tmp1q+64*1]
4185 mova m7, [tmp2q+64*1]
4186 add tmp1q, 64*2
4187 add tmp2q, 64*2
4188 psubsw m6, m1, m3
4189 punpcklwd m5, m3, m1
4190 pabsw m6, m6
4191 punpckhwd m3, m1
4192 psubusw m6, m8, m6
4193 psrlw m6, 10
4194 psubw m2, m9, m6
4195 punpcklwd m1, m6, m2
4196 punpckhwd m6, m2
4197 mova m0, m10
4198 vpdpwssd m0, m5, m1
4199 mova m1, m10
4200 vpdpwssd m1, m3, m6
4201 psubsw m5, m4, m7
4202 punpcklwd m6, m7, m4
4203 pabsw m5, m5
4204 punpckhwd m7, m4
4205 psubusw m5, m8, m5
4206 psrlw m5, 10
4207 psubw m3, m9, m5
4208 punpcklwd m4, m5, m3
4209 psrad m0, 4
4210 punpckhwd m5, m3
4211 psrad m1, 4
4212 packusdw m0, m1
4213 mova m1, m10
4214 vpdpwssd m1, m6, m4
4215 mova m4, m10
4216 vpdpwssd m4, m7, m5
4217 mova m5, m12
4218 vpdpwssd m5, m14, m2
4219 mova m2, m12
4220 vpdpwssd m2, m14, m3
4221 psrad m1, 4
4222 psrad m4, 4
4223 packusdw m1, m4
4224 vpermt2b m5, m13, m2
4225 vpsrlvw m0, m11
4226 vpsrlvw m1, m11
4227 mova [maskq], ym5
4228 add maskq, 32
4229 ret
4230
4231 cglobal w_mask_444_16bpc, 4, 8, 13, dst, stride, tmp1, tmp2, w, h, mask, stride3
4232 %define base r7-w_mask_444_avx512icl_table
4233 lea r7, [w_mask_444_avx512icl_table]
4234 tzcnt wd, wm
4235 mov r6d, r8m ; pixel_max
4236 movifnidn hd, hm
4237 shr r6d, 11
4238 movsxd wq, [r7+wq*4]
4239 vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32
4240 vpbroadcastd m9, [base+pw_64]
4241 vpbroadcastd m10, [base+mask_round+r6*4]
4242 mova m11, [w_mask_end444]
4243 vpbroadcastd m12, [base+bidir_shift+r6*4]
4244 mov maskq, maskmp
4245 add wq, r7
4246 call .main
4247 lea stride3q, [strideq*3]
4248 jmp wq
4249 .w4:
4250 movq [dstq+strideq*0], xm0
4251 movhps [dstq+strideq*1], xm0
4252 vextracti32x4 xm2, ym0, 1
4253 movq [dstq+strideq*2], xm2
4254 movhps [dstq+stride3q ], xm2
4255 cmp hd, 8
4256 jl .w4_end
4257 vextracti32x4 xm2, m0, 2
4258 lea dstq, [dstq+strideq*4]
4259 movq [dstq+strideq*0], xm2
4260 movhps [dstq+strideq*1], xm2
4261 vextracti32x4 xm0, m0, 3
4262 movq [dstq+strideq*2], xm0
4263 movhps [dstq+stride3q ], xm0
4264 je .w4_end
4265 lea dstq, [dstq+strideq*4]
4266 movq [dstq+strideq*0], xm1
4267 movhps [dstq+strideq*1], xm1
4268 vextracti32x4 xm2, ym1, 1
4269 movq [dstq+strideq*2], xm2
4270 movhps [dstq+stride3q ], xm2
4271 vextracti32x4 xm2, m1, 2
4272 lea dstq, [dstq+strideq*4]
4273 movq [dstq+strideq*0], xm2
4274 movhps [dstq+strideq*1], xm2
4275 vextracti32x4 xm1, m1, 3
4276 movq [dstq+strideq*2], xm1
4277 movhps [dstq+stride3q ], xm1
4278 .w4_end:
4279 RET
4280 .w8_loop:
4281 call .main
4282 lea dstq, [dstq+strideq*4]
4283 .w8:
4284 mova [dstq+strideq*0], xm0
4285 vextracti32x4 [dstq+strideq*1], ym0, 1
4286 vextracti32x4 [dstq+strideq*2], m0, 2
4287 vextracti32x4 [dstq+stride3q ], m0, 3
4288 sub hd, 8
4289 jl .w8_end
4290 lea dstq, [dstq+strideq*4]
4291 mova [dstq+strideq*0], xm1
4292 vextracti32x4 [dstq+strideq*1], ym1, 1
4293 vextracti32x4 [dstq+strideq*2], m1, 2
4294 vextracti32x4 [dstq+stride3q ], m1, 3
4295 jg .w8_loop
4296 .w8_end:
4297 RET
4298 .w16_loop:
4299 call .main
4300 lea dstq, [dstq+strideq*4]
4301 .w16:
4302 mova [dstq+strideq*0], ym0
4303 vextracti32x8 [dstq+strideq*1], m0, 1
4304 mova [dstq+strideq*2], ym1
4305 vextracti32x8 [dstq+stride3q ], m1, 1
4306 sub hd, 4
4307 jg .w16_loop
4308 RET
4309 .w32_loop:
4310 call .main
4311 lea dstq, [dstq+strideq*2]
4312 .w32:
4313 mova [dstq+strideq*0], m0
4314 mova [dstq+strideq*1], m1
4315 sub hd, 2
4316 jg .w32_loop
4317 RET
4318 .w64_loop:
4319 call .main
4320 add dstq, strideq
4321 .w64:
4322 mova [dstq+64*0], m0
4323 mova [dstq+64*1], m1
4324 dec hd
4325 jg .w64_loop
4326 RET
4327 .w128_loop:
4328 call .main
4329 add dstq, strideq
4330 .w128:
4331 mova [dstq+64*0], m0
4332 mova [dstq+64*1], m1
4333 call .main
4334 mova [dstq+64*2], m0
4335 mova [dstq+64*3], m1
4336 dec hd
4337 jg .w128_loop
4338 RET
4339 ALIGN function_align
4340 .main:
4341 mova m1, [tmp1q+64*0]
4342 mova m3, [tmp2q+64*0]
4343 mova m4, [tmp1q+64*1]
4344 mova m7, [tmp2q+64*1]
4345 add tmp1q, 64*2
4346 add tmp2q, 64*2
4347 psubsw m6, m1, m3
4348 punpcklwd m5, m3, m1
4349 pabsw m6, m6
4350 punpckhwd m3, m1
4351 psubusw m6, m8, m6
4352 psrlw m6, 10
4353 psubw m2, m9, m6
4354 punpcklwd m1, m6, m2
4355 punpckhwd m6, m2
4356 mova m0, m10
4357 vpdpwssd m0, m5, m1
4358 mova m1, m10
4359 vpdpwssd m1, m3, m6
4360 psubsw m5, m4, m7
4361 punpcklwd m6, m7, m4
4362 pabsw m5, m5
4363 punpckhwd m7, m4
4364 psubusw m5, m8, m5
4365 psrlw m5, 10
4366 psubw m3, m9, m5
4367 punpcklwd m4, m5, m3
4368 psrad m0, 4
4369 punpckhwd m5, m3
4370 psrad m1, 4
4371 packusdw m0, m1
4372 mova m1, m10
4373 vpdpwssd m1, m6, m4
4374 mova m4, m10
4375 vpdpwssd m4, m7, m5
4376 vpermt2b m2, m11, m3
4377 psrad m1, 4
4378 psrad m4, 4
4379 packusdw m1, m4
4380 vpsrlvw m0, m12
4381 vpsrlvw m1, m12
4382 mova [maskq], m2
4383 add maskq, 64
4384 ret
4385
4386 cglobal blend_16bpc, 3, 7, 7, dst, ds, tmp, w, h, mask
4387 %define base r6-blend_avx512icl_table
4388 lea r6, [blend_avx512icl_table]
4389 tzcnt wd, wm
4390 movifnidn hd, hm
4391 movsxd wq, [r6+wq*4]
4392 movifnidn maskq, maskmp
4393 vpbroadcastd m6, [base+pw_m512]
4394 add wq, r6
4395 lea r6, [dsq*3]
4396 jmp wq
4397 .w4:
4398 pmovzxbw ym19, [maskq]
4399 movq xm16, [dstq+dsq*0]
4400 movhps xm16, [dstq+dsq*1]
4401 vpbroadcastq ym17, [dstq+dsq*2]
4402 vpbroadcastq ym18, [dstq+r6 ]
4403 pmullw ym19, ym6
4404 vpblendd ym16, ym17, 0x30
4405 vpblendd ym16, ym18, 0xc0
4406 psubw ym17, ym16, [tmpq]
4407 add maskq, 16
4408 add tmpq, 32
4409 pmulhrsw ym17, ym19
4410 paddw ym16, ym17
4411 vextracti128 xm17, ym16, 1
4412 movq [dstq+dsq*0], xm16
4413 movhps [dstq+dsq*1], xm16
4414 movq [dstq+dsq*2], xm17
4415 movhps [dstq+r6 ], xm17
4416 lea dstq, [dstq+dsq*4]
4417 sub hd, 4
4418 jg .w4
4419 vzeroupper
4420 RET
4421 .w8:
4422 pmovzxbw m2, [maskq]
4423 mova xm0, [dstq+dsq*0]
4424 vinserti32x4 ym0, [dstq+dsq*1], 1
4425 vinserti32x4 m0, [dstq+dsq*2], 2
4426 vinserti32x4 m0, [dstq+r6 ], 3
4427 pmullw m2, m6
4428 psubw m1, m0, [tmpq]
4429 add maskq, 32
4430 add tmpq, 64
4431 pmulhrsw m1, m2
4432 paddw m0, m1
4433 mova [dstq+dsq*0], xm0
4434 vextracti32x4 [dstq+dsq*1], ym0, 1
4435 vextracti32x4 [dstq+dsq*2], m0, 2
4436 vextracti32x4 [dstq+r6 ], m0, 3
4437 lea dstq, [dstq+dsq*4]
4438 sub hd, 4
4439 jg .w8
4440 RET
4441 .w16:
4442 pmovzxbw m4, [maskq+32*0]
4443 pmovzxbw m5, [maskq+32*1]
4444 mova ym0, [dstq+dsq*0]
4445 vinserti32x8 m0, [dstq+dsq*1], 1
4446 mova ym1, [dstq+dsq*2]
4447 vinserti32x8 m1, [dstq+r6 ], 1
4448 pmullw m4, m6
4449 pmullw m5, m6
4450 psubw m2, m0, [tmpq+64*0]
4451 psubw m3, m1, [tmpq+64*1]
4452 add maskq, 32*2
4453 add tmpq, 64*2
4454 pmulhrsw m2, m4
4455 pmulhrsw m3, m5
4456 paddw m0, m2
4457 paddw m1, m3
4458 mova [dstq+dsq*0], ym0
4459 vextracti32x8 [dstq+dsq*1], m0, 1
4460 mova [dstq+dsq*2], ym1
4461 vextracti32x8 [dstq+r6 ], m1, 1
4462 lea dstq, [dstq+dsq*4]
4463 sub hd, 4
4464 jg .w16
4465 RET
4466 .w32:
4467 pmovzxbw m4, [maskq+32*0]
4468 pmovzxbw m5, [maskq+32*1]
4469 mova m0, [dstq+dsq*0]
4470 mova m1, [dstq+dsq*1]
4471 pmullw m4, m6
4472 pmullw m5, m6
4473 psubw m2, m0, [tmpq+ 64*0]
4474 psubw m3, m1, [tmpq+ 64*1]
4475 add maskq, 32*2
4476 add tmpq, 64*2
4477 pmulhrsw m2, m4
4478 pmulhrsw m3, m5
4479 paddw m0, m2
4480 paddw m1, m3
4481 mova [dstq+dsq*0], m0
4482 mova [dstq+dsq*1], m1
4483 lea dstq, [dstq+dsq*2]
4484 sub hd, 2
4485 jg .w32
4486 RET
4487
4488 cglobal blend_v_16bpc, 3, 6, 5, dst, ds, tmp, w, h
4489 lea r5, [blend_v_avx512icl_table]
4490 tzcnt wd, wm
4491 movifnidn hd, hm
4492 movsxd wq, [r5+wq*4]
4493 add wq, r5
4494 jmp wq
4495 .w2:
4496 vpbroadcastd xmm2, [obmc_masks_avx2+2*2]
4497 .w2_loop:
4498 movd xmm0, [dstq+dsq*0]
4499 pinsrd xmm0, [dstq+dsq*1], 1
4500 movq xmm1, [tmpq]
4501 add tmpq, 4*2
4502 psubw xmm1, xmm0, xmm1
4503 pmulhrsw xmm1, xmm2
4504 paddw xmm0, xmm1
4505 movd [dstq+dsq*0], xmm0
4506 pextrd [dstq+dsq*1], xmm0, 1
4507 lea dstq, [dstq+dsq*2]
4508 sub hd, 2
4509 jg .w2_loop
4510 RET
4511 .w4:
4512 vpbroadcastq xmm2, [obmc_masks_avx2+4*2]
4513 .w4_loop:
4514 movq xmm0, [dstq+dsq*0]
4515 movhps xmm0, [dstq+dsq*1]
4516 psubw xmm1, xmm0, [tmpq]
4517 add tmpq, 8*2
4518 pmulhrsw xmm1, xmm2
4519 paddw xmm0, xmm1
4520 movq [dstq+dsq*0], xmm0
4521 movhps [dstq+dsq*1], xmm0
4522 lea dstq, [dstq+dsq*2]
4523 sub hd, 2
4524 jg .w4_loop
4525 RET
4526 .w8:
4527 vbroadcasti32x4 ym2, [obmc_masks_avx2+8*2]
4528 .w8_loop:
4529 mova xm0, [dstq+dsq*0]
4530 vinserti32x4 ym0, [dstq+dsq*1], 1
4531 psubw ym1, ym0, [tmpq]
4532 add tmpq, 16*2
4533 pmulhrsw ym1, ym2
4534 paddw ym0, ym1
4535 mova [dstq+dsq*0], xm0
4536 vextracti32x4 [dstq+dsq*1], ym0, 1
4537 lea dstq, [dstq+dsq*2]
4538 sub hd, 2
4539 jg .w8_loop
4540 RET
4541 .w16:
4542 vbroadcasti32x8 m2, [obmc_masks_avx2+16*2]
4543 .w16_loop:
4544 mova ym0, [dstq+dsq*0]
4545 vinserti32x8 m0, [dstq+dsq*1], 1
4546 psubw m1, m0, [tmpq]
4547 add tmpq, 32*2
4548 pmulhrsw m1, m2
4549 paddw m0, m1
4550 mova [dstq+dsq*0], ym0
4551 vextracti32x8 [dstq+dsq*1], m0, 1
4552 lea dstq, [dstq+dsq*2]
4553 sub hd, 2
4554 jg .w16_loop
4555 RET
4556 .w32:
4557 mova m4, [obmc_masks_avx2+32*2]
4558 .w32_loop:
4559 mova m0, [dstq+dsq*0]
4560 psubw m2, m0, [tmpq+ 64*0]
4561 mova m1, [dstq+dsq*1]
4562 psubw m3, m1, [tmpq+ 64*1]
4563 add tmpq, 64*2
4564 pmulhrsw m2, m4
4565 pmulhrsw m3, m4
4566 paddw m0, m2
4567 paddw m1, m3
4568 mova [dstq+dsq*0], m0
4569 mova [dstq+dsq*1], m1
4570 lea dstq, [dstq+dsq*2]
4571 sub hd, 2
4572 jg .w32_loop
4573 RET
4574
4575 cglobal blend_h_16bpc, 3, 7, 9, dst, ds, tmp, w, h, mask
4576 %define base r6-$$
4577 lea r6, [$$]
4578 tzcnt wd, wm
4579 mov hd, hm
4580 movsxd wq, [base+blend_h_avx512icl_table+wq*4]
4581 lea maskq, [base+obmc_masks_avx2+hq*2]
4582 lea hd, [hq*3]
4583 lea wq, [base+blend_h_avx512icl_table+wq]
4584 shr hd, 2 ; h * 3/4
4585 lea maskq, [maskq+hq*2]
4586 neg hq
4587 jmp wq
4588 .w2:
4589 movd xmm0, [dstq+dsq*0]
4590 pinsrd xmm0, [dstq+dsq*1], 1
4591 movd xmm2, [maskq+hq*2]
4592 movq xmm1, [tmpq]
4593 add tmpq, 4*2
4594 punpcklwd xmm2, xmm2
4595 psubw xmm1, xmm0, xmm1
4596 pmulhrsw xmm1, xmm2
4597 paddw xmm0, xmm1
4598 movd [dstq+dsq*0], xmm0
4599 pextrd [dstq+dsq*1], xmm0, 1
4600 lea dstq, [dstq+dsq*2]
4601 add hq, 2
4602 jl .w2
4603 RET
4604 .w4:
4605 mova xmm3, [blend_shuf]
4606 .w4_loop:
4607 movq xmm0, [dstq+dsq*0]
4608 movhps xmm0, [dstq+dsq*1]
4609 movd xmm2, [maskq+hq*2]
4610 psubw xmm1, xmm0, [tmpq]
4611 add tmpq, 8*2
4612 pshufb xmm2, xmm3
4613 pmulhrsw xmm1, xmm2
4614 paddw xmm0, xmm1
4615 movq [dstq+dsq*0], xmm0
4616 movhps [dstq+dsq*1], xmm0
4617 lea dstq, [dstq+dsq*2]
4618 add hq, 2
4619 jl .w4_loop
4620 RET
4621 .w8:
4622 vbroadcasti32x4 ym3, [blend_shuf]
4623 shufpd ym3, ym3, 0x0c
4624 .w8_loop:
4625 mova xm0, [dstq+dsq*0]
4626 vinserti32x4 ym0, [dstq+dsq*1], 1
4627 vpbroadcastd ym2, [maskq+hq*2]
4628 psubw ym1, ym0, [tmpq]
4629 add tmpq, 16*2
4630 pshufb ym2, ym3
4631 pmulhrsw ym1, ym2
4632 paddw ym0, ym1
4633 mova [dstq+dsq*0], xm0
4634 vextracti32x4 [dstq+dsq*1], ym0, 1
4635 lea dstq, [dstq+dsq*2]
4636 add hq, 2
4637 jl .w8_loop
4638 RET
4639 .w16:
4640 vbroadcasti32x4 m3, [blend_shuf]
4641 shufpd m3, m3, 0xf0
4642 .w16_loop:
4643 mova ym0, [dstq+dsq*0]
4644 vinserti32x8 m0, [dstq+dsq*1], 1
4645 vpbroadcastd m2, [maskq+hq*2]
4646 psubw m1, m0, [tmpq]
4647 add tmpq, 32*2
4648 pshufb m2, m3
4649 pmulhrsw m1, m2
4650 paddw m0, m1
4651 mova [dstq+dsq*0], ym0
4652 vextracti32x8 [dstq+dsq*1], m0, 1
4653 lea dstq, [dstq+dsq*2]
4654 add hq, 2
4655 jl .w16_loop
4656 RET
4657 .w32:
4658 vpbroadcastw m4, [maskq+hq*2]
4659 vpbroadcastw m5, [maskq+hq*2+2]
4660 mova m0, [dstq+dsq*0]
4661 psubw m2, m0, [tmpq+ 64*0]
4662 mova m1, [dstq+dsq*1]
4663 psubw m3, m1, [tmpq+ 64*1]
4664 add tmpq, 64*2
4665 pmulhrsw m2, m4
4666 pmulhrsw m3, m5
4667 paddw m0, m2
4668 paddw m1, m3
4669 mova [dstq+dsq*0], m0
4670 mova [dstq+dsq*1], m1
4671 lea dstq, [dstq+dsq*2]
4672 add hq, 2
4673 jl .w32
4674 RET
4675 .w64:
4676 vpbroadcastw m4, [maskq+hq*2]
4677 mova m0, [dstq+64*0]
4678 psubw m2, m0, [tmpq+64*0]
4679 mova m1, [dstq+64*1]
4680 psubw m3, m1, [tmpq+64*1]
4681 add tmpq, 64*2
4682 pmulhrsw m2, m4
4683 pmulhrsw m3, m4
4684 paddw m0, m2
4685 paddw m1, m3
4686 mova [dstq+64*0], m0
4687 mova [dstq+64*1], m1
4688 add dstq, dsq
4689 inc hq
4690 jl .w64
4691 RET
4692 .w128:
4693 vpbroadcastw m8, [maskq+hq*2]
4694 mova m0, [dstq+64*0]
4695 psubw m4, m0, [tmpq+64*0]
4696 mova m1, [dstq+64*1]
4697 psubw m5, m1, [tmpq+64*1]
4698 mova m2, [dstq+64*2]
4699 psubw m6, m2, [tmpq+64*2]
4700 mova m3, [dstq+64*3]
4701 psubw m7, m3, [tmpq+64*3]
4702 add tmpq, 64*4
4703 REPX {pmulhrsw x, m8}, m4, m5, m6, m7
4704 paddw m0, m4
4705 paddw m1, m5
4706 paddw m2, m6
4707 paddw m3, m7
4708 mova [dstq+64*0], m0
4709 mova [dstq+64*1], m1
4710 mova [dstq+64*2], m2
4711 mova [dstq+64*3], m3
4712 add dstq, dsq
4713 inc hq
4714 jl .w128
4715 RET
4716
4717 cglobal resize_16bpc, 6, 12, 32, dst, dst_stride, src, src_stride, \
4718 dst_w, h, src_w, dx, mx0, pxmax
4719 sub dword mx0m, 4<<14
4720 sub dword src_wm, 8
4721 mov r6, ~0
4722 vpbroadcastd m5, dxm
4723 vpbroadcastd m8, mx0m
4724 vpbroadcastd m6, src_wm
4725 kmovq k6, r6
4726 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, _, _, pxmax
4727 LEA r7, $$
4728 %define base r7-$$
4729 vpbroadcastd m3, [base+pd_16384]
4730 vpbroadcastd m7, [base+pd_63]
4731 mova m24, [base+resize_permA]
4732 mova m25, [base+resize_permB]
4733 mova m26, [base+resize_permC]
4734 mova m27, [base+resize_permD]
4735 vbroadcasti32x4 m28, [base+resize_shufA]
4736 vbroadcasti32x4 m29, [base+resize_shufB]
4737 mova m30, [base+resize_permE]
4738 vpbroadcastw ym31, pxmaxm
4739 vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15]
4740 pslld m5, 4 ; dx*16
4741 pslld m6, 14
4742 pxor m2, m2
4743 .loop_y:
4744 xor xd, xd
4745 mova m4, m8 ; per-line working version of mx
4746 .loop_x:
4747 pmaxsd m0, m4, m2
4748 psrad m9, m4, 8 ; filter offset (unmasked)
4749 pminsd m0, m6 ; iclip(mx, 0, src_w-8)
4750 psubd m1, m4, m0 ; pshufb offset
4751 psrad m0, 14 ; clipped src_x offset
4752 psrad m1, 14 ; pshufb edge_emu offset
4753 vptestmd k5, m1, m1
4754 pand m9, m7 ; filter offset (masked)
4755 ktestw k5, k5
4756 jz .load
4757 vpbroadcastq m14, [base+pd_0_4]
4758 vpermq m10, m0, q1100
4759 vpermq m11, m0, q3322
4760 vpermq m20, m1, q1100
4761 vpermq m21, m1, q3322
4762 punpckldq m10, m10
4763 punpckldq m11, m11
4764 punpckldq m20, m20
4765 punpckldq m21, m21
4766 paddd m10, m14
4767 paddd m11, m14
4768 paddd m20, m14
4769 paddd m21, m14
4770 vextracti32x8 ym12, m10, 1
4771 vextracti32x8 ym13, m11, 1
4772 vextracti32x8 ym22, m20, 1
4773 vextracti32x8 ym23, m21, 1
4774 kmovq k1, k6
4775 kmovq k2, k6
4776 kmovq k3, k6
4777 kmovq k4, k6
4778 vpgatherdq m16{k1}, [srcq+ym10*2] ; 0 1 2 3
4779 vpgatherdq m17{k2}, [srcq+ym11*2] ; 4 5 6 7
4780 vpgatherdq m18{k3}, [srcq+ym12*2] ; 8 9 A B
4781 vpgatherdq m19{k4}, [srcq+ym13*2] ; C D E F
4782 kmovq k1, k6
4783 kmovq k2, k6
4784 kmovq k3, k6
4785 kmovq k4, k6
4786 vpgatherdq m0{k1}, [base+resize_shuf+8+ym20*2]
4787 vpgatherdq m1{k2}, [base+resize_shuf+8+ym21*2]
4788 vpgatherdq m14{k3}, [base+resize_shuf+8+ym22*2]
4789 vpgatherdq m15{k4}, [base+resize_shuf+8+ym23*2]
4790 pshufb m16, m0
4791 pshufb m17, m1
4792 pshufb m18, m14
4793 pshufb m19, m15
4794 mova m20, m24
4795 mova m22, m24
4796 mova m21, m25
4797 mova m23, m25
4798 vpermi2d m20, m16, m17 ; 0-3a 0-3b 4-7a 4-7b
4799 vpermi2d m21, m16, m17 ; 0-3c 0-3d 4-7c 4-7d
4800 vpermi2d m22, m18, m19 ; 8-Ba 8-Bb C-Fa C-Fb
4801 vpermi2d m23, m18, m19 ; 8-Bc 8-Bd C-Fc C-Fd
4802 mova m15, m26
4803 mova m17, m26
4804 mova m16, m27
4805 mova m18, m27
4806 vpermi2q m15, m20, m22 ; 0-3a 4-7a 8-Ba C-Fa
4807 vpermi2q m16, m20, m22 ; 0-3b 4-7b 8-Bb C-Fb
4808 vpermi2q m17, m21, m23 ; 0-3c 4-7c 8-Bc C-Fc
4809 vpermi2q m18, m21, m23 ; 0-3d 4-7d 8-Bd C-Fd
4810 kmovq k1, k6
4811 kmovq k2, k6
4812 vpgatherdd m11{k1}, [base+resize_filter+m9*8+0]
4813 vpgatherdd m13{k2}, [base+resize_filter+m9*8+4]
4814 pshufb m10, m11, m28
4815 pshufb m11, m11, m29
4816 pshufb m12, m13, m28
4817 pshufb m13, m13, m29
4818 jmp .filter
4819 .load:
4820 kmovq k1, k6
4821 kmovq k2, k6
4822 kmovq k3, k6
4823 kmovq k4, k6
4824 vpgatherdd m11{k1}, [base+resize_filter+m9*8+0]
4825 vpgatherdd m13{k2}, [base+resize_filter+m9*8+4]
4826 pshufb m10, m11, m28
4827 pshufb m11, m11, m29
4828 pshufb m12, m13, m28
4829 pshufb m13, m13, m29
4830 vpgatherdd m15{k3}, [srcq+m0*2+ 0]
4831 vpgatherdd m16{k4}, [srcq+m0*2+ 4]
4832 kmovq k1, k6
4833 kmovq k2, k6
4834 vpgatherdd m17{k1}, [srcq+m0*2+ 8]
4835 vpgatherdd m18{k2}, [srcq+m0*2+12]
4836 .filter:
4837 mova m14, m2
4838 vpdpwssd m14, m15, m10
4839 vpdpwssd m14, m16, m11
4840 vpdpwssd m14, m17, m12
4841 vpdpwssd m14, m18, m13
4842 psubd m14, m3, m14
4843 psrad m14, 15
4844 packusdw m14, m14
4845 vpermq m14, m30, m14
4846 pminsw ym14, ym31
4847 mova [dstq+xq*2], ym14
4848 paddd m4, m5
4849 add xd, 16
4850 cmp xd, dst_wd
4851 jl .loop_x
4852 add dstq, dst_strideq
4853 add srcq, src_strideq
4854 dec hd
4855 jg .loop_y
4856 RET
4857
4858 %endif ; ARCH_X86_64