Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/x86/mc_sse.asm

raw
1 ; Copyright © 2018, VideoLAN and dav1d authors
2 ; Copyright © 2018, Two Orioles, LLC
3 ; Copyright © 2018, VideoLabs
4 ; All rights reserved.
5 ;
6 ; Redistribution and use in source and binary forms, with or without
7 ; modification, are permitted provided that the following conditions are met:
8 ;
9 ; 1. Redistributions of source code must retain the above copyright notice, this
10 ; list of conditions and the following disclaimer.
11 ;
12 ; 2. Redistributions in binary form must reproduce the above copyright notice,
13 ; this list of conditions and the following disclaimer in the documentation
14 ; and/or other materials provided with the distribution.
15 ;
16 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 %include "config.asm"
28 %include "src/asm/x86/x86inc.asm"
29
30 SECTION_RODATA 16
31
32 ; dav1d_obmc_masks[] with 64-x interleaved
33 obmc_masks: db 0, 0, 0, 0
34 ; 2 @4
35 db 45, 19, 64, 0
36 ; 4 @8
37 db 39, 25, 50, 14, 59, 5, 64, 0
38 ; 8 @16
39 db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0
40 ; 16 @32
41 db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10
42 db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0
43 ; 32 @64
44 db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20
45 db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9
46 db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2
47
48 warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8
49 warp_8x8_shufB: db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12
50 warp_8x8_shufC: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10
51 warp_8x8_shufD: db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14
52 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
53 subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12
54 db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14
55 subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
56 subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
57 subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
58 subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11
59 subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
60 bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12
61 bilin_h_shuf8: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
62 unpckw: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
63 rescale_mul: dd 0, 1, 2, 3
64 resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7
65
66 wm_420_sign: times 4 dw 258
67 times 4 dw 257
68 wm_422_sign: times 8 db 128
69 times 8 db 127
70
71 pb_8x0_8x8: times 8 db 0
72 times 8 db 8
73 bdct_lb_dw: times 4 db 0
74 times 4 db 4
75 times 4 db 8
76 times 4 db 12
77
78 pb_64: times 16 db 64
79 pw_m256: times 8 dw -256
80 pw_1: times 8 dw 1
81 pw_2: times 8 dw 2
82 pw_8: times 8 dw 8
83 pw_15: times 8 dw 15
84 pw_26: times 8 dw 26
85 pw_34: times 8 dw 34
86 pw_512: times 8 dw 512
87 pw_1024: times 8 dw 1024
88 pw_2048: times 8 dw 2048
89 pw_6903: times 8 dw 6903
90 pw_8192: times 8 dw 8192
91 pd_32: times 4 dd 32
92 pd_63: times 4 dd 63
93 pd_512: times 4 dd 512
94 pd_16384: times 4 dd 16484
95 pd_32768: times 4 dd 32768
96 pd_262144:times 4 dd 262144
97 pd_0x3ff: times 4 dd 0x3ff
98 pd_0x4000:times 4 dd 0x4000
99 pq_0x40000000: times 2 dq 0x40000000
100
101 const mc_warp_filter2 ; dav1d_mc_warp_filter[] reordered for pmaddubsw usage
102 ; [-1, 0)
103 db 0, 127, 0, 0, 0, 1, 0, 0, 0, 127, 0, 0, -1, 2, 0, 0
104 db 1, 127, -1, 0, -3, 4, 0, 0, 1, 126, -2, 0, -4, 6, 1, 0
105 db 1, 126, -3, 0, -5, 8, 1, 0, 1, 125, -4, 0, -6, 11, 1, 0
106 db 1, 124, -4, 0, -7, 13, 1, 0, 2, 123, -5, 0, -8, 15, 1, 0
107 db 2, 122, -6, 0, -9, 18, 1, 0, 2, 121, -6, 0, -10, 20, 1, 0
108 db 2, 120, -7, 0, -11, 22, 2, 0, 2, 119, -8, 0, -12, 25, 2, 0
109 db 3, 117, -8, 0, -13, 27, 2, 0, 3, 116, -9, 0, -13, 29, 2, 0
110 db 3, 114, -10, 0, -14, 32, 3, 0, 3, 113, -10, 0, -15, 35, 2, 0
111 db 3, 111, -11, 0, -15, 37, 3, 0, 3, 109, -11, 0, -16, 40, 3, 0
112 db 3, 108, -12, 0, -16, 42, 3, 0, 4, 106, -13, 0, -17, 45, 3, 0
113 db 4, 104, -13, 0, -17, 47, 3, 0, 4, 102, -14, 0, -17, 50, 3, 0
114 db 4, 100, -14, 0, -17, 52, 3, 0, 4, 98, -15, 0, -18, 55, 4, 0
115 db 4, 96, -15, 0, -18, 58, 3, 0, 4, 94, -16, 0, -18, 60, 4, 0
116 db 4, 91, -16, 0, -18, 63, 4, 0, 4, 89, -16, 0, -18, 65, 4, 0
117 db 4, 87, -17, 0, -18, 68, 4, 0, 4, 85, -17, 0, -18, 70, 4, 0
118 db 4, 82, -17, 0, -18, 73, 4, 0, 4, 80, -17, 0, -18, 75, 4, 0
119 db 4, 78, -18, 0, -18, 78, 4, 0, 4, 75, -18, 0, -17, 80, 4, 0
120 db 4, 73, -18, 0, -17, 82, 4, 0, 4, 70, -18, 0, -17, 85, 4, 0
121 db 4, 68, -18, 0, -17, 87, 4, 0, 4, 65, -18, 0, -16, 89, 4, 0
122 db 4, 63, -18, 0, -16, 91, 4, 0, 4, 60, -18, 0, -16, 94, 4, 0
123 db 3, 58, -18, 0, -15, 96, 4, 0, 4, 55, -18, 0, -15, 98, 4, 0
124 db 3, 52, -17, 0, -14, 100, 4, 0, 3, 50, -17, 0, -14, 102, 4, 0
125 db 3, 47, -17, 0, -13, 104, 4, 0, 3, 45, -17, 0, -13, 106, 4, 0
126 db 3, 42, -16, 0, -12, 108, 3, 0, 3, 40, -16, 0, -11, 109, 3, 0
127 db 3, 37, -15, 0, -11, 111, 3, 0, 2, 35, -15, 0, -10, 113, 3, 0
128 db 3, 32, -14, 0, -10, 114, 3, 0, 2, 29, -13, 0, -9, 116, 3, 0
129 db 2, 27, -13, 0, -8, 117, 3, 0, 2, 25, -12, 0, -8, 119, 2, 0
130 db 2, 22, -11, 0, -7, 120, 2, 0, 1, 20, -10, 0, -6, 121, 2, 0
131 db 1, 18, -9, 0, -6, 122, 2, 0, 1, 15, -8, 0, -5, 123, 2, 0
132 db 1, 13, -7, 0, -4, 124, 1, 0, 1, 11, -6, 0, -4, 125, 1, 0
133 db 1, 8, -5, 0, -3, 126, 1, 0, 1, 6, -4, 0, -2, 126, 1, 0
134 db 0, 4, -3, 0, -1, 127, 1, 0, 0, 2, -1, 0, 0, 127, 0, 0
135 ; [0, 1)
136 db 0, 0, 1, 0, 0, 127, 0, 0, 0, -1, 2, 0, 0, 127, 0, 0
137 db 0, -3, 4, 1, 1, 127, -2, 0, 0, -5, 6, 1, 1, 127, -2, 0
138 db 0, -6, 8, 1, 2, 126, -3, 0, -1, -7, 11, 2, 2, 126, -4, -1
139 db -1, -8, 13, 2, 3, 125, -5, -1, -1, -10, 16, 3, 3, 124, -6, -1
140 db -1, -11, 18, 3, 4, 123, -7, -1, -1, -12, 20, 3, 4, 122, -7, -1
141 db -1, -13, 23, 3, 4, 121, -8, -1, -2, -14, 25, 4, 5, 120, -9, -1
142 db -1, -15, 27, 4, 5, 119, -10, -1, -1, -16, 30, 4, 5, 118, -11, -1
143 db -2, -17, 33, 5, 6, 116, -12, -1, -2, -17, 35, 5, 6, 114, -12, -1
144 db -2, -18, 38, 5, 6, 113, -13, -1, -2, -19, 41, 6, 7, 111, -14, -2
145 db -2, -19, 43, 6, 7, 110, -15, -2, -2, -20, 46, 6, 7, 108, -15, -2
146 db -2, -20, 49, 6, 7, 106, -16, -2, -2, -21, 51, 7, 7, 104, -16, -2
147 db -2, -21, 54, 7, 7, 102, -17, -2, -2, -21, 56, 7, 8, 100, -18, -2
148 db -2, -22, 59, 7, 8, 98, -18, -2, -2, -22, 62, 7, 8, 96, -19, -2
149 db -2, -22, 64, 7, 8, 94, -19, -2, -2, -22, 67, 8, 8, 91, -20, -2
150 db -2, -22, 69, 8, 8, 89, -20, -2, -2, -22, 72, 8, 8, 87, -21, -2
151 db -2, -21, 74, 8, 8, 84, -21, -2, -2, -22, 77, 8, 8, 82, -21, -2
152 db -2, -21, 79, 8, 8, 79, -21, -2, -2, -21, 82, 8, 8, 77, -22, -2
153 db -2, -21, 84, 8, 8, 74, -21, -2, -2, -21, 87, 8, 8, 72, -22, -2
154 db -2, -20, 89, 8, 8, 69, -22, -2, -2, -20, 91, 8, 8, 67, -22, -2
155 db -2, -19, 94, 8, 7, 64, -22, -2, -2, -19, 96, 8, 7, 62, -22, -2
156 db -2, -18, 98, 8, 7, 59, -22, -2, -2, -18, 100, 8, 7, 56, -21, -2
157 db -2, -17, 102, 7, 7, 54, -21, -2, -2, -16, 104, 7, 7, 51, -21, -2
158 db -2, -16, 106, 7, 6, 49, -20, -2, -2, -15, 108, 7, 6, 46, -20, -2
159 db -2, -15, 110, 7, 6, 43, -19, -2, -2, -14, 111, 7, 6, 41, -19, -2
160 db -1, -13, 113, 6, 5, 38, -18, -2, -1, -12, 114, 6, 5, 35, -17, -2
161 db -1, -12, 116, 6, 5, 33, -17, -2, -1, -11, 118, 5, 4, 30, -16, -1
162 db -1, -10, 119, 5, 4, 27, -15, -1, -1, -9, 120, 5, 4, 25, -14, -2
163 db -1, -8, 121, 4, 3, 23, -13, -1, -1, -7, 122, 4, 3, 20, -12, -1
164 db -1, -7, 123, 4, 3, 18, -11, -1, -1, -6, 124, 3, 3, 16, -10, -1
165 db -1, -5, 125, 3, 2, 13, -8, -1, -1, -4, 126, 2, 2, 11, -7, -1
166 db 0, -3, 126, 2, 1, 8, -6, 0, 0, -2, 127, 1, 1, 6, -5, 0
167 db 0, -2, 127, 1, 1, 4, -3, 0, 0, 0, 127, 0, 0, 2, -1, 0
168 ; [1, 2)
169 db 0, 0, 127, 0, 0, 1, 0, 0, 0, 0, 127, 0, 0, -1, 2, 0
170 db 0, 1, 127, -1, 0, -3, 4, 0, 0, 1, 126, -2, 0, -4, 6, 1
171 db 0, 1, 126, -3, 0, -5, 8, 1, 0, 1, 125, -4, 0, -6, 11, 1
172 db 0, 1, 124, -4, 0, -7, 13, 1, 0, 2, 123, -5, 0, -8, 15, 1
173 db 0, 2, 122, -6, 0, -9, 18, 1, 0, 2, 121, -6, 0, -10, 20, 1
174 db 0, 2, 120, -7, 0, -11, 22, 2, 0, 2, 119, -8, 0, -12, 25, 2
175 db 0, 3, 117, -8, 0, -13, 27, 2, 0, 3, 116, -9, 0, -13, 29, 2
176 db 0, 3, 114, -10, 0, -14, 32, 3, 0, 3, 113, -10, 0, -15, 35, 2
177 db 0, 3, 111, -11, 0, -15, 37, 3, 0, 3, 109, -11, 0, -16, 40, 3
178 db 0, 3, 108, -12, 0, -16, 42, 3, 0, 4, 106, -13, 0, -17, 45, 3
179 db 0, 4, 104, -13, 0, -17, 47, 3, 0, 4, 102, -14, 0, -17, 50, 3
180 db 0, 4, 100, -14, 0, -17, 52, 3, 0, 4, 98, -15, 0, -18, 55, 4
181 db 0, 4, 96, -15, 0, -18, 58, 3, 0, 4, 94, -16, 0, -18, 60, 4
182 db 0, 4, 91, -16, 0, -18, 63, 4, 0, 4, 89, -16, 0, -18, 65, 4
183 db 0, 4, 87, -17, 0, -18, 68, 4, 0, 4, 85, -17, 0, -18, 70, 4
184 db 0, 4, 82, -17, 0, -18, 73, 4, 0, 4, 80, -17, 0, -18, 75, 4
185 db 0, 4, 78, -18, 0, -18, 78, 4, 0, 4, 75, -18, 0, -17, 80, 4
186 db 0, 4, 73, -18, 0, -17, 82, 4, 0, 4, 70, -18, 0, -17, 85, 4
187 db 0, 4, 68, -18, 0, -17, 87, 4, 0, 4, 65, -18, 0, -16, 89, 4
188 db 0, 4, 63, -18, 0, -16, 91, 4, 0, 4, 60, -18, 0, -16, 94, 4
189 db 0, 3, 58, -18, 0, -15, 96, 4, 0, 4, 55, -18, 0, -15, 98, 4
190 db 0, 3, 52, -17, 0, -14, 100, 4, 0, 3, 50, -17, 0, -14, 102, 4
191 db 0, 3, 47, -17, 0, -13, 104, 4, 0, 3, 45, -17, 0, -13, 106, 4
192 db 0, 3, 42, -16, 0, -12, 108, 3, 0, 3, 40, -16, 0, -11, 109, 3
193 db 0, 3, 37, -15, 0, -11, 111, 3, 0, 2, 35, -15, 0, -10, 113, 3
194 db 0, 3, 32, -14, 0, -10, 114, 3, 0, 2, 29, -13, 0, -9, 116, 3
195 db 0, 2, 27, -13, 0, -8, 117, 3, 0, 2, 25, -12, 0, -8, 119, 2
196 db 0, 2, 22, -11, 0, -7, 120, 2, 0, 1, 20, -10, 0, -6, 121, 2
197 db 0, 1, 18, -9, 0, -6, 122, 2, 0, 1, 15, -8, 0, -5, 123, 2
198 db 0, 1, 13, -7, 0, -4, 124, 1, 0, 1, 11, -6, 0, -4, 125, 1
199 db 0, 1, 8, -5, 0, -3, 126, 1, 0, 1, 6, -4, 0, -2, 126, 1
200 db 0, 0, 4, -3, 0, -1, 127, 1, 0, 0, 2, -1, 0, 0, 127, 0
201 db 0, 0, 2, -1, 0, 0, 127, 0
202
203 pw_258: times 2 dw 258
204
205 cextern mc_subpel_filters
206 %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
207
208 %macro BIDIR_JMP_TABLE 2-*
209 ;evaluated at definition time (in loop below)
210 %xdefine %1_%2_table (%%table - 2*%3)
211 %xdefine %%base %1_%2_table
212 %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
213 ; dynamically generated label
214 %%table:
215 %rep %0 - 2 ; repeat for num args
216 dd %%prefix %+ .w%3 - %%base
217 %rotate 1
218 %endrep
219 %endmacro
220
221 BIDIR_JMP_TABLE avg, ssse3, 4, 8, 16, 32, 64, 128
222 BIDIR_JMP_TABLE w_avg, ssse3, 4, 8, 16, 32, 64, 128
223 BIDIR_JMP_TABLE mask, ssse3, 4, 8, 16, 32, 64, 128
224 BIDIR_JMP_TABLE w_mask_420, ssse3, 4, 8, 16, 32, 64, 128
225 BIDIR_JMP_TABLE w_mask_422, ssse3, 4, 8, 16, 32, 64, 128
226 BIDIR_JMP_TABLE w_mask_444, ssse3, 4, 8, 16, 32, 64, 128
227 BIDIR_JMP_TABLE blend, ssse3, 4, 8, 16, 32
228 BIDIR_JMP_TABLE blend_v, ssse3, 2, 4, 8, 16, 32
229 BIDIR_JMP_TABLE blend_h, ssse3, 2, 4, 8, 16, 16, 16, 16
230
231 %macro BASE_JMP_TABLE 3-*
232 %xdefine %1_%2_table (%%table - %3)
233 %xdefine %%base %1_%2
234 %%table:
235 %rep %0 - 2
236 dw %%base %+ _w%3 - %%base
237 %rotate 1
238 %endrep
239 %endmacro
240
241 %xdefine prep_sse2 mangle(private_prefix %+ _prep_bilin_8bpc_sse2.prep)
242 %xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_8bpc_ssse3.put)
243 %xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_8bpc_ssse3.prep)
244
245 BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128
246 BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128
247
248 %macro HV_JMP_TABLE 5-*
249 %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3)
250 %xdefine %%base %1_%3
251 %assign %%types %4
252 %if %%types & 1
253 %xdefine %1_%2_h_%3_table (%%h - %5)
254 %%h:
255 %rep %0 - 4
256 dw %%prefix %+ .h_w%5 - %%base
257 %rotate 1
258 %endrep
259 %rotate 4
260 %endif
261 %if %%types & 2
262 %xdefine %1_%2_v_%3_table (%%v - %5)
263 %%v:
264 %rep %0 - 4
265 dw %%prefix %+ .v_w%5 - %%base
266 %rotate 1
267 %endrep
268 %rotate 4
269 %endif
270 %if %%types & 4
271 %xdefine %1_%2_hv_%3_table (%%hv - %5)
272 %%hv:
273 %rep %0 - 4
274 dw %%prefix %+ .hv_w%5 - %%base
275 %rotate 1
276 %endrep
277 %endif
278 %endmacro
279
280 HV_JMP_TABLE prep, 8tap, sse2, 1, 4, 8, 16, 32, 64, 128
281 HV_JMP_TABLE prep, bilin, sse2, 7, 4, 8, 16, 32, 64, 128
282 HV_JMP_TABLE put, 8tap, ssse3, 3, 2, 4, 8, 16, 32, 64, 128
283 HV_JMP_TABLE prep, 8tap, ssse3, 1, 4, 8, 16, 32, 64, 128
284 HV_JMP_TABLE put, bilin, ssse3, 7, 2, 4, 8, 16, 32, 64, 128
285 HV_JMP_TABLE prep, bilin, ssse3, 7, 4, 8, 16, 32, 64, 128
286
287 %macro SCALED_JMP_TABLE 2-*
288 %xdefine %1_%2_table (%%table - %3)
289 %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2)
290 %%table:
291 %rep %0 - 2
292 dw %%base %+ .w%3 - %%base
293 %rotate 1
294 %endrep
295 %rotate 2
296 %%dy_1024:
297 %xdefine %1_%2_dy1_table (%%dy_1024 - %3)
298 %rep %0 - 2
299 dw %%base %+ .dy1_w%3 - %%base
300 %rotate 1
301 %endrep
302 %rotate 2
303 %%dy_2048:
304 %xdefine %1_%2_dy2_table (%%dy_2048 - %3)
305 %rep %0 - 2
306 dw %%base %+ .dy2_w%3 - %%base
307 %rotate 1
308 %endrep
309 %endmacro
310
311 SCALED_JMP_TABLE put_8tap_scaled, ssse3, 2, 4, 8, 16, 32, 64, 128
312 SCALED_JMP_TABLE prep_8tap_scaled, ssse3, 4, 8, 16, 32, 64, 128
313
314 %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
315
316 SECTION .text
317
318 INIT_XMM ssse3
319
320 %if ARCH_X86_32
321 DECLARE_REG_TMP 1
322 %define base t0-put_ssse3
323 %else
324 DECLARE_REG_TMP 7
325 %define base 0
326 %endif
327
328 %macro RESTORE_DSQ_32 1
329 %if ARCH_X86_32
330 mov %1, dsm ; restore dsq
331 %endif
332 %endmacro
333
334 cglobal put_bilin_8bpc, 1, 8, 0, dst, ds, src, ss, w, h, mxy
335 movifnidn mxyd, r6m ; mx
336 LEA t0, put_ssse3
337 movifnidn srcq, srcmp
338 movifnidn ssq, ssmp
339 tzcnt wd, wm
340 mov hd, hm
341 test mxyd, mxyd
342 jnz .h
343 mov mxyd, r7m ; my
344 test mxyd, mxyd
345 jnz .v
346 .put:
347 movzx wd, word [t0+wq*2+table_offset(put,)]
348 add wq, t0
349 RESTORE_DSQ_32 t0
350 jmp wq
351 .put_w2:
352 movzx r4d, word [srcq+ssq*0]
353 movzx r6d, word [srcq+ssq*1]
354 lea srcq, [srcq+ssq*2]
355 mov [dstq+dsq*0], r4w
356 mov [dstq+dsq*1], r6w
357 lea dstq, [dstq+dsq*2]
358 sub hd, 2
359 jg .put_w2
360 RET
361 .put_w4:
362 mov r4d, [srcq+ssq*0]
363 mov r6d, [srcq+ssq*1]
364 lea srcq, [srcq+ssq*2]
365 mov [dstq+dsq*0], r4d
366 mov [dstq+dsq*1], r6d
367 lea dstq, [dstq+dsq*2]
368 sub hd, 2
369 jg .put_w4
370 RET
371 .put_w8:
372 movq m0, [srcq+ssq*0]
373 movq m1, [srcq+ssq*1]
374 lea srcq, [srcq+ssq*2]
375 movq [dstq+dsq*0], m0
376 movq [dstq+dsq*1], m1
377 lea dstq, [dstq+dsq*2]
378 sub hd, 2
379 jg .put_w8
380 RET
381 .put_w16:
382 movu m0, [srcq+ssq*0]
383 movu m1, [srcq+ssq*1]
384 lea srcq, [srcq+ssq*2]
385 mova [dstq+dsq*0], m0
386 mova [dstq+dsq*1], m1
387 lea dstq, [dstq+dsq*2]
388 sub hd, 2
389 jg .put_w16
390 RET
391 .put_w32:
392 movu m0, [srcq+ssq*0+16*0]
393 movu m1, [srcq+ssq*0+16*1]
394 movu m2, [srcq+ssq*1+16*0]
395 movu m3, [srcq+ssq*1+16*1]
396 lea srcq, [srcq+ssq*2]
397 mova [dstq+dsq*0+16*0], m0
398 mova [dstq+dsq*0+16*1], m1
399 mova [dstq+dsq*1+16*0], m2
400 mova [dstq+dsq*1+16*1], m3
401 lea dstq, [dstq+dsq*2]
402 sub hd, 2
403 jg .put_w32
404 RET
405 .put_w64:
406 movu m0, [srcq+16*0]
407 movu m1, [srcq+16*1]
408 movu m2, [srcq+16*2]
409 movu m3, [srcq+16*3]
410 add srcq, ssq
411 mova [dstq+16*0], m0
412 mova [dstq+16*1], m1
413 mova [dstq+16*2], m2
414 mova [dstq+16*3], m3
415 add dstq, dsq
416 dec hd
417 jg .put_w64
418 RET
419 .put_w128:
420 movu m0, [srcq+16*0]
421 movu m1, [srcq+16*1]
422 movu m2, [srcq+16*2]
423 movu m3, [srcq+16*3]
424 mova [dstq+16*0], m0
425 mova [dstq+16*1], m1
426 mova [dstq+16*2], m2
427 mova [dstq+16*3], m3
428 movu m0, [srcq+16*4]
429 movu m1, [srcq+16*5]
430 movu m2, [srcq+16*6]
431 movu m3, [srcq+16*7]
432 mova [dstq+16*4], m0
433 mova [dstq+16*5], m1
434 mova [dstq+16*6], m2
435 mova [dstq+16*7], m3
436 add srcq, ssq
437 add dstq, dsq
438 dec hd
439 jg .put_w128
440 RET
441 .h:
442 ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
443 ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
444 imul mxyd, 0x00ff00ff
445 mova m4, [base+bilin_h_shuf8]
446 mova m0, [base+bilin_h_shuf4]
447 add mxyd, 0x00100010
448 movd m5, mxyd
449 mov mxyd, r7m ; my
450 pshufd m5, m5, q0000
451 test mxyd, mxyd
452 jnz .hv
453 movzx wd, word [t0+wq*2+table_offset(put, _bilin_h)]
454 mova m3, [base+pw_2048]
455 add wq, t0
456 movifnidn dsq, dsmp
457 jmp wq
458 .h_w2:
459 pshufd m4, m4, q3120 ; m4 = {1, 0, 2, 1, 5, 4, 6, 5}
460 .h_w2_loop:
461 movd m0, [srcq+ssq*0]
462 movd m1, [srcq+ssq*1]
463 lea srcq, [srcq+ssq*2]
464 punpckldq m0, m1
465 pshufb m0, m4
466 pmaddubsw m0, m5
467 pmulhrsw m0, m3
468 packuswb m0, m0
469 movd r6d, m0
470 mov [dstq+dsq*0], r6w
471 shr r6d, 16
472 mov [dstq+dsq*1], r6w
473 lea dstq, [dstq+dsq*2]
474 sub hd, 2
475 jg .h_w2_loop
476 RET
477 .h_w4:
478 movq m4, [srcq+ssq*0]
479 movhps m4, [srcq+ssq*1]
480 lea srcq, [srcq+ssq*2]
481 pshufb m4, m0
482 pmaddubsw m4, m5
483 pmulhrsw m4, m3
484 packuswb m4, m4
485 movd [dstq+dsq*0], m4
486 psrlq m4, 32
487 movd [dstq+dsq*1], m4
488 lea dstq, [dstq+dsq*2]
489 sub hd, 2
490 jg .h_w4
491 RET
492 .h_w8:
493 movu m0, [srcq+ssq*0]
494 movu m1, [srcq+ssq*1]
495 lea srcq, [srcq+ssq*2]
496 pshufb m0, m4
497 pshufb m1, m4
498 pmaddubsw m0, m5
499 pmaddubsw m1, m5
500 pmulhrsw m0, m3
501 pmulhrsw m1, m3
502 packuswb m0, m1
503 movq [dstq+dsq*0], m0
504 movhps [dstq+dsq*1], m0
505 lea dstq, [dstq+dsq*2]
506 sub hd, 2
507 jg .h_w8
508 RET
509 .h_w16:
510 movu m0, [srcq+8*0]
511 movu m1, [srcq+8*1]
512 add srcq, ssq
513 pshufb m0, m4
514 pshufb m1, m4
515 pmaddubsw m0, m5
516 pmaddubsw m1, m5
517 pmulhrsw m0, m3
518 pmulhrsw m1, m3
519 packuswb m0, m1
520 mova [dstq], m0
521 add dstq, dsq
522 dec hd
523 jg .h_w16
524 RET
525 .h_w32:
526 movu m0, [srcq+mmsize*0+8*0]
527 movu m1, [srcq+mmsize*0+8*1]
528 pshufb m0, m4
529 pshufb m1, m4
530 pmaddubsw m0, m5
531 pmaddubsw m1, m5
532 pmulhrsw m0, m3
533 pmulhrsw m1, m3
534 packuswb m0, m1
535 movu m1, [srcq+mmsize*1+8*0]
536 movu m2, [srcq+mmsize*1+8*1]
537 add srcq, ssq
538 pshufb m1, m4
539 pshufb m2, m4
540 pmaddubsw m1, m5
541 pmaddubsw m2, m5
542 pmulhrsw m1, m3
543 pmulhrsw m2, m3
544 packuswb m1, m2
545 mova [dstq+16*0], m0
546 mova [dstq+16*1], m1
547 add dstq, dsq
548 dec hd
549 jg .h_w32
550 RET
551 .h_w64:
552 mov r6, -16*3
553 .h_w64_loop:
554 movu m0, [srcq+r6+16*3+8*0]
555 movu m1, [srcq+r6+16*3+8*1]
556 pshufb m0, m4
557 pshufb m1, m4
558 pmaddubsw m0, m5
559 pmaddubsw m1, m5
560 pmulhrsw m0, m3
561 pmulhrsw m1, m3
562 packuswb m0, m1
563 mova [dstq+r6+16*3], m0
564 add r6, 16
565 jle .h_w64_loop
566 add srcq, ssq
567 add dstq, dsq
568 dec hd
569 jg .h_w64
570 RET
571 .h_w128:
572 mov r6, -16*7
573 .h_w128_loop:
574 movu m0, [srcq+r6+16*7+8*0]
575 movu m1, [srcq+r6+16*7+8*1]
576 pshufb m0, m4
577 pshufb m1, m4
578 pmaddubsw m0, m5
579 pmaddubsw m1, m5
580 pmulhrsw m0, m3
581 pmulhrsw m1, m3
582 packuswb m0, m1
583 mova [dstq+r6+16*7], m0
584 add r6, 16
585 jle .h_w128_loop
586 add srcq, ssq
587 add dstq, dsq
588 dec hd
589 jg .h_w128
590 RET
591 .v:
592 movzx wd, word [t0+wq*2+table_offset(put, _bilin_v)]
593 imul mxyd, 0x00ff00ff
594 mova m5, [base+pw_2048]
595 add mxyd, 0x00100010
596 add wq, t0
597 movd m4, mxyd
598 pshufd m4, m4, q0000
599 movifnidn dsq, dsmp
600 jmp wq
601 .v_w2:
602 movd m0, [srcq+ssq*0]
603 .v_w2_loop:
604 pinsrw m0, [srcq+ssq*1], 1 ; 0 1
605 lea srcq, [srcq+ssq*2]
606 pshuflw m1, m0, q2301
607 pinsrw m0, [srcq+ssq*0], 0 ; 2 1
608 punpcklbw m1, m0
609 pmaddubsw m1, m4
610 pmulhrsw m1, m5
611 packuswb m1, m1
612 movd r6d, m1
613 mov [dstq+dsq*1], r6w
614 shr r6d, 16
615 mov [dstq+dsq*0], r6w
616 lea dstq, [dstq+dsq*2]
617 sub hd, 2
618 jg .v_w2_loop
619 RET
620 .v_w4:
621 movd m0, [srcq+ssq*0]
622 .v_w4_loop:
623 movd m2, [srcq+ssq*1]
624 lea srcq, [srcq+ssq*2]
625 mova m1, m0
626 movd m0, [srcq+ssq*0]
627 punpckldq m1, m2 ; 0 1
628 punpckldq m2, m0 ; 1 2
629 punpcklbw m1, m2
630 pmaddubsw m1, m4
631 pmulhrsw m1, m5
632 packuswb m1, m1
633 movd [dstq+dsq*0], m1
634 psrlq m1, 32
635 movd [dstq+dsq*1], m1
636 ;
637 lea dstq, [dstq+dsq*2]
638 sub hd, 2
639 jg .v_w4_loop
640 RET
641 .v_w8:
642 movq m0, [srcq+ssq*0]
643 .v_w8_loop:
644 movq m2, [srcq+ssq*1]
645 lea srcq, [srcq+ssq*2]
646 mova m1, m0
647 movq m0, [srcq+ssq*0]
648 punpcklbw m1, m2
649 punpcklbw m2, m0
650 pmaddubsw m1, m4
651 pmaddubsw m2, m4
652 pmulhrsw m1, m5
653 pmulhrsw m2, m5
654 packuswb m1, m2
655 movq [dstq+dsq*0], m1
656 movhps [dstq+dsq*1], m1
657 lea dstq, [dstq+dsq*2]
658 sub hd, 2
659 jg .v_w8_loop
660 RET
661 %macro PUT_BILIN_V_W16 0
662 movu m0, [srcq+ssq*0]
663 %%loop:
664 movu m3, [srcq+ssq*1]
665 lea srcq, [srcq+ssq*2]
666 mova m1, m0
667 mova m2, m0
668 movu m0, [srcq+ssq*0]
669 punpcklbw m1, m3
670 punpckhbw m2, m3
671 pmaddubsw m1, m4
672 pmaddubsw m2, m4
673 pmulhrsw m1, m5
674 pmulhrsw m2, m5
675 packuswb m1, m2
676 punpcklbw m2, m3, m0
677 punpckhbw m3, m0
678 pmaddubsw m2, m4
679 pmaddubsw m3, m4
680 pmulhrsw m2, m5
681 pmulhrsw m3, m5
682 packuswb m2, m3
683 mova [dstq+dsq*0], m1
684 mova [dstq+dsq*1], m2
685 lea dstq, [dstq+dsq*2]
686 sub hd, 2
687 jg %%loop
688 %endmacro
689 .v_w16:
690 PUT_BILIN_V_W16
691 RET
692 .v_w128:
693 lea r6d, [hq+(7<<16)]
694 jmp .v_w16gt
695 .v_w64:
696 lea r6d, [hq+(3<<16)]
697 jmp .v_w16gt
698 .v_w32:
699 lea r6d, [hq+(1<<16)]
700 .v_w16gt:
701 mov r4, srcq
702 %if ARCH_X86_64
703 mov r7, dstq
704 %endif
705 .v_w16gt_loop:
706 PUT_BILIN_V_W16
707 %if ARCH_X86_64
708 add r4, 16
709 add r7, 16
710 movzx hd, r6b
711 mov srcq, r4
712 mov dstq, r7
713 %else
714 mov dstq, dstmp
715 add r4, 16
716 movzx hd, r6w
717 add dstq, 16
718 mov srcq, r4
719 mov dstmp, dstq
720 %endif
721 sub r6d, 1<<16
722 jg .v_w16gt
723 RET
724 .hv:
725 ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8
726 ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4
727 movzx wd, word [t0+wq*2+table_offset(put, _bilin_hv)]
728 WIN64_SPILL_XMM 8
729 shl mxyd, 11 ; can't shift by 12 due to signed overflow
730 mova m7, [base+pw_15]
731 movd m6, mxyd
732 add wq, t0
733 pshuflw m6, m6, q0000
734 paddb m5, m5
735 punpcklqdq m6, m6
736 jmp wq
737 .hv_w2:
738 RESTORE_DSQ_32 t0
739 movd m0, [srcq+ssq*0]
740 punpckldq m0, m0
741 pshufb m0, m4
742 pmaddubsw m0, m5
743 .hv_w2_loop:
744 movd m1, [srcq+ssq*1]
745 lea srcq, [srcq+ssq*2]
746 movd m2, [srcq+ssq*0]
747 punpckldq m1, m2
748 pshufb m1, m4
749 pmaddubsw m1, m5 ; 1 _ 2 _
750 shufps m2, m0, m1, q1032 ; 0 _ 1 _
751 mova m0, m1
752 psubw m1, m2 ; 2 * (src[x + src_stride] - src[x])
753 pmulhw m1, m6 ; (my * (src[x + src_stride] - src[x]) >> 4
754 pavgw m2, m7 ; src[x] + 8
755 paddw m1, m2 ; src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8
756 psrlw m1, 4
757 packuswb m1, m1
758 %if ARCH_X86_64
759 movq r6, m1
760 %else
761 pshuflw m1, m1, q2020
762 movd r6d, m1
763 %endif
764 mov [dstq+dsq*0], r6w
765 shr r6, gprsize*4
766 mov [dstq+dsq*1], r6w
767 lea dstq, [dstq+dsq*2]
768 sub hd, 2
769 jg .hv_w2_loop
770 RET
771 .hv_w4:
772 mova m4, [base+bilin_h_shuf4]
773 movddup m0, [srcq+ssq*0]
774 movifnidn dsq, dsmp
775 pshufb m0, m4
776 pmaddubsw m0, m5
777 .hv_w4_loop:
778 movq m1, [srcq+ssq*1]
779 lea srcq, [srcq+ssq*2]
780 movhps m1, [srcq+ssq*0]
781 pshufb m1, m4
782 pmaddubsw m1, m5 ; 1 2
783 shufps m2, m0, m1, q1032 ; 0 1
784 mova m0, m1
785 psubw m1, m2
786 pmulhw m1, m6
787 pavgw m2, m7
788 paddw m1, m2
789 psrlw m1, 4
790 packuswb m1, m1
791 movd [dstq+dsq*0], m1
792 psrlq m1, 32
793 movd [dstq+dsq*1], m1
794 lea dstq, [dstq+dsq*2]
795 sub hd, 2
796 jg .hv_w4_loop
797 RET
798 .hv_w8:
799 movu m0, [srcq+ssq*0]
800 movifnidn dsq, dsmp
801 pshufb m0, m4
802 pmaddubsw m0, m5
803 .hv_w8_loop:
804 movu m2, [srcq+ssq*1]
805 lea srcq, [srcq+ssq*2]
806 pshufb m2, m4
807 pmaddubsw m2, m5
808 psubw m1, m2, m0
809 pmulhw m1, m6
810 pavgw m0, m7
811 paddw m1, m0
812 movu m0, [srcq+ssq*0]
813 pshufb m0, m4
814 pmaddubsw m0, m5
815 psubw m3, m0, m2
816 pmulhw m3, m6
817 pavgw m2, m7
818 paddw m3, m2
819 psrlw m1, 4
820 psrlw m3, 4
821 packuswb m1, m3
822 movq [dstq+dsq*0], m1
823 movhps [dstq+dsq*1], m1
824 lea dstq, [dstq+dsq*2]
825 sub hd, 2
826 jg .hv_w8_loop
827 RET
828 .hv_w128:
829 lea r6d, [hq+(7<<16)]
830 jmp .hv_w16_start
831 .hv_w64:
832 lea r6d, [hq+(3<<16)]
833 jmp .hv_w16_start
834 .hv_w32:
835 lea r6d, [hq+(1<<16)]
836 .hv_w16_start:
837 mov r4, srcq
838 %if ARCH_X86_32
839 %define m8 [dstq]
840 %else
841 mov r7, dstq
842 %endif
843 .hv_w16:
844 movifnidn dsq, dsmp
845 %if WIN64
846 movaps r4m, m8
847 %endif
848 .hv_w16_loop0:
849 movu m0, [srcq+8*0]
850 movu m1, [srcq+8*1]
851 pshufb m0, m4
852 pshufb m1, m4
853 pmaddubsw m0, m5
854 pmaddubsw m1, m5
855 .hv_w16_loop:
856 add srcq, ssq
857 movu m2, [srcq+8*0]
858 movu m3, [srcq+8*1]
859 pshufb m2, m4
860 pshufb m3, m4
861 pmaddubsw m2, m5
862 pmaddubsw m3, m5
863 mova m8, m2
864 psubw m2, m0
865 pmulhw m2, m6
866 pavgw m0, m7
867 paddw m2, m0
868 mova m0, m3
869 psubw m3, m1
870 pmulhw m3, m6
871 pavgw m1, m7
872 paddw m3, m1
873 mova m1, m0
874 mova m0, m8
875 psrlw m2, 4
876 psrlw m3, 4
877 packuswb m2, m3
878 mova [dstq], m2
879 add dstq, dsmp
880 dec hd
881 jg .hv_w16_loop
882 %if ARCH_X86_32
883 mov dstq, dstm
884 add r4, 16
885 movzx hd, r6w
886 add dstq, 16
887 mov srcq, r4
888 mov dstm, dstq
889 %else
890 add r4, 16
891 add r7, 16
892 movzx hd, r6b
893 mov srcq, r4
894 mov dstq, r7
895 %endif
896 sub r6d, 1<<16
897 jg .hv_w16_loop0
898 %if WIN64
899 movaps m8, r4m
900 %endif
901 RET
902
903 %macro PSHUFB_BILIN_H8 2 ; dst, src
904 %if cpuflag(ssse3)
905 pshufb %1, %2
906 %else
907 psrldq %2, %1, 1
908 punpcklbw %1, %2
909 %endif
910 %endmacro
911
912 %macro PSHUFB_BILIN_H4 3 ; dst, src, tmp
913 %if cpuflag(ssse3)
914 pshufb %1, %2
915 %else
916 psrldq %2, %1, 1
917 punpckhbw %3, %1, %2
918 punpcklbw %1, %2
919 punpcklqdq %1, %3
920 %endif
921 %endmacro
922
923 %macro PMADDUBSW 5 ; dst/src1, src2, zero, tmp, reset_zero
924 %if cpuflag(ssse3)
925 pmaddubsw %1, %2
926 %else
927 %if %5 == 1
928 pxor %3, %3
929 %endif
930 punpckhbw %4, %1, %3
931 punpcklbw %1, %1, %3
932 pmaddwd %4, %2
933 pmaddwd %1, %2
934 packssdw %1, %4
935 %endif
936 %endmacro
937
938 %macro PMULHRSW 5 ; dst, src, tmp, rndval, shift
939 %if cpuflag(ssse3)
940 pmulhrsw %1, %2
941 %else
942 punpckhwd %3, %1, %4
943 punpcklwd %1, %4
944 pmaddwd %3, %2
945 pmaddwd %1, %2
946 psrad %3, %5
947 psrad %1, %5
948 packssdw %1, %3
949 %endif
950 %endmacro
951
952 %macro PREP_BILIN 0
953 %if ARCH_X86_32
954 %define base r6-prep%+SUFFIX
955 %else
956 %define base 0
957 %endif
958
959 cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
960 movifnidn mxyd, r5m ; mx
961 LEA r6, prep%+SUFFIX
962 tzcnt wd, wm
963 movifnidn hd, hm
964 test mxyd, mxyd
965 jnz .h
966 mov mxyd, r6m ; my
967 test mxyd, mxyd
968 jnz .v
969 .prep:
970 %if notcpuflag(ssse3)
971 add r6, prep_ssse3 - prep_sse2
972 jmp prep_ssse3
973 %else
974 movzx wd, word [r6+wq*2+table_offset(prep,)]
975 pxor m4, m4
976 add wq, r6
977 lea stride3q, [strideq*3]
978 jmp wq
979 .prep_w4:
980 movd m0, [srcq+strideq*0]
981 movd m1, [srcq+strideq*1]
982 movd m2, [srcq+strideq*2]
983 movd m3, [srcq+stride3q ]
984 lea srcq, [srcq+strideq*4]
985 punpckldq m0, m1
986 punpckldq m2, m3
987 punpcklbw m0, m4
988 punpcklbw m2, m4
989 psllw m0, 4
990 psllw m2, 4
991 mova [tmpq+16*0], m0
992 mova [tmpq+16*1], m2
993 add tmpq, 16*2
994 sub hd, 4
995 jg .prep_w4
996 RET
997 .prep_w8:
998 movq m0, [srcq+strideq*0]
999 movq m1, [srcq+strideq*1]
1000 movq m2, [srcq+strideq*2]
1001 movq m3, [srcq+stride3q ]
1002 lea srcq, [srcq+strideq*4]
1003 punpcklbw m0, m4
1004 punpcklbw m1, m4
1005 punpcklbw m2, m4
1006 punpcklbw m3, m4
1007 psllw m0, 4
1008 psllw m1, 4
1009 psllw m2, 4
1010 psllw m3, 4
1011 mova [tmpq+16*0], m0
1012 mova [tmpq+16*1], m1
1013 mova [tmpq+16*2], m2
1014 mova [tmpq+16*3], m3
1015 add tmpq, 16*4
1016 sub hd, 4
1017 jg .prep_w8
1018 RET
1019 .prep_w16:
1020 movu m1, [srcq+strideq*0]
1021 movu m3, [srcq+strideq*1]
1022 lea srcq, [srcq+strideq*2]
1023 punpcklbw m0, m1, m4
1024 punpckhbw m1, m4
1025 punpcklbw m2, m3, m4
1026 punpckhbw m3, m4
1027 psllw m0, 4
1028 psllw m1, 4
1029 psllw m2, 4
1030 psllw m3, 4
1031 mova [tmpq+16*0], m0
1032 mova [tmpq+16*1], m1
1033 mova [tmpq+16*2], m2
1034 mova [tmpq+16*3], m3
1035 add tmpq, 16*4
1036 sub hd, 2
1037 jg .prep_w16
1038 RET
1039 .prep_w128:
1040 mov r3, -128
1041 jmp .prep_w32_start
1042 .prep_w64:
1043 mov r3, -64
1044 jmp .prep_w32_start
1045 .prep_w32:
1046 mov r3, -32
1047 .prep_w32_start:
1048 sub srcq, r3
1049 .prep_w32_vloop:
1050 mov r6, r3
1051 .prep_w32_hloop:
1052 movu m1, [srcq+r6+16*0]
1053 movu m3, [srcq+r6+16*1]
1054 punpcklbw m0, m1, m4
1055 punpckhbw m1, m4
1056 punpcklbw m2, m3, m4
1057 punpckhbw m3, m4
1058 psllw m0, 4
1059 psllw m1, 4
1060 psllw m2, 4
1061 psllw m3, 4
1062 mova [tmpq+16*0], m0
1063 mova [tmpq+16*1], m1
1064 mova [tmpq+16*2], m2
1065 mova [tmpq+16*3], m3
1066 add tmpq, 16*4
1067 add r6, 32
1068 jl .prep_w32_hloop
1069 add srcq, strideq
1070 dec hd
1071 jg .prep_w32_vloop
1072 RET
1073 %endif
1074 .h:
1075 ; 16 * src[x] + (mx * (src[x + 1] - src[x]))
1076 ; = (16 - mx) * src[x] + mx * src[x + 1]
1077 %if cpuflag(ssse3)
1078 imul mxyd, 0x00ff00ff
1079 mova m4, [base+bilin_h_shuf8]
1080 add mxyd, 0x00100010
1081 %else
1082 imul mxyd, 0xffff
1083 add mxyd, 16
1084 %endif
1085 movd m5, mxyd
1086 mov mxyd, r6m ; my
1087 pshufd m5, m5, q0000
1088 test mxyd, mxyd
1089 jnz .hv
1090 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)]
1091 %if notcpuflag(ssse3)
1092 WIN64_SPILL_XMM 8
1093 pxor m6, m6
1094 %endif
1095 add wq, r6
1096 jmp wq
1097 .h_w4:
1098 %if cpuflag(ssse3)
1099 mova m4, [base+bilin_h_shuf4]
1100 %endif
1101 lea stride3q, [strideq*3]
1102 .h_w4_loop:
1103 movq m0, [srcq+strideq*0]
1104 movhps m0, [srcq+strideq*1]
1105 movq m1, [srcq+strideq*2]
1106 movhps m1, [srcq+stride3q ]
1107 lea srcq, [srcq+strideq*4]
1108 PSHUFB_BILIN_H4 m0, m4, m2
1109 PMADDUBSW m0, m5, m6, m2, 0
1110 PSHUFB_BILIN_H4 m1, m4, m2
1111 PMADDUBSW m1, m5, m6, m2, 0
1112 mova [tmpq+0 ], m0
1113 mova [tmpq+16], m1
1114 add tmpq, 32
1115 sub hd, 4
1116 jg .h_w4_loop
1117 RET
1118 .h_w8:
1119 lea stride3q, [strideq*3]
1120 .h_w8_loop:
1121 movu m0, [srcq+strideq*0]
1122 movu m1, [srcq+strideq*1]
1123 movu m2, [srcq+strideq*2]
1124 movu m3, [srcq+stride3q ]
1125 lea srcq, [srcq+strideq*4]
1126 PSHUFB_BILIN_H8 m0, m4
1127 PSHUFB_BILIN_H8 m1, m4
1128 PSHUFB_BILIN_H8 m2, m4
1129 PSHUFB_BILIN_H8 m3, m4
1130 PMADDUBSW m0, m5, m6, m7, 0
1131 PMADDUBSW m1, m5, m6, m7, 0
1132 PMADDUBSW m2, m5, m6, m7, 0
1133 PMADDUBSW m3, m5, m6, m7, 0
1134 mova [tmpq+16*0], m0
1135 mova [tmpq+16*1], m1
1136 mova [tmpq+16*2], m2
1137 mova [tmpq+16*3], m3
1138 add tmpq, 16*4
1139 sub hd, 4
1140 jg .h_w8_loop
1141 RET
1142 .h_w16:
1143 movu m0, [srcq+strideq*0+8*0]
1144 movu m1, [srcq+strideq*0+8*1]
1145 movu m2, [srcq+strideq*1+8*0]
1146 movu m3, [srcq+strideq*1+8*1]
1147 lea srcq, [srcq+strideq*2]
1148 PSHUFB_BILIN_H8 m0, m4
1149 PSHUFB_BILIN_H8 m1, m4
1150 PSHUFB_BILIN_H8 m2, m4
1151 PSHUFB_BILIN_H8 m3, m4
1152 PMADDUBSW m0, m5, m6, m7, 0
1153 PMADDUBSW m1, m5, m6, m7, 0
1154 PMADDUBSW m2, m5, m6, m7, 0
1155 PMADDUBSW m3, m5, m6, m7, 0
1156 mova [tmpq+16*0], m0
1157 mova [tmpq+16*1], m1
1158 mova [tmpq+16*2], m2
1159 mova [tmpq+16*3], m3
1160 add tmpq, 16*4
1161 sub hd, 2
1162 jg .h_w16
1163 RET
1164 .h_w128:
1165 mov r3, -128
1166 jmp .h_w32_start
1167 .h_w64:
1168 mov r3, -64
1169 jmp .h_w32_start
1170 .h_w32:
1171 mov r3, -32
1172 .h_w32_start:
1173 sub srcq, r3
1174 .h_w32_vloop:
1175 mov r6, r3
1176 .h_w32_hloop:
1177 movu m0, [srcq+r6+8*0]
1178 movu m1, [srcq+r6+8*1]
1179 movu m2, [srcq+r6+8*2]
1180 movu m3, [srcq+r6+8*3]
1181 PSHUFB_BILIN_H8 m0, m4
1182 PSHUFB_BILIN_H8 m1, m4
1183 PSHUFB_BILIN_H8 m2, m4
1184 PSHUFB_BILIN_H8 m3, m4
1185 PMADDUBSW m0, m5, m6, m7, 0
1186 PMADDUBSW m1, m5, m6, m7, 0
1187 PMADDUBSW m2, m5, m6, m7, 0
1188 PMADDUBSW m3, m5, m6, m7, 0
1189 mova [tmpq+16*0], m0
1190 mova [tmpq+16*1], m1
1191 mova [tmpq+16*2], m2
1192 mova [tmpq+16*3], m3
1193 add tmpq, 16*4
1194 add r6, 32
1195 jl .h_w32_hloop
1196 add srcq, strideq
1197 dec hd
1198 jg .h_w32_vloop
1199 RET
1200 .v:
1201 %if notcpuflag(ssse3)
1202 %assign stack_offset stack_offset - stack_size_padded
1203 WIN64_SPILL_XMM 8
1204 %endif
1205 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)]
1206 %if cpuflag(ssse3)
1207 imul mxyd, 0x00ff00ff
1208 add mxyd, 0x00100010
1209 %else
1210 imul mxyd, 0xffff
1211 pxor m6, m6
1212 add mxyd, 16
1213 %endif
1214 add wq, r6
1215 lea stride3q, [strideq*3]
1216 movd m5, mxyd
1217 pshufd m5, m5, q0000
1218 jmp wq
1219 .v_w4:
1220 movd m0, [srcq+strideq*0]
1221 .v_w4_loop:
1222 movd m1, [srcq+strideq*1]
1223 movd m2, [srcq+strideq*2]
1224 movd m3, [srcq+stride3q ]
1225 lea srcq, [srcq+strideq*4]
1226 punpckldq m0, m1
1227 punpckldq m1, m2
1228 punpcklbw m0, m1 ; 01 12
1229 PMADDUBSW m0, m5, m6, m7, 0
1230 mova [tmpq+16*0], m0
1231 movd m0, [srcq+strideq*0]
1232 punpckldq m2, m3
1233 punpckldq m3, m0
1234 punpcklbw m2, m3 ; 23 34
1235 PMADDUBSW m2, m5, m6, m7, 0
1236 mova [tmpq+16*1], m2
1237 add tmpq, 16*2
1238 sub hd, 4
1239 jg .v_w4_loop
1240 RET
1241 .v_w8:
1242 movq m0, [srcq+strideq*0]
1243 .v_w8_loop:
1244 movq m1, [srcq+strideq*1]
1245 movq m2, [srcq+strideq*2]
1246 movq m3, [srcq+stride3q ]
1247 lea srcq, [srcq+strideq*4]
1248 punpcklbw m0, m1 ; 01
1249 punpcklbw m1, m2 ; 12
1250 PMADDUBSW m0, m5, m6, m7, 0
1251 PMADDUBSW m1, m5, m6, m7, 0
1252 mova [tmpq+16*0], m0
1253 movq m0, [srcq+strideq*0]
1254 punpcklbw m2, m3 ; 23
1255 punpcklbw m3, m0 ; 34
1256 PMADDUBSW m2, m5, m6, m7, 0
1257 mova [tmpq+16*1], m1
1258 PMADDUBSW m3, m5, m6, m7, 0
1259 mova [tmpq+16*2], m2
1260 mova [tmpq+16*3], m3
1261 add tmpq, 16*4
1262 sub hd, 4
1263 jg .v_w8_loop
1264 RET
1265 .v_w16:
1266 movu m0, [srcq+strideq*0]
1267 .v_w16_loop:
1268 movu m1, [srcq+strideq*1]
1269 movu m2, [srcq+strideq*2]
1270 movu m3, [srcq+stride3q ]
1271 lea srcq, [srcq+strideq*4]
1272 punpcklbw m4, m0, m1
1273 punpckhbw m0, m1
1274 PMADDUBSW m4, m5, m6, m7, 0
1275 PMADDUBSW m0, m5, m6, m7, 0
1276 mova [tmpq+16*0], m4
1277 punpcklbw m4, m1, m2
1278 punpckhbw m1, m2
1279 PMADDUBSW m4, m5, m6, m7, 0
1280 mova [tmpq+16*1], m0
1281 movu m0, [srcq+strideq*0]
1282 PMADDUBSW m1, m5, m6, m7, 0
1283 mova [tmpq+16*2], m4
1284 punpcklbw m4, m2, m3
1285 punpckhbw m2, m3
1286 PMADDUBSW m4, m5, m6, m7, 0
1287 mova [tmpq+16*3], m1
1288 PMADDUBSW m2, m5, m6, m7, 0
1289 mova [tmpq+16*4], m4
1290 punpcklbw m4, m3, m0
1291 punpckhbw m3, m0
1292 PMADDUBSW m4, m5, m6, m7, 0
1293 mova [tmpq+16*5], m2
1294 PMADDUBSW m3, m5, m6, m7, 0
1295 mova [tmpq+16*6], m4
1296 mova [tmpq+16*7], m3
1297 add tmpq, 16*8
1298 sub hd, 4
1299 jg .v_w16_loop
1300 RET
1301 .v_w128:
1302 lea r3d, [hq+(3<<8)]
1303 mov r6d, 256
1304 jmp .v_w32_start
1305 .v_w64:
1306 lea r3d, [hq+(1<<8)]
1307 mov r6d, 128
1308 jmp .v_w32_start
1309 .v_w32:
1310 xor r3d, r3d
1311 mov r6d, 64
1312 .v_w32_start:
1313 %if ARCH_X86_64
1314 %if WIN64
1315 PUSH r7
1316 %endif
1317 mov r7, tmpq
1318 %endif
1319 mov r5, srcq
1320 .v_w32_hloop:
1321 movu m0, [srcq+strideq*0+16*0]
1322 movu m1, [srcq+strideq*0+16*1]
1323 .v_w32_vloop:
1324 movu m2, [srcq+strideq*1+16*0]
1325 movu m3, [srcq+strideq*1+16*1]
1326 lea srcq, [srcq+strideq*2]
1327 punpcklbw m4, m0, m2
1328 punpckhbw m0, m2
1329 PMADDUBSW m4, m5, m6, m7, 0
1330 PMADDUBSW m0, m5, m6, m7, 0
1331 mova [tmpq+16*0], m4
1332 mova [tmpq+16*1], m0
1333 movu m0, [srcq+strideq*0+16*0]
1334 punpcklbw m4, m1, m3
1335 punpckhbw m1, m3
1336 PMADDUBSW m4, m5, m6, m7, 0
1337 PMADDUBSW m1, m5, m6, m7, 0
1338 mova [tmpq+16*2], m4
1339 mova [tmpq+16*3], m1
1340 movu m1, [srcq+strideq*0+16*1]
1341 add tmpq, r6
1342 punpcklbw m4, m2, m0
1343 punpckhbw m2, m0
1344 PMADDUBSW m4, m5, m6, m7, 0
1345 PMADDUBSW m2, m5, m6, m7, 0
1346 mova [tmpq+16*0], m4
1347 mova [tmpq+16*1], m2
1348 punpcklbw m4, m3, m1
1349 punpckhbw m3, m1
1350 PMADDUBSW m4, m5, m6, m7, 0
1351 PMADDUBSW m3, m5, m6, m7, 0
1352 mova [tmpq+16*2], m4
1353 mova [tmpq+16*3], m3
1354 add tmpq, r6
1355 sub hd, 2
1356 jg .v_w32_vloop
1357 add r5, 32
1358 movzx hd, r3b
1359 mov srcq, r5
1360 %if ARCH_X86_64
1361 add r7, 16*4
1362 mov tmpq, r7
1363 %else
1364 mov tmpq, tmpmp
1365 add tmpq, 16*4
1366 mov tmpmp, tmpq
1367 %endif
1368 sub r3d, 1<<8
1369 jg .v_w32_hloop
1370 %if WIN64
1371 POP r7
1372 %endif
1373 RET
1374 .hv:
1375 ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4
1376 ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4)
1377 movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)]
1378 %assign stack_offset stack_offset - stack_size_padded
1379 %if cpuflag(ssse3)
1380 imul mxyd, 0x08000800
1381 WIN64_SPILL_XMM 8
1382 %else
1383 or mxyd, 1<<16
1384 WIN64_SPILL_XMM 9
1385 %if ARCH_X86_64
1386 mova m8, [base+pw_8]
1387 %else
1388 %define m8 [base+pw_8]
1389 %endif
1390 pxor m7, m7
1391 %endif
1392 movd m6, mxyd
1393 add wq, r6
1394 pshufd m6, m6, q0000
1395 jmp wq
1396 .hv_w4:
1397 %if cpuflag(ssse3)
1398 mova m4, [base+bilin_h_shuf4]
1399 movddup m0, [srcq+strideq*0]
1400 %else
1401 movhps m0, [srcq+strideq*0]
1402 %endif
1403 lea r3, [strideq*3]
1404 PSHUFB_BILIN_H4 m0, m4, m3
1405 PMADDUBSW m0, m5, m7, m4, 0 ; _ 0
1406 .hv_w4_loop:
1407 movq m1, [srcq+strideq*1]
1408 movhps m1, [srcq+strideq*2]
1409 movq m2, [srcq+r3 ]
1410 lea srcq, [srcq+strideq*4]
1411 movhps m2, [srcq+strideq*0]
1412 PSHUFB_BILIN_H4 m1, m4, m3
1413 PSHUFB_BILIN_H4 m2, m4, m3
1414 PMADDUBSW m1, m5, m7, m4, 0 ; 1 2
1415 PMADDUBSW m2, m5, m7, m4, 0 ; 3 4
1416 shufpd m0, m1, 0x01 ; 0 1
1417 shufpd m3, m1, m2, 0x01 ; 2 3
1418 psubw m1, m0
1419 PMULHRSW m1, m6, m4, m8, 4
1420 paddw m1, m0
1421 mova m0, m2
1422 psubw m2, m3
1423 PMULHRSW m2, m6, m4, m8, 4
1424 paddw m2, m3
1425 mova [tmpq+16*0], m1
1426 mova [tmpq+16*1], m2
1427 add tmpq, 32
1428 sub hd, 4
1429 jg .hv_w4_loop
1430 RET
1431 .hv_w8:
1432 movu m0, [srcq+strideq*0]
1433 PSHUFB_BILIN_H8 m0, m4
1434 PMADDUBSW m0, m5, m7, m4, 0 ; 0
1435 .hv_w8_loop:
1436 movu m1, [srcq+strideq*1]
1437 lea srcq, [srcq+strideq*2]
1438 movu m2, [srcq+strideq*0]
1439 PSHUFB_BILIN_H8 m1, m4
1440 PSHUFB_BILIN_H8 m2, m4
1441 PMADDUBSW m1, m5, m7, m4, 0 ; 1
1442 PMADDUBSW m2, m5, m7, m4, 0 ; 2
1443 psubw m3, m1, m0
1444 PMULHRSW m3, m6, m4, m8, 4
1445 paddw m3, m0
1446 mova m0, m2
1447 psubw m2, m1
1448 PMULHRSW m2, m6, m4, m8, 4
1449 paddw m2, m1
1450 mova [tmpq+16*0], m3
1451 mova [tmpq+16*1], m2
1452 add tmpq, 16*2
1453 sub hd, 2
1454 jg .hv_w8_loop
1455 RET
1456 .hv_w128:
1457 lea r3d, [hq+(7<<8)]
1458 mov r5d, 256
1459 jmp .hv_w16_start
1460 .hv_w64:
1461 lea r3d, [hq+(3<<8)]
1462 mov r5d, 128
1463 jmp .hv_w16_start
1464 .hv_w32:
1465 lea r3d, [hq+(1<<8)]
1466 mov r5d, 64
1467 jmp .hv_w16_start
1468 .hv_w16:
1469 xor r3d, r3d
1470 mov r5d, 32
1471 .hv_w16_start:
1472 %if ARCH_X86_64 || cpuflag(ssse3)
1473 mov r6, srcq
1474 %endif
1475 %if ARCH_X86_64
1476 %if WIN64
1477 PUSH r7
1478 %endif
1479 mov r7, tmpq
1480 %endif
1481 .hv_w16_hloop:
1482 movu m0, [srcq+strideq*0+8*0]
1483 movu m1, [srcq+strideq*0+8*1]
1484 PSHUFB_BILIN_H8 m0, m4
1485 PSHUFB_BILIN_H8 m1, m4
1486 PMADDUBSW m0, m5, m7, m4, 0 ; 0a
1487 PMADDUBSW m1, m5, m7, m4, 0 ; 0b
1488 .hv_w16_vloop:
1489 movu m2, [srcq+strideq*1+8*0]
1490 PSHUFB_BILIN_H8 m2, m4
1491 PMADDUBSW m2, m5, m7, m4, 0 ; 1a
1492 psubw m3, m2, m0
1493 PMULHRSW m3, m6, m4, m8, 4
1494 paddw m3, m0
1495 mova [tmpq+16*0], m3
1496 movu m3, [srcq+strideq*1+8*1]
1497 lea srcq, [srcq+strideq*2]
1498 PSHUFB_BILIN_H8 m3, m4
1499 PMADDUBSW m3, m5, m7, m4, 0 ; 1b
1500 psubw m0, m3, m1
1501 PMULHRSW m0, m6, m4, m8, 4
1502 paddw m0, m1
1503 mova [tmpq+16*1], m0
1504 add tmpq, r5
1505 movu m0, [srcq+strideq*0+8*0]
1506 PSHUFB_BILIN_H8 m0, m4
1507 PMADDUBSW m0, m5, m7, m4, 0 ; 2a
1508 psubw m1, m0, m2
1509 PMULHRSW m1, m6, m4, m8, 4
1510 paddw m1, m2
1511 mova [tmpq+16*0], m1
1512 movu m1, [srcq+strideq*0+8*1]
1513 PSHUFB_BILIN_H8 m1, m4
1514 PMADDUBSW m1, m5, m7, m4, 0 ; 2b
1515 psubw m2, m1, m3
1516 PMULHRSW m2, m6, m4, m8, 4
1517 paddw m2, m3
1518 mova [tmpq+16*1], m2
1519 add tmpq, r5
1520 sub hd, 2
1521 jg .hv_w16_vloop
1522 movzx hd, r3b
1523 %if ARCH_X86_64
1524 add r6, 16
1525 add r7, 2*16
1526 mov srcq, r6
1527 mov tmpq, r7
1528 %elif cpuflag(ssse3)
1529 mov tmpq, tmpm
1530 add r6, 16
1531 add tmpq, 2*16
1532 mov srcq, r6
1533 mov tmpm, tmpq
1534 %else
1535 mov srcq, srcm
1536 mov tmpq, tmpm
1537 add srcq, 16
1538 add tmpq, 2*16
1539 mov srcm, srcq
1540 mov tmpm, tmpq
1541 %endif
1542 sub r3d, 1<<8
1543 jg .hv_w16_hloop
1544 %if WIN64
1545 POP r7
1546 %endif
1547 RET
1548 %endmacro
1549
1550 ; int8_t subpel_filters[5][15][8]
1551 %assign FILTER_REGULAR (0*15 << 16) | 3*15
1552 %assign FILTER_SMOOTH (1*15 << 16) | 4*15
1553 %assign FILTER_SHARP (2*15 << 16) | 3*15
1554
1555 %macro FN 4 ; prefix, type, type_h, type_v
1556 cglobal %1_%2_8bpc
1557 mov t0d, FILTER_%3
1558 %ifidn %3, %4
1559 mov t1d, t0d
1560 %else
1561 mov t1d, FILTER_%4
1562 %endif
1563 %ifnidn %2, regular ; skip the jump in the last filter
1564 jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX)
1565 %endif
1566 %endmacro
1567
1568 %if ARCH_X86_32
1569 DECLARE_REG_TMP 1, 2
1570 %elif WIN64
1571 DECLARE_REG_TMP 4, 5
1572 %else
1573 DECLARE_REG_TMP 7, 8
1574 %endif
1575
1576 FN put_8tap, sharp, SHARP, SHARP
1577 FN put_8tap, sharp_smooth, SHARP, SMOOTH
1578 FN put_8tap, smooth_sharp, SMOOTH, SHARP
1579 FN put_8tap, smooth, SMOOTH, SMOOTH
1580 FN put_8tap, sharp_regular, SHARP, REGULAR
1581 FN put_8tap, regular_sharp, REGULAR, SHARP
1582 FN put_8tap, smooth_regular, SMOOTH, REGULAR
1583 FN put_8tap, regular_smooth, REGULAR, SMOOTH
1584 FN put_8tap, regular, REGULAR, REGULAR
1585
1586 %if ARCH_X86_32
1587 %define base_reg r1
1588 %define base base_reg-put_ssse3
1589 %else
1590 %define base_reg r8
1591 %define base 0
1592 %endif
1593
1594 cglobal put_8tap_8bpc, 1, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
1595 %assign org_stack_offset stack_offset
1596 imul mxd, mxm, 0x010101
1597 add mxd, t0d ; 8tap_h, mx, 4tap_h
1598 %if ARCH_X86_64
1599 imul myd, mym, 0x010101
1600 add myd, t1d ; 8tap_v, my, 4tap_v
1601 %else
1602 imul ssd, mym, 0x010101
1603 add ssd, t1d ; 8tap_v, my, 4tap_v
1604 mov srcq, srcm
1605 %endif
1606 mov wd, wm
1607 movifnidn hd, hm
1608 LEA base_reg, put_ssse3
1609 test mxd, 0xf00
1610 jnz .h
1611 %if ARCH_X86_32
1612 test ssd, 0xf00
1613 %else
1614 test myd, 0xf00
1615 %endif
1616 jnz .v
1617 tzcnt wd, wd
1618 movzx wd, word [base_reg+wq*2+table_offset(put,)]
1619 add wq, base_reg
1620 ; put_bilin mangling jump
1621 %assign stack_offset org_stack_offset
1622 movifnidn dsq, dsmp
1623 movifnidn ssq, ssmp
1624 %if WIN64
1625 pop r8
1626 %endif
1627 lea r6, [ssq*3]
1628 jmp wq
1629 .h:
1630 %if ARCH_X86_32
1631 test ssd, 0xf00
1632 %else
1633 test myd, 0xf00
1634 %endif
1635 jnz .hv
1636 movifnidn ssq, ssmp
1637 WIN64_SPILL_XMM 12
1638 cmp wd, 4
1639 jl .h_w2
1640 je .h_w4
1641 tzcnt wd, wd
1642 %if ARCH_X86_64
1643 mova m10, [base+subpel_h_shufA]
1644 mova m11, [base+subpel_h_shufB]
1645 mova m9, [base+subpel_h_shufC]
1646 %endif
1647 shr mxd, 16
1648 sub srcq, 3
1649 movzx wd, word [base_reg+wq*2+table_offset(put, _8tap_h)]
1650 movq m6, [base_reg+mxq*8+subpel_filters-put_ssse3]
1651 mova m7, [base+pw_34] ; 2 + (8 << 2)
1652 pshufd m5, m6, q0000
1653 pshufd m6, m6, q1111
1654 add wq, base_reg
1655 jmp wq
1656 .h_w2:
1657 %if ARCH_X86_32
1658 and mxd, 0x7f
1659 %else
1660 movzx mxd, mxb
1661 %endif
1662 dec srcq
1663 mova m4, [base+subpel_h_shuf4]
1664 movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2]
1665 mova m5, [base+pw_34] ; 2 + (8 << 2)
1666 pshufd m3, m3, q0000
1667 movifnidn dsq, dsmp
1668 .h_w2_loop:
1669 movq m0, [srcq+ssq*0]
1670 movhps m0, [srcq+ssq*1]
1671 lea srcq, [srcq+ssq*2]
1672 pshufb m0, m4
1673 pmaddubsw m0, m3
1674 phaddw m0, m0
1675 paddw m0, m5 ; pw34
1676 psraw m0, 6
1677 packuswb m0, m0
1678 movd r6d, m0
1679 mov [dstq+dsq*0], r6w
1680 shr r6d, 16
1681 mov [dstq+dsq*1], r6w
1682 lea dstq, [dstq+dsq*2]
1683 sub hd, 2
1684 jg .h_w2_loop
1685 RET
1686 .h_w4:
1687 %if ARCH_X86_32
1688 and mxd, 0x7f
1689 %else
1690 movzx mxd, mxb
1691 %endif
1692 dec srcq
1693 movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2]
1694 mova m6, [base+subpel_h_shufA]
1695 mova m5, [base+pw_34] ; 2 + (8 << 2)
1696 pshufd m3, m3, q0000
1697 movifnidn dsq, dsmp
1698 .h_w4_loop:
1699 movq m0, [srcq+ssq*0] ; 1
1700 movq m1, [srcq+ssq*1] ; 2
1701 lea srcq, [srcq+ssq*2]
1702 pshufb m0, m6 ; subpel_h_shufA
1703 pshufb m1, m6 ; subpel_h_shufA
1704 pmaddubsw m0, m3 ; subpel_filters
1705 pmaddubsw m1, m3 ; subpel_filters
1706 phaddw m0, m1
1707 paddw m0, m5 ; pw34
1708 psraw m0, 6
1709 packuswb m0, m0
1710 movd [dstq+dsq*0], m0
1711 psrlq m0, 32
1712 movd [dstq+dsq*1], m0
1713 lea dstq, [dstq+dsq*2]
1714 sub hd, 2
1715 jg .h_w4_loop
1716 RET
1717 %macro PUT_8TAP_H 4 ; dst/src, tmp[1-3]
1718 %if ARCH_X86_32
1719 pshufb %2, %1, [base+subpel_h_shufB]
1720 pshufb %3, %1, [base+subpel_h_shufC]
1721 pshufb %1, [base+subpel_h_shufA]
1722 %else
1723 pshufb %2, %1, m11; subpel_h_shufB
1724 pshufb %3, %1, m9 ; subpel_h_shufC
1725 pshufb %1, m10 ; subpel_h_shufA
1726 %endif
1727 pmaddubsw %4, %2, m5 ; subpel +0 B0
1728 pmaddubsw %2, m6 ; subpel +4 B4
1729 pmaddubsw %3, m6 ; C4
1730 pmaddubsw %1, m5 ; A0
1731 paddw %3, %4 ; C4+B0
1732 paddw %1, %2 ; A0+B4
1733 phaddw %1, %3
1734 paddw %1, m7 ; pw34
1735 psraw %1, 6
1736 %endmacro
1737 .h_w8:
1738 movu m0, [srcq+ssq*0]
1739 movu m1, [srcq+ssq*1]
1740 lea srcq, [srcq+ssq*2]
1741 PUT_8TAP_H m0, m2, m3, m4
1742 PUT_8TAP_H m1, m2, m3, m4
1743 packuswb m0, m1
1744 %if ARCH_X86_32
1745 movq [dstq], m0
1746 add dstq, dsm
1747 movhps [dstq], m0
1748 add dstq, dsm
1749 %else
1750 movq [dstq+dsq*0], m0
1751 movhps [dstq+dsq*1], m0
1752 lea dstq, [dstq+dsq*2]
1753 %endif
1754 sub hd, 2
1755 jg .h_w8
1756 RET
1757 .h_w128:
1758 mov r4, -16*7
1759 jmp .h_w16_start
1760 .h_w64:
1761 mov r4, -16*3
1762 jmp .h_w16_start
1763 .h_w32:
1764 mov r4, -16*1
1765 jmp .h_w16_start
1766 .h_w16:
1767 xor r4d, r4d
1768 .h_w16_start:
1769 sub srcq, r4
1770 sub dstq, r4
1771 .h_w16_loop_v:
1772 mov r6, r4
1773 .h_w16_loop_h:
1774 movu m0, [srcq+r6+8*0]
1775 movu m1, [srcq+r6+8*1]
1776 PUT_8TAP_H m0, m2, m3, m4
1777 PUT_8TAP_H m1, m2, m3, m4
1778 packuswb m0, m1
1779 mova [dstq+r6], m0
1780 add r6, 16
1781 jle .h_w16_loop_h
1782 add srcq, ssq
1783 add dstq, dsmp
1784 dec hd
1785 jg .h_w16_loop_v
1786 RET
1787 .v:
1788 %if ARCH_X86_32
1789 movzx mxd, ssb
1790 shr ssd, 16
1791 cmp hd, 6
1792 cmovs ssd, mxd
1793 movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3]
1794 %else
1795 %assign stack_offset org_stack_offset
1796 WIN64_SPILL_XMM 16
1797 movzx mxd, myb
1798 shr myd, 16
1799 cmp hd, 6
1800 cmovs myd, mxd
1801 movq m0, [base_reg+myq*8+subpel_filters-put_ssse3]
1802 %endif
1803 tzcnt r6d, wd
1804 movzx r6d, word [base_reg+r6*2+table_offset(put, _8tap_v)]
1805 punpcklwd m0, m0
1806 mova m7, [base+pw_512]
1807 add r6, base_reg
1808 %if ARCH_X86_32
1809 %define subpel0 [rsp+mmsize*0]
1810 %define subpel1 [rsp+mmsize*1]
1811 %define subpel2 [rsp+mmsize*2]
1812 %define subpel3 [rsp+mmsize*3]
1813 %assign regs_used 2 ; use r1 (ds) as tmp for stack alignment if needed
1814 ALLOC_STACK -16*4
1815 %assign regs_used 7
1816 pshufd m1, m0, q0000
1817 mova subpel0, m1
1818 pshufd m1, m0, q1111
1819 mova subpel1, m1
1820 pshufd m1, m0, q2222
1821 mova subpel2, m1
1822 pshufd m1, m0, q3333
1823 mova subpel3, m1
1824 mov ssq, [rstk+stack_offset+gprsize*4]
1825 lea ssq, [ssq*3]
1826 sub srcq, ssq
1827 mov ssq, [rstk+stack_offset+gprsize*4]
1828 mov dsq, [rstk+stack_offset+gprsize*2]
1829 %else
1830 %define subpel0 m8
1831 %define subpel1 m9
1832 %define subpel2 m10
1833 %define subpel3 m11
1834 lea ss3q, [ssq*3]
1835 pshufd m8, m0, q0000
1836 sub srcq, ss3q
1837 pshufd m9, m0, q1111
1838 pshufd m10, m0, q2222
1839 pshufd m11, m0, q3333
1840 %endif
1841 jmp r6
1842 .v_w2:
1843 movd m1, [srcq+ssq*0]
1844 movd m0, [srcq+ssq*1]
1845 %if ARCH_X86_32
1846 lea srcq, [srcq+ssq*2]
1847 movd m2, [srcq+ssq*0]
1848 movd m5, [srcq+ssq*1]
1849 lea srcq, [srcq+ssq*2]
1850 movd m3, [srcq+ssq*0]
1851 movd m4, [srcq+ssq*1]
1852 lea srcq, [srcq+ssq*2]
1853 %else
1854 movd m2, [srcq+ssq*2]
1855 add srcq, ss3q
1856 movd m5, [srcq+ssq*0]
1857 movd m3, [srcq+ssq*1]
1858 movd m4, [srcq+ssq*2]
1859 add srcq, ss3q
1860 %endif
1861 punpcklwd m1, m0 ; 0 1
1862 punpcklwd m0, m2 ; 1 2
1863 punpcklbw m1, m0 ; 01 12
1864 movd m0, [srcq+ssq*0]
1865 punpcklwd m2, m5 ; 2 3
1866 punpcklwd m5, m3 ; 3 4
1867 punpcklwd m3, m4 ; 4 5
1868 punpcklwd m4, m0 ; 5 6
1869 punpcklbw m2, m5 ; 23 34
1870 punpcklbw m3, m4 ; 45 56
1871 .v_w2_loop:
1872 movd m4, [srcq+ssq*1]
1873 lea srcq, [srcq+ssq*2]
1874 pmaddubsw m5, m1, subpel0 ; a0 b0
1875 mova m1, m2
1876 pmaddubsw m2, subpel1 ; a1 b1
1877 paddw m5, m2
1878 mova m2, m3
1879 pmaddubsw m3, subpel2 ; a2 b2
1880 paddw m5, m3
1881 punpcklwd m3, m0, m4 ; 6 7
1882 movd m0, [srcq+ssq*0]
1883 punpcklwd m4, m0 ; 7 8
1884 punpcklbw m3, m4 ; 67 78
1885 pmaddubsw m4, m3, subpel3 ; a3 b3
1886 paddw m5, m4
1887 pmulhrsw m5, m7
1888 packuswb m5, m5
1889 movd r6d, m5
1890 mov [dstq+dsq*0], r6w
1891 shr r6d, 16
1892 mov [dstq+dsq*1], r6w
1893 lea dstq, [dstq+dsq*2]
1894 sub hd, 2
1895 jg .v_w2_loop
1896 RET
1897 .v_w4:
1898 %if ARCH_X86_32
1899 .v_w8:
1900 .v_w16:
1901 .v_w32:
1902 .v_w64:
1903 .v_w128:
1904 shl wd, 14
1905 %if STACK_ALIGNMENT < 16
1906 %define dstm [rsp+mmsize*4+gprsize]
1907 mov dstm, dstq
1908 %endif
1909 lea r6d, [hq+wq-(1<<16)]
1910 mov r4, srcq
1911 .v_w4_loop0:
1912 %endif
1913 movd m1, [srcq+ssq*0]
1914 movd m0, [srcq+ssq*1]
1915 %if ARCH_X86_32
1916 lea srcq, [srcq+ssq*2]
1917 movd m2, [srcq+ssq*0]
1918 movd m5, [srcq+ssq*1]
1919 lea srcq, [srcq+ssq*2]
1920 movd m3, [srcq+ssq*0]
1921 movd m4, [srcq+ssq*1]
1922 lea srcq, [srcq+ssq*2]
1923 %else
1924 movd m2, [srcq+ssq*2]
1925 add srcq, ss3q
1926 movd m5, [srcq+ssq*0]
1927 movd m3, [srcq+ssq*1]
1928 movd m4, [srcq+ssq*2]
1929 add srcq, ss3q
1930 %endif
1931 punpckldq m1, m0 ; 0 1
1932 punpckldq m0, m2 ; 1 2
1933 punpcklbw m1, m0 ; 01 12
1934 movd m0, [srcq+ssq*0]
1935 punpckldq m2, m5 ; 2 3
1936 punpckldq m5, m3 ; 3 4
1937 punpckldq m3, m4 ; 4 5
1938 punpckldq m4, m0 ; 5 6
1939 punpcklbw m2, m5 ; 23 34
1940 punpcklbw m3, m4 ; 45 56
1941 .v_w4_loop:
1942 movd m4, [srcq+ssq*1]
1943 lea srcq, [srcq+ssq*2]
1944 pmaddubsw m5, m1, subpel0 ; a0 b0
1945 mova m1, m2
1946 pmaddubsw m2, subpel1 ; a1 b1
1947 paddw m5, m2
1948 mova m2, m3
1949 pmaddubsw m3, subpel2 ; a2 b2
1950 paddw m5, m3
1951 punpckldq m3, m0, m4 ; 6 7 _ _
1952 movd m0, [srcq+ssq*0]
1953 punpckldq m4, m0 ; 7 8 _ _
1954 punpcklbw m3, m4 ; 67 78
1955 pmaddubsw m4, m3, subpel3 ; a3 b3
1956 paddw m5, m4
1957 pmulhrsw m5, m7
1958 packuswb m5, m5
1959 movd [dstq+dsq*0], m5
1960 psrlq m5, 32
1961 movd [dstq+dsq*1], m5
1962 lea dstq, [dstq+dsq*2]
1963 sub hd, 2
1964 jg .v_w4_loop
1965 %if ARCH_X86_32
1966 mov dstq, dstm
1967 add r4, 4
1968 movzx hd, r6w
1969 add dstq, 4
1970 mov srcq, r4
1971 mov dstm, dstq
1972 sub r6d, 1<<16
1973 jg .v_w4_loop0
1974 %endif
1975 RET
1976 %if ARCH_X86_64
1977 .v_w8:
1978 .v_w16:
1979 .v_w32:
1980 .v_w64:
1981 .v_w128:
1982 lea r6d, [wq*8-64]
1983 mov r4, srcq
1984 mov r7, dstq
1985 lea r6d, [hq+r6*4]
1986 .v_w8_loop0:
1987 movq m1, [srcq+ssq*0]
1988 movq m2, [srcq+ssq*1]
1989 movq m3, [srcq+ssq*2]
1990 add srcq, ss3q
1991 movq m4, [srcq+ssq*0]
1992 movq m5, [srcq+ssq*1]
1993 movq m6, [srcq+ssq*2]
1994 add srcq, ss3q
1995 movq m0, [srcq+ssq*0]
1996 punpcklbw m1, m2 ; 01
1997 punpcklbw m2, m3 ; 12
1998 punpcklbw m3, m4 ; 23
1999 punpcklbw m4, m5 ; 34
2000 punpcklbw m5, m6 ; 45
2001 punpcklbw m6, m0 ; 56
2002 .v_w8_loop:
2003 movq m13, [srcq+ssq*1]
2004 lea srcq, [srcq+ssq*2]
2005 pmaddubsw m14, m1, subpel0 ; a0
2006 mova m1, m3
2007 pmaddubsw m15, m2, subpel0 ; b0
2008 mova m2, m4
2009 pmaddubsw m3, subpel1 ; a1
2010 mova m12, m0
2011 pmaddubsw m4, subpel1 ; b1
2012 movq m0, [srcq+ssq*0]
2013 paddw m14, m3
2014 paddw m15, m4
2015 mova m3, m5
2016 pmaddubsw m5, subpel2 ; a2
2017 mova m4, m6
2018 pmaddubsw m6, subpel2 ; b2
2019 punpcklbw m12, m13 ; 67
2020 punpcklbw m13, m0 ; 78
2021 paddw m14, m5
2022 mova m5, m12
2023 pmaddubsw m12, subpel3 ; a3
2024 paddw m15, m6
2025 mova m6, m13
2026 pmaddubsw m13, subpel3 ; b3
2027 paddw m14, m12
2028 paddw m15, m13
2029 pmulhrsw m14, m7
2030 pmulhrsw m15, m7
2031 packuswb m14, m15
2032 movq [dstq+dsq*0], m14
2033 movhps [dstq+dsq*1], m14
2034 lea dstq, [dstq+dsq*2]
2035 sub hd, 2
2036 jg .v_w8_loop
2037 add r4, 8
2038 add r7, 8
2039 movzx hd, r6b
2040 mov srcq, r4
2041 mov dstq, r7
2042 sub r6d, 1<<8
2043 jg .v_w8_loop0
2044 RET
2045 %endif ;ARCH_X86_64
2046 %undef subpel0
2047 %undef subpel1
2048 %undef subpel2
2049 %undef subpel3
2050 .hv:
2051 %assign stack_offset org_stack_offset
2052 cmp wd, 4
2053 jg .hv_w8
2054 %if ARCH_X86_32
2055 and mxd, 0x7f
2056 %else
2057 movzx mxd, mxb
2058 %endif
2059 dec srcq
2060 movd m1, [base_reg+mxq*8+subpel_filters-put_ssse3+2]
2061 %if ARCH_X86_32
2062 movzx mxd, ssb
2063 shr ssd, 16
2064 cmp hd, 6
2065 cmovs ssd, mxd
2066 movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3]
2067 mov ssq, ssmp
2068 lea r6, [ssq*3]
2069 sub srcq, r6
2070 %define base_reg r6
2071 mov r6, r1; use as new base
2072 %assign regs_used 2
2073 ALLOC_STACK -mmsize*14
2074 %assign regs_used 7
2075 mov dsq, [rstk+stack_offset+gprsize*2]
2076 %define subpelv0 [rsp+mmsize*0]
2077 %define subpelv1 [rsp+mmsize*1]
2078 %define subpelv2 [rsp+mmsize*2]
2079 %define subpelv3 [rsp+mmsize*3]
2080 punpcklbw m0, m0
2081 psraw m0, 8 ; sign-extend
2082 pshufd m6, m0, q0000
2083 mova subpelv0, m6
2084 pshufd m6, m0, q1111
2085 mova subpelv1, m6
2086 pshufd m6, m0, q2222
2087 mova subpelv2, m6
2088 pshufd m6, m0, q3333
2089 mova subpelv3, m6
2090 %else
2091 movzx mxd, myb
2092 shr myd, 16
2093 cmp hd, 6
2094 cmovs myd, mxd
2095 movq m0, [base_reg+myq*8+subpel_filters-put_ssse3]
2096 ALLOC_STACK mmsize*14, 14
2097 lea ss3q, [ssq*3]
2098 sub srcq, ss3q
2099 %define subpelv0 m10
2100 %define subpelv1 m11
2101 %define subpelv2 m12
2102 %define subpelv3 m13
2103 punpcklbw m0, m0
2104 psraw m0, 8 ; sign-extend
2105 mova m8, [base+pw_8192]
2106 mova m9, [base+pd_512]
2107 pshufd m10, m0, q0000
2108 pshufd m11, m0, q1111
2109 pshufd m12, m0, q2222
2110 pshufd m13, m0, q3333
2111 %endif
2112 pshufd m7, m1, q0000
2113 cmp wd, 4
2114 je .hv_w4
2115 .hv_w2:
2116 mova m6, [base+subpel_h_shuf4]
2117 movq m2, [srcq+ssq*0] ; 0
2118 movhps m2, [srcq+ssq*1] ; 0 _ 1
2119 %if ARCH_X86_32
2120 %define w8192reg [base+pw_8192]
2121 %define d512reg [base+pd_512]
2122 lea srcq, [srcq+ssq*2]
2123 movq m0, [srcq+ssq*0] ; 2
2124 movhps m0, [srcq+ssq*1] ; 2 _ 3
2125 lea srcq, [srcq+ssq*2]
2126 %else
2127 %define w8192reg m8
2128 %define d512reg m9
2129 movq m0, [srcq+ssq*2] ; 2
2130 add srcq, ss3q
2131 movhps m0, [srcq+ssq*0] ; 2 _ 3
2132 %endif
2133 pshufb m2, m6 ; 0 ~ 1 ~
2134 pshufb m0, m6 ; 2 ~ 3 ~
2135 pmaddubsw m2, m7 ; subpel_filters
2136 pmaddubsw m0, m7 ; subpel_filters
2137 phaddw m2, m0 ; 0 1 2 3
2138 pmulhrsw m2, w8192reg
2139 %if ARCH_X86_32
2140 movq m3, [srcq+ssq*0] ; 4
2141 movhps m3, [srcq+ssq*1] ; 4 _ 5
2142 lea srcq, [srcq+ssq*2]
2143 %else
2144 movq m3, [srcq+ssq*1] ; 4
2145 movhps m3, [srcq+ssq*2] ; 4 _ 5
2146 add srcq, ss3q
2147 %endif
2148 movq m0, [srcq+ssq*0] ; 6
2149 pshufb m3, m6 ; 4 ~ 5 ~
2150 pshufb m0, m6 ; 6 ~
2151 pmaddubsw m3, m7 ; subpel_filters
2152 pmaddubsw m0, m7 ; subpel_filters
2153 phaddw m3, m0 ; 4 5 6 _
2154 pmulhrsw m3, w8192reg
2155 palignr m4, m3, m2, 4; V 1 2 3 4
2156 punpcklwd m1, m2, m4 ; V 01 12 0 1 1 2
2157 punpckhwd m2, m4 ; V 23 34 2 3 3 4
2158 pshufd m0, m3, q2121; V 5 6 5 6
2159 punpcklwd m3, m0 ; V 45 56 4 5 5 6
2160 .hv_w2_loop:
2161 movq m4, [srcq+ssq*1] ; V 7
2162 lea srcq, [srcq+ssq*2] ; V
2163 movhps m4, [srcq+ssq*0] ; V 7 8
2164 pshufb m4, m6
2165 pmaddubsw m4, m7
2166 pmaddwd m5, m1, subpelv0; V a0 b0
2167 mova m1, m2 ; V
2168 pmaddwd m2, subpelv1 ; V a1 b1
2169 paddd m5, m2 ; V
2170 mova m2, m3 ; V
2171 pmaddwd m3, subpelv2 ; a2 b2
2172 phaddw m4, m4
2173 pmulhrsw m4, w8192reg
2174 paddd m5, m3 ; V
2175 palignr m3, m4, m0, 12
2176 mova m0, m4
2177 punpcklwd m3, m0 ; V 67 78
2178 pmaddwd m4, m3, subpelv3 ; V a3 b3
2179 paddd m5, d512reg
2180 paddd m5, m4
2181 psrad m5, 10
2182 packssdw m5, m5
2183 packuswb m5, m5
2184 movd r4d, m5
2185 mov [dstq+dsq*0], r4w
2186 shr r4d, 16
2187 mov [dstq+dsq*1], r4w
2188 lea dstq, [dstq+dsq*2]
2189 sub hd, 2
2190 jg .hv_w2_loop
2191 RET
2192 %undef w8192reg
2193 %undef d512reg
2194 .hv_w4:
2195 %define hv4_line_0_0 4
2196 %define hv4_line_0_1 5
2197 %define hv4_line_0_2 6
2198 %define hv4_line_0_3 7
2199 %define hv4_line_0_4 8
2200 %define hv4_line_0_5 9
2201 %define hv4_line_1_0 10
2202 %define hv4_line_1_1 11
2203 %define hv4_line_1_2 12
2204 %define hv4_line_1_3 13
2205 %macro SAVELINE_W4 3
2206 mova [rsp+mmsize*hv4_line_%3_%2], %1
2207 %endmacro
2208 %macro RESTORELINE_W4 3
2209 mova %1, [rsp+mmsize*hv4_line_%3_%2]
2210 %endmacro
2211 %if ARCH_X86_32
2212 %define w8192reg [base+pw_8192]
2213 %define d512reg [base+pd_512]
2214 %else
2215 %define w8192reg m8
2216 %define d512reg m9
2217 %endif
2218 ; lower shuffle 0 1 2 3 4
2219 mova m6, [base+subpel_h_shuf4]
2220 movq m5, [srcq+ssq*0] ; 0 _ _ _
2221 movhps m5, [srcq+ssq*1] ; 0 _ 1 _
2222 %if ARCH_X86_32
2223 lea srcq, [srcq+ssq*2]
2224 movq m4, [srcq+ssq*0] ; 2 _ _ _
2225 movhps m4, [srcq+ssq*1] ; 2 _ 3 _
2226 lea srcq, [srcq+ssq*2]
2227 %else
2228 movq m4, [srcq+ssq*2] ; 2 _ _ _
2229 movhps m4, [srcq+ss3q ] ; 2 _ 3 _
2230 lea srcq, [srcq+ssq*4]
2231 %endif
2232 pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~
2233 pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~
2234 pmaddubsw m2, m7 ;H subpel_filters
2235 pmaddubsw m0, m7 ;H subpel_filters
2236 phaddw m2, m0 ;H 0 1 2 3
2237 pmulhrsw m2, w8192reg ;H pw_8192
2238 SAVELINE_W4 m2, 2, 0
2239 ; upper shuffle 2 3 4 5 6
2240 mova m6, [base+subpel_h_shuf4+16]
2241 pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~
2242 pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~
2243 pmaddubsw m2, m7 ;H subpel_filters
2244 pmaddubsw m0, m7 ;H subpel_filters
2245 phaddw m2, m0 ;H 0 1 2 3
2246 pmulhrsw m2, w8192reg ;H pw_8192
2247 ;
2248 ; lower shuffle
2249 mova m6, [base+subpel_h_shuf4]
2250 movq m5, [srcq+ssq*0] ; 4 _ _ _
2251 movhps m5, [srcq+ssq*1] ; 4 _ 5 _
2252 %if ARCH_X86_32
2253 lea srcq, [srcq+ssq*2]
2254 movq m4, [srcq+ssq*0] ; 6 _ _ _
2255 add srcq, ssq
2256 %else
2257 movq m4, [srcq+ssq*2] ; 6 _ _ _
2258 add srcq, ss3q
2259 %endif
2260 pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~
2261 pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~
2262 pmaddubsw m3, m7 ;H subpel_filters
2263 pmaddubsw m0, m7 ;H subpel_filters
2264 phaddw m3, m0 ;H 4 5 6 7
2265 pmulhrsw m3, w8192reg ;H pw_8192
2266 SAVELINE_W4 m3, 3, 0
2267 ; upper shuffle
2268 mova m6, [base+subpel_h_shuf4+16]
2269 pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~
2270 pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~
2271 pmaddubsw m3, m7 ;H subpel_filters
2272 pmaddubsw m0, m7 ;H subpel_filters
2273 phaddw m3, m0 ;H 4 5 6 7
2274 pmulhrsw m3, w8192reg ;H pw_8192
2275 ;process high
2276 palignr m4, m3, m2, 4;V 1 2 3 4
2277 punpcklwd m1, m2, m4 ; V 01 12
2278 punpckhwd m2, m4 ; V 23 34
2279 pshufd m0, m3, q2121;V 5 6 5 6
2280 punpcklwd m3, m0 ; V 45 56
2281 SAVELINE_W4 m0, 0, 1
2282 SAVELINE_W4 m1, 1, 1
2283 SAVELINE_W4 m2, 2, 1
2284 SAVELINE_W4 m3, 3, 1
2285 ;process low
2286 RESTORELINE_W4 m2, 2, 0
2287 RESTORELINE_W4 m3, 3, 0
2288 palignr m4, m3, m2, 4;V 1 2 3 4
2289 punpcklwd m1, m2, m4 ; V 01 12
2290 punpckhwd m2, m4 ; V 23 34
2291 pshufd m0, m3, q2121;V 5 6 5 6
2292 punpcklwd m3, m0 ; V 45 56
2293 .hv_w4_loop:
2294 ;process low
2295 pmaddwd m5, m1, subpelv0 ; V a0 b0
2296 mova m1, m2
2297 pmaddwd m2, subpelv1; V a1 b1
2298 paddd m5, m2
2299 mova m2, m3
2300 pmaddwd m3, subpelv2; V a2 b2
2301 paddd m5, m3
2302 mova m6, [base+subpel_h_shuf4]
2303 movq m4, [srcq+ssq*0] ; 7
2304 movhps m4, [srcq+ssq*1] ; 7 _ 8 _
2305 pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~
2306 pmaddubsw m4, m7 ;H subpel_filters
2307 phaddw m4, m4 ;H 7 8 7 8
2308 pmulhrsw m4, w8192reg ;H pw_8192
2309 palignr m3, m4, m0, 12 ; 6 7 8 7
2310 mova m0, m4
2311 punpcklwd m3, m4 ; 67 78
2312 pmaddwd m4, m3, subpelv3; a3 b3
2313 paddd m5, d512reg ; pd_512
2314 paddd m5, m4
2315 psrad m5, 10
2316 SAVELINE_W4 m0, 0, 0
2317 SAVELINE_W4 m1, 1, 0
2318 SAVELINE_W4 m2, 2, 0
2319 SAVELINE_W4 m3, 3, 0
2320 SAVELINE_W4 m5, 5, 0
2321 ;process high
2322 RESTORELINE_W4 m0, 0, 1
2323 RESTORELINE_W4 m1, 1, 1
2324 RESTORELINE_W4 m2, 2, 1
2325 RESTORELINE_W4 m3, 3, 1
2326 pmaddwd m5, m1, subpelv0; V a0 b0
2327 mova m1, m2
2328 pmaddwd m2, subpelv1; V a1 b1
2329 paddd m5, m2
2330 mova m2, m3
2331 pmaddwd m3, subpelv2; V a2 b2
2332 paddd m5, m3
2333 mova m6, [base+subpel_h_shuf4+16]
2334 movq m4, [srcq+ssq*0] ; 7
2335 movhps m4, [srcq+ssq*1] ; 7 _ 8 _
2336 lea srcq, [srcq+ssq*2]
2337 pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~
2338 pmaddubsw m4, m7 ;H subpel_filters
2339 phaddw m4, m4 ;H 7 8 7 8
2340 pmulhrsw m4, w8192reg ;H pw_8192
2341 palignr m3, m4, m0, 12 ; 6 7 8 7
2342 mova m0, m4
2343 punpcklwd m3, m4 ; 67 78
2344 pmaddwd m4, m3, subpelv3; a3 b3
2345 paddd m5, d512reg ; pd_512
2346 paddd m5, m4
2347 psrad m4, m5, 10
2348 RESTORELINE_W4 m5, 5, 0
2349 packssdw m5, m4 ; d -> w
2350 packuswb m5, m5 ; w -> b
2351 pshuflw m5, m5, q3120
2352 movd [dstq+dsq*0], m5
2353 psrlq m5, 32
2354 movd [dstq+dsq*1], m5
2355 lea dstq, [dstq+dsq*2]
2356 sub hd, 2
2357 SAVELINE_W4 m0, 0, 1
2358 SAVELINE_W4 m1, 1, 1
2359 SAVELINE_W4 m2, 2, 1
2360 SAVELINE_W4 m3, 3, 1
2361 RESTORELINE_W4 m0, 0, 0
2362 RESTORELINE_W4 m1, 1, 0
2363 RESTORELINE_W4 m2, 2, 0
2364 RESTORELINE_W4 m3, 3, 0
2365 jg .hv_w4_loop
2366 RET
2367 %undef subpelv0
2368 %undef subpelv1
2369 %undef subpelv2
2370 %undef subpelv3
2371 .hv_w8:
2372 %assign stack_offset org_stack_offset
2373 %define hv8_line_1 0
2374 %define hv8_line_2 1
2375 %define hv8_line_3 2
2376 %define hv8_line_4 3
2377 %define hv8_line_6 4
2378 %macro SAVELINE_W8 2
2379 mova [rsp+hv8_line_%1*mmsize], %2
2380 %endmacro
2381 %macro RESTORELINE_W8 2
2382 mova %2, [rsp+hv8_line_%1*mmsize]
2383 %endmacro
2384 shr mxd, 16
2385 sub srcq, 3
2386 %if ARCH_X86_32
2387 %define base_reg r1
2388 %define subpelh0 [rsp+mmsize*5]
2389 %define subpelh1 [rsp+mmsize*6]
2390 %define subpelv0 [rsp+mmsize*7]
2391 %define subpelv1 [rsp+mmsize*8]
2392 %define subpelv2 [rsp+mmsize*9]
2393 %define subpelv3 [rsp+mmsize*10]
2394 %define accuv0 [rsp+mmsize*11]
2395 %define accuv1 [rsp+mmsize*12]
2396 movq m1, [base_reg+mxq*8+subpel_filters-put_ssse3]
2397 movzx mxd, ssb
2398 shr ssd, 16
2399 cmp hd, 6
2400 cmovs ssd, mxd
2401 movq m5, [base_reg+ssq*8+subpel_filters-put_ssse3]
2402 mov ssq, ssmp
2403 ALLOC_STACK -mmsize*13
2404 %if STACK_ALIGNMENT < 16
2405 %define dstm [rsp+mmsize*13+gprsize*1]
2406 %define dsm [rsp+mmsize*13+gprsize*2]
2407 mov r6, [rstk+stack_offset+gprsize*2]
2408 mov dsm, r6
2409 %endif
2410 pshufd m0, m1, q0000
2411 pshufd m1, m1, q1111
2412 punpcklbw m5, m5
2413 psraw m5, 8 ; sign-extend
2414 pshufd m2, m5, q0000
2415 pshufd m3, m5, q1111
2416 pshufd m4, m5, q2222
2417 pshufd m5, m5, q3333
2418 mova subpelh0, m0
2419 mova subpelh1, m1
2420 mova subpelv0, m2
2421 mova subpelv1, m3
2422 mova subpelv2, m4
2423 mova subpelv3, m5
2424 lea r6, [ssq*3]
2425 mov dstm, dstq
2426 sub srcq, r6
2427 %else
2428 ALLOC_STACK 16*5, 16
2429 %define subpelh0 m10
2430 %define subpelh1 m11
2431 %define subpelv0 m12
2432 %define subpelv1 m13
2433 %define subpelv2 m14
2434 %define subpelv3 m15
2435 %define accuv0 m8
2436 %define accuv1 m9
2437 movq m0, [base_reg+mxq*8+subpel_filters-put_ssse3]
2438 movzx mxd, myb
2439 shr myd, 16
2440 cmp hd, 6
2441 cmovs myd, mxd
2442 movq m1, [base_reg+myq*8+subpel_filters-put_ssse3]
2443 pshufd subpelh0, m0, q0000
2444 pshufd subpelh1, m0, q1111
2445 punpcklbw m1, m1
2446 psraw m1, 8 ; sign-extend
2447 pshufd subpelv0, m1, q0000
2448 pshufd subpelv1, m1, q1111
2449 pshufd subpelv2, m1, q2222
2450 pshufd subpelv3, m1, q3333
2451 lea ss3q, [ssq*3]
2452 mov r7, dstq
2453 sub srcq, ss3q
2454 %endif
2455 shl wd, 14
2456 lea r6d, [hq+wq-(1<<16)]
2457 mov r4, srcq
2458 .hv_w8_loop0:
2459 movu m4, [srcq+ssq*0] ; 0 = _ _
2460 movu m5, [srcq+ssq*1] ; 1 = _ _
2461 %if ARCH_X86_32
2462 lea srcq, [srcq+ssq*2]
2463 %endif
2464 %macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3]
2465 %if ARCH_X86_32
2466 pshufb %3, %1, [base+subpel_h_shufB]
2467 pshufb %4, %1, [base+subpel_h_shufC]
2468 pshufb %1, [base+subpel_h_shufA]
2469 %else
2470 pshufb %3, %1, %6 ; subpel_h_shufB
2471 pshufb %4, %1, %7 ; subpel_h_shufC
2472 pshufb %1, %5 ; subpel_h_shufA
2473 %endif
2474 pmaddubsw %2, %3, subpelh0 ; subpel +0 C0
2475 pmaddubsw %4, subpelh1; subpel +4 B4
2476 pmaddubsw %3, subpelh1; C4
2477 pmaddubsw %1, subpelh0; A0
2478 paddw %2, %4 ; C0+B4
2479 paddw %1, %3 ; A0+C4
2480 phaddw %1, %2
2481 %endmacro
2482 %if ARCH_X86_64
2483 mova m7, [base+subpel_h_shufA]
2484 mova m8, [base+subpel_h_shufB]
2485 mova m9, [base+subpel_h_shufC]
2486 %endif
2487 HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 0 ~ ~ ~
2488 HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 1 ~ ~ ~
2489 %if ARCH_X86_32
2490 movu m6, [srcq+ssq*0] ; 2 = _ _
2491 movu m0, [srcq+ssq*1] ; 3 = _ _
2492 lea srcq, [srcq+ssq*2]
2493 %else
2494 movu m6, [srcq+ssq*2] ; 2 = _ _
2495 add srcq, ss3q
2496 movu m0, [srcq+ssq*0] ; 3 = _ _
2497 %endif
2498 HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 2 ~ ~ ~
2499 HV_H_W8 m0, m1, m2, m3, m7, m8, m9 ; 3 ~ ~ ~
2500 mova m7, [base+pw_8192]
2501 pmulhrsw m4, m7 ; H pw_8192
2502 pmulhrsw m5, m7 ; H pw_8192
2503 pmulhrsw m6, m7 ; H pw_8192
2504 pmulhrsw m0, m7 ; H pw_8192
2505 punpcklwd m1, m4, m5 ; 0 1 ~
2506 punpcklwd m2, m5, m6 ; 1 2 ~
2507 punpcklwd m3, m6, m0 ; 2 3 ~
2508 SAVELINE_W8 1, m1
2509 SAVELINE_W8 2, m2
2510 SAVELINE_W8 3, m3
2511 mova m7, [base+subpel_h_shufA]
2512 %if ARCH_X86_32
2513 movu m4, [srcq+ssq*0] ; 4 = _ _
2514 movu m5, [srcq+ssq*1] ; 5 = _ _
2515 lea srcq, [srcq+ssq*2]
2516 %else
2517 movu m4, [srcq+ssq*1] ; 4 = _ _
2518 movu m5, [srcq+ssq*2] ; 5 = _ _
2519 add srcq, ss3q
2520 %endif
2521 movu m6, [srcq+ssq*0] ; 6 = _ _
2522 HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 4 ~ ~ ~
2523 HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 5 ~ ~ ~
2524 HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 6 ~ ~ ~
2525 mova m7, [base+pw_8192]
2526 pmulhrsw m1, m4, m7 ; H pw_8192 4 ~
2527 pmulhrsw m2, m5, m7 ; H pw_8192 5 ~
2528 pmulhrsw m3, m6, m7 ; H pw_8192 6 ~
2529 punpcklwd m4, m0, m1 ; 3 4 ~
2530 punpcklwd m5, m1, m2 ; 4 5 ~
2531 punpcklwd m6, m2, m3 ; 5 6 ~
2532 SAVELINE_W8 6, m3
2533 RESTORELINE_W8 1, m1
2534 RESTORELINE_W8 2, m2
2535 RESTORELINE_W8 3, m3
2536 .hv_w8_loop:
2537 ; m8 accu for V a
2538 ; m9 accu for V b
2539 SAVELINE_W8 1, m3
2540 SAVELINE_W8 2, m4
2541 SAVELINE_W8 3, m5
2542 SAVELINE_W8 4, m6
2543 %if ARCH_X86_32
2544 pmaddwd m0, m1, subpelv0 ; a0
2545 pmaddwd m7, m2, subpelv0 ; b0
2546 pmaddwd m3, subpelv1 ; a1
2547 pmaddwd m4, subpelv1 ; b1
2548 paddd m0, m3
2549 paddd m7, m4
2550 pmaddwd m5, subpelv2 ; a2
2551 pmaddwd m6, subpelv2 ; b2
2552 paddd m0, m5
2553 paddd m7, m6
2554 mova m5, [base+pd_512]
2555 paddd m0, m5 ; pd_512
2556 paddd m7, m5 ; pd_512
2557 mova accuv0, m0
2558 mova accuv1, m7
2559 %else
2560 pmaddwd m8, m1, subpelv0 ; a0
2561 pmaddwd m9, m2, subpelv0 ; b0
2562 pmaddwd m3, subpelv1 ; a1
2563 pmaddwd m4, subpelv1 ; b1
2564 paddd m8, m3
2565 paddd m9, m4
2566 pmaddwd m5, subpelv2 ; a2
2567 pmaddwd m6, subpelv2 ; b2
2568 paddd m8, m5
2569 paddd m9, m6
2570 mova m7, [base+pd_512]
2571 paddd m8, m7 ; pd_512
2572 paddd m9, m7 ; pd_512
2573 mova m7, [base+subpel_h_shufB]
2574 mova m6, [base+subpel_h_shufC]
2575 mova m5, [base+subpel_h_shufA]
2576 %endif
2577 movu m0, [srcq+ssq*1] ; 7
2578 movu m4, [srcq+ssq*2] ; 8
2579 lea srcq, [srcq+ssq*2]
2580 HV_H_W8 m0, m1, m2, m3, m5, m7, m6
2581 HV_H_W8 m4, m1, m2, m3, m5, m7, m6
2582 mova m5, [base+pw_8192]
2583 pmulhrsw m0, m5 ; H pw_8192
2584 pmulhrsw m4, m5 ; H pw_8192
2585 RESTORELINE_W8 6, m6
2586 punpcklwd m5, m6, m0 ; 6 7 ~
2587 punpcklwd m6, m0, m4 ; 7 8 ~
2588 pmaddwd m1, m5, subpelv3 ; a3
2589 paddd m2, m1, accuv0
2590 pmaddwd m1, m6, subpelv3 ; b3
2591 paddd m1, m1, accuv1 ; H + V
2592 psrad m2, 10
2593 psrad m1, 10
2594 packssdw m2, m1 ; d -> w
2595 packuswb m2, m1 ; w -> b
2596 movd [dstq+dsq*0], m2
2597 psrlq m2, 32
2598 %if ARCH_X86_32
2599 add dstq, dsm
2600 movd [dstq+dsq*0], m2
2601 add dstq, dsm
2602 %else
2603 movd [dstq+dsq*1], m2
2604 lea dstq, [dstq+dsq*2]
2605 %endif
2606 sub hd, 2
2607 jle .hv_w8_outer
2608 SAVELINE_W8 6, m4
2609 RESTORELINE_W8 1, m1
2610 RESTORELINE_W8 2, m2
2611 RESTORELINE_W8 3, m3
2612 RESTORELINE_W8 4, m4
2613 jmp .hv_w8_loop
2614 .hv_w8_outer:
2615 %if ARCH_X86_32
2616 mov dstq, dstm
2617 add r4, 4
2618 movzx hd, r6w
2619 add dstq, 4
2620 mov srcq, r4
2621 mov dstm, dstq
2622 %else
2623 add r4, 4
2624 add r7, 4
2625 movzx hd, r6b
2626 mov srcq, r4
2627 mov dstq, r7
2628 %endif
2629 sub r6d, 1<<16
2630 jg .hv_w8_loop0
2631 RET
2632
2633 %macro PSHUFB_SUBPEL_H_4 5 ; dst/src1, src2/mask, tmp1, tmp2, reset_mask
2634 %if cpuflag(ssse3)
2635 pshufb %1, %2
2636 %else
2637 %if %5 == 1
2638 pcmpeqd %2, %2
2639 psrlq %2, 32
2640 %endif
2641 psrldq %3, %1, 1
2642 pshufd %3, %3, q2301
2643 pand %1, %2
2644 pandn %4, %2, %3
2645 por %1, %4
2646 %endif
2647 %endmacro
2648
2649 %macro PSHUFB_SUBPEL_H_4a 6 ; dst, src1, src2/mask, tmp1, tmp2, reset_mask
2650 %ifnidn %1, %2
2651 mova %1, %2
2652 %endif
2653 PSHUFB_SUBPEL_H_4 %1, %3, %4, %5, %6
2654 %endmacro
2655
2656 %macro PSHUFB_SUBPEL_H_4b 6 ; dst, src1, src2/mask, tmp1, tmp2, reset_mask
2657 %if notcpuflag(ssse3)
2658 psrlq %1, %2, 16
2659 %elifnidn %1, %2
2660 mova %1, %2
2661 %endif
2662 PSHUFB_SUBPEL_H_4 %1, %3, %4, %5, %6
2663 %endmacro
2664
2665 %macro PALIGNR 4-5 ; dst, src1, src2, shift[, tmp]
2666 %if cpuflag(ssse3)
2667 palignr %1, %2, %3, %4
2668 %else
2669 %if %0 == 4
2670 %assign %%i regnumof%+%1 + 1
2671 %define %%tmp m %+ %%i
2672 %else
2673 %define %%tmp %5
2674 %endif
2675 psrldq %1, %3, %4
2676 pslldq %%tmp, %2, 16-%4
2677 por %1, %%tmp
2678 %endif
2679 %endmacro
2680
2681 %macro PHADDW 4 ; dst, src, pw_1/tmp, load_pw_1
2682 %if cpuflag(ssse3)
2683 phaddw %1, %2
2684 %elifnidn %1, %2
2685 %if %4 == 1
2686 mova %3, [base+pw_1]
2687 %endif
2688 pmaddwd %1, %3
2689 pmaddwd %2, %3
2690 packssdw %1, %2
2691 %else
2692 %if %4 == 1
2693 pmaddwd %1, [base+pw_1]
2694 %else
2695 pmaddwd %1, %3
2696 %endif
2697 packssdw %1, %1
2698 %endif
2699 %endmacro
2700
2701 %macro PMULHRSW_POW2 4 ; dst, src1, src2, shift
2702 %if cpuflag(ssse3)
2703 pmulhrsw %1, %2, %3
2704 %else
2705 paddw %1, %2, %3
2706 psraw %1, %4
2707 %endif
2708 %endmacro
2709
2710 %macro PMULHRSW_8192 3 ; dst, src1, src2
2711 PMULHRSW_POW2 %1, %2, %3, 2
2712 %endmacro
2713
2714 %macro PREP_8TAP_H_LOAD4 5 ; dst, src_memloc, tmp[1-2]
2715 movd %1, [%2+0]
2716 movd %3, [%2+1]
2717 movd %4, [%2+2]
2718 movd %5, [%2+3]
2719 punpckldq %1, %3
2720 punpckldq %4, %5
2721 punpcklqdq %1, %4
2722 %endmacro
2723
2724 %macro PREP_8TAP_H_LOAD 2 ; dst0, src_memloc
2725 %if cpuflag(ssse3)
2726 movu m%1, [%2]
2727 pshufb m2, m%1, m11 ; subpel_h_shufB
2728 pshufb m3, m%1, m9 ; subpel_h_shufC
2729 pshufb m%1, m10 ; subpel_h_shufA
2730 %else
2731 %if ARCH_X86_64
2732 SWAP m12, m5
2733 SWAP m13, m6
2734 SWAP m14, m7
2735 %define %%mx0 m%+%%i
2736 %define %%mx1 m%+%%j
2737 %assign %%i 0
2738 %rep 12
2739 movd %%mx0, [%2+%%i]
2740 %assign %%i %%i+1
2741 %endrep
2742 %assign %%i 0
2743 %rep 6
2744 %assign %%j %%i+1
2745 punpckldq %%mx0, %%mx1
2746 %assign %%i %%i+2
2747 %endrep
2748 %assign %%i 0
2749 %rep 3
2750 %assign %%j %%i+2
2751 punpcklqdq %%mx0, %%mx1
2752 %assign %%i %%i+4
2753 %endrep
2754 SWAP m%1, m0
2755 SWAP m2, m4
2756 SWAP m3, m8
2757 SWAP m5, m12
2758 SWAP m6, m13
2759 SWAP m7, m14
2760 %else
2761 PREP_8TAP_H_LOAD4 m0, %2+0, m1, m4, m7
2762 PREP_8TAP_H_LOAD4 m2, %2+4, m1, m4, m7
2763 PREP_8TAP_H_LOAD4 m3, %2+8, m1, m4, m7
2764 SWAP m%1, m0
2765 %endif
2766 %endif
2767 %endmacro
2768
2769 %macro PREP_8TAP_H 2 ; dst, src_memloc
2770 PREP_8TAP_H_LOAD %1, %2
2771 %if ARCH_X86_64 && notcpuflag(ssse3)
2772 SWAP m8, m1
2773 SWAP m9, m7
2774 %endif
2775 %xdefine mX m%+%1
2776 %assign %%i regnumof%+mX
2777 %define mX m%+%%i
2778 mova m4, m2
2779 PMADDUBSW m4, m5, m1, m7, 1 ; subpel +0 B0
2780 PMADDUBSW m2, m6, m1, m7, 0 ; subpel +4 B4
2781 PMADDUBSW m3, m6, m1, m7, 0 ; subpel +4 C4
2782 PMADDUBSW mX, m5, m1, m7, 0 ; subpel +0 A0
2783 %undef mX
2784 %if ARCH_X86_64 && notcpuflag(ssse3)
2785 SWAP m1, m8
2786 SWAP m7, m9
2787 %endif
2788 paddw m3, m4
2789 paddw m%1, m2
2790 PHADDW m%1, m3, m15, ARCH_X86_32
2791 %if ARCH_X86_64 || cpuflag(ssse3)
2792 PMULHRSW_8192 m%1, m%1, m7
2793 %else
2794 PMULHRSW_8192 m%1, m%1, [base+pw_2]
2795 %endif
2796 %endmacro
2797
2798 %macro PREP_8TAP_HV 4 ; dst, src_memloc, tmp[1-2]
2799 %if cpuflag(ssse3)
2800 movu %1, [%2]
2801 pshufb m2, %1, shufB
2802 pshufb m3, %1, shufC
2803 pshufb %1, shufA
2804 %else
2805 PREP_8TAP_H_LOAD4 %1, %2+0, m1, %3, %4
2806 PREP_8TAP_H_LOAD4 m2, %2+4, m1, %3, %4
2807 PREP_8TAP_H_LOAD4 m3, %2+8, m1, %3, %4
2808 %endif
2809 mova m1, m2
2810 PMADDUBSW m1, subpelh0, %3, %4, 1 ; subpel +0 C0
2811 PMADDUBSW m3, subpelh1, %3, %4, 0 ; subpel +4 B4
2812 PMADDUBSW m2, subpelh1, %3, %4, 0 ; C4
2813 PMADDUBSW %1, subpelh0, %3, %4, 0 ; A0
2814 paddw m1, m3 ; C0+B4
2815 paddw %1, m2 ; A0+C4
2816 PHADDW %1, m1, %3, 1
2817 %endmacro
2818
2819 %macro PREP_8TAP 0
2820 %if ARCH_X86_32
2821 DECLARE_REG_TMP 1, 2
2822 %elif WIN64
2823 DECLARE_REG_TMP 6, 4
2824 %else
2825 DECLARE_REG_TMP 6, 7
2826 %endif
2827
2828 FN prep_8tap, sharp, SHARP, SHARP
2829 FN prep_8tap, sharp_smooth, SHARP, SMOOTH
2830 FN prep_8tap, smooth_sharp, SMOOTH, SHARP
2831 FN prep_8tap, smooth, SMOOTH, SMOOTH
2832 FN prep_8tap, sharp_regular, SHARP, REGULAR
2833 FN prep_8tap, regular_sharp, REGULAR, SHARP
2834 FN prep_8tap, smooth_regular, SMOOTH, REGULAR
2835 FN prep_8tap, regular_smooth, REGULAR, SMOOTH
2836 FN prep_8tap, regular, REGULAR, REGULAR
2837
2838 %if ARCH_X86_32
2839 %define base_reg r2
2840 %define base base_reg-prep%+SUFFIX
2841 %else
2842 %define base_reg r7
2843 %define base 0
2844 %endif
2845 cglobal prep_8tap_8bpc, 1, 9, 0, tmp, src, stride, w, h, mx, my, stride3
2846 %assign org_stack_offset stack_offset
2847 imul mxd, mxm, 0x010101
2848 add mxd, t0d ; 8tap_h, mx, 4tap_h
2849 imul myd, mym, 0x010101
2850 add myd, t1d ; 8tap_v, my, 4tap_v
2851 mov wd, wm
2852 movifnidn srcd, srcm
2853 movifnidn hd, hm
2854 test mxd, 0xf00
2855 jnz .h
2856 test myd, 0xf00
2857 jnz .v
2858 LEA base_reg, prep_ssse3
2859 tzcnt wd, wd
2860 movzx wd, word [base_reg-prep_ssse3+prep_ssse3_table+wq*2]
2861 pxor m4, m4
2862 add wq, base_reg
2863 movifnidn strided, stridem
2864 lea r6, [strideq*3]
2865 %assign stack_offset org_stack_offset
2866 %if WIN64
2867 pop r8
2868 pop r7
2869 %endif
2870 jmp wq
2871 .h:
2872 LEA base_reg, prep%+SUFFIX
2873 test myd, 0xf00
2874 jnz .hv
2875 %if cpuflag(ssse3)
2876 WIN64_SPILL_XMM 12
2877 %else
2878 WIN64_SPILL_XMM 16
2879 %endif
2880 %if ARCH_X86_32
2881 %define strideq r6
2882 mov strideq, stridem
2883 %endif
2884 cmp wd, 4
2885 je .h_w4
2886 tzcnt wd, wd
2887 %if cpuflag(ssse3)
2888 %if ARCH_X86_64
2889 mova m10, [base+subpel_h_shufA]
2890 mova m11, [base+subpel_h_shufB]
2891 mova m9, [base+subpel_h_shufC]
2892 %else
2893 %define m10 [base+subpel_h_shufA]
2894 %define m11 [base+subpel_h_shufB]
2895 %define m9 [base+subpel_h_shufC]
2896 %endif
2897 %endif
2898 shr mxd, 16
2899 sub srcq, 3
2900 movzx wd, word [base_reg+wq*2+table_offset(prep, _8tap_h)]
2901 movq m6, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX]
2902 %if cpuflag(ssse3)
2903 mova m7, [base+pw_8192]
2904 pshufd m5, m6, q0000
2905 pshufd m6, m6, q1111
2906 %else
2907 punpcklbw m6, m6
2908 psraw m6, 8
2909 %if ARCH_X86_64
2910 mova m7, [pw_2]
2911 mova m15, [pw_1]
2912 %else
2913 %define m15 m4
2914 %endif
2915 pshufd m5, m6, q1010
2916 punpckhqdq m6, m6
2917 %endif
2918 add wq, base_reg
2919 jmp wq
2920 .h_w4:
2921 %if ARCH_X86_32
2922 and mxd, 0x7f
2923 %else
2924 movzx mxd, mxb
2925 %endif
2926 dec srcq
2927 movd m4, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+2]
2928 %if cpuflag(ssse3)
2929 mova m6, [base+pw_8192]
2930 mova m5, [base+subpel_h_shufA]
2931 pshufd m4, m4, q0000
2932 %else
2933 mova m6, [base+pw_2]
2934 %if ARCH_X86_64
2935 mova m14, [pw_1]
2936 %else
2937 %define m14 m7
2938 %endif
2939 punpcklbw m4, m4
2940 psraw m4, 8
2941 punpcklqdq m4, m4
2942 %endif
2943 %if ARCH_X86_64
2944 lea stride3q, [strideq*3]
2945 %endif
2946 .h_w4_loop:
2947 %if cpuflag(ssse3)
2948 movq m0, [srcq+strideq*0] ; 0
2949 movq m1, [srcq+strideq*1] ; 1
2950 %if ARCH_X86_32
2951 lea srcq, [srcq+strideq*2]
2952 movq m2, [srcq+strideq*0] ; 2
2953 movq m3, [srcq+strideq*1] ; 3
2954 lea srcq, [srcq+strideq*2]
2955 %else
2956 movq m2, [srcq+strideq*2] ; 2
2957 movq m3, [srcq+stride3q ] ; 3
2958 lea srcq, [srcq+strideq*4]
2959 %endif
2960 pshufb m0, m5
2961 pshufb m1, m5
2962 pshufb m2, m5
2963 pshufb m3, m5
2964 %elif ARCH_X86_64
2965 movd m0, [srcq+strideq*0+0]
2966 movd m12, [srcq+strideq*0+1]
2967 movd m1, [srcq+strideq*1+0]
2968 movd m5, [srcq+strideq*1+1]
2969 movd m2, [srcq+strideq*2+0]
2970 movd m13, [srcq+strideq*2+1]
2971 movd m3, [srcq+stride3q +0]
2972 movd m7, [srcq+stride3q +1]
2973 punpckldq m0, m12
2974 punpckldq m1, m5
2975 punpckldq m2, m13
2976 punpckldq m3, m7
2977 movd m12, [srcq+strideq*0+2]
2978 movd m8, [srcq+strideq*0+3]
2979 movd m5, [srcq+strideq*1+2]
2980 movd m9, [srcq+strideq*1+3]
2981 movd m13, [srcq+strideq*2+2]
2982 movd m10, [srcq+strideq*2+3]
2983 movd m7, [srcq+stride3q +2]
2984 movd m11, [srcq+stride3q +3]
2985 lea srcq, [srcq+strideq*4]
2986 punpckldq m12, m8
2987 punpckldq m5, m9
2988 punpckldq m13, m10
2989 punpckldq m7, m11
2990 punpcklqdq m0, m12 ; 0
2991 punpcklqdq m1, m5 ; 1
2992 punpcklqdq m2, m13 ; 2
2993 punpcklqdq m3, m7 ; 3
2994 %else
2995 movd m0, [srcq+strideq*0+0]
2996 movd m1, [srcq+strideq*0+1]
2997 movd m2, [srcq+strideq*0+2]
2998 movd m3, [srcq+strideq*0+3]
2999 punpckldq m0, m1
3000 punpckldq m2, m3
3001 punpcklqdq m0, m2 ; 0
3002 movd m1, [srcq+strideq*1+0]
3003 movd m2, [srcq+strideq*1+1]
3004 movd m3, [srcq+strideq*1+2]
3005 movd m7, [srcq+strideq*1+3]
3006 lea srcq, [srcq+strideq*2]
3007 punpckldq m1, m2
3008 punpckldq m3, m7
3009 punpcklqdq m1, m3 ; 1
3010 movd m2, [srcq+strideq*0+0]
3011 movd m3, [srcq+strideq*0+1]
3012 movd m7, [srcq+strideq*0+2]
3013 movd m5, [srcq+strideq*0+3]
3014 punpckldq m2, m3
3015 punpckldq m7, m5
3016 punpcklqdq m2, m7 ; 2
3017 movd m3, [srcq+strideq*1+0]
3018 movd m7, [srcq+strideq*1+1]
3019 punpckldq m3, m7
3020 movd m7, [srcq+strideq*1+2]
3021 movd m5, [srcq+strideq*1+3]
3022 lea srcq, [srcq+strideq*2]
3023 punpckldq m7, m5
3024 punpcklqdq m3, m7 ; 3
3025 %endif
3026 PMADDUBSW m0, m4, m5, m7, 1 ; subpel_filters + 2
3027 PMADDUBSW m1, m4, m5, m7, 0
3028 PMADDUBSW m2, m4, m5, m7, 0
3029 PMADDUBSW m3, m4, m5, m7, 0
3030 PHADDW m0, m1, m14, ARCH_X86_32
3031 PHADDW m2, m3, m14, 0
3032 PMULHRSW_8192 m0, m0, m6
3033 PMULHRSW_8192 m2, m2, m6
3034 mova [tmpq+16*0], m0
3035 mova [tmpq+16*1], m2
3036 add tmpq, 32
3037 sub hd, 4
3038 jg .h_w4_loop
3039 RET
3040 .h_w8:
3041 %if cpuflag(ssse3)
3042 PREP_8TAP_H 0, srcq+strideq*0
3043 PREP_8TAP_H 1, srcq+strideq*1
3044 mova [tmpq+16*0], m0
3045 mova [tmpq+16*1], m1
3046 lea srcq, [srcq+strideq*2]
3047 add tmpq, 32
3048 sub hd, 2
3049 %else
3050 PREP_8TAP_H 0, srcq
3051 mova [tmpq], m0
3052 add srcq, strideq
3053 add tmpq, 16
3054 dec hd
3055 %endif
3056 jg .h_w8
3057 RET
3058 .h_w16:
3059 mov r3, -16*1
3060 jmp .h_start
3061 .h_w32:
3062 mov r3, -16*2
3063 jmp .h_start
3064 .h_w64:
3065 mov r3, -16*4
3066 jmp .h_start
3067 .h_w128:
3068 mov r3, -16*8
3069 .h_start:
3070 sub srcq, r3
3071 mov r5, r3
3072 .h_loop:
3073 %if cpuflag(ssse3)
3074 PREP_8TAP_H 0, srcq+r3+8*0
3075 PREP_8TAP_H 1, srcq+r3+8*1
3076 mova [tmpq+16*0], m0
3077 mova [tmpq+16*1], m1
3078 add tmpq, 32
3079 add r3, 16
3080 %else
3081 PREP_8TAP_H 0, srcq+r3
3082 mova [tmpq], m0
3083 add tmpq, 16
3084 add r3, 8
3085 %endif
3086 jl .h_loop
3087 add srcq, strideq
3088 mov r3, r5
3089 dec hd
3090 jg .h_loop
3091 RET
3092 .v:
3093 LEA base_reg, prep%+SUFFIX
3094 %if ARCH_X86_32
3095 mov mxd, myd
3096 and mxd, 0x7f
3097 %else
3098 %assign stack_offset org_stack_offset
3099 WIN64_SPILL_XMM 16
3100 movzx mxd, myb
3101 %endif
3102 shr myd, 16
3103 cmp hd, 6
3104 cmovs myd, mxd
3105 movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX]
3106 %if cpuflag(ssse3)
3107 mova m2, [base+pw_512]
3108 mova m7, [base+pw_8192]
3109 punpcklwd m0, m0
3110 %else
3111 punpcklbw m0, m0
3112 psraw m0, 8
3113 %endif
3114 %if ARCH_X86_32
3115 %define subpel0 [rsp+mmsize*0]
3116 %define subpel1 [rsp+mmsize*1]
3117 %define subpel2 [rsp+mmsize*2]
3118 %define subpel3 [rsp+mmsize*3]
3119 %assign regs_used 6 ; use r5 (mx) as tmp for stack alignment if needed
3120 %if cpuflag(ssse3)
3121 ALLOC_STACK -mmsize*4
3122 %else
3123 ALLOC_STACK -mmsize*5
3124 %endif
3125 %assign regs_used 7
3126 mov strideq, [rstk+stack_offset+gprsize*3]
3127 pshufd m1, m0, q0000
3128 mova subpel0, m1
3129 pshufd m1, m0, q1111
3130 mova subpel1, m1
3131 lea r5, [strideq*3]
3132 pshufd m1, m0, q2222
3133 mova subpel2, m1
3134 pshufd m1, m0, q3333
3135 mova subpel3, m1
3136 sub srcq, r5
3137 %else
3138 %define subpel0 m8
3139 %define subpel1 m9
3140 %define subpel2 m10
3141 %define subpel3 m11
3142 pshufd m8, m0, q0000
3143 pshufd m9, m0, q1111
3144 lea stride3q, [strideq*3]
3145 pshufd m10, m0, q2222
3146 pshufd m11, m0, q3333
3147 sub srcq, stride3q
3148 cmp wd, 8
3149 jns .v_w8
3150 %endif
3151 .v_w4:
3152 %if notcpuflag(ssse3)
3153 pxor m6, m6
3154 %if ARCH_X86_64
3155 mova m7, [base+pw_2]
3156 %endif
3157 %endif
3158 %if ARCH_X86_32
3159 %if STACK_ALIGNMENT < mmsize
3160 %define srcm [esp+stack_size+gprsize*1]
3161 %define tmpm [esp+stack_size+gprsize*2]
3162 %endif
3163 mov tmpm, tmpq
3164 mov srcm, srcq
3165 lea r5d, [wq - 4] ; horizontal loop
3166 shl r5d, (16 - 2) ; (wq / 4) << 16
3167 mov r5w, hw
3168 .v_w4_loop0:
3169 %endif
3170 movd m1, [srcq+strideq*0]
3171 movd m0, [srcq+strideq*1]
3172 %if ARCH_X86_32
3173 lea srcq, [srcq+strideq*2]
3174 movd m2, [srcq+strideq*0]
3175 movd m4, [srcq+strideq*1]
3176 lea srcq, [srcq+strideq*2]
3177 movd m3, [srcq+strideq*0]
3178 movd m5, [srcq+strideq*1]
3179 lea srcq, [srcq+strideq*2]
3180 %else
3181 movd m2, [srcq+strideq*2]
3182 add srcq, stride3q
3183 movd m4, [srcq+strideq*0]
3184 movd m3, [srcq+strideq*1]
3185 movd m5, [srcq+strideq*2]
3186 add srcq, stride3q
3187 %endif
3188 punpckldq m1, m0 ; 0 1
3189 punpckldq m0, m2 ; 1 2
3190 punpcklbw m1, m0 ; 01 12
3191 movd m0, [srcq+strideq*0]
3192 punpckldq m2, m4 ; 2 3
3193 punpckldq m4, m3 ; 3 4
3194 punpckldq m3, m5 ; 4 5
3195 punpckldq m5, m0 ; 5 6
3196 punpcklbw m2, m4 ; 23 34
3197 punpcklbw m3, m5 ; 45 56
3198 .v_w4_loop:
3199 %if ARCH_X86_32 && notcpuflag(ssse3)
3200 mova m7, subpel0
3201 %define subpel0 m7
3202 %endif
3203 mova m5, m1
3204 PMADDUBSW m5, subpel0, m6, m4, 0 ; a0 b0
3205 %if ARCH_X86_32 && notcpuflag(ssse3)
3206 mova m7, subpel1
3207 %define subpel1 m7
3208 %endif
3209 mova m1, m2
3210 PMADDUBSW m2, subpel1, m6, m4, 0 ; a1 b1
3211 paddw m5, m2
3212 %if ARCH_X86_32 && notcpuflag(ssse3)
3213 mova m7, subpel2
3214 %define subpel2 m7
3215 %endif
3216 mova m2, m3
3217 PMADDUBSW m3, subpel2, m6, m4, 0 ; a2 b2
3218 movd m4, [srcq+strideq*1]
3219 lea srcq, [srcq+strideq*2]
3220 paddw m5, m3
3221 punpckldq m3, m0, m4 ; 6 7 _ _
3222 movd m0, [srcq+strideq*0]
3223 punpckldq m4, m0 ; 7 8 _ _
3224 punpcklbw m3, m4 ; 67 78
3225 %if notcpuflag(ssse3)
3226 %if ARCH_X86_64
3227 SWAP m12, m0
3228 %else
3229 mova [esp+mmsize*4], m0
3230 mova m7, subpel3
3231 %define subpel3 m7
3232 %endif
3233 %endif
3234 mova m4, m3
3235 PMADDUBSW m4, subpel3, m6, m0, 0 ; a3 b3
3236 paddw m5, m4
3237 %if ARCH_X86_64 || cpuflag(ssse3)
3238 %if notcpuflag(ssse3)
3239 SWAP m0, m12
3240 %endif
3241 PMULHRSW_8192 m5, m5, m7
3242 %else
3243 mova m0, [esp+mmsize*4]
3244 PMULHRSW_8192 m5, m5, [base+pw_2]
3245 %endif
3246 movq [tmpq+wq*0], m5
3247 movhps [tmpq+wq*2], m5
3248 lea tmpq, [tmpq+wq*4]
3249 sub hd, 2
3250 jg .v_w4_loop
3251 %if ARCH_X86_32
3252 mov srcq, srcm
3253 mov tmpq, tmpm
3254 movzx hd, r5w
3255 add srcq, 4
3256 add tmpq, 8
3257 mov srcm, srcq
3258 mov tmpm, tmpq
3259 sub r5d, 1<<16 ; horizontal--
3260 jg .v_w4_loop0
3261 %endif
3262 RET
3263 %if ARCH_X86_64
3264 .v_w8:
3265 lea r6d, [wq*8-64]
3266 mov r5, srcq
3267 mov r8, tmpq
3268 lea r6d, [hq+r6*4]
3269 .v_w8_loop0:
3270 movq m1, [srcq+strideq*0]
3271 movq m2, [srcq+strideq*1]
3272 movq m3, [srcq+strideq*2]
3273 add srcq, stride3q
3274 movq m4, [srcq+strideq*0]
3275 movq m5, [srcq+strideq*1]
3276 movq m6, [srcq+strideq*2]
3277 add srcq, stride3q
3278 movq m0, [srcq+strideq*0]
3279 punpcklbw m1, m2 ; 01
3280 punpcklbw m2, m3 ; 12
3281 punpcklbw m3, m4 ; 23
3282 punpcklbw m4, m5 ; 34
3283 punpcklbw m5, m6 ; 45
3284 punpcklbw m6, m0 ; 56
3285 .v_w8_loop:
3286 movq m13, [srcq+strideq*1]
3287 lea srcq, [srcq+strideq*2]
3288 %if cpuflag(ssse3)
3289 pmaddubsw m14, m1, subpel0 ; a0
3290 pmaddubsw m15, m2, subpel0 ; b0
3291 mova m1, m3
3292 mova m2, m4
3293 pmaddubsw m3, subpel1 ; a1
3294 pmaddubsw m4, subpel1 ; b1
3295 paddw m14, m3
3296 paddw m15, m4
3297 mova m3, m5
3298 mova m4, m6
3299 pmaddubsw m5, subpel2 ; a2
3300 pmaddubsw m6, subpel2 ; b2
3301 punpcklbw m12, m0, m13 ; 67
3302 movq m0, [srcq+strideq*0]
3303 punpcklbw m13, m0 ; 78
3304 paddw m14, m5
3305 mova m5, m12
3306 pmaddubsw m12, subpel3 ; a3
3307 paddw m15, m6
3308 mova m6, m13
3309 pmaddubsw m13, subpel3 ; b3
3310 paddw m14, m12
3311 paddw m15, m13
3312 pmulhrsw m14, m7
3313 pmulhrsw m15, m7
3314 %else
3315 mova m14, m1
3316 PMADDUBSW m14, subpel0, m7, m12, 1 ; a0
3317 mova m15, m2
3318 PMADDUBSW m15, subpel0, m7, m12, 0 ; b0
3319 mova m1, m3
3320 PMADDUBSW m3, subpel1, m7, m12, 0 ; a1
3321 mova m2, m4
3322 PMADDUBSW m4, subpel1, m7, m12, 0 ; b1
3323 paddw m14, m3
3324 mova m3, m5
3325 PMADDUBSW m5, subpel2, m7, m12, 0 ; a2
3326 paddw m15, m4
3327 mova m4, m6
3328 PMADDUBSW m6, subpel2, m7, m12, 0 ; b2
3329 paddw m15, m6
3330 punpcklbw m12, m0, m13 ; 67
3331 movq m0, [srcq+strideq*0]
3332 punpcklbw m13, m0 ; 78
3333 paddw m14, m5
3334 mova m5, m12
3335 PMADDUBSW m12, subpel3, m7, m6, 0 ; a3
3336 paddw m14, m12
3337 mova m6, m13
3338 PMADDUBSW m13, subpel3, m7, m12, 0 ; b3
3339 paddw m15, m13
3340 PMULHRSW_8192 m14, m14, [base+pw_2]
3341 PMULHRSW_8192 m15, m15, [base+pw_2]
3342 %endif
3343 movu [tmpq+wq*0], m14
3344 movu [tmpq+wq*2], m15
3345 lea tmpq, [tmpq+wq*4]
3346 sub hd, 2
3347 jg .v_w8_loop
3348 add r5, 8
3349 add r8, 16
3350 movzx hd, r6b
3351 mov srcq, r5
3352 mov tmpq, r8
3353 sub r6d, 1<<8
3354 jg .v_w8_loop0
3355 RET
3356 %endif ;ARCH_X86_64
3357 %undef subpel0
3358 %undef subpel1
3359 %undef subpel2
3360 %undef subpel3
3361 .hv:
3362 %assign stack_offset org_stack_offset
3363 cmp wd, 4
3364 jg .hv_w8
3365 and mxd, 0x7f
3366 movd m1, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+2]
3367 %if ARCH_X86_32
3368 mov mxd, myd
3369 shr myd, 16
3370 and mxd, 0x7f
3371 cmp hd, 6
3372 cmovs myd, mxd
3373 movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX]
3374 mov strideq, stridem
3375 %assign regs_used 6
3376 ALLOC_STACK -mmsize*14
3377 %assign regs_used 7
3378 lea r5, [strideq*3+1]
3379 sub srcq, r5
3380 %define subpelv0 [rsp+mmsize*0]
3381 %define subpelv1 [rsp+mmsize*1]
3382 %define subpelv2 [rsp+mmsize*2]
3383 %define subpelv3 [rsp+mmsize*3]
3384 punpcklbw m0, m0
3385 psraw m0, 8
3386 pshufd m6, m0, q0000
3387 mova subpelv0, m6
3388 pshufd m6, m0, q1111
3389 mova subpelv1, m6
3390 pshufd m6, m0, q2222
3391 mova subpelv2, m6
3392 pshufd m6, m0, q3333
3393 mova subpelv3, m6
3394 %else
3395 movzx mxd, myb
3396 shr myd, 16
3397 cmp hd, 6
3398 cmovs myd, mxd
3399 movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX]
3400 %if cpuflag(ssse3)
3401 ALLOC_STACK mmsize*14, 14
3402 %else
3403 ALLOC_STACK mmsize*14, 16
3404 %endif
3405 lea stride3q, [strideq*3]
3406 sub srcq, stride3q
3407 dec srcq
3408 %define subpelv0 m10
3409 %define subpelv1 m11
3410 %define subpelv2 m12
3411 %define subpelv3 m13
3412 punpcklbw m0, m0
3413 psraw m0, 8
3414 %if cpuflag(ssse3)
3415 mova m8, [base+pw_8192]
3416 %else
3417 mova m8, [base+pw_2]
3418 %endif
3419 mova m9, [base+pd_32]
3420 pshufd m10, m0, q0000
3421 pshufd m11, m0, q1111
3422 pshufd m12, m0, q2222
3423 pshufd m13, m0, q3333
3424 %endif
3425 pshufd m7, m1, q0000
3426 %if notcpuflag(ssse3)
3427 punpcklbw m7, m7
3428 psraw m7, 8
3429 %endif
3430 %define hv4_line_0_0 4
3431 %define hv4_line_0_1 5
3432 %define hv4_line_0_2 6
3433 %define hv4_line_0_3 7
3434 %define hv4_line_0_4 8
3435 %define hv4_line_0_5 9
3436 %define hv4_line_1_0 10
3437 %define hv4_line_1_1 11
3438 %define hv4_line_1_2 12
3439 %define hv4_line_1_3 13
3440 %if ARCH_X86_32
3441 %if cpuflag(ssse3)
3442 %define w8192reg [base+pw_8192]
3443 %else
3444 %define w8192reg [base+pw_2]
3445 %endif
3446 %define d32reg [base+pd_32]
3447 %else
3448 %define w8192reg m8
3449 %define d32reg m9
3450 %endif
3451 ; lower shuffle 0 1 2 3 4
3452 %if cpuflag(ssse3)
3453 mova m6, [base+subpel_h_shuf4]
3454 %else
3455 %if ARCH_X86_64
3456 mova m15, [pw_1]
3457 %else
3458 %define m15 m1
3459 %endif
3460 %endif
3461 movq m5, [srcq+strideq*0] ; 0 _ _ _
3462 movhps m5, [srcq+strideq*1] ; 0 _ 1 _
3463 %if ARCH_X86_32
3464 lea srcq, [srcq+strideq*2]
3465 movq m4, [srcq+strideq*0] ; 2 _ _ _
3466 movhps m4, [srcq+strideq*1] ; 2 _ 3 _
3467 lea srcq, [srcq+strideq*2]
3468 %else
3469 movq m4, [srcq+strideq*2] ; 2 _ _ _
3470 movhps m4, [srcq+stride3q ] ; 2 _ 3 _
3471 lea srcq, [srcq+strideq*4]
3472 %endif
3473 PSHUFB_SUBPEL_H_4a m2, m5, m6, m1, m3, 1 ;H subpel_h_shuf4 0~1~
3474 PSHUFB_SUBPEL_H_4a m0, m4, m6, m1, m3, 0 ;H subpel_h_shuf4 2~3~
3475 PMADDUBSW m2, m7, m1, m3, 1 ;H subpel_filters
3476 PMADDUBSW m0, m7, m1, m3, 0 ;H subpel_filters
3477 PHADDW m2, m0, m15, ARCH_X86_32 ;H 0 1 2 3
3478 PMULHRSW_8192 m2, m2, w8192reg
3479 SAVELINE_W4 m2, 2, 0
3480 ; upper shuffle 2 3 4 5 6
3481 %if cpuflag(ssse3)
3482 mova m6, [base+subpel_h_shuf4+16]
3483 %endif
3484 PSHUFB_SUBPEL_H_4b m2, m5, m6, m1, m3, 0 ;H subpel_h_shuf4 0~1~
3485 PSHUFB_SUBPEL_H_4b m0, m4, m6, m1, m3, 0 ;H subpel_h_shuf4 2~3~
3486 PMADDUBSW m2, m7, m1, m3, 1 ;H subpel_filters
3487 PMADDUBSW m0, m7, m1, m3, 0 ;H subpel_filters
3488 PHADDW m2, m0, m15, ARCH_X86_32 ;H 0 1 2 3
3489 PMULHRSW_8192 m2, m2, w8192reg
3490 %if notcpuflag(ssse3)
3491 %if ARCH_X86_64
3492 SWAP m14, m2
3493 %else
3494 mova [esp+mmsize*4], m2
3495 %endif
3496 %endif
3497 ; lower shuffle
3498 %if cpuflag(ssse3)
3499 mova m6, [base+subpel_h_shuf4]
3500 %endif
3501 movq m5, [srcq+strideq*0] ; 4 _ _ _
3502 movhps m5, [srcq+strideq*1] ; 4 _ 5 _
3503 %if ARCH_X86_32
3504 lea srcq, [srcq+strideq*2]
3505 movq m4, [srcq+strideq*0] ; 6 _ _ _
3506 add srcq, strideq
3507 %else
3508 movq m4, [srcq+strideq*2] ; 6 _ _ _
3509 add srcq, stride3q
3510 %endif
3511 PSHUFB_SUBPEL_H_4a m3, m5, m6, m1, m2, 0 ;H subpel_h_shuf4 4~5~
3512 PSHUFB_SUBPEL_H_4a m0, m4, m6, m1, m2, 0 ;H subpel_h_shuf4 6~6~
3513 PMADDUBSW m3, m7, m1, m2, 1 ;H subpel_filters
3514 PMADDUBSW m0, m7, m1, m2, 0 ;H subpel_filters
3515 PHADDW m3, m0, m15, ARCH_X86_32 ;H 4 5 6 7
3516 PMULHRSW_8192 m3, m3, w8192reg
3517 SAVELINE_W4 m3, 3, 0
3518 ; upper shuffle
3519 %if cpuflag(ssse3)
3520 mova m6, [base+subpel_h_shuf4+16]
3521 %endif
3522 PSHUFB_SUBPEL_H_4b m3, m5, m6, m1, m2, 0 ;H subpel_h_shuf4 4~5~
3523 PSHUFB_SUBPEL_H_4b m0, m4, m6, m1, m2, 0 ;H subpel_h_shuf4 6~6~
3524 PMADDUBSW m3, m7, m1, m2, 1 ;H subpel_filters
3525 PMADDUBSW m0, m7, m1, m2, 0 ;H subpel_filters
3526 PHADDW m3, m0, m15, ARCH_X86_32 ;H 4 5 6 7
3527 PMULHRSW_8192 m3, m3, w8192reg
3528 %if notcpuflag(ssse3)
3529 %if ARCH_X86_64
3530 SWAP m2, m14
3531 %else
3532 mova m2, [esp+mmsize*4]
3533 %endif
3534 %endif
3535 ;process high
3536 PALIGNR m4, m3, m2, 4;V 1 2 3 4
3537 punpcklwd m1, m2, m4 ; V 01 12
3538 punpckhwd m2, m4 ; V 23 34
3539 pshufd m0, m3, q2121;V 5 6 5 6
3540 punpcklwd m3, m0 ; V 45 56
3541 SAVELINE_W4 m0, 0, 1
3542 SAVELINE_W4 m1, 1, 1
3543 SAVELINE_W4 m2, 2, 1
3544 SAVELINE_W4 m3, 3, 1
3545 ;process low
3546 RESTORELINE_W4 m2, 2, 0
3547 RESTORELINE_W4 m3, 3, 0
3548 PALIGNR m4, m3, m2, 4;V 1 2 3 4
3549 punpcklwd m1, m2, m4 ; V 01 12
3550 punpckhwd m2, m4 ; V 23 34
3551 pshufd m0, m3, q2121;V 5 6 5 6
3552 punpcklwd m3, m0 ; V 45 56
3553 .hv_w4_loop:
3554 ;process low
3555 pmaddwd m5, m1, subpelv0 ; V a0 b0
3556 mova m1, m2
3557 pmaddwd m2, subpelv1; V a1 b1
3558 paddd m5, m2
3559 mova m2, m3
3560 pmaddwd m3, subpelv2; V a2 b2
3561 paddd m5, m3
3562 %if notcpuflag(ssse3)
3563 %if ARCH_X86_64
3564 SWAP m14, m5
3565 %else
3566 mova [esp+mmsize*4], m5
3567 %define m15 m3
3568 %endif
3569 %endif
3570 %if cpuflag(ssse3)
3571 mova m6, [base+subpel_h_shuf4]
3572 %endif
3573 movq m4, [srcq+strideq*0] ; 7
3574 movhps m4, [srcq+strideq*1] ; 7 _ 8 _
3575 PSHUFB_SUBPEL_H_4a m4, m4, m6, m3, m5, 0 ; H subpel_h_shuf4 7~8~
3576 PMADDUBSW m4, m7, m3, m5, 1 ; H subpel_filters
3577 PHADDW m4, m4, m15, ARCH_X86_32 ; H 7878
3578 PMULHRSW_8192 m4, m4, w8192reg
3579 PALIGNR m3, m4, m0, 12, m5 ; 6787
3580 mova m0, m4
3581 punpcklwd m3, m4 ; 67 78
3582 pmaddwd m4, m3, subpelv3; a3 b3
3583 %if notcpuflag(ssse3)
3584 %if ARCH_X86_64
3585 SWAP m5, m14
3586 %else
3587 mova m5, [esp+mmsize*4]
3588 %endif
3589 %endif
3590 paddd m5, d32reg ; pd_32
3591 paddd m5, m4
3592 psrad m5, 6
3593 SAVELINE_W4 m0, 0, 0
3594 SAVELINE_W4 m1, 1, 0
3595 SAVELINE_W4 m2, 2, 0
3596 SAVELINE_W4 m3, 3, 0
3597 SAVELINE_W4 m5, 5, 0
3598 ;process high
3599 RESTORELINE_W4 m0, 0, 1
3600 RESTORELINE_W4 m1, 1, 1
3601 RESTORELINE_W4 m2, 2, 1
3602 RESTORELINE_W4 m3, 3, 1
3603 pmaddwd m5, m1, subpelv0; V a0 b0
3604 mova m1, m2
3605 pmaddwd m2, subpelv1; V a1 b1
3606 paddd m5, m2
3607 mova m2, m3
3608 pmaddwd m3, subpelv2; V a2 b2
3609 paddd m5, m3
3610 %if notcpuflag(ssse3)
3611 %if ARCH_X86_64
3612 SWAP m14, m5
3613 %else
3614 mova [esp+0xA0], m5
3615 %endif
3616 %endif
3617 %if cpuflag(ssse3)
3618 mova m6, [base+subpel_h_shuf4+16]
3619 %endif
3620 movq m4, [srcq+strideq*0] ; 7
3621 movhps m4, [srcq+strideq*1] ; 7 _ 8 _
3622 PSHUFB_SUBPEL_H_4b m4, m4, m6, m3, m5, 0 ; H subpel_h_shuf4 7~8~
3623 PMADDUBSW m4, m7, m3, m5, 1 ; H subpel_filters
3624 PHADDW m4, m4, m15, ARCH_X86_32 ; H 7878
3625 PMULHRSW_8192 m4, m4, w8192reg
3626 PALIGNR m3, m4, m0, 12, m5 ; 6787
3627 mova m0, m4
3628 punpcklwd m3, m4 ; 67 78
3629 pmaddwd m4, m3, subpelv3; a3 b3
3630 %if notcpuflag(ssse3)
3631 %if ARCH_X86_64
3632 SWAP m5, m14
3633 %else
3634 mova m5, [esp+0xA0]
3635 %endif
3636 %endif
3637 paddd m5, d32reg ; pd_32
3638 paddd m5, m4
3639 psrad m4, m5, 6
3640 RESTORELINE_W4 m5, 5, 0
3641 packssdw m5, m4
3642 pshufd m5, m5, q3120
3643 movu [tmpq], m5
3644 lea srcq, [srcq+strideq*2]
3645 add tmpq, 16
3646 sub hd, 2
3647 SAVELINE_W4 m0, 0, 1
3648 SAVELINE_W4 m1, 1, 1
3649 SAVELINE_W4 m2, 2, 1
3650 SAVELINE_W4 m3, 3, 1
3651 RESTORELINE_W4 m0, 0, 0
3652 RESTORELINE_W4 m1, 1, 0
3653 RESTORELINE_W4 m2, 2, 0
3654 RESTORELINE_W4 m3, 3, 0
3655 jg .hv_w4_loop
3656 RET
3657 %undef subpelv0
3658 %undef subpelv1
3659 %undef subpelv2
3660 %undef subpelv3
3661 .hv_w8:
3662 %assign stack_offset org_stack_offset
3663 %define hv8_line_1 0
3664 %define hv8_line_2 1
3665 %define hv8_line_3 2
3666 %define hv8_line_4 3
3667 %define hv8_line_6 4
3668 shr mxd, 16
3669 %if ARCH_X86_32
3670 %define subpelh0 [rsp+mmsize*5]
3671 %define subpelh1 [rsp+mmsize*6]
3672 %define subpelv0 [rsp+mmsize*7]
3673 %define subpelv1 [rsp+mmsize*8]
3674 %define subpelv2 [rsp+mmsize*9]
3675 %define subpelv3 [rsp+mmsize*10]
3676 %define accuv0 [rsp+mmsize*11]
3677 %define accuv1 [rsp+mmsize*12]
3678 movq m1, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX]
3679 mov mxd, myd
3680 shr myd, 16
3681 and mxd, 0x7f
3682 cmp hd, 6
3683 cmovs myd, mxd
3684 movq m5, [base_reg+myq*8+subpel_filters-prep%+SUFFIX]
3685 mov strideq, stridem
3686 %assign regs_used 6
3687 ALLOC_STACK -mmsize*14
3688 %assign regs_used 7
3689 %if STACK_ALIGNMENT < mmsize
3690 %define tmpm [rsp+mmsize*13+gprsize*1]
3691 %define srcm [rsp+mmsize*13+gprsize*2]
3692 %define stridem [rsp+mmsize*13+gprsize*3]
3693 mov tmpm, tmpq
3694 mov stridem, strideq
3695 %endif
3696 %if cpuflag(ssse3)
3697 pshufd m0, m1, q0000
3698 pshufd m1, m1, q1111
3699 %else
3700 punpcklbw m1, m1
3701 psraw m1, 8
3702 pshufd m0, m1, q1010
3703 punpckhqdq m1, m1
3704 %endif
3705 punpcklbw m5, m5
3706 psraw m5, 8
3707 pshufd m2, m5, q0000
3708 pshufd m3, m5, q1111
3709 pshufd m4, m5, q2222
3710 pshufd m5, m5, q3333
3711 mova subpelh0, m0
3712 mova subpelh1, m1
3713 mova subpelv0, m2
3714 mova subpelv1, m3
3715 mova subpelv2, m4
3716 mova subpelv3, m5
3717 lea r5, [strideq*3+3]
3718 sub srcq, r5
3719 mov srcm, srcq
3720 %else
3721 ALLOC_STACK mmsize*5, 16
3722 %define subpelh0 m10
3723 %define subpelh1 m11
3724 %define subpelv0 m12
3725 %define subpelv1 m13
3726 %define subpelv2 m14
3727 %define subpelv3 m15
3728 %define accuv0 m8
3729 %define accuv1 m9
3730 movq m0, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX]
3731 movzx mxd, myb
3732 shr myd, 16
3733 cmp hd, 6
3734 cmovs myd, mxd
3735 movq m1, [base_reg+myq*8+subpel_filters-prep%+SUFFIX]
3736 %if cpuflag(ssse3)
3737 pshufd subpelh0, m0, q0000
3738 pshufd subpelh1, m0, q1111
3739 %else
3740 punpcklbw m0, m0
3741 psraw m0, 8
3742 pshufd subpelh0, m0, q1010
3743 pshufd subpelh1, m0, q3232
3744 mova m7, [base+pw_2]
3745 %endif
3746 punpcklbw m1, m1
3747 psraw m1, 8
3748 pshufd subpelv0, m1, q0000
3749 pshufd subpelv1, m1, q1111
3750 pshufd subpelv2, m1, q2222
3751 pshufd subpelv3, m1, q3333
3752 lea stride3q, [strideq*3]
3753 sub srcq, 3
3754 sub srcq, stride3q
3755 mov r6, srcq
3756 mov r8, tmpq
3757 %endif
3758 lea r5d, [wq-4]
3759 shl r5d, 14
3760 add r5d, hd
3761 .hv_w8_loop0:
3762 %if cpuflag(ssse3)
3763 %if ARCH_X86_64
3764 mova m7, [base+subpel_h_shufA]
3765 mova m8, [base+subpel_h_shufB]
3766 mova m9, [base+subpel_h_shufC]
3767 %define shufA m7
3768 %define shufB m8
3769 %define shufC m9
3770 %else
3771 %define shufA [base+subpel_h_shufA]
3772 %define shufB [base+subpel_h_shufB]
3773 %define shufC [base+subpel_h_shufC]
3774 %endif
3775 %endif
3776 PREP_8TAP_HV m4, srcq+strideq*0, m7, m0
3777 PREP_8TAP_HV m5, srcq+strideq*1, m7, m0
3778 %if ARCH_X86_64
3779 PREP_8TAP_HV m6, srcq+strideq*2, m7, m0
3780 add srcq, stride3q
3781 PREP_8TAP_HV m0, srcq+strideq*0, m7, m9
3782 %else
3783 lea srcq, [srcq+strideq*2]
3784 %if notcpuflag(ssse3)
3785 mova [esp], m4
3786 %endif
3787 PREP_8TAP_HV m6, srcq+strideq*0, m7, m4
3788 PREP_8TAP_HV m0, srcq+strideq*1, m7, m4
3789 lea srcq, [srcq+strideq*2]
3790 %endif
3791 %if cpuflag(ssse3)
3792 mova m7, [base+pw_8192]
3793 %else
3794 mova m7, [base+pw_2]
3795 %if ARCH_X86_32
3796 mova m4, [esp]
3797 %endif
3798 %endif
3799 PMULHRSW_8192 m4, m4, m7
3800 PMULHRSW_8192 m5, m5, m7
3801 PMULHRSW_8192 m6, m6, m7
3802 PMULHRSW_8192 m0, m0, m7
3803 punpcklwd m1, m4, m5 ; 01
3804 punpcklwd m2, m5, m6 ; 12
3805 punpcklwd m3, m6, m0 ; 23
3806 SAVELINE_W8 1, m1
3807 SAVELINE_W8 2, m2
3808 SAVELINE_W8 3, m3
3809 %if cpuflag(ssse3)
3810 mova m7, [base+subpel_h_shufA]
3811 %endif
3812 %if ARCH_X86_64
3813 PREP_8TAP_HV m4, srcq+strideq*1, m8, m9
3814 PREP_8TAP_HV m5, srcq+strideq*2, m8, m9
3815 add srcq, stride3q
3816 PREP_8TAP_HV m6, srcq+strideq*0, m8, m9
3817 %else
3818 %if notcpuflag(ssse3)
3819 mova [esp+0x30], m0
3820 %endif
3821 PREP_8TAP_HV m4, srcq+strideq*0, m7, m0
3822 PREP_8TAP_HV m5, srcq+strideq*1, m7, m0
3823 lea srcq, [srcq+strideq*2]
3824 PREP_8TAP_HV m6, srcq+strideq*0, m7, m0
3825 %endif
3826 %if cpuflag(ssse3)
3827 mova m7, [base+pw_8192]
3828 %elif ARCH_X86_32
3829 mova m0, [esp+0x30]
3830 mova m7, [base+pw_2]
3831 %endif
3832 PMULHRSW_8192 m1, m4, m7
3833 PMULHRSW_8192 m2, m5, m7
3834 PMULHRSW_8192 m3, m6, m7
3835 punpcklwd m4, m0, m1 ; 34
3836 punpcklwd m5, m1, m2 ; 45
3837 punpcklwd m6, m2, m3 ; 56
3838 SAVELINE_W8 6, m3
3839 RESTORELINE_W8 1, m1
3840 RESTORELINE_W8 2, m2
3841 RESTORELINE_W8 3, m3
3842 .hv_w8_loop:
3843 SAVELINE_W8 1, m3
3844 SAVELINE_W8 2, m4
3845 SAVELINE_W8 3, m5
3846 SAVELINE_W8 4, m6
3847 %if ARCH_X86_32
3848 pmaddwd m0, m1, subpelv0 ; a0
3849 pmaddwd m7, m2, subpelv0 ; b0
3850 pmaddwd m3, subpelv1 ; a1
3851 pmaddwd m4, subpelv1 ; b1
3852 paddd m0, m3
3853 paddd m7, m4
3854 pmaddwd m5, subpelv2 ; a2
3855 pmaddwd m6, subpelv2 ; b2
3856 paddd m0, m5
3857 paddd m7, m6
3858 mova m5, [base+pd_32]
3859 paddd m0, m5
3860 paddd m7, m5
3861 mova accuv0, m0
3862 mova accuv1, m7
3863 %else
3864 pmaddwd accuv0, m1, subpelv0 ; a0
3865 pmaddwd accuv1, m2, subpelv0 ; b0
3866 pmaddwd m3, subpelv1 ; a1
3867 pmaddwd m4, subpelv1 ; b1
3868 paddd accuv0, m3
3869 paddd accuv1, m4
3870 pmaddwd m5, subpelv2 ; a2
3871 pmaddwd m6, subpelv2 ; b2
3872 paddd accuv0, m5
3873 paddd accuv1, m6
3874 mova m7, [base+pd_32]
3875 paddd accuv0, m7
3876 paddd accuv1, m7
3877 %if cpuflag(ssse3)
3878 mova m7, [base+subpel_h_shufB]
3879 mova m6, [base+subpel_h_shufC]
3880 mova m5, [base+subpel_h_shufA]
3881 %define shufA m5
3882 %define shufB m7
3883 %define shufC m6
3884 %endif
3885 %endif
3886 PREP_8TAP_HV m0, srcq+strideq*1, m5, m6
3887 lea srcq, [srcq+strideq*2]
3888 PREP_8TAP_HV m4, srcq+strideq*0, m5, m6
3889 %if cpuflag(ssse3)
3890 mova m5, [base+pw_8192]
3891 %else
3892 mova m5, [base+pw_2]
3893 %endif
3894 PMULHRSW_8192 m0, m0, m5
3895 PMULHRSW_8192 m4, m4, m5
3896 RESTORELINE_W8 6, m6
3897 punpcklwd m5, m6, m0 ; 67
3898 punpcklwd m6, m0, m4 ; 78
3899 pmaddwd m1, m5, subpelv3 ; a3
3900 paddd m2, m1, accuv0
3901 pmaddwd m1, m6, subpelv3 ; b3
3902 paddd m1, m1, accuv1
3903 psrad m2, 6
3904 psrad m1, 6
3905 packssdw m2, m1
3906 movq [tmpq+wq*0], m2
3907 movhps [tmpq+wq*2], m2
3908 lea tmpq, [tmpq+wq*4]
3909 sub hd, 2
3910 jle .hv_w8_outer
3911 SAVELINE_W8 6, m4
3912 RESTORELINE_W8 1, m1
3913 RESTORELINE_W8 2, m2
3914 RESTORELINE_W8 3, m3
3915 RESTORELINE_W8 4, m4
3916 jmp .hv_w8_loop
3917 .hv_w8_outer:
3918 %if ARCH_X86_32
3919 mov srcq, srcm
3920 mov tmpq, tmpm
3921 movzx hd, r5w
3922 add srcq, 4
3923 add tmpq, 8
3924 mov srcm, srcq
3925 mov tmpm, tmpq
3926 %else
3927 add r6, 4
3928 add r8, 8
3929 movzx hd, r5b
3930 mov srcq, r6
3931 mov tmpq, r8
3932 %endif
3933 sub r5d, 1<<16
3934 jg .hv_w8_loop0
3935 RET
3936 %endmacro
3937
3938 %macro movifprep 2
3939 %if isprep
3940 mov %1, %2
3941 %endif
3942 %endmacro
3943
3944 %macro SAVE_REG 1
3945 %xdefine r%1_save r%1
3946 %xdefine r%1q_save r%1q
3947 %xdefine r%1d_save r%1d
3948 %if ARCH_X86_32
3949 %define r%1m_save [rstk+stack_offset+(%1+1)*4]
3950 %endif
3951 %endmacro
3952
3953 %macro LOAD_REG 1
3954 %xdefine r%1 r%1_save
3955 %xdefine r%1q r%1q_save
3956 %xdefine r%1d r%1d_save
3957 %if ARCH_X86_32
3958 %define r%1m r%1m_save
3959 %endif
3960 %undef r%1d_save
3961 %undef r%1q_save
3962 %undef r%1_save
3963 %endmacro
3964
3965 %macro REMAP_REG 2-3
3966 %xdefine r%1 r%2
3967 %xdefine r%1q r%2q
3968 %xdefine r%1d r%2d
3969 %if ARCH_X86_32
3970 %if %3 == 0
3971 %xdefine r%1m r%2m
3972 %else
3973 %define r%1m [rstk+stack_offset+(%1+1)*4]
3974 %endif
3975 %endif
3976 %endmacro
3977
3978 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0
3979 %if isprep
3980 %if ARCH_X86_64
3981 SAVE_REG 14
3982 %assign %%i 14
3983 %rep 14
3984 %assign %%j %%i-1
3985 REMAP_REG %%i, %%j
3986 %assign %%i %%i-1
3987 %endrep
3988 %else
3989 SAVE_REG 5
3990 %assign %%i 5
3991 %rep 5
3992 %assign %%j %%i-1
3993 REMAP_REG %%i, %%j, 0
3994 %assign %%i %%i-1
3995 %endrep
3996 %endif
3997 %endif
3998 %endmacro
3999
4000 %macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0
4001 %if isprep
4002 %assign %%i 1
4003 %if ARCH_X86_64
4004 %rep 13
4005 %assign %%j %%i+1
4006 REMAP_REG %%i, %%j
4007 %assign %%i %%i+1
4008 %endrep
4009 LOAD_REG 14
4010 %else
4011 %rep 4
4012 %assign %%j %%i+1
4013 REMAP_REG %%i, %%j, 1
4014 %assign %%i %%i+1
4015 %endrep
4016 LOAD_REG 5
4017 %endif
4018 %endif
4019 %endmacro
4020
4021 %macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged
4022 MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
4023 RET
4024 %if %1
4025 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
4026 %endif
4027 %endmacro
4028
4029 %if ARCH_X86_64
4030 %macro MC_8TAP_SCALED_H 12 ; dst[0-1], tmp[0-5], weights[0-3]
4031 SWAP m%2, m%5
4032 movq m%1, [srcq+ r4]
4033 movq m%2, [srcq+ r6]
4034 movhps m%1, [srcq+ r7]
4035 movhps m%2, [srcq+ r9]
4036 movq m%3, [srcq+r10]
4037 movq m%4, [srcq+r11]
4038 movhps m%3, [srcq+r13]
4039 movhps m%4, [srcq+ rX]
4040 add srcq, ssq
4041 movq m%5, [srcq+ r4]
4042 movq m%6, [srcq+ r6]
4043 movhps m%5, [srcq+ r7]
4044 movhps m%6, [srcq+ r9]
4045 movq m%7, [srcq+r10]
4046 movq m%8, [srcq+r11]
4047 movhps m%7, [srcq+r13]
4048 movhps m%8, [srcq+ rX]
4049 add srcq, ssq
4050 pmaddubsw m%1, m%9
4051 pmaddubsw m%5, m%9
4052 pmaddubsw m%2, m%10
4053 pmaddubsw m%6, m%10
4054 pmaddubsw m%3, m%11
4055 pmaddubsw m%7, m%11
4056 pmaddubsw m%4, m%12
4057 pmaddubsw m%8, m%12
4058 phaddw m%1, m%2
4059 phaddw m%5, m%6
4060 phaddw m%3, m%4
4061 phaddw m%7, m%8
4062 phaddw m%1, m%3
4063 phaddw m%5, m%7
4064 pmulhrsw m%1, m12
4065 pmulhrsw m%5, m12
4066 SWAP m%2, m%5
4067 %endmacro
4068 %else
4069 %macro MC_8TAP_SCALED_H 2-3 1 ; weights_mem_start, h_mem_start, load_fh_offsets
4070 %if %3 == 1
4071 mov r0, [esp+ 0]
4072 mov rX, [esp+ 8]
4073 mov r4, [esp+ 4]
4074 mov r5, [esp+12]
4075 %endif
4076 movq m0, [srcq+r0]
4077 movq m1, [srcq+rX]
4078 movhps m0, [srcq+r4]
4079 movhps m1, [srcq+r5]
4080 add srcq, ssq
4081 movq m4, [srcq+r0]
4082 movq m5, [srcq+rX]
4083 movhps m4, [srcq+r4]
4084 movhps m5, [srcq+r5]
4085 mov r0, [esp+16]
4086 mov rX, [esp+24]
4087 mov r4, [esp+20]
4088 mov r5, [esp+28]
4089 sub srcq, ssq
4090 movq m2, [srcq+r0]
4091 movq m3, [srcq+rX]
4092 movhps m2, [srcq+r4]
4093 movhps m3, [srcq+r5]
4094 add srcq, ssq
4095 movq m6, [srcq+r0]
4096 movq m7, [srcq+rX]
4097 movhps m6, [srcq+r4]
4098 movhps m7, [srcq+r5]
4099 add srcq, ssq
4100 pmaddubsw m0, [esp+%1+ 0]
4101 pmaddubsw m4, [esp+%1+ 0]
4102 pmaddubsw m1, [esp+%1+16]
4103 pmaddubsw m5, [esp+%1+16]
4104 pmaddubsw m2, [esp+%1+32]
4105 pmaddubsw m6, [esp+%1+32]
4106 pmaddubsw m3, [esp+%1+48]
4107 pmaddubsw m7, [esp+%1+48]
4108 phaddw m0, m1
4109 phaddw m4, m5
4110 phaddw m2, m3
4111 phaddw m6, m7
4112 phaddw m0, m2
4113 phaddw m4, m6
4114 pmulhrsw m0, m12
4115 pmulhrsw m4, m12
4116 %if %2 != 0
4117 mova [esp+%2+ 0], m0
4118 mova [esp+%2+16], m4
4119 %endif
4120 %endmacro
4121 %endif
4122
4123 %macro MC_8TAP_SCALED 1
4124 %ifidn %1, put
4125 %assign isprep 0
4126 %if ARCH_X86_64
4127 %if required_stack_alignment <= STACK_ALIGNMENT
4128 cglobal put_8tap_scaled_8bpc, 2, 15, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy
4129 %else
4130 cglobal put_8tap_scaled_8bpc, 2, 14, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy
4131 %endif
4132 %else ; ARCH_X86_32
4133 %if required_stack_alignment <= STACK_ALIGNMENT
4134 cglobal put_8tap_scaled_8bpc, 0, 7, 8, 0x200, dst, ds, src, ss, w, h, mx, my, dx, dy
4135 %else
4136 cglobal put_8tap_scaled_8bpc, 0, 7, 8, -0x200-0x20, dst, ds, src, ss, w, h, mx, my, dx, dy
4137 %endif
4138 %endif
4139 %xdefine base_reg r12
4140 %define rndshift 10
4141 %else ; prep
4142 %assign isprep 1
4143 %if ARCH_X86_64
4144 %if required_stack_alignment <= STACK_ALIGNMENT
4145 cglobal prep_8tap_scaled_8bpc, 2, 15, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy
4146 %xdefine tmp_stridem r14q
4147 %else
4148 cglobal prep_8tap_scaled_8bpc, 2, 14, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy
4149 %define tmp_stridem qword [rsp+0x138]
4150 %endif
4151 %xdefine base_reg r11
4152 %else ; ARCH_X86_32
4153 %if required_stack_alignment <= STACK_ALIGNMENT
4154 cglobal prep_8tap_scaled_8bpc, 0, 7, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy
4155 %else
4156 cglobal prep_8tap_scaled_8bpc, 0, 6, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy
4157 %endif
4158 %define tmp_stridem dword [esp+0x138]
4159 %endif
4160 %define rndshift 6
4161 %endif
4162 %if ARCH_X86_32
4163 mov [esp+0x1f0], t0d
4164 mov [esp+0x1f4], t1d
4165 %if !isprep && required_stack_alignment > STACK_ALIGNMENT
4166 mov dstd, dstm
4167 mov dsd, dsm
4168 mov srcd, srcm
4169 mov ssd, ssm
4170 mov hd, hm
4171 mov r4, mxm
4172 %define r0m [esp+0x200]
4173 %define dsm [esp+0x204]
4174 %define dsmp dsm
4175 %define r1m dsm
4176 %define r2m [esp+0x208]
4177 %define ssm [esp+0x20c]
4178 %define r3m ssm
4179 %define hm [esp+0x210]
4180 %define mxm [esp+0x214]
4181 mov r0m, dstd
4182 mov dsm, dsd
4183 mov r2m, srcd
4184 mov ssm, ssd
4185 mov hm, hd
4186 mov r0, mym
4187 mov r1, dxm
4188 mov r2, dym
4189 %define mym [esp+0x218]
4190 %define dxm [esp+0x09c]
4191 %define dym [esp+0x21c]
4192 mov mxm, r4
4193 mov mym, r0
4194 mov dxm, r1
4195 mov dym, r2
4196 tzcnt wd, wm
4197 %endif
4198 %if isprep && required_stack_alignment > STACK_ALIGNMENT
4199 %xdefine base_reg r5
4200 %else
4201 %xdefine base_reg r6
4202 %endif
4203 mov ssd, ssm
4204 %endif
4205 LEA base_reg, %1_8tap_scaled_8bpc_ssse3
4206 %xdefine base base_reg-%1_8tap_scaled_8bpc_ssse3
4207 %if ARCH_X86_64 || isprep || required_stack_alignment <= STACK_ALIGNMENT
4208 tzcnt wd, wm
4209 %endif
4210 %if ARCH_X86_32
4211 %define m8 m0
4212 %define m9 m1
4213 %define m14 m4
4214 %define m15 m3
4215 %endif
4216 movd m8, dxm
4217 movd m14, mxm
4218 pshufd m8, m8, q0000
4219 pshufd m14, m14, q0000
4220 %if isprep && UNIX64
4221 mov r5d, t0d
4222 DECLARE_REG_TMP 5, 7
4223 %endif
4224 %if ARCH_X86_64
4225 mov dyd, dym
4226 %endif
4227 %ifidn %1, put
4228 %if WIN64
4229 mov r8d, hm
4230 DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3
4231 %define hm r5m
4232 %define dxm r8m
4233 %elif ARCH_X86_64
4234 DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3
4235 %define hm r6m
4236 %endif
4237 %if ARCH_X86_64
4238 %if required_stack_alignment > STACK_ALIGNMENT
4239 %define dsm [rsp+0x138]
4240 %define rX r1
4241 %define rXd r1d
4242 %else
4243 %define dsm dsq
4244 %define rX r14
4245 %define rXd r14d
4246 %endif
4247 %else
4248 %define rX r1
4249 %endif
4250 %else ; prep
4251 %if WIN64
4252 mov r7d, hm
4253 DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3
4254 %define hm r4m
4255 %define dxm r7m
4256 %elif ARCH_X86_64
4257 DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3
4258 %define hm [rsp+0x94]
4259 %endif
4260 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
4261 %if ARCH_X86_64
4262 %define rX r14
4263 %define rXd r14d
4264 %else
4265 %define rX r3
4266 %endif
4267 %endif
4268 %if ARCH_X86_64
4269 mova m10, [base+pd_0x3ff]
4270 mova m12, [base+pw_8192]
4271 %ifidn %1, put
4272 mova m13, [base+pd_512]
4273 %else
4274 mova m13, [base+pd_32]
4275 %endif
4276 %else
4277 %define m10 [base+pd_0x3ff]
4278 %define m12 [base+pw_8192]
4279 %ifidn %1, put
4280 %define m13 [base+pd_512]
4281 %else
4282 %define m13 [base+pd_32]
4283 %endif
4284 %endif
4285 pxor m9, m9
4286 %if ARCH_X86_64
4287 lea ss3q, [ssq*3]
4288 movzx r7d, t1b
4289 shr t1d, 16
4290 cmp hd, 6
4291 cmovs t1d, r7d
4292 sub srcq, ss3q
4293 %else
4294 MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
4295 mov r1, [esp+0x1f4]
4296 lea r0, [ssq*3]
4297 movzx r2, r1b
4298 shr r1, 16
4299 cmp dword hm, 6
4300 cmovs r1, r2
4301 mov [esp+0x1f4], r1
4302 mov r1, r1m
4303 mov r2, r2m
4304 sub srcq, r0
4305 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
4306 %define ss3q r0
4307 %define myd r4
4308 %define dyd dword dym
4309 %define hd dword hm
4310 %endif
4311 cmp dyd, 1024
4312 je .dy1
4313 cmp dyd, 2048
4314 je .dy2
4315 movzx wd, word [base+%1_8tap_scaled_ssse3_table+wq*2]
4316 add wq, base_reg
4317 jmp wq
4318 %ifidn %1, put
4319 .w2:
4320 %if ARCH_X86_64
4321 mov myd, mym
4322 movzx t0d, t0b
4323 dec srcq
4324 movd m15, t0d
4325 %else
4326 movzx r4, byte [esp+0x1f0]
4327 dec srcq
4328 movd m15, r4
4329 %endif
4330 punpckldq m9, m8
4331 SWAP m8, m9
4332 paddd m14, m8 ; mx+dx*[0-1]
4333 %if ARCH_X86_64
4334 mova m11, [base+pd_0x4000]
4335 %else
4336 %define m11 [base+pd_0x4000]
4337 %endif
4338 pshufd m15, m15, q0000
4339 pand m8, m14, m10
4340 psrld m8, 6
4341 paddd m15, m8
4342 movd r4d, m15
4343 psrldq m15, 4
4344 %if ARCH_X86_64
4345 movd r6d, m15
4346 %else
4347 movd r3d, m15
4348 %endif
4349 mova m5, [base+bdct_lb_dw]
4350 mova m6, [base+subpel_s_shuf2]
4351 movd m15, [base+subpel_filters+r4*8+2]
4352 %if ARCH_X86_64
4353 movd m7, [base+subpel_filters+r6*8+2]
4354 %else
4355 movd m7, [base+subpel_filters+r3*8+2]
4356 %endif
4357 pxor m9, m9
4358 pcmpeqd m8, m9
4359 psrld m14, 10
4360 %if ARCH_X86_32
4361 mov r3, r3m
4362 pshufb m14, m5
4363 paddb m14, m6
4364 mova [rsp+0x180], m14
4365 SWAP m5, m0
4366 SWAP m6, m3
4367 %define m8 m5
4368 %define m15 m6
4369 %endif
4370 movq m0, [srcq+ssq*0]
4371 movq m2, [srcq+ssq*2]
4372 movhps m0, [srcq+ssq*1]
4373 movhps m2, [srcq+ss3q ]
4374 lea srcq, [srcq+ssq*4]
4375 %if ARCH_X86_64
4376 pshufb m14, m5
4377 paddb m14, m6
4378 %endif
4379 movq m1, [srcq+ssq*0]
4380 movq m3, [srcq+ssq*2]
4381 movhps m1, [srcq+ssq*1]
4382 movhps m3, [srcq+ss3q ]
4383 lea srcq, [srcq+ssq*4]
4384 punpckldq m15, m7
4385 punpcklqdq m15, m15
4386 %if ARCH_X86_64
4387 pand m11, m8
4388 pandn m8, m15
4389 SWAP m15, m8
4390 por m15, m11
4391 %else
4392 pand m7, m8, m11
4393 pandn m8, m15
4394 %define m8 m6
4395 %define m15 m5
4396 por m15, m7
4397 mova [rsp+0x190], m15
4398 %endif
4399 pshufb m0, m14
4400 pshufb m2, m14
4401 pshufb m1, m14
4402 pshufb m3, m14
4403 pmaddubsw m0, m15
4404 pmaddubsw m2, m15
4405 pmaddubsw m1, m15
4406 pmaddubsw m3, m15
4407 phaddw m0, m2
4408 phaddw m1, m3
4409 pmulhrsw m0, m12 ; 0 1 2 3
4410 pmulhrsw m1, m12 ; 4 5 6 7
4411 palignr m2, m1, m0, 4 ; 1 2 3 4
4412 punpcklwd m3, m0, m2 ; 01 12
4413 punpckhwd m0, m2 ; 23 34
4414 pshufd m5, m1, q0321 ; 5 6 7 _
4415 punpcklwd m2, m1, m5 ; 45 56
4416 punpckhwd m4, m1, m5 ; 67 __
4417 %if ARCH_X86_32
4418 mov myd, mym
4419 mov r0, r0m
4420 mova [rsp+0x1a0], m3
4421 mova [rsp+0x1b0], m0
4422 mova [rsp+0x1c0], m2
4423 mova [rsp+0x1d0], m4
4424 %endif
4425 .w2_loop:
4426 and myd, 0x3ff
4427 %if ARCH_X86_64
4428 mov r6d, 64 << 24
4429 mov r4d, myd
4430 shr r4d, 6
4431 lea r4d, [t1+r4]
4432 cmovnz r6q, [base+subpel_filters+r4*8]
4433 movq m11, r6q
4434 punpcklbw m11, m11
4435 psraw m11, 8
4436 pshufd m8, m11, q0000
4437 pshufd m9, m11, q1111
4438 pshufd m10, m11, q2222
4439 pshufd m11, m11, q3333
4440 pmaddwd m5, m3, m8
4441 pmaddwd m6, m0, m9
4442 pmaddwd m7, m2, m10
4443 pmaddwd m8, m4, m11
4444 paddd m5, m6
4445 paddd m7, m8
4446 %else
4447 mov mym, myd
4448 mov r1, [esp+0x1f4]
4449 xor r3, r3
4450 shr r4, 6
4451 lea r1, [r1+r4]
4452 mov r4, 64 << 24
4453 cmovnz r4, [base+subpel_filters+r1*8+0]
4454 cmovnz r3, [base+subpel_filters+r1*8+4]
4455 movd m7, r4
4456 movd m6, r3
4457 punpckldq m7, m6
4458 punpcklbw m7, m7
4459 psraw m7, 8
4460 pshufd m5, m7, q0000
4461 pshufd m6, m7, q1111
4462 pmaddwd m3, m5
4463 pmaddwd m0, m6
4464 pshufd m5, m7, q2222
4465 pshufd m7, m7, q3333
4466 pmaddwd m2, m5
4467 pmaddwd m4, m7
4468 paddd m3, m0
4469 paddd m2, m4
4470 SWAP m5, m3
4471 SWAP m7, m2
4472 %endif
4473 paddd m5, m13
4474 paddd m5, m7
4475 psrad m5, 10
4476 packssdw m5, m5
4477 packuswb m5, m5
4478 %if ARCH_X86_64
4479 pextrw r6d, m5, 0
4480 mov [dstq], r6w
4481 add dstq, dsq
4482 dec hd
4483 jz .ret
4484 add myd, dyd
4485 %else
4486 pextrw r3d, m5, 0
4487 mov [dstq], r3w
4488 add dstq, dsm
4489 dec hd
4490 jz .ret
4491 mov myd, mym
4492 add myd, dym
4493 %endif
4494 test myd, ~0x3ff
4495 %if ARCH_X86_32
4496 SWAP m3, m5
4497 SWAP m2, m7
4498 mova m3, [rsp+0x1a0]
4499 mova m0, [rsp+0x1b0]
4500 mova m2, [rsp+0x1c0]
4501 mova m4, [rsp+0x1d0]
4502 %define m14 [esp+0x180]
4503 %define m15 [esp+0x190]
4504 %endif
4505 jz .w2_loop
4506 %if ARCH_X86_32
4507 mov r3, r3m
4508 %endif
4509 movq m5, [srcq]
4510 test myd, 0x400
4511 jz .w2_skip_line
4512 add srcq, ssq
4513 shufps m3, m0, q1032 ; 01 12
4514 shufps m0, m2, q1032 ; 23 34
4515 shufps m2, m4, q1032 ; 45 56
4516 pshufb m5, m14
4517 pmaddubsw m5, m15
4518 phaddw m5, m5
4519 pmulhrsw m5, m12
4520 palignr m4, m5, m1, 12
4521 punpcklqdq m1, m4, m4 ; 6 7 6 7
4522 punpcklwd m4, m1, m5 ; 67 __
4523 %if ARCH_X86_32
4524 mova [rsp+0x1a0], m3
4525 mova [rsp+0x1b0], m0
4526 mova [rsp+0x1c0], m2
4527 mova [rsp+0x1d0], m4
4528 %endif
4529 jmp .w2_loop
4530 .w2_skip_line:
4531 movhps m5, [srcq+ssq*1]
4532 lea srcq, [srcq+ssq*2]
4533 mova m3, m0 ; 01 12
4534 mova m0, m2 ; 23 34
4535 pshufb m5, m14
4536 pmaddubsw m5, m15
4537 phaddw m5, m5
4538 pmulhrsw m5, m12 ; 6 7 6 7
4539 palignr m4, m5, m1, 8 ; 4 5 6 7
4540 pshufd m5, m4, q0321 ; 5 6 7 _
4541 mova m1, m4
4542 punpcklwd m2, m4, m5 ; 45 56
4543 punpckhwd m4, m5 ; 67 __
4544 %if ARCH_X86_32
4545 mova [rsp+0x1a0], m3
4546 mova [rsp+0x1b0], m0
4547 mova [rsp+0x1c0], m2
4548 mova [rsp+0x1d0], m4
4549 %endif
4550 jmp .w2_loop
4551 %endif
4552 INIT_XMM ssse3
4553 .w4:
4554 %if ARCH_X86_64
4555 mov myd, mym
4556 movzx t0d, t0b
4557 dec srcq
4558 movd m15, t0d
4559 %else
4560 %define m8 m0
4561 %xdefine m14 m4
4562 %define m15 m3
4563 movzx r4, byte [esp+0x1f0]
4564 dec srcq
4565 movd m15, r4
4566 %endif
4567 pmaddwd m8, [base+rescale_mul]
4568 %if ARCH_X86_64
4569 mova m11, [base+pd_0x4000]
4570 %else
4571 %define m11 [base+pd_0x4000]
4572 %endif
4573 pshufd m15, m15, q0000
4574 paddd m14, m8 ; mx+dx*[0-3]
4575 pand m0, m14, m10
4576 psrld m0, 6
4577 paddd m15, m0
4578 psrldq m7, m15, 8
4579 %if ARCH_X86_64
4580 movd r4d, m15
4581 movd r11d, m7
4582 psrldq m15, 4
4583 psrldq m7, 4
4584 movd r6d, m15
4585 movd r13d, m7
4586 movd m15, [base+subpel_filters+ r4*8+2]
4587 movd m2, [base+subpel_filters+r11*8+2]
4588 movd m3, [base+subpel_filters+ r6*8+2]
4589 movd m4, [base+subpel_filters+r13*8+2]
4590 %else
4591 movd r0, m15
4592 movd rX, m7
4593 psrldq m15, 4
4594 psrldq m7, 4
4595 movd r4, m15
4596 movd r5, m7
4597 movd m1, [base+subpel_filters+r0*8+2]
4598 movd m2, [base+subpel_filters+rX*8+2]
4599 movd m3, [base+subpel_filters+r4*8+2]
4600 movd m7, [base+subpel_filters+r5*8+2]
4601 movifprep r3, r3m
4602 SWAP m4, m7
4603 %define m15 m1
4604 %endif
4605 mova m5, [base+bdct_lb_dw]
4606 movq m6, [base+subpel_s_shuf2]
4607 psrld m14, 10
4608 punpckldq m15, m3
4609 punpckldq m2, m4
4610 punpcklqdq m15, m2
4611 punpcklqdq m6, m6
4612 pshufb m14, m5
4613 paddb m14, m6
4614 %if ARCH_X86_64
4615 pcmpeqd m0, m9
4616 pand m11, m0
4617 %else
4618 mova [esp+0x180], m14
4619 SWAP m7, m4
4620 pxor m3, m3
4621 pcmpeqd m0, m3
4622 pand m2, m11, m0
4623 %define m11 m2
4624 %endif
4625 pandn m0, m15
4626 %if ARCH_X86_64
4627 SWAP m15, m0
4628 %else
4629 %define m15 m0
4630 %endif
4631 por m15, m11
4632 %if ARCH_X86_64
4633 movu m7, [srcq+ssq*0]
4634 movu m9, [srcq+ssq*1]
4635 movu m8, [srcq+ssq*2]
4636 movu m10, [srcq+ss3q ]
4637 lea srcq, [srcq+ssq*4]
4638 movu m2, [srcq+ssq*0]
4639 movu m4, [srcq+ssq*1]
4640 movu m3, [srcq+ssq*2]
4641 movu m5, [srcq+ss3q ]
4642 lea srcq, [srcq+ssq*4]
4643 pshufb m7, m14
4644 pshufb m9, m14
4645 pshufb m8, m14
4646 pshufb m10, m14
4647 pshufb m2, m14
4648 pshufb m4, m14
4649 pshufb m3, m14
4650 pshufb m5, m14
4651 pmaddubsw m7, m15
4652 pmaddubsw m9, m15
4653 pmaddubsw m8, m15
4654 pmaddubsw m10, m15
4655 pmaddubsw m2, m15
4656 pmaddubsw m4, m15
4657 pmaddubsw m3, m15
4658 pmaddubsw m5, m15
4659 phaddw m7, m9
4660 phaddw m8, m10
4661 phaddw m9, m2, m4
4662 phaddw m3, m5
4663 pmulhrsw m7, m12 ; 0 1
4664 pmulhrsw m8, m12 ; 2 3
4665 pmulhrsw m9, m12 ; 4 5
4666 pmulhrsw m3, m12 ; 6 7
4667 shufps m4, m7, m8, q1032 ; 1 2
4668 shufps m5, m8, m9, q1032 ; 3 4
4669 shufps m6, m9, m3, q1032 ; 5 6
4670 psrldq m11, m3, 8 ; 7 _
4671 punpcklwd m0, m7, m4 ; 01
4672 punpckhwd m7, m4 ; 12
4673 punpcklwd m1, m8, m5 ; 23
4674 punpckhwd m8, m5 ; 34
4675 punpcklwd m2, m9, m6 ; 45
4676 punpckhwd m9, m6 ; 56
4677 punpcklwd m3, m11 ; 67
4678 mova [rsp+0x00], m7
4679 mova [rsp+0x10], m8
4680 mova [rsp+0x20], m9
4681 %else
4682 mova [esp+0x190], m15
4683 lea ss3q, [ssq*3]
4684 movu m2, [srcq+ssq*0]
4685 movu m3, [srcq+ssq*1]
4686 movu m7, [srcq+ssq*2]
4687 movu m6, [srcq+ss3q ]
4688 lea srcq, [srcq+ssq*4]
4689 pshufb m2, m14
4690 pshufb m3, m14
4691 pshufb m7, m14
4692 pshufb m6, m14
4693 pmaddubsw m2, m15
4694 pmaddubsw m3, m15
4695 pmaddubsw m7, m15
4696 pmaddubsw m6, m15
4697 phaddw m2, m3
4698 phaddw m7, m6
4699 movu m1, [srcq+ssq*0]
4700 movu m5, [srcq+ssq*1]
4701 movu m3, [srcq+ssq*2]
4702 movu m6, [srcq+ss3q ]
4703 lea srcq, [srcq+ssq*4]
4704 pshufb m1, m14
4705 pshufb m5, m14
4706 pshufb m3, m14
4707 pshufb m6, m14
4708 pmaddubsw m1, m15
4709 pmaddubsw m5, m15
4710 pmaddubsw m3, m15
4711 pmaddubsw m6, m15
4712 phaddw m1, m5
4713 phaddw m3, m6
4714 pmulhrsw m2, m12
4715 pmulhrsw m7, m12
4716 pmulhrsw m1, m12
4717 pmulhrsw m3, m12
4718 shufps m4, m2, m7, q1032 ; 1 2
4719 shufps m5, m7, m1, q1032 ; 3 4
4720 shufps m6, m1, m3, q1032 ; 5 6
4721 psrldq m0, m3, 8 ; 7 _
4722 mova [esp+0x1a0], m0
4723 %define m11 [esp+0x1a0]
4724 punpcklwd m0, m2, m4 ; 01
4725 punpckhwd m2, m4 ; 12
4726 punpcklwd m4, m7, m5 ; 23
4727 punpckhwd m7, m5 ; 34
4728 punpcklwd m5, m1, m6 ; 45
4729 punpckhwd m1, m6 ; 56
4730 punpcklwd m3, [esp+0x1a0] ; 67
4731 mov myd, mym
4732 mov r0, r0m
4733 mova [esp+0x1b0], m0 ; 01
4734 mova [esp+0x1c0], m4 ; 23
4735 mova [esp+0x1d0], m5 ; 45
4736 mova [esp+0x1e0], m3 ; 67
4737 mova [rsp+0x00], m2 ; 12
4738 mova [rsp+0x10], m7 ; 34
4739 mova [rsp+0x20], m1 ; 56
4740 SWAP m1, m4
4741 SWAP m2, m5
4742 %endif
4743 .w4_loop:
4744 and myd, 0x3ff
4745 %if ARCH_X86_64
4746 mov r6d, 64 << 24
4747 mov r4d, myd
4748 shr r4d, 6
4749 lea r4d, [t1+r4]
4750 cmovnz r6q, [base+subpel_filters+r4*8]
4751 movq m10, r6q
4752 punpcklbw m10, m10
4753 psraw m10, 8
4754 pshufd m7, m10, q0000
4755 pshufd m8, m10, q1111
4756 pshufd m9, m10, q2222
4757 pshufd m10, m10, q3333
4758 pmaddwd m4, m0, m7
4759 pmaddwd m5, m1, m8
4760 pmaddwd m6, m2, m9
4761 pmaddwd m7, m3, m10
4762 paddd m4, m5
4763 paddd m6, m7
4764 paddd m4, m13
4765 paddd m4, m6
4766 %else
4767 mov mym, myd
4768 mov r5, [esp+0x1f4]
4769 xor r3, r3
4770 shr r4, 6
4771 lea r5, [r5+r4]
4772 mov r4, 64 << 24
4773 cmovnz r4, [base+subpel_filters+r5*8+0]
4774 cmovnz r3, [base+subpel_filters+r5*8+4]
4775 movd m7, r4
4776 movd m6, r3
4777 punpckldq m7, m6
4778 punpcklbw m7, m7
4779 psraw m7, 8
4780 pshufd m4, m7, q0000
4781 pshufd m5, m7, q1111
4782 pshufd m6, m7, q2222
4783 pshufd m7, m7, q3333
4784 pmaddwd m0, m4
4785 pmaddwd m1, m5
4786 pmaddwd m2, m6
4787 pmaddwd m3, m7
4788 paddd m0, m1
4789 paddd m2, m3
4790 paddd m0, m13
4791 paddd m0, m2
4792 SWAP m4, m0
4793 %endif
4794 psrad m4, rndshift
4795 packssdw m4, m4
4796 %ifidn %1, put
4797 packuswb m4, m4
4798 movd [dstq], m4
4799 add dstq, dsmp
4800 %else
4801 movq [tmpq], m4
4802 add tmpq, 8
4803 %endif
4804 dec hd
4805 jz .ret
4806 %if ARCH_X86_64
4807 add myd, dyd
4808 test myd, ~0x3ff
4809 jz .w4_loop
4810 %else
4811 SWAP m0, m4
4812 mov myd, mym
4813 mov r3, r3m
4814 add myd, dym
4815 test myd, ~0x3ff
4816 jnz .w4_next_line
4817 mova m0, [esp+0x1b0]
4818 mova m1, [esp+0x1c0]
4819 mova m2, [esp+0x1d0]
4820 mova m3, [esp+0x1e0]
4821 jmp .w4_loop
4822 .w4_next_line:
4823 %define m14 [esp+0x180]
4824 %define m15 [esp+0x190]
4825 %endif
4826 movu m4, [srcq]
4827 test myd, 0x400
4828 jz .w4_skip_line
4829 %if ARCH_X86_64
4830 mova m0, [rsp+0x00]
4831 mova [rsp+0x00], m1
4832 mova m1, [rsp+0x10]
4833 mova [rsp+0x10], m2
4834 mova m2, [rsp+0x20]
4835 mova [rsp+0x20], m3
4836 %else
4837 mova m5, [esp+0x1c0]
4838 mova m0, [rsp+0x000]
4839 mova [rsp+0x00], m5
4840 mova [esp+0x1b0], m0
4841 mova m6, [esp+0x1d0]
4842 mova m1, [rsp+0x010]
4843 mova [rsp+0x10], m6
4844 mova [esp+0x1c0], m1
4845 mova m7, [esp+0x1e0]
4846 mova m2, [rsp+0x020]
4847 mova [rsp+0x20], m7
4848 mova [esp+0x1d0], m2
4849 %endif
4850 pshufb m4, m14
4851 pmaddubsw m4, m15
4852 phaddw m4, m4
4853 pmulhrsw m4, m12
4854 punpcklwd m3, m11, m4
4855 %if ARCH_X86_32
4856 mova [esp+0x1e0], m3
4857 %endif
4858 mova m11, m4
4859 add srcq, ssq
4860 jmp .w4_loop
4861 .w4_skip_line:
4862 %if ARCH_X86_32
4863 mova m0, [esp+0x1c0]
4864 mova m1, [esp+0x1d0]
4865 mova m2, [esp+0x1e0]
4866 %endif
4867 movu m5, [srcq+ssq*1]
4868 lea srcq, [srcq+ssq*2]
4869 mova m6, [rsp+0x10]
4870 mova m7, [rsp+0x20]
4871 pshufb m4, m14
4872 pshufb m5, m14
4873 pmaddubsw m4, m15
4874 pmaddubsw m5, m15
4875 phaddw m4, m5
4876 pmulhrsw m4, m12
4877 punpcklwd m5, m11, m4
4878 mova [rsp+0x00], m6
4879 mova [rsp+0x10], m7
4880 mova [rsp+0x20], m5
4881 %if ARCH_X86_64
4882 psrldq m11, m4, 8
4883 mova m0, m1
4884 mova m1, m2
4885 mova m2, m3
4886 punpcklwd m3, m4, m11
4887 %else
4888 psrldq m6, m4, 8
4889 punpcklwd m3, m4, m6
4890 mova [esp+0x1a0], m6
4891 mova [esp+0x1b0], m0
4892 mova [esp+0x1c0], m1
4893 mova [esp+0x1d0], m2
4894 mova [esp+0x1e0], m3
4895 %endif
4896 jmp .w4_loop
4897 INIT_XMM ssse3
4898 .w8:
4899 mov dword [rsp+0x90], 1
4900 movifprep tmp_stridem, 16
4901 jmp .w_start
4902 .w16:
4903 mov dword [rsp+0x90], 2
4904 movifprep tmp_stridem, 32
4905 jmp .w_start
4906 .w32:
4907 mov dword [rsp+0x90], 4
4908 movifprep tmp_stridem, 64
4909 jmp .w_start
4910 .w64:
4911 mov dword [rsp+0x90], 8
4912 movifprep tmp_stridem, 128
4913 jmp .w_start
4914 .w128:
4915 mov dword [rsp+0x90], 16
4916 movifprep tmp_stridem, 256
4917 .w_start:
4918 %ifidn %1, put
4919 movifnidn dsm, dsq
4920 %endif
4921 %if ARCH_X86_64
4922 shr t0d, 16
4923 movd m15, t0d
4924 %else
4925 %define m8 m0
4926 %xdefine m14 m4
4927 %define m15 m3
4928 %if isprep
4929 %define ssq ssm
4930 %endif
4931 mov r4, [esp+0x1f0]
4932 shr r4, 16
4933 movd m15, r4
4934 mov r0, r0m
4935 mov myd, mym
4936 %endif
4937 sub srcq, 3
4938 pslld m7, m8, 2 ; dx*4
4939 pmaddwd m8, [base+rescale_mul] ; dx*[0-3]
4940 pshufd m15, m15, q0000
4941 paddd m14, m8 ; mx+dx*[0-3]
4942 mova [rsp+0x100], m7
4943 mova [rsp+0x120], m15
4944 mov [rsp+0x098], srcq
4945 mov [rsp+0x130], r0q ; dstq / tmpq
4946 %if ARCH_X86_64 && UNIX64
4947 mov hm, hd
4948 %elif ARCH_X86_32
4949 mov r5, hm
4950 mov [esp+0x094], myd
4951 mov [esp+0x134], r5
4952 %endif
4953 jmp .hloop
4954 .hloop_prep:
4955 dec dword [rsp+0x090]
4956 jz .ret
4957 %if ARCH_X86_64
4958 add qword [rsp+0x130], 8*(isprep+1)
4959 mov hd, hm
4960 %else
4961 add dword [esp+0x130], 8*(isprep+1)
4962 mov myd, [esp+0x094]
4963 mov r5, [esp+0x134]
4964 mov r0, [esp+0x130]
4965 %endif
4966 mova m7, [rsp+0x100]
4967 mova m14, [rsp+0x110]
4968 %if ARCH_X86_64
4969 mova m10, [base+pd_0x3ff]
4970 %endif
4971 mova m15, [rsp+0x120]
4972 pxor m9, m9
4973 mov srcq, [rsp+0x098]
4974 %if ARCH_X86_64
4975 mov r0q, [rsp+0x130] ; dstq / tmpq
4976 %else
4977 mov mym, myd
4978 mov hm, r5
4979 mov r0m, r0
4980 mov r3, r3m
4981 %endif
4982 paddd m14, m7
4983 .hloop:
4984 %if ARCH_X86_64
4985 mova m11, [base+pq_0x40000000]
4986 %else
4987 %define m11 [base+pq_0x40000000]
4988 %endif
4989 psrld m2, m14, 10
4990 mova [rsp], m2
4991 pand m6, m14, m10
4992 psrld m6, 6
4993 paddd m5, m15, m6
4994 pcmpeqd m6, m9
4995 psrldq m2, m5, 8
4996 %if ARCH_X86_64
4997 movd r4d, m5
4998 movd r6d, m2
4999 psrldq m5, 4
5000 psrldq m2, 4
5001 movd r7d, m5
5002 movd r9d, m2
5003 movq m0, [base+subpel_filters+r4*8]
5004 movq m1, [base+subpel_filters+r6*8]
5005 movhps m0, [base+subpel_filters+r7*8]
5006 movhps m1, [base+subpel_filters+r9*8]
5007 %else
5008 movd r0, m5
5009 movd rX, m2
5010 psrldq m5, 4
5011 psrldq m2, 4
5012 movd r4, m5
5013 movd r5, m2
5014 movq m0, [base+subpel_filters+r0*8]
5015 movq m1, [base+subpel_filters+rX*8]
5016 movhps m0, [base+subpel_filters+r4*8]
5017 movhps m1, [base+subpel_filters+r5*8]
5018 pxor m2, m2
5019 %define m9 m2
5020 %endif
5021 paddd m14, m7 ; mx+dx*[4-7]
5022 pand m5, m14, m10
5023 psrld m5, 6
5024 paddd m15, m5
5025 pcmpeqd m5, m9
5026 mova [rsp+0x110], m14
5027 psrldq m4, m15, 8
5028 %if ARCH_X86_64
5029 movd r10d, m15
5030 movd r11d, m4
5031 psrldq m15, 4
5032 psrldq m4, 4
5033 movd r13d, m15
5034 movd rXd, m4
5035 movq m2, [base+subpel_filters+r10*8]
5036 movq m3, [base+subpel_filters+r11*8]
5037 movhps m2, [base+subpel_filters+r13*8]
5038 movhps m3, [base+subpel_filters+ rX*8]
5039 psrld m14, 10
5040 psrldq m4, m14, 8
5041 movd r10d, m14
5042 movd r11d, m4
5043 psrldq m14, 4
5044 psrldq m4, 4
5045 movd r13d, m14
5046 movd rXd, m4
5047 mov r4d, [rsp+ 0]
5048 mov r6d, [rsp+ 8]
5049 mov r7d, [rsp+ 4]
5050 mov r9d, [rsp+12]
5051 pshufd m4, m6, q1100
5052 pshufd m6, m6, q3322
5053 pshufd m14, m5, q1100
5054 pshufd m5, m5, q3322
5055 pand m7, m11, m4
5056 pand m8, m11, m6
5057 pand m15, m11, m14
5058 pand m11, m11, m5
5059 pandn m4, m0
5060 pandn m6, m1
5061 pandn m14, m2
5062 pandn m5, m3
5063 por m7, m4
5064 por m8, m6
5065 por m15, m14
5066 por m11, m5
5067 mova [rsp+0x10], m7
5068 mova [rsp+0x20], m8
5069 mova [rsp+0x30], m15
5070 mova [rsp+0x40], m11
5071 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10, 7, 8, 15, 11 ; 0-1
5072 mova [rsp+0x50], m1
5073 mova [rsp+0x60], m2
5074 MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10, 7, 8, 15, 11 ; 2-3
5075 mova [rsp+0x70], m3
5076 mova [rsp+0x80], m4
5077 MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10, 7, 8, 15, 11 ; 4-5
5078 MC_8TAP_SCALED_H 0,14, 1, 2, 3, 4, 9, 10, 7, 8, 15, 11 ; 6-7
5079 SWAP m7, m0
5080 SWAP m8, m14
5081 mova m1, [rsp+0x50]
5082 mova m2, [rsp+0x60]
5083 mova m3, [rsp+0x70]
5084 mova m9, [rsp+0x80]
5085 mov myd, mym
5086 mov dyd, dym
5087 punpcklwd m4, m5, m6 ; 45a
5088 punpckhwd m5, m6 ; 45b
5089 punpcklwd m6, m7, m8 ; 67a
5090 punpckhwd m7, m8 ; 67b
5091 punpcklwd m0, m1, m2 ; 01a
5092 punpckhwd m1, m2 ; 01b
5093 punpcklwd m2, m3, m9 ; 23a
5094 punpckhwd m3, m9 ; 23b
5095 mova [rsp+0x50], m4
5096 mova [rsp+0x60], m5
5097 mova [rsp+0x70], m6
5098 mova [rsp+0x80], m7
5099 SWAP m14, m8
5100 .vloop:
5101 and myd, 0x3ff
5102 mov r6d, 64 << 24
5103 mov r4d, myd
5104 shr r4d, 6
5105 lea r4d, [t1+r4]
5106 cmovnz r6q, [base+subpel_filters+r4*8]
5107 movq m11, r6q
5108 punpcklbw m11, m11
5109 psraw m11, 8
5110 pshufd m5, m11, q0000
5111 pshufd m7, m11, q1111
5112 pshufd m10, m11, q2222
5113 pshufd m11, m11, q3333
5114 pmaddwd m4, m5, m0
5115 pmaddwd m5, m5, m1
5116 pmaddwd m6, m7, m2
5117 pmaddwd m7, m7, m3
5118 paddd m4, m13
5119 paddd m5, m13
5120 paddd m4, m6
5121 paddd m5, m7
5122 pmaddwd m6, [rsp+0x50], m10
5123 pmaddwd m7, [rsp+0x60], m10
5124 pmaddwd m8, [rsp+0x70], m11
5125 pmaddwd m9, [rsp+0x80], m11
5126 paddd m4, m6
5127 paddd m5, m7
5128 paddd m4, m8
5129 paddd m5, m9
5130 %else
5131 movd r0, m15
5132 movd rX, m4
5133 psrldq m15, 4
5134 psrldq m4, 4
5135 movd r4, m15
5136 movd r5, m4
5137 mova m14, [esp+0x110]
5138 movq m2, [base+subpel_filters+r0*8]
5139 movq m3, [base+subpel_filters+rX*8]
5140 movhps m2, [base+subpel_filters+r4*8]
5141 movhps m3, [base+subpel_filters+r5*8]
5142 psrld m14, 10
5143 mova [esp+16], m14
5144 mov r0, [esp+ 0]
5145 mov rX, [esp+ 8]
5146 mov r4, [esp+ 4]
5147 mov r5, [esp+12]
5148 mova [esp+0x20], m0
5149 mova [esp+0x30], m1
5150 mova [esp+0x40], m2
5151 mova [esp+0x50], m3
5152 pshufd m4, m6, q1100
5153 pshufd m6, m6, q3322
5154 pshufd m7, m5, q1100
5155 pshufd m5, m5, q3322
5156 pand m0, m11, m4
5157 pand m1, m11, m6
5158 pand m2, m11, m7
5159 pand m3, m11, m5
5160 pandn m4, [esp+0x20]
5161 pandn m6, [esp+0x30]
5162 pandn m7, [esp+0x40]
5163 pandn m5, [esp+0x50]
5164 por m0, m4
5165 por m1, m6
5166 por m2, m7
5167 por m3, m5
5168 mova [esp+0x20], m0
5169 mova [esp+0x30], m1
5170 mova [esp+0x40], m2
5171 mova [esp+0x50], m3
5172 MC_8TAP_SCALED_H 0x20, 0x140, 0 ; 0-1
5173 MC_8TAP_SCALED_H 0x20, 0x160 ; 2-3
5174 MC_8TAP_SCALED_H 0x20, 0x180 ; 4-5
5175 MC_8TAP_SCALED_H 0x20, 0x1a0 ; 6-7
5176 mova m5, [esp+0x180]
5177 mova m6, [esp+0x190]
5178 mova m7, [esp+0x1a0]
5179 mova m0, [esp+0x1b0]
5180 mov myd, mym
5181 punpcklwd m4, m5, m6 ; 45a
5182 punpckhwd m5, m6 ; 45b
5183 punpcklwd m6, m7, m0 ; 67a
5184 punpckhwd m7, m0 ; 67b
5185 mova [esp+0x180], m4
5186 mova [esp+0x190], m5
5187 mova [esp+0x1a0], m6
5188 mova [esp+0x1b0], m7
5189 mova m1, [esp+0x140]
5190 mova m2, [esp+0x150]
5191 mova m3, [esp+0x160]
5192 mova m4, [esp+0x170]
5193 punpcklwd m0, m1, m2 ; 01a
5194 punpckhwd m1, m2 ; 01b
5195 punpcklwd m2, m3, m4 ; 23a
5196 punpckhwd m3, m4 ; 23b
5197 mova [esp+0x140], m0
5198 mova [esp+0x150], m1
5199 mova [esp+0x160], m2
5200 mova [esp+0x170], m3
5201 .vloop:
5202 mov r0, r0m
5203 mov r5, [esp+0x1f4]
5204 and myd, 0x3ff
5205 mov mym, myd
5206 xor r3, r3
5207 shr r4, 6
5208 lea r5, [r5+r4]
5209 mov r4, 64 << 24
5210 cmovnz r4, [base+subpel_filters+r5*8+0]
5211 cmovnz r3, [base+subpel_filters+r5*8+4]
5212 movd m7, r4
5213 movd m6, r3
5214 punpckldq m7, m6
5215 punpcklbw m7, m7
5216 psraw m7, 8
5217 pshufd m4, m7, q0000
5218 pshufd m5, m7, q1111
5219 pmaddwd m0, m4
5220 pmaddwd m1, m4
5221 pmaddwd m2, m5
5222 pmaddwd m3, m5
5223 pshufd m6, m7, q2222
5224 pshufd m7, m7, q3333
5225 paddd m0, m2
5226 paddd m1, m3
5227 pmaddwd m2, [esp+0x180], m6
5228 pmaddwd m3, [esp+0x190], m6
5229 pmaddwd m4, [esp+0x1a0], m7
5230 pmaddwd m5, [esp+0x1b0], m7
5231 paddd m0, m2
5232 paddd m1, m3
5233 paddd m0, m13
5234 paddd m1, m13
5235 paddd m4, m0
5236 paddd m5, m1
5237 %endif
5238 psrad m4, rndshift
5239 psrad m5, rndshift
5240 packssdw m4, m5
5241 %ifidn %1, put
5242 packuswb m4, m4
5243 movq [dstq], m4
5244 add dstq, dsm
5245 %else
5246 mova [tmpq], m4
5247 add tmpq, tmp_stridem
5248 %endif
5249 dec hd
5250 jz .hloop_prep
5251 %if ARCH_X86_64
5252 add myd, dyd
5253 test myd, ~0x3ff
5254 jz .vloop
5255 test myd, 0x400
5256 mov [rsp+0x140], myd
5257 mov r4d, [rsp+ 0]
5258 mov r6d, [rsp+ 8]
5259 mov r7d, [rsp+ 4]
5260 mov r9d, [rsp+12]
5261 jz .skip_line
5262 mova m14, [base+unpckw]
5263 movq m6, [srcq+r10]
5264 movq m7, [srcq+r11]
5265 movhps m6, [srcq+r13]
5266 movhps m7, [srcq+ rX]
5267 movq m4, [srcq+ r4]
5268 movq m5, [srcq+ r6]
5269 movhps m4, [srcq+ r7]
5270 movhps m5, [srcq+ r9]
5271 add srcq, ssq
5272 mov myd, [rsp+0x140]
5273 mov dyd, dym
5274 pshufd m9, m14, q1032
5275 pshufb m0, m14 ; 0a 1a
5276 pshufb m1, m14 ; 0b 1b
5277 pshufb m2, m9 ; 3a 2a
5278 pshufb m3, m9 ; 3b 2b
5279 pmaddubsw m6, [rsp+0x30]
5280 pmaddubsw m7, [rsp+0x40]
5281 pmaddubsw m4, [rsp+0x10]
5282 pmaddubsw m5, [rsp+0x20]
5283 phaddw m6, m7
5284 phaddw m4, m5
5285 phaddw m4, m6
5286 pmulhrsw m4, m12
5287 pshufb m5, [rsp+0x50], m14 ; 4a 5a
5288 pshufb m6, [rsp+0x60], m14 ; 4b 5b
5289 pshufb m7, [rsp+0x70], m9 ; 7a 6a
5290 pshufb m8, [rsp+0x80], m9 ; 7b 6b
5291 punpckhwd m0, m2 ; 12a
5292 punpckhwd m1, m3 ; 12b
5293 punpcklwd m2, m5 ; 34a
5294 punpcklwd m3, m6 ; 34b
5295 punpckhwd m5, m7 ; 56a
5296 punpckhwd m6, m8 ; 56b
5297 punpcklwd m7, m4 ; 78a
5298 punpckhqdq m4, m4
5299 punpcklwd m8, m4 ; 78b
5300 mova [rsp+0x50], m5
5301 mova [rsp+0x60], m6
5302 mova [rsp+0x70], m7
5303 mova [rsp+0x80], m8
5304 jmp .vloop
5305 .skip_line:
5306 mova m0, [rsp+0x10]
5307 mova m1, [rsp+0x20]
5308 mova m14, [rsp+0x30]
5309 mova m15, [rsp+0x40]
5310 MC_8TAP_SCALED_H 4, 8, 5, 6, 7, 9, 10, 11, 0, 1, 14, 15
5311 mov myd, [rsp+0x140]
5312 mov dyd, dym
5313 mova m0, m2 ; 01a
5314 mova m1, m3 ; 01b
5315 mova m2, [rsp+0x50] ; 23a
5316 mova m3, [rsp+0x60] ; 23b
5317 mova m5, [rsp+0x70] ; 45a
5318 mova m6, [rsp+0x80] ; 45b
5319 punpcklwd m7, m4, m8 ; 67a
5320 punpckhwd m4, m8 ; 67b
5321 mova [rsp+0x50], m5
5322 mova [rsp+0x60], m6
5323 mova [rsp+0x70], m7
5324 mova [rsp+0x80], m4
5325 %else
5326 mov r0m, r0
5327 mov myd, mym
5328 mov r3, r3m
5329 add myd, dym
5330 test myd, ~0x3ff
5331 mov mym, myd
5332 jnz .next_line
5333 mova m0, [esp+0x140]
5334 mova m1, [esp+0x150]
5335 mova m2, [esp+0x160]
5336 mova m3, [esp+0x170]
5337 jmp .vloop
5338 .next_line:
5339 test myd, 0x400
5340 mov r0, [esp+ 0]
5341 mov rX, [esp+ 8]
5342 mov r4, [esp+ 4]
5343 mov r5, [esp+12]
5344 jz .skip_line
5345 mova m6, [base+unpckw]
5346 mova m0, [esp+0x140]
5347 mova m1, [esp+0x150]
5348 mova m7, [esp+0x180]
5349 movq m4, [srcq+r0]
5350 movq m5, [srcq+rX]
5351 movhps m4, [srcq+r4]
5352 movhps m5, [srcq+r5]
5353 pshufb m0, m6 ; 0a 1a
5354 pshufb m1, m6 ; 0b 1b
5355 pshufb m7, m6 ; 4a 5a
5356 mov r0, [esp+16]
5357 mov rX, [esp+24]
5358 mov r4, [esp+20]
5359 mov r5, [esp+28]
5360 movq m3, [srcq+r0]
5361 movq m2, [srcq+rX]
5362 movhps m3, [srcq+r4]
5363 movhps m2, [srcq+r5]
5364 add srcq, ssq
5365 pmaddubsw m4, [esp+0x20]
5366 pmaddubsw m5, [esp+0x30]
5367 pmaddubsw m3, [esp+0x40]
5368 pmaddubsw m2, [esp+0x50]
5369 phaddw m4, m5
5370 phaddw m3, m2
5371 mova m5, [esp+0x190]
5372 mova m2, [esp+0x160]
5373 phaddw m4, m3
5374 mova m3, [esp+0x170]
5375 pmulhrsw m4, m12 ; 8a 8b
5376 mov myd, mym
5377 pshufb m5, m6 ; 4b 5b
5378 pshufd m6, m6, q1032
5379 pshufb m2, m6 ; 3a 2a
5380 pshufb m3, m6 ; 3b 2b
5381 punpckhwd m0, m2 ; 12a
5382 punpckhwd m1, m3 ; 12b
5383 mova [esp+0x140], m0
5384 mova [esp+0x150], m1
5385 mova m0, [esp+0x1a0]
5386 mova m1, [esp+0x1b0]
5387 punpcklwd m2, m7 ; 34a
5388 punpcklwd m3, m5 ; 34b
5389 mova [esp+0x160], m2
5390 mova [esp+0x170], m3
5391 pshufb m0, m6 ; 7a 6a
5392 pshufb m1, m6 ; 7b 6b
5393 punpckhwd m7, m0 ; 56a
5394 punpckhwd m5, m1 ; 56b
5395 punpcklwd m0, m4
5396 punpckhqdq m4, m4
5397 punpcklwd m1, m4
5398 mova [esp+0x180], m7
5399 mova [esp+0x190], m5
5400 mova [esp+0x1a0], m0
5401 mova [esp+0x1b0], m1
5402 mova m0, [esp+0x140]
5403 mova m1, [esp+0x150]
5404 jmp .vloop
5405 .skip_line:
5406 MC_8TAP_SCALED_H 0x20, 0x1c0, 0
5407 mov myd, mym
5408 mova m0, [esp+0x160]
5409 mova m1, [esp+0x170]
5410 mova m2, [esp+0x180]
5411 mova m3, [esp+0x190]
5412 mova [esp+0x140], m0
5413 mova [esp+0x150], m1
5414 mova m4, [esp+0x1a0]
5415 mova m5, [esp+0x1b0]
5416 mova [esp+0x160], m2
5417 mova [esp+0x170], m3
5418 mova m6, [esp+0x1c0]
5419 mova m7, [esp+0x1d0]
5420 mova [esp+0x180], m4
5421 mova [esp+0x190], m5
5422 punpcklwd m4, m6, m7
5423 punpckhwd m6, m7
5424 mova [esp+0x1a0], m4
5425 mova [esp+0x1b0], m6
5426 %endif
5427 jmp .vloop
5428 INIT_XMM ssse3
5429 .dy1:
5430 movzx wd, word [base+%1_8tap_scaled_ssse3_dy1_table+wq*2]
5431 add wq, base_reg
5432 jmp wq
5433 %ifidn %1, put
5434 .dy1_w2:
5435 %if ARCH_X86_64
5436 mov myd, mym
5437 movzx t0d, t0b
5438 dec srcq
5439 movd m15, t0d
5440 %else
5441 %define m8 m0
5442 %define m9 m1
5443 %define m14 m4
5444 %define m15 m3
5445 movzx r5, byte [esp+0x1f0]
5446 dec srcd
5447 movd m15, r5
5448 %endif
5449 punpckldq m9, m8
5450 SWAP m8, m9
5451 paddd m14, m8 ; mx+dx*[0-1]
5452 %if ARCH_X86_64
5453 mova m11, [base+pd_0x4000]
5454 %else
5455 %define m11 [base+pd_0x4000]
5456 %endif
5457 pshufd m15, m15, q0000
5458 pand m8, m14, m10
5459 psrld m8, 6
5460 paddd m15, m8
5461 movd r4d, m15
5462 psrldq m15, 4
5463 %if ARCH_X86_64
5464 movd r6d, m15
5465 %else
5466 movd r3d, m15
5467 %endif
5468 mova m5, [base+bdct_lb_dw]
5469 mova m6, [base+subpel_s_shuf2]
5470 movd m15, [base+subpel_filters+r4*8+2]
5471 %if ARCH_X86_64
5472 movd m7, [base+subpel_filters+r6*8+2]
5473 %else
5474 movd m7, [base+subpel_filters+r3*8+2]
5475 %endif
5476 pxor m9, m9
5477 pcmpeqd m8, m9
5478 psrld m14, 10
5479 %if ARCH_X86_32
5480 mov r3, r3m
5481 pshufb m14, m5
5482 paddb m14, m6
5483 mova [esp+0x00], m14
5484 %define m14 [esp+0x00]
5485 SWAP m5, m0
5486 SWAP m6, m3
5487 %define m8 m5
5488 %define m15 m6
5489 %endif
5490 movq m0, [srcq+ssq*0]
5491 movq m2, [srcq+ssq*2]
5492 movhps m0, [srcq+ssq*1]
5493 movhps m2, [srcq+ss3q ]
5494 lea srcq, [srcq+ssq*4]
5495 %if ARCH_X86_64
5496 shr myd, 6
5497 mov r4d, 64 << 24
5498 lea myd, [t1+myq]
5499 cmovnz r4q, [base+subpel_filters+myq*8]
5500 pshufb m14, m5
5501 paddb m14, m6
5502 movq m10, r4
5503 %else
5504 mov myd, mym
5505 mov r5, [esp+0x1f4]
5506 xor r3, r3
5507 shr myd, 6
5508 lea r5, [r5+myd]
5509 mov r4, 64 << 24
5510 cmovnz r4, [base+subpel_filters+r5*8+0]
5511 cmovnz r3, [base+subpel_filters+r5*8+4]
5512 %define m10 m4
5513 movd m10, r4
5514 movd m3, r3
5515 mov r3, r3m
5516 punpckldq m10, m3
5517 %endif
5518 movq m1, [srcq+ssq*0]
5519 movq m3, [srcq+ssq*2]
5520 movhps m1, [srcq+ssq*1]
5521 add srcq, ss3q
5522 punpcklbw m10, m10
5523 psraw m10, 8
5524 punpckldq m15, m7
5525 punpcklqdq m15, m15
5526 %if ARCH_X86_64
5527 pand m11, m8
5528 %else
5529 pand m7, m11, m8
5530 %define m11 m7
5531 %endif
5532 pandn m8, m15
5533 SWAP m15, m8
5534 por m15, m11
5535 %if ARCH_X86_64
5536 pshufd m8, m10, q0000
5537 pshufd m9, m10, q1111
5538 pshufd m11, m10, q3333
5539 pshufd m10, m10, q2222
5540 %else
5541 mova [esp+0x10], m15
5542 %define m15 [esp+0x10]
5543 mov r0, r0m
5544 pshufd m5, m4, q0000
5545 pshufd m6, m4, q1111
5546 pshufd m7, m4, q2222
5547 pshufd m4, m4, q3333
5548 %define m8 [esp+0x20]
5549 %define m9 [esp+0x30]
5550 %define m10 [esp+0x40]
5551 %define m11 [esp+0x50]
5552 mova m8, m5
5553 mova m9, m6
5554 mova m10, m7
5555 mova m11, m4
5556 %endif
5557 pshufb m0, m14
5558 pshufb m2, m14
5559 pshufb m1, m14
5560 pshufb m3, m14
5561 pmaddubsw m0, m15
5562 pmaddubsw m2, m15
5563 pmaddubsw m1, m15
5564 pmaddubsw m3, m15
5565 phaddw m0, m2
5566 phaddw m1, m3
5567 pmulhrsw m0, m12
5568 pmulhrsw m1, m12
5569 palignr m2, m1, m0, 4
5570 pshufd m4, m1, q2121
5571 punpcklwd m3, m0, m2 ; 01 12
5572 punpckhwd m0, m2 ; 23 34
5573 punpcklwd m2, m1, m4 ; 45 56
5574 .dy1_w2_loop:
5575 movq m1, [srcq+ssq*0]
5576 movhps m1, [srcq+ssq*1]
5577 lea srcq, [srcq+ssq*2]
5578 pmaddwd m5, m3, m8
5579 pmaddwd m6, m0, m9
5580 pmaddwd m7, m2, m10
5581 mova m3, m0
5582 mova m0, m2
5583 paddd m5, m13
5584 paddd m6, m7
5585 pshufb m1, m14
5586 pmaddubsw m1, m15
5587 phaddw m1, m1
5588 pmulhrsw m1, m12
5589 palignr m7, m1, m4, 12
5590 punpcklwd m2, m7, m1 ; 67 78
5591 pmaddwd m7, m2, m11
5592 mova m4, m1
5593 paddd m5, m6
5594 paddd m5, m7
5595 psrad m5, rndshift
5596 packssdw m5, m5
5597 packuswb m5, m5
5598 movd r4d, m5
5599 mov [dstq+dsq*0], r4w
5600 shr r4d, 16
5601 mov [dstq+dsq*1], r4w
5602 lea dstq, [dstq+dsq*2]
5603 sub hd, 2
5604 jg .dy1_w2_loop
5605 RET
5606 %endif
5607 INIT_XMM ssse3
5608 .dy1_w4:
5609 %if ARCH_X86_64
5610 mov myd, mym
5611 movzx t0d, t0b
5612 dec srcq
5613 movd m15, t0d
5614 %else
5615 %define m10 [base+pd_0x3ff]
5616 %define m11 [base+pd_0x4000]
5617 %define m8 m0
5618 %xdefine m14 m4
5619 %define m15 m3
5620 %if isprep
5621 %define ssq r3
5622 %endif
5623 movzx r4, byte [esp+0x1f0]
5624 dec srcq
5625 movd m15, r4
5626 %endif
5627 pmaddwd m8, [base+rescale_mul]
5628 %if ARCH_X86_64
5629 mova m11, [base+pd_0x4000]
5630 %endif
5631 pshufd m15, m15, q0000
5632 paddd m14, m8 ; mx+dx*[0-3]
5633 pand m8, m14, m10
5634 psrld m8, 6
5635 paddd m15, m8
5636 psrldq m7, m15, 8
5637 %if ARCH_X86_64
5638 movd r4d, m15
5639 movd r11d, m7
5640 psrldq m15, 4
5641 psrldq m7, 4
5642 movd r6d, m15
5643 movd r13d, m7
5644 movd m15, [base+subpel_filters+ r4*8+2]
5645 movd m2, [base+subpel_filters+r11*8+2]
5646 movd m3, [base+subpel_filters+ r6*8+2]
5647 movd m4, [base+subpel_filters+r13*8+2]
5648 shr myd, 6
5649 mov r4d, 64 << 24
5650 lea myd, [t1+myq]
5651 cmovnz r4q, [base+subpel_filters+myq*8]
5652 %else
5653 movd r1, m15
5654 movd r3, m7
5655 psrldq m15, 4
5656 psrldq m7, 4
5657 movd r4, m15
5658 movd r5, m7
5659 %define m15 m5
5660 SWAP m4, m7
5661 movd m15, [base+subpel_filters+r1*8+2]
5662 movd m2, [base+subpel_filters+r3*8+2]
5663 movd m3, [base+subpel_filters+r4*8+2]
5664 movd m4, [base+subpel_filters+r5*8+2]
5665 mov myd, mym
5666 mov rX, [esp+0x1f4]
5667 xor r5, r5
5668 shr myd, 6
5669 lea rX, [rX+myd]
5670 mov r4, 64 << 24
5671 cmovnz r4, [base+subpel_filters+rX*8+0]
5672 cmovnz r5, [base+subpel_filters+rX*8+4]
5673 mov r3, r3m
5674 %if isprep
5675 lea ss3q, [ssq*3]
5676 %endif
5677 %endif
5678 punpckldq m15, m3
5679 punpckldq m2, m4
5680 punpcklqdq m15, m2
5681 movq m6, [base+subpel_s_shuf2]
5682 %if ARCH_X86_64
5683 pcmpeqd m8, m9
5684 psrld m14, 10
5685 pshufb m14, [base+bdct_lb_dw]
5686 movu m0, [srcq+ssq*0]
5687 movu m1, [srcq+ssq*1]
5688 movu m2, [srcq+ssq*2]
5689 movu m3, [srcq+ss3q ]
5690 lea srcq, [srcq+ssq*4]
5691 punpcklqdq m6, m6
5692 movu m4, [srcq+ssq*0]
5693 movu m5, [srcq+ssq*1]
5694 movu m7, [srcq+ssq*2]
5695 add srcq, ss3q
5696 pand m11, m8
5697 pandn m8, m15
5698 SWAP m15, m8
5699 por m15, m11
5700 paddb m14, m6
5701 movq m10, r4q
5702 punpcklbw m10, m10
5703 psraw m10, 8
5704 pshufb m0, m14
5705 pshufb m1, m14
5706 pshufb m2, m14
5707 pshufb m3, m14
5708 pshufb m4, m14
5709 pshufb m5, m14
5710 pshufb m7, m14
5711 pmaddubsw m0, m15
5712 pmaddubsw m1, m15
5713 pmaddubsw m2, m15
5714 pmaddubsw m3, m15
5715 pmaddubsw m4, m15
5716 pmaddubsw m5, m15
5717 pmaddubsw m7, m15
5718 phaddw m0, m1
5719 phaddw m2, m3
5720 phaddw m4, m5
5721 phaddw m6, m7, m7
5722 pmulhrsw m0, m12 ; 0 1
5723 pmulhrsw m2, m12 ; 2 3
5724 pmulhrsw m4, m12 ; 4 5
5725 pmulhrsw m6, m12 ; 6 _
5726 shufps m1, m0, m2, q1032 ; 1 2
5727 shufps m3, m2, m4, q1032 ; 3 4
5728 shufps m5, m4, m6, q1032 ; 5 6
5729 punpcklwd m7, m0, m1 ; 01
5730 punpckhwd m0, m1 ; 12
5731 punpcklwd m8, m2, m3 ; 23
5732 punpckhwd m2, m3 ; 34
5733 punpcklwd m9, m4, m5 ; 45
5734 punpckhwd m4, m5 ; 56
5735 %else
5736 pxor m3, m3
5737 pcmpeqd m8, m3
5738 psrld m14, 10
5739 pshufb m14, [base+bdct_lb_dw]
5740 movu m1, [srcq+ssq*0]
5741 movu m2, [srcq+ssq*1]
5742 movu m3, [srcq+ssq*2]
5743 add srcq, ss3q
5744 punpcklqdq m6, m6
5745 SWAP m4, m7
5746 pand m7, m11, m8
5747 pandn m8, m15
5748 SWAP m5, m0
5749 por m15, m7
5750 paddb m14, m6
5751 movu m0, [srcq+ssq*0]
5752 movu m7, [srcq+ssq*1]
5753 movu m6, [srcq+ssq*2]
5754 pshufb m1, m14
5755 pshufb m2, m14
5756 pshufb m3, m14
5757 pshufb m0, m14
5758 pshufb m7, m14
5759 pshufb m6, m14
5760 pmaddubsw m1, m15
5761 pmaddubsw m2, m15
5762 pmaddubsw m3, m15
5763 mova [esp+0x00], m14
5764 mova [esp+0x10], m15
5765 pmaddubsw m0, m15
5766 pmaddubsw m7, m15
5767 pmaddubsw m6, m15
5768 phaddw m1, m2
5769 movu m2, [srcq+ss3q ]
5770 lea srcq, [srcq+ssq*4]
5771 mov r0, r0m
5772 phaddw m3, m0
5773 pshufb m2, m14
5774 pmaddubsw m2, m15
5775 %define m14 [esp+0x00]
5776 %define m15 [esp+0x10]
5777 phaddw m7, m6
5778 phaddw m2, m2
5779 movd m6, r4
5780 movd m0, r5
5781 punpckldq m6, m0
5782 punpcklbw m6, m6
5783 psraw m6, 8
5784 mova [esp+0x20], m6
5785 pmulhrsw m1, m12 ; 0 1
5786 pmulhrsw m3, m12 ; 2 3
5787 pmulhrsw m7, m12 ; 4 5
5788 pmulhrsw m2, m12 ; 6 _
5789 shufps m0, m1, m3, q1032 ; 1 2
5790 shufps m4, m3, m7, q1032 ; 3 4
5791 shufps m5, m7, m2, q1032 ; 5 6
5792 punpcklwd m6, m1, m0 ; 01
5793 punpckhwd m1, m0 ; 12
5794 mova [esp+0x30], m1
5795 punpcklwd m1, m3, m4 ; 23
5796 punpckhwd m3, m4 ; 34
5797 mova [esp+0x40], m3
5798 punpcklwd m3, m7, m5 ; 45
5799 punpckhwd m7, m5 ; 56
5800 mova [esp+0x50], m7
5801 mova [esp+0x60], m2
5802 mova m0, [esp+0x20]
5803 %xdefine m8 m1
5804 %xdefine m9 m3
5805 %xdefine m10 m0
5806 SWAP m7, m6
5807 SWAP m1, m4
5808 SWAP m3, m2
5809 %endif
5810 pshufd m1, m10, q0000
5811 pshufd m3, m10, q1111
5812 pshufd m5, m10, q2222
5813 pshufd m10, m10, q3333
5814 %if ARCH_X86_64
5815 mova [rsp+0x00], m8
5816 mova [rsp+0x10], m2
5817 mova [rsp+0x20], m9
5818 mova [rsp+0x30], m4
5819 %else
5820 mova [esp+0x70], m8
5821 mova [esp+0x80], m9
5822 mova [esp+0x90], m1
5823 mova [esp+0xa0], m3
5824 mova [esp+0xb0], m5
5825 mova [esp+0xc0], m10
5826 %ifidn %1, put
5827 mov dsd, dsm
5828 %endif
5829 %define m11 m6
5830 %endif
5831 .dy1_w4_loop:
5832 %if ARCH_X86_64
5833 movu m11, [srcq+ssq*0]
5834 pmaddwd m7, m1
5835 pmaddwd m8, m3
5836 pmaddwd m0, m1
5837 pmaddwd m2, m3
5838 pmaddwd m9, m5
5839 pmaddwd m4, m5
5840 paddd m7, m8
5841 paddd m0, m2
5842 movu m8, [srcq+ssq*1]
5843 lea srcq, [srcq+ssq*2]
5844 pshufb m11, m14
5845 pmaddubsw m11, m15
5846 paddd m7, m13
5847 paddd m0, m13
5848 paddd m7, m9
5849 paddd m0, m4
5850 pshufb m8, m14
5851 pmaddubsw m8, m15
5852 phaddw m11, m8
5853 mova m8, [rsp+0x20]
5854 pmulhrsw m11, m12
5855 punpcklwd m9, m6, m11 ; 67
5856 psrldq m6, m11, 8
5857 punpcklwd m4, m11, m6 ; 78
5858 pmaddwd m2, m9, m10
5859 pmaddwd m11, m4, m10
5860 paddd m7, m2
5861 mova m2, [rsp+0x30]
5862 paddd m0, m11
5863 %else
5864 SWAP m7, m6
5865 SWAP m1, m4
5866 SWAP m3, m2
5867 movu m5, [srcq+ssq*0]
5868 mova m0, [esp+0x30]
5869 mova m2, [esp+0x40]
5870 mova m4, [esp+0x50]
5871 pmaddwd m6, [esp+0x90]
5872 pmaddwd m1, [esp+0xa0]
5873 pmaddwd m0, [esp+0x90]
5874 pmaddwd m2, [esp+0xa0]
5875 pmaddwd m3, [esp+0xb0]
5876 pmaddwd m4, [esp+0xb0]
5877 paddd m6, m1
5878 paddd m0, m2
5879 movu m7, [srcq+ssq*1]
5880 lea srcq, [srcq+ssq*2]
5881 pshufb m5, m14
5882 pmaddubsw m5, m15
5883 paddd m6, m13
5884 paddd m0, m13
5885 paddd m6, m3
5886 paddd m0, m4
5887 pshufb m7, m14
5888 pmaddubsw m7, m15
5889 phaddw m5, m7
5890 mova m7, [rsp+0x80]
5891 pmulhrsw m5, m12
5892 punpcklwd m3, [esp+0x60], m5 ; 67
5893 psrldq m1, m5, 8
5894 punpcklwd m4, m5, m1 ; 78
5895 pmaddwd m2, m3, [esp+0xc0]
5896 pmaddwd m5, m4, [esp+0xc0]
5897 mova [esp+0x60], m1
5898 paddd m6, m2
5899 mova m2, [esp+0x50]
5900 paddd m0, m5
5901 SWAP m7, m6
5902 %endif
5903 psrad m7, rndshift
5904 psrad m0, rndshift
5905 packssdw m7, m0
5906 %if ARCH_X86_64
5907 mova m0, [rsp+0x10]
5908 %else
5909 mova m0, [esp+0x40]
5910 %define m11 m5
5911 %endif
5912 %ifidn %1, put
5913 packuswb m7, m7
5914 psrldq m11, m7, 4
5915 movd [dstq+dsq*0], m7
5916 movd [dstq+dsq*1], m11
5917 lea dstq, [dstq+dsq*2]
5918 %else
5919 mova [tmpq], m7
5920 add tmpq, 16
5921 %endif
5922 sub hd, 2
5923 jz .ret
5924 %if ARCH_X86_64
5925 mova m7, [rsp+0x00]
5926 mova [rsp+0x00], m8
5927 mova [rsp+0x10], m2
5928 mova [rsp+0x20], m9
5929 mova [rsp+0x30], m4
5930 %else
5931 mova m7, [esp+0x70] ; 01
5932 mova m1, [esp+0x80] ; 23
5933 mova m2, [esp+0x50] ; 34
5934 mova [esp+0x30], m0
5935 mova [esp+0x70], m1
5936 mova [esp+0x40], m2
5937 mova [esp+0x80], m3
5938 mova [esp+0x50], m4
5939 %endif
5940 jmp .dy1_w4_loop
5941 INIT_XMM ssse3
5942 .dy1_w8:
5943 mov dword [rsp+0x90], 1
5944 movifprep tmp_stridem, 16
5945 jmp .dy1_w_start
5946 .dy1_w16:
5947 mov dword [rsp+0x90], 2
5948 movifprep tmp_stridem, 32
5949 jmp .dy1_w_start
5950 .dy1_w32:
5951 mov dword [rsp+0x90], 4
5952 movifprep tmp_stridem, 64
5953 jmp .dy1_w_start
5954 .dy1_w64:
5955 mov dword [rsp+0x90], 8
5956 movifprep tmp_stridem, 128
5957 jmp .dy1_w_start
5958 .dy1_w128:
5959 mov dword [rsp+0x90], 16
5960 movifprep tmp_stridem, 256
5961 .dy1_w_start:
5962 mov myd, mym
5963 %ifidn %1, put
5964 movifnidn dsm, dsq
5965 %endif
5966 %if ARCH_X86_64
5967 shr t0d, 16
5968 sub srcq, 3
5969 shr myd, 6
5970 mov r4d, 64 << 24
5971 lea myd, [t1+myq]
5972 cmovnz r4q, [base+subpel_filters+myq*8]
5973 movd m15, t0d
5974 %else
5975 %define m8 m0
5976 %define m9 m1
5977 %xdefine m14 m4
5978 %xdefine m15 m3
5979 %if isprep
5980 %define ssq ssm
5981 %endif
5982 mov r5, [esp+0x1f0]
5983 mov r3, [esp+0x1f4]
5984 shr r5, 16
5985 sub srcq, 3
5986 movd m15, r5
5987 xor r5, r5
5988 shr myd, 6
5989 lea r3, [r3+myd]
5990 mov r4, 64 << 24
5991 cmovnz r4, [base+subpel_filters+r3*8+0]
5992 cmovnz r5, [base+subpel_filters+r3*8+4]
5993 mov r0, r0m
5994 mov r3, r3m
5995 %endif
5996 pslld m7, m8, 2 ; dx*4
5997 pmaddwd m8, [base+rescale_mul] ; dx*[0-3]
5998 pshufd m15, m15, q0000
5999 paddd m14, m8 ; mx+dx*[0-3]
6000 %if ARCH_X86_64
6001 movq m3, r4q
6002 punpcklbw m3, m3
6003 psraw m3, 8
6004 %else
6005 movd m5, r4
6006 movd m6, r5
6007 punpckldq m5, m6
6008 punpcklbw m5, m5
6009 psraw m5, 8
6010 SWAP m3, m5
6011 %endif
6012 mova [rsp+0x100], m7
6013 mova [rsp+0x120], m15
6014 mov [rsp+0x098], srcq
6015 mov [rsp+0x130], r0q ; dstq / tmpq
6016 pshufd m0, m3, q0000
6017 pshufd m1, m3, q1111
6018 pshufd m2, m3, q2222
6019 pshufd m3, m3, q3333
6020 mova [rsp+0x140], m0
6021 mova [rsp+0x150], m1
6022 mova [rsp+0x160], m2
6023 mova [rsp+0x170], m3
6024 %if ARCH_X86_64 && UNIX64
6025 mov hm, hd
6026 %elif ARCH_X86_32
6027 SWAP m5, m3
6028 mov r5, hm
6029 mov [esp+0x134], r5
6030 %endif
6031 jmp .dy1_hloop
6032 .dy1_hloop_prep:
6033 dec dword [rsp+0x090]
6034 jz .ret
6035 %if ARCH_X86_64
6036 add qword [rsp+0x130], 8*(isprep+1)
6037 mov hd, hm
6038 %else
6039 add dword [rsp+0x130], 8*(isprep+1)
6040 mov r5, [esp+0x134]
6041 mov r0, [esp+0x130]
6042 %endif
6043 mova m7, [rsp+0x100]
6044 mova m14, [rsp+0x110]
6045 %if ARCH_X86_64
6046 mova m10, [base+pd_0x3ff]
6047 %else
6048 %define m10 [base+pd_0x3ff]
6049 %endif
6050 mova m15, [rsp+0x120]
6051 mov srcq, [rsp+0x098]
6052 %if ARCH_X86_64
6053 mov r0q, [rsp+0x130] ; dstq / tmpq
6054 %else
6055 mov hm, r5
6056 mov r0m, r0
6057 mov r3, r3m
6058 %endif
6059 paddd m14, m7
6060 .dy1_hloop:
6061 pxor m9, m9
6062 %if ARCH_X86_64
6063 mova m11, [base+pq_0x40000000]
6064 %else
6065 %define m11 [base+pq_0x40000000]
6066 %endif
6067 psrld m2, m14, 10
6068 mova [rsp], m2
6069 pand m6, m14, m10
6070 psrld m6, 6
6071 paddd m5, m15, m6
6072 pcmpeqd m6, m9
6073 psrldq m2, m5, 8
6074 %if ARCH_X86_64
6075 movd r4d, m5
6076 movd r6d, m2
6077 psrldq m5, 4
6078 psrldq m2, 4
6079 movd r7d, m5
6080 movd r9d, m2
6081 movq m0, [base+subpel_filters+r4*8]
6082 movq m1, [base+subpel_filters+r6*8]
6083 movhps m0, [base+subpel_filters+r7*8]
6084 movhps m1, [base+subpel_filters+r9*8]
6085 %else
6086 movd r0, m5
6087 movd rX, m2
6088 psrldq m5, 4
6089 psrldq m2, 4
6090 movd r4, m5
6091 movd r5, m2
6092 movq m0, [base+subpel_filters+r0*8]
6093 movq m1, [base+subpel_filters+rX*8]
6094 movhps m0, [base+subpel_filters+r4*8]
6095 movhps m1, [base+subpel_filters+r5*8]
6096 pxor m2, m2
6097 %define m9 m2
6098 %endif
6099 paddd m14, m7 ; mx+dx*[4-7]
6100 pand m5, m14, m10
6101 psrld m5, 6
6102 paddd m15, m5
6103 pcmpeqd m5, m9
6104 mova [rsp+0x110], m14
6105 psrldq m4, m15, 8
6106 %if ARCH_X86_64
6107 movd r10d, m15
6108 movd r11d, m4
6109 psrldq m15, 4
6110 psrldq m4, 4
6111 movd r13d, m15
6112 movd rXd, m4
6113 movq m2, [base+subpel_filters+r10*8]
6114 movq m3, [base+subpel_filters+r11*8]
6115 movhps m2, [base+subpel_filters+r13*8]
6116 movhps m3, [base+subpel_filters+ rX*8]
6117 psrld m14, 10
6118 psrldq m4, m14, 8
6119 movd r10d, m14
6120 movd r11d, m4
6121 psrldq m14, 4
6122 psrldq m4, 4
6123 movd r13d, m14
6124 movd rXd, m4
6125 mov r4d, [rsp+ 0]
6126 mov r6d, [rsp+ 8]
6127 mov r7d, [rsp+ 4]
6128 mov r9d, [rsp+12]
6129 pshufd m4, m6, q1100
6130 pshufd m6, m6, q3322
6131 pshufd m7, m5, q1100
6132 pshufd m5, m5, q3322
6133 pand m8, m11, m4
6134 pand m9, m11, m6
6135 pand m15, m11, m7
6136 pand m11, m11, m5
6137 pandn m4, m0
6138 pandn m6, m1
6139 pandn m7, m2
6140 pandn m5, m3
6141 por m8, m4
6142 por m9, m6
6143 por m15, m7
6144 por m11, m5
6145 mova [rsp+0x10], m8
6146 mova [rsp+0x20], m9
6147 mova [rsp+0x30], m15
6148 mova [rsp+0x40], m11
6149 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 10, 8, 9, 15, 11 ; 0-1
6150 mova [rsp+0x50], m1
6151 mova [rsp+0x60], m2
6152 MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 7, 10, 8, 9, 15, 11 ; 2-3
6153 mova [rsp+0x70], m3
6154 mova [rsp+0x80], m4
6155 MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 4-5
6156 MC_8TAP_SCALED_H 0,14, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 6-7
6157 SWAP m7, m0
6158 SWAP m8, m14
6159 mova m1, [rsp+0x50]
6160 mova m2, [rsp+0x60]
6161 mova m3, [rsp+0x70]
6162 mova m15, [rsp+0x80]
6163 punpcklwd m4, m5, m6 ; 45a
6164 punpckhwd m5, m6 ; 45b
6165 punpcklwd m6, m7, m8 ; 67a
6166 punpckhwd m7, m8 ; 67b
6167 SWAP m14, m8
6168 mova m8, [rsp+0x140]
6169 mova m9, [rsp+0x150]
6170 mova m10, [rsp+0x160]
6171 mova m11, [rsp+0x170]
6172 punpcklwd m0, m1, m2 ; 01a
6173 punpckhwd m1, m2 ; 01b
6174 punpcklwd m2, m3, m15; 23a
6175 punpckhwd m3, m15 ; 23b
6176 mova [rsp+0x50], m4
6177 mova [rsp+0x60], m5
6178 mova [rsp+0x70], m6
6179 mova [rsp+0x80], m7
6180 mova m14, [base+unpckw]
6181 %else
6182 movd r0, m15
6183 movd rX, m4
6184 psrldq m15, 4
6185 psrldq m4, 4
6186 movd r4, m15
6187 movd r5, m4
6188 mova m14, [esp+0x110]
6189 movq m2, [base+subpel_filters+r0*8]
6190 movq m3, [base+subpel_filters+rX*8]
6191 movhps m2, [base+subpel_filters+r4*8]
6192 movhps m3, [base+subpel_filters+r5*8]
6193 psrld m14, 10
6194 mova [esp+16], m14
6195 mov r0, [esp+ 0]
6196 mov rX, [esp+ 8]
6197 mov r4, [esp+ 4]
6198 mov r5, [esp+12]
6199 mova [esp+0x20], m0
6200 mova [esp+0x30], m1
6201 mova [esp+0x40], m2
6202 mova [esp+0x50], m3
6203 pshufd m4, m6, q1100
6204 pshufd m6, m6, q3322
6205 pshufd m7, m5, q1100
6206 pshufd m5, m5, q3322
6207 pand m0, m11, m4
6208 pand m1, m11, m6
6209 pand m2, m11, m7
6210 pand m3, m11, m5
6211 pandn m4, [esp+0x20]
6212 pandn m6, [esp+0x30]
6213 pandn m7, [esp+0x40]
6214 pandn m5, [esp+0x50]
6215 por m0, m4
6216 por m1, m6
6217 por m2, m7
6218 por m3, m5
6219 mova [esp+0x20], m0
6220 mova [esp+0x30], m1
6221 mova [esp+0x40], m2
6222 mova [esp+0x50], m3
6223 MC_8TAP_SCALED_H 0x20, 0x60, 0 ; 0-1
6224 MC_8TAP_SCALED_H 0x20, 0x180 ; 2-3
6225 MC_8TAP_SCALED_H 0x20, 0x1a0 ; 4-5
6226 MC_8TAP_SCALED_H 0x20, 0x1c0 ; 6-7
6227 mova m5, [esp+0x1a0]
6228 mova m6, [esp+0x1b0]
6229 mova m7, [esp+0x1c0]
6230 mova m0, [esp+0x1d0]
6231 punpcklwd m4, m5, m6 ; 45a
6232 punpckhwd m5, m6 ; 45b
6233 punpcklwd m6, m7, m0 ; 67a
6234 punpckhwd m7, m0 ; 67b
6235 mova [esp+0x1a0], m4
6236 mova [esp+0x1b0], m5
6237 mova [esp+0x1c0], m6
6238 mova [esp+0x1d0], m7
6239 mova m1, [esp+0x060]
6240 mova m2, [esp+0x070]
6241 mova m3, [esp+0x180]
6242 mova m4, [esp+0x190]
6243 punpcklwd m0, m1, m2 ; 01a
6244 punpckhwd m1, m2 ; 01b
6245 punpcklwd m2, m3, m4 ; 23a
6246 punpckhwd m3, m4 ; 23b
6247 mova [esp+0x060], m0
6248 mova [esp+0x070], m1
6249 mova [esp+0x180], m2
6250 mova [esp+0x190], m3
6251 %define m8 [esp+0x140]
6252 %define m9 [esp+0x150]
6253 %define m10 [esp+0x160]
6254 %define m11 [esp+0x170]
6255 %endif
6256 .dy1_vloop:
6257 %if ARCH_X86_32
6258 mov r0, r0m
6259 %endif
6260 pmaddwd m4, m0, m8
6261 pmaddwd m5, m1, m8
6262 pmaddwd m6, m2, m9
6263 pmaddwd m7, m3, m9
6264 paddd m4, m13
6265 paddd m5, m13
6266 paddd m4, m6
6267 paddd m5, m7
6268 %if ARCH_X86_64
6269 pmaddwd m6, [rsp+0x50], m10
6270 pmaddwd m7, [rsp+0x60], m10
6271 %else
6272 pmaddwd m6, [rsp+0x1a0], m10
6273 pmaddwd m7, [rsp+0x1b0], m10
6274 %endif
6275 paddd m4, m6
6276 paddd m5, m7
6277 %if ARCH_X86_64
6278 pmaddwd m6, [rsp+0x70], m11
6279 pmaddwd m7, [rsp+0x80], m11
6280 %else
6281 pmaddwd m6, [rsp+0x1c0], m11
6282 pmaddwd m7, [rsp+0x1d0], m11
6283 %endif
6284 paddd m4, m6
6285 paddd m5, m7
6286 psrad m4, rndshift
6287 psrad m5, rndshift
6288 packssdw m4, m5
6289 %ifidn %1, put
6290 packuswb m4, m4
6291 movq [dstq], m4
6292 add dstq, dsm
6293 %else
6294 mova [tmpq], m4
6295 add tmpq, tmp_stridem
6296 %endif
6297 %if ARCH_X86_32
6298 mov r0m, r0
6299 %endif
6300 dec hd
6301 jz .dy1_hloop_prep
6302 %if ARCH_X86_64
6303 movq m4, [srcq+ r4]
6304 movq m5, [srcq+ r6]
6305 movhps m4, [srcq+ r7]
6306 movhps m5, [srcq+ r9]
6307 movq m6, [srcq+r10]
6308 movq m7, [srcq+r11]
6309 movhps m6, [srcq+r13]
6310 movhps m7, [srcq+ rX]
6311 add srcq, ssq
6312 pshufd m15, m14, q1032
6313 pshufb m0, m14 ; 0a 1a
6314 pshufb m1, m14 ; 0b 1b
6315 pshufb m2, m15 ; 3a 2a
6316 pshufb m3, m15 ; 3b 2b
6317 pmaddubsw m4, [rsp+0x10]
6318 pmaddubsw m5, [rsp+0x20]
6319 pmaddubsw m6, [rsp+0x30]
6320 pmaddubsw m7, [rsp+0x40]
6321 phaddw m4, m5
6322 phaddw m6, m7
6323 phaddw m4, m6
6324 pmulhrsw m4, m12
6325 pshufb m5, [rsp+0x70], m15 ; 7a 6a
6326 pshufb m7, [rsp+0x80], m15 ; 7b 6b
6327 pshufb m6, [rsp+0x50], m14 ; 4a 5a
6328 pshufb m15, [rsp+0x60], m14 ; 4b 5b
6329 punpckhwd m0, m2 ; 12a
6330 punpckhwd m1, m3 ; 12b
6331 punpcklwd m2, m6 ; 34a
6332 punpcklwd m3, m15 ; 34b
6333 punpckhwd m6, m5 ; 56a
6334 punpckhwd m15, m7 ; 56b
6335 punpcklwd m5, m4 ; 78a
6336 psrldq m4, 8
6337 punpcklwd m7, m4 ; 78b
6338 mova [rsp+0x50], m6
6339 mova [rsp+0x60], m15
6340 mova [rsp+0x70], m5
6341 mova [rsp+0x80], m7
6342 %else
6343 mov r0, [esp+ 0]
6344 mov rX, [esp+ 8]
6345 mov r4, [esp+ 4]
6346 mov r5, [esp+12]
6347 mova m6, [base+unpckw]
6348 mova m0, [esp+0x060]
6349 mova m1, [esp+0x070]
6350 mova m7, [esp+0x1a0]
6351 movq m4, [srcq+r0]
6352 movq m5, [srcq+rX]
6353 movhps m4, [srcq+r4]
6354 movhps m5, [srcq+r5]
6355 pshufb m0, m6 ; 0a 1a
6356 pshufb m1, m6 ; 0b 1b
6357 pshufb m7, m6 ; 4a 5a
6358 mov r0, [esp+16]
6359 mov rX, [esp+24]
6360 mov r4, [esp+20]
6361 mov r5, [esp+28]
6362 movq m3, [srcq+r0]
6363 movq m2, [srcq+rX]
6364 movhps m3, [srcq+r4]
6365 movhps m2, [srcq+r5]
6366 add srcq, ssq
6367 pmaddubsw m4, [esp+0x20]
6368 pmaddubsw m5, [esp+0x30]
6369 pmaddubsw m3, [esp+0x40]
6370 pmaddubsw m2, [esp+0x50]
6371 phaddw m4, m5
6372 phaddw m3, m2
6373 mova m5, [esp+0x1b0]
6374 mova m2, [esp+0x180]
6375 phaddw m4, m3
6376 mova m3, [esp+0x190]
6377 pmulhrsw m4, m12 ; 8a 8b
6378 pshufb m5, m6 ; 4b 5b
6379 pshufd m6, m6, q1032
6380 pshufb m2, m6 ; 3a 2a
6381 pshufb m3, m6 ; 3b 2b
6382 punpckhwd m0, m2 ; 12a
6383 punpckhwd m1, m3 ; 12b
6384 mova [esp+0x60], m0
6385 mova [esp+0x70], m1
6386 mova m0, [esp+0x1c0]
6387 mova m1, [esp+0x1d0]
6388 punpcklwd m2, m7 ; 34a
6389 punpcklwd m3, m5 ; 34b
6390 mova [esp+0x180], m2
6391 mova [esp+0x190], m3
6392 pshufb m0, m6 ; 7a 6a
6393 pshufb m1, m6 ; 7b 6b
6394 punpckhwd m7, m0 ; 56a
6395 punpckhwd m5, m1 ; 56b
6396 punpcklwd m0, m4
6397 punpckhqdq m4, m4
6398 punpcklwd m1, m4
6399 mova [esp+0x1a0], m7
6400 mova [esp+0x1b0], m5
6401 mova [esp+0x1c0], m0
6402 mova [esp+0x1d0], m1
6403 mova m0, [esp+0x60]
6404 mova m1, [esp+0x70]
6405 %endif
6406 jmp .dy1_vloop
6407 INIT_XMM ssse3
6408 .dy2:
6409 movzx wd, word [base+%1_8tap_scaled_ssse3_dy2_table+wq*2]
6410 add wq, base_reg
6411 jmp wq
6412 %ifidn %1, put
6413 .dy2_w2:
6414 %if ARCH_X86_64
6415 mov myd, mym
6416 movzx t0d, t0b
6417 dec srcq
6418 movd m15, t0d
6419 %else
6420 %define m10 [base+pd_0x3ff]
6421 %define m11 [base+pd_0x4000]
6422 %define m8 m0
6423 %define m9 m1
6424 %define m14 m4
6425 %define m15 m3
6426 movzx r5, byte [esp+0x1f0]
6427 dec srcd
6428 movd m15, r5
6429 %endif
6430 punpckldq m9, m8
6431 SWAP m8, m9
6432 paddd m14, m8 ; mx+dx*[0-1]
6433 %if ARCH_X86_64
6434 mova m11, [base+pd_0x4000]
6435 %endif
6436 pshufd m15, m15, q0000
6437 pand m8, m14, m10
6438 psrld m8, 6
6439 paddd m15, m8
6440 movd r4d, m15
6441 psrldq m15, 4
6442 %if ARCH_X86_64
6443 movd r6d, m15
6444 %else
6445 movd r3d, m15
6446 %endif
6447 mova m5, [base+bdct_lb_dw]
6448 mova m6, [base+subpel_s_shuf2]
6449 movd m15, [base+subpel_filters+r4*8+2]
6450 %if ARCH_X86_64
6451 movd m7, [base+subpel_filters+r6*8+2]
6452 %else
6453 movd m7, [base+subpel_filters+r3*8+2]
6454 %endif
6455 pxor m9, m9
6456 pcmpeqd m8, m9
6457 psrld m14, 10
6458 %if ARCH_X86_32
6459 mov r3, r3m
6460 pshufb m14, m5
6461 paddb m14, m6
6462 mova [esp+0x00], m14
6463 %define m14 [esp+0x00]
6464 SWAP m5, m0
6465 SWAP m6, m3
6466 %define m8 m5
6467 %define m15 m6
6468 %endif
6469 movq m0, [srcq+ssq*0]
6470 movq m1, [srcq+ssq*1]
6471 movhps m0, [srcq+ssq*2]
6472 movhps m1, [srcq+ss3q ]
6473 lea srcq, [srcq+ssq*4]
6474 %if ARCH_X86_64
6475 shr myd, 6
6476 mov r4d, 64 << 24
6477 lea myd, [t1+myq]
6478 cmovnz r4q, [base+subpel_filters+myq*8]
6479 pshufb m14, m5
6480 paddb m14, m6
6481 movq m10, r4q
6482 %else
6483 mov myd, mym
6484 mov r3, [esp+0x1f4]
6485 xor r5, r5
6486 shr myd, 6
6487 lea r3, [r3+myd]
6488 mov r4, 64 << 24
6489 cmovnz r4, [base+subpel_filters+r3*8+0]
6490 cmovnz r5, [base+subpel_filters+r3*8+4]
6491 mov r3, r3m
6492 %define m10 m4
6493 movd m10, r4
6494 movd m3, r5
6495 punpckldq m10, m3
6496 %endif
6497 movq m3, [srcq+ssq*0]
6498 movhps m3, [srcq+ssq*1]
6499 lea srcq, [srcq+ssq*2]
6500 punpcklbw m10, m10
6501 psraw m10, 8
6502 punpckldq m15, m7
6503 punpcklqdq m15, m15
6504 %if ARCH_X86_64
6505 pand m11, m8
6506 %else
6507 pand m7, m11, m8
6508 %define m11 m7
6509 %endif
6510 pandn m8, m15
6511 SWAP m15, m8
6512 por m15, m11
6513 %if ARCH_X86_64
6514 pshufd m8, m10, q0000
6515 pshufd m9, m10, q1111
6516 pshufd m11, m10, q3333
6517 pshufd m10, m10, q2222
6518 %else
6519 mova [esp+0x10], m15
6520 %define m15 [esp+0x10]
6521 mov r5, r0m
6522 %define dstq r5
6523 mov dsd, dsm
6524 pshufd m5, m4, q0000
6525 pshufd m6, m4, q1111
6526 pshufd m7, m4, q2222
6527 pshufd m4, m4, q3333
6528 %define m8 [esp+0x20]
6529 %define m9 [esp+0x30]
6530 %define m10 [esp+0x40]
6531 %define m11 [esp+0x50]
6532 mova m8, m5
6533 mova m9, m6
6534 mova m10, m7
6535 mova m11, m4
6536 %endif
6537 pshufb m0, m14
6538 pshufb m1, m14
6539 pshufb m3, m14
6540 pmaddubsw m0, m15
6541 pmaddubsw m1, m15
6542 pmaddubsw m3, m15
6543 pslldq m2, m3, 8
6544 phaddw m0, m2
6545 phaddw m1, m3
6546 pmulhrsw m0, m12 ; 0 2 _ 4
6547 pmulhrsw m1, m12 ; 1 3 _ 5
6548 pshufd m2, m0, q3110 ; 0 2 2 4
6549 pshufd m1, m1, q3110 ; 1 3 3 5
6550 punpcklwd m3, m2, m1 ; 01 23
6551 punpckhwd m2, m1 ; 23 45
6552 .dy2_w2_loop:
6553 movq m6, [srcq+ssq*0]
6554 movq m7, [srcq+ssq*1]
6555 movhps m6, [srcq+ssq*2]
6556 movhps m7, [srcq+ss3q ]
6557 lea srcq, [srcq+ssq*4]
6558 pmaddwd m4, m3, m8
6559 pmaddwd m5, m2, m9
6560 pshufb m6, m14
6561 pshufb m7, m14
6562 pmaddubsw m6, m15
6563 pmaddubsw m7, m15
6564 phaddw m6, m7
6565 pmulhrsw m6, m12
6566 psrldq m7, m6, 8
6567 palignr m6, m0, 8
6568 palignr m7, m1, 8
6569 mova m0, m6
6570 mova m1, m7
6571 pshufd m6, m6, q3221
6572 pshufd m7, m7, q3221
6573 punpcklwd m3, m6, m7 ; 45 67
6574 punpckhwd m2, m6, m7 ; 67 89
6575 pmaddwd m6, m3, m10
6576 pmaddwd m7, m2, m11
6577 paddd m4, m5
6578 paddd m4, m13
6579 paddd m6, m7
6580 paddd m4, m6
6581 psrad m4, rndshift
6582 packssdw m4, m4
6583 packuswb m4, m4
6584 movd r4d, m4
6585 mov [dstq+dsq*0], r4w
6586 shr r4d, 16
6587 mov [dstq+dsq*1], r4w
6588 lea dstq, [dstq+dsq*2]
6589 sub hd, 2
6590 jg .dy2_w2_loop
6591 RET
6592 %endif
6593 INIT_XMM ssse3
6594 .dy2_w4:
6595 %if ARCH_X86_64
6596 mov myd, mym
6597 movzx t0d, t0b
6598 dec srcq
6599 movd m15, t0d
6600 %else
6601 %define m10 [base+pd_0x3ff]
6602 %define m11 [base+pd_0x4000]
6603 %define m8 m0
6604 %xdefine m14 m4
6605 %define m15 m3
6606 %define dstq r0
6607 %if isprep
6608 %define ssq r3
6609 %endif
6610 movzx r4, byte [esp+0x1f0]
6611 dec srcq
6612 movd m15, r4
6613 %endif
6614 pmaddwd m8, [base+rescale_mul]
6615 %if ARCH_X86_64
6616 mova m11, [base+pd_0x4000]
6617 %endif
6618 pshufd m15, m15, q0000
6619 paddd m14, m8 ; mx+dx*[0-3]
6620 pand m8, m14, m10
6621 psrld m8, 6
6622 paddd m15, m8
6623 psrldq m7, m15, 8
6624 %if ARCH_X86_64
6625 movd r4d, m15
6626 movd r11d, m7
6627 psrldq m15, 4
6628 psrldq m7, 4
6629 movd r6d, m15
6630 movd r13d, m7
6631 movd m15, [base+subpel_filters+ r4*8+2]
6632 movd m2, [base+subpel_filters+r11*8+2]
6633 movd m3, [base+subpel_filters+ r6*8+2]
6634 movd m4, [base+subpel_filters+r13*8+2]
6635 movq m6, [base+subpel_s_shuf2]
6636 shr myd, 6
6637 mov r4d, 64 << 24
6638 lea myd, [t1+myq]
6639 cmovnz r4q, [base+subpel_filters+myq*8]
6640 %else
6641 movd r1, m15
6642 movd r3, m7
6643 psrldq m15, 4
6644 psrldq m7, 4
6645 movd r4, m15
6646 movd r5, m7
6647 %define m15 m5
6648 SWAP m4, m7
6649 movd m15, [base+subpel_filters+r1*8+2]
6650 movd m2, [base+subpel_filters+r3*8+2]
6651 movd m3, [base+subpel_filters+r4*8+2]
6652 movd m4, [base+subpel_filters+r5*8+2]
6653 movq m6, [base+subpel_s_shuf2]
6654 mov myd, mym
6655 mov r3, [esp+0x1f4]
6656 xor r5, r5
6657 shr myd, 6
6658 lea r3, [r3+myd]
6659 mov r4, 64 << 24
6660 cmovnz r4, [base+subpel_filters+r3*8+0]
6661 cmovnz r5, [base+subpel_filters+r3*8+4]
6662 mov r3, r3m
6663 %if isprep
6664 lea ss3q, [ssq*3]
6665 %endif
6666 %endif
6667 punpckldq m15, m3
6668 punpckldq m2, m4
6669 punpcklqdq m15, m2
6670 %if ARCH_X86_64
6671 pcmpeqd m8, m9
6672 psrld m14, 10
6673 movu m0, [srcq+ssq*0]
6674 movu m2, [srcq+ssq*2]
6675 movu m1, [srcq+ssq*1]
6676 movu m3, [srcq+ss3q ]
6677 lea srcq, [srcq+ssq*4]
6678 punpcklqdq m6, m6
6679 pshufb m14, [base+bdct_lb_dw]
6680 movu m4, [srcq+ssq*0]
6681 movu m5, [srcq+ssq*1]
6682 lea srcq, [srcq+ssq*2]
6683 pand m11, m8
6684 pandn m8, m15
6685 SWAP m15, m8
6686 por m15, m11
6687 paddb m14, m6
6688 movq m11, r4q
6689 punpcklbw m11, m11
6690 psraw m11, 8
6691 pshufb m0, m14
6692 pshufb m2, m14
6693 pshufb m1, m14
6694 pshufb m3, m14
6695 pshufb m4, m14
6696 pshufb m5, m14
6697 pmaddubsw m0, m15
6698 pmaddubsw m2, m15
6699 pmaddubsw m1, m15
6700 pmaddubsw m3, m15
6701 pmaddubsw m4, m15
6702 pmaddubsw m5, m15
6703 phaddw m0, m2
6704 phaddw m1, m3
6705 phaddw m4, m5
6706 pmulhrsw m0, m12 ; 0 2
6707 pmulhrsw m1, m12 ; 1 3
6708 pmulhrsw m4, m12 ; 4 5
6709 pshufd m8, m11, q0000
6710 pshufd m9, m11, q1111
6711 pshufd m10, m11, q2222
6712 pshufd m11, m11, q3333
6713 %else
6714 pxor m3, m3
6715 pcmpeqd m8, m3
6716 psrld m14, 10
6717 pshufb m14, [base+bdct_lb_dw]
6718 movu m1, [srcq+ssq*0]
6719 movu m2, [srcq+ssq*2]
6720 movu m3, [srcq+ssq*1]
6721 add srcq, ss3q
6722 punpcklqdq m6, m6
6723 SWAP m4, m7
6724 pand m7, m11, m8
6725 pandn m8, m15
6726 SWAP m15, m8
6727 por m15, m7
6728 paddb m14, m6
6729 movu m0, [srcq+ssq*0]
6730 movu m7, [srcq+ssq*1]
6731 movu m6, [srcq+ssq*2]
6732 add srcq, ss3q
6733 pshufb m1, m14
6734 pshufb m2, m14
6735 pshufb m3, m14
6736 pshufb m0, m14
6737 pshufb m7, m14
6738 pshufb m6, m14
6739 pmaddubsw m1, m15
6740 pmaddubsw m2, m15
6741 pmaddubsw m3, m15
6742 mova [esp+0x00], m14
6743 mova [esp+0x10], m15
6744 pmaddubsw m0, m15
6745 pmaddubsw m7, m15
6746 pmaddubsw m6, m15
6747 %define m14 [esp+0x00]
6748 %define m15 [esp+0x10]
6749 phaddw m1, m2
6750 phaddw m3, m0
6751 phaddw m7, m6
6752 %ifidn %1, put
6753 mov dsd, dsm
6754 %define dstq r5
6755 %else
6756 %define tmpq r5
6757 %endif
6758 movd m6, r4
6759 movd m0, r5
6760 punpckldq m6, m0
6761 punpcklbw m6, m6
6762 psraw m6, 8
6763 mov r5, r0m
6764 pmulhrsw m1, m12 ; 0 2
6765 pmulhrsw m3, m12 ; 1 3
6766 pmulhrsw m7, m12 ; 4 5
6767 SWAP m0, m1, m3
6768 SWAP m4, m7
6769 pshufd m2, m6, q0000
6770 pshufd m3, m6, q1111
6771 pshufd m7, m6, q2222
6772 pshufd m6, m6, q3333
6773 mova [esp+0x30], m2
6774 mova [esp+0x40], m3
6775 mova [esp+0x50], m7
6776 mova [esp+0x60], m6
6777 %define m8 [esp+0x30]
6778 %define m9 [esp+0x40]
6779 %define m10 [esp+0x50]
6780 %define m11 [esp+0x60]
6781 %endif
6782 psrldq m5, m4, 8 ; 5 _
6783 punpckhwd m2, m0, m1 ; 23
6784 punpcklwd m0, m1 ; 01
6785 punpcklwd m4, m5 ; 45
6786 .dy2_w4_loop:
6787 pmaddwd m0, m8 ; a0
6788 pmaddwd m5, m2, m8 ; b0
6789 pmaddwd m2, m9 ; a1
6790 pmaddwd m7, m4, m9 ; b1
6791 pmaddwd m3, m4, m10 ; a2
6792 paddd m0, m13
6793 paddd m5, m13
6794 paddd m0, m2
6795 paddd m5, m7
6796 paddd m0, m3
6797 movu m6, [srcq+ssq*0]
6798 movu m7, [srcq+ssq*1]
6799 movu m3, [srcq+ssq*2]
6800 movu m1, [srcq+ss3q ]
6801 lea srcq, [srcq+ssq*4]
6802 pshufb m6, m14
6803 pshufb m7, m14
6804 pshufb m3, m14
6805 pshufb m1, m14
6806 pmaddubsw m6, m15
6807 pmaddubsw m7, m15
6808 pmaddubsw m3, m15
6809 pmaddubsw m1, m15
6810 phaddw m6, m7
6811 phaddw m3, m1
6812 pmulhrsw m6, m12 ; 6 7
6813 pmulhrsw m3, m12 ; 8 9
6814 psrldq m7, m6, 8
6815 psrldq m1, m3, 8
6816 punpcklwd m6, m7 ; 67
6817 punpcklwd m3, m1 ; 89
6818 mova m2, m6
6819 pmaddwd m1, m6, m10 ; b2
6820 pmaddwd m6, m11 ; a3
6821 pmaddwd m7, m3, m11 ; b3
6822 paddd m5, m1
6823 paddd m0, m6
6824 paddd m5, m7
6825 psrad m0, rndshift
6826 psrad m5, rndshift
6827 packssdw m0, m5
6828 %ifidn %1, put
6829 packuswb m0, m0
6830 psrldq m1, m0, 4
6831 movd [dstq+dsq*0], m0
6832 movd [dstq+dsq*1], m1
6833 lea dstq, [dstq+dsq*2]
6834 %else
6835 mova [tmpq], m0
6836 add tmpq, 16
6837 %endif
6838 mova m0, m4
6839 mova m4, m3
6840 sub hd, 2
6841 jg .dy2_w4_loop
6842 MC_8TAP_SCALED_RET
6843 INIT_XMM ssse3
6844 .dy2_w8:
6845 mov dword [rsp+0x90], 1
6846 movifprep tmp_stridem, 16
6847 jmp .dy2_w_start
6848 .dy2_w16:
6849 mov dword [rsp+0x90], 2
6850 movifprep tmp_stridem, 32
6851 jmp .dy2_w_start
6852 .dy2_w32:
6853 mov dword [rsp+0x90], 4
6854 movifprep tmp_stridem, 64
6855 jmp .dy2_w_start
6856 .dy2_w64:
6857 mov dword [rsp+0x90], 8
6858 movifprep tmp_stridem, 128
6859 jmp .dy2_w_start
6860 .dy2_w128:
6861 mov dword [rsp+0x90], 16
6862 movifprep tmp_stridem, 256
6863 .dy2_w_start:
6864 mov myd, mym
6865 %ifidn %1, put
6866 movifnidn dsm, dsq
6867 %endif
6868 %if ARCH_X86_64
6869 shr t0d, 16
6870 sub srcq, 3
6871 shr myd, 6
6872 mov r4d, 64 << 24
6873 lea myd, [t1+myq]
6874 cmovnz r4q, [base+subpel_filters+myq*8]
6875 movd m15, t0d
6876 %else
6877 %define m10 [base+pd_0x3ff]
6878 %define m11 [base+pd_0x4000]
6879 %define m8 m0
6880 %define m9 m1
6881 %xdefine m14 m4
6882 %xdefine m15 m3
6883 %if isprep
6884 %define tmpq r0
6885 %define ssq ssm
6886 %else
6887 %define dstq r0
6888 %endif
6889 mov r5, [esp+0x1f0]
6890 mov r3, [esp+0x1f4]
6891 shr r5, 16
6892 sub srcq, 3
6893 movd m15, r5
6894 xor r5, r5
6895 shr myd, 6
6896 lea r3, [r3+myd]
6897 mov r4, 64 << 24
6898 cmovnz r4, [base+subpel_filters+r3*8+0]
6899 cmovnz r5, [base+subpel_filters+r3*8+4]
6900 mov r0, r0m
6901 mov r3, r3m
6902 %endif
6903 pslld m7, m8, 2 ; dx*4
6904 pmaddwd m8, [base+rescale_mul] ; dx*[0-3]
6905 pshufd m15, m15, q0000
6906 paddd m14, m8 ; mx+dx*[0-3]
6907 %if ARCH_X86_64
6908 movq m3, r4q
6909 punpcklbw m3, m3
6910 psraw m3, 8
6911 %else
6912 movd m5, r4
6913 movd m6, r5
6914 punpckldq m5, m6
6915 punpcklbw m5, m5
6916 psraw m5, 8
6917 SWAP m3, m5
6918 %endif
6919 mova [rsp+0x100], m7
6920 mova [rsp+0x120], m15
6921 mov [rsp+0x098], srcq
6922 mov [rsp+0x130], r0q ; dstq / tmpq
6923 pshufd m0, m3, q0000
6924 pshufd m1, m3, q1111
6925 pshufd m2, m3, q2222
6926 pshufd m3, m3, q3333
6927 mova [rsp+0x140], m0
6928 mova [rsp+0x150], m1
6929 mova [rsp+0x160], m2
6930 mova [rsp+0x170], m3
6931 %if ARCH_X86_64 && UNIX64
6932 mov hm, hd
6933 %elif ARCH_X86_32
6934 SWAP m5, m3
6935 mov r5, hm
6936 mov [esp+0x134], r5
6937 %endif
6938 jmp .dy2_hloop
6939 .dy2_hloop_prep:
6940 dec dword [rsp+0x090]
6941 jz .ret
6942 %if ARCH_X86_64
6943 add qword [rsp+0x130], 8*(isprep+1)
6944 mov hd, hm
6945 %else
6946 add dword [rsp+0x130], 8*(isprep+1)
6947 mov r5, [esp+0x134]
6948 mov r0, [esp+0x130]
6949 %endif
6950 mova m7, [rsp+0x100]
6951 mova m14, [rsp+0x110]
6952 %if ARCH_X86_64
6953 mova m10, [base+pd_0x3ff]
6954 %else
6955 %define m10 [base+pd_0x3ff]
6956 %endif
6957 mova m15, [rsp+0x120]
6958 mov srcq, [rsp+0x098]
6959 %if ARCH_X86_64
6960 mov r0q, [rsp+0x130] ; dstq / tmpq
6961 %else
6962 mov hm, r5
6963 mov r0m, r0
6964 mov r3, r3m
6965 %endif
6966 paddd m14, m7
6967 .dy2_hloop:
6968 pxor m9, m9
6969 %if ARCH_X86_64
6970 mova m11, [base+pq_0x40000000]
6971 %else
6972 %define m11 [base+pq_0x40000000]
6973 %endif
6974 psrld m2, m14, 10
6975 mova [rsp], m2
6976 pand m6, m14, m10
6977 psrld m6, 6
6978 paddd m5, m15, m6
6979 pcmpeqd m6, m9
6980 psrldq m2, m5, 8
6981 %if ARCH_X86_64
6982 movd r4d, m5
6983 movd r6d, m2
6984 psrldq m5, 4
6985 psrldq m2, 4
6986 movd r7d, m5
6987 movd r9d, m2
6988 movq m0, [base+subpel_filters+r4*8]
6989 movq m1, [base+subpel_filters+r6*8]
6990 movhps m0, [base+subpel_filters+r7*8]
6991 movhps m1, [base+subpel_filters+r9*8]
6992 %else
6993 movd r0, m5
6994 movd rX, m2
6995 psrldq m5, 4
6996 psrldq m2, 4
6997 movd r4, m5
6998 movd r5, m2
6999 movq m0, [base+subpel_filters+r0*8]
7000 movq m1, [base+subpel_filters+rX*8]
7001 movhps m0, [base+subpel_filters+r4*8]
7002 movhps m1, [base+subpel_filters+r5*8]
7003 pxor m2, m2
7004 %define m9 m2
7005 %endif
7006 paddd m14, m7 ; mx+dx*[4-7]
7007 pand m5, m14, m10
7008 psrld m5, 6
7009 paddd m15, m5
7010 pcmpeqd m5, m9
7011 mova [rsp+0x110], m14
7012 psrldq m4, m15, 8
7013 %if ARCH_X86_64
7014 movd r10d, m15
7015 movd r11d, m4
7016 psrldq m15, 4
7017 psrldq m4, 4
7018 movd r13d, m15
7019 movd rXd, m4
7020 movq m2, [base+subpel_filters+r10*8]
7021 movq m3, [base+subpel_filters+r11*8]
7022 movhps m2, [base+subpel_filters+r13*8]
7023 movhps m3, [base+subpel_filters+ rX*8]
7024 psrld m14, 10
7025 psrldq m4, m14, 8
7026 movd r10d, m14
7027 movd r11d, m4
7028 psrldq m14, 4
7029 psrldq m4, 4
7030 movd r13d, m14
7031 movd rXd, m4
7032 mov r4d, [rsp+ 0]
7033 mov r6d, [rsp+ 8]
7034 mov r7d, [rsp+ 4]
7035 mov r9d, [rsp+12]
7036 pshufd m4, m6, q1100
7037 pshufd m6, m6, q3322
7038 pshufd m7, m5, q1100
7039 pshufd m5, m5, q3322
7040 pand m8, m11, m4
7041 pand m9, m11, m6
7042 pand m15, m11, m7
7043 pand m11, m11, m5
7044 pandn m4, m0
7045 pandn m6, m1
7046 pandn m7, m2
7047 pandn m5, m3
7048 por m8, m4
7049 por m9, m6
7050 por m15, m7
7051 por m11, m5
7052 mova [rsp+0x10], m8
7053 mova [rsp+0x20], m9
7054 mova [rsp+0x30], m15
7055 mova [rsp+0x40], m11
7056 MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 10, 8, 9, 15, 11 ; 0-1
7057 mova [rsp+0x50], m1
7058 mova [rsp+0x60], m2
7059 MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 7, 10, 8, 9, 15, 11 ; 2-3
7060 mova [rsp+0x70], m3
7061 mova [rsp+0x80], m4
7062 MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 4-5
7063 MC_8TAP_SCALED_H 0,14, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 6-7
7064 SWAP m7, m0
7065 SWAP m8, m14
7066 mova m1, [rsp+0x50]
7067 mova m2, [rsp+0x60]
7068 mova m3, [rsp+0x70]
7069 mova m15, [rsp+0x80]
7070 punpcklwd m4, m5, m6 ; 45a
7071 punpckhwd m5, m6 ; 45b
7072 punpcklwd m6, m7, m8 ; 67a
7073 punpckhwd m7, m8 ; 67b
7074 SWAP m14, m8
7075 mova m8, [rsp+0x140]
7076 mova m9, [rsp+0x150]
7077 mova m10, [rsp+0x160]
7078 mova m11, [rsp+0x170]
7079 punpcklwd m0, m1, m2 ; 01a
7080 punpckhwd m1, m2 ; 01b
7081 punpcklwd m2, m3, m15; 23a
7082 punpckhwd m3, m15 ; 23b
7083 mova [rsp+0x50], m4
7084 mova [rsp+0x60], m5
7085 mova [rsp+0x70], m6
7086 mova [rsp+0x80], m7
7087 %else
7088 movd r0, m15
7089 movd rX, m4
7090 psrldq m15, 4
7091 psrldq m4, 4
7092 movd r4, m15
7093 movd r5, m4
7094 mova m14, [esp+0x110]
7095 movq m2, [base+subpel_filters+r0*8]
7096 movq m3, [base+subpel_filters+rX*8]
7097 movhps m2, [base+subpel_filters+r4*8]
7098 movhps m3, [base+subpel_filters+r5*8]
7099 psrld m14, 10
7100 mova [esp+16], m14
7101 mov r0, [esp+ 0]
7102 mov rX, [esp+ 8]
7103 mov r4, [esp+ 4]
7104 mov r5, [esp+12]
7105 mova [esp+0x20], m0
7106 mova [esp+0x30], m1
7107 mova [esp+0x40], m2
7108 mova [esp+0x50], m3
7109 pshufd m4, m6, q1100
7110 pshufd m6, m6, q3322
7111 pshufd m7, m5, q1100
7112 pshufd m5, m5, q3322
7113 pand m0, m11, m4
7114 pand m1, m11, m6
7115 pand m2, m11, m7
7116 pand m3, m11, m5
7117 pandn m4, [esp+0x20]
7118 pandn m6, [esp+0x30]
7119 pandn m7, [esp+0x40]
7120 pandn m5, [esp+0x50]
7121 por m0, m4
7122 por m1, m6
7123 por m2, m7
7124 por m3, m5
7125 mova [esp+0x20], m0
7126 mova [esp+0x30], m1
7127 mova [esp+0x40], m2
7128 mova [esp+0x50], m3
7129 MC_8TAP_SCALED_H 0x20, 0x60, 0 ; 0-1
7130 MC_8TAP_SCALED_H 0x20, 0x180 ; 2-3
7131 MC_8TAP_SCALED_H 0x20, 0x1a0 ; 4-5
7132 MC_8TAP_SCALED_H 0x20, 0x1c0 ; 6-7
7133 mova m5, [esp+0x1a0]
7134 mova m6, [esp+0x1b0]
7135 mova m7, [esp+0x1c0]
7136 mova m0, [esp+0x1d0]
7137 punpcklwd m4, m5, m6 ; 45a
7138 punpckhwd m5, m6 ; 45b
7139 punpcklwd m6, m7, m0 ; 67a
7140 punpckhwd m7, m0 ; 67b
7141 mova [esp+0x1a0], m4
7142 mova [esp+0x1b0], m5
7143 mova [esp+0x1c0], m6
7144 mova [esp+0x1d0], m7
7145 mova m1, [esp+0x060]
7146 mova m2, [esp+0x070]
7147 mova m3, [esp+0x180]
7148 mova m4, [esp+0x190]
7149 punpcklwd m0, m1, m2 ; 01a
7150 punpckhwd m1, m2 ; 01b
7151 punpcklwd m2, m3, m4 ; 23a
7152 punpckhwd m3, m4 ; 23b
7153 mova [esp+0x180], m2
7154 mova [esp+0x190], m3
7155 %define m8 [esp+0x140]
7156 %define m9 [esp+0x150]
7157 %define m10 [esp+0x160]
7158 %define m11 [esp+0x170]
7159 %endif
7160 .dy2_vloop:
7161 %if ARCH_X86_32
7162 mov r0, r0m
7163 %endif
7164 pmaddwd m4, m0, m8
7165 pmaddwd m5, m1, m8
7166 pmaddwd m6, m2, m9
7167 pmaddwd m7, m3, m9
7168 paddd m4, m13
7169 paddd m5, m13
7170 paddd m4, m6
7171 paddd m5, m7
7172 %if ARCH_X86_64
7173 pmaddwd m6, [rsp+0x50], m10
7174 pmaddwd m7, [rsp+0x60], m10
7175 %else
7176 pmaddwd m6, [esp+0x1a0], m10
7177 pmaddwd m7, [esp+0x1b0], m10
7178 %endif
7179 paddd m4, m6
7180 paddd m5, m7
7181 %if ARCH_X86_64
7182 pmaddwd m6, [rsp+0x70], m11
7183 pmaddwd m7, [rsp+0x80], m11
7184 %else
7185 pmaddwd m6, [esp+0x1c0], m11
7186 pmaddwd m7, [esp+0x1d0], m11
7187 %endif
7188 paddd m4, m6
7189 paddd m5, m7
7190 psrad m4, rndshift
7191 psrad m5, rndshift
7192 packssdw m4, m5
7193 %ifidn %1, put
7194 packuswb m4, m4
7195 movq [dstq], m4
7196 add dstq, dsm
7197 %else
7198 mova [tmpq], m4
7199 add tmpq, tmp_stridem
7200 %endif
7201 %if ARCH_X86_32
7202 mov r0m, r0
7203 %endif
7204 dec hd
7205 jz .dy2_hloop_prep
7206 %if ARCH_X86_64
7207 mova m8, [rsp+0x10]
7208 mova m9, [rsp+0x20]
7209 mova m10, [rsp+0x30]
7210 mova m11, [rsp+0x40]
7211 mova m0, m2 ; 01a
7212 mova m1, m3 ; 01b
7213 MC_8TAP_SCALED_H 2, 6, 3, 4, 5, 7, 14, 15, 8, 9, 10, 11
7214 mova m3, [rsp+0x50] ; 23a
7215 mova m4, [rsp+0x60] ; 23b
7216 mova m5, [rsp+0x70] ; 45a
7217 mova m7, [rsp+0x80] ; 45b
7218 mova m8, [rsp+0x140]
7219 mova m9, [rsp+0x150]
7220 mova m10, [rsp+0x160]
7221 mova m11, [rsp+0x170]
7222 punpcklwd m14, m2, m6 ; 67a
7223 punpckhwd m2, m6 ; 67b
7224 mova [rsp+0x50], m5
7225 mova [rsp+0x60], m7
7226 mova [rsp+0x70], m14
7227 mova [rsp+0x80], m2
7228 mova m2, m3
7229 mova m3, m4
7230 %else
7231 MC_8TAP_SCALED_H 0x20, 0
7232 punpcklwd m6, m0, m4
7233 punpckhwd m7, m0, m4
7234 mova m0, [esp+0x180] ; 01a
7235 mova m1, [esp+0x190] ; 01b
7236 mova m2, [rsp+0x1a0] ; 23a
7237 mova m3, [esp+0x1b0] ; 23b
7238 mova m4, [esp+0x1c0] ; 45a
7239 mova m5, [esp+0x1d0] ; 45b
7240 mova [esp+0x180], m2
7241 mova [esp+0x190], m3
7242 mova [esp+0x1a0], m4
7243 mova [esp+0x1b0], m5
7244 mova [esp+0x1c0], m6 ; 67a
7245 mova [esp+0x1d0], m7 ; 67b
7246 %endif
7247 jmp .dy2_vloop
7248 .ret:
7249 MC_8TAP_SCALED_RET 0
7250 %if ARCH_X86_32 && !isprep && required_stack_alignment > STACK_ALIGNMENT
7251 %define r0m [rstk+stack_offset+ 4]
7252 %define r1m [rstk+stack_offset+ 8]
7253 %define r2m [rstk+stack_offset+12]
7254 %define r3m [rstk+stack_offset+16]
7255 %endif
7256 %undef isprep
7257 %endmacro
7258
7259 %macro BILIN_SCALED_FN 1
7260 cglobal %1_bilin_scaled_8bpc
7261 mov t0d, (5*15 << 16) | 5*15
7262 mov t1d, (5*15 << 16) | 5*15
7263 jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX)
7264 %endmacro
7265
7266 %if WIN64
7267 DECLARE_REG_TMP 6, 5
7268 %elif ARCH_X86_64
7269 DECLARE_REG_TMP 6, 8
7270 %else
7271 DECLARE_REG_TMP 1, 2
7272 %endif
7273 BILIN_SCALED_FN put
7274 FN put_8tap_scaled, sharp, SHARP, SHARP
7275 FN put_8tap_scaled, sharp_smooth, SHARP, SMOOTH
7276 FN put_8tap_scaled, smooth_sharp, SMOOTH, SHARP
7277 FN put_8tap_scaled, smooth, SMOOTH, SMOOTH
7278 FN put_8tap_scaled, sharp_regular, SHARP, REGULAR
7279 FN put_8tap_scaled, regular_sharp, REGULAR, SHARP
7280 FN put_8tap_scaled, smooth_regular, SMOOTH, REGULAR
7281 FN put_8tap_scaled, regular_smooth, REGULAR, SMOOTH
7282 FN put_8tap_scaled, regular, REGULAR, REGULAR
7283 MC_8TAP_SCALED put
7284
7285 %if WIN64
7286 DECLARE_REG_TMP 5, 4
7287 %elif ARCH_X86_64
7288 DECLARE_REG_TMP 6, 7
7289 %else
7290 DECLARE_REG_TMP 1, 2
7291 %endif
7292 BILIN_SCALED_FN prep
7293 FN prep_8tap_scaled, sharp, SHARP, SHARP
7294 FN prep_8tap_scaled, sharp_smooth, SHARP, SMOOTH
7295 FN prep_8tap_scaled, smooth_sharp, SMOOTH, SHARP
7296 FN prep_8tap_scaled, smooth, SMOOTH, SMOOTH
7297 FN prep_8tap_scaled, sharp_regular, SHARP, REGULAR
7298 FN prep_8tap_scaled, regular_sharp, REGULAR, SHARP
7299 FN prep_8tap_scaled, smooth_regular, SMOOTH, REGULAR
7300 FN prep_8tap_scaled, regular_smooth, REGULAR, SMOOTH
7301 FN prep_8tap_scaled, regular, REGULAR, REGULAR
7302 MC_8TAP_SCALED prep
7303
7304 %if ARCH_X86_32
7305 %macro SAVE_ALPHA_BETA 0
7306 mov alpham, alphad
7307 mov betam, betad
7308 %endmacro
7309
7310 %macro SAVE_DELTA_GAMMA 0
7311 mov deltam, deltad
7312 mov gammam, gammad
7313 %endmacro
7314
7315 %macro LOAD_ALPHA_BETA_MX 0
7316 mov mym, myd
7317 mov alphad, alpham
7318 mov betad, betam
7319 mov mxd, mxm
7320 %endmacro
7321
7322 %macro LOAD_DELTA_GAMMA_MY 0
7323 mov mxm, mxd
7324 mov deltad, deltam
7325 mov gammad, gammam
7326 mov myd, mym
7327 %endmacro
7328
7329 %define PIC_reg r2
7330 %define PIC_base_offset $$
7331 %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset)
7332 %else
7333 %define SAVE_ALPHA_BETA
7334 %define SAVE_DELTA_GAMMA
7335 %define PIC_sym(sym) sym
7336 %endif
7337
7338 %if ARCH_X86_32
7339 %if STACK_ALIGNMENT < required_stack_alignment
7340 %assign copy_args 8*4
7341 %else
7342 %assign copy_args 0
7343 %endif
7344 %endif
7345
7346 %macro RELOC_ARGS 0
7347 %if copy_args
7348 mov r0, r0m
7349 mov r1, r1m
7350 mov r2, r2m
7351 mov r3, r3m
7352 mov r5, r5m
7353 mov dstm, r0
7354 mov dsm, r1
7355 mov srcm, r2
7356 mov ssm, r3
7357 mov mxm, r5
7358 mov r0, r6m
7359 mov mym, r0
7360 %endif
7361 %endmacro
7362
7363 %macro BLENDHWDW 2 ; blend high words from dwords, src1, src2
7364 %if cpuflag(sse4)
7365 pblendw %1, %2, 0xAA
7366 %else
7367 pand %2, m10
7368 por %1, %2
7369 %endif
7370 %endmacro
7371
7372 %macro WARP_V 10 ; dst0, dst1, 0, 2, 4, 6, 1, 3, 5, 7
7373 %if ARCH_X86_32
7374 %define m8 m4
7375 %define m9 m5
7376 %define m14 m6
7377 %define m15 m7
7378 %define m11 m7
7379 %endif
7380 %if notcpuflag(ssse3) || ARCH_X86_32
7381 pxor m11, m11
7382 %endif
7383 lea tmp1d, [myq+deltaq*4]
7384 lea tmp2d, [myq+deltaq*1]
7385 shr myd, 10
7386 shr tmp1d, 10
7387 movq m2, [filterq+myq *8] ; a
7388 movq m8, [filterq+tmp1q*8] ; e
7389 lea tmp1d, [tmp2q+deltaq*4]
7390 lea myd, [tmp2q+deltaq*1]
7391 shr tmp2d, 10
7392 shr tmp1d, 10
7393 movq m3, [filterq+tmp2q*8] ; b
7394 movq m0, [filterq+tmp1q*8] ; f
7395 punpcklwd m2, m3
7396 punpcklwd m8, m0
7397 lea tmp1d, [myq+deltaq*4]
7398 lea tmp2d, [myq+deltaq*1]
7399 shr myd, 10
7400 shr tmp1d, 10
7401 movq m0, [filterq+myq *8] ; c
7402 movq m9, [filterq+tmp1q*8] ; g
7403 lea tmp1d, [tmp2q+deltaq*4]
7404 lea myd, [tmp2q+gammaq] ; my += gamma
7405 shr tmp2d, 10
7406 shr tmp1d, 10
7407 movq m3, [filterq+tmp2q*8] ; d
7408 movq m1, [filterq+tmp1q*8] ; h
7409 punpcklwd m0, m3
7410 punpcklwd m9, m1
7411 punpckldq m1, m2, m0
7412 punpckhdq m2, m0
7413 punpcklbw m0, m11, m1 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8
7414 punpckhbw m3, m11, m1 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8
7415 punpcklbw m1, m11, m2 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8
7416 punpckhbw m14, m11, m2 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8
7417 pmaddwd m0, %3
7418 pmaddwd m3, %5
7419 pmaddwd m1, %7
7420 pmaddwd m14, %9
7421 paddd m0, m3
7422 paddd m1, m14
7423 paddd m0, m1
7424 mova %1, m0
7425 %if ARCH_X86_64
7426 SWAP m3, m14
7427 %endif
7428 punpckldq m0, m8, m9
7429 punpckhdq m8, m9
7430 punpcklbw m1, m11, m0 ; e0 e2 f0 f2 g0 g2 h0 h2 << 8
7431 punpckhbw m14, m11, m0 ; e4 e6 f4 f6 g4 g6 h4 h6 << 8
7432 punpcklbw m2, m11, m8 ; e1 e3 f1 f3 g1 g3 h1 h3 << 8
7433 punpckhbw m15, m11, m8 ; e5 e7 f5 f7 g5 g7 h5 h7 << 8
7434 pmaddwd m1, %4
7435 pmaddwd m14, %6
7436 pmaddwd m2, %8
7437 pmaddwd m15, %10
7438 paddd m1, m14
7439 paddd m2, m15
7440 paddd m1, m2
7441 mova %2, m1
7442 %if ARCH_X86_64
7443 SWAP m14, m3
7444 %endif
7445 %endmacro
7446
7447 %if ARCH_X86_64
7448 %define counterd r4d
7449 %else
7450 %if copy_args == 0
7451 %define counterd dword r4m
7452 %else
7453 %define counterd dword [esp+stack_size-4*7]
7454 %endif
7455 %endif
7456
7457 %macro WARP_AFFINE_8X8T 0
7458 %if ARCH_X86_64
7459 cglobal warp_affine_8x8t_8bpc, 6, 14, 16, 0x90, tmp, ts
7460 %else
7461 cglobal warp_affine_8x8t_8bpc, 0, 7, 16, -0x130-copy_args, tmp, ts
7462 %if copy_args
7463 %define tmpm [esp+stack_size-4*1]
7464 %define tsm [esp+stack_size-4*2]
7465 %endif
7466 %endif
7467 call mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).main
7468 .loop:
7469 %if ARCH_X86_32
7470 %define m12 m4
7471 %define m13 m5
7472 %define m14 m6
7473 %define m15 m7
7474 mova m12, [esp+0xC0]
7475 mova m13, [esp+0xD0]
7476 mova m14, [esp+0xE0]
7477 mova m15, [esp+0xF0]
7478 %endif
7479 %if cpuflag(ssse3)
7480 psrad m12, 13
7481 psrad m13, 13
7482 psrad m14, 13
7483 psrad m15, 13
7484 packssdw m12, m13
7485 packssdw m14, m15
7486 mova m13, [PIC_sym(pw_8192)]
7487 pmulhrsw m12, m13 ; (x + (1 << 6)) >> 7
7488 pmulhrsw m14, m13
7489 %else
7490 %if ARCH_X86_32
7491 %define m10 m0
7492 %endif
7493 mova m10, [PIC_sym(pd_16384)]
7494 paddd m12, m10
7495 paddd m13, m10
7496 paddd m14, m10
7497 paddd m15, m10
7498 psrad m12, 15
7499 psrad m13, 15
7500 psrad m14, 15
7501 psrad m15, 15
7502 packssdw m12, m13
7503 packssdw m14, m15
7504 %endif
7505 mova [tmpq+tsq*0], m12
7506 mova [tmpq+tsq*2], m14
7507 dec counterd
7508 jz mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).end
7509 %if ARCH_X86_32
7510 mov tmpm, tmpd
7511 mov r0, [esp+0x100]
7512 mov r1, [esp+0x104]
7513 %endif
7514 call mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).main2
7515 lea tmpq, [tmpq+tsq*4]
7516 jmp .loop
7517 %endmacro
7518
7519 %macro WARP_AFFINE_8X8 0
7520 %if ARCH_X86_64
7521 cglobal warp_affine_8x8_8bpc, 6, 14, 16, 0x90, \
7522 dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \
7523 filter, tmp1, delta, my, gamma
7524 %else
7525 cglobal warp_affine_8x8_8bpc, 0, 7, 16, -0x130-copy_args, \
7526 dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \
7527 filter, tmp1, delta, my, gamma
7528 %define alphaq r0
7529 %define alphad r0
7530 %define alpham [esp+gprsize+0x100]
7531 %define betaq r1
7532 %define betad r1
7533 %define betam [esp+gprsize+0x104]
7534 %define deltaq r0
7535 %define deltad r0
7536 %define deltam [esp+gprsize+0x108]
7537 %define gammaq r1
7538 %define gammad r1
7539 %define gammam [esp+gprsize+0x10C]
7540 %define filterq r3
7541 %define tmp1q r4
7542 %define tmp1d r4
7543 %define tmp1m [esp+gprsize+0x110]
7544 %define myq r5
7545 %define myd r5
7546 %define mym r6m
7547 %if copy_args
7548 %define dstm [esp+stack_size-4*1]
7549 %define dsm [esp+stack_size-4*2]
7550 %define srcm [esp+stack_size-4*3]
7551 %define ssm [esp+stack_size-4*4]
7552 %define mxm [esp+stack_size-4*5]
7553 %define mym [esp+stack_size-4*6]
7554 %endif
7555 %endif
7556 call .main
7557 jmp .start
7558 .loop:
7559 %if ARCH_X86_32
7560 mov dstm, dstd
7561 mov alphad, [esp+0x100]
7562 mov betad, [esp+0x104]
7563 %endif
7564 call .main2
7565 lea dstq, [dstq+dsq*2]
7566 .start:
7567 %if notcpuflag(sse4)
7568 %if cpuflag(ssse3)
7569 %define roundval pw_8192
7570 %else
7571 %define roundval pd_262144
7572 %endif
7573 %if ARCH_X86_64
7574 mova m10, [PIC_sym(roundval)]
7575 %else
7576 %define m10 [PIC_sym(roundval)]
7577 %endif
7578 %endif
7579 %if ARCH_X86_32
7580 %define m12 m5
7581 %define m13 m6
7582 mova m12, [esp+0xC0]
7583 mova m13, [esp+0xD0]
7584 %endif
7585 %if cpuflag(sse4)
7586 %if ARCH_X86_32
7587 %define m11 m4
7588 pxor m11, m11
7589 %endif
7590 psrad m12, 18
7591 psrad m13, 18
7592 packusdw m12, m13
7593 pavgw m12, m11 ; (x + (1 << 10)) >> 11
7594 %else
7595 %if cpuflag(ssse3)
7596 psrad m12, 17
7597 psrad m13, 17
7598 packssdw m12, m13
7599 pmulhrsw m12, m10
7600 %else
7601 paddd m12, m10
7602 paddd m13, m10
7603 psrad m12, 19
7604 psrad m13, 19
7605 packssdw m12, m13
7606 %endif
7607 %endif
7608 %if ARCH_X86_32
7609 %define m14 m6
7610 %define m15 m7
7611 mova m14, [esp+0xE0]
7612 mova m15, [esp+0xF0]
7613 %endif
7614 %if cpuflag(sse4)
7615 psrad m14, 18
7616 psrad m15, 18
7617 packusdw m14, m15
7618 pavgw m14, m11 ; (x + (1 << 10)) >> 11
7619 %else
7620 %if cpuflag(ssse3)
7621 psrad m14, 17
7622 psrad m15, 17
7623 packssdw m14, m15
7624 pmulhrsw m14, m10
7625 %else
7626 paddd m14, m10
7627 paddd m15, m10
7628 psrad m14, 19
7629 psrad m15, 19
7630 packssdw m14, m15
7631 %endif
7632 %endif
7633 packuswb m12, m14
7634 movq [dstq+dsq*0], m12
7635 movhps [dstq+dsq*1], m12
7636 dec counterd
7637 jg .loop
7638 .end:
7639 RET
7640 ALIGN function_align
7641 .main:
7642 %assign stack_offset stack_offset+gprsize
7643 %if ARCH_X86_32
7644 %assign stack_size stack_size+4
7645 %if copy_args
7646 %assign stack_offset stack_offset-4
7647 %endif
7648 RELOC_ARGS
7649 LEA PIC_reg, $$
7650 %define PIC_mem [esp+gprsize+0x114]
7651 mov abcdd, abcdm
7652 %if copy_args == 0
7653 mov ssd, ssm
7654 mov mxd, mxm
7655 %endif
7656 mov PIC_mem, PIC_reg
7657 mov srcd, srcm
7658 %endif
7659 movsx deltad, word [abcdq+2*2]
7660 movsx gammad, word [abcdq+2*3]
7661 lea tmp1d, [deltaq*3]
7662 sub gammad, tmp1d ; gamma -= delta*3
7663 SAVE_DELTA_GAMMA
7664 %if ARCH_X86_32
7665 mov abcdd, abcdm
7666 %endif
7667 movsx alphad, word [abcdq+2*0]
7668 movsx betad, word [abcdq+2*1]
7669 lea tmp1q, [ssq*3+3]
7670 add mxd, 512+(64<<10)
7671 lea tmp2d, [alphaq*3]
7672 sub srcq, tmp1q ; src -= src_stride*3 + 3
7673 %if ARCH_X86_32
7674 mov srcm, srcd
7675 mov PIC_reg, PIC_mem
7676 %endif
7677 sub betad, tmp2d ; beta -= alpha*3
7678 lea filterq, [PIC_sym(mc_warp_filter2)]
7679 %if ARCH_X86_64
7680 mov myd, r6m
7681 %if cpuflag(ssse3)
7682 pxor m11, m11
7683 %endif
7684 %endif
7685 call .h
7686 psrld m2, m0, 16
7687 psrld m3, m1, 16
7688 %if ARCH_X86_32
7689 %if notcpuflag(ssse3)
7690 mova [esp+gprsize+0x00], m2
7691 %endif
7692 mova [esp+gprsize+0x10], m3
7693 %endif
7694 call .h
7695 psrld m4, m0, 16
7696 psrld m5, m1, 16
7697 %if ARCH_X86_32
7698 mova [esp+gprsize+0x20], m4
7699 mova [esp+gprsize+0x30], m5
7700 %endif
7701 call .h
7702 %if ARCH_X86_64
7703 %define blendmask [rsp+gprsize+0x80]
7704 %else
7705 %if notcpuflag(ssse3)
7706 mova m2, [esp+gprsize+0x00]
7707 %endif
7708 mova m3, [esp+gprsize+0x10]
7709 %define blendmask [esp+gprsize+0x120]
7710 %define m10 m7
7711 %endif
7712 pcmpeqd m10, m10
7713 pslld m10, 16
7714 mova blendmask, m10
7715 BLENDHWDW m2, m0 ; 0
7716 BLENDHWDW m3, m1 ; 2
7717 mova [rsp+gprsize+0x00], m2
7718 mova [rsp+gprsize+0x10], m3
7719 call .h
7720 %if ARCH_X86_32
7721 mova m4, [esp+gprsize+0x20]
7722 mova m5, [esp+gprsize+0x30]
7723 %endif
7724 mova m10, blendmask
7725 BLENDHWDW m4, m0 ; 1
7726 BLENDHWDW m5, m1 ; 3
7727 mova [rsp+gprsize+0x20], m4
7728 mova [rsp+gprsize+0x30], m5
7729 call .h
7730 %if ARCH_X86_32
7731 %if notcpuflag(ssse3)
7732 mova m2, [esp+gprsize+0x00]
7733 %endif
7734 mova m3, [esp+gprsize+0x10]
7735 %define m10 m5
7736 %endif
7737 psrld m6, m2, 16
7738 psrld m7, m3, 16
7739 mova m10, blendmask
7740 BLENDHWDW m6, m0 ; 2
7741 BLENDHWDW m7, m1 ; 4
7742 mova [rsp+gprsize+0x40], m6
7743 mova [rsp+gprsize+0x50], m7
7744 call .h
7745 %if ARCH_X86_32
7746 mova m4, [esp+gprsize+0x20]
7747 mova m5, [esp+gprsize+0x30]
7748 %endif
7749 psrld m2, m4, 16
7750 psrld m3, m5, 16
7751 mova m10, blendmask
7752 BLENDHWDW m2, m0 ; 3
7753 BLENDHWDW m3, m1 ; 5
7754 mova [rsp+gprsize+0x60], m2
7755 mova [rsp+gprsize+0x70], m3
7756 call .h
7757 %if ARCH_X86_32
7758 mova m6, [esp+gprsize+0x40]
7759 mova m7, [esp+gprsize+0x50]
7760 %define m10 m7
7761 %endif
7762 psrld m4, m6, 16
7763 psrld m5, m7, 16
7764 mova m10, blendmask
7765 BLENDHWDW m4, m0 ; 4
7766 BLENDHWDW m5, m1 ; 6
7767 %if ARCH_X86_64
7768 add myd, 512+(64<<10)
7769 mova m6, m2
7770 mova m7, m3
7771 %else
7772 mova [esp+gprsize+0x80], m4
7773 mova [esp+gprsize+0x90], m5
7774 add dword mym, 512+(64<<10)
7775 %endif
7776 mov counterd, 4
7777 SAVE_ALPHA_BETA
7778 .main2:
7779 call .h
7780 %if ARCH_X86_32
7781 mova m6, [esp+gprsize+0x60]
7782 mova m7, [esp+gprsize+0x70]
7783 %define m10 m5
7784 %endif
7785 psrld m6, 16
7786 psrld m7, 16
7787 mova m10, blendmask
7788 BLENDHWDW m6, m0 ; 5
7789 BLENDHWDW m7, m1 ; 7
7790 %if ARCH_X86_64
7791 WARP_V m12, m13, [rsp+gprsize+0x00], [rsp+gprsize+0x10], \
7792 m4, m5, \
7793 [rsp+gprsize+0x20], [rsp+gprsize+0x30], \
7794 m6, m7
7795 %else
7796 mova [esp+gprsize+0xA0], m6
7797 mova [esp+gprsize+0xB0], m7
7798 LOAD_DELTA_GAMMA_MY
7799 WARP_V [esp+gprsize+0xC0], [esp+gprsize+0xD0], \
7800 [esp+gprsize+0x00], [esp+gprsize+0x10], \
7801 [esp+gprsize+0x80], [esp+gprsize+0x90], \
7802 [esp+gprsize+0x20], [esp+gprsize+0x30], \
7803 [esp+gprsize+0xA0], [esp+gprsize+0xB0]
7804 LOAD_ALPHA_BETA_MX
7805 %endif
7806 call .h
7807 mova m2, [rsp+gprsize+0x40]
7808 mova m3, [rsp+gprsize+0x50]
7809 %if ARCH_X86_32
7810 mova m4, [rsp+gprsize+0x80]
7811 mova m5, [rsp+gprsize+0x90]
7812 %define m10 m7
7813 %endif
7814 mova [rsp+gprsize+0x00], m2
7815 mova [rsp+gprsize+0x10], m3
7816 mova [rsp+gprsize+0x40], m4
7817 mova [rsp+gprsize+0x50], m5
7818 psrld m4, 16
7819 psrld m5, 16
7820 mova m10, blendmask
7821 BLENDHWDW m4, m0 ; 6
7822 BLENDHWDW m5, m1 ; 8
7823 %if ARCH_X86_64
7824 WARP_V m14, m15, [rsp+gprsize+0x20], [rsp+gprsize+0x30], \
7825 m6, m7, \
7826 [rsp+gprsize+0x00], [rsp+gprsize+0x10], \
7827 m4, m5
7828 %else
7829 mova [esp+gprsize+0x80], m4
7830 mova [esp+gprsize+0x90], m5
7831 LOAD_DELTA_GAMMA_MY
7832 WARP_V [esp+gprsize+0xE0], [esp+gprsize+0xF0], \
7833 [esp+gprsize+0x20], [esp+gprsize+0x30], \
7834 [esp+gprsize+0xA0], [esp+gprsize+0xB0], \
7835 [esp+gprsize+0x00], [esp+gprsize+0x10], \
7836 [esp+gprsize+0x80], [esp+gprsize+0x90]
7837 mov mym, myd
7838 mov dstd, dstm
7839 mov dsd, dsm
7840 mov mxd, mxm
7841 %endif
7842 mova m2, [rsp+gprsize+0x60]
7843 mova m3, [rsp+gprsize+0x70]
7844 %if ARCH_X86_32
7845 mova m6, [esp+gprsize+0xA0]
7846 mova m7, [esp+gprsize+0xB0]
7847 %endif
7848 mova [rsp+gprsize+0x20], m2
7849 mova [rsp+gprsize+0x30], m3
7850 mova [rsp+gprsize+0x60], m6
7851 mova [rsp+gprsize+0x70], m7
7852 ret
7853 ALIGN function_align
7854 .h:
7855 %if ARCH_X86_32
7856 %define m8 m3
7857 %define m9 m4
7858 %define m10 m5
7859 %define m14 m6
7860 %define m15 m7
7861 %endif
7862 lea tmp1d, [mxq+alphaq*4]
7863 lea tmp2d, [mxq+alphaq*1]
7864 %if ARCH_X86_32
7865 %assign stack_offset stack_offset+4
7866 %assign stack_size stack_size+4
7867 %define PIC_mem [esp+gprsize*2+0x114]
7868 mov PIC_mem, PIC_reg
7869 mov srcd, srcm
7870 %endif
7871 movu m10, [srcq]
7872 %if ARCH_X86_32
7873 add srcd, ssm
7874 mov srcm, srcd
7875 mov PIC_reg, PIC_mem
7876 %else
7877 add srcq, ssq
7878 %endif
7879 shr mxd, 10
7880 shr tmp1d, 10
7881 movq m1, [filterq+mxq *8] ; 0 X
7882 movq m8, [filterq+tmp1q*8] ; 4 X
7883 lea tmp1d, [tmp2q+alphaq*4]
7884 lea mxd, [tmp2q+alphaq*1]
7885 shr tmp2d, 10
7886 shr tmp1d, 10
7887 movhps m1, [filterq+tmp2q*8] ; 0 1
7888 movhps m8, [filterq+tmp1q*8] ; 4 5
7889 lea tmp1d, [mxq+alphaq*4]
7890 lea tmp2d, [mxq+alphaq*1]
7891 shr mxd, 10
7892 shr tmp1d, 10
7893 %if cpuflag(ssse3)
7894 movq m14, [filterq+mxq *8] ; 2 X
7895 movq m9, [filterq+tmp1q*8] ; 6 X
7896 lea tmp1d, [tmp2q+alphaq*4]
7897 lea mxd, [tmp2q+betaq] ; mx += beta
7898 shr tmp2d, 10
7899 shr tmp1d, 10
7900 movhps m14, [filterq+tmp2q*8] ; 2 3
7901 movhps m9, [filterq+tmp1q*8] ; 6 7
7902 pshufb m0, m10, [PIC_sym(warp_8x8_shufA)]
7903 pmaddubsw m0, m1
7904 pshufb m1, m10, [PIC_sym(warp_8x8_shufB)]
7905 pmaddubsw m1, m8
7906 pshufb m15, m10, [PIC_sym(warp_8x8_shufC)]
7907 pmaddubsw m15, m14
7908 pshufb m10, m10, [PIC_sym(warp_8x8_shufD)]
7909 pmaddubsw m10, m9
7910 phaddw m0, m15
7911 phaddw m1, m10
7912 %else
7913 %if ARCH_X86_32
7914 %define m11 m2
7915 %endif
7916 pcmpeqw m0, m0
7917 psrlw m14, m0, 8
7918 psrlw m15, m10, 8 ; 01 03 05 07 09 11 13 15
7919 pand m14, m10 ; 00 02 04 06 08 10 12 14
7920 packuswb m14, m15 ; 00 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15
7921 psrldq m9, m0, 4
7922 pshufd m0, m14, q0220
7923 pand m0, m9
7924 psrldq m14, 1 ; 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __
7925 pslldq m15, m14, 12
7926 por m0, m15 ; shufA
7927 psrlw m15, m0, 8
7928 psraw m11, m1, 8
7929 psllw m0, 8
7930 psllw m1, 8
7931 psrlw m0, 8
7932 psraw m1, 8
7933 pmullw m15, m11
7934 pmullw m0, m1
7935 paddw m0, m15 ; pmaddubsw m0, m1
7936 pshufd m15, m14, q0220
7937 pand m15, m9
7938 psrldq m14, 1 ; 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __
7939 pslldq m1, m14, 12
7940 por m15, m1 ; shufC
7941 pshufd m1, m14, q0220
7942 pand m1, m9
7943 psrldq m14, 1 ; 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __
7944 pslldq m11, m14, 12
7945 por m1, m11 ; shufB
7946 pshufd m10, m14, q0220
7947 pand m10, m9
7948 psrldq m14, 1 ; 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ __
7949 pslldq m14, m14, 12
7950 por m10, m14 ; shufD
7951 psrlw m9, m1, 8
7952 psraw m11, m8, 8
7953 psllw m1, 8
7954 psllw m8, 8
7955 psrlw m1, 8
7956 psraw m8, 8
7957 pmullw m9, m11
7958 pmullw m1, m8
7959 paddw m1, m9 ; pmaddubsw m1, m8
7960 movq m14, [filterq+mxq *8] ; 2 X
7961 movq m9, [filterq+tmp1q*8] ; 6 X
7962 lea tmp1d, [tmp2q+alphaq*4]
7963 lea mxd, [tmp2q+betaq] ; mx += beta
7964 shr tmp2d, 10
7965 shr tmp1d, 10
7966 movhps m14, [filterq+tmp2q*8] ; 2 3
7967 movhps m9, [filterq+tmp1q*8] ; 6 7
7968 psrlw m8, m15, 8
7969 psraw m11, m14, 8
7970 psllw m15, 8
7971 psllw m14, 8
7972 psrlw m15, 8
7973 psraw m14, 8
7974 pmullw m8, m11
7975 pmullw m15, m14
7976 paddw m15, m8 ; pmaddubsw m15, m14
7977 psrlw m8, m10, 8
7978 psraw m11, m9, 8
7979 psllw m10, 8
7980 psllw m9, 8
7981 psrlw m10, 8
7982 psraw m9, 8
7983 pmullw m8, m11
7984 pmullw m10, m9
7985 paddw m10, m8 ; pmaddubsw m10, m9
7986 pslld m8, m0, 16
7987 pslld m9, m1, 16
7988 pslld m14, m15, 16
7989 pslld m11, m10, 16
7990 paddw m0, m8
7991 paddw m1, m9
7992 paddw m15, m14
7993 paddw m10, m11
7994 psrad m0, 16
7995 psrad m1, 16
7996 psrad m15, 16
7997 psrad m10, 16
7998 packssdw m0, m15 ; phaddw m0, m15
7999 packssdw m1, m10 ; phaddw m1, m10
8000 %endif
8001 mova m14, [PIC_sym(pw_8192)]
8002 mova m9, [PIC_sym(pd_32768)]
8003 pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13
8004 pmaddwd m1, m14
8005 paddd m0, m9 ; rounded 14-bit result in upper 16 bits of dword
8006 paddd m1, m9
8007 ret
8008 %endmacro
8009
8010 %if WIN64
8011 DECLARE_REG_TMP 6, 4
8012 %else
8013 DECLARE_REG_TMP 6, 7
8014 %endif
8015
8016 %macro BIDIR_FN 1 ; op
8017 %1 0
8018 lea stride3q, [strideq*3]
8019 jmp wq
8020 .w4_loop:
8021 %1_INC_PTR 2
8022 %1 0
8023 lea dstq, [dstq+strideq*4]
8024 .w4: ; tile 4x
8025 movd [dstq ], m0 ; copy dw[0]
8026 pshuflw m1, m0, q1032 ; swap dw[1] and dw[0]
8027 movd [dstq+strideq*1], m1 ; copy dw[1]
8028 punpckhqdq m0, m0 ; swap dw[3,2] with dw[1,0]
8029 movd [dstq+strideq*2], m0 ; dw[2]
8030 psrlq m0, 32 ; shift right in dw[3]
8031 movd [dstq+stride3q ], m0 ; copy
8032 sub hd, 4
8033 jg .w4_loop
8034 RET
8035 .w8_loop:
8036 %1_INC_PTR 2
8037 %1 0
8038 lea dstq, [dstq+strideq*2]
8039 .w8:
8040 movq [dstq ], m0
8041 movhps [dstq+strideq*1], m0
8042 sub hd, 2
8043 jg .w8_loop
8044 RET
8045 .w16_loop:
8046 %1_INC_PTR 2
8047 %1 0
8048 lea dstq, [dstq+strideq]
8049 .w16:
8050 mova [dstq ], m0
8051 dec hd
8052 jg .w16_loop
8053 RET
8054 .w32_loop:
8055 %1_INC_PTR 4
8056 %1 0
8057 lea dstq, [dstq+strideq]
8058 .w32:
8059 mova [dstq ], m0
8060 %1 2
8061 mova [dstq + 16 ], m0
8062 dec hd
8063 jg .w32_loop
8064 RET
8065 .w64_loop:
8066 %1_INC_PTR 8
8067 %1 0
8068 add dstq, strideq
8069 .w64:
8070 %assign i 0
8071 %rep 4
8072 mova [dstq + i*16 ], m0
8073 %assign i i+1
8074 %if i < 4
8075 %1 2*i
8076 %endif
8077 %endrep
8078 dec hd
8079 jg .w64_loop
8080 RET
8081 .w128_loop:
8082 %1_INC_PTR 16
8083 %1 0
8084 add dstq, strideq
8085 .w128:
8086 %assign i 0
8087 %rep 8
8088 mova [dstq + i*16 ], m0
8089 %assign i i+1
8090 %if i < 8
8091 %1 2*i
8092 %endif
8093 %endrep
8094 dec hd
8095 jg .w128_loop
8096 RET
8097 %endmacro
8098
8099 %macro AVG 1 ; src_offset
8100 ; writes AVG of tmp1 tmp2 uint16 coeffs into uint8 pixel
8101 mova m0, [tmp1q+(%1+0)*mmsize] ; load 8 coef(2bytes) from tmp1
8102 paddw m0, [tmp2q+(%1+0)*mmsize] ; load/add 8 coef(2bytes) tmp2
8103 mova m1, [tmp1q+(%1+1)*mmsize]
8104 paddw m1, [tmp2q+(%1+1)*mmsize]
8105 pmulhrsw m0, m2
8106 pmulhrsw m1, m2
8107 packuswb m0, m1 ; pack/trunc 16 bits from m0 & m1 to 8 bit
8108 %endmacro
8109
8110 %macro AVG_INC_PTR 1
8111 add tmp1q, %1*mmsize
8112 add tmp2q, %1*mmsize
8113 %endmacro
8114
8115 cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3
8116 LEA r6, avg_ssse3_table
8117 tzcnt wd, wm ; leading zeros
8118 movifnidn hd, hm ; move h(stack) to h(register) if not already that register
8119 movsxd wq, dword [r6+wq*4] ; push table entry matching the tile width (tzcnt) in widen reg
8120 mova m2, [pw_1024+r6-avg_ssse3_table] ; fill m2 with shift/align
8121 add wq, r6
8122 BIDIR_FN AVG
8123
8124 %macro W_AVG 1 ; src_offset
8125 ; (a * weight + b * (16 - weight) + 128) >> 8
8126 ; = ((a - b) * weight + (b << 4) + 128) >> 8
8127 ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4
8128 ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4
8129 mova m2, [tmp1q+(%1+0)*mmsize]
8130 mova m0, m2
8131 psubw m2, [tmp2q+(%1+0)*mmsize]
8132 mova m3, [tmp1q+(%1+1)*mmsize]
8133 mova m1, m3
8134 psubw m3, [tmp2q+(%1+1)*mmsize]
8135 pmulhw m2, m4
8136 pmulhw m3, m4
8137 paddw m0, m2
8138 paddw m1, m3
8139 pmulhrsw m0, m5
8140 pmulhrsw m1, m5
8141 packuswb m0, m1
8142 %endmacro
8143
8144 %define W_AVG_INC_PTR AVG_INC_PTR
8145
8146 cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
8147 LEA r6, w_avg_ssse3_table
8148 tzcnt wd, wm
8149 movd m4, r6m
8150 movifnidn hd, hm
8151 pxor m0, m0
8152 movsxd wq, dword [r6+wq*4]
8153 mova m5, [pw_2048+r6-w_avg_ssse3_table]
8154 pshufb m4, m0
8155 psllw m4, 12 ; (weight-16) << 12 when interpreted as signed
8156 add wq, r6
8157 cmp dword r6m, 7
8158 jg .weight_gt7
8159 mov r6, tmp1q
8160 psubw m0, m4
8161 mov tmp1q, tmp2q
8162 mova m4, m0 ; -weight
8163 mov tmp2q, r6
8164 .weight_gt7:
8165 BIDIR_FN W_AVG
8166
8167 %macro MASK 1 ; src_offset
8168 ; (a * m + b * (64 - m) + 512) >> 10
8169 ; = ((a - b) * m + (b << 6) + 512) >> 10
8170 ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4
8171 mova m3, [maskq+(%1+0)*(mmsize/2)]
8172 mova m0, [tmp2q+(%1+0)*mmsize] ; b
8173 psubw m1, m0, [tmp1q+(%1+0)*mmsize] ; b - a
8174 mova m6, m3 ; m
8175 psubb m3, m4, m6 ; -m
8176 paddw m1, m1 ; (b - a) << 1
8177 paddb m3, m3 ; -m << 1
8178 punpcklbw m2, m4, m3 ; -m << 9 (<< 8 when ext as uint16)
8179 pmulhw m1, m2 ; (-m * (b - a)) << 10
8180 paddw m0, m1 ; + b
8181 mova m1, [tmp2q+(%1+1)*mmsize] ; b
8182 psubw m2, m1, [tmp1q+(%1+1)*mmsize] ; b - a
8183 paddw m2, m2 ; (b - a) << 1
8184 mova m6, m3 ; (-m << 1)
8185 punpckhbw m3, m4, m6 ; (-m << 9)
8186 pmulhw m2, m3 ; (-m << 9)
8187 paddw m1, m2 ; (-m * (b - a)) << 10
8188 pmulhrsw m0, m5 ; round
8189 pmulhrsw m1, m5 ; round
8190 packuswb m0, m1 ; interleave 16 -> 8
8191 %endmacro
8192
8193 %macro MASK_INC_PTR 1
8194 add maskq, %1*mmsize/2
8195 add tmp1q, %1*mmsize
8196 add tmp2q, %1*mmsize
8197 %endmacro
8198
8199 %if ARCH_X86_64
8200 cglobal mask_8bpc, 4, 8, 7, dst, stride, tmp1, tmp2, w, h, mask, stride3
8201 movifnidn hd, hm
8202 %else
8203 cglobal mask_8bpc, 4, 7, 7, dst, stride, tmp1, tmp2, w, mask, stride3
8204 %define hd dword r5m
8205 %endif
8206 %define base r6-mask_ssse3_table
8207 LEA r6, mask_ssse3_table
8208 tzcnt wd, wm
8209 movsxd wq, dword [r6+wq*4]
8210 pxor m4, m4
8211 mova m5, [base+pw_2048]
8212 add wq, r6
8213 mov maskq, r6m
8214 BIDIR_FN MASK
8215 %undef hd
8216
8217 %macro W_MASK_420_END 1-*
8218 %rep %0
8219 call .main
8220 paddw m2, [maskq+16*%1]
8221 mova [maskq+16*%1], m2
8222 mova [dstq+strideq*1+16*(2*%1+0)], m0
8223 call .main
8224 psubw m3, m7, m2
8225 psubw m1, m7, [maskq+16*%1]
8226 psubw m3, [dstq+strideq*1+16*(2*%1+1)]
8227 psrlw m1, 2
8228 psrlw m3, 2
8229 packuswb m1, m3
8230 mova [maskq+16*%1], m1
8231 mova [dstq+strideq*1+16*(2*%1+1)], m0
8232 %rotate 1
8233 %endrep
8234 %endmacro
8235
8236 %if UNIX64
8237 DECLARE_REG_TMP 7
8238 %else
8239 DECLARE_REG_TMP 5
8240 %endif
8241
8242 cglobal w_mask_420_8bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask
8243 %define base t0-w_mask_420_ssse3_table
8244 LEA t0, w_mask_420_ssse3_table
8245 tzcnt wd, wm
8246 mov r6d, r7m ; sign
8247 sub tmp2q, tmp1q
8248 movsxd wq, [t0+wq*4]
8249 mova m6, [base+pw_2048]
8250 movddup m7, [base+wm_420_sign+r6*8] ; 258 - sign
8251 add wq, t0
8252 %if ARCH_X86_64
8253 mova m8, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
8254 movifnidn hd, hm
8255 %else
8256 %define m8 [base+pw_6903]
8257 %define hd dword hm
8258 %endif
8259 mov maskq, maskmp
8260 call .main
8261 jmp wq
8262 .w4_loop:
8263 call .main
8264 add maskq, 4
8265 lea dstq, [dstq+strideq*2]
8266 .w4:
8267 pshufd m3, m2, q2020
8268 pshufd m2, m2, q3131
8269 psubw m1, m7, m3
8270 psubw m1, m2
8271 psrlw m1, 2
8272 packuswb m1, m1
8273 movd [maskq], m1
8274 movd [dstq+strideq*0], m0
8275 pshuflw m1, m0, q1032
8276 movd [dstq+strideq*1], m1
8277 punpckhqdq m0, m0
8278 lea dstq, [dstq+strideq*2]
8279 movd [dstq+strideq*0], m0
8280 pshuflw m1, m0, q1032
8281 movd [dstq+strideq*1], m1
8282 sub hd, 4
8283 jg .w4_loop
8284 RET
8285 .w8_loop:
8286 call .main
8287 add maskq, 4
8288 lea dstq, [dstq+strideq*2]
8289 .w8:
8290 movhlps m3, m2
8291 psubw m1, m7, m2
8292 psubw m1, m3
8293 psrlw m1, 2
8294 packuswb m1, m1
8295 movd [maskq], m1
8296 movq [dstq+strideq*0], m0
8297 movhps [dstq+strideq*1], m0
8298 sub hd, 2
8299 jg .w8_loop
8300 RET
8301 .w16_loop:
8302 call .main
8303 add maskq, 8
8304 lea dstq, [dstq+strideq*2]
8305 .w16:
8306 mova [dstq+strideq*1], m2
8307 mova [dstq+strideq*0], m0
8308 call .main
8309 psubw m1, m7, [dstq+strideq*1]
8310 psubw m1, m2
8311 psrlw m1, 2
8312 packuswb m1, m1
8313 movq [maskq], m1
8314 mova [dstq+strideq*1], m0
8315 sub hd, 2
8316 jg .w16_loop
8317 RET
8318 .w32_loop:
8319 call .main
8320 add maskq, 16
8321 lea dstq, [dstq+strideq*2]
8322 .w32:
8323 mova [maskq], m2
8324 mova [dstq+strideq*0+16*0], m0
8325 call .main
8326 mova [dstq+strideq*1+16*1], m2
8327 mova [dstq+strideq*0+16*1], m0
8328 W_MASK_420_END 0
8329 sub hd, 2
8330 jg .w32_loop
8331 RET
8332 .w64_loop:
8333 call .main
8334 add maskq, 16*2
8335 lea dstq, [dstq+strideq*2]
8336 .w64:
8337 mova [maskq+16*0], m2
8338 mova [dstq+strideq*0+16*0], m0
8339 call .main
8340 mova [dstq+strideq*1+16*1], m2
8341 mova [dstq+strideq*0+16*1], m0
8342 call .main
8343 mova [maskq+16*1], m2
8344 mova [dstq+strideq*0+16*2], m0
8345 call .main
8346 mova [dstq+strideq*1+16*3], m2
8347 mova [dstq+strideq*0+16*3], m0
8348 W_MASK_420_END 0, 1
8349 sub hd, 2
8350 jg .w64_loop
8351 RET
8352 .w128_loop:
8353 call .main
8354 add maskq, 16*4
8355 lea dstq, [dstq+strideq*2]
8356 .w128:
8357 mova [maskq+16*0], m2
8358 mova [dstq+strideq*0+16*0], m0
8359 call .main
8360 mova [dstq+strideq*1+16*1], m2
8361 mova [dstq+strideq*0+16*1], m0
8362 call .main
8363 mova [maskq+16*1], m2
8364 mova [dstq+strideq*0+16*2], m0
8365 call .main
8366 mova [dstq+strideq*1+16*3], m2
8367 mova [dstq+strideq*0+16*3], m0
8368 call .main
8369 mova [maskq+16*2], m2
8370 mova [dstq+strideq*0+16*4], m0
8371 call .main
8372 mova [dstq+strideq*1+16*5], m2
8373 mova [dstq+strideq*0+16*5], m0
8374 call .main
8375 mova [maskq+16*3], m2
8376 mova [dstq+strideq*0+16*6], m0
8377 call .main
8378 mova [dstq+strideq*1+16*7], m2
8379 mova [dstq+strideq*0+16*7], m0
8380 W_MASK_420_END 0, 1, 2, 3
8381 sub hd, 2
8382 jg .w128_loop
8383 RET
8384 ALIGN function_align
8385 .main:
8386 mova m0, [tmp1q +16*0]
8387 mova m3, [tmp1q+tmp2q+16*0]
8388 mova m1, [tmp1q +16*1]
8389 mova m4, [tmp1q+tmp2q+16*1]
8390 add tmp1q, 16*2
8391 psubw m3, m0
8392 psubw m4, m1
8393 pabsw m5, m3
8394 psubusw m2, m8, m5
8395 psrlw m2, 8 ; 64 - m
8396 psllw m5, m2, 10
8397 pmulhw m3, m5
8398 pabsw m5, m4
8399 paddw m0, m3
8400 psubusw m3, m8, m5
8401 psrlw m3, 8
8402 phaddw m2, m3
8403 psllw m3, 10
8404 pmulhw m4, m3
8405 paddw m1, m4
8406 pmulhrsw m0, m6
8407 pmulhrsw m1, m6
8408 packuswb m0, m1
8409 ret
8410
8411 %macro W_MASK_422_BACKUP 1 ; mask_offset
8412 %if ARCH_X86_64
8413 mova m10, m2
8414 %else
8415 mova [maskq+16*%1], m2
8416 %endif
8417 %endmacro
8418
8419 %macro W_MASK_422_END 1 ; mask_offset
8420 %if ARCH_X86_64
8421 packuswb m10, m2
8422 psubb m1, m7, m10
8423 pavgb m1, m9
8424 %else
8425 mova m3, [maskq+16*%1]
8426 packuswb m3, m2
8427 pxor m2, m2
8428 psubb m1, m7, m3
8429 pavgb m1, m2
8430 %endif
8431 mova [maskq+16*%1], m1
8432 %endmacro
8433
8434 cglobal w_mask_422_8bpc, 4, 7, 11, dst, stride, tmp1, tmp2, w, h, mask
8435 %define base t0-w_mask_422_ssse3_table
8436 LEA t0, w_mask_422_ssse3_table
8437 tzcnt wd, wm
8438 mov r6d, r7m ; sign
8439 sub tmp2q, tmp1q
8440 movsxd wq, [t0+wq*4]
8441 mova m6, [base+pw_2048]
8442 movddup m7, [base+wm_422_sign+r6*8] ; 128 - sign
8443 add wq, t0
8444 %if ARCH_X86_64
8445 mova m8, [base+pw_6903]
8446 pxor m9, m9
8447 movifnidn hd, hm
8448 %else
8449 add t0, w_mask_420_ssse3_table-w_mask_422_ssse3_table
8450 %define hd dword hm
8451 %endif
8452 mov maskq, maskmp
8453 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8454 jmp wq
8455 .w4_loop:
8456 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8457 add maskq, 8
8458 lea dstq, [dstq+strideq*2]
8459 .w4:
8460 packuswb m2, m2
8461 psubb m1, m7, m2
8462 %if ARCH_X86_64
8463 pavgb m1, m9
8464 %else
8465 pxor m2, m2
8466 pavgb m1, m2
8467 %endif
8468 movq [maskq], m1
8469 movd [dstq+strideq*0], m0
8470 pshuflw m1, m0, q1032
8471 movd [dstq+strideq*1], m1
8472 punpckhqdq m0, m0
8473 lea dstq, [dstq+strideq*2]
8474 movd [dstq+strideq*0], m0
8475 pshuflw m1, m0, q1032
8476 movd [dstq+strideq*1], m1
8477 sub hd, 4
8478 jg .w4_loop
8479 RET
8480 .w8_loop:
8481 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8482 add maskq, 16
8483 lea dstq, [dstq+strideq*2]
8484 .w8:
8485 W_MASK_422_BACKUP 0
8486 movq [dstq+strideq*0], m0
8487 movhps [dstq+strideq*1], m0
8488 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8489 lea dstq, [dstq+strideq*2]
8490 W_MASK_422_END 0
8491 movq [dstq+strideq*0], m0
8492 movhps [dstq+strideq*1], m0
8493 sub hd, 4
8494 jg .w8_loop
8495 RET
8496 .w16_loop:
8497 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8498 add maskq, 16
8499 lea dstq, [dstq+strideq*2]
8500 .w16:
8501 W_MASK_422_BACKUP 0
8502 mova [dstq+strideq*0], m0
8503 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8504 W_MASK_422_END 0
8505 mova [dstq+strideq*1], m0
8506 sub hd, 2
8507 jg .w16_loop
8508 RET
8509 .w32_loop:
8510 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8511 add maskq, 16
8512 add dstq, strideq
8513 .w32:
8514 W_MASK_422_BACKUP 0
8515 mova [dstq+16*0], m0
8516 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8517 W_MASK_422_END 0
8518 mova [dstq+16*1], m0
8519 dec hd
8520 jg .w32_loop
8521 RET
8522 .w64_loop:
8523 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8524 add maskq, 16*2
8525 add dstq, strideq
8526 .w64:
8527 W_MASK_422_BACKUP 0
8528 mova [dstq+16*0], m0
8529 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8530 W_MASK_422_END 0
8531 mova [dstq+16*1], m0
8532 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8533 W_MASK_422_BACKUP 1
8534 mova [dstq+16*2], m0
8535 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8536 W_MASK_422_END 1
8537 mova [dstq+16*3], m0
8538 dec hd
8539 jg .w64_loop
8540 RET
8541 .w128_loop:
8542 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8543 add maskq, 16*4
8544 add dstq, strideq
8545 .w128:
8546 W_MASK_422_BACKUP 0
8547 mova [dstq+16*0], m0
8548 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8549 W_MASK_422_END 0
8550 mova [dstq+16*1], m0
8551 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8552 W_MASK_422_BACKUP 1
8553 mova [dstq+16*2], m0
8554 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8555 W_MASK_422_END 1
8556 mova [dstq+16*3], m0
8557 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8558 W_MASK_422_BACKUP 2
8559 mova [dstq+16*4], m0
8560 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8561 W_MASK_422_END 2
8562 mova [dstq+16*5], m0
8563 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8564 W_MASK_422_BACKUP 3
8565 mova [dstq+16*6], m0
8566 call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main
8567 W_MASK_422_END 3
8568 mova [dstq+16*7], m0
8569 dec hd
8570 jg .w128_loop
8571 RET
8572
8573 cglobal w_mask_444_8bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask
8574 %define base t0-w_mask_444_ssse3_table
8575 LEA t0, w_mask_444_ssse3_table
8576 tzcnt wd, wm
8577 mov maskq, maskmp
8578 sub tmp2q, tmp1q
8579 movsxd wq, [t0+wq*4]
8580 mova m6, [base+pw_6903]
8581 mova m7, [base+pw_2048]
8582 add wq, t0
8583 %if ARCH_X86_64
8584 mova m8, [base+pb_64]
8585 movifnidn hd, hm
8586 %else
8587 %define m8 [base+pb_64]
8588 %define hd dword hm
8589 %endif
8590 call .main
8591 jmp wq
8592 .w4_loop:
8593 call .main
8594 lea dstq, [dstq+strideq*2]
8595 .w4:
8596 movd [dstq+strideq*0], m0
8597 pshuflw m1, m0, q1032
8598 movd [dstq+strideq*1], m1
8599 punpckhqdq m0, m0
8600 lea dstq, [dstq+strideq*2]
8601 movd [dstq+strideq*0], m0
8602 pshuflw m1, m0, q1032
8603 movd [dstq+strideq*1], m1
8604 sub hd, 4
8605 jg .w4_loop
8606 RET
8607 .w8_loop:
8608 call .main
8609 lea dstq, [dstq+strideq*2]
8610 .w8:
8611 movq [dstq+strideq*0], m0
8612 movhps [dstq+strideq*1], m0
8613 sub hd, 2
8614 jg .w8_loop
8615 RET
8616 .w16_loop:
8617 call .main
8618 lea dstq, [dstq+strideq*2]
8619 .w16:
8620 mova [dstq+strideq*0], m0
8621 call .main
8622 mova [dstq+strideq*1], m0
8623 sub hd, 2
8624 jg .w16_loop
8625 RET
8626 .w32_loop:
8627 call .main
8628 add dstq, strideq
8629 .w32:
8630 mova [dstq+16*0], m0
8631 call .main
8632 mova [dstq+16*1], m0
8633 dec hd
8634 jg .w32_loop
8635 RET
8636 .w64_loop:
8637 call .main
8638 add dstq, strideq
8639 .w64:
8640 mova [dstq+16*0], m0
8641 call .main
8642 mova [dstq+16*1], m0
8643 call .main
8644 mova [dstq+16*2], m0
8645 call .main
8646 mova [dstq+16*3], m0
8647 dec hd
8648 jg .w64_loop
8649 RET
8650 .w128_loop:
8651 call .main
8652 add dstq, strideq
8653 .w128:
8654 mova [dstq+16*0], m0
8655 call .main
8656 mova [dstq+16*1], m0
8657 call .main
8658 mova [dstq+16*2], m0
8659 call .main
8660 mova [dstq+16*3], m0
8661 call .main
8662 mova [dstq+16*4], m0
8663 call .main
8664 mova [dstq+16*5], m0
8665 call .main
8666 mova [dstq+16*6], m0
8667 call .main
8668 mova [dstq+16*7], m0
8669 dec hd
8670 jg .w128_loop
8671 RET
8672 ALIGN function_align
8673 .main:
8674 mova m0, [tmp1q +16*0]
8675 mova m3, [tmp1q+tmp2q+16*0]
8676 mova m1, [tmp1q +16*1]
8677 mova m4, [tmp1q+tmp2q+16*1]
8678 add tmp1q, 16*2
8679 psubw m3, m0
8680 psubw m4, m1
8681 pabsw m5, m3
8682 psubusw m2, m6, m5
8683 psrlw m2, 8 ; 64 - m
8684 psllw m5, m2, 10
8685 pmulhw m3, m5
8686 pabsw m5, m4
8687 paddw m0, m3
8688 psubusw m3, m6, m5
8689 psrlw m3, 8
8690 packuswb m2, m3
8691 psllw m3, 10
8692 pmulhw m4, m3
8693 psubb m3, m8, m2
8694 paddw m1, m4
8695 pmulhrsw m0, m7
8696 pmulhrsw m1, m7
8697 mova [maskq], m3
8698 add maskq, 16
8699 packuswb m0, m1
8700 ret
8701
8702 %macro BLEND_64M 4; a, b, mask1, mask2
8703 punpcklbw m0, %1, %2; {b;a}[7..0]
8704 punpckhbw %1, %2 ; {b;a}[15..8]
8705 pmaddubsw m0, %3 ; {b*m[0] + (64-m[0])*a}[7..0] u16
8706 pmaddubsw %1, %4 ; {b*m[1] + (64-m[1])*a}[15..8] u16
8707 pmulhrsw m0, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16
8708 pmulhrsw %1, m5 ; {((b*m[1] + (64-m[0])*a) + 1) / 32}[15..8] u16
8709 packuswb m0, %1 ; {blendpx}[15..0] u8
8710 %endmacro
8711
8712 %macro BLEND 2; a, b
8713 psubb m3, m4, m0 ; m3 = (64 - m)
8714 punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0]
8715 punpckhbw m3, m0 ; {m;(64-m)}[15..8]
8716 BLEND_64M %1, %2, m2, m3
8717 %endmacro
8718
8719 cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask
8720 %define base r6-blend_ssse3_table
8721 LEA r6, blend_ssse3_table
8722 tzcnt wd, wm
8723 movifnidn hd, hm
8724 movifnidn maskq, maskmp
8725 movsxd wq, dword [r6+wq*4]
8726 mova m4, [base+pb_64]
8727 mova m5, [base+pw_512]
8728 add wq, r6
8729 lea r6, [dsq*3]
8730 jmp wq
8731 .w4:
8732 movq m0, [maskq]; m
8733 movd m1, [dstq+dsq*0] ; a
8734 movd m6, [dstq+dsq*1]
8735 punpckldq m1, m6
8736 movq m6, [tmpq] ; b
8737 psubb m3, m4, m0 ; m3 = (64 - m)
8738 punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0]
8739 punpcklbw m1, m6 ; {b;a}[7..0]
8740 pmaddubsw m1, m2 ; {b*m[0] + (64-m[0])*a}[7..0] u16
8741 pmulhrsw m1, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16
8742 packuswb m1, m0 ; {blendpx}[15..0] u8
8743 movd [dstq+dsq*0], m1
8744 psrlq m1, 32
8745 movd [dstq+dsq*1], m1
8746 add maskq, 8
8747 add tmpq, 8
8748 lea dstq, [dstq+dsq*2] ; dst_stride * 2
8749 sub hd, 2
8750 jg .w4
8751 RET
8752 .w8:
8753 mova m0, [maskq]; m
8754 movq m1, [dstq+dsq*0] ; a
8755 movhps m1, [dstq+dsq*1]
8756 mova m6, [tmpq] ; b
8757 BLEND m1, m6
8758 movq [dstq+dsq*0], m0
8759 movhps [dstq+dsq*1], m0
8760 add maskq, 16
8761 add tmpq, 16
8762 lea dstq, [dstq+dsq*2] ; dst_stride * 2
8763 sub hd, 2
8764 jg .w8
8765 RET
8766 .w16:
8767 mova m0, [maskq]; m
8768 mova m1, [dstq] ; a
8769 mova m6, [tmpq] ; b
8770 BLEND m1, m6
8771 mova [dstq], m0
8772 add maskq, 16
8773 add tmpq, 16
8774 add dstq, dsq ; dst_stride
8775 dec hd
8776 jg .w16
8777 RET
8778 .w32:
8779 %assign i 0
8780 %rep 2
8781 mova m0, [maskq+16*i]; m
8782 mova m1, [dstq+16*i] ; a
8783 mova m6, [tmpq+16*i] ; b
8784 BLEND m1, m6
8785 mova [dstq+i*16], m0
8786 %assign i i+1
8787 %endrep
8788 add maskq, 32
8789 add tmpq, 32
8790 add dstq, dsq ; dst_stride
8791 dec hd
8792 jg .w32
8793 RET
8794
8795 cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask
8796 %define base r5-blend_v_ssse3_table
8797 LEA r5, blend_v_ssse3_table
8798 tzcnt wd, wm
8799 movifnidn hd, hm
8800 movsxd wq, dword [r5+wq*4]
8801 mova m5, [base+pw_512]
8802 add wq, r5
8803 add maskq, obmc_masks-blend_v_ssse3_table
8804 jmp wq
8805 .w2:
8806 movd m3, [maskq+4]
8807 punpckldq m3, m3
8808 ; 2 mask blend is provided for 4 pixels / 2 lines
8809 .w2_loop:
8810 movd m1, [dstq+dsq*0] ; a {..;a;a}
8811 pinsrw m1, [dstq+dsq*1], 1
8812 movd m2, [tmpq] ; b
8813 punpcklbw m0, m1, m2; {b;a}[7..0]
8814 pmaddubsw m0, m3 ; {b*m + (64-m)*a}[7..0] u16
8815 pmulhrsw m0, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16
8816 packuswb m0, m1 ; {blendpx}[8..0] u8
8817 movd r3d, m0
8818 mov [dstq+dsq*0], r3w
8819 shr r3d, 16
8820 mov [dstq+dsq*1], r3w
8821 add tmpq, 2*2
8822 lea dstq, [dstq + dsq * 2]
8823 sub hd, 2
8824 jg .w2_loop
8825 RET
8826 .w4:
8827 movddup m3, [maskq+8]
8828 ; 4 mask blend is provided for 8 pixels / 2 lines
8829 .w4_loop:
8830 movd m1, [dstq+dsq*0] ; a
8831 movd m2, [dstq+dsq*1] ;
8832 punpckldq m1, m2
8833 movq m2, [tmpq] ; b
8834 punpcklbw m1, m2 ; {b;a}[7..0]
8835 pmaddubsw m1, m3 ; {b*m + (64-m)*a}[7..0] u16
8836 pmulhrsw m1, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16
8837 packuswb m1, m1 ; {blendpx}[8..0] u8
8838 movd [dstq], m1
8839 psrlq m1, 32
8840 movd [dstq+dsq*1], m1
8841 add tmpq, 2*4
8842 lea dstq, [dstq+dsq*2]
8843 sub hd, 2
8844 jg .w4_loop
8845 RET
8846 .w8:
8847 mova m3, [maskq+16]
8848 ; 8 mask blend is provided for 16 pixels
8849 .w8_loop:
8850 movq m1, [dstq+dsq*0] ; a
8851 movhps m1, [dstq+dsq*1]
8852 mova m2, [tmpq]; b
8853 BLEND_64M m1, m2, m3, m3
8854 movq [dstq+dsq*0], m0
8855 movhps [dstq+dsq*1], m0
8856 add tmpq, 16
8857 lea dstq, [dstq+dsq*2]
8858 sub hd, 2
8859 jg .w8_loop
8860 RET
8861 .w16:
8862 ; 16 mask blend is provided for 32 pixels
8863 mova m3, [maskq+32] ; obmc_masks_16[0] (64-m[0])
8864 mova m4, [maskq+48] ; obmc_masks_16[1] (64-m[1])
8865 .w16_loop:
8866 mova m1, [dstq] ; a
8867 mova m2, [tmpq] ; b
8868 BLEND_64M m1, m2, m3, m4
8869 mova [dstq], m0
8870 add tmpq, 16
8871 add dstq, dsq
8872 dec hd
8873 jg .w16_loop
8874 RET
8875 .w32:
8876 %if WIN64
8877 mova [rsp+8], xmm6
8878 %endif
8879 mova m3, [maskq+64] ; obmc_masks_32[0] (64-m[0])
8880 mova m4, [maskq+80] ; obmc_masks_32[1] (64-m[1])
8881 mova m6, [maskq+96] ; obmc_masks_32[2] (64-m[2])
8882 ; 16 mask blend is provided for 64 pixels
8883 .w32_loop:
8884 mova m1, [dstq+16*0] ; a
8885 mova m2, [tmpq+16*0] ; b
8886 BLEND_64M m1, m2, m3, m4
8887 movq m1, [dstq+16*1] ; a
8888 punpcklbw m1, [tmpq+16*1] ; b
8889 pmaddubsw m1, m6
8890 pmulhrsw m1, m5
8891 packuswb m1, m1
8892 mova [dstq+16*0], m0
8893 movq [dstq+16*1], m1
8894 add tmpq, 32
8895 add dstq, dsq
8896 dec hd
8897 jg .w32_loop
8898 %if WIN64
8899 mova xmm6, [rsp+8]
8900 %endif
8901 RET
8902
8903 cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask
8904 %define base t0-blend_h_ssse3_table
8905 %if ARCH_X86_32
8906 ; We need to keep the PIC pointer for w4, reload wd from stack instead
8907 DECLARE_REG_TMP 6
8908 %else
8909 DECLARE_REG_TMP 5
8910 mov r6d, wd
8911 %endif
8912 LEA t0, blend_h_ssse3_table
8913 tzcnt wd, wm
8914 mov hd, hm
8915 movsxd wq, dword [t0+wq*4]
8916 mova m5, [base+pw_512]
8917 add wq, t0
8918 lea maskq, [base+obmc_masks+hq*2]
8919 lea hd, [hq*3]
8920 shr hd, 2 ; h * 3/4
8921 lea maskq, [maskq+hq*2]
8922 neg hq
8923 jmp wq
8924 .w2:
8925 movd m0, [dstq+dsq*0]
8926 pinsrw m0, [dstq+dsq*1], 1
8927 movd m2, [maskq+hq*2]
8928 movd m1, [tmpq]
8929 punpcklwd m2, m2
8930 punpcklbw m0, m1
8931 pmaddubsw m0, m2
8932 pmulhrsw m0, m5
8933 packuswb m0, m0
8934 movd r3d, m0
8935 mov [dstq+dsq*0], r3w
8936 shr r3d, 16
8937 mov [dstq+dsq*1], r3w
8938 lea dstq, [dstq+dsq*2]
8939 add tmpq, 2*2
8940 add hq, 2
8941 jl .w2
8942 RET
8943 .w4:
8944 %if ARCH_X86_32
8945 mova m3, [base+blend_shuf]
8946 %else
8947 mova m3, [blend_shuf]
8948 %endif
8949 .w4_loop:
8950 movd m0, [dstq+dsq*0]
8951 movd m2, [dstq+dsq*1]
8952 punpckldq m0, m2 ; a
8953 movq m1, [tmpq] ; b
8954 movq m2, [maskq+hq*2] ; m
8955 pshufb m2, m3
8956 punpcklbw m0, m1
8957 pmaddubsw m0, m2
8958 pmulhrsw m0, m5
8959 packuswb m0, m0
8960 movd [dstq+dsq*0], m0
8961 psrlq m0, 32
8962 movd [dstq+dsq*1], m0
8963 lea dstq, [dstq+dsq*2]
8964 add tmpq, 4*2
8965 add hq, 2
8966 jl .w4_loop
8967 RET
8968 .w8:
8969 movd m4, [maskq+hq*2]
8970 punpcklwd m4, m4
8971 pshufd m3, m4, q0000
8972 pshufd m4, m4, q1111
8973 movq m1, [dstq+dsq*0] ; a
8974 movhps m1, [dstq+dsq*1]
8975 mova m2, [tmpq]
8976 BLEND_64M m1, m2, m3, m4
8977 movq [dstq+dsq*0], m0
8978 movhps [dstq+dsq*1], m0
8979 lea dstq, [dstq+dsq*2]
8980 add tmpq, 8*2
8981 add hq, 2
8982 jl .w8
8983 RET
8984 ; w16/w32/w64/w128
8985 .w16:
8986 %if ARCH_X86_32
8987 mov r6d, wm
8988 %endif
8989 sub dsq, r6
8990 .w16_loop0:
8991 movd m3, [maskq+hq*2]
8992 pshuflw m3, m3, q0000
8993 punpcklqdq m3, m3
8994 mov wd, r6d
8995 .w16_loop:
8996 mova m1, [dstq] ; a
8997 mova m2, [tmpq] ; b
8998 BLEND_64M m1, m2, m3, m3
8999 mova [dstq], m0
9000 add dstq, 16
9001 add tmpq, 16
9002 sub wd, 16
9003 jg .w16_loop
9004 add dstq, dsq
9005 inc hq
9006 jl .w16_loop0
9007 RET
9008
9009 ; emu_edge args:
9010 ; const intptr_t bw, const intptr_t bh, const intptr_t iw, const intptr_t ih,
9011 ; const intptr_t x, const intptr_t y, pixel *dst, const ptrdiff_t dst_stride,
9012 ; const pixel *ref, const ptrdiff_t ref_stride
9013 ;
9014 ; bw, bh total filled size
9015 ; iw, ih, copied block -> fill bottom, right
9016 ; x, y, offset in bw/bh -> fill top, left
9017 cglobal emu_edge_8bpc, 10, 13, 2, bw, bh, iw, ih, x, \
9018 y, dst, dstride, src, sstride, \
9019 bottomext, rightext, blk
9020 ; we assume that the buffer (stride) is larger than width, so we can
9021 ; safely overwrite by a few bytes
9022 pxor m1, m1
9023
9024 %if ARCH_X86_64
9025 %define reg_zero r12q
9026 %define reg_tmp r10
9027 %define reg_src srcq
9028 %define reg_bottomext bottomextq
9029 %define reg_rightext rightextq
9030 %define reg_blkm r9m
9031 %else
9032 %define reg_zero r6
9033 %define reg_tmp r0
9034 %define reg_src r1
9035 %define reg_bottomext r0
9036 %define reg_rightext r1
9037 %define reg_blkm r2m
9038 %endif
9039 ;
9040 ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride)
9041 xor reg_zero, reg_zero
9042 lea reg_tmp, [ihq-1]
9043 cmp yq, ihq
9044 cmovs reg_tmp, yq
9045 test yq, yq
9046 cmovs reg_tmp, reg_zero
9047 %if ARCH_X86_64
9048 imul reg_tmp, sstrideq
9049 add srcq, reg_tmp
9050 %else
9051 imul reg_tmp, sstridem
9052 mov reg_src, srcm
9053 add reg_src, reg_tmp
9054 %endif
9055 ;
9056 ; ref += iclip(x, 0, iw - 1)
9057 lea reg_tmp, [iwq-1]
9058 cmp xq, iwq
9059 cmovs reg_tmp, xq
9060 test xq, xq
9061 cmovs reg_tmp, reg_zero
9062 add reg_src, reg_tmp
9063 %if ARCH_X86_32
9064 mov srcm, reg_src
9065 %endif
9066 ;
9067 ; bottom_ext = iclip(y + bh - ih, 0, bh - 1)
9068 %if ARCH_X86_32
9069 mov r1, r1m ; restore bh
9070 %endif
9071 lea reg_bottomext, [yq+bhq]
9072 sub reg_bottomext, ihq
9073 lea r3, [bhq-1]
9074 cmovs reg_bottomext, reg_zero
9075 ;
9076
9077 DEFINE_ARGS bw, bh, iw, ih, x, \
9078 topext, dst, dstride, src, sstride, \
9079 bottomext, rightext, blk
9080
9081 ; top_ext = iclip(-y, 0, bh - 1)
9082 neg topextq
9083 cmovs topextq, reg_zero
9084 cmp reg_bottomext, bhq
9085 cmovns reg_bottomext, r3
9086 cmp topextq, bhq
9087 cmovg topextq, r3
9088 %if ARCH_X86_32
9089 mov r4m, reg_bottomext
9090 ;
9091 ; right_ext = iclip(x + bw - iw, 0, bw - 1)
9092 mov r0, r0m ; restore bw
9093 %endif
9094 lea reg_rightext, [xq+bwq]
9095 sub reg_rightext, iwq
9096 lea r2, [bwq-1]
9097 cmovs reg_rightext, reg_zero
9098
9099 DEFINE_ARGS bw, bh, iw, ih, leftext, \
9100 topext, dst, dstride, src, sstride, \
9101 bottomext, rightext, blk
9102
9103 ; left_ext = iclip(-x, 0, bw - 1)
9104 neg leftextq
9105 cmovs leftextq, reg_zero
9106 cmp reg_rightext, bwq
9107 cmovns reg_rightext, r2
9108 %if ARCH_X86_32
9109 mov r3m, r1
9110 %endif
9111 cmp leftextq, bwq
9112 cmovns leftextq, r2
9113
9114 %undef reg_zero
9115 %undef reg_tmp
9116 %undef reg_src
9117 %undef reg_bottomext
9118 %undef reg_rightext
9119
9120 DEFINE_ARGS bw, centerh, centerw, dummy, leftext, \
9121 topext, dst, dstride, src, sstride, \
9122 bottomext, rightext, blk
9123
9124 ; center_h = bh - top_ext - bottom_ext
9125 %if ARCH_X86_64
9126 lea r3, [bottomextq+topextq]
9127 sub centerhq, r3
9128 %else
9129 mov r1, centerhm ; restore r1
9130 sub centerhq, topextq
9131 sub centerhq, r4m
9132 mov r1m, centerhq
9133 %endif
9134 ;
9135 ; blk += top_ext * PXSTRIDE(dst_stride)
9136 mov r2, topextq
9137 %if ARCH_X86_64
9138 imul r2, dstrideq
9139 %else
9140 mov r6, r6m ; restore dstq
9141 imul r2, dstridem
9142 %endif
9143 add dstq, r2
9144 mov reg_blkm, dstq ; save pointer for ext
9145 ;
9146 ; center_w = bw - left_ext - right_ext
9147 mov centerwq, bwq
9148 %if ARCH_X86_64
9149 lea r3, [rightextq+leftextq]
9150 sub centerwq, r3
9151 %else
9152 sub centerwq, r3m
9153 sub centerwq, leftextq
9154 %endif
9155
9156 ; vloop Macro
9157 %macro v_loop 3 ; need_left_ext, need_right_ext, suffix
9158 %if ARCH_X86_64
9159 %define reg_tmp r12
9160 %else
9161 %define reg_tmp r0
9162 %endif
9163 .v_loop_%3:
9164 %if ARCH_X86_32
9165 mov r0, r0m
9166 mov r1, r1m
9167 %endif
9168 %if %1
9169 ; left extension
9170 %if ARCH_X86_64
9171 movd m0, [srcq]
9172 %else
9173 mov r3, srcm
9174 movd m0, [r3]
9175 %endif
9176 pshufb m0, m1
9177 xor r3, r3
9178 .left_loop_%3:
9179 mova [dstq+r3], m0
9180 add r3, mmsize
9181 cmp r3, leftextq
9182 jl .left_loop_%3
9183 ; body
9184 lea reg_tmp, [dstq+leftextq]
9185 %endif
9186 xor r3, r3
9187 .body_loop_%3:
9188 %if ARCH_X86_64
9189 movu m0, [srcq+r3]
9190 %else
9191 mov r1, srcm
9192 movu m0, [r1+r3]
9193 %endif
9194 %if %1
9195 movu [reg_tmp+r3], m0
9196 %else
9197 movu [dstq+r3], m0
9198 %endif
9199 add r3, mmsize
9200 cmp r3, centerwq
9201 jl .body_loop_%3
9202 %if %2
9203 ; right extension
9204 %if %1
9205 add reg_tmp, centerwq
9206 %else
9207 lea reg_tmp, [dstq+centerwq]
9208 %endif
9209 %if ARCH_X86_64
9210 movd m0, [srcq+centerwq-1]
9211 %else
9212 mov r3, srcm
9213 movd m0, [r3+centerwq-1]
9214 %endif
9215 pshufb m0, m1
9216 xor r3, r3
9217 .right_loop_%3:
9218 movu [reg_tmp+r3], m0
9219 add r3, mmsize
9220 %if ARCH_X86_64
9221 cmp r3, rightextq
9222 %else
9223 cmp r3, r3m
9224 %endif
9225 jl .right_loop_%3
9226 %endif
9227 %if ARCH_X86_64
9228 add dstq, dstrideq
9229 add srcq, sstrideq
9230 dec centerhq
9231 jg .v_loop_%3
9232 %else
9233 add dstq, dstridem
9234 mov r0, sstridem
9235 add srcm, r0
9236 sub dword centerhm, 1
9237 jg .v_loop_%3
9238 mov r0, r0m ; restore r0
9239 %endif
9240 %endmacro ; vloop MACRO
9241
9242 test leftextq, leftextq
9243 jnz .need_left_ext
9244 %if ARCH_X86_64
9245 test rightextq, rightextq
9246 jnz .need_right_ext
9247 %else
9248 cmp leftextq, r3m ; leftextq == 0
9249 jne .need_right_ext
9250 %endif
9251 v_loop 0, 0, 0
9252 jmp .body_done
9253
9254 ;left right extensions
9255 .need_left_ext:
9256 %if ARCH_X86_64
9257 test rightextq, rightextq
9258 %else
9259 mov r3, r3m
9260 test r3, r3
9261 %endif
9262 jnz .need_left_right_ext
9263 v_loop 1, 0, 1
9264 jmp .body_done
9265
9266 .need_left_right_ext:
9267 v_loop 1, 1, 2
9268 jmp .body_done
9269
9270 .need_right_ext:
9271 v_loop 0, 1, 3
9272
9273 .body_done:
9274 ; r0 ; bw
9275 ; r1 ;; x loop
9276 ; r4 ;; y loop
9277 ; r5 ; topextq
9278 ; r6 ;dstq
9279 ; r7 ;dstrideq
9280 ; r8 ; srcq
9281 %if ARCH_X86_64
9282 %define reg_dstride dstrideq
9283 %else
9284 %define reg_dstride r2
9285 %endif
9286 ;
9287 ; bottom edge extension
9288 %if ARCH_X86_64
9289 test bottomextq, bottomextq
9290 jz .top
9291 %else
9292 xor r1, r1
9293 cmp r1, r4m
9294 je .top
9295 %endif
9296 ;
9297 %if ARCH_X86_64
9298 mov srcq, dstq
9299 sub srcq, dstrideq
9300 xor r1, r1
9301 %else
9302 mov r3, dstq
9303 mov reg_dstride, dstridem
9304 sub r3, reg_dstride
9305 mov srcm, r3
9306 %endif
9307 ;
9308 .bottom_x_loop:
9309 %if ARCH_X86_64
9310 mova m0, [srcq+r1]
9311 lea r3, [dstq+r1]
9312 mov r4, bottomextq
9313 %else
9314 mov r3, srcm
9315 mova m0, [r3+r1]
9316 lea r3, [dstq+r1]
9317 mov r4, r4m
9318 %endif
9319 ;
9320 .bottom_y_loop:
9321 mova [r3], m0
9322 add r3, reg_dstride
9323 dec r4
9324 jg .bottom_y_loop
9325 add r1, mmsize
9326 cmp r1, bwq
9327 jl .bottom_x_loop
9328
9329 .top:
9330 ; top edge extension
9331 test topextq, topextq
9332 jz .end
9333 %if ARCH_X86_64
9334 mov srcq, reg_blkm
9335 %else
9336 mov r3, reg_blkm
9337 mov reg_dstride, dstridem
9338 %endif
9339 mov dstq, dstm
9340 xor r1, r1
9341 ;
9342 .top_x_loop:
9343 %if ARCH_X86_64
9344 mova m0, [srcq+r1]
9345 %else
9346 mov r3, reg_blkm
9347 mova m0, [r3+r1]
9348 %endif
9349 lea r3, [dstq+r1]
9350 mov r4, topextq
9351 ;
9352 .top_y_loop:
9353 mova [r3], m0
9354 add r3, reg_dstride
9355 dec r4
9356 jg .top_y_loop
9357 add r1, mmsize
9358 cmp r1, bwq
9359 jl .top_x_loop
9360
9361 .end:
9362 RET
9363
9364 %undef reg_dstride
9365 %undef reg_blkm
9366 %undef reg_tmp
9367
9368 cextern resize_filter
9369
9370 %macro SCRATCH 3
9371 %if ARCH_X86_32
9372 mova [rsp+%3*mmsize], m%1
9373 %define m%2 [rsp+%3*mmsize]
9374 %else
9375 SWAP %1, %2
9376 %endif
9377 %endmacro
9378
9379 %if ARCH_X86_64
9380 cglobal resize_8bpc, 0, 12, 14, dst, dst_stride, src, src_stride, \
9381 dst_w, h, src_w, dx, mx0
9382 %elif STACK_ALIGNMENT >= 16
9383 cglobal resize_8bpc, 0, 7, 8, 3 * 16, dst, dst_stride, src, src_stride, \
9384 dst_w, h, src_w, dx, mx0
9385 %else
9386 cglobal resize_8bpc, 0, 6, 8, 3 * 16, dst, dst_stride, src, src_stride, \
9387 dst_w, h, src_w, dx, mx0
9388 %endif
9389 movifnidn dstq, dstmp
9390 movifnidn srcq, srcmp
9391 %if STACK_ALIGNMENT >= 16
9392 movifnidn dst_wd, dst_wm
9393 %endif
9394 %if ARCH_X86_64
9395 movifnidn hd, hm
9396 %endif
9397 sub dword mx0m, 4<<14
9398 sub dword src_wm, 8
9399 movd m7, dxm
9400 movd m6, mx0m
9401 movd m5, src_wm
9402 pshufd m7, m7, q0000
9403 pshufd m6, m6, q0000
9404 pshufd m5, m5, q0000
9405
9406 %if ARCH_X86_64
9407 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x
9408 LEA r7, $$
9409 %define base r7-$$
9410 %else
9411 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x
9412 %define hd dword r5m
9413 %if STACK_ALIGNMENT >= 16
9414 LEA r6, $$
9415 %define base r6-$$
9416 %else
9417 LEA r4, $$
9418 %define base r4-$$
9419 %endif
9420 %endif
9421
9422 %if ARCH_X86_64
9423 mova m10, [base+pw_m256]
9424 mova m9, [base+pd_63]
9425 mova m8, [base+pb_8x0_8x8]
9426 %else
9427 %define m10 [base+pw_m256]
9428 %define m9 [base+pd_63]
9429 %define m8 [base+pb_8x0_8x8]
9430 %endif
9431 pmaddwd m4, m7, [base+rescale_mul] ; dx*[0,1,2,3]
9432 pslld m7, 2 ; dx*4
9433 pslld m5, 14
9434 paddd m6, m4 ; mx+[0..3]*dx
9435 SCRATCH 7, 13, 0
9436 SCRATCH 6, 12, 1
9437 SCRATCH 5, 11, 2
9438
9439 ; m10 = pmulhrsw constant for x=(x+64)>>7
9440 ; m12 = mx+[0..3]*dx, m13 = dx*4, m11 = src_w, m9 = 0x3f, m8=0,8
9441
9442 .loop_y:
9443 xor xd, xd
9444 mova m0, m12 ; per-line working version of mx
9445
9446 .loop_x:
9447 pxor m1, m1
9448 pcmpgtd m1, m0
9449 pandn m1, m0
9450 psrad m2, m0, 8 ; filter offset (unmasked)
9451 pcmpgtd m3, m11, m1
9452 pand m1, m3
9453 pandn m3, m11
9454 por m1, m3
9455 psubd m3, m0, m1 ; pshufb offset
9456 psrad m1, 14 ; clipped src_x offset
9457 psrad m3, 14 ; pshufb edge_emu offset
9458 pand m2, m9 ; filter offset (masked)
9459
9460 ; load source pixels
9461 %if ARCH_X86_64
9462 movd r8d, m1
9463 pshuflw m1, m1, q3232
9464 movd r9d, m1
9465 punpckhqdq m1, m1
9466 movd r10d, m1
9467 psrlq m1, 32
9468 movd r11d, m1
9469 movq m4, [srcq+r8]
9470 movq m5, [srcq+r10]
9471 movhps m4, [srcq+r9]
9472 movhps m5, [srcq+r11]
9473 %else
9474 movd r3d, m1
9475 pshufd m1, m1, q3312
9476 movd r1d, m1
9477 pshuflw m1, m1, q3232
9478 movq m4, [srcq+r3]
9479 movq m5, [srcq+r1]
9480 movd r3d, m1
9481 punpckhqdq m1, m1
9482 movd r1d, m1
9483 movhps m4, [srcq+r3]
9484 movhps m5, [srcq+r1]
9485 %endif
9486
9487 ; if no emulation is required, we don't need to shuffle or emulate edges
9488 ; this also saves 2 quasi-vpgatherdqs
9489 pxor m6, m6
9490 pcmpeqb m6, m3
9491 %if ARCH_X86_64
9492 pmovmskb r8d, m6
9493 cmp r8d, 0xffff
9494 %else
9495 pmovmskb r3d, m6
9496 cmp r3d, 0xffff
9497 %endif
9498 je .filter
9499
9500 %if ARCH_X86_64
9501 movd r8d, m3
9502 pshuflw m3, m3, q3232
9503 movd r9d, m3
9504 punpckhqdq m3, m3
9505 movd r10d, m3
9506 psrlq m3, 32
9507 movd r11d, m3
9508 movsxd r8, r8d
9509 movsxd r9, r9d
9510 movsxd r10, r10d
9511 movsxd r11, r11d
9512 movq m6, [base+resize_shuf+4+r8]
9513 movq m7, [base+resize_shuf+4+r10]
9514 movhps m6, [base+resize_shuf+4+r9]
9515 movhps m7, [base+resize_shuf+4+r11]
9516 %else
9517 movd r3d, m3
9518 pshufd m3, m3, q3312
9519 movd r1d, m3
9520 pshuflw m3, m3, q3232
9521 movq m6, [base+resize_shuf+4+r3]
9522 movq m7, [base+resize_shuf+4+r1]
9523 movd r3d, m3
9524 punpckhqdq m3, m3
9525 movd r1d, m3
9526 movhps m6, [base+resize_shuf+4+r3]
9527 movhps m7, [base+resize_shuf+4+r1]
9528 %endif
9529
9530 paddb m6, m8
9531 paddb m7, m8
9532 pshufb m4, m6
9533 pshufb m5, m7
9534
9535 .filter:
9536 %if ARCH_X86_64
9537 movd r8d, m2
9538 pshuflw m2, m2, q3232
9539 movd r9d, m2
9540 punpckhqdq m2, m2
9541 movd r10d, m2
9542 psrlq m2, 32
9543 movd r11d, m2
9544 movq m6, [base+resize_filter+r8*8]
9545 movq m7, [base+resize_filter+r10*8]
9546 movhps m6, [base+resize_filter+r9*8]
9547 movhps m7, [base+resize_filter+r11*8]
9548 %else
9549 movd r3d, m2
9550 pshufd m2, m2, q3312
9551 movd r1d, m2
9552 pshuflw m2, m2, q3232
9553 movq m6, [base+resize_filter+r3*8]
9554 movq m7, [base+resize_filter+r1*8]
9555 movd r3d, m2
9556 punpckhqdq m2, m2
9557 movd r1d, m2
9558 movhps m6, [base+resize_filter+r3*8]
9559 movhps m7, [base+resize_filter+r1*8]
9560 %endif
9561
9562 pmaddubsw m4, m6
9563 pmaddubsw m5, m7
9564 phaddw m4, m5
9565 phaddsw m4, m4
9566 pmulhrsw m4, m10 ; x=(x+64)>>7
9567 packuswb m4, m4
9568 movd [dstq+xq], m4
9569
9570 paddd m0, m13
9571 add xd, 4
9572 %if STACK_ALIGNMENT >= 16
9573 cmp xd, dst_wd
9574 %else
9575 cmp xd, dst_wm
9576 %endif
9577 jl .loop_x
9578
9579 add dstq, dst_stridemp
9580 add srcq, src_stridemp
9581 dec hd
9582 jg .loop_y
9583 RET
9584
9585 INIT_XMM ssse3
9586 PREP_BILIN
9587 PREP_8TAP
9588 WARP_AFFINE_8X8
9589 WARP_AFFINE_8X8T
9590
9591 INIT_XMM sse4
9592 WARP_AFFINE_8X8
9593 WARP_AFFINE_8X8T
9594
9595 INIT_XMM sse2
9596 PREP_BILIN
9597 PREP_8TAP
9598 WARP_AFFINE_8X8
9599 WARP_AFFINE_8X8T