Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/x86/mc_avx512.asm

raw
1 ; Copyright © 2020, VideoLAN and dav1d authors
2 ; Copyright © 2020, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "src/asm/x86/x86inc.asm"
28
29 %if ARCH_X86_64
30
31 SECTION_RODATA 64
32
33 obmc_masks:
34 pw_512: times 2 dw 512
35 ; 2
36 db 45, 19, 64, 0
37 ; 4
38 db 39, 25, 50, 14, 59, 5, 64, 0
39 ; 8
40 db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0
41 ; 16
42 db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10
43 db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0
44 ; 32
45 db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20
46 db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9
47 db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2
48 db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0
49
50 warp_8x8_permA: db 4, 5, 6, 7, 16, 17, 18, 19, 5, 6, 7, 8, 17, 18, 19, 20
51 db 6, 7, 8, 9, 18, 19, 20, 21, 7, 8, 9, 10, 19, 20, 21, 22
52 db 8, 9, 10, 11, 20, 21, 22, 23, 9, 10, 11, 12, 21, 22, 23, 24
53 db 10, 11, 12, 13, 22, 23, 24, 25, 11, 12, 13, 14, 23, 24, 25, 26
54 warp_8x8_permB: db 0, 1, 2, 3, 20, 21, 22, 23, 1, 2, 3, 4, 21, 22, 23, 24
55 db 2, 3, 4, 5, 22, 23, 24, 25, 3, 4, 5, 6, 23, 24, 25, 26
56 db 4, 5, 6, 7, 24, 25, 26, 27, 5, 6, 7, 8, 25, 26, 27, 28
57 db 6, 7, 8, 9, 26, 27, 28, 29, 7, 8, 9, 10, 27, 28, 29, 30
58 warp_8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13
59 warp_8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15
60 pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7
61 warp_8x8_hpack: db 3, 11, 3, 11, 35, 43, 35, 43
62 pd_16384: dd 16384
63 pd_262144: dd 262144
64 warp_8x8_end: db 0, 4, 16, 20, 32, 36, 48, 52, 2, 6, 18, 22, 34, 38, 50, 54
65 warp_8x8t_end: db 2, 3, 10, 11, 18, 19, 26, 27, 34, 35, 42, 43, 50, 51, 58, 59
66 db 6, 7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, 63
67 bidir_sctr_w4: dd 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
68 wm_420_perm4: db 1, 3, 9, 11, 5, 7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31
69 db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63
70 db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30
71 db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62
72 wm_420_perm8: db 1, 3, 17, 19, 5, 7, 21, 23, 9, 11, 25, 27, 13, 15, 29, 31
73 db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63
74 db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30
75 db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62
76 wm_420_perm16: db 1, 3, 33, 35, 5, 7, 37, 39, 9, 11, 41, 43, 13, 15, 45, 47
77 db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63
78 db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46
79 db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62
80 wm_420_mask: db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63
81 db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127
82 db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
83 db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
84 wm_422_mask: db 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62
85 db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
86 db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126
87 db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
88 wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
89 db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63
90 db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
91 db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
92 bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7
93 db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15
94 db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39
95 db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47
96 bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7
97 db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15
98 db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23
99 db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31
100 bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7
101 db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23
102 db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87
103 db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39
104 bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7
105 db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15
106 db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23
107 db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31
108 bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7
109 db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15
110 db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23
111 db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31
112 bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7
113 spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
114 db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
115 db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
116 db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
117 spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
118 db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18
119 db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42
120 db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50
121 spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
122 db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
123 db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
124 db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54
125 spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
126 db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
127 db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
128 db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
129 spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
130 db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18
131 db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26
132 db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34
133 spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
134 db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
135 db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
136 db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
137 spel_v_perm16: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7
138 db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
139 db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23
140 db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
141 spel_v_perm32: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39
142 db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47
143 db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55
144 db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
145 spel_hv_perm4a: db 8, 9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23
146 db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31
147 spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39
148 db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47
149 spel_hv_perm4c: db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55
150 db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63
151 spel_hv_perm4d: db 18, 19, 0, 1, 22, 23, 4, 5, 26, 27, 8, 9, 30, 31, 12, 13
152 db 0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29
153 spel_hv_perm8a: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
154 db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31
155 db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39
156 db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47
157 spel_hv_perm8b: db 32, 33, 48, 49, 34, 35, 50, 51, 36, 37, 52, 53, 38, 39, 54, 55
158 db 40, 41, 56, 57, 42, 43, 58, 59, 44, 45, 60, 61, 46, 47, 62, 63
159 db 48, 49, 64, 65, 50, 51, 66, 67, 52, 53, 68, 69, 54, 55, 70, 71
160 db 56, 57, 72, 73, 58, 59, 74, 75, 60, 61, 76, 77, 62, 63, 78, 79
161 spel_hv_perm8c: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13
162 db 50, 51, 16, 17, 54, 55, 20, 21, 58, 59, 24, 25, 62, 63, 28, 29
163 db 0, 1, 32, 33, 4, 5, 36, 37, 8, 9, 40, 41, 12, 13, 44, 45
164 db 16, 17, 48, 49, 20, 21, 52, 53, 24, 25, 56, 57, 28, 29, 60, 61
165 spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55
166 db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63
167 spel_hv_perm16a:db 0, 1, 2, 3, 32, 33, 34, 35, 1, 2, 3, 4, 33, 34, 35, 36
168 db 2, 3, 4, 5, 34, 35, 36, 37, 3, 4, 5, 6, 35, 36, 37, 38
169 spel_hv_perm16c:db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44
170 db 10, 11, 12, 13, 42, 43, 44, 45, 11, 12, 13, 14, 43, 44, 45, 46
171 db 16, 17, 18, 19, 48, 49, 50, 51, 17, 18, 19, 20, 49, 50, 51, 52
172 db 18, 19, 20, 21, 50, 51, 52, 53, 19, 20, 21, 22, 51, 52, 53, 54
173 spel_hv_perm16b:db 4, 5, 6, 7, 36, 37, 38, 39, 5, 6, 7, 8, 37, 38, 39, 40
174 db 6, 7, 8, 9, 38, 39, 40, 41, 7, 8, 9, 10, 39, 40, 41, 42
175 db 12, 13, 14, 15, 44, 45, 46, 47, 13, 14, 15, 16, 45, 46, 47, 48
176 db 14, 15, 16, 17, 46, 47, 48, 49, 15, 16, 17, 18, 47, 48, 49, 50
177 spel_hv_perm16d:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8
178 db 2, 3, 4, 5, 3, 4, 5, 6, 6, 7, 8, 9, 7, 8, 9, 10
179 db 8, 9, 10, 11, 9, 10, 11, 12, 12, 13, 14, 15, 13, 14, 15, 16
180 db 10, 11, 12, 13, 11, 12, 13, 14, 14, 15, 16, 17, 15, 16, 17, 18
181 spel_hv_perm16e:db 4, 5, 6, 7, 5, 6, 7, 8, 8, 9, 10, 11, 9, 10, 11, 12
182 db 6, 7, 8, 9, 7, 8, 9, 10, 10, 11, 12, 13, 11, 12, 13, 14
183 db 12, 13, 14, 15, 13, 14, 15, 16, 16, 17, 18, 19, 17, 18, 19, 20
184 db 14, 15, 16, 17, 15, 16, 17, 18, 18, 19, 20, 21, 19, 20, 21, 22
185 spel_hv_end: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55
186 deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11
187 subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12
188 db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14
189 subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
190 subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
191 subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
192 bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11
193 bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7
194 bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7
195 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
196 rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
197 resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7
198 resize_permA: dd 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
199 resize_permB: dd 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
200 resize_permC: dd 0, 4, 8, 12
201 pb_02461357: db 0, 2, 4, 6, 1, 3, 5, 7
202
203 wm_420_perm64: dq 0xfedcba9876543210
204 wm_sign: dd 0x40804080, 0xc0c0c0c0, 0x40404040
205
206 pb_8x0_8x8: times 8 db 0
207 times 8 db 8
208 pb_127: times 4 db 127
209 pw_m128 times 2 dw -128
210 pw_m256: times 2 dw -256
211 pw_1024: times 2 dw 1024
212 pw_2048: times 2 dw 2048
213 pw_6903: times 2 dw 6903
214 pw_8192: times 2 dw 8192
215 pd_32: dd 32
216 pd_34: dd 34
217 pd_63: dd 63
218 pd_512: dd 512
219 pd_32768: dd 32768
220
221 %define pb_m64 (wm_sign+4)
222 %define pb_64 (wm_sign+8)
223 %define pd_2 (pd_0to7+8)
224
225 cextern mc_subpel_filters
226 %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
227 cextern mc_warp_filter
228 cextern resize_filter
229
230 %macro BASE_JMP_TABLE 3-*
231 %xdefine %1_%2_table (%%table - %3)
232 %xdefine %%base %1_%2
233 %%table:
234 %rep %0 - 2
235 dw %%base %+ _w%3 - %%base
236 %rotate 1
237 %endrep
238 %endmacro
239
240 %macro HV_JMP_TABLE 5-*
241 %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3)
242 %xdefine %%base %1_%3
243 %assign %%types %4
244 %if %%types & 1
245 %xdefine %1_%2_h_%3_table (%%h - %5)
246 %%h:
247 %rep %0 - 4
248 dw %%prefix %+ .h_w%5 - %%base
249 %rotate 1
250 %endrep
251 %rotate 4
252 %endif
253 %if %%types & 2
254 %xdefine %1_%2_v_%3_table (%%v - %5)
255 %%v:
256 %rep %0 - 4
257 dw %%prefix %+ .v_w%5 - %%base
258 %rotate 1
259 %endrep
260 %rotate 4
261 %endif
262 %if %%types & 4
263 %xdefine %1_%2_hv_%3_table (%%hv - %5)
264 %%hv:
265 %rep %0 - 4
266 dw %%prefix %+ .hv_w%5 - %%base
267 %rotate 1
268 %endrep
269 %endif
270 %endmacro
271
272 %macro BIDIR_JMP_TABLE 2-*
273 %xdefine %1_%2_table (%%table - 2*%3)
274 %xdefine %%base %1_%2_table
275 %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
276 %%table:
277 %rep %0 - 2
278 dd %%prefix %+ .w%3 - %%base
279 %rotate 1
280 %endrep
281 %endmacro
282
283 %xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_8bpc_avx512icl.put)
284 %xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_8bpc_avx512icl.prep)
285
286 %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
287
288 BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128
289 BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128
290 HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128
291 HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128
292 HV_JMP_TABLE put, 8tap, avx512icl, 3, 2, 4, 8, 16, 32, 64, 128
293 HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128
294 BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128
295 BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128
296 BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128
297 BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128
298 BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128
299 BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128
300 BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32
301 BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32
302 BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128
303
304 SECTION .text
305
306 %macro WRAP_YMM 1+
307 INIT_YMM cpuname
308 %1
309 INIT_ZMM cpuname
310 %endmacro
311
312 INIT_ZMM avx512icl
313 cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
314 movifnidn mxyd, r6m ; mx
315 lea r7, [put_avx512icl]
316 tzcnt wd, wm
317 movifnidn hd, hm
318 test mxyd, mxyd
319 jnz .h
320 mov mxyd, r7m ; my
321 test mxyd, mxyd
322 jnz .v
323 .put:
324 movzx wd, word [r7+wq*2+table_offset(put,)]
325 add wq, r7
326 jmp wq
327 .put_w2:
328 movzx r6d, word [srcq+ssq*0]
329 movzx r7d, word [srcq+ssq*1]
330 lea srcq, [srcq+ssq*2]
331 mov [dstq+dsq*0], r6w
332 mov [dstq+dsq*1], r7w
333 lea dstq, [dstq+dsq*2]
334 sub hd, 2
335 jg .put_w2
336 RET
337 .put_w4:
338 mov r6d, [srcq+ssq*0]
339 mov r7d, [srcq+ssq*1]
340 lea srcq, [srcq+ssq*2]
341 mov [dstq+dsq*0], r6d
342 mov [dstq+dsq*1], r7d
343 lea dstq, [dstq+dsq*2]
344 sub hd, 2
345 jg .put_w4
346 RET
347 .put_w8:
348 mov r6, [srcq+ssq*0]
349 mov r7, [srcq+ssq*1]
350 lea srcq, [srcq+ssq*2]
351 mov [dstq+dsq*0], r6
352 mov [dstq+dsq*1], r7
353 lea dstq, [dstq+dsq*2]
354 sub hd, 2
355 jg .put_w8
356 RET
357 .put_w16:
358 movu xmm0, [srcq+ssq*0]
359 movu xmm1, [srcq+ssq*1]
360 lea srcq, [srcq+ssq*2]
361 mova [dstq+dsq*0], xmm0
362 mova [dstq+dsq*1], xmm1
363 lea dstq, [dstq+dsq*2]
364 sub hd, 2
365 jg .put_w16
366 RET
367 .put_w32:
368 movu ym0, [srcq+ssq*0]
369 movu ym1, [srcq+ssq*1]
370 lea srcq, [srcq+ssq*2]
371 mova [dstq+dsq*0], ym0
372 mova [dstq+dsq*1], ym1
373 lea dstq, [dstq+dsq*2]
374 sub hd, 2
375 jg .put_w32
376 RET
377 .put_w64:
378 movu m0, [srcq+ssq*0]
379 movu m1, [srcq+ssq*1]
380 lea srcq, [srcq+ssq*2]
381 mova [dstq+dsq*0], m0
382 mova [dstq+dsq*1], m1
383 lea dstq, [dstq+dsq*2]
384 sub hd, 2
385 jg .put_w64
386 RET
387 .put_w128:
388 movu m0, [srcq+ssq*0+64*0]
389 movu m1, [srcq+ssq*0+64*1]
390 movu m2, [srcq+ssq*1+64*0]
391 movu m3, [srcq+ssq*1+64*1]
392 lea srcq, [srcq+ssq*2]
393 mova [dstq+dsq*0+64*0], m0
394 mova [dstq+dsq*0+64*1], m1
395 mova [dstq+dsq*1+64*0], m2
396 mova [dstq+dsq*1+64*1], m3
397 lea dstq, [dstq+dsq*2]
398 sub hd, 2
399 jg .put_w128
400 RET
401 .h:
402 ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
403 ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
404 imul mxyd, 0xff01
405 vbroadcasti128 m4, [bilin_h_shuf8]
406 add mxyd, 16 << 8
407 vpbroadcastw m5, mxyd
408 mov mxyd, r7m ; my
409 test mxyd, mxyd
410 jnz .hv
411 movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)]
412 vpbroadcastd m3, [pw_2048]
413 add wq, r7
414 jmp wq
415 .h_w2:
416 movd xmm0, [srcq+ssq*0]
417 pinsrd xmm0, [srcq+ssq*1], 1
418 lea srcq, [srcq+ssq*2]
419 pshufb xmm0, xm4
420 pmaddubsw xmm0, xm5
421 pmulhrsw xmm0, xm3
422 packuswb xmm0, xmm0
423 pextrw [dstq+dsq*0], xmm0, 0
424 pextrw [dstq+dsq*1], xmm0, 2
425 lea dstq, [dstq+dsq*2]
426 sub hd, 2
427 jg .h_w2
428 RET
429 .h_w4:
430 mova xmm4, [bilin_h_shuf4]
431 .h_w4_loop:
432 movq xmm0, [srcq+ssq*0]
433 movhps xmm0, [srcq+ssq*1]
434 lea srcq, [srcq+ssq*2]
435 pshufb xmm0, xmm4
436 pmaddubsw xmm0, xm5
437 pmulhrsw xmm0, xm3
438 packuswb xmm0, xmm0
439 movd [dstq+dsq*0], xmm0
440 pextrd [dstq+dsq*1], xmm0, 1
441 lea dstq, [dstq+dsq*2]
442 sub hd, 2
443 jg .h_w4_loop
444 RET
445 .h_w8:
446 movu xm0, [srcq+ssq*0]
447 vinserti32x4 ym0, [srcq+ssq*1], 1
448 lea srcq, [srcq+ssq*2]
449 pshufb ym0, ym4
450 pmaddubsw ym0, ym5
451 pmulhrsw ym0, ym3
452 vpmovuswb xm0, ym0
453 movq [dstq+dsq*0], xm0
454 movhps [dstq+dsq*1], xm0
455 lea dstq, [dstq+dsq*2]
456 sub hd, 2
457 jg .h_w8
458 RET
459 .h_w16:
460 mova m4, [bilin_h_perm16]
461 .h_w16_loop:
462 movu ym0, [srcq+ssq*0]
463 vinserti32x8 m0, [srcq+ssq*1], 1
464 lea srcq, [srcq+ssq*2]
465 vpermb m0, m4, m0
466 pmaddubsw m0, m5
467 pmulhrsw m0, m3
468 vpmovuswb ym0, m0
469 mova [dstq+dsq*0], xm0
470 vextracti128 [dstq+dsq*1], ym0, 1
471 lea dstq, [dstq+dsq*2]
472 sub hd, 2
473 jg .h_w16_loop
474 RET
475 .h_w32:
476 movu ym0, [srcq+ssq*0+8*0]
477 vinserti32x8 m0, [srcq+ssq*1+8*0], 1
478 movu ym1, [srcq+ssq*0+8*1]
479 vinserti32x8 m1, [srcq+ssq*1+8*1], 1
480 lea srcq, [srcq+ssq*2]
481 pshufb m0, m4
482 pshufb m1, m4
483 pmaddubsw m0, m5
484 pmaddubsw m1, m5
485 pmulhrsw m0, m3
486 pmulhrsw m1, m3
487 packuswb m0, m1
488 mova [dstq+dsq*0], ym0
489 vextracti32x8 [dstq+dsq*1], m0, 1
490 lea dstq, [dstq+dsq*2]
491 sub hd, 2
492 jg .h_w32
493 RET
494 .h_w64:
495 movu m0, [srcq+8*0]
496 movu m1, [srcq+8*1]
497 pshufb m0, m4
498 pshufb m1, m4
499 pmaddubsw m0, m5
500 pmaddubsw m1, m5
501 pmulhrsw m0, m3
502 pmulhrsw m1, m3
503 packuswb m0, m1
504 add srcq, ssq
505 mova [dstq], m0
506 add dstq, dsq
507 dec hd
508 jg .h_w64
509 RET
510 .h_w128:
511 movu m0, [srcq+8*0]
512 movu m2, [srcq+8*1]
513 movu m1, [srcq+8*8]
514 movu m6, [srcq+8*9]
515 add srcq, ssq
516 REPX {pshufb x, m4}, m0, m2, m1, m6
517 REPX {pmaddubsw x, m5}, m0, m2, m1, m6
518 REPX {pmulhrsw x, m3}, m0, m2, m1, m6
519 packuswb m0, m2
520 packuswb m1, m6
521 mova [dstq+64*0], m0
522 mova [dstq+64*1], m1
523 add dstq, dsq
524 dec hd
525 jg .h_w128
526 RET
527 .v:
528 movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)]
529 imul mxyd, 0xff01
530 vpbroadcastd m5, [pw_2048]
531 add mxyd, 16 << 8
532 add wq, r7
533 vpbroadcastw m4, mxyd
534 jmp wq
535 .v_w2:
536 movd xmm0, [srcq+ssq*0]
537 .v_w2_loop:
538 pinsrw xmm1, xmm0, [srcq+ssq*1], 1 ; 0 1
539 lea srcq, [srcq+ssq*2]
540 pinsrw xmm0, xmm1, [srcq+ssq*0], 0 ; 2 1
541 pshuflw xmm1, xmm1, q2301 ; 1 0
542 punpcklbw xmm1, xmm0, xmm1
543 pmaddubsw xmm1, xm4
544 pmulhrsw xmm1, xm5
545 packuswb xmm1, xmm1
546 pextrw [dstq+dsq*0], xmm1, 1
547 pextrw [dstq+dsq*1], xmm1, 0
548 lea dstq, [dstq+dsq*2]
549 sub hd, 2
550 jg .v_w2_loop
551 RET
552 .v_w4:
553 movd xmm0, [srcq+ssq*0]
554 .v_w4_loop:
555 vpbroadcastd xmm1, [srcq+ssq*1]
556 lea srcq, [srcq+ssq*2]
557 vpblendd xmm2, xmm1, xmm0, 0x01 ; 0 1
558 vpbroadcastd xmm0, [srcq+ssq*0]
559 vpblendd xmm1, xmm0, 0x02 ; 1 2
560 punpcklbw xmm1, xmm2
561 pmaddubsw xmm1, xm4
562 pmulhrsw xmm1, xm5
563 packuswb xmm1, xmm1
564 movd [dstq+dsq*0], xmm1
565 pextrd [dstq+dsq*1], xmm1, 1
566 lea dstq, [dstq+dsq*2]
567 sub hd, 2
568 jg .v_w4_loop
569 RET
570 .v_w8:
571 movq xmm0, [srcq+ssq*0]
572 .v_w8_loop:
573 movq xmm3, [srcq+ssq*1]
574 lea srcq, [srcq+ssq*2]
575 punpcklbw xmm1, xmm3, xmm0
576 movq xmm0, [srcq+ssq*0]
577 punpcklbw xmm2, xmm0, xmm3
578 pmaddubsw xmm1, xm4
579 pmaddubsw xmm2, xm4
580 pmulhrsw xmm1, xm5
581 pmulhrsw xmm2, xm5
582 packuswb xmm1, xmm2
583 movq [dstq+dsq*0], xmm1
584 movhps [dstq+dsq*1], xmm1
585 lea dstq, [dstq+dsq*2]
586 sub hd, 2
587 jg .v_w8_loop
588 RET
589 .v_w16:
590 movu xmm0, [srcq+ssq*0]
591 .v_w16_loop:
592 vbroadcasti128 ymm2, [srcq+ssq*1]
593 lea srcq, [srcq+ssq*2]
594 vpblendd ymm3, ymm2, ymm0, 0x0f ; 0 1
595 vbroadcasti128 ymm0, [srcq+ssq*0]
596 vpblendd ymm2, ymm2, ymm0, 0xf0 ; 1 2
597 punpcklbw ymm1, ymm2, ymm3
598 punpckhbw ymm2, ymm3
599 pmaddubsw ymm1, ym4
600 pmaddubsw ymm2, ym4
601 pmulhrsw ymm1, ym5
602 pmulhrsw ymm2, ym5
603 packuswb ymm1, ymm2
604 mova [dstq+dsq*0], xmm1
605 vextracti128 [dstq+dsq*1], ymm1, 1
606 lea dstq, [dstq+dsq*2]
607 sub hd, 2
608 jg .v_w16_loop
609 vzeroupper
610 RET
611 .v_w32:
612 movu ym0, [srcq+ssq*0]
613 kxnorb k1, k1, k1
614 .v_w32_loop:
615 vbroadcasti32x8 m2, [srcq+ssq*1]
616 lea srcq, [srcq+ssq*2]
617 vpblendmd m3{k1}, m2, m0 ; 0 1
618 vbroadcasti32x8 m0, [srcq+ssq*0]
619 vpblendmd m2{k1}, m0, m2 ; 1 2
620 punpcklbw m1, m2, m3
621 punpckhbw m2, m3
622 pmaddubsw m1, m4
623 pmaddubsw m2, m4
624 pmulhrsw m1, m5
625 pmulhrsw m2, m5
626 packuswb m1, m2
627 mova [dstq+dsq*0], ym1
628 vextracti32x8 [dstq+dsq*1], m1, 1
629 lea dstq, [dstq+dsq*2]
630 sub hd, 2
631 jg .v_w32_loop
632 RET
633 .v_w64:
634 movu m0, [srcq+ssq*0]
635 .v_w64_loop:
636 movu m3, [srcq+ssq*1]
637 lea srcq, [srcq+ssq*2]
638 punpcklbw m1, m3, m0
639 punpckhbw m6, m3, m0
640 movu m0, [srcq+ssq*0]
641 pmaddubsw m1, m4
642 pmaddubsw m6, m4
643 punpcklbw m2, m0, m3
644 punpckhbw m7, m0, m3
645 pmaddubsw m2, m4
646 pmaddubsw m7, m4
647 REPX {pmulhrsw x, m5}, m1, m6, m2, m7
648 packuswb m1, m6
649 packuswb m2, m7
650 mova [dstq+dsq*0], m1
651 mova [dstq+dsq*1], m2
652 lea dstq, [dstq+dsq*2]
653 sub hd, 2
654 jg .v_w64_loop
655 RET
656 .v_w128:
657 movu m0, [srcq+64*0]
658 movu m1, [srcq+64*1]
659 .v_w128_loop:
660 add srcq, ssq
661 movu m2, [srcq+64*0]
662 movu m3, [srcq+64*1]
663 punpcklbw m6, m2, m0
664 pmaddubsw m6, m4
665 punpckhbw m0, m2, m0
666 pmaddubsw m0, m4
667 punpcklbw m7, m3, m1
668 pmaddubsw m7, m4
669 punpckhbw m1, m3, m1
670 pmaddubsw m1, m4
671 REPX {pmulhrsw x, m5}, m6, m0, m7, m1
672 packuswb m6, m0
673 mova m0, m2
674 packuswb m7, m1
675 mova m1, m3
676 mova [dstq+64*0], m6
677 mova [dstq+64*1], m7
678 add dstq, dsq
679 dec hd
680 jg .v_w128_loop
681 RET
682 .hv:
683 ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8
684 ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4
685 movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)]
686 WIN64_SPILL_XMM 8
687 shl mxyd, 11 ; can't shift by 12 due to signed overflow
688 vpbroadcastd m7, [pw_2048]
689 add wq, r7
690 vpbroadcastw m6, mxyd
691 jmp wq
692 .hv_w2:
693 vpbroadcastd xmm0, [srcq+ssq*0]
694 pshufb xmm0, xm4
695 pmaddubsw xmm0, xm5
696 .hv_w2_loop:
697 movd xmm1, [srcq+ssq*1]
698 lea srcq, [srcq+ssq*2]
699 pinsrd xmm1, [srcq+ssq*0], 1
700 pshufb xmm1, xm4
701 pmaddubsw xmm1, xm5 ; 1 _ 2 _
702 shufps xmm2, xmm0, xmm1, q1032 ; 0 _ 1 _
703 mova xmm0, xmm1
704 psubw xmm1, xmm2
705 paddw xmm1, xmm1
706 pmulhw xmm1, xm6
707 paddw xmm1, xmm2
708 pmulhrsw xmm1, xm7
709 packuswb xmm1, xmm1
710 pextrw [dstq+dsq*0], xmm1, 0
711 pextrw [dstq+dsq*1], xmm1, 2
712 lea dstq, [dstq+dsq*2]
713 sub hd, 2
714 jg .hv_w2_loop
715 RET
716 .hv_w4:
717 mova xmm4, [bilin_h_shuf4]
718 movddup xmm0, [srcq+ssq*0]
719 pshufb xmm0, xmm4
720 pmaddubsw xmm0, xm5
721 .hv_w4_loop:
722 movq xmm1, [srcq+ssq*1]
723 lea srcq, [srcq+ssq*2]
724 movhps xmm1, [srcq+ssq*0]
725 pshufb xmm1, xmm4
726 pmaddubsw xmm1, xm5 ; 1 2
727 shufps xmm2, xmm0, xmm1, q1032 ; 0 1
728 mova xmm0, xmm1
729 psubw xmm1, xmm2
730 paddw xmm1, xmm1
731 pmulhw xmm1, xm6
732 paddw xmm1, xmm2
733 pmulhrsw xmm1, xm7
734 packuswb xmm1, xmm1
735 movd [dstq+dsq*0], xmm1
736 pextrd [dstq+dsq*1], xmm1, 1
737 lea dstq, [dstq+dsq*2]
738 sub hd, 2
739 jg .hv_w4_loop
740 RET
741 .hv_w8:
742 vbroadcasti128 ym0, [srcq+ssq*0]
743 pshufb ym0, ym4
744 pmaddubsw ym0, ym5
745 .hv_w8_loop:
746 movu xm1, [srcq+ssq*1]
747 lea srcq, [srcq+ssq*2]
748 vinserti128 ym1, [srcq+ssq*0], 1
749 pshufb ym1, ym4
750 pmaddubsw ym1, ym5 ; 1 2
751 valignq ym2, ym1, ym0, 2
752 mova ym0, ym1
753 psubw ym1, ym2
754 paddw ym1, ym1
755 pmulhw ym1, ym6
756 paddw ym1, ym2
757 pmulhrsw ym1, ym7
758 vpmovuswb xm1, ym1
759 movq [dstq+dsq*0], xm1
760 movhps [dstq+dsq*1], xm1
761 lea dstq, [dstq+dsq*2]
762 sub hd, 2
763 jg .hv_w8_loop
764 RET
765 .hv_w16:
766 vbroadcasti32x8 m0, [srcq+ssq*0]
767 mova m4, [bilin_h_perm16]
768 vpermb m0, m4, m0
769 pmaddubsw m0, m5
770 .hv_w16_loop:
771 movu ym1, [srcq+ssq*1]
772 lea srcq, [srcq+ssq*2]
773 vinserti32x8 m1, [srcq+ssq*0], 1
774 vpermb m1, m4, m1
775 pmaddubsw m1, m5 ; 1 2
776 valignq m2, m1, m0, 4 ; 0 1
777 mova m0, m1
778 psubw m1, m2
779 paddw m1, m1
780 pmulhw m1, m6
781 paddw m1, m2
782 pmulhrsw m1, m7
783 vpmovuswb ym1, m1
784 mova [dstq+dsq*0], xm1
785 vextracti32x4 [dstq+dsq*1], ym1, 1
786 lea dstq, [dstq+dsq*2]
787 sub hd, 2
788 jg .hv_w16_loop
789 RET
790 .hv_w32:
791 mova m4, [bilin_h_perm32]
792 vpermb m0, m4, [srcq+ssq*0]
793 pmovzxbq m8, [pb_02461357]
794 pmaddubsw m0, m5
795 .hv_w32_loop:
796 vpermb m2, m4, [srcq+ssq*1]
797 lea srcq, [srcq+ssq*2]
798 vpermb m3, m4, [srcq+ssq*0]
799 pmaddubsw m2, m5
800 psubw m1, m2, m0
801 paddw m1, m1
802 pmulhw m1, m6
803 paddw m1, m0
804 pmaddubsw m0, m3, m5
805 psubw m3, m0, m2
806 paddw m3, m3
807 pmulhw m3, m6
808 paddw m3, m2
809 pmulhrsw m1, m7
810 pmulhrsw m3, m7
811 packuswb m1, m3
812 vpermq m1, m8, m1
813 mova [dstq+dsq*0], ym1
814 vextracti32x8 [dstq+dsq*1], m1, 1
815 lea dstq, [dstq+dsq*2]
816 sub hd, 2
817 jg .hv_w32_loop
818 RET
819 .hv_w64:
820 movu m0, [srcq+8*0]
821 movu m1, [srcq+8*1]
822 pshufb m0, m4
823 pshufb m1, m4
824 pmaddubsw m0, m5
825 pmaddubsw m1, m5
826 .hv_w64_loop:
827 add srcq, ssq
828 movu m2, [srcq+8*0]
829 movu m3, [srcq+8*1]
830 pshufb m2, m4
831 pshufb m3, m4
832 pmaddubsw m2, m5
833 pmaddubsw m3, m5
834 psubw m8, m2, m0
835 psubw m9, m3, m1
836 paddw m8, m8
837 pmulhw m8, m6
838 paddw m9, m9
839 pmulhw m9, m6
840 paddw m8, m0
841 pmulhrsw m8, m7
842 paddw m9, m1
843 pmulhrsw m9, m7
844 mova m0, m2
845 mova m1, m3
846 packuswb m8, m9
847 mova [dstq], m8
848 add dstq, dsq
849 dec hd
850 jg .hv_w64_loop
851 RET
852 .hv_w128:
853 movu m0, [srcq+8*0]
854 movu m1, [srcq+8*1]
855 movu m2, [srcq+8*8]
856 movu m3, [srcq+8*9]
857 REPX {pshufb x, m4}, m0, m1, m2, m3
858 REPX {pmaddubsw x, m5}, m0, m1, m2, m3
859 .hv_w128_loop:
860 add srcq, ssq
861 movu m8, [srcq+8*0]
862 movu m9, [srcq+8*1]
863 movu m10, [srcq+8*8]
864 movu m11, [srcq+8*9]
865 REPX {pshufb x, m4}, m8, m9, m10, m11
866 REPX {pmaddubsw x, m5}, m8, m9, m10, m11
867 psubw m12, m8, m0
868 psubw m13, m9, m1
869 psubw m14, m10, m2
870 psubw m15, m11, m3
871 paddw m12, m12
872 pmulhw m12, m6
873 paddw m13, m13
874 pmulhw m13, m6
875 paddw m14, m14
876 pmulhw m14, m6
877 paddw m15, m15
878 pmulhw m15, m6
879 paddw m12, m0
880 pmulhrsw m12, m7
881 paddw m13, m1
882 pmulhrsw m13, m7
883 paddw m14, m2
884 pmulhrsw m14, m7
885 paddw m15, m3
886 pmulhrsw m15, m7
887 mova m0, m8
888 mova m1, m9
889 mova m2, m10
890 mova m3, m11
891 packuswb m12, m13
892 packuswb m14, m15
893 mova [dstq+64*0], m12
894 mova [dstq+64*1], m14
895 add dstq, dsq
896 dec hd
897 jg .hv_w128_loop
898 RET
899
900 DECLARE_REG_TMP 3, 5, 6
901
902 cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
903 movifnidn mxyd, r5m ; mx
904 lea t2, [prep_avx512icl]
905 tzcnt wd, wm
906 movifnidn hd, hm
907 test mxyd, mxyd
908 jnz .h
909 mov mxyd, r6m ; my
910 test mxyd, mxyd
911 jnz .v
912 .prep:
913 movzx wd, word [t2+wq*2+table_offset(prep,)]
914 add wq, t2
915 lea stride3q, [strideq*3]
916 jmp wq
917 .prep_w4:
918 movd xmm0, [srcq+strideq*0]
919 pinsrd xmm0, [srcq+strideq*1], 1
920 pinsrd xmm0, [srcq+strideq*2], 2
921 pinsrd xmm0, [srcq+stride3q ], 3
922 lea srcq, [srcq+strideq*4]
923 pmovzxbw ym0, xmm0
924 psllw ym0, 4
925 mova [tmpq], ym0
926 add tmpq, 32
927 sub hd, 4
928 jg .prep_w4
929 RET
930 .prep_w8:
931 movq xmm0, [srcq+strideq*0]
932 movq xmm1, [srcq+strideq*1]
933 vinserti128 ym0, ymm0, [srcq+strideq*2], 1
934 vinserti128 ym1, ymm1, [srcq+stride3q ], 1
935 lea srcq, [srcq+strideq*4]
936 punpcklqdq ym0, ym1
937 pmovzxbw m0, ym0
938 psllw m0, 4
939 mova [tmpq], m0
940 add tmpq, 32*2
941 sub hd, 4
942 jg .prep_w8
943 RET
944 .prep_w16:
945 movu xmm0, [srcq+strideq*0]
946 vinserti128 ym0, ymm0, [srcq+strideq*1], 1
947 movu xmm1, [srcq+strideq*2]
948 vinserti128 ym1, ymm1, [srcq+stride3q ], 1
949 lea srcq, [srcq+strideq*4]
950 pmovzxbw m0, ym0
951 pmovzxbw m1, ym1
952 psllw m0, 4
953 psllw m1, 4
954 mova [tmpq+64*0], m0
955 mova [tmpq+64*1], m1
956 add tmpq, 32*4
957 sub hd, 4
958 jg .prep_w16
959 RET
960 .prep_w32:
961 pmovzxbw m0, [srcq+strideq*0]
962 pmovzxbw m1, [srcq+strideq*1]
963 pmovzxbw m2, [srcq+strideq*2]
964 pmovzxbw m3, [srcq+stride3q ]
965 lea srcq, [srcq+strideq*4]
966 REPX {psllw x, 4}, m0, m1, m2, m3
967 mova [tmpq+64*0], m0
968 mova [tmpq+64*1], m1
969 mova [tmpq+64*2], m2
970 mova [tmpq+64*3], m3
971 add tmpq, 64*4
972 sub hd, 4
973 jg .prep_w32
974 RET
975 .prep_w64:
976 pmovzxbw m0, [srcq+strideq*0+32*0]
977 pmovzxbw m1, [srcq+strideq*0+32*1]
978 pmovzxbw m2, [srcq+strideq*1+32*0]
979 pmovzxbw m3, [srcq+strideq*1+32*1]
980 lea srcq, [srcq+strideq*2]
981 REPX {psllw x, 4}, m0, m1, m2, m3
982 mova [tmpq+64*0], m0
983 mova [tmpq+64*1], m1
984 mova [tmpq+64*2], m2
985 mova [tmpq+64*3], m3
986 add tmpq, 64*4
987 sub hd, 2
988 jg .prep_w64
989 RET
990 .prep_w128:
991 pmovzxbw m0, [srcq+32*0]
992 pmovzxbw m1, [srcq+32*1]
993 pmovzxbw m2, [srcq+32*2]
994 pmovzxbw m3, [srcq+32*3]
995 REPX {psllw x, 4}, m0, m1, m2, m3
996 mova [tmpq+64*0], m0
997 mova [tmpq+64*1], m1
998 mova [tmpq+64*2], m2
999 mova [tmpq+64*3], m3
1000 add tmpq, 64*4
1001 add srcq, strideq
1002 dec hd
1003 jg .prep_w128
1004 RET
1005 .h:
1006 ; 16 * src[x] + (mx * (src[x + 1] - src[x]))
1007 ; = (16 - mx) * src[x] + mx * src[x + 1]
1008 imul mxyd, 0xff01
1009 add mxyd, 16 << 8
1010 vpbroadcastw m5, mxyd
1011 mov mxyd, r6m ; my
1012 test mxyd, mxyd
1013 jnz .hv
1014 movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)]
1015 add wq, t2
1016 lea stride3q, [strideq*3]
1017 jmp wq
1018 .h_w4:
1019 vbroadcasti32x4 ym4, [bilin_h_shuf4]
1020 .h_w4_loop:
1021 movq xmm0, [srcq+strideq*0]
1022 movq xmm1, [srcq+strideq*1]
1023 vinserti32x4 ym0, ymm0, [srcq+strideq*2], 1
1024 vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1
1025 lea srcq, [srcq+strideq*4]
1026 punpcklqdq ym0, ym1
1027 pshufb ym0, ym4
1028 pmaddubsw ym0, ym5
1029 mova [tmpq], ym0
1030 add tmpq, 32
1031 sub hd, 4
1032 jg .h_w4_loop
1033 RET
1034 .h_w8:
1035 vbroadcasti32x4 m4, [bilin_h_shuf8]
1036 .h_w8_loop:
1037 movu xmm0, [srcq+strideq*0]
1038 vinserti32x4 ym0, ymm0, [srcq+strideq*1], 1
1039 vinserti32x4 m0, [srcq+strideq*2], 2
1040 vinserti32x4 m0, [srcq+stride3q ], 3
1041 lea srcq, [srcq+strideq*4]
1042 pshufb m0, m4
1043 pmaddubsw m0, m5
1044 mova [tmpq], m0
1045 add tmpq, 64
1046 sub hd, 4
1047 jg .h_w8_loop
1048 RET
1049 .h_w16:
1050 mova m4, [bilin_h_perm16]
1051 .h_w16_loop:
1052 movu ym0, [srcq+strideq*0]
1053 vinserti32x8 m0, [srcq+strideq*1], 1
1054 movu ym1, [srcq+strideq*2]
1055 vinserti32x8 m1, [srcq+stride3q ], 1
1056 lea srcq, [srcq+strideq*4]
1057 vpermb m0, m4, m0
1058 vpermb m1, m4, m1
1059 pmaddubsw m0, m5
1060 pmaddubsw m1, m5
1061 mova [tmpq+64*0], m0
1062 mova [tmpq+64*1], m1
1063 add tmpq, 64*2
1064 sub hd, 4
1065 jg .h_w16_loop
1066 RET
1067 .h_w32:
1068 mova m4, [bilin_h_perm32]
1069 .h_w32_loop:
1070 vpermb m0, m4, [srcq+strideq*0]
1071 vpermb m1, m4, [srcq+strideq*1]
1072 vpermb m2, m4, [srcq+strideq*2]
1073 vpermb m3, m4, [srcq+stride3q ]
1074 lea srcq, [srcq+strideq*4]
1075 pmaddubsw m0, m5
1076 pmaddubsw m1, m5
1077 pmaddubsw m2, m5
1078 pmaddubsw m3, m5
1079 mova [tmpq+64*0], m0
1080 mova [tmpq+64*1], m1
1081 mova [tmpq+64*2], m2
1082 mova [tmpq+64*3], m3
1083 add tmpq, 64*4
1084 sub hd, 4
1085 jg .h_w32_loop
1086 RET
1087 .h_w64:
1088 mova m4, [bilin_h_perm32]
1089 .h_w64_loop:
1090 vpermb m0, m4, [srcq+strideq*0+32*0]
1091 vpermb m1, m4, [srcq+strideq*0+32*1]
1092 vpermb m2, m4, [srcq+strideq*1+32*0]
1093 vpermb m3, m4, [srcq+strideq*1+32*1]
1094 lea srcq, [srcq+strideq*2]
1095 pmaddubsw m0, m5
1096 pmaddubsw m1, m5
1097 pmaddubsw m2, m5
1098 pmaddubsw m3, m5
1099 mova [tmpq+64*0], m0
1100 mova [tmpq+64*1], m1
1101 mova [tmpq+64*2], m2
1102 mova [tmpq+64*3], m3
1103 add tmpq, 64*4
1104 sub hd, 2
1105 jg .h_w64_loop
1106 RET
1107 .h_w128:
1108 mova m4, [bilin_h_perm32]
1109 .h_w128_loop:
1110 vpermb m0, m4, [srcq+32*0]
1111 vpermb m1, m4, [srcq+32*1]
1112 vpermb m2, m4, [srcq+32*2]
1113 vpermb m3, m4, [srcq+32*3]
1114 pmaddubsw m0, m5
1115 pmaddubsw m1, m5
1116 pmaddubsw m2, m5
1117 pmaddubsw m3, m5
1118 mova [tmpq+64*0], m0
1119 mova [tmpq+64*1], m1
1120 mova [tmpq+64*2], m2
1121 mova [tmpq+64*3], m3
1122 add tmpq, 64*4
1123 add srcq, strideq
1124 dec hd
1125 jg .h_w128_loop
1126 RET
1127 .v:
1128 WIN64_SPILL_XMM 7
1129 movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)]
1130 imul mxyd, 0xff01
1131 add mxyd, 16 << 8
1132 add wq, t2
1133 lea stride3q, [strideq*3]
1134 vpbroadcastw m6, mxyd
1135 jmp wq
1136 .v_w4:
1137 vpbroadcastd xm0, [srcq+strideq*0]
1138 mov r3d, 0x29
1139 vbroadcasti32x4 ym3, [bilin_v_shuf4]
1140 kmovb k1, r3d
1141 .v_w4_loop:
1142 vpblendmd xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____
1143 vpbroadcastd ym2, [srcq+strideq*2]
1144 vpbroadcastd ym2{k1}, [srcq+stride3q ] ; __2_ 23__
1145 lea srcq, [srcq+strideq*4]
1146 vpbroadcastd ym0, [srcq+strideq*0]
1147 punpckhqdq ym2{k1}, ym1, ym0 ; 012_ 234_
1148 pshufb ym2, ym3
1149 pmaddubsw ym2, ym6
1150 mova [tmpq], ym2
1151 add tmpq, 32
1152 sub hd, 4
1153 jg .v_w4_loop
1154 RET
1155 .v_w8:
1156 mova m5, [bilin_v_perm8]
1157 vbroadcasti32x4 ym0, [srcq+strideq*0]
1158 .v_w8_loop:
1159 vinserti32x4 ym1, ym0, [srcq+strideq*1], 1
1160 vpbroadcastq ym0, [srcq+strideq*2]
1161 vinserti32x4 m1, [srcq+stride3q ], 2
1162 lea srcq, [srcq+strideq*4]
1163 vinserti32x4 ym0, [srcq+strideq*0], 0
1164 vpermt2b m1, m5, m0
1165 pmaddubsw m1, m6
1166 mova [tmpq], m1
1167 add tmpq, 64
1168 sub hd, 4
1169 jg .v_w8_loop
1170 RET
1171 .v_w16:
1172 mova m5, [bilin_v_perm16]
1173 movu xm0, [srcq+strideq*0]
1174 .v_w16_loop:
1175 movu xm2, [srcq+strideq*2]
1176 vinserti32x4 ym1, ym0, [srcq+strideq*1], 1
1177 vpermt2b m1, m5, m2
1178 vinserti32x4 ym2, [srcq+stride3q ], 1
1179 lea srcq, [srcq+strideq*4]
1180 movu xm0, [srcq+strideq*0]
1181 vpermt2b m2, m5, m0
1182 pmaddubsw m1, m6
1183 pmaddubsw m2, m6
1184 mova [tmpq+64*0], m1
1185 mova [tmpq+64*1], m2
1186 add tmpq, 64*2
1187 sub hd, 4
1188 jg .v_w16_loop
1189 RET
1190 .v_w32:
1191 mova m5, [bilin_v_perm32]
1192 movu ym0, [srcq+strideq*0]
1193 .v_w32_loop:
1194 movu ym2, [srcq+strideq*1]
1195 movu ym3, [srcq+strideq*2]
1196 movu ym4, [srcq+stride3q ]
1197 lea srcq, [srcq+strideq*4]
1198 vpermt2b m0, m5, m2
1199 vpermt2b m2, m5, m3
1200 vpermt2b m3, m5, m4
1201 pmaddubsw m1, m0, m6
1202 movu ym0, [srcq+strideq*0]
1203 vpermt2b m4, m5, m0
1204 pmaddubsw m2, m6
1205 pmaddubsw m3, m6
1206 pmaddubsw m4, m6
1207 mova [tmpq+64*0], m1
1208 mova [tmpq+64*1], m2
1209 mova [tmpq+64*2], m3
1210 mova [tmpq+64*3], m4
1211 add tmpq, 64*4
1212 sub hd, 4
1213 jg .v_w32_loop
1214 RET
1215 .v_w64:
1216 mova m5, [bilin_v_perm64]
1217 vpermq m0, m5, [srcq+strideq*0]
1218 .v_w64_loop:
1219 vpermq m1, m5, [srcq+strideq*1]
1220 lea srcq, [srcq+strideq*2]
1221 punpcklbw m4, m1, m0
1222 punpckhbw m2, m1, m0
1223 vpermq m0, m5, [srcq+strideq*0]
1224 punpcklbw m3, m0, m1
1225 punpckhbw m1, m0, m1
1226 pmaddubsw m4, m6
1227 pmaddubsw m2, m6
1228 pmaddubsw m3, m6
1229 pmaddubsw m1, m6
1230 mova [tmpq+64*0], m4
1231 mova [tmpq+64*1], m2
1232 mova [tmpq+64*2], m3
1233 mova [tmpq+64*3], m1
1234 add tmpq, 64*4
1235 sub hd, 2
1236 jg .v_w64_loop
1237 RET
1238 .v_w128:
1239 mova m5, [bilin_v_perm64]
1240 vpermq m0, m5, [srcq+strideq*0+ 0]
1241 vpermq m1, m5, [srcq+strideq*0+64]
1242 .v_w128_loop:
1243 vpermq m2, m5, [srcq+strideq*1+ 0]
1244 vpermq m3, m5, [srcq+strideq*1+64]
1245 lea srcq, [srcq+strideq*2]
1246 punpcklbw m4, m2, m0
1247 punpckhbw m0, m2, m0
1248 pmaddubsw m4, m6
1249 pmaddubsw m0, m6
1250 mova [tmpq+64*0], m4
1251 mova [tmpq+64*1], m0
1252 punpcklbw m4, m3, m1
1253 punpckhbw m1, m3, m1
1254 pmaddubsw m4, m6
1255 pmaddubsw m1, m6
1256 mova [tmpq+64*2], m4
1257 mova [tmpq+64*3], m1
1258 vpermq m0, m5, [srcq+strideq*0+ 0]
1259 vpermq m1, m5, [srcq+strideq*0+64]
1260 punpcklbw m4, m0, m2
1261 punpckhbw m2, m0, m2
1262 pmaddubsw m4, m6
1263 pmaddubsw m2, m6
1264 mova [tmpq+64*4], m4
1265 mova [tmpq+64*5], m2
1266 punpcklbw m4, m1, m3
1267 punpckhbw m3, m1, m3
1268 pmaddubsw m4, m6
1269 pmaddubsw m3, m6
1270 mova [tmpq+64*6], m4
1271 mova [tmpq+64*7], m3
1272 add tmpq, 64*8
1273 sub hd, 2
1274 jg .v_w128_loop
1275 RET
1276 .hv:
1277 ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4
1278 ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4)
1279 %assign stack_offset stack_offset - stack_size_padded
1280 WIN64_SPILL_XMM 7
1281 movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)]
1282 shl mxyd, 11
1283 vpbroadcastw m6, mxyd
1284 add wq, t2
1285 lea stride3q, [strideq*3]
1286 jmp wq
1287 .hv_w4:
1288 vbroadcasti32x4 ym4, [bilin_h_shuf4]
1289 vpbroadcastq ym0, [srcq+strideq*0]
1290 pshufb ym0, ym4
1291 pmaddubsw ym0, ym5
1292 .hv_w4_loop:
1293 movq xmm1, [srcq+strideq*1]
1294 movq xmm2, [srcq+strideq*2]
1295 vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1
1296 lea srcq, [srcq+strideq*4]
1297 vinserti32x4 ym2, ymm2, [srcq+strideq*0], 1
1298 punpcklqdq ym1, ym2
1299 pshufb ym1, ym4
1300 pmaddubsw ym1, ym5 ; 1 2 3 4
1301 valignq ym2, ym1, ym0, 3 ; 0 1 2 3
1302 mova ym0, ym1
1303 psubw ym1, ym2
1304 pmulhrsw ym1, ym6
1305 paddw ym1, ym2
1306 mova [tmpq], ym1
1307 add tmpq, 32
1308 sub hd, 4
1309 jg .hv_w4_loop
1310 RET
1311 .hv_w8:
1312 vbroadcasti32x4 m4, [bilin_h_shuf8]
1313 vbroadcasti32x4 m0, [srcq+strideq*0]
1314 pshufb m0, m4
1315 pmaddubsw m0, m5
1316 .hv_w8_loop:
1317 movu xmm1, [srcq+strideq*1]
1318 vinserti128 ym1, ymm1, [srcq+strideq*2], 1
1319 vinserti128 m1, [srcq+stride3q ], 2
1320 lea srcq, [srcq+strideq*4]
1321 vinserti128 m1, [srcq+strideq*0], 3
1322 pshufb m1, m4
1323 pmaddubsw m1, m5 ; 1 2 3 4
1324 valignq m2, m1, m0, 6 ; 0 1 2 3
1325 mova m0, m1
1326 psubw m1, m2
1327 pmulhrsw m1, m6
1328 paddw m1, m2
1329 mova [tmpq], m1
1330 add tmpq, 64
1331 sub hd, 4
1332 jg .hv_w8_loop
1333 RET
1334 .hv_w16:
1335 mova m4, [bilin_h_perm16]
1336 vbroadcasti32x8 m0, [srcq+strideq*0]
1337 vpermb m0, m4, m0
1338 pmaddubsw m0, m5
1339 .hv_w16_loop:
1340 movu ym1, [srcq+strideq*1]
1341 vinserti32x8 m1, [srcq+strideq*2], 1
1342 movu ym2, [srcq+stride3q ]
1343 lea srcq, [srcq+strideq*4]
1344 vinserti32x8 m2, [srcq+strideq*0], 1
1345 vpermb m1, m4, m1
1346 vpermb m2, m4, m2
1347 pmaddubsw m1, m5 ; 1 2
1348 vshufi32x4 m3, m0, m1, q1032 ; 0 1
1349 pmaddubsw m0, m2, m5 ; 3 4
1350 vshufi32x4 m2, m1, m0, q1032 ; 2 3
1351 psubw m1, m3
1352 pmulhrsw m1, m6
1353 paddw m1, m3
1354 psubw m3, m0, m2
1355 pmulhrsw m3, m6
1356 paddw m3, m2
1357 mova [tmpq+64*0], m1
1358 mova [tmpq+64*1], m3
1359 add tmpq, 64*2
1360 sub hd, 4
1361 jg .hv_w16_loop
1362 RET
1363 .hv_w32:
1364 mova m4, [bilin_h_perm32]
1365 vpermb m0, m4, [srcq+strideq*0]
1366 pmaddubsw m0, m5
1367 .hv_w32_loop:
1368 vpermb m1, m4, [srcq+strideq*1]
1369 lea srcq, [srcq+strideq*2]
1370 vpermb m2, m4, [srcq+strideq*0]
1371 pmaddubsw m1, m5
1372 psubw m3, m1, m0
1373 pmulhrsw m3, m6
1374 paddw m3, m0
1375 pmaddubsw m0, m2, m5
1376 psubw m2, m0, m1
1377 pmulhrsw m2, m6
1378 paddw m2, m1
1379 mova [tmpq+64*0], m3
1380 mova [tmpq+64*1], m2
1381 add tmpq, 64*2
1382 sub hd, 2
1383 jg .hv_w32_loop
1384 RET
1385 .hv_w64:
1386 mova m4, [bilin_h_perm32]
1387 vpermb m0, m4, [srcq+32*0]
1388 vpermb m1, m4, [srcq+32*1]
1389 pmaddubsw m0, m5
1390 pmaddubsw m1, m5
1391 .hv_w64_loop:
1392 add srcq, strideq
1393 vpermb m2, m4, [srcq+32*0]
1394 vpermb m3, m4, [srcq+32*1]
1395 pmaddubsw m2, m5
1396 pmaddubsw m3, m5
1397 psubw m7, m2, m0
1398 psubw m8, m3, m1
1399 pmulhrsw m7, m6
1400 pmulhrsw m8, m6
1401 paddw m7, m0
1402 mova m0, m2
1403 paddw m8, m1
1404 mova m1, m3
1405 mova [tmpq+64*0], m7
1406 mova [tmpq+64*1], m8
1407 add tmpq, 64*2
1408 dec hd
1409 jg .hv_w64_loop
1410 RET
1411 .hv_w128:
1412 mova m4, [bilin_h_perm32]
1413 vpermb m0, m4, [srcq+32*0]
1414 vpermb m1, m4, [srcq+32*1]
1415 vpermb m2, m4, [srcq+32*2]
1416 vpermb m3, m4, [srcq+32*3]
1417 REPX {pmaddubsw x, m5}, m0, m1, m2, m3
1418 .hv_w128_loop:
1419 add srcq, strideq
1420 vpermb m7, m4, [srcq+32*0]
1421 vpermb m8, m4, [srcq+32*1]
1422 vpermb m9, m4, [srcq+32*2]
1423 vpermb m10, m4, [srcq+32*3]
1424 REPX {pmaddubsw x, m5}, m7, m8, m9, m10
1425 psubw m11, m7, m0
1426 psubw m12, m8, m1
1427 psubw m13, m9, m2
1428 psubw m14, m10, m3
1429 REPX {pmulhrsw x, m6}, m11, m12, m13, m14
1430 paddw m11, m0
1431 mova m0, m7
1432 paddw m12, m1
1433 mova m1, m8
1434 paddw m13, m2
1435 mova m2, m9
1436 paddw m14, m3
1437 mova m3, m10
1438 mova [tmpq+64*0], m11
1439 mova [tmpq+64*1], m12
1440 mova [tmpq+64*2], m13
1441 mova [tmpq+64*3], m14
1442 add tmpq, 64*4
1443 dec hd
1444 jg .hv_w128_loop
1445 RET
1446
1447 ; int8_t subpel_filters[5][15][8]
1448 %assign FILTER_REGULAR (0*15 << 16) | 3*15
1449 %assign FILTER_SMOOTH (1*15 << 16) | 4*15
1450 %assign FILTER_SHARP (2*15 << 16) | 3*15
1451
1452 %macro FN 4 ; fn, type, type_h, type_v
1453 cglobal %1_%2_8bpc
1454 mov t0d, FILTER_%3
1455 %ifidn %3, %4
1456 mov t1d, t0d
1457 %else
1458 mov t1d, FILTER_%4
1459 %endif
1460 %ifnidn %2, regular ; skip the jump in the last filter
1461 jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX)
1462 %endif
1463 %endmacro
1464
1465 %macro PUT_8TAP_H 4-5 0 ; dst/src, tmp[1-3], vpermb
1466 %if %5
1467 vpermb m%2, m6, m%1
1468 vpermb m%3, m7, m%1
1469 vpermb m%4, m8, m%1
1470 %else
1471 %if %2 < %4 ; reuse a previous value if possible
1472 pshufb m%2, m%1, m6
1473 %endif
1474 pshufb m%3, m%1, m7
1475 pshufb m%4, m%1, m8
1476 %endif
1477 mova m%1, m5
1478 vpdpbusd m%1, m%2, m9
1479 mova m%2, m5
1480 vpdpbusd m%2, m%3, m9
1481 vpdpbusd m%1, m%3, m10
1482 vpdpbusd m%2, m%4, m10
1483 packusdw m%1, m%2
1484 psrlw m%1, 6
1485 %endmacro
1486
1487 %if WIN64
1488 DECLARE_REG_TMP 4, 5
1489 %else
1490 DECLARE_REG_TMP 7, 8
1491 %endif
1492
1493 %define PUT_8TAP_FN FN put_8tap,
1494
1495 PUT_8TAP_FN sharp, SHARP, SHARP
1496 PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH
1497 PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP
1498 PUT_8TAP_FN smooth, SMOOTH, SMOOTH
1499 PUT_8TAP_FN sharp_regular, SHARP, REGULAR
1500 PUT_8TAP_FN regular_sharp, REGULAR, SHARP
1501 PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR
1502 PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH
1503 PUT_8TAP_FN regular, REGULAR, REGULAR
1504
1505 cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
1506 %define base r8-put_avx512icl
1507 imul mxd, mxm, 0x010101
1508 add mxd, t0d ; 8tap_h, mx, 4tap_h
1509 imul myd, mym, 0x010101
1510 add myd, t1d ; 8tap_v, my, 4tap_v
1511 lea r8, [put_avx512icl]
1512 movsxd wq, wm
1513 movifnidn hd, hm
1514 test mxd, 0xf00
1515 jnz .h
1516 test myd, 0xf00
1517 jnz .v
1518 tzcnt wd, wd
1519 movzx wd, word [r8+wq*2+table_offset(put,)]
1520 add wq, r8
1521 lea r6, [ssq*3]
1522 lea r7, [dsq*3]
1523 %if WIN64
1524 pop r8
1525 %endif
1526 jmp wq
1527 .h:
1528 test myd, 0xf00
1529 jnz .hv
1530 vpbroadcastd m5, [pd_34] ; 2 + (8 << 2)
1531 WIN64_SPILL_XMM 11
1532 cmp wd, 4
1533 jl .h_w2
1534 vbroadcasti128 m6, [subpel_h_shufA]
1535 je .h_w4
1536 tzcnt wd, wd
1537 vbroadcasti128 m7, [subpel_h_shufB]
1538 vbroadcasti128 m8, [subpel_h_shufC]
1539 shr mxd, 16
1540 sub srcq, 3
1541 movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)]
1542 vpbroadcastd m9, [base+mxq*8+subpel_filters+0]
1543 vpbroadcastd m10, [base+mxq*8+subpel_filters+4]
1544 add wq, r8
1545 jmp wq
1546 .h_w2:
1547 movzx mxd, mxb
1548 dec srcq
1549 mova xmm4, [subpel_h_shuf4]
1550 vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
1551 .h_w2_loop:
1552 movq xmm0, [srcq+ssq*0]
1553 movhps xmm0, [srcq+ssq*1]
1554 lea srcq, [srcq+ssq*2]
1555 pshufb xmm0, xmm4
1556 mova xmm1, xm5
1557 vpdpbusd xmm1, xmm0, xmm3
1558 packssdw xmm0, xmm1, xmm1
1559 psraw xmm0, 6
1560 packuswb xmm0, xm0
1561 pextrw [dstq+dsq*0], xmm0, 0
1562 pextrw [dstq+dsq*1], xmm0, 1
1563 lea dstq, [dstq+dsq*2]
1564 sub hd, 2
1565 jg .h_w2_loop
1566 RET
1567 .h_w4:
1568 movzx mxd, mxb
1569 dec srcq
1570 vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
1571 .h_w4_loop:
1572 movq xmm0, [srcq+ssq*0]
1573 movq xmm1, [srcq+ssq*1]
1574 lea srcq, [srcq+ssq*2]
1575 pshufb xmm0, xm6
1576 pshufb xmm1, xm6
1577 mova xmm2, xm5
1578 vpdpbusd xmm2, xmm0, xmm3
1579 mova xmm0, xm5
1580 vpdpbusd xmm0, xmm1, xmm3
1581 packssdw xmm0, xmm2, xmm0
1582 psraw xmm0, 6
1583 packuswb xmm0, xmm0
1584 movd [dstq+dsq*0], xmm0
1585 pextrd [dstq+dsq*1], xmm0, 1
1586 lea dstq, [dstq+dsq*2]
1587 sub hd, 2
1588 jg .h_w4_loop
1589 RET
1590 .h_w8:
1591 movu xm0, [srcq+ssq*0]
1592 vinserti32x4 ym0, [srcq+ssq*1], 1
1593 lea srcq, [srcq+ssq*2]
1594 WRAP_YMM PUT_8TAP_H 0, 1, 2, 3
1595 vpmovuswb xm0, ym0
1596 movq [dstq+dsq*0], xm0
1597 movhps [dstq+dsq*1], xm0
1598 lea dstq, [dstq+dsq*2]
1599 sub hd, 2
1600 jg .h_w8
1601 RET
1602 .h_w16:
1603 mova m6, [spel_h_perm16a]
1604 mova m7, [spel_h_perm16b]
1605 mova m8, [spel_h_perm16c]
1606 .h_w16_loop:
1607 movu ym0, [srcq+ssq*0]
1608 vinserti32x8 m0, [srcq+ssq*1], 1
1609 lea srcq, [srcq+ssq*2]
1610 PUT_8TAP_H 0, 1, 2, 3, 1
1611 vpmovuswb ym0, m0
1612 mova [dstq+dsq*0], xm0
1613 vextracti128 [dstq+dsq*1], ym0, 1
1614 lea dstq, [dstq+dsq*2]
1615 sub hd, 2
1616 jg .h_w16_loop
1617 RET
1618 .h_w32:
1619 movu ym0, [srcq+ssq*0+8*0]
1620 vinserti32x8 m0, [srcq+ssq*1+8*0], 1
1621 movu ym1, [srcq+ssq*0+8*1]
1622 vinserti32x8 m1, [srcq+ssq*1+8*1], 1
1623 lea srcq, [srcq+ssq*2]
1624 PUT_8TAP_H 0, 2, 3, 4
1625 PUT_8TAP_H 1, 4, 3, 2
1626 packuswb m0, m1
1627 mova [dstq+dsq*0], ym0
1628 vextracti32x8 [dstq+dsq*1], m0, 1
1629 lea dstq, [dstq+dsq*2]
1630 sub hd, 2
1631 jg .h_w32
1632 RET
1633 .h_w64:
1634 movu m0, [srcq+8*0]
1635 movu m1, [srcq+8*1]
1636 add srcq, ssq
1637 PUT_8TAP_H 0, 2, 3, 4
1638 PUT_8TAP_H 1, 4, 3, 2
1639 packuswb m0, m1
1640 mova [dstq], m0
1641 add dstq, dsq
1642 dec hd
1643 jg .h_w64
1644 RET
1645 .h_w128:
1646 movu m0, [srcq+8*0]
1647 movu m2, [srcq+8*1]
1648 movu m1, [srcq+8*8]
1649 movu m3, [srcq+8*9]
1650 add srcq, ssq
1651 PUT_8TAP_H 0, 4, 11, 12
1652 PUT_8TAP_H 2, 12, 11, 4
1653 PUT_8TAP_H 1, 4, 11, 12
1654 PUT_8TAP_H 3, 12, 11, 4
1655 packuswb m0, m2
1656 packuswb m1, m3
1657 mova [dstq+64*0], m0
1658 mova [dstq+64*1], m1
1659 add dstq, dsq
1660 dec hd
1661 jg .h_w128
1662 RET
1663 .v:
1664 movzx mxd, myb
1665 shr myd, 16
1666 cmp hd, 6
1667 cmovs myd, mxd
1668 tzcnt r6d, wd
1669 movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)]
1670 vpbroadcastd m7, [pw_512]
1671 lea myq, [base+subpel_filters+myq*8]
1672 vpbroadcastw m8, [myq+0]
1673 vpbroadcastw m9, [myq+2]
1674 vpbroadcastw m10, [myq+4]
1675 vpbroadcastw m11, [myq+6]
1676 add r6, r8
1677 lea ss3q, [ssq*3]
1678 sub srcq, ss3q
1679 jmp r6
1680 .v_w2:
1681 movd xmm2, [srcq+ssq*0]
1682 pinsrw xmm2, [srcq+ssq*1], 2
1683 pinsrw xmm2, [srcq+ssq*2], 4
1684 add srcq, ss3q
1685 pinsrw xmm2, [srcq+ssq*0], 6 ; 0 1 2 3
1686 movd xmm3, [srcq+ssq*1]
1687 vpbroadcastd xmm1, [srcq+ssq*2]
1688 add srcq, ss3q
1689 vpbroadcastd xmm0, [srcq+ssq*0]
1690 vpblendd xmm3, xmm3, xmm1, 0x02 ; 4 5
1691 vpblendd xmm1, xmm1, xmm0, 0x02 ; 5 6
1692 palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4
1693 punpcklbw xmm3, xmm1 ; 45 56
1694 punpcklbw xmm1, xmm2, xmm4 ; 01 12
1695 punpckhbw xmm2, xmm4 ; 23 34
1696 .v_w2_loop:
1697 pmaddubsw xmm5, xmm1, xm8 ; a0 b0
1698 mova xmm1, xmm2
1699 pmaddubsw xmm2, xm9 ; a1 b1
1700 paddw xmm5, xmm2
1701 mova xmm2, xmm3
1702 pmaddubsw xmm3, xm10 ; a2 b2
1703 paddw xmm5, xmm3
1704 vpbroadcastd xmm4, [srcq+ssq*1]
1705 lea srcq, [srcq+ssq*2]
1706 vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7
1707 vpbroadcastd xmm0, [srcq+ssq*0]
1708 vpblendd xmm4, xmm4, xmm0, 0x02 ; 7 8
1709 punpcklbw xmm3, xmm4 ; 67 78
1710 pmaddubsw xmm4, xmm3, xm11 ; a3 b3
1711 paddw xmm5, xmm4
1712 pmulhrsw xmm5, xm7
1713 packuswb xmm5, xmm5
1714 pextrw [dstq+dsq*0], xmm5, 0
1715 pextrw [dstq+dsq*1], xmm5, 2
1716 lea dstq, [dstq+dsq*2]
1717 sub hd, 2
1718 jg .v_w2_loop
1719 RET
1720 .v_w4:
1721 movd xmm2, [srcq+ssq*0]
1722 pinsrd xmm2, [srcq+ssq*1], 1
1723 pinsrd xmm2, [srcq+ssq*2], 2
1724 add srcq, ss3q
1725 pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3
1726 movd xmm3, [srcq+ssq*1]
1727 vpbroadcastd xmm1, [srcq+ssq*2]
1728 add srcq, ss3q
1729 vpbroadcastd xmm0, [srcq+ssq*0]
1730 vpblendd xmm3, xmm3, xmm1, 0x02 ; 4 5
1731 vpblendd xmm1, xmm1, xmm0, 0x02 ; 5 6
1732 palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4
1733 punpcklbw xmm3, xmm1 ; 45 56
1734 punpcklbw xmm1, xmm2, xmm4 ; 01 12
1735 punpckhbw xmm2, xmm4 ; 23 34
1736 .v_w4_loop:
1737 vpbroadcastd xmm4, [srcq+ssq*1]
1738 lea srcq, [srcq+ssq*2]
1739 pmaddubsw xmm5, xmm1, xm8 ; a0 b0
1740 mova xmm1, xmm2
1741 pmaddubsw xmm2, xm9 ; a1 b1
1742 paddw xmm5, xmm2
1743 mova xmm2, xmm3
1744 pmaddubsw xmm3, xm10 ; a2 b2
1745 paddw xmm5, xmm3
1746 vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7
1747 vpbroadcastd xmm0, [srcq+ssq*0]
1748 vpblendd xmm4, xmm4, xmm0, 0x02 ; 7 8
1749 punpcklbw xmm3, xmm4 ; 67 78
1750 pmaddubsw xmm4, xmm3, xm11 ; a3 b3
1751 paddw xmm5, xmm4
1752 pmulhrsw xmm5, xm7
1753 packuswb xmm5, xmm5
1754 movd [dstq+dsq*0], xmm5
1755 pextrd [dstq+dsq*1], xmm5, 1
1756 lea dstq, [dstq+dsq*2]
1757 sub hd, 2
1758 jg .v_w4_loop
1759 RET
1760 .v_w8:
1761 movq xmm1, [srcq+ssq*0]
1762 vpbroadcastq ymm0, [srcq+ssq*1]
1763 vpbroadcastq ymm2, [srcq+ssq*2]
1764 add srcq, ss3q
1765 vpbroadcastq ymm5, [srcq+ssq*0]
1766 vpbroadcastq ymm3, [srcq+ssq*1]
1767 vpbroadcastq ymm4, [srcq+ssq*2]
1768 add srcq, ss3q
1769 vpblendd ymm1, ymm0, 0x30
1770 vpblendd ymm0, ymm2, 0x30
1771 punpcklbw ymm1, ymm0 ; 01 12
1772 vpbroadcastq ymm0, [srcq+ssq*0]
1773 vpblendd ymm2, ymm5, 0x30
1774 vpblendd ymm5, ymm3, 0x30
1775 punpcklbw ymm2, ymm5 ; 23 34
1776 vpblendd ymm3, ymm4, 0x30
1777 vpblendd ymm4, ymm0, 0x30
1778 punpcklbw ymm3, ymm4 ; 45 56
1779 .v_w8_loop:
1780 vpbroadcastq ymm4, [srcq+ssq*1]
1781 lea srcq, [srcq+ssq*2]
1782 pmaddubsw ymm5, ymm1, ym8 ; a0 b0
1783 mova ymm1, ymm2
1784 pmaddubsw ymm2, ym9 ; a1 b1
1785 paddw ymm5, ymm2
1786 mova ymm2, ymm3
1787 pmaddubsw ymm3, ym10 ; a2 b2
1788 paddw ymm5, ymm3
1789 vpblendd ymm3, ymm0, ymm4, 0x30
1790 vpbroadcastq ymm0, [srcq+ssq*0]
1791 vpblendd ymm4, ymm4, ymm0, 0x30
1792 punpcklbw ymm3, ymm4 ; 67 78
1793 pmaddubsw ymm4, ymm3, ym11 ; a3 b3
1794 paddw ymm5, ymm4
1795 pmulhrsw ymm5, ym7
1796 vextracti128 xmm4, ymm5, 1
1797 packuswb xmm5, xmm4
1798 movq [dstq+dsq*0], xmm5
1799 movhps [dstq+dsq*1], xmm5
1800 lea dstq, [dstq+dsq*2]
1801 sub hd, 2
1802 jg .v_w8_loop
1803 vzeroupper
1804 RET
1805 .v_w16:
1806 mova m12, [spel_v_perm16]
1807 vbroadcasti32x4 m1, [srcq+ssq*0]
1808 vbroadcasti32x4 ym4, [srcq+ssq*1]
1809 mov r6d, 0x0f
1810 vbroadcasti32x4 m2, [srcq+ssq*2]
1811 add srcq, ss3q
1812 vbroadcasti32x4 ym5, [srcq+ssq*0]
1813 kmovb k1, r6d
1814 vbroadcasti32x4 m3, [srcq+ssq*1]
1815 vbroadcasti32x4 ym6, [srcq+ssq*2]
1816 add srcq, ss3q
1817 vbroadcasti32x4 m0, [srcq+ssq*0]
1818 vshufpd m1{k1}, m4, m2, 0xcc
1819 vshufpd m2{k1}, m5, m3, 0xcc
1820 vshufpd m3{k1}, m6, m0, 0xcc
1821 vpermb m1, m12, m1 ; 01 12
1822 vpermb m2, m12, m2 ; 23 34
1823 vpermb m3, m12, m3 ; 45 56
1824 .v_w16_loop:
1825 pmaddubsw m4, m1, m8 ; a0 b0
1826 mova m1, m2
1827 pmaddubsw m5, m2, m9 ; a1 b1
1828 mova m2, m3
1829 pmaddubsw m6, m3, m10 ; a2 b2
1830 mova m3, m0
1831 paddw m4, m5
1832 vbroadcasti32x4 ym5, [srcq+ssq*1]
1833 lea srcq, [srcq+ssq*2]
1834 vbroadcasti32x4 m0, [srcq+ssq*0]
1835 vshufpd m3{k1}, m5, m0, 0xcc
1836 vpermb m3, m12, m3 ; 67 78
1837 pmaddubsw m5, m3, m11 ; a3 b3
1838 paddw m4, m6
1839 paddw m4, m5
1840 pmulhrsw m4, m7
1841 vextracti32x8 ym5, m4, 1
1842 packuswb ym4, ym5
1843 mova [dstq+dsq*0], xm4
1844 vextracti32x4 [dstq+dsq*1], ym4, 1
1845 lea dstq, [dstq+dsq*2]
1846 sub hd, 2
1847 jg .v_w16_loop
1848 RET
1849 .v_w32:
1850 mova m12, [spel_v_perm32]
1851 pmovzxbq m14, [pb_02461357]
1852 vpshrdw m13, m12, m12, 8
1853 movu ym0, [srcq+ssq*0]
1854 vinserti32x8 m0, [srcq+ssq*1], 1
1855 vpermb m1, m12, m0 ; 01
1856 vinserti32x8 m0, [srcq+ssq*2], 0
1857 add srcq, ss3q
1858 vpermb m2, m13, m0 ; 12
1859 vinserti32x8 m0, [srcq+ssq*0], 1
1860 vpermb m3, m12, m0 ; 23
1861 vinserti32x8 m0, [srcq+ssq*1], 0
1862 vpermb m4, m13, m0 ; 34
1863 vinserti32x8 m0, [srcq+ssq*2], 1
1864 add srcq, ss3q
1865 vpermb m5, m12, m0 ; 45
1866 vinserti32x8 m0, [srcq+ssq*0], 0
1867 vpermb m6, m13, m0 ; 56
1868 .v_w32_loop:
1869 vinserti32x8 m0, [srcq+ssq*1], 1
1870 lea srcq, [srcq+ssq*2]
1871 pmaddubsw m15, m1, m8
1872 mova m1, m3
1873 pmaddubsw m16, m2, m8
1874 mova m2, m4
1875 pmaddubsw m17, m3, m9
1876 mova m3, m5
1877 pmaddubsw m18, m4, m9
1878 mova m4, m6
1879 pmaddubsw m19, m5, m10
1880 vpermb m5, m12, m0 ; 67
1881 vinserti32x8 m0, [srcq+ssq*0], 0
1882 pmaddubsw m20, m6, m10
1883 vpermb m6, m13, m0 ; 78
1884 paddw m15, m17
1885 pmaddubsw m17, m5, m11
1886 paddw m16, m18
1887 pmaddubsw m18, m6, m11
1888 paddw m15, m19
1889 paddw m16, m20
1890 paddw m15, m17
1891 paddw m16, m18
1892 pmulhrsw m15, m7
1893 pmulhrsw m16, m7
1894 packuswb m15, m16
1895 vpermq m15, m14, m15
1896 mova [dstq+dsq*0], ym15
1897 vextracti32x8 [dstq+dsq*1], m15, 1
1898 lea dstq, [dstq+dsq*2]
1899 sub hd, 2
1900 jg .v_w32_loop
1901 vzeroupper
1902 RET
1903 .v_w64:
1904 .v_w128:
1905 lea r6d, [hq+wq*4-256]
1906 mov r4, srcq
1907 mov r7, dstq
1908 .v_loop0:
1909 movu m2, [srcq+ssq*0]
1910 movu m4, [srcq+ssq*1]
1911 movu m6, [srcq+ssq*2]
1912 add srcq, ss3q
1913 movu m13, [srcq+ssq*0]
1914 movu m15, [srcq+ssq*1]
1915 movu m17, [srcq+ssq*2]
1916 add srcq, ss3q
1917 movu m0, [srcq+ssq*0]
1918 punpcklbw m1, m2, m4 ; 01l
1919 punpckhbw m2, m4 ; 01h
1920 punpcklbw m3, m4, m6 ; 12l
1921 punpckhbw m4, m6 ; 12h
1922 punpcklbw m5, m6, m13 ; 23l
1923 punpckhbw m6, m13 ; 23h
1924 punpcklbw m12, m13, m15 ; 34l
1925 punpckhbw m13, m15 ; 34h
1926 punpcklbw m14, m15, m17 ; 45l
1927 punpckhbw m15, m17 ; 45h
1928 punpcklbw m16, m17, m0 ; 56l
1929 punpckhbw m17, m0 ; 56h
1930 .v_loop:
1931 pmaddubsw m18, m1, m8 ; a0l
1932 mova m1, m5
1933 pmaddubsw m19, m2, m8 ; a0h
1934 mova m2, m6
1935 pmaddubsw m20, m3, m8 ; b0l
1936 mova m3, m12
1937 pmaddubsw m21, m4, m8 ; b0h
1938 mova m4, m13
1939 pmaddubsw m5, m9 ; a1l
1940 pmaddubsw m6, m9 ; a1h
1941 pmaddubsw m12, m9 ; b1l
1942 pmaddubsw m13, m9 ; b1h
1943 paddw m18, m5
1944 mova m5, m14
1945 pmaddubsw m14, m10 ; a2l
1946 paddw m19, m6
1947 mova m6, m15
1948 pmaddubsw m15, m10 ; a2h
1949 paddw m20, m12
1950 mova m12, m16
1951 pmaddubsw m16, m10 ; b2l
1952 paddw m21, m13
1953 mova m13, m17
1954 pmaddubsw m17, m10 ; b2h
1955 paddw m18, m14
1956 paddw m19, m15
1957 paddw m20, m16
1958 paddw m21, m17
1959 movu m17, [srcq+ssq*1]
1960 lea srcq, [srcq+ssq*2]
1961 punpcklbw m14, m0, m17 ; 67l
1962 punpckhbw m15, m0, m17 ; 67h
1963 pmaddubsw m16, m14, m11 ; a3l
1964 pmaddubsw m0, m15, m11 ; a3h
1965 paddw m18, m16
1966 paddw m19, m0
1967 movu m0, [srcq+ssq*0]
1968 punpcklbw m16, m17, m0 ; 78l
1969 punpckhbw m17, m0 ; 78h
1970 pmulhrsw m18, m7
1971 pmulhrsw m19, m7
1972 packuswb m18, m19
1973 mova [dstq+dsq*0], m18
1974 pmaddubsw m18, m16, m11 ; b3l
1975 pmaddubsw m19, m17, m11 ; b3h
1976 paddw m18, m20
1977 paddw m19, m21
1978 pmulhrsw m18, m7
1979 pmulhrsw m19, m7
1980 packuswb m18, m19
1981 mova [dstq+dsq*1], m18
1982 lea dstq, [dstq+dsq*2]
1983 sub hd, 2
1984 jg .v_loop
1985 add r4, 64
1986 add r7, 64
1987 movzx hd, r6b
1988 mov srcq, r4
1989 mov dstq, r7
1990 sub r6d, 256
1991 jg .v_loop0
1992 vzeroupper
1993 RET
1994 .hv:
1995 cmp wd, 4
1996 jg .hv_w8
1997 movzx mxd, mxb
1998 dec srcq
1999 vpbroadcastd m7, [base+subpel_filters+mxq*8+2]
2000 movzx mxd, myb
2001 shr myd, 16
2002 cmp hd, 6
2003 cmovs myd, mxd
2004 vpbroadcastd m8, [pd_2]
2005 vpbroadcastq ym0, [base+subpel_filters+myq*8]
2006 lea ss3q, [ssq*3]
2007 vpbroadcastd ym9, [pd_32768]
2008 mov r6, srcq
2009 punpcklbw ym0, ym8, ym0
2010 sub r6, ss3q
2011 psraw ym0, 2 ; << 6
2012 mova xm14, [spel_hv_end]
2013 pshufd ym10, ym0, q0000
2014 pshufd ym11, ym0, q1111
2015 pshufd ym12, ym0, q2222
2016 pshufd ym13, ym0, q3333
2017 cmp wd, 4
2018 je .hv_w4
2019 vbroadcasti128 ym6, [subpel_h_shuf4]
2020 movq xmm2, [r6+ssq*0]
2021 movhps xmm2, [r6+ssq*1]
2022 movq xmm0, [r6+ssq*2]
2023 movhps xmm0, [srcq+ssq*0]
2024 vpbroadcastq ymm3, [srcq+ssq*1]
2025 vpbroadcastq ymm4, [srcq+ssq*2]
2026 add srcq, ss3q
2027 vpbroadcastq ymm1, [srcq+ssq*0]
2028 vpblendd ymm2, ymm3, 0x30
2029 vpblendd ymm0, ymm1, 0x30 ; 2 3 6 _
2030 vpblendd ymm2, ymm4, 0xc0 ; 0 1 4 5
2031 pshufb ymm2, ym6
2032 pshufb ymm0, ym6
2033 mova ymm1, ym8
2034 vpdpbusd ymm1, ymm2, ym7
2035 mova ymm2, ym8
2036 vpdpbusd ymm2, ymm0, ym7
2037 packssdw ymm2, ymm1, ymm2
2038 psraw ymm2, 2
2039 vextracti128 xmm3, ymm2, 1
2040 palignr xmm4, xmm3, xmm2, 4
2041 punpcklwd xmm1, xmm2, xmm4 ; 01 12
2042 punpckhwd xmm2, xmm4 ; 23 34
2043 pshufd xmm0, xmm3, q2121
2044 punpcklwd xmm3, xmm0 ; 45 56
2045 .hv_w2_loop:
2046 movq xmm4, [srcq+ssq*1]
2047 lea srcq, [srcq+ssq*2]
2048 movhps xmm4, [srcq+ssq*0]
2049 mova xmm5, xm9
2050 vpdpwssd xmm5, xmm1, xm10 ; a0 b0
2051 mova xmm1, xmm2
2052 vpdpwssd xmm5, xmm2, xm11 ; a1 b1
2053 pshufb xmm4, xm6
2054 mova xmm2, xmm3
2055 vpdpwssd xmm5, xmm3, xm12 ; a2 b2
2056 mova xmm3, xm8
2057 vpdpbusd xmm3, xmm4, xm7
2058 packssdw xmm4, xmm3, xmm3
2059 psraw xmm4, 2
2060 palignr xmm3, xmm4, xmm0, 12
2061 mova xmm0, xmm4
2062 punpcklwd xmm3, xmm4 ; 67 78
2063 vpdpwssd xmm5, xmm3, xm13 ; a3 b3
2064 packuswb xmm5, xmm5
2065 pshufb xmm5, xm14
2066 pextrw [dstq+dsq*0], xmm5, 0
2067 pextrw [dstq+dsq*1], xmm5, 1
2068 lea dstq, [dstq+dsq*2]
2069 sub hd, 2
2070 jg .hv_w2_loop
2071 vzeroupper
2072 RET
2073 .hv_w4:
2074 movq xmm1, [r6+ssq*0]
2075 vpbroadcastq ym2, [r6+ssq*1]
2076 vinserti32x4 ym1, ymm1, [r6+ssq*2], 1
2077 vinserti32x4 m2, [srcq+ssq*0], 2
2078 vinserti32x4 m1, [srcq+ssq*1], 2
2079 vinserti32x4 m2, [srcq+ssq*2], 3 ; _ 1 3 5
2080 vbroadcasti32x4 m6, [subpel_h_shufA]
2081 add srcq, ss3q
2082 vinserti32x4 m1, [srcq+ssq*0], 3 ; 0 2 4 6
2083 pshufb m2, m6
2084 pshufb m1, m6
2085 mova m0, m8
2086 vpdpbusd m0, m2, m7
2087 mova m4, m8
2088 vpdpbusd m4, m1, m7
2089 mova ym1, [spel_hv_perm4a]
2090 mova ym2, [spel_hv_perm4b]
2091 mova ym3, [spel_hv_perm4c]
2092 packssdw m0, m4
2093 psraw m0, 2 ; _ 0 1 2 3 4 5 6
2094 mov r6d, 0x5555
2095 vpermb ym1, ym1, ym0 ; 01 12
2096 vpermb m2, m2, m0 ; 23 34
2097 vpermb m3, m3, m0 ; 45 56
2098 kmovw k1, r6d
2099 mova ym15, [spel_hv_perm4d]
2100 .hv_w4_loop:
2101 movq xmm4, [srcq+ssq*1]
2102 lea srcq, [srcq+ssq*2]
2103 vinserti32x4 ym4, ymm4, [srcq+ssq*0], 1
2104 mova ym5, ym9
2105 vpdpwssd ym5, ym1, ym10 ; a0 b0
2106 mova ym1, ym2
2107 pshufb ym4, ym6
2108 mova ym0, ym8
2109 vpdpbusd ym0, ym4, ym7
2110 vpdpwssd ym5, ym2, ym11 ; a1 b1
2111 mova ym2, ym3
2112 vpdpwssd ym5, ym3, ym12 ; a2 b2
2113 vpsraw ym3{k1}, ym0, 2 ; 7 8
2114 vpermb ym3, ym15, ym3 ; 67 78
2115 vpdpwssd ym5, ym3, ym13 ; a3 b3
2116 packuswb ym5, ym5
2117 vpermb ym5, ym14, ym5
2118 movd [dstq+dsq*0], xm5
2119 pextrd [dstq+dsq*1], xm5, 1
2120 lea dstq, [dstq+dsq*2]
2121 sub hd, 2
2122 jg .hv_w4_loop
2123 RET
2124 .hv_w8:
2125 shr mxd, 16
2126 sub srcq, 3
2127 vpbroadcastd m10, [base+subpel_filters+mxq*8+0]
2128 vpbroadcastd m11, [base+subpel_filters+mxq*8+4]
2129 movzx mxd, myb
2130 shr myd, 16
2131 cmp hd, 6
2132 cmovs myd, mxd
2133 vpbroadcastd m8, [pd_2]
2134 vpbroadcastq m0, [base+subpel_filters+myq*8]
2135 vpbroadcastd m9, [pd_32768]
2136 punpcklbw m0, m8, m0
2137 lea ss3q, [ssq*3]
2138 psraw m0, 2 ; << 6
2139 pshufd m12, m0, q0000
2140 pshufd m13, m0, q1111
2141 pshufd m14, m0, q2222
2142 pshufd m15, m0, q3333
2143 cmp wd, 8
2144 jne .hv_w16
2145 mov r6, srcq
2146 sub r6, ss3q
2147 movu xmm1, [r6+ssq*0]
2148 vinserti128 ymm1, [r6+ssq*1], 1
2149 movu xmm2, [srcq+ssq*1]
2150 vinserti32x4 m6, zmm1, [r6+ssq*2], 2
2151 vinserti128 ymm2, [srcq+ssq*2], 1
2152 vinserti32x4 m6, [srcq+ssq*0], 3 ; 0 1 2 3
2153 add srcq, ss3q
2154 vbroadcasti32x4 m4, [subpel_h_shufA]
2155 vinserti32x4 m0, zmm2, [srcq+ssq*0], 2 ; 4 5 6 _
2156 vbroadcasti32x4 m7, [subpel_h_shufB]
2157 vbroadcasti32x4 m17, [subpel_h_shufC]
2158 pshufb m1, m6, m4 ; 0 1 2 3 0123
2159 mova m2, m8
2160 vpdpbusd m2, m1, m10
2161 pshufb m5, m6, m7 ; 0 1 2 3 4567
2162 mova m1, m8
2163 vpdpbusd m1, m5, m10
2164 pshufb m4, m0, m4 ; 4 5 6 _ 0123
2165 mova m3, m8
2166 vpdpbusd m3, m4, m10
2167 pshufb m7, m0, m7 ; 4 5 6 _ 4567
2168 mova m4, m8
2169 vpdpbusd m4, m7, m10
2170 pshufb m6, m17
2171 vpdpbusd m2, m5, m11
2172 vpdpbusd m1, m6, m11
2173 pshufb m6, m0, m17
2174 vpdpbusd m3, m7, m11
2175 vpdpbusd m4, m6, m11
2176 mova m5, [spel_hv_perm8a]
2177 mova m0, [spel_hv_perm8b]
2178 mov r6, 0x55555555ff00
2179 packssdw m2, m1
2180 packssdw m3, m4
2181 mova m18, [spel_hv_perm8c]
2182 psraw m2, 2 ; 0 1 2 3
2183 psraw m3, 2 ; 4 5 6 _
2184 vpermb m1, m5, m2 ; 01 12
2185 vbroadcasti32x8 m6, [subpel_h_shufA]
2186 kmovq k1, r6
2187 vpermt2b m2, m0, m3 ; 23 34
2188 vbroadcasti32x8 m7, [subpel_h_shufB]
2189 kshiftrq k2, k1, 16
2190 mova xm16, [spel_hv_end]
2191 vpermb m3, m5, m3 ; 45 56
2192 .hv_w8_loop:
2193 vbroadcasti32x4 ym4, [srcq+ssq*1]
2194 lea srcq, [srcq+ssq*2]
2195 vbroadcasti32x4 m4{k1}, [srcq+ssq*0]
2196 mova m0, m9
2197 vpdpwssd m0, m1, m12 ; a0 b0
2198 pshufb m1, m4, m6 ; 7 8 0123 4567
2199 mova m5, m8
2200 vpdpbusd m5, m1, m10
2201 pshufb m4, m7 ; 7 8 4567 89ab
2202 vpdpwssd m0, m2, m13 ; a1 b1
2203 mova m1, m2
2204 vpdpbusd m5, m4, m11
2205 mova m2, m3
2206 vpdpwssd m0, m3, m14 ; a2 b2
2207 psraw m3{k2}, m5, 2 ; 75 86
2208 vpermb m3, m18, m3 ; 67 78
2209 vpdpwssd m0, m3, m15 ; a3 b3
2210 packuswb m0, m0
2211 vpermb zmm1, m16, m0
2212 movq [dstq+dsq*0], xmm1
2213 movhps [dstq+dsq*1], xmm1
2214 lea dstq, [dstq+dsq*2]
2215 sub hd, 2
2216 jg .hv_w8_loop
2217 vzeroupper
2218 RET
2219 .hv_w16:
2220 movu m7, [spel_hv_perm16a]
2221 sub srcq, ss3q
2222 mova m20, [spel_hv_perm16b]
2223 lea r6d, [wq*2-32]
2224 mova m21, [spel_hv_perm16c]
2225 mov r4, srcq
2226 mov r7, dstq
2227 mova ym16, [spel_hv_end16]
2228 lea r6d, [hq+r6*8]
2229 .hv_w16_loop0:
2230 movu ym17, [srcq+ssq*0]
2231 vinserti32x8 m17, [srcq+ssq*1], 1 ; 0 1
2232 movu ym18, [srcq+ssq*2]
2233 add srcq, ss3q
2234 vinserti32x8 m18, [srcq+ssq*0], 1 ; 2 3
2235 movu ym19, [srcq+ssq*1]
2236 vinserti32x8 m19, [srcq+ssq*2], 1 ; 4 5
2237 add srcq, ss3q
2238 vpermb m2, m7, m17 ; 0 1 0123 89ab
2239 vpermb m0, m20, m17 ; 0 1 4567 cdef
2240 vpermb m4, m7, m18 ; 2 3 0123 89ab
2241 mova m1, m8
2242 vpdpbusd m1, m2, m10
2243 vpermb m5, m20, m18 ; 2 3 4567 cdef
2244 mova m2, m8
2245 vpdpbusd m2, m0, m10
2246 vpermb m17, m21, m17 ; 0 1 89ab ghij
2247 mova m3, m8
2248 vpdpbusd m3, m4, m10
2249 vpermb m6, m7, m19 ; 4 5 0123 89ab
2250 mova m4, m8
2251 vpdpbusd m4, m5, m10
2252 vpermb m18, m21, m18 ; 2 3 89ab ghij
2253 vpdpbusd m1, m0, m11
2254 movu ym0, [srcq+ssq*0] ; 6
2255 vpdpbusd m2, m17, m11
2256 vpermb m17, m20, m19 ; 4 5 4567 cdef
2257 vpdpbusd m3, m5, m11
2258 mova m5, m8
2259 vpdpbusd m5, m6, m10
2260 mova m6, m8
2261 vpdpbusd m6, m17, m10
2262 vpdpbusd m4, m18, m11
2263 mova m18, [spel_hv_perm16d]
2264 vpermb m18, m18, m0 ; 6 0145 2367 89cd abef
2265 vpdpbusd m5, m17, m11
2266 vpermb m19, m21, m19 ; 4 5 89ab ghij
2267 mova m17, m8
2268 vpdpbusd m17, m18, m10
2269 mova m18, [spel_hv_perm16e]
2270 vpermb m0, m18, m0 ; 6 4589 67ab cdgh efij
2271 packssdw m1, m2 ; 01
2272 vpdpbusd m6, m19, m11
2273 packssdw m3, m4 ; 23
2274 vpdpbusd m17, m0, m11
2275 psraw m1, 2
2276 packssdw m5, m6 ; 45
2277 psraw m3, 2
2278 vpshrdd m2, m1, m3, 16 ; 12
2279 psraw m5, 2
2280 vpshrdd m4, m3, m5, 16 ; 34
2281 psraw m17, 2
2282 vpshrdd m6, m5, m17, 16 ; 56
2283 .hv_w16_loop:
2284 movu ym18, [srcq+ssq*1]
2285 lea srcq, [srcq+ssq*2]
2286 vinserti32x8 m18, [srcq+ssq*0], 1
2287 mova m0, m9
2288 vpdpwssd m0, m1, m12 ; a0
2289 vpermb m1, m7, m18 ; 7 8 0123 89ab
2290 mova m17, m9
2291 vpdpwssd m17, m2, m12 ; b0
2292 vpermb m2, m20, m18 ; 7 8 4567 cdef
2293 mova m19, m8
2294 vpdpbusd m19, m1, m10
2295 vpermb m18, m21, m18
2296 mova m1, m8
2297 vpdpbusd m1, m2, m10
2298 vpdpwssd m0, m3, m13 ; a1
2299 vpdpwssd m17, m4, m13 ; b1
2300 vpdpbusd m19, m2, m11
2301 mova m2, m4
2302 vpdpbusd m1, m18, m11
2303 mova m4, m6
2304 vpdpwssd m0, m5, m14 ; a2
2305 vpdpwssd m17, m6, m14 ; b2
2306 packssdw m19, m1
2307 mova m1, m3
2308 mova m3, m5
2309 psraw m6, m19, 2 ; 7 8
2310 vpshrdd m5, m4, m6, 16 ; 6 7
2311 vpdpwssd m17, m6, m15 ; b3
2312 vpdpwssd m0, m5, m15 ; a3
2313 packuswb m0, m17
2314 vpermb zmm1, m16, m0
2315 mova [dstq+dsq*0], xmm1
2316 vextracti128 [dstq+dsq*1], ymm1, 1
2317 lea dstq, [dstq+dsq*2]
2318 sub hd, 2
2319 jg .hv_w16_loop
2320 add r4, 16
2321 add r7, 16
2322 movzx hd, r6b
2323 mov srcq, r4
2324 mov dstq, r7
2325 sub r6d, 1<<8
2326 jg .hv_w16_loop0
2327 vzeroupper
2328 RET
2329
2330 %macro PREP_8TAP_H 0
2331 vpermb m10, m5, m0
2332 vpermb m11, m5, m1
2333 vpermb m12, m6, m0
2334 vpermb m13, m6, m1
2335 vpermb m14, m7, m0
2336 vpermb m15, m7, m1
2337 mova m0, m4
2338 vpdpbusd m0, m10, m8
2339 mova m2, m4
2340 vpdpbusd m2, m12, m8
2341 mova m1, m4
2342 vpdpbusd m1, m11, m8
2343 mova m3, m4
2344 vpdpbusd m3, m13, m8
2345 vpdpbusd m0, m12, m9
2346 vpdpbusd m2, m14, m9
2347 vpdpbusd m1, m13, m9
2348 vpdpbusd m3, m15, m9
2349 packssdw m0, m2
2350 packssdw m1, m3
2351 psraw m0, 2
2352 psraw m1, 2
2353 mova [tmpq+64*0], m0
2354 mova [tmpq+64*1], m1
2355 %endmacro
2356
2357 %if WIN64
2358 DECLARE_REG_TMP 6, 4
2359 %else
2360 DECLARE_REG_TMP 6, 7
2361 %endif
2362
2363 %define PREP_8TAP_FN FN prep_8tap,
2364
2365 PREP_8TAP_FN sharp, SHARP, SHARP
2366 PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH
2367 PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP
2368 PREP_8TAP_FN smooth, SMOOTH, SMOOTH
2369 PREP_8TAP_FN sharp_regular, SHARP, REGULAR
2370 PREP_8TAP_FN regular_sharp, REGULAR, SHARP
2371 PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR
2372 PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH
2373 PREP_8TAP_FN regular, REGULAR, REGULAR
2374
2375 cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
2376 imul mxd, mxm, 0x010101
2377 add mxd, t0d ; 8tap_h, mx, 4tap_h
2378 imul myd, mym, 0x010101
2379 add myd, t1d ; 8tap_v, my, 4tap_v
2380 lea r7, [prep_avx512icl]
2381 movsxd wq, wm
2382 movifnidn hd, hm
2383 test mxd, 0xf00
2384 jnz .h
2385 test myd, 0xf00
2386 jnz .v
2387 tzcnt wd, wd
2388 movzx wd, word [r7+wq*2+table_offset(prep,)]
2389 add wq, r7
2390 lea r6, [strideq*3]
2391 %if WIN64
2392 pop r7
2393 %endif
2394 jmp wq
2395 .h:
2396 test myd, 0xf00
2397 jnz .hv
2398 vpbroadcastd m4, [pd_2]
2399 WIN64_SPILL_XMM 10
2400 cmp wd, 4
2401 je .h_w4
2402 tzcnt wd, wd
2403 shr mxd, 16
2404 sub srcq, 3
2405 movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)]
2406 vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+0]
2407 vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep_avx512icl+4]
2408 add wq, r7
2409 jmp wq
2410 .h_w4:
2411 movzx mxd, mxb
2412 vbroadcasti128 ym5, [subpel_h_shufA]
2413 mov r3d, 0x4
2414 dec srcq
2415 vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep_avx512icl+2]
2416 kmovb k1, r3d
2417 lea stride3q, [strideq*3]
2418 .h_w4_loop:
2419 movq xm2, [srcq+strideq*0]
2420 movq xm3, [srcq+strideq*1]
2421 vpbroadcastq ym2{k1}, [srcq+strideq*2]
2422 vpbroadcastq ym3{k1}, [srcq+stride3q ]
2423 lea srcq, [srcq+strideq*4]
2424 pshufb ym2, ym5
2425 pshufb ym3, ym5
2426 mova ym0, ym4
2427 vpdpbusd ym0, ym2, ym6
2428 mova ym1, ym4
2429 vpdpbusd ym1, ym3, ym6
2430 packssdw ym0, ym1
2431 psraw ym0, 2
2432 mova [tmpq], ym0
2433 add tmpq, 32
2434 sub hd, 4
2435 jg .h_w4_loop
2436 RET
2437 .h_w8:
2438 vbroadcasti128 m5, [subpel_h_shufA]
2439 vbroadcasti128 m6, [subpel_h_shufB]
2440 vbroadcasti128 m7, [subpel_h_shufC]
2441 lea stride3q, [strideq*3]
2442 .h_w8_loop:
2443 movu xmm3, [srcq+strideq*0]
2444 vinserti128 ym3, ymm3, [srcq+strideq*1], 1
2445 vinserti128 m3, [srcq+strideq*2], 2
2446 vinserti128 m3, [srcq+stride3q ], 3
2447 lea srcq, [srcq+strideq*4]
2448 pshufb m1, m3, m5
2449 pshufb m2, m3, m6
2450 mova m0, m4
2451 vpdpbusd m0, m1, m8
2452 mova m1, m4
2453 vpdpbusd m1, m2, m8
2454 pshufb m3, m7
2455 vpdpbusd m0, m2, m9
2456 vpdpbusd m1, m3, m9
2457 packssdw m0, m1
2458 psraw m0, 2
2459 mova [tmpq], m0
2460 add tmpq, 64
2461 sub hd, 4
2462 jg .h_w8_loop
2463 RET
2464 .h_w16:
2465 mova m5, [spel_h_perm16a]
2466 mova m6, [spel_h_perm16b]
2467 mova m7, [spel_h_perm16c]
2468 lea stride3q, [strideq*3]
2469 .h_w16_loop:
2470 movu ym0, [srcq+strideq*0]
2471 movu ym1, [srcq+strideq*2]
2472 vinserti32x8 m0, [srcq+strideq*1], 1
2473 vinserti32x8 m1, [srcq+stride3q ], 1
2474 lea srcq, [srcq+strideq*4]
2475 PREP_8TAP_H
2476 add tmpq, 64*2
2477 sub hd, 4
2478 jg .h_w16_loop
2479 RET
2480 .h_w32:
2481 mova m5, [spel_h_perm32a]
2482 mova m6, [spel_h_perm32b]
2483 mova m7, [spel_h_perm32c]
2484 .h_w32_loop:
2485 movu m0, [srcq+strideq*0]
2486 movu m1, [srcq+strideq*1]
2487 lea srcq, [srcq+strideq*2]
2488 PREP_8TAP_H
2489 add tmpq, 64*2
2490 sub hd, 2
2491 jg .h_w32_loop
2492 RET
2493 .h_w64:
2494 xor r6d, r6d
2495 jmp .h_start
2496 .h_w128:
2497 mov r6, -64*1
2498 .h_start:
2499 mova m5, [spel_h_perm32a]
2500 mova m6, [spel_h_perm32b]
2501 mova m7, [spel_h_perm32c]
2502 sub srcq, r6
2503 mov r5, r6
2504 .h_loop:
2505 movu m0, [srcq+r6+32*0]
2506 movu m1, [srcq+r6+32*1]
2507 PREP_8TAP_H
2508 add tmpq, 64*2
2509 add r6, 64
2510 jle .h_loop
2511 add srcq, strideq
2512 mov r6, r5
2513 dec hd
2514 jg .h_loop
2515 RET
2516 .v:
2517 movzx mxd, myb ; Select 4-tap/8-tap filter multipliers.
2518 shr myd, 16 ; Note that the code is 8-tap only, having
2519 tzcnt wd, wd
2520 cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4
2521 cmove myd, mxd ; had a negligible effect on performance.
2522 ; TODO: Would a 6-tap code path be worth it?
2523 lea myq, [r7+myq*8+subpel_filters-prep_avx512icl]
2524 movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)]
2525 add wq, r7
2526 lea stride3q, [strideq*3]
2527 sub srcq, stride3q
2528 vpbroadcastd m7, [pw_8192]
2529 vpbroadcastw m8, [myq+0]
2530 vpbroadcastw m9, [myq+2]
2531 vpbroadcastw m10, [myq+4]
2532 vpbroadcastw m11, [myq+6]
2533 jmp wq
2534 .v_w4:
2535 movd xmm0, [srcq+strideq*0]
2536 vpbroadcastd ymm1, [srcq+strideq*2]
2537 vpbroadcastd xmm2, [srcq+strideq*1]
2538 vpbroadcastd ymm3, [srcq+stride3q ]
2539 lea srcq, [srcq+strideq*4]
2540 vpblendd ymm1, ymm0, 0x01 ; 0 2 2 _ 2 _ _ _
2541 vpblendd ymm3, ymm2, 0x03 ; 1 1 3 3 3 3 _ _
2542 vpbroadcastd ymm0, [srcq+strideq*0]
2543 vpbroadcastd ymm2, [srcq+strideq*1]
2544 vpblendd ymm1, ymm0, 0x68 ; 0 2 2 4 2 4 4 _
2545 vpbroadcastd ymm0, [srcq+strideq*2]
2546 vbroadcasti128 ymm5, [deint_shuf4]
2547 vpblendd ymm3, ymm2, 0xc0 ; 1 1 3 3 3 3 5 5
2548 vpblendd ymm2, ymm3, ymm1, 0x55 ; 0 1 2 3 2 3 4 5
2549 vpblendd ymm3, ymm1, 0xaa ; 1 2 3 4 3 4 5 _
2550 punpcklbw ymm1, ymm2, ymm3 ; 01 12 23 34
2551 vpblendd ymm3, ymm0, 0x80 ; 1 2 3 4 3 4 5 6
2552 punpckhbw ymm2, ymm3 ; 23 34 45 56
2553 .v_w4_loop:
2554 pinsrd xmm0, [srcq+stride3q ], 1
2555 lea srcq, [srcq+strideq*4]
2556 vpbroadcastd ymm3, [srcq+strideq*0]
2557 vpbroadcastd ymm4, [srcq+strideq*1]
2558 vpblendd ymm3, ymm4, 0x20 ; _ _ 8 _ 8 9 _ _
2559 vpblendd ymm3, ymm0, 0x03 ; 6 7 8 _ 8 9 _ _
2560 vpbroadcastd ymm0, [srcq+strideq*2]
2561 vpblendd ymm3, ymm0, 0x40 ; 6 7 8 _ 8 9 a _
2562 pshufb ymm3, ymm5 ; 67 78 89 9a
2563 pmaddubsw ymm4, ymm1, ym8
2564 vperm2i128 ymm1, ymm2, ymm3, 0x21 ; 45 56 67 78
2565 pmaddubsw ymm2, ym9
2566 paddw ymm4, ymm2
2567 mova ymm2, ymm3
2568 pmaddubsw ymm3, ym11
2569 paddw ymm3, ymm4
2570 pmaddubsw ymm4, ymm1, ym10
2571 paddw ymm3, ymm4
2572 pmulhrsw ymm3, ym7
2573 mova [tmpq], ymm3
2574 add tmpq, 32
2575 sub hd, 4
2576 jg .v_w4_loop
2577 vzeroupper
2578 RET
2579 .v_w8:
2580 mov r3d, 0xf044
2581 kmovw k1, r3d
2582 kshiftrw k2, k1, 8
2583 movq xm0, [srcq+strideq*0]
2584 vpbroadcastq ym1, [srcq+strideq*1]
2585 vpbroadcastq m2, [srcq+strideq*2]
2586 vpbroadcastq m3, [srcq+stride3q ]
2587 lea srcq, [srcq+strideq*4]
2588 vpbroadcastq m4, [srcq+strideq*0]
2589 vpbroadcastq m5, [srcq+strideq*1]
2590 vpbroadcastq m6, [srcq+strideq*2]
2591 vmovdqa64 ym0{k1}, ym1
2592 vmovdqa64 ym1{k1}, ym2
2593 vmovdqa64 m2{k1}, m3
2594 vmovdqa64 m3{k1}, m4
2595 vmovdqa64 m4{k1}, m5
2596 vmovdqa64 m5{k1}, m6
2597 punpcklbw ym0, ym1 ; 01 12 __ __
2598 punpcklbw m2, m3 ; 23 34 23 34
2599 punpcklbw m4, m5 ; 45 56 45 56
2600 vmovdqa64 m0{k2}, m2 ; 01 12 23 34
2601 vmovdqa64 m2{k2}, m4 ; 23 34 45 56
2602 .v_w8_loop:
2603 vpbroadcastq m1, [srcq+stride3q ]
2604 lea srcq, [srcq+strideq*4]
2605 vpbroadcastq m3, [srcq+strideq*0]
2606 vpbroadcastq m5, [srcq+strideq*1]
2607 pmaddubsw m14, m0, m8
2608 pmaddubsw m15, m2, m9
2609 vpblendmq m0{k1}, m6, m1
2610 vpblendmq m2{k1}, m1, m3
2611 vpbroadcastq m6, [srcq+strideq*2]
2612 paddw m14, m15
2613 punpcklbw m2, m0, m2 ; 67 78 67 78
2614 vpblendmq m12{k1}, m3, m5
2615 vpblendmq m13{k1}, m5, m6
2616 vpblendmq m0{k2}, m4, m2 ; 45 56 67 78
2617 punpcklbw m4, m12, m13 ; 89 9a 89 9a
2618 vmovdqa64 m2{k2}, m4 ; 67 78 89 9a
2619 pmaddubsw m12, m0, m10
2620 pmaddubsw m13, m2, m11
2621 paddw m14, m12
2622 paddw m14, m13
2623 pmulhrsw m14, m7
2624 mova [tmpq], m14
2625 add tmpq, 64
2626 sub hd, 4
2627 jg .v_w8_loop
2628 RET
2629 .v_w16:
2630 mov r3d, 0xf0
2631 kmovb k1, r3d
2632 vbroadcasti128 m0, [srcq+strideq*0]
2633 vbroadcasti128 m1, [srcq+strideq*1]
2634 vbroadcasti128 m2, [srcq+strideq*2]
2635 vbroadcasti128 m3, [srcq+stride3q ]
2636 lea srcq, [srcq+strideq*4]
2637 vbroadcasti128 m4, [srcq+strideq*0]
2638 vbroadcasti128 m5, [srcq+strideq*1]
2639 vbroadcasti128 m6, [srcq+strideq*2]
2640 vmovdqa64 m0{k1}, m1
2641 vmovdqa64 m1{k1}, m2
2642 vmovdqa64 m2{k1}, m3
2643 vmovdqa64 m3{k1}, m4
2644 vmovdqa64 m4{k1}, m5
2645 vmovdqa64 m5{k1}, m6
2646 shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b
2647 shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b
2648 shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_--
2649 shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_--
2650 punpckhbw m2, m0, m1 ; 23a 23b 34a 34b
2651 punpcklbw m0, m1 ; 01a 01b 12a 12b
2652 punpcklbw m4, m5 ; 45a 45b 56a 56b
2653 .v_w16_loop:
2654 vbroadcasti128 m3, [srcq+stride3q ]
2655 lea srcq, [srcq+strideq*4]
2656 vbroadcasti128 m5, [srcq+strideq*0]
2657 vpblendmq m1{k1}, m6, m3
2658 vmovdqa64 m3{k1}, m5
2659 pmaddubsw m12, m0, m8
2660 pmaddubsw m13, m2, m8
2661 pmaddubsw m14, m2, m9
2662 pmaddubsw m15, m4, m9
2663 pmaddubsw m0, m4, m10
2664 vbroadcasti128 m2, [srcq+strideq*1]
2665 vbroadcasti128 m6, [srcq+strideq*2]
2666 paddw m12, m14
2667 paddw m13, m15
2668 paddw m12, m0
2669 vmovdqa64 m5{k1}, m2
2670 vmovdqa64 m2{k1}, m6
2671 mova m0, m4
2672 shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b
2673 shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab
2674 punpcklbw m2, m1, m3 ; 67a 67b 78a 78b
2675 punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab
2676 pmaddubsw m14, m2, m10
2677 pmaddubsw m15, m2, m11
2678 paddw m13, m14
2679 paddw m12, m15
2680 pmaddubsw m14, m4, m11
2681 paddw m13, m14
2682 pmulhrsw m12, m7
2683 pmulhrsw m13, m7
2684 mova [tmpq+ 0], m12
2685 mova [tmpq+64], m13
2686 add tmpq, 64*2
2687 sub hd, 4
2688 jg .v_w16_loop
2689 RET
2690 .v_w32:
2691 mova m18, [bilin_v_perm64]
2692 movu ym0, [srcq+strideq*0]
2693 movu ym1, [srcq+strideq*1]
2694 lea srcq, [srcq+strideq*2]
2695 movu ym2, [srcq+strideq*0]
2696 movu ym3, [srcq+strideq*1]
2697 lea srcq, [srcq+strideq*2]
2698 movu ym4, [srcq+strideq*0]
2699 movu ym5, [srcq+strideq*1]
2700 lea srcq, [srcq+strideq*2]
2701 movu ym6, [srcq+strideq*0]
2702 vpermq m0, m18, m0
2703 vpermq m1, m18, m1
2704 vpermq m2, m18, m2
2705 vpermq m3, m18, m3
2706 vpermq m4, m18, m4
2707 vpermq m5, m18, m5
2708 vpermq m6, m18, m6
2709 punpcklbw m0, m1
2710 punpcklbw m1, m2
2711 punpcklbw m2, m3
2712 punpcklbw m3, m4
2713 punpcklbw m4, m5
2714 punpcklbw m5, m6
2715 .v_w32_loop:
2716 movu ym12, [srcq+strideq*1]
2717 lea srcq, [srcq+strideq*2]
2718 movu ym13, [srcq+strideq*0]
2719 pmaddubsw m14, m0, m8
2720 pmaddubsw m16, m2, m9
2721 pmaddubsw m15, m1, m8
2722 pmaddubsw m17, m3, m9
2723 mova m0, m2
2724 mova m1, m3
2725 vpermq m12, m18, m12
2726 vpermq m13, m18, m13
2727 paddw m14, m16
2728 paddw m15, m17
2729 pmaddubsw m16, m4, m10
2730 pmaddubsw m17, m5, m10
2731 punpcklbw m6, m12
2732 punpcklbw m12, m13
2733 mova m2, m4
2734 mova m3, m5
2735 paddw m14, m16
2736 paddw m15, m17
2737 pmaddubsw m16, m6, m11
2738 pmaddubsw m17, m12, m11
2739 mova m4, m6
2740 mova m5, m12
2741 paddw m14, m16
2742 paddw m15, m17
2743 pmulhrsw m14, m7
2744 pmulhrsw m15, m7
2745 mova m6, m13
2746 mova [tmpq+ 0], m14
2747 mova [tmpq+64], m15
2748 add tmpq, 64*2
2749 sub hd, 2
2750 jg .v_w32_loop
2751 vzeroupper
2752 RET
2753 .v_w64:
2754 mov wd, 64
2755 jmp .v_start
2756 .v_w128:
2757 mov wd, 128
2758 .v_start:
2759 WIN64_SPILL_XMM 27
2760 mova m26, [bilin_v_perm64]
2761 lea r6d, [hq+wq*2]
2762 mov r5, srcq
2763 mov r7, tmpq
2764 .v_loop0:
2765 vpermq m0, m26, [srcq+strideq*0]
2766 vpermq m1, m26, [srcq+strideq*1]
2767 lea srcq, [srcq+strideq*2]
2768 vpermq m2, m26, [srcq+strideq*0]
2769 vpermq m3, m26, [srcq+strideq*1]
2770 lea srcq, [srcq+strideq*2]
2771 vpermq m4, m26, [srcq+strideq*0]
2772 vpermq m5, m26, [srcq+strideq*1]
2773 lea srcq, [srcq+strideq*2]
2774 vpermq m6, m26, [srcq+strideq*0]
2775 punpckhbw m12, m0, m1
2776 punpcklbw m0, m1
2777 punpckhbw m13, m1, m2
2778 punpcklbw m1, m2
2779 punpckhbw m14, m2, m3
2780 punpcklbw m2, m3
2781 punpckhbw m15, m3, m4
2782 punpcklbw m3, m4
2783 punpckhbw m16, m4, m5
2784 punpcklbw m4, m5
2785 punpckhbw m17, m5, m6
2786 punpcklbw m5, m6
2787 .v_loop:
2788 vpermq m18, m26, [srcq+strideq*1]
2789 lea srcq, [srcq+strideq*2]
2790 vpermq m19, m26, [srcq+strideq*0]
2791 pmaddubsw m20, m0, m8
2792 pmaddubsw m21, m12, m8
2793 pmaddubsw m22, m1, m8
2794 pmaddubsw m23, m13, m8
2795 mova m0, m2
2796 mova m12, m14
2797 mova m1, m3
2798 mova m13, m15
2799 pmaddubsw m2, m9
2800 pmaddubsw m14, m9
2801 pmaddubsw m3, m9
2802 pmaddubsw m15, m9
2803 punpckhbw m24, m6, m18
2804 punpcklbw m6, m18
2805 paddw m20, m2
2806 paddw m21, m14
2807 paddw m22, m3
2808 paddw m23, m15
2809 mova m2, m4
2810 mova m14, m16
2811 mova m3, m5
2812 mova m15, m17
2813 pmaddubsw m4, m10
2814 pmaddubsw m16, m10
2815 pmaddubsw m5, m10
2816 pmaddubsw m17, m10
2817 punpckhbw m25, m18, m19
2818 punpcklbw m18, m19
2819 paddw m20, m4
2820 paddw m21, m16
2821 paddw m22, m5
2822 paddw m23, m17
2823 mova m4, m6
2824 mova m16, m24
2825 mova m5, m18
2826 mova m17, m25
2827 pmaddubsw m6, m11
2828 pmaddubsw m24, m11
2829 pmaddubsw m18, m11
2830 pmaddubsw m25, m11
2831 paddw m20, m6
2832 paddw m21, m24
2833 paddw m22, m18
2834 paddw m23, m25
2835 pmulhrsw m20, m7
2836 pmulhrsw m21, m7
2837 pmulhrsw m22, m7
2838 pmulhrsw m23, m7
2839 mova m6, m19
2840 mova [tmpq+wq*0+ 0], m20
2841 mova [tmpq+wq*0+64], m21
2842 mova [tmpq+wq*2+ 0], m22
2843 mova [tmpq+wq*2+64], m23
2844 lea tmpq, [tmpq+wq*4]
2845 sub hd, 2
2846 jg .v_loop
2847 add r5, 64
2848 add r7, 128
2849 movzx hd, r6b
2850 mov srcq, r5
2851 mov tmpq, r7
2852 sub r6d, 1<<8
2853 jg .v_loop0
2854 RET
2855 .hv:
2856 %assign stack_offset stack_offset - stack_size_padded
2857 %assign stack_size_padded 0
2858 WIN64_SPILL_XMM 16
2859 cmp wd, 4
2860 je .hv_w4
2861 shr mxd, 16
2862 sub srcq, 3
2863 vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep_avx512icl+0]
2864 vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep_avx512icl+4]
2865 movzx mxd, myb
2866 shr myd, 16
2867 cmp hd, 4
2868 cmove myd, mxd
2869 tzcnt wd, wd
2870 vpbroadcastd m8, [pd_2]
2871 movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)]
2872 vpbroadcastd m9, [pd_32]
2873 add wq, r7
2874 vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl]
2875 lea stride3q, [strideq*3]
2876 sub srcq, stride3q
2877 punpcklbw m0, m0
2878 psraw m0, 8 ; sign-extend
2879 pshufd m12, m0, q0000
2880 pshufd m13, m0, q1111
2881 pshufd m14, m0, q2222
2882 pshufd m15, m0, q3333
2883 jmp wq
2884 .hv_w4:
2885 movzx mxd, mxb
2886 dec srcq
2887 vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+2]
2888 movzx mxd, myb
2889 shr myd, 16
2890 cmp hd, 4
2891 cmove myd, mxd
2892 vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl]
2893 lea stride3q, [strideq*3]
2894 sub srcq, stride3q
2895 mov r3d, 0x04
2896 kmovb k1, r3d
2897 kshiftlb k2, k1, 2
2898 kshiftlb k3, k1, 4
2899 vpbroadcastd m10, [pd_2]
2900 vbroadcasti128 m16, [subpel_h_shufA]
2901 punpcklbw m0, m0
2902 psraw m0, 8 ; sign-extend
2903 vpbroadcastd m11, [pd_32]
2904 pshufd m12, m0, q0000
2905 pshufd m13, m0, q1111
2906 pshufd m14, m0, q2222
2907 pshufd m15, m0, q3333
2908 movq xm3, [srcq+strideq*0]
2909 vpbroadcastq ym2, [srcq+strideq*1]
2910 vpbroadcastq ym3{k1}, [srcq+strideq*2]
2911 vpbroadcastq m2{k2}, [srcq+stride3q ]
2912 lea srcq, [srcq+strideq*4]
2913 vpbroadcastq m3{k2}, [srcq+strideq*0]
2914 vpbroadcastq m2{k3}, [srcq+strideq*1]
2915 vpbroadcastq m3{k3}, [srcq+strideq*2]
2916 mova m17, [spel_hv_perm4a]
2917 movu m18, [spel_hv_perm4b]
2918 mova m0, m10
2919 mova m1, m10
2920 pshufb m2, m16
2921 pshufb m3, m16
2922 vpdpbusd m0, m2, m8
2923 vpdpbusd m1, m3, m8
2924 packssdw m0, m1 ; _ 0 1 2 3 4 5 6
2925 psraw m0, 2
2926 vpermb m1, m17, m0 ; 01 12 23 34
2927 vpermb m2, m18, m0 ; 23 34 45 56
2928 .hv_w4_loop:
2929 movq xm3, [srcq+stride3q ]
2930 lea srcq, [srcq+strideq*4]
2931 movq xm4, [srcq+strideq*0]
2932 vpbroadcastq ym3{k1}, [srcq+strideq*1]
2933 vpbroadcastq ym4{k1}, [srcq+strideq*2]
2934 mova ym5, ym10
2935 mova ym6, ym10
2936 pshufb ym3, ym16
2937 pshufb ym4, ym16
2938 vpdpbusd ym5, ym3, ym8
2939 vpdpbusd ym6, ym4, ym8
2940 mova m7, m11
2941 packssdw ym5, ym6 ; 7 8 9 a _ _ _ _
2942 psraw ym5, 2
2943 valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a
2944 vpdpwssd m7, m1, m12
2945 vpdpwssd m7, m2, m13
2946 vpermb m1, m17, m0 ; 45 56 67 78
2947 vpermb m2, m18, m0 ; 67 78 89 9a
2948 vpdpwssd m7, m1, m14
2949 vpdpwssd m7, m2, m15
2950 psrad m7, 6
2951 vpmovdw [tmpq], m7
2952 add tmpq, 32
2953 sub hd, 4
2954 jg .hv_w4_loop
2955 vzeroupper
2956 RET
2957 .hv_w8:
2958 WIN64_SPILL_XMM 24
2959 vbroadcasti128 m16, [subpel_h_shufA]
2960 vbroadcasti128 m17, [subpel_h_shufB]
2961 vbroadcasti128 m18, [subpel_h_shufC]
2962 vinserti128 ym0, [srcq+strideq*0], 1
2963 vinserti128 m0, [srcq+strideq*1], 2
2964 vinserti128 m0, [srcq+strideq*2], 3
2965 movu xm1, [srcq+stride3q ]
2966 lea srcq, [srcq+strideq*4]
2967 vinserti128 ym1, [srcq+strideq*0], 1
2968 vinserti128 m1, [srcq+strideq*1], 2
2969 vinserti128 m1, [srcq+strideq*2], 3
2970 mova m2, m8
2971 mova m4, m8
2972 mova m3, m8
2973 mova m5, m8
2974 pshufb m20, m0, m16
2975 pshufb m21, m0, m17
2976 pshufb m22, m0, m18
2977 pshufb m23, m1, m16
2978 pshufb m6, m1, m17
2979 pshufb m7, m1, m18
2980 vpdpbusd m2, m20, m10
2981 vpdpbusd m4, m21, m10
2982 vpdpbusd m2, m21, m11
2983 vpdpbusd m4, m22, m11
2984 vpdpbusd m3, m23, m10
2985 vpdpbusd m5, m6, m10
2986 vpdpbusd m3, m6, m11
2987 vpdpbusd m5, m7, m11
2988 packssdw m2, m4
2989 packssdw m3, m5
2990 psraw m2, 2 ; _ 0 1 2
2991 psraw m3, 2 ; 3 4 5 6
2992 valignq m0, m3, m2, 2 ; 0 1 2 3
2993 valignq m1, m3, m2, 4 ; 1 2 3 4
2994 valignq m2, m3, m2, 6 ; 2 3 4 5
2995 punpcklwd m4, m0, m1 ; 01a 12a 23a 34a
2996 punpckhwd m5, m0, m1 ; 01b 12b 23b 34b
2997 punpcklwd m6, m2, m3 ; 23a 34a 45a 56a
2998 punpckhwd m7, m2, m3 ; 23b 34b 45b 56b
2999 .hv_w8_loop:
3000 movu xm19, [srcq+stride3q ]
3001 lea srcq, [srcq+strideq*4]
3002 vinserti128 ym19, [srcq+strideq*0], 1
3003 vinserti128 m19, [srcq+strideq*1], 2
3004 vinserti128 m19, [srcq+strideq*2], 3
3005 mova m20, m9
3006 mova m21, m9
3007 mova m22, m8
3008 mova m23, m8
3009 vpdpwssd m20, m4, m12
3010 vpdpwssd m21, m5, m12
3011 vpdpwssd m20, m6, m13
3012 vpdpwssd m21, m7, m13
3013 pshufb m0, m19, m16
3014 pshufb m1, m19, m17
3015 pshufb m2, m19, m18
3016 vpdpbusd m22, m0, m10
3017 vpdpbusd m23, m1, m10
3018 vpdpbusd m22, m1, m11
3019 vpdpbusd m23, m2, m11
3020 packssdw m22, m23
3021 psraw m22, 2 ; 7 8 9 A
3022 valignq m0, m22, m3, 2 ; 4 5 6 7
3023 valignq m1, m22, m3, 4 ; 5 6 7 8
3024 valignq m2, m22, m3, 6 ; 6 7 8 9
3025 mova m3, m22
3026 punpcklwd m4, m0, m1 ; 45a 56a 67a 78a
3027 punpckhwd m5, m0, m1 ; 45b 56b 67b 78b
3028 punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa
3029 punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab
3030 vpdpwssd m20, m4, m14
3031 vpdpwssd m21, m5, m14
3032 vpdpwssd m20, m6, m15
3033 vpdpwssd m21, m7, m15
3034 psrad m20, 6
3035 psrad m21, 6
3036 packssdw m20, m21
3037 mova [tmpq], m20
3038 add tmpq, 64
3039 sub hd, 4
3040 jg .hv_w8_loop
3041 RET
3042 .hv_w16:
3043 mov wd, 16*2
3044 jmp .hv_start
3045 .hv_w32:
3046 mov wd, 32*2
3047 jmp .hv_start
3048 .hv_w64:
3049 mov wd, 64*2
3050 jmp .hv_start
3051 .hv_w128:
3052 mov wd, 128*2
3053 .hv_start:
3054 WIN64_SPILL_XMM 31
3055 mova m16, [spel_h_perm16a]
3056 mova m17, [spel_h_perm16b]
3057 mova m18, [spel_h_perm16c]
3058 lea r6d, [hq+wq*8-256]
3059 mov r5, srcq
3060 mov r7, tmpq
3061 .hv_loop0:
3062 movu ym0, [srcq+strideq*0]
3063 vinserti32x8 m0, [srcq+strideq*1], 1
3064 lea srcq, [srcq+strideq*2]
3065 movu ym1, [srcq+strideq*0]
3066 vinserti32x8 m1, [srcq+strideq*1], 1
3067 lea srcq, [srcq+strideq*2]
3068 movu ym2, [srcq+strideq*0]
3069 vinserti32x8 m2, [srcq+strideq*1], 1
3070 lea srcq, [srcq+strideq*2]
3071 movu ym3, [srcq+strideq*0]
3072 mova m4, m8
3073 mova m5, m8
3074 mova m6, m8
3075 mova m7, m8
3076 vpermb m19, m16, m0
3077 vpermb m20, m17, m0
3078 vpermb m21, m18, m0
3079 vpermb m22, m16, m1
3080 vpermb m23, m17, m1
3081 vpermb m24, m18, m1
3082 vpermb m25, m16, m2
3083 vpermb m26, m17, m2
3084 vpermb m27, m18, m2
3085 vpermb ym28, ym16, ym3
3086 vpermb ym29, ym17, ym3
3087 vpermb ym30, ym18, ym3
3088 mova m0, m8
3089 mova m1, m8
3090 mova ym2, ym8
3091 mova ym3, ym8
3092 vpdpbusd m4, m19, m10
3093 vpdpbusd m5, m20, m10
3094 vpdpbusd m6, m22, m10
3095 vpdpbusd m7, m23, m10
3096 vpdpbusd m0, m25, m10
3097 vpdpbusd m1, m26, m10
3098 vpdpbusd ym2, ym28, ym10
3099 vpdpbusd ym3, ym29, ym10
3100 vpdpbusd m4, m20, m11
3101 vpdpbusd m5, m21, m11
3102 vpdpbusd m6, m23, m11
3103 vpdpbusd m7, m24, m11
3104 vpdpbusd m0, m26, m11
3105 vpdpbusd m1, m27, m11
3106 vpdpbusd ym2, ym29, ym11
3107 vpdpbusd ym3, ym30, ym11
3108 packssdw m4, m5
3109 packssdw m6, m7
3110 packssdw m0, m1
3111 packssdw ym2, ym3
3112 psraw m4, 2 ; 0a 0b 1a 1b
3113 psraw m6, 2 ; 2a 2b 3a 3b
3114 psraw m0, 2 ; 4a 4b 5a 5b
3115 psraw ym2, 2 ; 6a 6b __ __
3116 vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b
3117 vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b
3118 vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b
3119 punpcklwd m2, m4, m5 ; 01a 01c 12a 12c
3120 punpckhwd m3, m4, m5 ; 01b 01d 12b 12d
3121 punpcklwd m4, m6, m7 ; 23a 23c 34a 34c
3122 punpckhwd m5, m6, m7 ; 23b 23d 34b 34d
3123 punpcklwd m6, m0, m1 ; 45a 45c 56a 56c
3124 punpckhwd m7, m0, m1 ; 45b 45d 56b 56d
3125 .hv_loop:
3126 movu ym19, [srcq+strideq*1]
3127 lea srcq, [srcq+strideq*2]
3128 vinserti32x8 m19, [srcq+strideq*0], 1
3129 mova m20, m9
3130 mova m21, m9
3131 mova m22, m8
3132 mova m23, m8
3133 vpdpwssd m20, m2, m12
3134 vpdpwssd m21, m3, m12
3135 vpdpwssd m20, m4, m13
3136 vpdpwssd m21, m5, m13
3137 vpermb m24, m16, m19
3138 vpermb m25, m17, m19
3139 vpermb m26, m18, m19
3140 vpdpbusd m22, m24, m10
3141 vpdpbusd m23, m25, m10
3142 vpdpbusd m22, m25, m11
3143 vpdpbusd m23, m26, m11
3144 packssdw m22, m23
3145 psraw m22, 2 ; 7a 7b 8a 8b
3146 vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b
3147 mova m2, m4
3148 mova m3, m5
3149 mova m1, m22
3150 mova m4, m6
3151 mova m5, m7
3152 punpcklwd m6, m0, m1 ; 67a 67c 78a 78c
3153 punpckhwd m7, m0, m1 ; 67b 67d 78b 78d
3154 vpdpwssd m20, m4, m14
3155 vpdpwssd m21, m5, m14
3156 vpdpwssd m20, m6, m15
3157 vpdpwssd m21, m7, m15
3158 psrad m20, 6
3159 psrad m21, 6
3160 packssdw m20, m21
3161 mova [tmpq+wq*0], ym20
3162 vextracti32x8 [tmpq+wq*1], m20, 1
3163 lea tmpq, [tmpq+wq*2]
3164 sub hd, 2
3165 jg .hv_loop
3166 add r5, 16
3167 add r7, 32
3168 movzx hd, r6b
3169 mov srcq, r5
3170 mov tmpq, r7
3171 sub r6d, 1<<8
3172 jg .hv_loop0
3173 RET
3174
3175 cglobal warp_affine_8x8t_8bpc, 4, 7, 22, tmp, ts
3176 vpbroadcastd m9, [pd_16384]
3177 mova ym15, [warp_8x8t_end]
3178 call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main
3179 jmp .start
3180 .loop:
3181 call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main2
3182 lea tmpq, [tmpq+tsq*4]
3183 .start:
3184 paddd m16, m16
3185 vpermb m16, m15, m16
3186 mova [tmpq+tsq*0], xm16
3187 vextracti128 [tmpq+tsq*2], ym16, 1
3188 sub r6d, 0x1800
3189 jg .loop
3190 RET
3191
3192 cglobal warp_affine_8x8_8bpc, 4, 7, 22, dst, ds, src, ss, abcd, filter
3193 vpbroadcastd m9, [pd_262144]
3194 mova xm15, [warp_8x8_end]
3195 call .main
3196 jmp .start
3197 .loop:
3198 call .main2
3199 lea dstq, [dstq+dsq*2]
3200 .start:
3201 psrad m16, 19
3202 packuswb m16, m16
3203 vpermb m16, m15, m16
3204 movq [dstq+dsq*0], xm16
3205 movhps [dstq+dsq*1], xm16
3206 sub r6d, 0x1800
3207 jg .loop
3208 RET
3209 ALIGN function_align
3210 .main:
3211 vpbroadcastd m1, [pd_512]
3212 %if WIN64
3213 mov abcdq, r5mp
3214 vpaddd ym18, ym1, r6m {1to8} ; mx
3215 %else
3216 add r5d, 512
3217 vpbroadcastd ym18, r5d
3218 %endif
3219 vpaddd ym20, ym1, r7m {1to8} ; my
3220 mova ym16, [pd_0to7]
3221 vpbroadcastd ym19, [abcdq+4*0]
3222 vpbroadcastd ym21, [abcdq+4*1]
3223 lea r4, [ssq*3+3]
3224 mova m10, [warp_8x8_permA]
3225 mov r6d, 0x5555
3226 mova m11, [warp_8x8_permB]
3227 lea filterq, [mc_warp_filter+64*8]
3228 vpbroadcastq m12, [warp_8x8_hpack]
3229 sub srcq, r4 ; src -= src_stride*3 + 3
3230 vbroadcasti32x4 m13, [warp_8x8_permC]
3231 kxnorb k2, k2, k2
3232 vbroadcasti32x4 m14, [warp_8x8_permD]
3233 vpdpwssd ym18, ym19, ym16 ; alpha
3234 vpdpwssd ym20, ym21, ym16 ; gamma
3235 vbroadcasti32x4 m0, [srcq]
3236 psrad ym19, 16 ; beta
3237 psrad ym21, 16 ; delta
3238 kmovw k1, r6d
3239 psrad ym16, ym18, 10
3240 kmovb k3, k2
3241 paddd ym18, ym19
3242 vpgatherdq m2{k2}, [filterq+ym16*8] ; filter_x0
3243 psrld m1, 8 ; pd_2
3244 pshufb m0, m11
3245 paddd m8, m1, m1 ; pd_4
3246 vpdpbusd m1, m0, m2
3247 call .h
3248 psllq m2, m1, 45
3249 pslld m1, 13
3250 paddd m1, m2
3251 vpshrdq m1, m0, 48 ; 01 12
3252 call .h
3253 vpshrdq m2, m1, m0, 48 ; 23 34
3254 call .h
3255 vpshrdq m3, m2, m0, 48 ; 45 56
3256 .main2:
3257 call .h
3258 psrad ym17, ym20, 10
3259 kmovb k2, k3
3260 paddd ym20, ym21
3261 vpgatherdq m7{k3}, [filterq+ym17*8] ; filter_y0
3262 psrad ym16, ym20, 10
3263 kmovb k3, k2
3264 paddd ym20, ym21
3265 vpgatherdq m17{k2}, [filterq+ym16*8] ; filter_y1
3266 shufps m5, m7, m17, q2020 ; a0 a1 a2 a3 b0 b1 b2 b3 A0 A1 A2 A3 B0 B1 B2 B3
3267 mova m16, m9
3268 pshufb m4, m5, m13 ; a0 a1 A0 A1 b0 b1 B0 B1
3269 vpdpwssd m16, m1, m4
3270 pshufb m5, m14 ; a2 a3 A2 A3 b2 b3 B2 B3
3271 mova m1, m2
3272 vpdpwssd m16, m2, m5
3273 shufps m5, m7, m17, q3131 ; a4 a5 a6 a7 b4 b5 b6 b7 A4 A5 A6 A7 B4 B5 B6 B7
3274 mova m2, m3
3275 pshufb m4, m5, m13 ; a4 a5 A4 A5 b4 b5 B4 B5
3276 vpdpwssd m16, m3, m4
3277 vpshrdq m3, m0, 48 ; 67 78
3278 pshufb m5, m14 ; a6 a7 A6 A7 b6 b7 B6 B7
3279 vpdpwssd m16, m3, m5
3280 ret
3281 ALIGN function_align
3282 .h:
3283 movu xm5, [srcq+ssq*1]
3284 psrad ym16, ym18, 10
3285 lea srcq, [srcq+ssq*2]
3286 vinserti32x4 ym5, [srcq+ssq*0], 1
3287 kmovb k2, k3
3288 paddd ym18, ym19
3289 vpgatherdq m6{k3}, [filterq+ym16*8] ; filter_x1
3290 psrad ym17, ym18, 10
3291 kmovb k3, k2
3292 paddd ym18, ym19
3293 vpgatherdq m16{k2}, [filterq+ym17*8] ; filter_x2
3294 mova m0, m8
3295 vpermb m4, m10, m5 ; a4 b0 a5 b1 a6 b2 a7 b3 a8 b4 a9 b5 aa b6 ab b7
3296 vpshldq m17, m16, m6, 32 ; a4 a5 a6 a7 b0 b1 b2 b3
3297 vpdpbusd m0, m4, m17
3298 vpermb m5, m11, m5 ; a0 b4 a1 b5 a2 b6 a3 b7 a4 b8 a5 b9 a6 ba a7 bb
3299 vmovdqa32 m16{k1}, m6 ; a0 a1 a2 a3 b4 b5 b6 b7
3300 vpdpbusd m0, m5, m16
3301 vpmultishiftqb m0, m12, m0 ; 1 1 2 2 (>> 3)
3302 ret
3303
3304 %macro BIDIR_FN 1 ; op
3305 lea stride3q, [strideq*3]
3306 jmp wq
3307 .w4:
3308 cmp hd, 8
3309 jg .w4_h16
3310 WRAP_YMM %1 0
3311 vextracti32x4 xm1, ym0, 1
3312 movd [dstq ], xm0
3313 pextrd [dstq+strideq*1], xm0, 1
3314 movd [dstq+strideq*2], xm1
3315 pextrd [dstq+stride3q ], xm1, 1
3316 jl .w4_ret
3317 lea dstq, [dstq+strideq*4]
3318 pextrd [dstq ], xm0, 2
3319 pextrd [dstq+strideq*1], xm0, 3
3320 pextrd [dstq+strideq*2], xm1, 2
3321 pextrd [dstq+stride3q ], xm1, 3
3322 .w4_ret:
3323 RET
3324 .w4_h16:
3325 vpbroadcastd m7, strided
3326 pmulld m7, [bidir_sctr_w4]
3327 %1 0
3328 kxnorw k1, k1, k1
3329 vpscatterdd [dstq+m7]{k1}, m0
3330 RET
3331 .w8:
3332 cmp hd, 4
3333 jne .w8_h8
3334 WRAP_YMM %1 0
3335 vextracti32x4 xm1, ym0, 1
3336 movq [dstq ], xm0
3337 movq [dstq+strideq*1], xm1
3338 movhps [dstq+strideq*2], xm0
3339 movhps [dstq+stride3q ], xm1
3340 RET
3341 .w8_loop:
3342 %1_INC_PTR 2
3343 lea dstq, [dstq+strideq*4]
3344 .w8_h8:
3345 %1 0
3346 vextracti32x4 xm1, ym0, 1
3347 vextracti32x4 xm2, m0, 2
3348 vextracti32x4 xm3, m0, 3
3349 movq [dstq ], xm0
3350 movq [dstq+strideq*1], xm1
3351 movq [dstq+strideq*2], xm2
3352 movq [dstq+stride3q ], xm3
3353 lea dstq, [dstq+strideq*4]
3354 movhps [dstq ], xm0
3355 movhps [dstq+strideq*1], xm1
3356 movhps [dstq+strideq*2], xm2
3357 movhps [dstq+stride3q ], xm3
3358 sub hd, 8
3359 jg .w8_loop
3360 RET
3361 .w16_loop:
3362 %1_INC_PTR 2
3363 lea dstq, [dstq+strideq*4]
3364 .w16:
3365 %1 0
3366 vpermq m0, m0, q3120
3367 mova [dstq ], xm0
3368 vextracti32x4 [dstq+strideq*1], m0, 2
3369 vextracti32x4 [dstq+strideq*2], ym0, 1
3370 vextracti32x4 [dstq+stride3q ], m0, 3
3371 sub hd, 4
3372 jg .w16_loop
3373 RET
3374 .w32:
3375 pmovzxbq m7, [pb_02461357]
3376 .w32_loop:
3377 %1 0
3378 %1_INC_PTR 2
3379 vpermq m0, m7, m0
3380 mova [dstq+strideq*0], ym0
3381 vextracti32x8 [dstq+strideq*1], m0, 1
3382 lea dstq, [dstq+strideq*2]
3383 sub hd, 2
3384 jg .w32_loop
3385 RET
3386 .w64:
3387 pmovzxbq m7, [pb_02461357]
3388 .w64_loop:
3389 %1 0
3390 %1_INC_PTR 2
3391 vpermq m0, m7, m0
3392 mova [dstq], m0
3393 add dstq, strideq
3394 dec hd
3395 jg .w64_loop
3396 RET
3397 .w128:
3398 pmovzxbq m7, [pb_02461357]
3399 .w128_loop:
3400 %1 0
3401 vpermq m6, m7, m0
3402 %1 2
3403 mova [dstq+64*0], m6
3404 %1_INC_PTR 4
3405 vpermq m6, m7, m0
3406 mova [dstq+64*1], m6
3407 add dstq, strideq
3408 dec hd
3409 jg .w128_loop
3410 RET
3411 %endmacro
3412
3413 %macro AVG 1 ; src_offset
3414 mova m0, [tmp1q+(%1+0)*mmsize]
3415 paddw m0, [tmp2q+(%1+0)*mmsize]
3416 mova m1, [tmp1q+(%1+1)*mmsize]
3417 paddw m1, [tmp2q+(%1+1)*mmsize]
3418 pmulhrsw m0, m4
3419 pmulhrsw m1, m4
3420 packuswb m0, m1
3421 %endmacro
3422
3423 %macro AVG_INC_PTR 1
3424 add tmp1q, %1*mmsize
3425 add tmp2q, %1*mmsize
3426 %endmacro
3427
3428 cglobal avg_8bpc, 4, 7, 5, dst, stride, tmp1, tmp2, w, h, stride3
3429 %define base r6-avg_avx512icl_table
3430 lea r6, [avg_avx512icl_table]
3431 tzcnt wd, wm
3432 movifnidn hd, hm
3433 movsxd wq, dword [r6+wq*4]
3434 vpbroadcastd m4, [base+pw_1024]
3435 add wq, r6
3436 BIDIR_FN AVG
3437
3438 %macro W_AVG 1 ; src_offset
3439 ; (a * weight + b * (16 - weight) + 128) >> 8
3440 ; = ((a - b) * weight + (b << 4) + 128) >> 8
3441 ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4
3442 ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4
3443 mova m0, [tmp1q+(%1+0)*mmsize]
3444 psubw m2, m0, [tmp2q+(%1+0)*mmsize]
3445 mova m1, [tmp1q+(%1+1)*mmsize]
3446 psubw m3, m1, [tmp2q+(%1+1)*mmsize]
3447 pmulhw m2, m4
3448 pmulhw m3, m4
3449 paddw m0, m2
3450 paddw m1, m3
3451 pmulhrsw m0, m5
3452 pmulhrsw m1, m5
3453 packuswb m0, m1
3454 %endmacro
3455
3456 %define W_AVG_INC_PTR AVG_INC_PTR
3457
3458 cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
3459 %define base r6-w_avg_avx512icl_table
3460 lea r6, [w_avg_avx512icl_table]
3461 tzcnt wd, wm
3462 movifnidn hd, hm
3463 vpbroadcastw m4, r6m ; weight
3464 movsxd wq, dword [r6+wq*4]
3465 vpbroadcastd m5, [base+pw_2048]
3466 psllw m4, 12 ; (weight-16) << 12 when interpreted as signed
3467 add wq, r6
3468 cmp dword r6m, 7
3469 jg .weight_gt7
3470 mov r6, tmp1q
3471 pxor m0, m0
3472 mov tmp1q, tmp2q
3473 psubw m4, m0, m4 ; -weight
3474 mov tmp2q, r6
3475 .weight_gt7:
3476 BIDIR_FN W_AVG
3477
3478 %macro MASK 1 ; src_offset
3479 ; (a * m + b * (64 - m) + 512) >> 10
3480 ; = ((a - b) * m + (b << 6) + 512) >> 10
3481 ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4
3482 %if mmsize == 64
3483 vpermq m3, m8, [maskq+%1*32]
3484 %else
3485 vpermq m3, [maskq+%1*16], q3120
3486 %endif
3487 mova m0, [tmp2q+(%1+0)*mmsize]
3488 psubw m1, m0, [tmp1q+(%1+0)*mmsize]
3489 psubb m3, m4, m3
3490 paddw m1, m1 ; (b - a) << 1
3491 paddb m3, m3
3492 punpcklbw m2, m4, m3 ; -m << 9
3493 pmulhw m1, m2
3494 paddw m0, m1
3495 mova m1, [tmp2q+(%1+1)*mmsize]
3496 psubw m2, m1, [tmp1q+(%1+1)*mmsize]
3497 paddw m2, m2
3498 punpckhbw m3, m4, m3
3499 pmulhw m2, m3
3500 paddw m1, m2
3501 pmulhrsw m0, m5
3502 pmulhrsw m1, m5
3503 packuswb m0, m1
3504 %endmacro
3505
3506 %macro MASK_INC_PTR 1
3507 add maskq, %1*32
3508 add tmp2q, %1*64
3509 add tmp1q, %1*64
3510 %endmacro
3511
3512 cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3
3513 %define base r7-mask_avx512icl_table
3514 lea r7, [mask_avx512icl_table]
3515 tzcnt wd, wm
3516 movifnidn hd, hm
3517 mov maskq, maskmp
3518 movsxd wq, dword [r7+wq*4]
3519 pxor m4, m4
3520 mova m8, [base+bilin_v_perm64]
3521 vpbroadcastd m5, [base+pw_2048]
3522 add wq, r7
3523 BIDIR_FN MASK
3524
3525 %macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4
3526 mova m%1, [tmp1q+mmsize*%3]
3527 mova m1, [tmp2q+mmsize*%3]
3528 psubw m1, m%1
3529 pabsw m%2, m1
3530 psubusw m%2, m6, m%2
3531 psrlw m%2, 8 ; 64 - m
3532 psllw m2, m%2, 10
3533 pmulhw m1, m2
3534 paddw m%1, m1
3535 mova m1, [tmp1q+mmsize*%4]
3536 mova m2, [tmp2q+mmsize*%4]
3537 psubw m2, m1
3538 pabsw m3, m2
3539 psubusw m3, m6, m3
3540 vpshldw m%2, m3, 8
3541 psllw m3, m%2, 10
3542 %if %5
3543 psubb m%2, m5, m%2
3544 %endif
3545 pmulhw m2, m3
3546 paddw m1, m2
3547 pmulhrsw m%1, m7
3548 pmulhrsw m1, m7
3549 packuswb m%1, m1
3550 %endmacro
3551
3552 cglobal w_mask_420_8bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
3553 %define base r7-w_mask_420_avx512icl_table
3554 lea r7, [w_mask_420_avx512icl_table]
3555 tzcnt wd, wm
3556 mov r6d, r7m ; sign
3557 movifnidn hd, hm
3558 movsxd wq, [r7+wq*4]
3559 vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
3560 vpbroadcastd m7, [base+pw_2048]
3561 vpbroadcastd m9, [base+pb_m64] ; -1 << 6
3562 mova ym10, [base+wm_420_mask+32]
3563 vpbroadcastd m8, [base+wm_sign+r6*8] ; (258 - sign) << 6
3564 add wq, r7
3565 mov maskq, maskmp
3566 lea stride3q, [strideq*3]
3567 jmp wq
3568 .w4:
3569 mova m5, [wm_420_perm4]
3570 cmp hd, 8
3571 jg .w4_h16
3572 WRAP_YMM W_MASK 0, 4, 0, 1
3573 vinserti128 ym5, [wm_420_perm4+32], 1
3574 vpermb ym4, ym5, ym4
3575 vpdpbusd ym8, ym4, ym9
3576 vextracti32x4 xm1, m0, 1
3577 movd [dstq+strideq*0], xm0
3578 pextrd [dstq+strideq*1], xm0, 1
3579 movd [dstq+strideq*2], xm1
3580 pextrd [dstq+stride3q ], xm1, 1
3581 jl .w4_end
3582 lea dstq, [dstq+strideq*4]
3583 pextrd [dstq+strideq*0], xm0, 2
3584 pextrd [dstq+strideq*1], xm0, 3
3585 pextrd [dstq+strideq*2], xm1, 2
3586 pextrd [dstq+stride3q ], xm1, 3
3587 .w4_end:
3588 vpermb ym8, ym10, ym8
3589 movq [maskq], xm8
3590 RET
3591 .w4_h16:
3592 vpbroadcastd m11, strided
3593 pmulld m11, [bidir_sctr_w4]
3594 W_MASK 0, 4, 0, 1
3595 vpermb m4, m5, m4
3596 vpdpbusd m8, m4, m9
3597 kxnorw k1, k1, k1
3598 vpermb m8, m10, m8
3599 mova [maskq], xm8
3600 vpscatterdd [dstq+m11]{k1}, m0
3601 RET
3602 .w8:
3603 mova m5, [wm_420_perm8]
3604 cmp hd, 4
3605 jne .w8_h8
3606 WRAP_YMM W_MASK 0, 4, 0, 1
3607 vinserti128 ym5, [wm_420_perm8+32], 1
3608 vpermb ym4, ym5, ym4
3609 vpdpbusd ym8, ym4, ym9
3610 vpermb m8, m10, m8
3611 mova [maskq], xm8
3612 vextracti32x4 xm1, ym0, 1
3613 movq [dstq+strideq*0], xm0
3614 movq [dstq+strideq*1], xm1
3615 movhps [dstq+strideq*2], xm0
3616 movhps [dstq+stride3q ], xm1
3617 RET
3618 .w8_loop:
3619 add tmp1q, 128
3620 add tmp2q, 128
3621 add maskq, 16
3622 lea dstq, [dstq+strideq*4]
3623 .w8_h8:
3624 W_MASK 0, 4, 0, 1
3625 vpermb m4, m5, m4
3626 mova m1, m8
3627 vpdpbusd m1, m4, m9
3628 vpermb m1, m10, m1
3629 mova [maskq], xm1
3630 vextracti32x4 xm1, ym0, 1
3631 vextracti32x4 xm2, m0, 2
3632 vextracti32x4 xm3, m0, 3
3633 movq [dstq+strideq*0], xm0
3634 movq [dstq+strideq*1], xm1
3635 movq [dstq+strideq*2], xm2
3636 movq [dstq+stride3q ], xm3
3637 lea dstq, [dstq+strideq*4]
3638 movhps [dstq+strideq*0], xm0
3639 movhps [dstq+strideq*1], xm1
3640 movhps [dstq+strideq*2], xm2
3641 movhps [dstq+stride3q ], xm3
3642 sub hd, 8
3643 jg .w8_loop
3644 RET
3645 .w16:
3646 mova m5, [wm_420_perm16]
3647 .w16_loop:
3648 W_MASK 0, 4, 0, 1
3649 vpermb m4, m5, m4
3650 mova m1, m8
3651 vpdpbusd m1, m4, m9
3652 add tmp1q, 128
3653 add tmp2q, 128
3654 vpermb m1, m10, m1
3655 vpermq m0, m0, q3120
3656 mova [maskq], xm1
3657 add maskq, 16
3658 mova [dstq+strideq*0], xm0
3659 vextracti32x4 [dstq+strideq*1], m0, 2
3660 vextracti32x4 [dstq+strideq*2], ym0, 1
3661 vextracti32x4 [dstq+stride3q ], m0, 3
3662 lea dstq, [dstq+strideq*4]
3663 sub hd, 4
3664 jg .w16_loop
3665 RET
3666 .w32:
3667 pmovzxbq m5, [pb_02461357]
3668 .w32_loop:
3669 W_MASK 0, 4, 0, 1
3670 mova m1, m8
3671 vpdpbusd m1, m4, m9
3672 add tmp1q, 128
3673 add tmp2q, 128
3674 vpermb m1, m10, m1
3675 vpermq m0, m5, m0
3676 mova [maskq], xm1
3677 add maskq, 16
3678 mova [dstq+strideq*0], ym0
3679 vextracti32x8 [dstq+strideq*1], m0, 1
3680 lea dstq, [dstq+strideq*2]
3681 sub hd, 2
3682 jg .w32_loop
3683 RET
3684 .w64:
3685 pmovzxbq m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14
3686 psrlq m13, m12, 4 ; 1, 3, 5, 7, 9, 11, 13, 15
3687 .w64_loop:
3688 W_MASK 0, 4, 0, 2
3689 W_MASK 11, 5, 1, 3
3690 mova m2, m8
3691 vpdpbusd m2, m4, m9
3692 mova m3, m8
3693 vpdpbusd m3, m5, m9
3694 add tmp1q, 256
3695 add tmp2q, 256
3696 vpermt2b m2, m10, m3
3697 mova m1, m0
3698 vpermt2q m0, m12, m11
3699 vpermt2q m1, m13, m11
3700 mova [maskq], ym2
3701 add maskq, 32
3702 mova [dstq+strideq*0], m0
3703 mova [dstq+strideq*1], m1
3704 lea dstq, [dstq+strideq*2]
3705 sub hd, 2
3706 jg .w64_loop
3707 RET
3708 .w128:
3709 pmovzxbq m14, [wm_420_perm64]
3710 mova m10, [wm_420_mask]
3711 psrlq m15, m14, 4
3712 .w128_loop:
3713 W_MASK 0, 12, 0, 4
3714 W_MASK 11, 13, 1, 5
3715 mova m4, m8
3716 vpdpbusd m4, m12, m9
3717 mova m5, m8
3718 vpdpbusd m5, m13, m9
3719 mova m1, m0
3720 vpermt2q m0, m14, m11
3721 vpermt2q m1, m15, m11
3722 mova [dstq+strideq*0+64*0], m0
3723 mova [dstq+strideq*1+64*0], m1
3724 W_MASK 0, 12, 2, 6
3725 W_MASK 11, 13, 3, 7
3726 vprold m4, 16
3727 vprold m5, 16
3728 vpdpbusd m4, m12, m9
3729 vpdpbusd m5, m13, m9
3730 add tmp1q, 512
3731 add tmp2q, 512
3732 vpermt2b m4, m10, m5
3733 mova m1, m0
3734 vpermt2q m0, m14, m11
3735 vpermt2q m1, m15, m11
3736 mova [maskq], m4
3737 add maskq, 64
3738 mova [dstq+strideq*0+64*1], m0
3739 mova [dstq+strideq*1+64*1], m1
3740 lea dstq, [dstq+strideq*2]
3741 sub hd, 2
3742 jg .w128_loop
3743 RET
3744
3745 cglobal w_mask_422_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3
3746 %define base r7-w_mask_422_avx512icl_table
3747 lea r7, [w_mask_422_avx512icl_table]
3748 tzcnt wd, wm
3749 mov r6d, r7m ; sign
3750 movifnidn hd, hm
3751 movsxd wq, dword [r7+wq*4]
3752 vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
3753 vpbroadcastd m7, [base+pw_2048]
3754 vpbroadcastd m9, [base+pw_m128]
3755 mova m10, [base+wm_422_mask]
3756 vpbroadcastd m11, [base+pb_127]
3757 add wq, r7
3758 vpbroadcastd m8, [base+wm_sign+4+r6*4]
3759 mov maskq, maskmp
3760 lea stride3q, [strideq*3]
3761 jmp wq
3762 .w4:
3763 cmp hd, 8
3764 jg .w4_h16
3765 WRAP_YMM W_MASK 0, 4, 0, 1
3766 movhps xm10, [wm_422_mask+16]
3767 vpdpwssd ym8, ym4, ym9
3768 vpermb ym8, ym10, ym8
3769 vextracti32x4 xm1, m0, 1
3770 movd [dstq+strideq*0], xm0
3771 pextrd [dstq+strideq*1], xm0, 1
3772 movd [dstq+strideq*2], xm1
3773 pextrd [dstq+stride3q ], xm1, 1
3774 jl .w4_end
3775 lea dstq, [dstq+strideq*4]
3776 pextrd [dstq+strideq*0], xm0, 2
3777 pextrd [dstq+strideq*1], xm0, 3
3778 pextrd [dstq+strideq*2], xm1, 2
3779 pextrd [dstq+stride3q ], xm1, 3
3780 .w4_end:
3781 pand xm8, xm11
3782 mova [maskq], xm8
3783 RET
3784 .w4_h16:
3785 vpbroadcastd m5, strided
3786 pmulld m5, [bidir_sctr_w4]
3787 W_MASK 0, 4, 0, 1
3788 vpdpwssd m8, m4, m9
3789 kxnorw k1, k1, k1
3790 vpermb m8, m10, m8
3791 pand ym8, ym11
3792 mova [maskq], ym8
3793 vpscatterdd [dstq+m5]{k1}, m0
3794 RET
3795 .w8:
3796 cmp hd, 4
3797 jne .w8_h8
3798 WRAP_YMM W_MASK 0, 4, 0, 1
3799 movhps xm10, [wm_422_mask+16]
3800 vpdpwssd ym8, ym4, ym9
3801 vpermb ym8, ym10, ym8
3802 pand xm8, xm11
3803 mova [maskq], xm8
3804 vextracti32x4 xm1, ym0, 1
3805 movq [dstq+strideq*0], xm0
3806 movq [dstq+strideq*1], xm1
3807 movhps [dstq+strideq*2], xm0
3808 movhps [dstq+stride3q ], xm1
3809 RET
3810 .w8_loop:
3811 add tmp1q, 128
3812 add tmp2q, 128
3813 add maskq, 32
3814 lea dstq, [dstq+strideq*4]
3815 .w8_h8:
3816 W_MASK 0, 4, 0, 1
3817 mova m1, m8
3818 vpdpwssd m1, m4, m9
3819 vpermb m1, m10, m1
3820 pand ym1, ym11
3821 mova [maskq], ym1
3822 vextracti32x4 xm1, ym0, 1
3823 vextracti32x4 xm2, m0, 2
3824 vextracti32x4 xm3, m0, 3
3825 movq [dstq+strideq*0], xm0
3826 movq [dstq+strideq*1], xm1
3827 movq [dstq+strideq*2], xm2
3828 movq [dstq+stride3q ], xm3
3829 lea dstq, [dstq+strideq*4]
3830 movhps [dstq+strideq*0], xm0
3831 movhps [dstq+strideq*1], xm1
3832 movhps [dstq+strideq*2], xm2
3833 movhps [dstq+stride3q ], xm3
3834 sub hd, 8
3835 jg .w8_loop
3836 RET
3837 .w16_loop:
3838 add tmp1q, 128
3839 add tmp2q, 128
3840 add maskq, 32
3841 lea dstq, [dstq+strideq*4]
3842 .w16:
3843 W_MASK 0, 4, 0, 1
3844 mova m1, m8
3845 vpdpwssd m1, m4, m9
3846 vpermb m1, m10, m1
3847 vpermq m0, m0, q3120
3848 pand ym1, ym11
3849 mova [maskq], ym1
3850 mova [dstq+strideq*0], xm0
3851 vextracti32x4 [dstq+strideq*1], m0, 2
3852 vextracti32x4 [dstq+strideq*2], ym0, 1
3853 vextracti32x4 [dstq+stride3q ], m0, 3
3854 sub hd, 4
3855 jg .w16_loop
3856 RET
3857 .w32:
3858 pmovzxbq m5, [pb_02461357]
3859 .w32_loop:
3860 W_MASK 0, 4, 0, 1
3861 mova m1, m8
3862 vpdpwssd m1, m4, m9
3863 add tmp1q, 128
3864 add tmp2q, 128
3865 vpermb m1, m10, m1
3866 vpermq m0, m5, m0
3867 pand ym1, ym11
3868 mova [maskq], ym1
3869 add maskq, 32
3870 mova [dstq+strideq*0], ym0
3871 vextracti32x8 [dstq+strideq*1], m0, 1
3872 lea dstq, [dstq+strideq*2]
3873 sub hd, 2
3874 jg .w32_loop
3875 RET
3876 .w64:
3877 pmovzxbq m5, [pb_02461357]
3878 .w64_loop:
3879 W_MASK 0, 4, 0, 1
3880 mova m1, m8
3881 vpdpwssd m1, m4, m9
3882 add tmp1q, 128
3883 add tmp2q, 128
3884 vpermb m1, m10, m1
3885 vpermq m0, m5, m0
3886 pand ym1, ym11
3887 mova [maskq], ym1
3888 add maskq, 32
3889 mova [dstq], m0
3890 add dstq, strideq
3891 dec hd
3892 jg .w64_loop
3893 RET
3894 .w128:
3895 pmovzxbq m13, [pb_02461357]
3896 .w128_loop:
3897 W_MASK 0, 4, 0, 1
3898 W_MASK 12, 5, 2, 3
3899 mova m2, m8
3900 vpdpwssd m2, m4, m9
3901 mova m3, m8
3902 vpdpwssd m3, m5, m9
3903 add tmp1q, 256
3904 add tmp2q, 256
3905 vpermt2b m2, m10, m3
3906 vpermq m0, m13, m0
3907 vpermq m1, m13, m12
3908 pand m2, m11
3909 mova [maskq], m2
3910 add maskq, 64
3911 mova [dstq+64*0], m0
3912 mova [dstq+64*1], m1
3913 add dstq, strideq
3914 dec hd
3915 jg .w128_loop
3916 RET
3917
3918 cglobal w_mask_444_8bpc, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3
3919 %define base r7-w_mask_444_avx512icl_table
3920 lea r7, [w_mask_444_avx512icl_table]
3921 tzcnt wd, wm
3922 movifnidn hd, hm
3923 movsxd wq, dword [r7+wq*4]
3924 vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
3925 vpbroadcastd m5, [base+pb_64]
3926 vpbroadcastd m7, [base+pw_2048]
3927 mova m8, [base+wm_444_mask]
3928 add wq, r7
3929 mov maskq, maskmp
3930 lea stride3q, [strideq*3]
3931 jmp wq
3932 .w4:
3933 cmp hd, 8
3934 jg .w4_h16
3935 WRAP_YMM W_MASK 0, 4, 0, 1, 1
3936 vinserti128 ym8, [wm_444_mask+32], 1
3937 vpermb ym4, ym8, ym4
3938 mova [maskq], ym4
3939 vextracti32x4 xm1, m0, 1
3940 movd [dstq+strideq*0], xm0
3941 pextrd [dstq+strideq*1], xm0, 1
3942 movd [dstq+strideq*2], xm1
3943 pextrd [dstq+stride3q ], xm1, 1
3944 jl .w4_end
3945 lea dstq, [dstq+strideq*4]
3946 pextrd [dstq+strideq*0], xm0, 2
3947 pextrd [dstq+strideq*1], xm0, 3
3948 pextrd [dstq+strideq*2], xm1, 2
3949 pextrd [dstq+stride3q ], xm1, 3
3950 .w4_end:
3951 RET
3952 .w4_h16:
3953 vpbroadcastd m9, strided
3954 pmulld m9, [bidir_sctr_w4]
3955 W_MASK 0, 4, 0, 1, 1
3956 vpermb m4, m8, m4
3957 kxnorw k1, k1, k1
3958 mova [maskq], m4
3959 vpscatterdd [dstq+m9]{k1}, m0
3960 RET
3961 .w8:
3962 cmp hd, 4
3963 jne .w8_h8
3964 WRAP_YMM W_MASK 0, 4, 0, 1, 1
3965 vinserti128 ym8, [wm_444_mask+32], 1
3966 vpermb ym4, ym8, ym4
3967 mova [maskq], ym4
3968 vextracti32x4 xm1, ym0, 1
3969 movq [dstq+strideq*0], xm0
3970 movq [dstq+strideq*1], xm1
3971 movhps [dstq+strideq*2], xm0
3972 movhps [dstq+stride3q ], xm1
3973 RET
3974 .w8_loop:
3975 add tmp1q, 128
3976 add tmp2q, 128
3977 add maskq, 64
3978 lea dstq, [dstq+strideq*4]
3979 .w8_h8:
3980 W_MASK 0, 4, 0, 1, 1
3981 vpermb m4, m8, m4
3982 mova [maskq], m4
3983 vextracti32x4 xm1, ym0, 1
3984 vextracti32x4 xm2, m0, 2
3985 vextracti32x4 xm3, m0, 3
3986 movq [dstq+strideq*0], xm0
3987 movq [dstq+strideq*1], xm1
3988 movq [dstq+strideq*2], xm2
3989 movq [dstq+stride3q ], xm3
3990 lea dstq, [dstq+strideq*4]
3991 movhps [dstq+strideq*0], xm0
3992 movhps [dstq+strideq*1], xm1
3993 movhps [dstq+strideq*2], xm2
3994 movhps [dstq+stride3q ], xm3
3995 sub hd, 8
3996 jg .w8_loop
3997 RET
3998 .w16_loop:
3999 add tmp1q, 128
4000 add tmp2q, 128
4001 add maskq, 64
4002 lea dstq, [dstq+strideq*4]
4003 .w16:
4004 W_MASK 0, 4, 0, 1, 1
4005 vpermb m4, m8, m4
4006 vpermq m0, m0, q3120
4007 mova [maskq], m4
4008 mova [dstq+strideq*0], xm0
4009 vextracti32x4 [dstq+strideq*1], m0, 2
4010 vextracti32x4 [dstq+strideq*2], ym0, 1
4011 vextracti32x4 [dstq+stride3q ], m0, 3
4012 sub hd, 4
4013 jg .w16_loop
4014 RET
4015 .w32:
4016 pmovzxbq m9, [pb_02461357]
4017 .w32_loop:
4018 W_MASK 0, 4, 0, 1, 1
4019 vpermb m4, m8, m4
4020 add tmp1q, 128
4021 add tmp2q, 128
4022 vpermq m0, m9, m0
4023 mova [maskq], m4
4024 add maskq, 64
4025 mova [dstq+strideq*0], ym0
4026 vextracti32x8 [dstq+strideq*1], m0, 1
4027 lea dstq, [dstq+strideq*2]
4028 sub hd, 2
4029 jg .w32_loop
4030 RET
4031 .w64:
4032 pmovzxbq m9, [pb_02461357]
4033 .w64_loop:
4034 W_MASK 0, 4, 0, 1, 1
4035 vpermb m4, m8, m4
4036 add tmp1q, 128
4037 add tmp2q, 128
4038 vpermq m0, m9, m0
4039 mova [maskq], m4
4040 add maskq, 64
4041 mova [dstq], m0
4042 add dstq, strideq
4043 dec hd
4044 jg .w64_loop
4045 RET
4046 .w128:
4047 pmovzxbq m11, [pb_02461357]
4048 .w128_loop:
4049 W_MASK 0, 4, 0, 1, 1
4050 W_MASK 10, 9, 2, 3, 1
4051 vpermb m4, m8, m4
4052 vpermb m9, m8, m9
4053 add tmp1q, 256
4054 add tmp2q, 256
4055 vpermq m0, m11, m0
4056 vpermq m10, m11, m10
4057 mova [maskq+64*0], m4
4058 mova [maskq+64*1], m9
4059 add maskq, 128
4060 mova [dstq+64*0], m0
4061 mova [dstq+64*1], m10
4062 add dstq, strideq
4063 dec hd
4064 jg .w128_loop
4065 RET
4066
4067 cglobal blend_8bpc, 3, 7, 8, dst, ds, tmp, w, h, mask
4068 %define base r6-blend_avx512icl_table
4069 lea r6, [blend_avx512icl_table]
4070 tzcnt wd, wm
4071 movifnidn maskq, maskmp
4072 movifnidn hd, hm
4073 movsxd wq, [r6+wq*4]
4074 vpbroadcastd m6, [base+pb_64]
4075 vpbroadcastd m7, [base+pw_512]
4076 sub tmpq, maskq
4077 add wq, r6
4078 lea r6, [dsq*3]
4079 jmp wq
4080 .w4:
4081 movd xmm0, [dstq+dsq*0]
4082 pinsrd xmm0, [dstq+dsq*1], 1
4083 vpbroadcastd xmm1, [dstq+dsq*2]
4084 pinsrd xmm1, [dstq+r6 ], 3
4085 mova xmm4, [maskq]
4086 mova xmm5, [maskq+tmpq]
4087 add maskq, 4*4
4088 psubb xmm3, xm6, xmm4
4089 punpcklbw xmm0, xmm5
4090 punpcklbw xmm2, xmm3, xmm4
4091 punpckhbw xmm1, xmm5
4092 punpckhbw xmm3, xmm4
4093 pmaddubsw xmm0, xmm2
4094 pmaddubsw xmm1, xmm3
4095 pmulhrsw xmm0, xm7
4096 pmulhrsw xmm1, xm7
4097 packuswb xmm0, xmm1
4098 movd [dstq+dsq*0], xmm0
4099 pextrd [dstq+dsq*1], xmm0, 1
4100 pextrd [dstq+dsq*2], xmm0, 2
4101 pextrd [dstq+r6 ], xmm0, 3
4102 lea dstq, [dstq+dsq*4]
4103 sub hd, 4
4104 jg .w4
4105 RET
4106 .w8:
4107 movq xmm0, [dstq+dsq*0]
4108 vpbroadcastq xmm1, [dstq+dsq*1]
4109 vpbroadcastq ymm2, [dstq+dsq*2]
4110 vpbroadcastq ymm3, [dstq+r6 ]
4111 mova ymm4, [maskq]
4112 mova ymm5, [maskq+tmpq]
4113 add maskq, 8*4
4114 vpblendd ymm0, ymm2, 0x30
4115 vpblendd ymm1, ymm3, 0xc0
4116 psubb ymm3, ym6, ymm4
4117 punpcklbw ymm0, ymm5
4118 punpcklbw ymm2, ymm3, ymm4
4119 punpckhbw ymm1, ymm5
4120 punpckhbw ymm3, ymm4
4121 pmaddubsw ymm0, ymm2
4122 pmaddubsw ymm1, ymm3
4123 pmulhrsw ymm0, ym7
4124 pmulhrsw ymm1, ym7
4125 packuswb ymm0, ymm1
4126 vextracti128 xmm1, ymm0, 1
4127 movq [dstq+dsq*0], xmm0
4128 movhps [dstq+dsq*1], xmm0
4129 movq [dstq+dsq*2], xmm1
4130 movhps [dstq+r6 ], xmm1
4131 lea dstq, [dstq+dsq*4]
4132 sub hd, 4
4133 jg .w8
4134 vzeroupper
4135 RET
4136 .w16:
4137 mova xm1, [dstq+dsq*0]
4138 vinserti32x4 ym1, [dstq+dsq*1], 1
4139 vinserti32x4 m1, [dstq+dsq*2], 2
4140 mova m4, [maskq]
4141 vinserti32x4 m1, [dstq+r6 ], 3
4142 mova m5, [maskq+tmpq]
4143 add maskq, 16*4
4144 psubb m3, m6, m4
4145 punpcklbw m0, m1, m5
4146 punpcklbw m2, m3, m4
4147 punpckhbw m1, m5
4148 punpckhbw m3, m4
4149 pmaddubsw m0, m2
4150 pmaddubsw m1, m3
4151 pmulhrsw m0, m7
4152 pmulhrsw m1, m7
4153 packuswb m0, m1
4154 mova [dstq+dsq*0], xm0
4155 vextracti32x4 [dstq+dsq*1], ym0, 1
4156 vextracti32x4 [dstq+dsq*2], m0, 2
4157 vextracti32x4 [dstq+r6 ], m0, 3
4158 lea dstq, [dstq+dsq*4]
4159 sub hd, 4
4160 jg .w16
4161 RET
4162 .w32:
4163 mova ym1, [dstq+dsq*0]
4164 vinserti32x8 m1, [dstq+dsq*1], 1
4165 mova m4, [maskq]
4166 mova m5, [maskq+tmpq]
4167 add maskq, 32*2
4168 psubb m3, m6, m4
4169 punpcklbw m0, m1, m5
4170 punpcklbw m2, m3, m4
4171 punpckhbw m1, m5
4172 punpckhbw m3, m4
4173 pmaddubsw m0, m2
4174 pmaddubsw m1, m3
4175 pmulhrsw m0, m7
4176 pmulhrsw m1, m7
4177 packuswb m0, m1
4178 mova [dstq+dsq*0], ym0
4179 vextracti32x8 [dstq+dsq*1], m0, 1
4180 lea dstq, [dstq+dsq*2]
4181 sub hd, 2
4182 jg .w32
4183 RET
4184
4185 cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask
4186 %define base r5-blend_v_avx512icl_table
4187 lea r5, [blend_v_avx512icl_table]
4188 tzcnt wd, wm
4189 movifnidn hd, hm
4190 movsxd wq, [r5+wq*4]
4191 vpbroadcastd m5, [base+pw_512]
4192 add wq, r5
4193 add maskq, obmc_masks-blend_v_avx512icl_table
4194 jmp wq
4195 .w2:
4196 vpbroadcastd xmm2, [maskq+2*2]
4197 .w2_s0_loop:
4198 movd xmm0, [dstq+dsq*0]
4199 pinsrw xmm0, [dstq+dsq*1], 1
4200 movd xmm1, [tmpq]
4201 add tmpq, 2*2
4202 punpcklbw xmm0, xmm1
4203 pmaddubsw xmm0, xmm2
4204 pmulhrsw xmm0, xm5
4205 packuswb xmm0, xmm0
4206 pextrw [dstq+dsq*0], xmm0, 0
4207 pextrw [dstq+dsq*1], xmm0, 1
4208 lea dstq, [dstq+dsq*2]
4209 sub hd, 2
4210 jg .w2_s0_loop
4211 RET
4212 .w4:
4213 vpbroadcastq xmm2, [maskq+4*2]
4214 .w4_loop:
4215 movd xmm0, [dstq+dsq*0]
4216 pinsrd xmm0, [dstq+dsq*1], 1
4217 movq xmm1, [tmpq]
4218 add tmpq, 4*2
4219 punpcklbw xmm0, xmm1
4220 pmaddubsw xmm0, xmm2
4221 pmulhrsw xmm0, xm5
4222 packuswb xmm0, xmm0
4223 movd [dstq+dsq*0], xmm0
4224 pextrd [dstq+dsq*1], xmm0, 1
4225 lea dstq, [dstq+dsq*2]
4226 sub hd, 2
4227 jg .w4_loop
4228 RET
4229 .w8:
4230 mova xmm3, [maskq+8*2]
4231 .w8_loop:
4232 movq xmm0, [dstq+dsq*0]
4233 vpbroadcastq xmm1, [dstq+dsq*1]
4234 mova xmm2, [tmpq]
4235 add tmpq, 8*2
4236 punpcklbw xmm0, xmm2
4237 punpckhbw xmm1, xmm2
4238 pmaddubsw xmm0, xmm3
4239 pmaddubsw xmm1, xmm3
4240 pmulhrsw xmm0, xm5
4241 pmulhrsw xmm1, xm5
4242 packuswb xmm0, xmm1
4243 movq [dstq+dsq*0], xmm0
4244 movhps [dstq+dsq*1], xmm0
4245 lea dstq, [dstq+dsq*2]
4246 sub hd, 2
4247 jg .w8_loop
4248 RET
4249 .w16:
4250 vbroadcasti32x4 ym3, [maskq+16*2]
4251 vbroadcasti32x4 ym4, [maskq+16*3]
4252 .w16_loop:
4253 mova xm1, [dstq+dsq*0]
4254 vinserti32x4 ym1, [dstq+dsq*1], 1
4255 mova ym2, [tmpq]
4256 add tmpq, 16*2
4257 punpcklbw ym0, ym1, ym2
4258 punpckhbw ym1, ym2
4259 pmaddubsw ym0, ym3
4260 pmaddubsw ym1, ym4
4261 pmulhrsw ym0, ym5
4262 pmulhrsw ym1, ym5
4263 packuswb ym0, ym1
4264 mova [dstq+dsq*0], xm0
4265 vextracti32x4 [dstq+dsq*1], m0, 1
4266 lea dstq, [dstq+dsq*2]
4267 sub hd, 2
4268 jg .w16_loop
4269 RET
4270 .w32:
4271 mova m4, [maskq+32*2]
4272 vshufi32x4 m3, m4, m4, q2020
4273 vshufi32x4 m4, m4, q3131
4274 .w32_loop:
4275 mova ym1, [dstq+dsq*0]
4276 vinserti32x8 m1, [dstq+dsq*1], 1
4277 mova m2, [tmpq]
4278 add tmpq, 32*2
4279 punpcklbw m0, m1, m2
4280 punpckhbw m1, m2
4281 pmaddubsw m0, m3
4282 pmaddubsw m1, m4
4283 pmulhrsw m0, m5
4284 pmulhrsw m1, m5
4285 packuswb m0, m1
4286 mova [dstq+dsq*0], ym0
4287 vextracti32x8 [dstq+dsq*1], m0, 1
4288 lea dstq, [dstq+dsq*2]
4289 sub hd, 2
4290 jg .w32_loop
4291 RET
4292
4293 cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask
4294 %define base r6-blend_h_avx512icl_table
4295 lea r6, [blend_h_avx512icl_table]
4296 tzcnt wd, wm
4297 mov hd, hm
4298 movsxd wq, [r6+wq*4]
4299 lea maskq, [base+obmc_masks+hq*2]
4300 vpbroadcastd m5, [base+pw_512]
4301 lea hd, [hq*3]
4302 add wq, r6
4303 shr hd, 2 ; h * 3/4
4304 lea maskq, [maskq+hq*2]
4305 neg hq
4306 jmp wq
4307 .w2:
4308 movd xmm0, [dstq+dsq*0]
4309 pinsrw xmm0, [dstq+dsq*1], 1
4310 movd xmm2, [maskq+hq*2]
4311 movd xmm1, [tmpq]
4312 add tmpq, 2*2
4313 punpcklwd xmm2, xmm2
4314 punpcklbw xmm0, xmm1
4315 pmaddubsw xmm0, xmm2
4316 pmulhrsw xmm0, xm5
4317 packuswb xmm0, xmm0
4318 pextrw [dstq+dsq*0], xmm0, 0
4319 pextrw [dstq+dsq*1], xmm0, 1
4320 lea dstq, [dstq+dsq*2]
4321 add hq, 2
4322 jl .w2
4323 RET
4324 .w4:
4325 mova xmm3, [blend_shuf]
4326 .w4_loop:
4327 movd xmm0, [dstq+dsq*0]
4328 pinsrd xmm0, [dstq+dsq*1], 1
4329 movd xmm2, [maskq+hq*2]
4330 movq xmm1, [tmpq]
4331 add tmpq, 4*2
4332 pshufb xmm2, xmm3
4333 punpcklbw xmm0, xmm1
4334 pmaddubsw xmm0, xmm2
4335 pmulhrsw xmm0, xm5
4336 packuswb xmm0, xmm0
4337 movd [dstq+dsq*0], xmm0
4338 pextrd [dstq+dsq*1], xmm0, 1
4339 lea dstq, [dstq+dsq*2]
4340 add hq, 2
4341 jl .w4_loop
4342 RET
4343 .w8:
4344 vbroadcasti128 ymm4, [blend_shuf]
4345 shufpd ymm4, ymm4, 0x03
4346 .w8_loop:
4347 vpbroadcastq ymm1, [dstq+dsq*0]
4348 movq xmm0, [dstq+dsq*1]
4349 vpblendd ymm0, ymm1, 0x30
4350 vpbroadcastd ymm3, [maskq+hq*2]
4351 movq xmm1, [tmpq+8*1]
4352 vinserti128 ymm1, [tmpq+8*0], 1
4353 add tmpq, 8*2
4354 pshufb ymm3, ymm4
4355 punpcklbw ymm0, ymm1
4356 pmaddubsw ymm0, ymm3
4357 pmulhrsw ymm0, ym5
4358 vextracti128 xmm1, ymm0, 1
4359 packuswb xmm0, xmm1
4360 movhps [dstq+dsq*0], xmm0
4361 movq [dstq+dsq*1], xmm0
4362 lea dstq, [dstq+dsq*2]
4363 add hq, 2
4364 jl .w8_loop
4365 vzeroupper
4366 RET
4367 .w16:
4368 vbroadcasti32x4 ym4, [blend_shuf]
4369 shufpd ym4, ym4, 0x0c
4370 .w16_loop:
4371 mova xm1, [dstq+dsq*0]
4372 vinserti32x4 ym1, [dstq+dsq*1], 1
4373 vpbroadcastd ym3, [maskq+hq*2]
4374 mova ym2, [tmpq]
4375 add tmpq, 16*2
4376 pshufb ym3, ym4
4377 punpcklbw ym0, ym1, ym2
4378 punpckhbw ym1, ym2
4379 pmaddubsw ym0, ym3
4380 pmaddubsw ym1, ym3
4381 pmulhrsw ym0, ym5
4382 pmulhrsw ym1, ym5
4383 packuswb ym0, ym1
4384 mova [dstq+dsq*0], xm0
4385 vextracti32x4 [dstq+dsq*1], m0, 1
4386 lea dstq, [dstq+dsq*2]
4387 add hq, 2
4388 jl .w16_loop
4389 RET
4390 .w32:
4391 vbroadcasti32x4 m4, [blend_shuf]
4392 shufpd m4, m4, 0xf0
4393 .w32_loop:
4394 mova ym1, [dstq+dsq*0]
4395 vinserti32x8 m1, [dstq+dsq*1], 1
4396 vpbroadcastd m3, [maskq+hq*2]
4397 mova m2, [tmpq]
4398 add tmpq, 32*2
4399 pshufb m3, m4
4400 punpcklbw m0, m1, m2
4401 punpckhbw m1, m2
4402 pmaddubsw m0, m3
4403 pmaddubsw m1, m3
4404 pmulhrsw m0, m5
4405 pmulhrsw m1, m5
4406 packuswb m0, m1
4407 mova [dstq+dsq*0], ym0
4408 vextracti32x8 [dstq+dsq*1], m0, 1
4409 lea dstq, [dstq+dsq*2]
4410 add hq, 2
4411 jl .w32_loop
4412 RET
4413 .w64:
4414 vpbroadcastw m3, [maskq+hq*2]
4415 mova m1, [dstq]
4416 mova m2, [tmpq]
4417 add tmpq, 32*2
4418 punpcklbw m0, m1, m2
4419 punpckhbw m1, m2
4420 pmaddubsw m0, m3
4421 pmaddubsw m1, m3
4422 pmulhrsw m0, m5
4423 pmulhrsw m1, m5
4424 packuswb m0, m1
4425 mova [dstq], m0
4426 add dstq, dsq
4427 inc hq
4428 jl .w64
4429 RET
4430 .w128:
4431 vpbroadcastw m6, [maskq+hq*2]
4432 mova m2, [dstq+64*0]
4433 mova m1, [tmpq+64*0]
4434 mova m3, [dstq+64*1]
4435 mova m4, [tmpq+64*1]
4436 add tmpq, 64*2
4437 punpcklbw m0, m2, m1
4438 punpckhbw m2, m1
4439 pmaddubsw m0, m6
4440 pmaddubsw m2, m6
4441 punpcklbw m1, m3, m4
4442 punpckhbw m3, m4
4443 pmaddubsw m1, m6
4444 pmaddubsw m3, m6
4445 REPX {pmulhrsw x, m5}, m0, m2, m1, m3
4446 packuswb m0, m2
4447 packuswb m1, m3
4448 mova [dstq+64*0], m0
4449 mova [dstq+64*1], m1
4450 add dstq, dsq
4451 inc hq
4452 jl .w128
4453 RET
4454
4455 cglobal resize_8bpc, 6, 12, 19, dst, dst_stride, src, src_stride, \
4456 dst_w, h, src_w, dx, mx0
4457 sub dword mx0m, 4<<14
4458 sub dword src_wm, 8
4459 mov r6, ~0
4460 vpbroadcastd m5, dxm
4461 vpbroadcastd m8, mx0m
4462 vpbroadcastd m6, src_wm
4463 kmovq k3, r6
4464 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x
4465 LEA r7, $$
4466 %define base r7-$$
4467 vpbroadcastd m3, [base+pw_m256]
4468 vpbroadcastd m7, [base+pd_63]
4469 vbroadcasti32x4 m15, [base+pb_8x0_8x8]
4470 vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15]
4471 pslld m5, 4 ; dx*16
4472 pslld m6, 14
4473 pxor m2, m2
4474 mova m16, [base+resize_permA]
4475 mova m17, [base+resize_permB]
4476 mova xm18, [base+resize_permC]
4477 .loop_y:
4478 xor xd, xd
4479 mova m4, m8 ; per-line working version of mx
4480 .loop_x:
4481 pmaxsd m0, m4, m2
4482 psrad m9, m4, 8 ; filter offset (unmasked)
4483 pminsd m0, m6 ; iclip(mx, 0, src_w-8)
4484 psubd m1, m4, m0 ; pshufb offset
4485 psrad m0, 14 ; clipped src_x offset
4486 psrad m1, 14 ; pshufb edge_emu offset
4487 vptestmd k4, m1, m1
4488 pand m9, m7 ; filter offset (masked)
4489 ktestw k4, k4
4490 jz .load
4491 vextracti32x8 ym12, m0, 1
4492 vextracti32x8 ym13, m1, 1
4493 kmovq k1, k3
4494 kmovq k2, k3
4495 vpgatherdq m10{k1}, [srcq+ym0]
4496 vpgatherdq m11{k2}, [srcq+ym12]
4497 kmovq k1, k3
4498 kmovq k2, k3
4499 vpgatherdq m14{k1}, [base+resize_shuf+4+ym1]
4500 vpgatherdq m0{k2}, [base+resize_shuf+4+ym13]
4501 mova m12, m16
4502 mova m13, m17
4503 paddb m14, m15
4504 paddb m0, m15
4505 pshufb m10, m14
4506 pshufb m11, m0
4507 vpermi2d m12, m10, m11
4508 vpermi2d m13, m10, m11
4509 jmp .filter
4510 .load:
4511 kmovq k1, k3
4512 kmovq k2, k3
4513 vpgatherdd m12{k1}, [srcq+m0+0]
4514 vpgatherdd m13{k2}, [srcq+m0+4]
4515 .filter:
4516 kmovq k1, k3
4517 kmovq k2, k3
4518 vpgatherdd m10{k1}, [base+resize_filter+m9*8+0]
4519 vpgatherdd m11{k2}, [base+resize_filter+m9*8+4]
4520 mova m14, m2
4521 vpdpbusd m14, m12, m10
4522 vpdpbusd m14, m13, m11
4523 packssdw m14, m14
4524 pmulhrsw m14, m3
4525 packuswb m14, m14
4526 vpermd m14, m18, m14
4527 mova [dstq+xq], xm14
4528 paddd m4, m5
4529 add xd, 16
4530 cmp xd, dst_wd
4531 jl .loop_x
4532 add dstq, dst_strideq
4533 add srcq, src_strideq
4534 dec hd
4535 jg .loop_y
4536 RET
4537
4538 %endif ; ARCH_X86_64