Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/mc16.S

raw
1 /*
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2018, Janne Grunau
4 * Copyright © 2020, Martin Storsjo
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice, this
11 * list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
24 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include "src/arm/asm.S"
30 #include "util.S"
31
32 #define PREP_BIAS 8192
33
34 .macro avg d0, d1, t0, t1, t2, t3
35 ld1 {\t0\().8h,\t1\().8h}, [x2], 32
36 ld1 {\t2\().8h,\t3\().8h}, [x3], 32
37 sqadd \t0\().8h, \t0\().8h, \t2\().8h
38 sqadd \t1\().8h, \t1\().8h, \t3\().8h
39 smax \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits
40 smax \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits
41 sqsub \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits
42 sqsub \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits
43 sshl \d0\().8h, \t0\().8h, v29.8h // -(intermediate_bits+1)
44 sshl \d1\().8h, \t1\().8h, v29.8h // -(intermediate_bits+1)
45 .endm
46
47 .macro w_avg d0, d1, t0, t1, t2, t3
48 ld1 {\t0\().8h,\t1\().8h}, [x2], 32
49 ld1 {\t2\().8h,\t3\().8h}, [x3], 32
50 // This difference requires a 17 bit range, and all bits are
51 // significant for the following multiplication.
52 ssubl \d0\().4s, \t2\().4h, \t0\().4h
53 ssubl2 \t0\().4s, \t2\().8h, \t0\().8h
54 ssubl \d1\().4s, \t3\().4h, \t1\().4h
55 ssubl2 \t1\().4s, \t3\().8h, \t1\().8h
56 mul \d0\().4s, \d0\().4s, v27.4s
57 mul \t0\().4s, \t0\().4s, v27.4s
58 mul \d1\().4s, \d1\().4s, v27.4s
59 mul \t1\().4s, \t1\().4s, v27.4s
60 sshr \d0\().4s, \d0\().4s, #4
61 sshr \t0\().4s, \t0\().4s, #4
62 sshr \d1\().4s, \d1\().4s, #4
63 sshr \t1\().4s, \t1\().4s, #4
64 saddw \d0\().4s, \d0\().4s, \t2\().4h
65 saddw2 \t0\().4s, \t0\().4s, \t2\().8h
66 saddw \d1\().4s, \d1\().4s, \t3\().4h
67 saddw2 \t1\().4s, \t1\().4s, \t3\().8h
68 uzp1 \d0\().8h, \d0\().8h, \t0\().8h // Same as xtn, xtn2
69 uzp1 \d1\().8h, \d1\().8h, \t1\().8h // Ditto
70 srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits
71 srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits
72 add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits
73 add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits
74 smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max
75 smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max
76 smax \d0\().8h, \d0\().8h, v30.8h // 0
77 smax \d1\().8h, \d1\().8h, v30.8h // 0
78 .endm
79
80 .macro mask d0, d1, t0, t1, t2, t3
81 ld1 {v27.16b}, [x6], 16
82 ld1 {\t0\().8h,\t1\().8h}, [x2], 32
83 neg v27.16b, v27.16b
84 ld1 {\t2\().8h,\t3\().8h}, [x3], 32
85 sxtl v26.8h, v27.8b
86 sxtl2 v27.8h, v27.16b
87 sxtl v24.4s, v26.4h
88 sxtl2 v25.4s, v26.8h
89 sxtl v26.4s, v27.4h
90 sxtl2 v27.4s, v27.8h
91 ssubl \d0\().4s, \t2\().4h, \t0\().4h
92 ssubl2 \t0\().4s, \t2\().8h, \t0\().8h
93 ssubl \d1\().4s, \t3\().4h, \t1\().4h
94 ssubl2 \t1\().4s, \t3\().8h, \t1\().8h
95 mul \d0\().4s, \d0\().4s, v24.4s
96 mul \t0\().4s, \t0\().4s, v25.4s
97 mul \d1\().4s, \d1\().4s, v26.4s
98 mul \t1\().4s, \t1\().4s, v27.4s
99 sshr \d0\().4s, \d0\().4s, #6
100 sshr \t0\().4s, \t0\().4s, #6
101 sshr \d1\().4s, \d1\().4s, #6
102 sshr \t1\().4s, \t1\().4s, #6
103 saddw \d0\().4s, \d0\().4s, \t2\().4h
104 saddw2 \t0\().4s, \t0\().4s, \t2\().8h
105 saddw \d1\().4s, \d1\().4s, \t3\().4h
106 saddw2 \t1\().4s, \t1\().4s, \t3\().8h
107 uzp1 \d0\().8h, \d0\().8h, \t0\().8h // Same as xtn, xtn2
108 uzp1 \d1\().8h, \d1\().8h, \t1\().8h // Ditto
109 srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits
110 srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits
111 add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits
112 add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits
113 smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max
114 smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max
115 smax \d0\().8h, \d0\().8h, v30.8h // 0
116 smax \d1\().8h, \d1\().8h, v30.8h // 0
117 .endm
118
119 .macro bidir_fn type, bdmax
120 function \type\()_16bpc_neon, export=1
121 clz w4, w4
122 .ifnc \type, avg
123 dup v31.8h, \bdmax // bitdepth_max
124 movi v30.8h, #0
125 .endif
126 clz w7, \bdmax
127 sub w7, w7, #18 // intermediate_bits = clz(bitdepth_max) - 18
128 .ifc \type, avg
129 mov w9, #1
130 mov w8, #-2*PREP_BIAS
131 lsl w9, w9, w7 // 1 << intermediate_bits
132 add w7, w7, #1
133 sub w8, w8, w9 // -2*PREP_BIAS - 1 << intermediate_bits
134 neg w7, w7 // -(intermediate_bits+1)
135 dup v28.8h, w8 // -2*PREP_BIAS - 1 << intermediate_bits
136 dup v29.8h, w7 // -(intermediate_bits+1)
137 .else
138 mov w8, #PREP_BIAS
139 lsr w8, w8, w7 // PREP_BIAS >> intermediate_bits
140 neg w7, w7 // -intermediate_bits
141 dup v28.8h, w8 // PREP_BIAS >> intermediate_bits
142 dup v29.8h, w7 // -intermediate_bits
143 .endif
144 .ifc \type, w_avg
145 dup v27.4s, w6
146 neg v27.4s, v27.4s
147 .endif
148 adr x7, L(\type\()_tbl)
149 sub w4, w4, #24
150 \type v4, v5, v0, v1, v2, v3
151 ldrh w4, [x7, x4, lsl #1]
152 sub x7, x7, w4, uxtw
153 br x7
154 40:
155 AARCH64_VALID_JUMP_TARGET
156 add x7, x0, x1
157 lsl x1, x1, #1
158 4:
159 subs w5, w5, #4
160 st1 {v4.d}[0], [x0], x1
161 st1 {v4.d}[1], [x7], x1
162 st1 {v5.d}[0], [x0], x1
163 st1 {v5.d}[1], [x7], x1
164 b.le 0f
165 \type v4, v5, v0, v1, v2, v3
166 b 4b
167 80:
168 AARCH64_VALID_JUMP_TARGET
169 add x7, x0, x1
170 lsl x1, x1, #1
171 8:
172 st1 {v4.8h}, [x0], x1
173 subs w5, w5, #2
174 st1 {v5.8h}, [x7], x1
175 b.le 0f
176 \type v4, v5, v0, v1, v2, v3
177 b 8b
178 16:
179 AARCH64_VALID_JUMP_TARGET
180 \type v6, v7, v0, v1, v2, v3
181 st1 {v4.8h, v5.8h}, [x0], x1
182 subs w5, w5, #2
183 st1 {v6.8h, v7.8h}, [x0], x1
184 b.le 0f
185 \type v4, v5, v0, v1, v2, v3
186 b 16b
187 32:
188 AARCH64_VALID_JUMP_TARGET
189 \type v6, v7, v0, v1, v2, v3
190 subs w5, w5, #1
191 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1
192 b.le 0f
193 \type v4, v5, v0, v1, v2, v3
194 b 32b
195 640:
196 AARCH64_VALID_JUMP_TARGET
197 add x7, x0, #64
198 64:
199 \type v6, v7, v0, v1, v2, v3
200 \type v16, v17, v0, v1, v2, v3
201 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1
202 \type v18, v19, v0, v1, v2, v3
203 subs w5, w5, #1
204 st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1
205 b.le 0f
206 \type v4, v5, v0, v1, v2, v3
207 b 64b
208 1280:
209 AARCH64_VALID_JUMP_TARGET
210 add x7, x0, #64
211 mov x8, #128
212 sub x1, x1, #128
213 128:
214 \type v6, v7, v0, v1, v2, v3
215 \type v16, v17, v0, v1, v2, v3
216 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x8
217 \type v18, v19, v0, v1, v2, v3
218 st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x8
219 \type v4, v5, v0, v1, v2, v3
220 \type v6, v7, v0, v1, v2, v3
221 \type v16, v17, v0, v1, v2, v3
222 subs w5, w5, #1
223 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1
224 \type v18, v19, v0, v1, v2, v3
225 st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1
226 b.le 0f
227 \type v4, v5, v0, v1, v2, v3
228 b 128b
229 0:
230 ret
231 L(\type\()_tbl):
232 .hword L(\type\()_tbl) - 1280b
233 .hword L(\type\()_tbl) - 640b
234 .hword L(\type\()_tbl) - 32b
235 .hword L(\type\()_tbl) - 16b
236 .hword L(\type\()_tbl) - 80b
237 .hword L(\type\()_tbl) - 40b
238 endfunc
239 .endm
240
241 bidir_fn avg, w6
242 bidir_fn w_avg, w7
243 bidir_fn mask, w7
244
245
246 .macro w_mask_fn type
247 function w_mask_\type\()_16bpc_neon, export=1
248 ldr w8, [sp]
249 clz w9, w4
250 adr x10, L(w_mask_\type\()_tbl)
251 dup v31.8h, w8 // bitdepth_max
252 sub w9, w9, #24
253 clz w8, w8 // clz(bitdepth_max)
254 ldrh w9, [x10, x9, lsl #1]
255 sub x10, x10, w9, uxtw
256 sub w8, w8, #12 // sh = intermediate_bits + 6 = clz(bitdepth_max) - 12
257 mov w9, #PREP_BIAS*64
258 neg w8, w8 // -sh
259 mov w11, #27615 // (64 + 1 - 38)<<mask_sh - 1 - mask_rnd
260 dup v30.4s, w9 // PREP_BIAS*64
261 dup v29.4s, w8 // -sh
262 dup v0.8h, w11
263 .if \type == 444
264 movi v1.16b, #64
265 .elseif \type == 422
266 dup v2.8b, w7
267 movi v3.8b, #129
268 sub v3.8b, v3.8b, v2.8b
269 .elseif \type == 420
270 dup v2.8h, w7
271 movi v3.8h, #1, lsl #8
272 sub v3.8h, v3.8h, v2.8h
273 .endif
274 add x12, x0, x1
275 lsl x1, x1, #1
276 br x10
277 4:
278 AARCH64_VALID_JUMP_TARGET
279 ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 (four rows at once)
280 ld1 {v6.8h, v7.8h}, [x3], #32 // tmp2 (four rows at once)
281 subs w5, w5, #4
282 sabd v20.8h, v4.8h, v6.8h // abs(tmp1 - tmp2)
283 sabd v21.8h, v5.8h, v7.8h
284 ssubl v16.4s, v6.4h, v4.4h // tmp2 - tmp1 (requires 17 bit)
285 ssubl2 v17.4s, v6.8h, v4.8h
286 ssubl v18.4s, v7.4h, v5.4h
287 ssubl2 v19.4s, v7.8h, v5.8h
288 uqsub v20.8h, v0.8h, v20.8h // 27615 - abs()
289 uqsub v21.8h, v0.8h, v21.8h
290 sshll2 v7.4s, v5.8h, #6 // tmp1 << 6
291 sshll v6.4s, v5.4h, #6
292 sshll2 v5.4s, v4.8h, #6
293 sshll v4.4s, v4.4h, #6
294 ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh
295 ushr v21.8h, v21.8h, #10
296 add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64
297 add v5.4s, v5.4s, v30.4s
298 add v6.4s, v6.4s, v30.4s
299 add v7.4s, v7.4s, v30.4s
300 uxtl v22.4s, v20.4h
301 uxtl2 v23.4s, v20.8h
302 uxtl v24.4s, v21.4h
303 uxtl2 v25.4s, v21.8h
304 mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m)
305 mla v5.4s, v17.4s, v23.4s
306 mla v6.4s, v18.4s, v24.4s
307 mla v7.4s, v19.4s, v25.4s
308 srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh
309 srshl v5.4s, v5.4s, v29.4s
310 srshl v6.4s, v6.4s, v29.4s
311 srshl v7.4s, v7.4s, v29.4s
312 sqxtun v4.4h, v4.4s // iclip_pixel
313 sqxtun2 v4.8h, v5.4s
314 sqxtun v5.4h, v6.4s
315 sqxtun2 v5.8h, v7.4s
316 umin v4.8h, v4.8h, v31.8h // iclip_pixel
317 umin v5.8h, v5.8h, v31.8h
318 .if \type == 444
319 uzp1 v20.16b, v20.16b, v21.16b // 64 - m
320 sub v20.16b, v1.16b, v20.16b // m
321 st1 {v20.16b}, [x6], #16
322 .elseif \type == 422
323 addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition)
324 xtn v20.8b, v20.8h
325 uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1
326 st1 {v20.8b}, [x6], #8
327 .elseif \type == 420
328 trn1 v24.2d, v20.2d, v21.2d
329 trn2 v25.2d, v20.2d, v21.2d
330 add v24.8h, v24.8h, v25.8h // (64 - my1) + (64 - my2) (row wise addition)
331 addp v20.8h, v24.8h, v24.8h // (128 - m) + (128 - n) (column wise addition)
332 sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n))
333 rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2
334 st1 {v20.s}[0], [x6], #4
335 .endif
336 st1 {v4.d}[0], [x0], x1
337 st1 {v4.d}[1], [x12], x1
338 st1 {v5.d}[0], [x0], x1
339 st1 {v5.d}[1], [x12], x1
340 b.gt 4b
341 ret
342 8:
343 AARCH64_VALID_JUMP_TARGET
344 ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1
345 ld1 {v6.8h, v7.8h}, [x3], #32 // tmp2
346 subs w5, w5, #2
347 sabd v20.8h, v4.8h, v6.8h // abs(tmp1 - tmp2)
348 sabd v21.8h, v5.8h, v7.8h
349 ssubl v16.4s, v6.4h, v4.4h // tmp2 - tmp1 (requires 17 bit)
350 ssubl2 v17.4s, v6.8h, v4.8h
351 ssubl v18.4s, v7.4h, v5.4h
352 ssubl2 v19.4s, v7.8h, v5.8h
353 uqsub v20.8h, v0.8h, v20.8h // 27615 - abs()
354 uqsub v21.8h, v0.8h, v21.8h
355 sshll2 v7.4s, v5.8h, #6 // tmp1 << 6
356 sshll v6.4s, v5.4h, #6
357 sshll2 v5.4s, v4.8h, #6
358 sshll v4.4s, v4.4h, #6
359 ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh
360 ushr v21.8h, v21.8h, #10
361 add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64
362 add v5.4s, v5.4s, v30.4s
363 add v6.4s, v6.4s, v30.4s
364 add v7.4s, v7.4s, v30.4s
365 uxtl v22.4s, v20.4h
366 uxtl2 v23.4s, v20.8h
367 uxtl v24.4s, v21.4h
368 uxtl2 v25.4s, v21.8h
369 mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m)
370 mla v5.4s, v17.4s, v23.4s
371 mla v6.4s, v18.4s, v24.4s
372 mla v7.4s, v19.4s, v25.4s
373 srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh
374 srshl v5.4s, v5.4s, v29.4s
375 srshl v6.4s, v6.4s, v29.4s
376 srshl v7.4s, v7.4s, v29.4s
377 sqxtun v4.4h, v4.4s // iclip_pixel
378 sqxtun2 v4.8h, v5.4s
379 sqxtun v5.4h, v6.4s
380 sqxtun2 v5.8h, v7.4s
381 umin v4.8h, v4.8h, v31.8h // iclip_pixel
382 umin v5.8h, v5.8h, v31.8h
383 .if \type == 444
384 uzp1 v20.16b, v20.16b, v21.16b // 64 - m
385 sub v20.16b, v1.16b, v20.16b // m
386 st1 {v20.16b}, [x6], #16
387 .elseif \type == 422
388 addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition)
389 xtn v20.8b, v20.8h
390 uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1
391 st1 {v20.8b}, [x6], #8
392 .elseif \type == 420
393 add v20.8h, v20.8h, v21.8h // (64 - my1) + (64 - my2) (row wise addition)
394 addp v20.8h, v20.8h, v20.8h // (128 - m) + (128 - n) (column wise addition)
395 sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n))
396 rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2
397 st1 {v20.s}[0], [x6], #4
398 .endif
399 st1 {v4.8h}, [x0], x1
400 st1 {v5.8h}, [x12], x1
401 b.gt 8b
402 ret
403 1280:
404 640:
405 320:
406 160:
407 AARCH64_VALID_JUMP_TARGET
408 mov w11, w4
409 sub x1, x1, w4, uxtw #1
410 .if \type == 444
411 add x10, x6, w4, uxtw
412 .elseif \type == 422
413 add x10, x6, x11, lsr #1
414 .endif
415 add x9, x3, w4, uxtw #1
416 add x7, x2, w4, uxtw #1
417 161:
418 mov w8, w4
419 16:
420 ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1
421 ld1 {v16.8h, v17.8h}, [x3], #32 // tmp2
422 ld1 {v6.8h, v7.8h}, [x7], #32
423 ld1 {v18.8h, v19.8h}, [x9], #32
424 subs w8, w8, #16
425 sabd v20.8h, v4.8h, v16.8h // abs(tmp1 - tmp2)
426 sabd v21.8h, v5.8h, v17.8h
427 ssubl v22.4s, v16.4h, v4.4h // tmp2 - tmp1 (requires 17 bit)
428 ssubl2 v23.4s, v16.8h, v4.8h
429 ssubl v24.4s, v17.4h, v5.4h
430 ssubl2 v25.4s, v17.8h, v5.8h
431 uqsub v20.8h, v0.8h, v20.8h // 27615 - abs()
432 uqsub v21.8h, v0.8h, v21.8h
433 sshll2 v27.4s, v5.8h, #6 // tmp1 << 6
434 sshll v26.4s, v5.4h, #6
435 sshll2 v5.4s, v4.8h, #6
436 sshll v4.4s, v4.4h, #6
437 ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh
438 ushr v21.8h, v21.8h, #10
439 add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64
440 add v5.4s, v5.4s, v30.4s
441 add v26.4s, v26.4s, v30.4s
442 add v27.4s, v27.4s, v30.4s
443 uxtl v16.4s, v20.4h
444 uxtl2 v17.4s, v20.8h
445 uxtl v28.4s, v21.4h
446 mla v4.4s, v22.4s, v16.4s // (tmp2-tmp1)*(64-m)
447 uxtl2 v16.4s, v21.8h
448 mla v5.4s, v23.4s, v17.4s
449 mla v26.4s, v24.4s, v28.4s
450 mla v27.4s, v25.4s, v16.4s
451 srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh
452 srshl v5.4s, v5.4s, v29.4s
453 srshl v26.4s, v26.4s, v29.4s
454 srshl v27.4s, v27.4s, v29.4s
455 sqxtun v4.4h, v4.4s // iclip_pixel
456 sqxtun2 v4.8h, v5.4s
457 sqxtun v5.4h, v26.4s
458 sqxtun2 v5.8h, v27.4s
459
460 // Start of other half
461 sabd v22.8h, v6.8h, v18.8h // abs(tmp1 - tmp2)
462 sabd v23.8h, v7.8h, v19.8h
463
464 umin v4.8h, v4.8h, v31.8h // iclip_pixel
465 umin v5.8h, v5.8h, v31.8h
466
467 ssubl v16.4s, v18.4h, v6.4h // tmp2 - tmp1 (requires 17 bit)
468 ssubl2 v17.4s, v18.8h, v6.8h
469 ssubl v18.4s, v19.4h, v7.4h
470 ssubl2 v19.4s, v19.8h, v7.8h
471 uqsub v22.8h, v0.8h, v22.8h // 27615 - abs()
472 uqsub v23.8h, v0.8h, v23.8h
473 sshll v24.4s, v6.4h, #6 // tmp1 << 6
474 sshll2 v25.4s, v6.8h, #6
475 sshll v26.4s, v7.4h, #6
476 sshll2 v27.4s, v7.8h, #6
477 ushr v22.8h, v22.8h, #10 // 64-m = (27615 - abs()) >> mask_sh
478 ushr v23.8h, v23.8h, #10
479 add v24.4s, v24.4s, v30.4s // += PREP_BIAS*64
480 add v25.4s, v25.4s, v30.4s
481 add v26.4s, v26.4s, v30.4s
482 add v27.4s, v27.4s, v30.4s
483 uxtl v6.4s, v22.4h
484 uxtl2 v7.4s, v22.8h
485 uxtl v28.4s, v23.4h
486 mla v24.4s, v16.4s, v6.4s // (tmp2-tmp1)*(64-m)
487 uxtl2 v6.4s, v23.8h
488 mla v25.4s, v17.4s, v7.4s
489 mla v26.4s, v18.4s, v28.4s
490 mla v27.4s, v19.4s, v6.4s
491 srshl v24.4s, v24.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh
492 srshl v25.4s, v25.4s, v29.4s
493 srshl v26.4s, v26.4s, v29.4s
494 srshl v27.4s, v27.4s, v29.4s
495 sqxtun v6.4h, v24.4s // iclip_pixel
496 sqxtun2 v6.8h, v25.4s
497 sqxtun v7.4h, v26.4s
498 sqxtun2 v7.8h, v27.4s
499 umin v6.8h, v6.8h, v31.8h // iclip_pixel
500 umin v7.8h, v7.8h, v31.8h
501 .if \type == 444
502 uzp1 v20.16b, v20.16b, v21.16b // 64 - m
503 uzp1 v21.16b, v22.16b, v23.16b
504 sub v20.16b, v1.16b, v20.16b // m
505 sub v21.16b, v1.16b, v21.16b
506 st1 {v20.16b}, [x6], #16
507 st1 {v21.16b}, [x10], #16
508 .elseif \type == 422
509 addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition)
510 addp v21.8h, v22.8h, v23.8h
511 xtn v20.8b, v20.8h
512 xtn v21.8b, v21.8h
513 uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1
514 uhsub v21.8b, v3.8b, v21.8b
515 st1 {v20.8b}, [x6], #8
516 st1 {v21.8b}, [x10], #8
517 .elseif \type == 420
518 add v20.8h, v20.8h, v22.8h // (64 - my1) + (64 - my2) (row wise addition)
519 add v21.8h, v21.8h, v23.8h
520 addp v20.8h, v20.8h, v21.8h // (128 - m) + (128 - n) (column wise addition)
521 sub v20.8h, v3.8h, v20.8h // (256 - sign) - ((128 - m) + (128 - n))
522 rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2
523 st1 {v20.8b}, [x6], #8
524 .endif
525 st1 {v4.8h, v5.8h}, [x0], #32
526 st1 {v6.8h, v7.8h}, [x12], #32
527 b.gt 16b
528 subs w5, w5, #2
529 add x2, x2, w4, uxtw #1
530 add x3, x3, w4, uxtw #1
531 add x7, x7, w4, uxtw #1
532 add x9, x9, w4, uxtw #1
533 .if \type == 444
534 add x6, x6, w4, uxtw
535 add x10, x10, w4, uxtw
536 .elseif \type == 422
537 add x6, x6, x11, lsr #1
538 add x10, x10, x11, lsr #1
539 .endif
540 add x0, x0, x1
541 add x12, x12, x1
542 b.gt 161b
543 ret
544 L(w_mask_\type\()_tbl):
545 .hword L(w_mask_\type\()_tbl) - 1280b
546 .hword L(w_mask_\type\()_tbl) - 640b
547 .hword L(w_mask_\type\()_tbl) - 320b
548 .hword L(w_mask_\type\()_tbl) - 160b
549 .hword L(w_mask_\type\()_tbl) - 8b
550 .hword L(w_mask_\type\()_tbl) - 4b
551 endfunc
552 .endm
553
554 w_mask_fn 444
555 w_mask_fn 422
556 w_mask_fn 420
557
558
559 function blend_16bpc_neon, export=1
560 adr x6, L(blend_tbl)
561 clz w3, w3
562 sub w3, w3, #26
563 ldrh w3, [x6, x3, lsl #1]
564 sub x6, x6, w3, uxtw
565 add x8, x0, x1
566 br x6
567 40:
568 AARCH64_VALID_JUMP_TARGET
569 lsl x1, x1, #1
570 4:
571 ld1 {v2.8b}, [x5], #8
572 ld1 {v1.8h}, [x2], #16
573 ld1 {v0.d}[0], [x0]
574 neg v2.8b, v2.8b // -m
575 subs w4, w4, #2
576 ld1 {v0.d}[1], [x8]
577 sxtl v2.8h, v2.8b
578 shl v2.8h, v2.8h, #9 // -m << 9
579 sub v1.8h, v0.8h, v1.8h // a - b
580 sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6
581 add v0.8h, v0.8h, v1.8h
582 st1 {v0.d}[0], [x0], x1
583 st1 {v0.d}[1], [x8], x1
584 b.gt 4b
585 ret
586 80:
587 AARCH64_VALID_JUMP_TARGET
588 lsl x1, x1, #1
589 8:
590 ld1 {v4.16b}, [x5], #16
591 ld1 {v2.8h, v3.8h}, [x2], #32
592 neg v5.16b, v4.16b // -m
593 ld1 {v0.8h}, [x0]
594 ld1 {v1.8h}, [x8]
595 sxtl v4.8h, v5.8b
596 sxtl2 v5.8h, v5.16b
597 shl v4.8h, v4.8h, #9 // -m << 9
598 shl v5.8h, v5.8h, #9
599 sub v2.8h, v0.8h, v2.8h // a - b
600 sub v3.8h, v1.8h, v3.8h
601 subs w4, w4, #2
602 sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6
603 sqrdmulh v3.8h, v3.8h, v5.8h
604 add v0.8h, v0.8h, v2.8h
605 add v1.8h, v1.8h, v3.8h
606 st1 {v0.8h}, [x0], x1
607 st1 {v1.8h}, [x8], x1
608 b.gt 8b
609 ret
610 160:
611 AARCH64_VALID_JUMP_TARGET
612 lsl x1, x1, #1
613 16:
614 ld1 {v16.16b, v17.16b}, [x5], #32
615 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64
616 subs w4, w4, #2
617 neg v18.16b, v16.16b // -m
618 neg v19.16b, v17.16b
619 ld1 {v0.8h, v1.8h}, [x0]
620 sxtl v16.8h, v18.8b
621 sxtl2 v17.8h, v18.16b
622 sxtl v18.8h, v19.8b
623 sxtl2 v19.8h, v19.16b
624 ld1 {v2.8h, v3.8h}, [x8]
625 shl v16.8h, v16.8h, #9 // -m << 9
626 shl v17.8h, v17.8h, #9
627 shl v18.8h, v18.8h, #9
628 shl v19.8h, v19.8h, #9
629 sub v4.8h, v0.8h, v4.8h // a - b
630 sub v5.8h, v1.8h, v5.8h
631 sub v6.8h, v2.8h, v6.8h
632 sub v7.8h, v3.8h, v7.8h
633 sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6
634 sqrdmulh v5.8h, v5.8h, v17.8h
635 sqrdmulh v6.8h, v6.8h, v18.8h
636 sqrdmulh v7.8h, v7.8h, v19.8h
637 add v0.8h, v0.8h, v4.8h
638 add v1.8h, v1.8h, v5.8h
639 add v2.8h, v2.8h, v6.8h
640 add v3.8h, v3.8h, v7.8h
641 st1 {v0.8h, v1.8h}, [x0], x1
642 st1 {v2.8h, v3.8h}, [x8], x1
643 b.gt 16b
644 ret
645 32:
646 AARCH64_VALID_JUMP_TARGET
647 ld1 {v16.16b, v17.16b}, [x5], #32
648 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64
649 subs w4, w4, #1
650 neg v18.16b, v16.16b // -m
651 neg v19.16b, v17.16b
652 sxtl v16.8h, v18.8b
653 sxtl2 v17.8h, v18.16b
654 sxtl v18.8h, v19.8b
655 sxtl2 v19.8h, v19.16b
656 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0]
657 shl v16.8h, v16.8h, #9 // -m << 9
658 shl v17.8h, v17.8h, #9
659 shl v18.8h, v18.8h, #9
660 shl v19.8h, v19.8h, #9
661 sub v4.8h, v0.8h, v4.8h // a - b
662 sub v5.8h, v1.8h, v5.8h
663 sub v6.8h, v2.8h, v6.8h
664 sub v7.8h, v3.8h, v7.8h
665 sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6
666 sqrdmulh v5.8h, v5.8h, v17.8h
667 sqrdmulh v6.8h, v6.8h, v18.8h
668 sqrdmulh v7.8h, v7.8h, v19.8h
669 add v0.8h, v0.8h, v4.8h
670 add v1.8h, v1.8h, v5.8h
671 add v2.8h, v2.8h, v6.8h
672 add v3.8h, v3.8h, v7.8h
673 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1
674 b.gt 32b
675 ret
676 L(blend_tbl):
677 .hword L(blend_tbl) - 32b
678 .hword L(blend_tbl) - 160b
679 .hword L(blend_tbl) - 80b
680 .hword L(blend_tbl) - 40b
681 endfunc
682
683 function blend_h_16bpc_neon, export=1
684 adr x6, L(blend_h_tbl)
685 movrel x5, X(obmc_masks)
686 add x5, x5, w4, uxtw
687 sub w4, w4, w4, lsr #2
688 clz w7, w3
689 add x8, x0, x1
690 lsl x1, x1, #1
691 sub w7, w7, #24
692 ldrh w7, [x6, x7, lsl #1]
693 sub x6, x6, w7, uxtw
694 br x6
695 2:
696 AARCH64_VALID_JUMP_TARGET
697 ld2r {v2.8b, v3.8b}, [x5], #2
698 ld1 {v1.4h}, [x2], #8
699 ext v2.8b, v2.8b, v3.8b, #6
700 subs w4, w4, #2
701 neg v2.8b, v2.8b // -m
702 ld1 {v0.s}[0], [x0]
703 ld1 {v0.s}[1], [x8]
704 sxtl v2.8h, v2.8b
705 shl v2.4h, v2.4h, #9 // -m << 9
706 sub v1.4h, v0.4h, v1.4h // a - b
707 sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6
708 add v0.4h, v0.4h, v1.4h
709 st1 {v0.s}[0], [x0], x1
710 st1 {v0.s}[1], [x8], x1
711 b.gt 2b
712 ret
713 4:
714 AARCH64_VALID_JUMP_TARGET
715 ld2r {v2.8b, v3.8b}, [x5], #2
716 ld1 {v1.8h}, [x2], #16
717 ext v2.8b, v2.8b, v3.8b, #4
718 subs w4, w4, #2
719 neg v2.8b, v2.8b // -m
720 ld1 {v0.d}[0], [x0]
721 ld1 {v0.d}[1], [x8]
722 sxtl v2.8h, v2.8b
723 shl v2.8h, v2.8h, #9 // -m << 9
724 sub v1.8h, v0.8h, v1.8h // a - b
725 sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6
726 add v0.8h, v0.8h, v1.8h
727 st1 {v0.d}[0], [x0], x1
728 st1 {v0.d}[1], [x8], x1
729 b.gt 4b
730 ret
731 8:
732 AARCH64_VALID_JUMP_TARGET
733 ld2r {v4.8b, v5.8b}, [x5], #2
734 ld1 {v2.8h, v3.8h}, [x2], #32
735 neg v4.8b, v4.8b // -m
736 neg v5.8b, v5.8b
737 ld1 {v0.8h}, [x0]
738 subs w4, w4, #2
739 sxtl v4.8h, v4.8b
740 sxtl v5.8h, v5.8b
741 ld1 {v1.8h}, [x8]
742 shl v4.8h, v4.8h, #9 // -m << 9
743 shl v5.8h, v5.8h, #9
744 sub v2.8h, v0.8h, v2.8h // a - b
745 sub v3.8h, v1.8h, v3.8h
746 sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6
747 sqrdmulh v3.8h, v3.8h, v5.8h
748 add v0.8h, v0.8h, v2.8h
749 add v1.8h, v1.8h, v3.8h
750 st1 {v0.8h}, [x0], x1
751 st1 {v1.8h}, [x8], x1
752 b.gt 8b
753 ret
754 16:
755 AARCH64_VALID_JUMP_TARGET
756 ld2r {v16.8b, v17.8b}, [x5], #2
757 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64
758 neg v16.8b, v16.8b // -m
759 neg v17.8b, v17.8b
760 ld1 {v0.8h, v1.8h}, [x0]
761 ld1 {v2.8h, v3.8h}, [x8]
762 subs w4, w4, #2
763 sxtl v16.8h, v16.8b
764 sxtl v17.8h, v17.8b
765 shl v16.8h, v16.8h, #9 // -m << 9
766 shl v17.8h, v17.8h, #9
767 sub v4.8h, v0.8h, v4.8h // a - b
768 sub v5.8h, v1.8h, v5.8h
769 sub v6.8h, v2.8h, v6.8h
770 sub v7.8h, v3.8h, v7.8h
771 sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6
772 sqrdmulh v5.8h, v5.8h, v16.8h
773 sqrdmulh v6.8h, v6.8h, v17.8h
774 sqrdmulh v7.8h, v7.8h, v17.8h
775 add v0.8h, v0.8h, v4.8h
776 add v1.8h, v1.8h, v5.8h
777 add v2.8h, v2.8h, v6.8h
778 add v3.8h, v3.8h, v7.8h
779 st1 {v0.8h, v1.8h}, [x0], x1
780 st1 {v2.8h, v3.8h}, [x8], x1
781 b.gt 16b
782 ret
783 1280:
784 640:
785 320:
786 AARCH64_VALID_JUMP_TARGET
787 sub x1, x1, w3, uxtw #1
788 add x7, x2, w3, uxtw #1
789 321:
790 ld2r {v24.8b, v25.8b}, [x5], #2
791 mov w6, w3
792 neg v24.8b, v24.8b // -m
793 neg v25.8b, v25.8b
794 sxtl v24.8h, v24.8b
795 sxtl v25.8h, v25.8b
796 shl v24.8h, v24.8h, #9 // -m << 9
797 shl v25.8h, v25.8h, #9
798 32:
799 ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64
800 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0]
801 subs w6, w6, #32
802 sub v16.8h, v0.8h, v16.8h // a - b
803 sub v17.8h, v1.8h, v17.8h
804 sub v18.8h, v2.8h, v18.8h
805 sub v19.8h, v3.8h, v19.8h
806 ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64
807 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8]
808 sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6
809 sqrdmulh v17.8h, v17.8h, v24.8h
810 sqrdmulh v18.8h, v18.8h, v24.8h
811 sqrdmulh v19.8h, v19.8h, v24.8h
812 sub v20.8h, v4.8h, v20.8h // a - b
813 sub v21.8h, v5.8h, v21.8h
814 sub v22.8h, v6.8h, v22.8h
815 sub v23.8h, v7.8h, v23.8h
816 add v0.8h, v0.8h, v16.8h
817 add v1.8h, v1.8h, v17.8h
818 add v2.8h, v2.8h, v18.8h
819 add v3.8h, v3.8h, v19.8h
820 sqrdmulh v20.8h, v20.8h, v25.8h // ((a-b)*-m + 32) >> 6
821 sqrdmulh v21.8h, v21.8h, v25.8h
822 sqrdmulh v22.8h, v22.8h, v25.8h
823 sqrdmulh v23.8h, v23.8h, v25.8h
824 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
825 add v4.8h, v4.8h, v20.8h
826 add v5.8h, v5.8h, v21.8h
827 add v6.8h, v6.8h, v22.8h
828 add v7.8h, v7.8h, v23.8h
829 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8], #64
830 b.gt 32b
831 subs w4, w4, #2
832 add x0, x0, x1
833 add x8, x8, x1
834 add x2, x2, w3, uxtw #1
835 add x7, x7, w3, uxtw #1
836 b.gt 321b
837 ret
838 L(blend_h_tbl):
839 .hword L(blend_h_tbl) - 1280b
840 .hword L(blend_h_tbl) - 640b
841 .hword L(blend_h_tbl) - 320b
842 .hword L(blend_h_tbl) - 16b
843 .hword L(blend_h_tbl) - 8b
844 .hword L(blend_h_tbl) - 4b
845 .hword L(blend_h_tbl) - 2b
846 endfunc
847
848 function blend_v_16bpc_neon, export=1
849 adr x6, L(blend_v_tbl)
850 movrel x5, X(obmc_masks)
851 add x5, x5, w3, uxtw
852 clz w3, w3
853 add x8, x0, x1
854 lsl x1, x1, #1
855 sub w3, w3, #26
856 ldrh w3, [x6, x3, lsl #1]
857 sub x6, x6, w3, uxtw
858 br x6
859 20:
860 AARCH64_VALID_JUMP_TARGET
861 ld1r {v2.8b}, [x5]
862 neg v2.8b, v2.8b // -m
863 sxtl v2.8h, v2.8b
864 shl v2.4h, v2.4h, #9 // -m << 9
865 2:
866 ld1 {v1.s}[0], [x2], #4
867 ld1 {v0.h}[0], [x0]
868 subs w4, w4, #2
869 ld1 {v1.h}[1], [x2]
870 ld1 {v0.h}[1], [x8]
871 add x2, x2, #4
872 sub v1.4h, v0.4h, v1.4h // a - b
873 sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6
874 add v0.4h, v0.4h, v1.4h
875 st1 {v0.h}[0], [x0], x1
876 st1 {v0.h}[1], [x8], x1
877 b.gt 2b
878 ret
879 40:
880 AARCH64_VALID_JUMP_TARGET
881 ld1r {v2.2s}, [x5]
882 sub x1, x1, #4
883 neg v2.8b, v2.8b // -m
884 sxtl v2.8h, v2.8b
885 shl v2.8h, v2.8h, #9 // -m << 9
886 4:
887 ld1 {v1.8h}, [x2], #16
888 ld1 {v0.d}[0], [x0]
889 ld1 {v0.d}[1], [x8]
890 subs w4, w4, #2
891 sub v1.8h, v0.8h, v1.8h // a - b
892 sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6
893 add v0.8h, v0.8h, v1.8h
894 st1 {v0.s}[0], [x0], #4
895 st1 {v0.s}[2], [x8], #4
896 st1 {v0.h}[2], [x0], x1
897 st1 {v0.h}[6], [x8], x1
898 b.gt 4b
899 ret
900 80:
901 AARCH64_VALID_JUMP_TARGET
902 ld1 {v4.8b}, [x5]
903 sub x1, x1, #8
904 neg v4.8b, v4.8b // -m
905 sxtl v4.8h, v4.8b
906 shl v4.8h, v4.8h, #9 // -m << 9
907 8:
908 ld1 {v2.8h, v3.8h}, [x2], #32
909 ld1 {v0.8h}, [x0]
910 ld1 {v1.8h}, [x8]
911 subs w4, w4, #2
912 sub v2.8h, v0.8h, v2.8h // a - b
913 sub v3.8h, v1.8h, v3.8h
914 sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6
915 sqrdmulh v3.8h, v3.8h, v4.8h
916 add v0.8h, v0.8h, v2.8h
917 add v1.8h, v1.8h, v3.8h
918 st1 {v0.d}[0], [x0], #8
919 st1 {v1.d}[0], [x8], #8
920 st1 {v0.s}[2], [x0], x1
921 st1 {v1.s}[2], [x8], x1
922 b.gt 8b
923 ret
924 160:
925 AARCH64_VALID_JUMP_TARGET
926 ld1 {v16.16b}, [x5]
927 sub x1, x1, #16
928 neg v17.16b, v16.16b // -m
929 sxtl v16.8h, v17.8b
930 sxtl2 v17.8h, v17.16b
931 shl v16.8h, v16.8h, #9 // -m << 9
932 shl v17.4h, v17.4h, #9
933 16:
934 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64
935 ld1 {v0.8h, v1.8h}, [x0]
936 subs w4, w4, #2
937 ld1 {v2.8h, v3.8h}, [x8]
938 sub v4.8h, v0.8h, v4.8h // a - b
939 sub v5.4h, v1.4h, v5.4h
940 sub v6.8h, v2.8h, v6.8h
941 sub v7.4h, v3.4h, v7.4h
942 sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6
943 sqrdmulh v5.4h, v5.4h, v17.4h
944 sqrdmulh v6.8h, v6.8h, v16.8h
945 sqrdmulh v7.4h, v7.4h, v17.4h
946 add v0.8h, v0.8h, v4.8h
947 add v1.4h, v1.4h, v5.4h
948 add v2.8h, v2.8h, v6.8h
949 add v3.4h, v3.4h, v7.4h
950 st1 {v0.8h}, [x0], #16
951 st1 {v2.8h}, [x8], #16
952 st1 {v1.4h}, [x0], x1
953 st1 {v3.4h}, [x8], x1
954 b.gt 16b
955 ret
956 320:
957 AARCH64_VALID_JUMP_TARGET
958 ld1 {v24.16b, v25.16b}, [x5]
959 neg v26.16b, v24.16b // -m
960 neg v27.8b, v25.8b
961 sxtl v24.8h, v26.8b
962 sxtl2 v25.8h, v26.16b
963 sxtl v26.8h, v27.8b
964 shl v24.8h, v24.8h, #9 // -m << 9
965 shl v25.8h, v25.8h, #9
966 shl v26.8h, v26.8h, #9
967 32:
968 ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64
969 ld1 {v0.8h, v1.8h, v2.8h}, [x0]
970 ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2], #64
971 ld1 {v4.8h, v5.8h, v6.8h}, [x8]
972 subs w4, w4, #2
973 sub v16.8h, v0.8h, v16.8h // a - b
974 sub v17.8h, v1.8h, v17.8h
975 sub v18.8h, v2.8h, v18.8h
976 sub v20.8h, v4.8h, v20.8h
977 sub v21.8h, v5.8h, v21.8h
978 sub v22.8h, v6.8h, v22.8h
979 sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6
980 sqrdmulh v17.8h, v17.8h, v25.8h
981 sqrdmulh v18.8h, v18.8h, v26.8h
982 sqrdmulh v20.8h, v20.8h, v24.8h
983 sqrdmulh v21.8h, v21.8h, v25.8h
984 sqrdmulh v22.8h, v22.8h, v26.8h
985 add v0.8h, v0.8h, v16.8h
986 add v1.8h, v1.8h, v17.8h
987 add v2.8h, v2.8h, v18.8h
988 add v4.8h, v4.8h, v20.8h
989 add v5.8h, v5.8h, v21.8h
990 add v6.8h, v6.8h, v22.8h
991 st1 {v0.8h, v1.8h, v2.8h}, [x0], x1
992 st1 {v4.8h, v5.8h, v6.8h}, [x8], x1
993 b.gt 32b
994 ret
995 L(blend_v_tbl):
996 .hword L(blend_v_tbl) - 320b
997 .hword L(blend_v_tbl) - 160b
998 .hword L(blend_v_tbl) - 80b
999 .hword L(blend_v_tbl) - 40b
1000 .hword L(blend_v_tbl) - 20b
1001 endfunc
1002
1003
1004 // This has got the same signature as the put_8tap functions,
1005 // and assumes that x9 is set to (clz(w)-24).
1006 function put_neon
1007 adr x10, L(put_tbl)
1008 ldrh w9, [x10, x9, lsl #1]
1009 sub x10, x10, w9, uxtw
1010 br x10
1011
1012 2:
1013 AARCH64_VALID_JUMP_TARGET
1014 ld1 {v0.s}[0], [x2], x3
1015 ld1 {v1.s}[0], [x2], x3
1016 subs w5, w5, #2
1017 st1 {v0.s}[0], [x0], x1
1018 st1 {v1.s}[0], [x0], x1
1019 b.gt 2b
1020 ret
1021 4:
1022 AARCH64_VALID_JUMP_TARGET
1023 ld1 {v0.4h}, [x2], x3
1024 ld1 {v1.4h}, [x2], x3
1025 subs w5, w5, #2
1026 st1 {v0.4h}, [x0], x1
1027 st1 {v1.4h}, [x0], x1
1028 b.gt 4b
1029 ret
1030 80:
1031 AARCH64_VALID_JUMP_TARGET
1032 add x8, x0, x1
1033 lsl x1, x1, #1
1034 add x9, x2, x3
1035 lsl x3, x3, #1
1036 8:
1037 ld1 {v0.8h}, [x2], x3
1038 ld1 {v1.8h}, [x9], x3
1039 subs w5, w5, #2
1040 st1 {v0.8h}, [x0], x1
1041 st1 {v1.8h}, [x8], x1
1042 b.gt 8b
1043 ret
1044 16:
1045 AARCH64_VALID_JUMP_TARGET
1046 ldp x6, x7, [x2]
1047 ldp x8, x9, [x2, #16]
1048 stp x6, x7, [x0]
1049 subs w5, w5, #1
1050 stp x8, x9, [x0, #16]
1051 add x2, x2, x3
1052 add x0, x0, x1
1053 b.gt 16b
1054 ret
1055 32:
1056 AARCH64_VALID_JUMP_TARGET
1057 ldp x6, x7, [x2]
1058 ldp x8, x9, [x2, #16]
1059 stp x6, x7, [x0]
1060 ldp x10, x11, [x2, #32]
1061 stp x8, x9, [x0, #16]
1062 subs w5, w5, #1
1063 ldp x12, x13, [x2, #48]
1064 stp x10, x11, [x0, #32]
1065 stp x12, x13, [x0, #48]
1066 add x2, x2, x3
1067 add x0, x0, x1
1068 b.gt 32b
1069 ret
1070 64:
1071 AARCH64_VALID_JUMP_TARGET
1072 ldp q0, q1, [x2]
1073 ldp q2, q3, [x2, #32]
1074 stp q0, q1, [x0]
1075 ldp q4, q5, [x2, #64]
1076 stp q2, q3, [x0, #32]
1077 ldp q6, q7, [x2, #96]
1078 subs w5, w5, #1
1079 stp q4, q5, [x0, #64]
1080 stp q6, q7, [x0, #96]
1081 add x2, x2, x3
1082 add x0, x0, x1
1083 b.gt 64b
1084 ret
1085 128:
1086 AARCH64_VALID_JUMP_TARGET
1087 ldp q0, q1, [x2]
1088 ldp q2, q3, [x2, #32]
1089 stp q0, q1, [x0]
1090 ldp q4, q5, [x2, #64]
1091 stp q2, q3, [x0, #32]
1092 ldp q6, q7, [x2, #96]
1093 subs w5, w5, #1
1094 stp q4, q5, [x0, #64]
1095 ldp q16, q17, [x2, #128]
1096 stp q6, q7, [x0, #96]
1097 ldp q18, q19, [x2, #160]
1098 stp q16, q17, [x0, #128]
1099 ldp q20, q21, [x2, #192]
1100 stp q18, q19, [x0, #160]
1101 ldp q22, q23, [x2, #224]
1102 stp q20, q21, [x0, #192]
1103 stp q22, q23, [x0, #224]
1104 add x2, x2, x3
1105 add x0, x0, x1
1106 b.gt 128b
1107 ret
1108
1109 L(put_tbl):
1110 .hword L(put_tbl) - 128b
1111 .hword L(put_tbl) - 64b
1112 .hword L(put_tbl) - 32b
1113 .hword L(put_tbl) - 16b
1114 .hword L(put_tbl) - 80b
1115 .hword L(put_tbl) - 4b
1116 .hword L(put_tbl) - 2b
1117 endfunc
1118
1119
1120 // This has got the same signature as the prep_8tap functions,
1121 // and assumes that x9 is set to (clz(w)-24), w7 to intermediate_bits and
1122 // x8 to w*2.
1123 function prep_neon
1124 adr x10, L(prep_tbl)
1125 ldrh w9, [x10, x9, lsl #1]
1126 dup v31.8h, w7 // intermediate_bits
1127 movi v30.8h, #(PREP_BIAS >> 8), lsl #8
1128 sub x10, x10, w9, uxtw
1129 br x10
1130
1131 40:
1132 AARCH64_VALID_JUMP_TARGET
1133 add x9, x1, x2
1134 lsl x2, x2, #1
1135 4:
1136 ld1 {v0.d}[0], [x1], x2
1137 ld1 {v0.d}[1], [x9], x2
1138 subs w4, w4, #2
1139 sshl v0.8h, v0.8h, v31.8h
1140 sub v0.8h, v0.8h, v30.8h
1141 st1 {v0.8h}, [x0], #16
1142 b.gt 4b
1143 ret
1144 80:
1145 AARCH64_VALID_JUMP_TARGET
1146 add x9, x1, x2
1147 lsl x2, x2, #1
1148 8:
1149 ld1 {v0.8h}, [x1], x2
1150 ld1 {v1.8h}, [x9], x2
1151 subs w4, w4, #2
1152 sshl v0.8h, v0.8h, v31.8h
1153 sshl v1.8h, v1.8h, v31.8h
1154 sub v0.8h, v0.8h, v30.8h
1155 sub v1.8h, v1.8h, v30.8h
1156 st1 {v0.8h, v1.8h}, [x0], #32
1157 b.gt 8b
1158 ret
1159 16:
1160 AARCH64_VALID_JUMP_TARGET
1161 ldp q0, q1, [x1]
1162 add x1, x1, x2
1163 sshl v0.8h, v0.8h, v31.8h
1164 ldp q2, q3, [x1]
1165 add x1, x1, x2
1166 subs w4, w4, #2
1167 sshl v1.8h, v1.8h, v31.8h
1168 sshl v2.8h, v2.8h, v31.8h
1169 sshl v3.8h, v3.8h, v31.8h
1170 sub v0.8h, v0.8h, v30.8h
1171 sub v1.8h, v1.8h, v30.8h
1172 sub v2.8h, v2.8h, v30.8h
1173 sub v3.8h, v3.8h, v30.8h
1174 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
1175 b.gt 16b
1176 ret
1177 32:
1178 AARCH64_VALID_JUMP_TARGET
1179 ldp q0, q1, [x1]
1180 sshl v0.8h, v0.8h, v31.8h
1181 ldp q2, q3, [x1, #32]
1182 add x1, x1, x2
1183 sshl v1.8h, v1.8h, v31.8h
1184 sshl v2.8h, v2.8h, v31.8h
1185 sshl v3.8h, v3.8h, v31.8h
1186 subs w4, w4, #1
1187 sub v0.8h, v0.8h, v30.8h
1188 sub v1.8h, v1.8h, v30.8h
1189 sub v2.8h, v2.8h, v30.8h
1190 sub v3.8h, v3.8h, v30.8h
1191 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
1192 b.gt 32b
1193 ret
1194 64:
1195 AARCH64_VALID_JUMP_TARGET
1196 ldp q0, q1, [x1]
1197 subs w4, w4, #1
1198 sshl v0.8h, v0.8h, v31.8h
1199 ldp q2, q3, [x1, #32]
1200 sshl v1.8h, v1.8h, v31.8h
1201 ldp q4, q5, [x1, #64]
1202 sshl v2.8h, v2.8h, v31.8h
1203 sshl v3.8h, v3.8h, v31.8h
1204 ldp q6, q7, [x1, #96]
1205 add x1, x1, x2
1206 sshl v4.8h, v4.8h, v31.8h
1207 sshl v5.8h, v5.8h, v31.8h
1208 sshl v6.8h, v6.8h, v31.8h
1209 sshl v7.8h, v7.8h, v31.8h
1210 sub v0.8h, v0.8h, v30.8h
1211 sub v1.8h, v1.8h, v30.8h
1212 sub v2.8h, v2.8h, v30.8h
1213 sub v3.8h, v3.8h, v30.8h
1214 stp q0, q1, [x0]
1215 sub v4.8h, v4.8h, v30.8h
1216 sub v5.8h, v5.8h, v30.8h
1217 stp q2, q3, [x0, #32]
1218 sub v6.8h, v6.8h, v30.8h
1219 sub v7.8h, v7.8h, v30.8h
1220 stp q4, q5, [x0, #64]
1221 stp q6, q7, [x0, #96]
1222 add x0, x0, x8
1223 b.gt 64b
1224 ret
1225 128:
1226 AARCH64_VALID_JUMP_TARGET
1227 ldp q0, q1, [x1]
1228 subs w4, w4, #1
1229 sshl v0.8h, v0.8h, v31.8h
1230 ldp q2, q3, [x1, #32]
1231 sshl v1.8h, v1.8h, v31.8h
1232 ldp q4, q5, [x1, #64]
1233 sshl v2.8h, v2.8h, v31.8h
1234 sshl v3.8h, v3.8h, v31.8h
1235 ldp q6, q7, [x1, #96]
1236 sshl v4.8h, v4.8h, v31.8h
1237 sshl v5.8h, v5.8h, v31.8h
1238 ldp q16, q17, [x1, #128]
1239 sshl v6.8h, v6.8h, v31.8h
1240 sshl v7.8h, v7.8h, v31.8h
1241 ldp q18, q19, [x1, #160]
1242 sshl v16.8h, v16.8h, v31.8h
1243 sshl v17.8h, v17.8h, v31.8h
1244 ldp q20, q21, [x1, #192]
1245 sshl v18.8h, v18.8h, v31.8h
1246 sshl v19.8h, v19.8h, v31.8h
1247 ldp q22, q23, [x1, #224]
1248 add x1, x1, x2
1249 sshl v20.8h, v20.8h, v31.8h
1250 sshl v21.8h, v21.8h, v31.8h
1251 sshl v22.8h, v22.8h, v31.8h
1252 sshl v23.8h, v23.8h, v31.8h
1253 sub v0.8h, v0.8h, v30.8h
1254 sub v1.8h, v1.8h, v30.8h
1255 sub v2.8h, v2.8h, v30.8h
1256 sub v3.8h, v3.8h, v30.8h
1257 stp q0, q1, [x0]
1258 sub v4.8h, v4.8h, v30.8h
1259 sub v5.8h, v5.8h, v30.8h
1260 stp q2, q3, [x0, #32]
1261 sub v6.8h, v6.8h, v30.8h
1262 sub v7.8h, v7.8h, v30.8h
1263 stp q4, q5, [x0, #64]
1264 sub v16.8h, v16.8h, v30.8h
1265 sub v17.8h, v17.8h, v30.8h
1266 stp q6, q7, [x0, #96]
1267 sub v18.8h, v18.8h, v30.8h
1268 sub v19.8h, v19.8h, v30.8h
1269 stp q16, q17, [x0, #128]
1270 sub v20.8h, v20.8h, v30.8h
1271 sub v21.8h, v21.8h, v30.8h
1272 stp q18, q19, [x0, #160]
1273 sub v22.8h, v22.8h, v30.8h
1274 sub v23.8h, v23.8h, v30.8h
1275 stp q20, q21, [x0, #192]
1276 stp q22, q23, [x0, #224]
1277 add x0, x0, x8
1278 b.gt 128b
1279 ret
1280
1281 L(prep_tbl):
1282 .hword L(prep_tbl) - 128b
1283 .hword L(prep_tbl) - 64b
1284 .hword L(prep_tbl) - 32b
1285 .hword L(prep_tbl) - 16b
1286 .hword L(prep_tbl) - 80b
1287 .hword L(prep_tbl) - 40b
1288 endfunc
1289
1290
1291 .macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6
1292 ld1 {\d0\wd}[0], [\s0], \strd
1293 ld1 {\d1\wd}[0], [\s1], \strd
1294 .ifnb \d2
1295 ld1 {\d2\wd}[0], [\s0], \strd
1296 ld1 {\d3\wd}[0], [\s1], \strd
1297 .endif
1298 .ifnb \d4
1299 ld1 {\d4\wd}[0], [\s0], \strd
1300 .endif
1301 .ifnb \d5
1302 ld1 {\d5\wd}[0], [\s1], \strd
1303 .endif
1304 .ifnb \d6
1305 ld1 {\d6\wd}[0], [\s0], \strd
1306 .endif
1307 .endm
1308 .macro load_reg s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6
1309 ld1 {\d0\wd}, [\s0], \strd
1310 ld1 {\d1\wd}, [\s1], \strd
1311 .ifnb \d2
1312 ld1 {\d2\wd}, [\s0], \strd
1313 ld1 {\d3\wd}, [\s1], \strd
1314 .endif
1315 .ifnb \d4
1316 ld1 {\d4\wd}, [\s0], \strd
1317 .endif
1318 .ifnb \d5
1319 ld1 {\d5\wd}, [\s1], \strd
1320 .endif
1321 .ifnb \d6
1322 ld1 {\d6\wd}, [\s0], \strd
1323 .endif
1324 .endm
1325 .macro load_regpair s0, s1, strd, wd, d0, d1, d2, d3, d4, d5
1326 ld1 {\d0\wd, \d1\wd}, [\s0], \strd
1327 .ifnb \d2
1328 ld1 {\d2\wd, \d3\wd}, [\s1], \strd
1329 .endif
1330 .ifnb \d4
1331 ld1 {\d4\wd, \d5\wd}, [\s0], \strd
1332 .endif
1333 .endm
1334 .macro load_s s0, s1, strd, d0, d1, d2, d3, d4, d5, d6
1335 load_slice \s0, \s1, \strd, .s, \d0, \d1, \d2, \d3, \d4, \d5, \d6
1336 .endm
1337 .macro load_4h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6
1338 load_reg \s0, \s1, \strd, .4h, \d0, \d1, \d2, \d3, \d4, \d5, \d6
1339 .endm
1340 .macro load_8h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6
1341 load_reg \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5, \d6
1342 .endm
1343 .macro load_16h s0, s1, strd, d0, d1, d2, d3, d4, d5
1344 load_regpair \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5
1345 .endm
1346 .macro interleave_1 wd, r0, r1, r2, r3, r4
1347 trn1 \r0\wd, \r0\wd, \r1\wd
1348 trn1 \r1\wd, \r1\wd, \r2\wd
1349 .ifnb \r3
1350 trn1 \r2\wd, \r2\wd, \r3\wd
1351 trn1 \r3\wd, \r3\wd, \r4\wd
1352 .endif
1353 .endm
1354 .macro interleave_1_s r0, r1, r2, r3, r4
1355 interleave_1 .2s, \r0, \r1, \r2, \r3, \r4
1356 .endm
1357 .macro umin_h c, wd, r0, r1, r2, r3
1358 umin \r0\wd, \r0\wd, \c\wd
1359 .ifnb \r1
1360 umin \r1\wd, \r1\wd, \c\wd
1361 .endif
1362 .ifnb \r2
1363 umin \r2\wd, \r2\wd, \c\wd
1364 umin \r3\wd, \r3\wd, \c\wd
1365 .endif
1366 .endm
1367 .macro sub_h c, wd, r0, r1, r2, r3
1368 sub \r0\wd, \r0\wd, \c\wd
1369 .ifnb \r1
1370 sub \r1\wd, \r1\wd, \c\wd
1371 .endif
1372 .ifnb \r2
1373 sub \r2\wd, \r2\wd, \c\wd
1374 sub \r3\wd, \r3\wd, \c\wd
1375 .endif
1376 .endm
1377 .macro smull_smlal_4 d, s0, s1, s2, s3
1378 smull \d\().4s, \s0\().4h, v0.h[0]
1379 smlal \d\().4s, \s1\().4h, v0.h[1]
1380 smlal \d\().4s, \s2\().4h, v0.h[2]
1381 smlal \d\().4s, \s3\().4h, v0.h[3]
1382 .endm
1383 .macro smull2_smlal2_4 d, s0, s1, s2, s3
1384 smull2 \d\().4s, \s0\().8h, v0.h[0]
1385 smlal2 \d\().4s, \s1\().8h, v0.h[1]
1386 smlal2 \d\().4s, \s2\().8h, v0.h[2]
1387 smlal2 \d\().4s, \s3\().8h, v0.h[3]
1388 .endm
1389 .macro smull_smlal_8 d, s0, s1, s2, s3, s4, s5, s6, s7
1390 smull \d\().4s, \s0\().4h, v0.h[0]
1391 smlal \d\().4s, \s1\().4h, v0.h[1]
1392 smlal \d\().4s, \s2\().4h, v0.h[2]
1393 smlal \d\().4s, \s3\().4h, v0.h[3]
1394 smlal \d\().4s, \s4\().4h, v0.h[4]
1395 smlal \d\().4s, \s5\().4h, v0.h[5]
1396 smlal \d\().4s, \s6\().4h, v0.h[6]
1397 smlal \d\().4s, \s7\().4h, v0.h[7]
1398 .endm
1399 .macro smull2_smlal2_8 d, s0, s1, s2, s3, s4, s5, s6, s7
1400 smull2 \d\().4s, \s0\().8h, v0.h[0]
1401 smlal2 \d\().4s, \s1\().8h, v0.h[1]
1402 smlal2 \d\().4s, \s2\().8h, v0.h[2]
1403 smlal2 \d\().4s, \s3\().8h, v0.h[3]
1404 smlal2 \d\().4s, \s4\().8h, v0.h[4]
1405 smlal2 \d\().4s, \s5\().8h, v0.h[5]
1406 smlal2 \d\().4s, \s6\().8h, v0.h[6]
1407 smlal2 \d\().4s, \s7\().8h, v0.h[7]
1408 .endm
1409 .macro sqrshrun_h shift, r0, r1, r2, r3
1410 sqrshrun \r0\().4h, \r0\().4s, #\shift
1411 .ifnb \r1
1412 sqrshrun2 \r0\().8h, \r1\().4s, #\shift
1413 .endif
1414 .ifnb \r2
1415 sqrshrun \r2\().4h, \r2\().4s, #\shift
1416 sqrshrun2 \r2\().8h, \r3\().4s, #\shift
1417 .endif
1418 .endm
1419 .macro xtn_h r0, r1, r2, r3
1420 uzp1 \r0\().8h, \r0\().8h, \r1\().8h // Same as xtn, xtn2
1421 .ifnb \r2
1422 uzp1 \r2\().8h, \r2\().8h, \r3\().8h // Ditto
1423 .endif
1424 .endm
1425 .macro srshl_s shift, r0, r1, r2, r3
1426 srshl \r0\().4s, \r0\().4s, \shift\().4s
1427 srshl \r1\().4s, \r1\().4s, \shift\().4s
1428 .ifnb \r2
1429 srshl \r2\().4s, \r2\().4s, \shift\().4s
1430 srshl \r3\().4s, \r3\().4s, \shift\().4s
1431 .endif
1432 .endm
1433 .macro st_s strd, reg, lanes
1434 st1 {\reg\().s}[0], [x0], \strd
1435 st1 {\reg\().s}[1], [x9], \strd
1436 .if \lanes > 2
1437 st1 {\reg\().s}[2], [x0], \strd
1438 st1 {\reg\().s}[3], [x9], \strd
1439 .endif
1440 .endm
1441 .macro st_d strd, r0, r1
1442 st1 {\r0\().d}[0], [x0], \strd
1443 st1 {\r0\().d}[1], [x9], \strd
1444 .ifnb \r1
1445 st1 {\r1\().d}[0], [x0], \strd
1446 st1 {\r1\().d}[1], [x9], \strd
1447 .endif
1448 .endm
1449 .macro shift_store_4 type, strd, r0, r1, r2, r3
1450 .ifc \type, put
1451 sqrshrun_h 6, \r0, \r1, \r2, \r3
1452 umin_h v31, .8h, \r0, \r2
1453 .else
1454 srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits)
1455 xtn_h \r0, \r1, \r2, \r3
1456 sub_h v29, .8h, \r0, \r2 // PREP_BIAS
1457 .endif
1458 st_d \strd, \r0, \r2
1459 .endm
1460 .macro st_reg strd, wd, r0, r1, r2, r3, r4, r5, r6, r7
1461 st1 {\r0\wd}, [x0], \strd
1462 st1 {\r1\wd}, [x9], \strd
1463 .ifnb \r2
1464 st1 {\r2\wd}, [x0], \strd
1465 st1 {\r3\wd}, [x9], \strd
1466 .endif
1467 .ifnb \r4
1468 st1 {\r4\wd}, [x0], \strd
1469 st1 {\r5\wd}, [x9], \strd
1470 st1 {\r6\wd}, [x0], \strd
1471 st1 {\r7\wd}, [x9], \strd
1472 .endif
1473 .endm
1474 .macro st_8h strd, r0, r1, r2, r3, r4, r5, r6, r7
1475 st_reg \strd, .8h, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7
1476 .endm
1477 .macro shift_store_8 type, strd, r0, r1, r2, r3
1478 .ifc \type, put
1479 sqrshrun_h 6, \r0, \r1, \r2, \r3
1480 umin_h v31, .8h, \r0, \r2
1481 .else
1482 srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits)
1483 xtn_h \r0, \r1, \r2, \r3
1484 sub_h v29, .8h, \r0, \r2 // PREP_BIAS
1485 .endif
1486 st_8h \strd, \r0, \r2
1487 .endm
1488 .macro shift_store_16 type, strd, dst, r0, r1, r2, r3
1489 .ifc \type, put
1490 sqrshrun_h 6, \r0, \r1, \r2, \r3
1491 umin \r0\().8h, \r0\().8h, v31.8h
1492 umin \r1\().8h, \r2\().8h, v31.8h
1493 .else
1494 srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits)
1495 xtn_h \r0, \r1, \r2, \r3
1496 sub \r0\().8h, \r0\().8h, v29.8h
1497 sub \r1\().8h, \r2\().8h, v29.8h
1498 .endif
1499 st1 {\r0\().8h, \r1\().8h}, [\dst], \strd
1500 .endm
1501
1502 .macro make_8tap_fn op, type, type_h, type_v
1503 function \op\()_8tap_\type\()_16bpc_neon, export=1
1504 mov w9, \type_h
1505 mov w10, \type_v
1506 b \op\()_8tap_neon
1507 endfunc
1508 .endm
1509
1510 // No spaces in these expressions, due to gas-preprocessor.
1511 #define REGULAR ((0*15<<7)|3*15)
1512 #define SMOOTH ((1*15<<7)|4*15)
1513 #define SHARP ((2*15<<7)|3*15)
1514
1515 .macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, xmx, my, xmy, bdmax, ds2, sr2
1516 make_8tap_fn \type, regular, REGULAR, REGULAR
1517 make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH
1518 make_8tap_fn \type, regular_sharp, REGULAR, SHARP
1519 make_8tap_fn \type, smooth, SMOOTH, SMOOTH
1520 make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR
1521 make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP
1522 make_8tap_fn \type, sharp, SHARP, SHARP
1523 make_8tap_fn \type, sharp_regular, SHARP, REGULAR
1524 make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH
1525
1526 function \type\()_8tap_neon
1527 .ifc \bdmax, w8
1528 ldr w8, [sp]
1529 .endif
1530 mov w11, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0)
1531 mul \mx, \mx, w11
1532 mul \my, \my, w11
1533 add \mx, \mx, w9 // mx, 8tap_h, 4tap_h
1534 add \my, \my, w10 // my, 8tap_v, 4tap_v
1535 .ifc \type, prep
1536 uxtw \d_strd, \w
1537 lsl \d_strd, \d_strd, #1
1538 .endif
1539
1540 dup v31.8h, \bdmax // bitdepth_max
1541 clz \bdmax, \bdmax
1542 clz w9, \w
1543 sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18
1544 mov w12, #6
1545 tst \mx, #(0x7f << 14)
1546 sub w9, w9, #24
1547 add w13, w12, \bdmax // 6 + intermediate_bits
1548 sub w12, w12, \bdmax // 6 - intermediate_bits
1549 movrel x11, X(mc_subpel_filters), -8
1550 b.ne L(\type\()_8tap_h)
1551 tst \my, #(0x7f << 14)
1552 b.ne L(\type\()_8tap_v)
1553 b \type\()_neon
1554
1555 L(\type\()_8tap_h):
1556 cmp \w, #4
1557 ubfx w10, \mx, #7, #7
1558 and \mx, \mx, #0x7f
1559 b.le 4f
1560 mov \mx, w10
1561 4:
1562 tst \my, #(0x7f << 14)
1563 add \xmx, x11, \mx, uxtw #3
1564 b.ne L(\type\()_8tap_hv)
1565
1566 adr x10, L(\type\()_8tap_h_tbl)
1567 dup v30.4s, w12 // 6 - intermediate_bits
1568 ldrh w9, [x10, x9, lsl #1]
1569 neg v30.4s, v30.4s // -(6-intermediate_bits)
1570 .ifc \type, put
1571 dup v29.8h, \bdmax // intermediate_bits
1572 .else
1573 movi v28.8h, #(PREP_BIAS >> 8), lsl #8
1574 .endif
1575 sub x10, x10, w9, uxtw
1576 .ifc \type, put
1577 neg v29.8h, v29.8h // -intermediate_bits
1578 .endif
1579 br x10
1580
1581 20: // 2xN h
1582 AARCH64_VALID_JUMP_TARGET
1583 .ifc \type, put
1584 add \xmx, \xmx, #2
1585 ld1 {v0.s}[0], [\xmx]
1586 sub \src, \src, #2
1587 add \ds2, \dst, \d_strd
1588 add \sr2, \src, \s_strd
1589 lsl \d_strd, \d_strd, #1
1590 lsl \s_strd, \s_strd, #1
1591 sxtl v0.8h, v0.8b
1592 2:
1593 ld1 {v4.8h}, [\src], \s_strd
1594 ld1 {v6.8h}, [\sr2], \s_strd
1595 ext v5.16b, v4.16b, v4.16b, #2
1596 ext v7.16b, v6.16b, v6.16b, #2
1597 subs \h, \h, #2
1598 trn1 v3.2s, v4.2s, v6.2s
1599 trn2 v6.2s, v4.2s, v6.2s
1600 trn1 v4.2s, v5.2s, v7.2s
1601 trn2 v7.2s, v5.2s, v7.2s
1602 smull v3.4s, v3.4h, v0.h[0]
1603 smlal v3.4s, v4.4h, v0.h[1]
1604 smlal v3.4s, v6.4h, v0.h[2]
1605 smlal v3.4s, v7.4h, v0.h[3]
1606 srshl v3.4s, v3.4s, v30.4s // -(6-intermediate_bits)
1607 sqxtun v3.4h, v3.4s
1608 srshl v3.4h, v3.4h, v29.4h // -intermediate_bits
1609 umin v3.4h, v3.4h, v31.4h
1610 st1 {v3.s}[0], [\dst], \d_strd
1611 st1 {v3.s}[1], [\ds2], \d_strd
1612 b.gt 2b
1613 ret
1614 .endif
1615
1616 40: // 4xN h
1617 AARCH64_VALID_JUMP_TARGET
1618 add \xmx, \xmx, #2
1619 ld1 {v0.s}[0], [\xmx]
1620 sub \src, \src, #2
1621 add \ds2, \dst, \d_strd
1622 add \sr2, \src, \s_strd
1623 lsl \d_strd, \d_strd, #1
1624 lsl \s_strd, \s_strd, #1
1625 sxtl v0.8h, v0.8b
1626 4:
1627 ld1 {v16.8h}, [\src], \s_strd
1628 ld1 {v20.8h}, [\sr2], \s_strd
1629 ext v17.16b, v16.16b, v16.16b, #2
1630 ext v18.16b, v16.16b, v16.16b, #4
1631 ext v19.16b, v16.16b, v16.16b, #6
1632 ext v21.16b, v20.16b, v20.16b, #2
1633 ext v22.16b, v20.16b, v20.16b, #4
1634 ext v23.16b, v20.16b, v20.16b, #6
1635 subs \h, \h, #2
1636 smull v16.4s, v16.4h, v0.h[0]
1637 smlal v16.4s, v17.4h, v0.h[1]
1638 smlal v16.4s, v18.4h, v0.h[2]
1639 smlal v16.4s, v19.4h, v0.h[3]
1640 smull v20.4s, v20.4h, v0.h[0]
1641 smlal v20.4s, v21.4h, v0.h[1]
1642 smlal v20.4s, v22.4h, v0.h[2]
1643 smlal v20.4s, v23.4h, v0.h[3]
1644 srshl v16.4s, v16.4s, v30.4s // -(6-intermediate_bits)
1645 srshl v20.4s, v20.4s, v30.4s // -(6-intermediate_bits)
1646 .ifc \type, put
1647 sqxtun v16.4h, v16.4s
1648 sqxtun2 v16.8h, v20.4s
1649 srshl v16.8h, v16.8h, v29.8h // -intermediate_bits
1650 umin v16.8h, v16.8h, v31.8h
1651 .else
1652 uzp1 v16.8h, v16.8h, v20.8h // Same as xtn, xtn2
1653 sub v16.8h, v16.8h, v28.8h // PREP_BIAS
1654 .endif
1655 st1 {v16.d}[0], [\dst], \d_strd
1656 st1 {v16.d}[1], [\ds2], \d_strd
1657 b.gt 4b
1658 ret
1659
1660 80:
1661 160:
1662 320:
1663 640:
1664 1280: // 8xN, 16xN, 32xN, ... h
1665 AARCH64_VALID_JUMP_TARGET
1666 ld1 {v0.8b}, [\xmx]
1667 sub \src, \src, #6
1668 add \ds2, \dst, \d_strd
1669 add \sr2, \src, \s_strd
1670 lsl \s_strd, \s_strd, #1
1671 sxtl v0.8h, v0.8b
1672
1673 sub \s_strd, \s_strd, \w, uxtw #1
1674 sub \s_strd, \s_strd, #16
1675 .ifc \type, put
1676 lsl \d_strd, \d_strd, #1
1677 sub \d_strd, \d_strd, \w, uxtw #1
1678 .endif
1679 81:
1680 ld1 {v16.8h, v17.8h}, [\src], #32
1681 ld1 {v20.8h, v21.8h}, [\sr2], #32
1682 mov \mx, \w
1683
1684 8:
1685 smull v18.4s, v16.4h, v0.h[0]
1686 smull2 v19.4s, v16.8h, v0.h[0]
1687 smull v22.4s, v20.4h, v0.h[0]
1688 smull2 v23.4s, v20.8h, v0.h[0]
1689 .irpc i, 1234567
1690 ext v24.16b, v16.16b, v17.16b, #(2*\i)
1691 ext v25.16b, v20.16b, v21.16b, #(2*\i)
1692 smlal v18.4s, v24.4h, v0.h[\i]
1693 smlal2 v19.4s, v24.8h, v0.h[\i]
1694 smlal v22.4s, v25.4h, v0.h[\i]
1695 smlal2 v23.4s, v25.8h, v0.h[\i]
1696 .endr
1697 subs \mx, \mx, #8
1698 srshl v18.4s, v18.4s, v30.4s // -(6-intermediate_bits)
1699 srshl v19.4s, v19.4s, v30.4s // -(6-intermediate_bits)
1700 srshl v22.4s, v22.4s, v30.4s // -(6-intermediate_bits)
1701 srshl v23.4s, v23.4s, v30.4s // -(6-intermediate_bits)
1702 .ifc \type, put
1703 sqxtun v18.4h, v18.4s
1704 sqxtun2 v18.8h, v19.4s
1705 sqxtun v22.4h, v22.4s
1706 sqxtun2 v22.8h, v23.4s
1707 srshl v18.8h, v18.8h, v29.8h // -intermediate_bits
1708 srshl v22.8h, v22.8h, v29.8h // -intermediate_bits
1709 umin v18.8h, v18.8h, v31.8h
1710 umin v22.8h, v22.8h, v31.8h
1711 .else
1712 uzp1 v18.8h, v18.8h, v19.8h // Same as xtn, xtn2
1713 uzp1 v22.8h, v22.8h, v23.8h // Ditto
1714 sub v18.8h, v18.8h, v28.8h // PREP_BIAS
1715 sub v22.8h, v22.8h, v28.8h // PREP_BIAS
1716 .endif
1717 st1 {v18.8h}, [\dst], #16
1718 st1 {v22.8h}, [\ds2], #16
1719 b.le 9f
1720
1721 mov v16.16b, v17.16b
1722 mov v20.16b, v21.16b
1723 ld1 {v17.8h}, [\src], #16
1724 ld1 {v21.8h}, [\sr2], #16
1725 b 8b
1726
1727 9:
1728 add \dst, \dst, \d_strd
1729 add \ds2, \ds2, \d_strd
1730 add \src, \src, \s_strd
1731 add \sr2, \sr2, \s_strd
1732
1733 subs \h, \h, #2
1734 b.gt 81b
1735 ret
1736
1737 L(\type\()_8tap_h_tbl):
1738 .hword L(\type\()_8tap_h_tbl) - 1280b
1739 .hword L(\type\()_8tap_h_tbl) - 640b
1740 .hword L(\type\()_8tap_h_tbl) - 320b
1741 .hword L(\type\()_8tap_h_tbl) - 160b
1742 .hword L(\type\()_8tap_h_tbl) - 80b
1743 .hword L(\type\()_8tap_h_tbl) - 40b
1744 .hword L(\type\()_8tap_h_tbl) - 20b
1745 .hword 0
1746
1747
1748 L(\type\()_8tap_v):
1749 cmp \h, #4
1750 ubfx w10, \my, #7, #7
1751 and \my, \my, #0x7f
1752 b.le 4f
1753 mov \my, w10
1754 4:
1755 add \xmy, x11, \my, uxtw #3
1756
1757 .ifc \type, prep
1758 dup v30.4s, w12 // 6 - intermediate_bits
1759 movi v29.8h, #(PREP_BIAS >> 8), lsl #8
1760 .endif
1761 adr x10, L(\type\()_8tap_v_tbl)
1762 ldrh w9, [x10, x9, lsl #1]
1763 .ifc \type, prep
1764 neg v30.4s, v30.4s // -(6-intermediate_bits)
1765 .endif
1766 sub x10, x10, w9, uxtw
1767 br x10
1768
1769 20: // 2xN v
1770 AARCH64_VALID_JUMP_TARGET
1771 .ifc \type, put
1772 b.gt 28f
1773
1774 cmp \h, #2
1775 add \xmy, \xmy, #2
1776 ld1 {v0.s}[0], [\xmy]
1777 sub \src, \src, \s_strd
1778 add \ds2, \dst, \d_strd
1779 add \sr2, \src, \s_strd
1780 lsl \s_strd, \s_strd, #1
1781 lsl \d_strd, \d_strd, #1
1782 sxtl v0.8h, v0.8b
1783
1784 // 2x2 v
1785 load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5
1786 interleave_1_s v1, v2, v3, v4, v5
1787 b.gt 24f
1788 smull_smlal_4 v6, v1, v2, v3, v4
1789 sqrshrun_h 6, v6
1790 umin_h v31, .8h, v6
1791 st_s \d_strd, v6, 2
1792 ret
1793
1794 24: // 2x4 v
1795 load_s \sr2, \src, \s_strd, v6, v7
1796 interleave_1_s v5, v6, v7
1797 smull_smlal_4 v16, v1, v2, v3, v4
1798 smull_smlal_4 v17, v3, v4, v5, v6
1799 sqrshrun_h 6, v16, v17
1800 umin_h v31, .8h, v16
1801 st_s \d_strd, v16, 4
1802 ret
1803
1804 28: // 2x6, 2x8, 2x12, 2x16 v
1805 ld1 {v0.8b}, [\xmy]
1806 sub \sr2, \src, \s_strd, lsl #1
1807 add \ds2, \dst, \d_strd
1808 sub \src, \sr2, \s_strd
1809 lsl \d_strd, \d_strd, #1
1810 lsl \s_strd, \s_strd, #1
1811 sxtl v0.8h, v0.8b
1812
1813 load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5, v6, v7
1814 interleave_1_s v1, v2, v3, v4, v5
1815 interleave_1_s v5, v6, v7
1816 216:
1817 subs \h, \h, #4
1818 load_s \sr2, \src, \s_strd, v16, v17, v18, v19
1819 interleave_1_s v7, v16, v17, v18, v19
1820 smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16
1821 smull_smlal_8 v25, v3, v4, v5, v6, v7, v16, v17, v18
1822 sqrshrun_h 6, v24, v25
1823 umin_h v31, .8h, v24
1824 st_s \d_strd, v24, 4
1825 b.le 0f
1826 cmp \h, #2
1827 mov v1.16b, v5.16b
1828 mov v2.16b, v6.16b
1829 mov v3.16b, v7.16b
1830 mov v4.16b, v16.16b
1831 mov v5.16b, v17.16b
1832 mov v6.16b, v18.16b
1833 mov v7.16b, v19.16b
1834 b.eq 26f
1835 b 216b
1836 26:
1837 load_s \sr2, \src, \s_strd, v16, v17
1838 interleave_1_s v7, v16, v17
1839 smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16
1840 sqrshrun_h 6, v24
1841 umin_h v31, .4h, v24
1842 st_s \d_strd, v24, 2
1843 0:
1844 ret
1845 .endif
1846
1847 40:
1848 AARCH64_VALID_JUMP_TARGET
1849 b.gt 480f
1850
1851 // 4x2, 4x4 v
1852 cmp \h, #2
1853 add \xmy, \xmy, #2
1854 ld1 {v0.s}[0], [\xmy]
1855 sub \src, \src, \s_strd
1856 add \ds2, \dst, \d_strd
1857 add \sr2, \src, \s_strd
1858 lsl \s_strd, \s_strd, #1
1859 lsl \d_strd, \d_strd, #1
1860 sxtl v0.8h, v0.8b
1861
1862 load_4h \src, \sr2, \s_strd, v1, v2, v3, v4, v5
1863 smull_smlal_4 v6, v1, v2, v3, v4
1864 smull_smlal_4 v7, v2, v3, v4, v5
1865 shift_store_4 \type, \d_strd, v6, v7
1866 b.le 0f
1867 load_4h \sr2, \src, \s_strd, v6, v7
1868 smull_smlal_4 v1, v3, v4, v5, v6
1869 smull_smlal_4 v2, v4, v5, v6, v7
1870 shift_store_4 \type, \d_strd, v1, v2
1871 0:
1872 ret
1873
1874 480: // 4x6, 4x8, 4x12, 4x16 v
1875 ld1 {v0.8b}, [\xmy]
1876 sub \sr2, \src, \s_strd, lsl #1
1877 add \ds2, \dst, \d_strd
1878 sub \src, \sr2, \s_strd
1879 lsl \s_strd, \s_strd, #1
1880 lsl \d_strd, \d_strd, #1
1881 sxtl v0.8h, v0.8b
1882
1883 load_4h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22
1884
1885 48:
1886 subs \h, \h, #4
1887 load_4h \sr2, \src, \s_strd, v23, v24, v25, v26
1888 smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23
1889 smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24
1890 smull_smlal_8 v3, v18, v19, v20, v21, v22, v23, v24, v25
1891 smull_smlal_8 v4, v19, v20, v21, v22, v23, v24, v25, v26
1892 shift_store_4 \type, \d_strd, v1, v2, v3, v4
1893 b.le 0f
1894 cmp \h, #2
1895 mov v16.8b, v20.8b
1896 mov v17.8b, v21.8b
1897 mov v18.8b, v22.8b
1898 mov v19.8b, v23.8b
1899 mov v20.8b, v24.8b
1900 mov v21.8b, v25.8b
1901 mov v22.8b, v26.8b
1902 b.eq 46f
1903 b 48b
1904 46:
1905 load_4h \sr2, \src, \s_strd, v23, v24
1906 smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23
1907 smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24
1908 shift_store_4 \type, \d_strd, v1, v2
1909 0:
1910 ret
1911
1912 80:
1913 AARCH64_VALID_JUMP_TARGET
1914 b.gt 880f
1915
1916 // 8x2, 8x4 v
1917 cmp \h, #2
1918 add \xmy, \xmy, #2
1919 ld1 {v0.s}[0], [\xmy]
1920 sub \src, \src, \s_strd
1921 add \ds2, \dst, \d_strd
1922 add \sr2, \src, \s_strd
1923 lsl \s_strd, \s_strd, #1
1924 lsl \d_strd, \d_strd, #1
1925 sxtl v0.8h, v0.8b
1926
1927 load_8h \src, \sr2, \s_strd, v1, v2, v3, v4, v5
1928 smull_smlal_4 v16, v1, v2, v3, v4
1929 smull2_smlal2_4 v17, v1, v2, v3, v4
1930 smull_smlal_4 v18, v2, v3, v4, v5
1931 smull2_smlal2_4 v19, v2, v3, v4, v5
1932 shift_store_8 \type, \d_strd, v16, v17, v18, v19
1933 b.le 0f
1934 load_8h \sr2, \src, \s_strd, v6, v7
1935 smull_smlal_4 v16, v3, v4, v5, v6
1936 smull2_smlal2_4 v17, v3, v4, v5, v6
1937 smull_smlal_4 v18, v4, v5, v6, v7
1938 smull2_smlal2_4 v19, v4, v5, v6, v7
1939 shift_store_8 \type, \d_strd, v16, v17, v18, v19
1940 0:
1941 ret
1942
1943 880: // 8x6, 8x8, 8x16, 8x32 v
1944 1680: // 16x8, 16x16, ...
1945 320: // 32x8, 32x16, ...
1946 640:
1947 1280:
1948 AARCH64_VALID_JUMP_TARGET
1949 ld1 {v0.8b}, [\xmy]
1950 sub \src, \src, \s_strd
1951 sub \src, \src, \s_strd, lsl #1
1952 sxtl v0.8h, v0.8b
1953 mov \my, \h
1954 168:
1955 add \ds2, \dst, \d_strd
1956 add \sr2, \src, \s_strd
1957 lsl \s_strd, \s_strd, #1
1958 lsl \d_strd, \d_strd, #1
1959
1960 load_8h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22
1961
1962 88:
1963 subs \h, \h, #2
1964 load_8h \sr2, \src, \s_strd, v23, v24
1965 smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23
1966 smull2_smlal2_8 v2, v16, v17, v18, v19, v20, v21, v22, v23
1967 smull_smlal_8 v3, v17, v18, v19, v20, v21, v22, v23, v24
1968 smull2_smlal2_8 v4, v17, v18, v19, v20, v21, v22, v23, v24
1969 shift_store_8 \type, \d_strd, v1, v2, v3, v4
1970 b.le 9f
1971 subs \h, \h, #2
1972 load_8h \sr2, \src, \s_strd, v25, v26
1973 smull_smlal_8 v1, v18, v19, v20, v21, v22, v23, v24, v25
1974 smull2_smlal2_8 v2, v18, v19, v20, v21, v22, v23, v24, v25
1975 smull_smlal_8 v3, v19, v20, v21, v22, v23, v24, v25, v26
1976 smull2_smlal2_8 v4, v19, v20, v21, v22, v23, v24, v25, v26
1977 shift_store_8 \type, \d_strd, v1, v2, v3, v4
1978 b.le 9f
1979 mov v16.16b, v20.16b
1980 mov v17.16b, v21.16b
1981 mov v18.16b, v22.16b
1982 mov v19.16b, v23.16b
1983 mov v20.16b, v24.16b
1984 mov v21.16b, v25.16b
1985 mov v22.16b, v26.16b
1986 b 88b
1987 9:
1988 subs \w, \w, #8
1989 b.le 0f
1990 asr \s_strd, \s_strd, #1
1991 asr \d_strd, \d_strd, #1
1992 msub \src, \s_strd, \xmy, \src
1993 msub \dst, \d_strd, \xmy, \dst
1994 sub \src, \src, \s_strd, lsl #3
1995 mov \h, \my
1996 add \src, \src, #16
1997 add \dst, \dst, #16
1998 b 168b
1999 0:
2000 ret
2001
2002 160:
2003 AARCH64_VALID_JUMP_TARGET
2004 b.gt 1680b
2005
2006 // 16x2, 16x4 v
2007 add \xmy, \xmy, #2
2008 ld1 {v0.s}[0], [\xmy]
2009 sub \src, \src, \s_strd
2010 sxtl v0.8h, v0.8b
2011
2012 load_16h \src, \src, \s_strd, v16, v17, v18, v19, v20, v21
2013 16:
2014 load_16h \src, \src, \s_strd, v22, v23
2015 subs \h, \h, #1
2016 smull_smlal_4 v1, v16, v18, v20, v22
2017 smull2_smlal2_4 v2, v16, v18, v20, v22
2018 smull_smlal_4 v3, v17, v19, v21, v23
2019 smull2_smlal2_4 v4, v17, v19, v21, v23
2020 shift_store_16 \type, \d_strd, x0, v1, v2, v3, v4
2021 b.le 0f
2022 mov v16.16b, v18.16b
2023 mov v17.16b, v19.16b
2024 mov v18.16b, v20.16b
2025 mov v19.16b, v21.16b
2026 mov v20.16b, v22.16b
2027 mov v21.16b, v23.16b
2028 b 16b
2029 0:
2030 ret
2031
2032 L(\type\()_8tap_v_tbl):
2033 .hword L(\type\()_8tap_v_tbl) - 1280b
2034 .hword L(\type\()_8tap_v_tbl) - 640b
2035 .hword L(\type\()_8tap_v_tbl) - 320b
2036 .hword L(\type\()_8tap_v_tbl) - 160b
2037 .hword L(\type\()_8tap_v_tbl) - 80b
2038 .hword L(\type\()_8tap_v_tbl) - 40b
2039 .hword L(\type\()_8tap_v_tbl) - 20b
2040 .hword 0
2041
2042 L(\type\()_8tap_hv):
2043 cmp \h, #4
2044 ubfx w10, \my, #7, #7
2045 and \my, \my, #0x7f
2046 b.le 4f
2047 mov \my, w10
2048 4:
2049 add \xmy, x11, \my, uxtw #3
2050
2051 adr x10, L(\type\()_8tap_hv_tbl)
2052 dup v30.4s, w12 // 6 - intermediate_bits
2053 ldrh w9, [x10, x9, lsl #1]
2054 neg v30.4s, v30.4s // -(6-intermediate_bits)
2055 .ifc \type, put
2056 dup v29.4s, w13 // 6 + intermediate_bits
2057 .else
2058 movi v29.8h, #(PREP_BIAS >> 8), lsl #8
2059 .endif
2060 sub x10, x10, w9, uxtw
2061 .ifc \type, put
2062 neg v29.4s, v29.4s // -(6+intermediate_bits)
2063 .endif
2064 br x10
2065
2066 20:
2067 AARCH64_VALID_JUMP_TARGET
2068 .ifc \type, put
2069 add \xmx, \xmx, #2
2070 ld1 {v0.s}[0], [\xmx]
2071 b.gt 280f
2072 add \xmy, \xmy, #2
2073 ld1 {v1.s}[0], [\xmy]
2074
2075 // 2x2, 2x4 hv
2076 sub \sr2, \src, #2
2077 sub \src, \sr2, \s_strd
2078 add \ds2, \dst, \d_strd
2079 lsl \s_strd, \s_strd, #1
2080 lsl \d_strd, \d_strd, #1
2081 sxtl v0.8h, v0.8b
2082 sxtl v1.8h, v1.8b
2083 mov x15, x30
2084
2085 ld1 {v27.8h}, [\src], \s_strd
2086 ext v28.16b, v27.16b, v27.16b, #2
2087 smull v27.4s, v27.4h, v0.4h
2088 smull v28.4s, v28.4h, v0.4h
2089 addp v27.4s, v27.4s, v28.4s
2090 addp v16.4s, v27.4s, v27.4s
2091 srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits)
2092 bl L(\type\()_8tap_filter_2)
2093 // The intermediates from the horizontal pass fit in 16 bit without
2094 // any bias; we could just as well keep them as .4s, but narrowing
2095 // them to .4h gives a significant speedup on out of order cores
2096 // (at the cost of a smaller slowdown on in-order cores such as A53).
2097 xtn v16.4h, v16.4s
2098
2099 trn1 v16.2s, v16.2s, v24.2s
2100 mov v17.8b, v24.8b
2101
2102 2:
2103 bl L(\type\()_8tap_filter_2)
2104
2105 ext v18.8b, v17.8b, v24.8b, #4
2106 smull v2.4s, v16.4h, v1.h[0]
2107 smlal v2.4s, v17.4h, v1.h[1]
2108 smlal v2.4s, v18.4h, v1.h[2]
2109 smlal v2.4s, v24.4h, v1.h[3]
2110
2111 srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits)
2112 sqxtun v2.4h, v2.4s
2113 umin v2.4h, v2.4h, v31.4h
2114 subs \h, \h, #2
2115 st1 {v2.s}[0], [\dst], \d_strd
2116 st1 {v2.s}[1], [\ds2], \d_strd
2117 b.le 0f
2118 mov v16.8b, v18.8b
2119 mov v17.8b, v24.8b
2120 b 2b
2121
2122 280: // 2x8, 2x16, 2x32 hv
2123 ld1 {v1.8b}, [\xmy]
2124 sub \src, \src, #2
2125 sub \sr2, \src, \s_strd, lsl #1
2126 sub \src, \sr2, \s_strd
2127 add \ds2, \dst, \d_strd
2128 lsl \s_strd, \s_strd, #1
2129 lsl \d_strd, \d_strd, #1
2130 sxtl v0.8h, v0.8b
2131 sxtl v1.8h, v1.8b
2132 mov x15, x30
2133
2134 ld1 {v27.8h}, [\src], \s_strd
2135 ext v28.16b, v27.16b, v27.16b, #2
2136 smull v27.4s, v27.4h, v0.4h
2137 smull v28.4s, v28.4h, v0.4h
2138 addp v27.4s, v27.4s, v28.4s
2139 addp v16.4s, v27.4s, v27.4s
2140 srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits)
2141 // The intermediates from the horizontal pass fit in 16 bit without
2142 // any bias; we could just as well keep them as .4s, but narrowing
2143 // them to .4h gives a significant speedup on out of order cores
2144 // (at the cost of a smaller slowdown on in-order cores such as A53).
2145
2146 bl L(\type\()_8tap_filter_2)
2147 xtn v16.4h, v16.4s
2148 trn1 v16.2s, v16.2s, v24.2s
2149 mov v17.8b, v24.8b
2150 bl L(\type\()_8tap_filter_2)
2151 ext v18.8b, v17.8b, v24.8b, #4
2152 mov v19.8b, v24.8b
2153 bl L(\type\()_8tap_filter_2)
2154 ext v20.8b, v19.8b, v24.8b, #4
2155 mov v21.8b, v24.8b
2156
2157 28:
2158 bl L(\type\()_8tap_filter_2)
2159 ext v22.8b, v21.8b, v24.8b, #4
2160 smull v3.4s, v16.4h, v1.h[0]
2161 smlal v3.4s, v17.4h, v1.h[1]
2162 smlal v3.4s, v18.4h, v1.h[2]
2163 smlal v3.4s, v19.4h, v1.h[3]
2164 smlal v3.4s, v20.4h, v1.h[4]
2165 smlal v3.4s, v21.4h, v1.h[5]
2166 smlal v3.4s, v22.4h, v1.h[6]
2167 smlal v3.4s, v24.4h, v1.h[7]
2168
2169 srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits)
2170 sqxtun v3.4h, v3.4s
2171 umin v3.4h, v3.4h, v31.4h
2172 subs \h, \h, #2
2173 st1 {v3.s}[0], [\dst], \d_strd
2174 st1 {v3.s}[1], [\ds2], \d_strd
2175 b.le 0f
2176 mov v16.8b, v18.8b
2177 mov v17.8b, v19.8b
2178 mov v18.8b, v20.8b
2179 mov v19.8b, v21.8b
2180 mov v20.8b, v22.8b
2181 mov v21.8b, v24.8b
2182 b 28b
2183
2184 0:
2185 ret x15
2186
2187 L(\type\()_8tap_filter_2):
2188 ld1 {v25.8h}, [\sr2], \s_strd
2189 ld1 {v27.8h}, [\src], \s_strd
2190 ext v26.16b, v25.16b, v25.16b, #2
2191 ext v28.16b, v27.16b, v27.16b, #2
2192 trn1 v24.2s, v25.2s, v27.2s
2193 trn2 v27.2s, v25.2s, v27.2s
2194 trn1 v25.2s, v26.2s, v28.2s
2195 trn2 v28.2s, v26.2s, v28.2s
2196 smull v24.4s, v24.4h, v0.h[0]
2197 smlal v24.4s, v25.4h, v0.h[1]
2198 smlal v24.4s, v27.4h, v0.h[2]
2199 smlal v24.4s, v28.4h, v0.h[3]
2200 srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits)
2201 xtn v24.4h, v24.4s
2202 ret
2203 .endif
2204
2205 40:
2206 AARCH64_VALID_JUMP_TARGET
2207 add \xmx, \xmx, #2
2208 ld1 {v0.s}[0], [\xmx]
2209 b.gt 480f
2210 add \xmy, \xmy, #2
2211 ld1 {v1.s}[0], [\xmy]
2212 sub \sr2, \src, #2
2213 sub \src, \sr2, \s_strd
2214 add \ds2, \dst, \d_strd
2215 lsl \s_strd, \s_strd, #1
2216 lsl \d_strd, \d_strd, #1
2217 sxtl v0.8h, v0.8b
2218 sxtl v1.8h, v1.8b
2219 mov x15, x30
2220
2221 // 4x2, 4x4 hv
2222 ld1 {v25.8h}, [\src], \s_strd
2223 ext v26.16b, v25.16b, v25.16b, #2
2224 ext v27.16b, v25.16b, v25.16b, #4
2225 ext v28.16b, v25.16b, v25.16b, #6
2226 smull v25.4s, v25.4h, v0.h[0]
2227 smlal v25.4s, v26.4h, v0.h[1]
2228 smlal v25.4s, v27.4h, v0.h[2]
2229 smlal v25.4s, v28.4h, v0.h[3]
2230 srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits)
2231 // The intermediates from the horizontal pass fit in 16 bit without
2232 // any bias; we could just as well keep them as .4s, but narrowing
2233 // them to .4h gives a significant speedup on out of order cores
2234 // (at the cost of a smaller slowdown on in-order cores such as A53).
2235 xtn v16.4h, v16.4s
2236
2237 bl L(\type\()_8tap_filter_4)
2238 mov v17.8b, v24.8b
2239 mov v18.8b, v25.8b
2240
2241 4:
2242 bl L(\type\()_8tap_filter_4)
2243 smull v2.4s, v16.4h, v1.h[0]
2244 smlal v2.4s, v17.4h, v1.h[1]
2245 smlal v2.4s, v18.4h, v1.h[2]
2246 smlal v2.4s, v24.4h, v1.h[3]
2247 smull v3.4s, v17.4h, v1.h[0]
2248 smlal v3.4s, v18.4h, v1.h[1]
2249 smlal v3.4s, v24.4h, v1.h[2]
2250 smlal v3.4s, v25.4h, v1.h[3]
2251 .ifc \type, put
2252 srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits)
2253 srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits)
2254 sqxtun v2.4h, v2.4s
2255 sqxtun2 v2.8h, v3.4s
2256 umin v2.8h, v2.8h, v31.8h
2257 .else
2258 rshrn v2.4h, v2.4s, #6
2259 rshrn2 v2.8h, v3.4s, #6
2260 sub v2.8h, v2.8h, v29.8h // PREP_BIAS
2261 .endif
2262 subs \h, \h, #2
2263
2264 st1 {v2.d}[0], [\dst], \d_strd
2265 st1 {v2.d}[1], [\ds2], \d_strd
2266 b.le 0f
2267 mov v16.8b, v18.8b
2268 mov v17.8b, v24.8b
2269 mov v18.8b, v25.8b
2270 b 4b
2271
2272 480: // 4x8, 4x16, 4x32 hv
2273 ld1 {v1.8b}, [\xmy]
2274 sub \src, \src, #2
2275 sub \sr2, \src, \s_strd, lsl #1
2276 sub \src, \sr2, \s_strd
2277 add \ds2, \dst, \d_strd
2278 lsl \s_strd, \s_strd, #1
2279 lsl \d_strd, \d_strd, #1
2280 sxtl v0.8h, v0.8b
2281 sxtl v1.8h, v1.8b
2282 mov x15, x30
2283
2284 ld1 {v25.8h}, [\src], \s_strd
2285 ext v26.16b, v25.16b, v25.16b, #2
2286 ext v27.16b, v25.16b, v25.16b, #4
2287 ext v28.16b, v25.16b, v25.16b, #6
2288 smull v25.4s, v25.4h, v0.h[0]
2289 smlal v25.4s, v26.4h, v0.h[1]
2290 smlal v25.4s, v27.4h, v0.h[2]
2291 smlal v25.4s, v28.4h, v0.h[3]
2292 srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits)
2293 // The intermediates from the horizontal pass fit in 16 bit without
2294 // any bias; we could just as well keep them as .4s, but narrowing
2295 // them to .4h gives a significant speedup on out of order cores
2296 // (at the cost of a smaller slowdown on in-order cores such as A53).
2297 xtn v16.4h, v16.4s
2298
2299 bl L(\type\()_8tap_filter_4)
2300 mov v17.8b, v24.8b
2301 mov v18.8b, v25.8b
2302 bl L(\type\()_8tap_filter_4)
2303 mov v19.8b, v24.8b
2304 mov v20.8b, v25.8b
2305 bl L(\type\()_8tap_filter_4)
2306 mov v21.8b, v24.8b
2307 mov v22.8b, v25.8b
2308
2309 48:
2310 bl L(\type\()_8tap_filter_4)
2311 smull v3.4s, v16.4h, v1.h[0]
2312 smlal v3.4s, v17.4h, v1.h[1]
2313 smlal v3.4s, v18.4h, v1.h[2]
2314 smlal v3.4s, v19.4h, v1.h[3]
2315 smlal v3.4s, v20.4h, v1.h[4]
2316 smlal v3.4s, v21.4h, v1.h[5]
2317 smlal v3.4s, v22.4h, v1.h[6]
2318 smlal v3.4s, v24.4h, v1.h[7]
2319 smull v4.4s, v17.4h, v1.h[0]
2320 smlal v4.4s, v18.4h, v1.h[1]
2321 smlal v4.4s, v19.4h, v1.h[2]
2322 smlal v4.4s, v20.4h, v1.h[3]
2323 smlal v4.4s, v21.4h, v1.h[4]
2324 smlal v4.4s, v22.4h, v1.h[5]
2325 smlal v4.4s, v24.4h, v1.h[6]
2326 smlal v4.4s, v25.4h, v1.h[7]
2327 .ifc \type, put
2328 srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits)
2329 srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits)
2330 sqxtun v3.4h, v3.4s
2331 sqxtun2 v3.8h, v4.4s
2332 umin v3.8h, v3.8h, v31.8h
2333 .else
2334 rshrn v3.4h, v3.4s, #6
2335 rshrn2 v3.8h, v4.4s, #6
2336 sub v3.8h, v3.8h, v29.8h // PREP_BIAS
2337 .endif
2338 subs \h, \h, #2
2339 st1 {v3.d}[0], [\dst], \d_strd
2340 st1 {v3.d}[1], [\ds2], \d_strd
2341 b.le 0f
2342 mov v16.8b, v18.8b
2343 mov v17.8b, v19.8b
2344 mov v18.8b, v20.8b
2345 mov v19.8b, v21.8b
2346 mov v20.8b, v22.8b
2347 mov v21.8b, v24.8b
2348 mov v22.8b, v25.8b
2349 b 48b
2350 0:
2351 ret x15
2352
2353 L(\type\()_8tap_filter_4):
2354 ld1 {v24.8h}, [\sr2], \s_strd
2355 ld1 {v25.8h}, [\src], \s_strd
2356 ext v26.16b, v24.16b, v24.16b, #2
2357 ext v27.16b, v24.16b, v24.16b, #4
2358 ext v28.16b, v24.16b, v24.16b, #6
2359 smull v24.4s, v24.4h, v0.h[0]
2360 smlal v24.4s, v26.4h, v0.h[1]
2361 smlal v24.4s, v27.4h, v0.h[2]
2362 smlal v24.4s, v28.4h, v0.h[3]
2363 ext v26.16b, v25.16b, v25.16b, #2
2364 ext v27.16b, v25.16b, v25.16b, #4
2365 ext v28.16b, v25.16b, v25.16b, #6
2366 smull v25.4s, v25.4h, v0.h[0]
2367 smlal v25.4s, v26.4h, v0.h[1]
2368 smlal v25.4s, v27.4h, v0.h[2]
2369 smlal v25.4s, v28.4h, v0.h[3]
2370 srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits)
2371 srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits)
2372 xtn v24.4h, v24.4s
2373 xtn v25.4h, v25.4s
2374 ret
2375
2376 80:
2377 160:
2378 320:
2379 AARCH64_VALID_JUMP_TARGET
2380 b.gt 880f
2381 add \xmy, \xmy, #2
2382 ld1 {v0.8b}, [\xmx]
2383 ld1 {v1.s}[0], [\xmy]
2384 sub \src, \src, #6
2385 sub \src, \src, \s_strd
2386 sxtl v0.8h, v0.8b
2387 sxtl v1.8h, v1.8b
2388 mov x15, x30
2389 mov \my, \h
2390
2391 164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv
2392 add \ds2, \dst, \d_strd
2393 add \sr2, \src, \s_strd
2394 lsl \d_strd, \d_strd, #1
2395 lsl \s_strd, \s_strd, #1
2396
2397 ld1 {v27.8h, v28.8h}, [\src], \s_strd
2398 smull v24.4s, v27.4h, v0.h[0]
2399 smull2 v25.4s, v27.8h, v0.h[0]
2400 .irpc i, 1234567
2401 ext v26.16b, v27.16b, v28.16b, #(2*\i)
2402 smlal v24.4s, v26.4h, v0.h[\i]
2403 smlal2 v25.4s, v26.8h, v0.h[\i]
2404 .endr
2405 srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits)
2406 srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits)
2407 // The intermediates from the horizontal pass fit in 16 bit without
2408 // any bias; we could just as well keep them as .4s, but narrowing
2409 // them to .4h gives a significant speedup on out of order cores
2410 // (at the cost of a smaller slowdown on in-order cores such as A53),
2411 // and conserves register space (no need to clobber v8-v15).
2412 uzp1 v16.8h, v24.8h, v25.8h // Same as xtn, xtn2
2413
2414 bl L(\type\()_8tap_filter_8)
2415 mov v17.16b, v23.16b
2416 mov v18.16b, v24.16b
2417
2418 8:
2419 smull v2.4s, v16.4h, v1.h[0]
2420 smull2 v3.4s, v16.8h, v1.h[0]
2421 bl L(\type\()_8tap_filter_8)
2422 smull v4.4s, v17.4h, v1.h[0]
2423 smull2 v5.4s, v17.8h, v1.h[0]
2424 smlal v2.4s, v17.4h, v1.h[1]
2425 smlal2 v3.4s, v17.8h, v1.h[1]
2426 smlal v4.4s, v18.4h, v1.h[1]
2427 smlal2 v5.4s, v18.8h, v1.h[1]
2428 smlal v2.4s, v18.4h, v1.h[2]
2429 smlal2 v3.4s, v18.8h, v1.h[2]
2430 smlal v4.4s, v23.4h, v1.h[2]
2431 smlal2 v5.4s, v23.8h, v1.h[2]
2432 smlal v2.4s, v23.4h, v1.h[3]
2433 smlal2 v3.4s, v23.8h, v1.h[3]
2434 smlal v4.4s, v24.4h, v1.h[3]
2435 smlal2 v5.4s, v24.8h, v1.h[3]
2436 .ifc \type, put
2437 srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits)
2438 srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits)
2439 srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits)
2440 srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits)
2441 sqxtun v2.4h, v2.4s
2442 sqxtun2 v2.8h, v3.4s
2443 sqxtun v3.4h, v4.4s
2444 sqxtun2 v3.8h, v5.4s
2445 umin v2.8h, v2.8h, v31.8h
2446 umin v3.8h, v3.8h, v31.8h
2447 .else
2448 rshrn v2.4h, v2.4s, #6
2449 rshrn2 v2.8h, v3.4s, #6
2450 rshrn v3.4h, v4.4s, #6
2451 rshrn2 v3.8h, v5.4s, #6
2452 sub v2.8h, v2.8h, v29.8h // PREP_BIAS
2453 sub v3.8h, v3.8h, v29.8h // PREP_BIAS
2454 .endif
2455 subs \h, \h, #2
2456 st1 {v2.8h}, [\dst], \d_strd
2457 st1 {v3.8h}, [\ds2], \d_strd
2458 b.le 9f
2459 mov v16.16b, v18.16b
2460 mov v17.16b, v23.16b
2461 mov v18.16b, v24.16b
2462 b 8b
2463 9:
2464 subs \w, \w, #8
2465 b.le 0f
2466 asr \s_strd, \s_strd, #1
2467 asr \d_strd, \d_strd, #1
2468 msub \src, \s_strd, \xmy, \src
2469 msub \dst, \d_strd, \xmy, \dst
2470 sub \src, \src, \s_strd, lsl #2
2471 mov \h, \my
2472 add \src, \src, #16
2473 add \dst, \dst, #16
2474 b 164b
2475
2476 880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv
2477 640:
2478 1280:
2479 AARCH64_VALID_JUMP_TARGET
2480 ld1 {v0.8b}, [\xmx]
2481 ld1 {v1.8b}, [\xmy]
2482 sub \src, \src, #6
2483 sub \src, \src, \s_strd
2484 sub \src, \src, \s_strd, lsl #1
2485 sxtl v0.8h, v0.8b
2486 sxtl v1.8h, v1.8b
2487 mov x15, x30
2488 mov \my, \h
2489
2490 168:
2491 add \ds2, \dst, \d_strd
2492 add \sr2, \src, \s_strd
2493 lsl \d_strd, \d_strd, #1
2494 lsl \s_strd, \s_strd, #1
2495
2496 ld1 {v27.8h, v28.8h}, [\src], \s_strd
2497 smull v24.4s, v27.4h, v0.h[0]
2498 smull2 v25.4s, v27.8h, v0.h[0]
2499 .irpc i, 1234567
2500 ext v26.16b, v27.16b, v28.16b, #(2*\i)
2501 smlal v24.4s, v26.4h, v0.h[\i]
2502 smlal2 v25.4s, v26.8h, v0.h[\i]
2503 .endr
2504 srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits)
2505 srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits)
2506 // The intermediates from the horizontal pass fit in 16 bit without
2507 // any bias; we could just as well keep them as .4s, but narrowing
2508 // them to .4h gives a significant speedup on out of order cores
2509 // (at the cost of a smaller slowdown on in-order cores such as A53),
2510 // and conserves register space (no need to clobber v8-v15).
2511 uzp1 v16.8h, v24.8h, v25.8h // Same as xtn, xtn2
2512
2513 bl L(\type\()_8tap_filter_8)
2514 mov v17.16b, v23.16b
2515 mov v18.16b, v24.16b
2516 bl L(\type\()_8tap_filter_8)
2517 mov v19.16b, v23.16b
2518 mov v20.16b, v24.16b
2519 bl L(\type\()_8tap_filter_8)
2520 mov v21.16b, v23.16b
2521 mov v22.16b, v24.16b
2522
2523 88:
2524 smull v2.4s, v16.4h, v1.h[0]
2525 smull2 v3.4s, v16.8h, v1.h[0]
2526 bl L(\type\()_8tap_filter_8)
2527 smull v4.4s, v17.4h, v1.h[0]
2528 smull2 v5.4s, v17.8h, v1.h[0]
2529 smlal v2.4s, v17.4h, v1.h[1]
2530 smlal2 v3.4s, v17.8h, v1.h[1]
2531 smlal v4.4s, v18.4h, v1.h[1]
2532 smlal2 v5.4s, v18.8h, v1.h[1]
2533 smlal v2.4s, v18.4h, v1.h[2]
2534 smlal2 v3.4s, v18.8h, v1.h[2]
2535 smlal v4.4s, v19.4h, v1.h[2]
2536 smlal2 v5.4s, v19.8h, v1.h[2]
2537 smlal v2.4s, v19.4h, v1.h[3]
2538 smlal2 v3.4s, v19.8h, v1.h[3]
2539 smlal v4.4s, v20.4h, v1.h[3]
2540 smlal2 v5.4s, v20.8h, v1.h[3]
2541 smlal v2.4s, v20.4h, v1.h[4]
2542 smlal2 v3.4s, v20.8h, v1.h[4]
2543 smlal v4.4s, v21.4h, v1.h[4]
2544 smlal2 v5.4s, v21.8h, v1.h[4]
2545 smlal v2.4s, v21.4h, v1.h[5]
2546 smlal2 v3.4s, v21.8h, v1.h[5]
2547 smlal v4.4s, v22.4h, v1.h[5]
2548 smlal2 v5.4s, v22.8h, v1.h[5]
2549 smlal v2.4s, v22.4h, v1.h[6]
2550 smlal2 v3.4s, v22.8h, v1.h[6]
2551 smlal v4.4s, v23.4h, v1.h[6]
2552 smlal2 v5.4s, v23.8h, v1.h[6]
2553 smlal v2.4s, v23.4h, v1.h[7]
2554 smlal2 v3.4s, v23.8h, v1.h[7]
2555 smlal v4.4s, v24.4h, v1.h[7]
2556 smlal2 v5.4s, v24.8h, v1.h[7]
2557 .ifc \type, put
2558 srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits)
2559 srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits)
2560 srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits)
2561 srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits)
2562 sqxtun v2.4h, v2.4s
2563 sqxtun2 v2.8h, v3.4s
2564 sqxtun v3.4h, v4.4s
2565 sqxtun2 v3.8h, v5.4s
2566 umin v2.8h, v2.8h, v31.8h
2567 umin v3.8h, v3.8h, v31.8h
2568 .else
2569 rshrn v2.4h, v2.4s, #6
2570 rshrn2 v2.8h, v3.4s, #6
2571 rshrn v3.4h, v4.4s, #6
2572 rshrn2 v3.8h, v5.4s, #6
2573 sub v2.8h, v2.8h, v29.8h // PREP_BIAS
2574 sub v3.8h, v3.8h, v29.8h // PREP_BIAS
2575 .endif
2576 subs \h, \h, #2
2577 st1 {v2.8h}, [\dst], \d_strd
2578 st1 {v3.8h}, [\ds2], \d_strd
2579 b.le 9f
2580 mov v16.16b, v18.16b
2581 mov v17.16b, v19.16b
2582 mov v18.16b, v20.16b
2583 mov v19.16b, v21.16b
2584 mov v20.16b, v22.16b
2585 mov v21.16b, v23.16b
2586 mov v22.16b, v24.16b
2587 b 88b
2588 9:
2589 subs \w, \w, #8
2590 b.le 0f
2591 asr \s_strd, \s_strd, #1
2592 asr \d_strd, \d_strd, #1
2593 msub \src, \s_strd, \xmy, \src
2594 msub \dst, \d_strd, \xmy, \dst
2595 sub \src, \src, \s_strd, lsl #3
2596 mov \h, \my
2597 add \src, \src, #16
2598 add \dst, \dst, #16
2599 b 168b
2600 0:
2601 ret x15
2602
2603 L(\type\()_8tap_filter_8):
2604 ld1 {v4.8h, v5.8h}, [\sr2], \s_strd
2605 ld1 {v6.8h, v7.8h}, [\src], \s_strd
2606 smull v25.4s, v4.4h, v0.h[0]
2607 smull2 v26.4s, v4.8h, v0.h[0]
2608 smull v27.4s, v6.4h, v0.h[0]
2609 smull2 v28.4s, v6.8h, v0.h[0]
2610 .irpc i, 1234567
2611 ext v23.16b, v4.16b, v5.16b, #(2*\i)
2612 ext v24.16b, v6.16b, v7.16b, #(2*\i)
2613 smlal v25.4s, v23.4h, v0.h[\i]
2614 smlal2 v26.4s, v23.8h, v0.h[\i]
2615 smlal v27.4s, v24.4h, v0.h[\i]
2616 smlal2 v28.4s, v24.8h, v0.h[\i]
2617 .endr
2618 srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits)
2619 srshl v26.4s, v26.4s, v30.4s // -(6-intermediate_bits)
2620 srshl v27.4s, v27.4s, v30.4s // -(6-intermediate_bits)
2621 srshl v28.4s, v28.4s, v30.4s // -(6-intermediate_bits)
2622 uzp1 v23.8h, v25.8h, v26.8h // Same as xtn, xtn2
2623 uzp1 v24.8h, v27.8h, v28.8h // Ditto
2624 ret
2625
2626 L(\type\()_8tap_hv_tbl):
2627 .hword L(\type\()_8tap_hv_tbl) - 1280b
2628 .hword L(\type\()_8tap_hv_tbl) - 640b
2629 .hword L(\type\()_8tap_hv_tbl) - 320b
2630 .hword L(\type\()_8tap_hv_tbl) - 160b
2631 .hword L(\type\()_8tap_hv_tbl) - 80b
2632 .hword L(\type\()_8tap_hv_tbl) - 40b
2633 .hword L(\type\()_8tap_hv_tbl) - 20b
2634 .hword 0
2635 endfunc
2636
2637
2638 function \type\()_bilin_16bpc_neon, export=1
2639 .ifc \bdmax, w8
2640 ldr w8, [sp]
2641 .endif
2642 dup v1.8h, \mx
2643 dup v3.8h, \my
2644 mov w10, #16
2645 sub w9, w10, \mx
2646 sub w10, w10, \my
2647 dup v0.8h, w9
2648 dup v2.8h, w10
2649 .ifc \type, prep
2650 uxtw \d_strd, \w
2651 lsl \d_strd, \d_strd, #1
2652 .endif
2653
2654 clz \bdmax, \bdmax // bitdepth_max
2655 clz w9, \w
2656 sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18
2657 mov w11, #4
2658 sub w9, w9, #24
2659 sub w11, w11, \bdmax // 4 - intermediate_bits
2660 add w12, \bdmax, #4 // 4 + intermediate_bits
2661 cbnz \mx, L(\type\()_bilin_h)
2662 cbnz \my, L(\type\()_bilin_v)
2663 b \type\()_neon
2664
2665 L(\type\()_bilin_h):
2666 cbnz \my, L(\type\()_bilin_hv)
2667
2668 adr x10, L(\type\()_bilin_h_tbl)
2669 dup v31.8h, w11 // 4 - intermediate_bits
2670 ldrh w9, [x10, x9, lsl #1]
2671 neg v31.8h, v31.8h // -(4-intermediate_bits)
2672 .ifc \type, put
2673 dup v30.8h, \bdmax // intermediate_bits
2674 .else
2675 movi v29.8h, #(PREP_BIAS >> 8), lsl #8
2676 .endif
2677 sub x10, x10, w9, uxtw
2678 .ifc \type, put
2679 neg v30.8h, v30.8h // -intermediate_bits
2680 .endif
2681 br x10
2682
2683 20: // 2xN h
2684 AARCH64_VALID_JUMP_TARGET
2685 .ifc \type, put
2686 add \ds2, \dst, \d_strd
2687 add \sr2, \src, \s_strd
2688 lsl \d_strd, \d_strd, #1
2689 lsl \s_strd, \s_strd, #1
2690 2:
2691 ld1 {v4.4h}, [\src], \s_strd
2692 ld1 {v6.4h}, [\sr2], \s_strd
2693 ext v5.8b, v4.8b, v4.8b, #2
2694 ext v7.8b, v6.8b, v6.8b, #2
2695 trn1 v4.2s, v4.2s, v6.2s
2696 trn1 v5.2s, v5.2s, v7.2s
2697 subs \h, \h, #2
2698 mul v4.4h, v4.4h, v0.4h
2699 mla v4.4h, v5.4h, v1.4h
2700 urshl v4.4h, v4.4h, v31.4h
2701 urshl v4.4h, v4.4h, v30.4h
2702 st1 {v4.s}[0], [\dst], \d_strd
2703 st1 {v4.s}[1], [\ds2], \d_strd
2704 b.gt 2b
2705 ret
2706 .endif
2707
2708 40: // 4xN h
2709 AARCH64_VALID_JUMP_TARGET
2710 add \ds2, \dst, \d_strd
2711 add \sr2, \src, \s_strd
2712 lsl \d_strd, \d_strd, #1
2713 lsl \s_strd, \s_strd, #1
2714 4:
2715 ld1 {v4.8h}, [\src], \s_strd
2716 ld1 {v6.8h}, [\sr2], \s_strd
2717 ext v5.16b, v4.16b, v4.16b, #2
2718 ext v7.16b, v6.16b, v6.16b, #2
2719 trn1 v4.2d, v4.2d, v6.2d
2720 trn1 v5.2d, v5.2d, v7.2d
2721 subs \h, \h, #2
2722 mul v4.8h, v4.8h, v0.8h
2723 mla v4.8h, v5.8h, v1.8h
2724 urshl v4.8h, v4.8h, v31.8h
2725 .ifc \type, put
2726 urshl v4.8h, v4.8h, v30.8h
2727 .else
2728 sub v4.8h, v4.8h, v29.8h
2729 .endif
2730 st1 {v4.d}[0], [\dst], \d_strd
2731 st1 {v4.d}[1], [\ds2], \d_strd
2732 b.gt 4b
2733 ret
2734
2735 80: // 8xN h
2736 AARCH64_VALID_JUMP_TARGET
2737 add \ds2, \dst, \d_strd
2738 add \sr2, \src, \s_strd
2739 lsl \d_strd, \d_strd, #1
2740 lsl \s_strd, \s_strd, #1
2741 8:
2742 ldr h5, [\src, #16]
2743 ldr h7, [\sr2, #16]
2744 ld1 {v4.8h}, [\src], \s_strd
2745 ld1 {v6.8h}, [\sr2], \s_strd
2746 ext v5.16b, v4.16b, v5.16b, #2
2747 ext v7.16b, v6.16b, v7.16b, #2
2748 subs \h, \h, #2
2749 mul v4.8h, v4.8h, v0.8h
2750 mla v4.8h, v5.8h, v1.8h
2751 mul v6.8h, v6.8h, v0.8h
2752 mla v6.8h, v7.8h, v1.8h
2753 urshl v4.8h, v4.8h, v31.8h
2754 urshl v6.8h, v6.8h, v31.8h
2755 .ifc \type, put
2756 urshl v4.8h, v4.8h, v30.8h
2757 urshl v6.8h, v6.8h, v30.8h
2758 .else
2759 sub v4.8h, v4.8h, v29.8h
2760 sub v6.8h, v6.8h, v29.8h
2761 .endif
2762 st1 {v4.8h}, [\dst], \d_strd
2763 st1 {v6.8h}, [\ds2], \d_strd
2764 b.gt 8b
2765 ret
2766 160:
2767 320:
2768 640:
2769 1280: // 16xN, 32xN, ... h
2770 AARCH64_VALID_JUMP_TARGET
2771 add \ds2, \dst, \d_strd
2772 add \sr2, \src, \s_strd
2773 lsl \s_strd, \s_strd, #1
2774
2775 sub \s_strd, \s_strd, \w, uxtw #1
2776 sub \s_strd, \s_strd, #16
2777 .ifc \type, put
2778 lsl \d_strd, \d_strd, #1
2779 sub \d_strd, \d_strd, \w, uxtw #1
2780 .endif
2781 161:
2782 ld1 {v16.8h}, [\src], #16
2783 ld1 {v21.8h}, [\sr2], #16
2784 mov \mx, \w
2785
2786 16:
2787 ld1 {v17.8h, v18.8h}, [\src], #32
2788 ld1 {v22.8h, v23.8h}, [\sr2], #32
2789 ext v19.16b, v16.16b, v17.16b, #2
2790 ext v20.16b, v17.16b, v18.16b, #2
2791 ext v24.16b, v21.16b, v22.16b, #2
2792 ext v25.16b, v22.16b, v23.16b, #2
2793 mul v16.8h, v16.8h, v0.8h
2794 mla v16.8h, v19.8h, v1.8h
2795 mul v17.8h, v17.8h, v0.8h
2796 mla v17.8h, v20.8h, v1.8h
2797 mul v21.8h, v21.8h, v0.8h
2798 mla v21.8h, v24.8h, v1.8h
2799 mul v22.8h, v22.8h, v0.8h
2800 mla v22.8h, v25.8h, v1.8h
2801 urshl v16.8h, v16.8h, v31.8h
2802 urshl v17.8h, v17.8h, v31.8h
2803 urshl v21.8h, v21.8h, v31.8h
2804 urshl v22.8h, v22.8h, v31.8h
2805 subs \mx, \mx, #16
2806 .ifc \type, put
2807 urshl v16.8h, v16.8h, v30.8h
2808 urshl v17.8h, v17.8h, v30.8h
2809 urshl v21.8h, v21.8h, v30.8h
2810 urshl v22.8h, v22.8h, v30.8h
2811 .else
2812 sub v16.8h, v16.8h, v29.8h
2813 sub v17.8h, v17.8h, v29.8h
2814 sub v21.8h, v21.8h, v29.8h
2815 sub v22.8h, v22.8h, v29.8h
2816 .endif
2817 st1 {v16.8h, v17.8h}, [\dst], #32
2818 st1 {v21.8h, v22.8h}, [\ds2], #32
2819 b.le 9f
2820
2821 mov v16.16b, v18.16b
2822 mov v21.16b, v23.16b
2823 b 16b
2824
2825 9:
2826 add \dst, \dst, \d_strd
2827 add \ds2, \ds2, \d_strd
2828 add \src, \src, \s_strd
2829 add \sr2, \sr2, \s_strd
2830
2831 subs \h, \h, #2
2832 b.gt 161b
2833 ret
2834
2835 L(\type\()_bilin_h_tbl):
2836 .hword L(\type\()_bilin_h_tbl) - 1280b
2837 .hword L(\type\()_bilin_h_tbl) - 640b
2838 .hword L(\type\()_bilin_h_tbl) - 320b
2839 .hword L(\type\()_bilin_h_tbl) - 160b
2840 .hword L(\type\()_bilin_h_tbl) - 80b
2841 .hword L(\type\()_bilin_h_tbl) - 40b
2842 .hword L(\type\()_bilin_h_tbl) - 20b
2843 .hword 0
2844
2845
2846 L(\type\()_bilin_v):
2847 cmp \h, #4
2848 adr x10, L(\type\()_bilin_v_tbl)
2849 .ifc \type, prep
2850 dup v31.8h, w11 // 4 - intermediate_bits
2851 .endif
2852 ldrh w9, [x10, x9, lsl #1]
2853 .ifc \type, prep
2854 movi v29.8h, #(PREP_BIAS >> 8), lsl #8
2855 neg v31.8h, v31.8h // -(4-intermediate_bits)
2856 .endif
2857 sub x10, x10, w9, uxtw
2858 br x10
2859
2860 20: // 2xN v
2861 AARCH64_VALID_JUMP_TARGET
2862 .ifc \type, put
2863 cmp \h, #2
2864 add \ds2, \dst, \d_strd
2865 add \sr2, \src, \s_strd
2866 lsl \s_strd, \s_strd, #1
2867 lsl \d_strd, \d_strd, #1
2868
2869 // 2x2 v
2870 ld1 {v16.s}[0], [\src], \s_strd
2871 b.gt 24f
2872 22:
2873 ld1 {v17.s}[0], [\sr2], \s_strd
2874 ld1 {v18.s}[0], [\src], \s_strd
2875 trn1 v16.2s, v16.2s, v17.2s
2876 trn1 v17.2s, v17.2s, v18.2s
2877 mul v4.4h, v16.4h, v2.4h
2878 mla v4.4h, v17.4h, v3.4h
2879 urshr v4.8h, v4.8h, #4
2880 st1 {v4.s}[0], [\dst]
2881 st1 {v4.s}[1], [\ds2]
2882 ret
2883 24: // 2x4, 2x6, 2x8, ... v
2884 ld1 {v17.s}[0], [\sr2], \s_strd
2885 ld1 {v18.s}[0], [\src], \s_strd
2886 ld1 {v19.s}[0], [\sr2], \s_strd
2887 ld1 {v20.s}[0], [\src], \s_strd
2888 sub \h, \h, #4
2889 trn1 v16.2s, v16.2s, v17.2s
2890 trn1 v17.2s, v17.2s, v18.2s
2891 trn1 v18.2s, v18.2s, v19.2s
2892 trn1 v19.2s, v19.2s, v20.2s
2893 trn1 v16.2d, v16.2d, v18.2d
2894 trn1 v17.2d, v17.2d, v19.2d
2895 mul v4.8h, v16.8h, v2.8h
2896 mla v4.8h, v17.8h, v3.8h
2897 cmp \h, #2
2898 urshr v4.8h, v4.8h, #4
2899 st1 {v4.s}[0], [\dst], \d_strd
2900 st1 {v4.s}[1], [\ds2], \d_strd
2901 st1 {v4.s}[2], [\dst], \d_strd
2902 st1 {v4.s}[3], [\ds2], \d_strd
2903 b.lt 0f
2904 mov v16.8b, v20.8b
2905 b.eq 22b
2906 b 24b
2907 0:
2908 ret
2909 .endif
2910
2911 40: // 4xN v
2912 AARCH64_VALID_JUMP_TARGET
2913 add \ds2, \dst, \d_strd
2914 add \sr2, \src, \s_strd
2915 lsl \s_strd, \s_strd, #1
2916 lsl \d_strd, \d_strd, #1
2917 ld1 {v16.4h}, [\src], \s_strd
2918 4:
2919 ld1 {v17.4h}, [\sr2], \s_strd
2920 ld1 {v18.4h}, [\src], \s_strd
2921 trn1 v16.2d, v16.2d, v17.2d
2922 trn1 v17.2d, v17.2d, v18.2d
2923 mul v4.8h, v16.8h, v2.8h
2924 mla v4.8h, v17.8h, v3.8h
2925 subs \h, \h, #2
2926 .ifc \type, put
2927 urshr v4.8h, v4.8h, #4
2928 .else
2929 urshl v4.8h, v4.8h, v31.8h
2930 sub v4.8h, v4.8h, v29.8h
2931 .endif
2932 st1 {v4.d}[0], [\dst], \d_strd
2933 st1 {v4.d}[1], [\ds2], \d_strd
2934 b.le 0f
2935 mov v16.8b, v18.8b
2936 b 4b
2937 0:
2938 ret
2939
2940 80: // 8xN v
2941 AARCH64_VALID_JUMP_TARGET
2942 add \ds2, \dst, \d_strd
2943 add \sr2, \src, \s_strd
2944 lsl \s_strd, \s_strd, #1
2945 lsl \d_strd, \d_strd, #1
2946 ld1 {v16.8h}, [\src], \s_strd
2947 8:
2948 ld1 {v17.8h}, [\sr2], \s_strd
2949 ld1 {v18.8h}, [\src], \s_strd
2950 mul v4.8h, v16.8h, v2.8h
2951 mla v4.8h, v17.8h, v3.8h
2952 mul v5.8h, v17.8h, v2.8h
2953 mla v5.8h, v18.8h, v3.8h
2954 subs \h, \h, #2
2955 .ifc \type, put
2956 urshr v4.8h, v4.8h, #4
2957 urshr v5.8h, v5.8h, #4
2958 .else
2959 urshl v4.8h, v4.8h, v31.8h
2960 urshl v5.8h, v5.8h, v31.8h
2961 sub v4.8h, v4.8h, v29.8h
2962 sub v5.8h, v5.8h, v29.8h
2963 .endif
2964 st1 {v4.8h}, [\dst], \d_strd
2965 st1 {v5.8h}, [\ds2], \d_strd
2966 b.le 0f
2967 mov v16.16b, v18.16b
2968 b 8b
2969 0:
2970 ret
2971
2972 160: // 16xN, 32xN, ...
2973 320:
2974 640:
2975 1280:
2976 AARCH64_VALID_JUMP_TARGET
2977 mov \my, \h
2978 1:
2979 add \ds2, \dst, \d_strd
2980 add \sr2, \src, \s_strd
2981 lsl \s_strd, \s_strd, #1
2982 lsl \d_strd, \d_strd, #1
2983
2984 ld1 {v16.8h, v17.8h}, [\src], \s_strd
2985 2:
2986 ld1 {v18.8h, v19.8h}, [\sr2], \s_strd
2987 ld1 {v20.8h, v21.8h}, [\src], \s_strd
2988 mul v4.8h, v16.8h, v2.8h
2989 mla v4.8h, v18.8h, v3.8h
2990 mul v5.8h, v17.8h, v2.8h
2991 mla v5.8h, v19.8h, v3.8h
2992 mul v6.8h, v18.8h, v2.8h
2993 mla v6.8h, v20.8h, v3.8h
2994 mul v7.8h, v19.8h, v2.8h
2995 mla v7.8h, v21.8h, v3.8h
2996 subs \h, \h, #2
2997 .ifc \type, put
2998 urshr v4.8h, v4.8h, #4
2999 urshr v5.8h, v5.8h, #4
3000 urshr v6.8h, v6.8h, #4
3001 urshr v7.8h, v7.8h, #4
3002 .else
3003 urshl v4.8h, v4.8h, v31.8h
3004 urshl v5.8h, v5.8h, v31.8h
3005 urshl v6.8h, v6.8h, v31.8h
3006 urshl v7.8h, v7.8h, v31.8h
3007 sub v4.8h, v4.8h, v29.8h
3008 sub v5.8h, v5.8h, v29.8h
3009 sub v6.8h, v6.8h, v29.8h
3010 sub v7.8h, v7.8h, v29.8h
3011 .endif
3012 st1 {v4.8h, v5.8h}, [\dst], \d_strd
3013 st1 {v6.8h, v7.8h}, [\ds2], \d_strd
3014 b.le 9f
3015 mov v16.16b, v20.16b
3016 mov v17.16b, v21.16b
3017 b 2b
3018 9:
3019 subs \w, \w, #16
3020 b.le 0f
3021 asr \s_strd, \s_strd, #1
3022 asr \d_strd, \d_strd, #1
3023 msub \src, \s_strd, \xmy, \src
3024 msub \dst, \d_strd, \xmy, \dst
3025 sub \src, \src, \s_strd, lsl #1
3026 mov \h, \my
3027 add \src, \src, #32
3028 add \dst, \dst, #32
3029 b 1b
3030 0:
3031 ret
3032
3033 L(\type\()_bilin_v_tbl):
3034 .hword L(\type\()_bilin_v_tbl) - 1280b
3035 .hword L(\type\()_bilin_v_tbl) - 640b
3036 .hword L(\type\()_bilin_v_tbl) - 320b
3037 .hword L(\type\()_bilin_v_tbl) - 160b
3038 .hword L(\type\()_bilin_v_tbl) - 80b
3039 .hword L(\type\()_bilin_v_tbl) - 40b
3040 .hword L(\type\()_bilin_v_tbl) - 20b
3041 .hword 0
3042
3043 L(\type\()_bilin_hv):
3044 adr x10, L(\type\()_bilin_hv_tbl)
3045 dup v31.8h, w11 // 4 - intermediate_bits
3046 ldrh w9, [x10, x9, lsl #1]
3047 neg v31.8h, v31.8h // -(4-intermediate_bits)
3048 .ifc \type, put
3049 dup v30.4s, w12 // 4 + intermediate_bits
3050 .else
3051 movi v29.8h, #(PREP_BIAS >> 8), lsl #8
3052 .endif
3053 sub x10, x10, w9, uxtw
3054 .ifc \type, put
3055 neg v30.4s, v30.4s // -(4+intermediate_bits)
3056 .endif
3057 br x10
3058
3059 20: // 2xN hv
3060 AARCH64_VALID_JUMP_TARGET
3061 .ifc \type, put
3062 add \sr2, \src, \s_strd
3063 add \ds2, \dst, \d_strd
3064 lsl \s_strd, \s_strd, #1
3065 lsl \d_strd, \d_strd, #1
3066
3067 ld1 {v20.4h}, [\src], \s_strd
3068 ext v21.8b, v20.8b, v20.8b, #2
3069 mul v16.4h, v20.4h, v0.4h
3070 mla v16.4h, v21.4h, v1.4h
3071 urshl v16.4h, v16.4h, v31.4h
3072
3073 2:
3074 ld1 {v22.4h}, [\sr2], \s_strd
3075 ld1 {v24.4h}, [\src], \s_strd
3076 ext v23.8b, v22.8b, v22.8b, #2
3077 ext v25.8b, v24.8b, v24.8b, #2
3078 trn1 v22.2s, v22.2s, v24.2s
3079 trn1 v23.2s, v23.2s, v25.2s
3080 mul v17.4h, v22.4h, v0.4h
3081 mla v17.4h, v23.4h, v1.4h
3082 urshl v17.4h, v17.4h, v31.4h
3083
3084 trn1 v16.2s, v16.2s, v17.2s
3085
3086 umull v4.4s, v16.4h, v2.4h
3087 umlal v4.4s, v17.4h, v3.4h
3088 urshl v4.4s, v4.4s, v30.4s
3089 xtn v4.4h, v4.4s
3090 subs \h, \h, #2
3091 st1 {v4.s}[0], [\dst], \d_strd
3092 st1 {v4.s}[1], [\ds2], \d_strd
3093 b.le 0f
3094 trn2 v16.2s, v17.2s, v17.2s
3095 b 2b
3096 0:
3097 ret
3098 .endif
3099
3100 40: // 4xN hv
3101 AARCH64_VALID_JUMP_TARGET
3102 add \sr2, \src, \s_strd
3103 add \ds2, \dst, \d_strd
3104 lsl \s_strd, \s_strd, #1
3105 lsl \d_strd, \d_strd, #1
3106
3107 ld1 {v20.8h}, [\src], \s_strd
3108 ext v21.16b, v20.16b, v20.16b, #2
3109 mul v16.4h, v20.4h, v0.4h
3110 mla v16.4h, v21.4h, v1.4h
3111 urshl v16.4h, v16.4h, v31.4h
3112
3113 4:
3114 ld1 {v22.8h}, [\sr2], \s_strd
3115 ld1 {v24.8h}, [\src], \s_strd
3116 ext v23.16b, v22.16b, v22.16b, #2
3117 ext v25.16b, v24.16b, v24.16b, #2
3118 trn1 v22.2d, v22.2d, v24.2d
3119 trn1 v23.2d, v23.2d, v25.2d
3120 mul v17.8h, v22.8h, v0.8h
3121 mla v17.8h, v23.8h, v1.8h
3122 urshl v17.8h, v17.8h, v31.8h
3123
3124 trn1 v16.2d, v16.2d, v17.2d
3125
3126 umull v4.4s, v16.4h, v2.4h
3127 umlal v4.4s, v17.4h, v3.4h
3128 umull2 v5.4s, v16.8h, v2.8h
3129 umlal2 v5.4s, v17.8h, v3.8h
3130 .ifc \type, put
3131 urshl v4.4s, v4.4s, v30.4s
3132 urshl v5.4s, v5.4s, v30.4s
3133 uzp1 v4.8h, v4.8h, v5.8h // Same as xtn, xtn2
3134 .else
3135 rshrn v4.4h, v4.4s, #4
3136 rshrn2 v4.8h, v5.4s, #4
3137 sub v4.8h, v4.8h, v29.8h
3138 .endif
3139 subs \h, \h, #2
3140 st1 {v4.d}[0], [\dst], \d_strd
3141 st1 {v4.d}[1], [\ds2], \d_strd
3142 b.le 0f
3143 trn2 v16.2d, v17.2d, v17.2d
3144 b 4b
3145 0:
3146 ret
3147
3148 80: // 8xN, 16xN, ... hv
3149 160:
3150 320:
3151 640:
3152 1280:
3153 AARCH64_VALID_JUMP_TARGET
3154 mov \my, \h
3155
3156 1:
3157 add \sr2, \src, \s_strd
3158 add \ds2, \dst, \d_strd
3159 lsl \s_strd, \s_strd, #1
3160 lsl \d_strd, \d_strd, #1
3161
3162 ldr h21, [\src, #16]
3163 ld1 {v20.8h}, [\src], \s_strd
3164 ext v21.16b, v20.16b, v21.16b, #2
3165 mul v16.8h, v20.8h, v0.8h
3166 mla v16.8h, v21.8h, v1.8h
3167 urshl v16.8h, v16.8h, v31.8h
3168
3169 2:
3170 ldr h23, [\sr2, #16]
3171 ld1 {v22.8h}, [\sr2], \s_strd
3172 ldr h25, [\src, #16]
3173 ld1 {v24.8h}, [\src], \s_strd
3174 ext v23.16b, v22.16b, v23.16b, #2
3175 ext v25.16b, v24.16b, v25.16b, #2
3176 mul v17.8h, v22.8h, v0.8h
3177 mla v17.8h, v23.8h, v1.8h
3178 mul v18.8h, v24.8h, v0.8h
3179 mla v18.8h, v25.8h, v1.8h
3180 urshl v17.8h, v17.8h, v31.8h
3181 urshl v18.8h, v18.8h, v31.8h
3182
3183 umull v4.4s, v16.4h, v2.4h
3184 umlal v4.4s, v17.4h, v3.4h
3185 umull2 v5.4s, v16.8h, v2.8h
3186 umlal2 v5.4s, v17.8h, v3.8h
3187 umull v6.4s, v17.4h, v2.4h
3188 umlal v6.4s, v18.4h, v3.4h
3189 umull2 v7.4s, v17.8h, v2.8h
3190 umlal2 v7.4s, v18.8h, v3.8h
3191 .ifc \type, put
3192 urshl v4.4s, v4.4s, v30.4s
3193 urshl v5.4s, v5.4s, v30.4s
3194 urshl v6.4s, v6.4s, v30.4s
3195 urshl v7.4s, v7.4s, v30.4s
3196 uzp1 v4.8h, v4.8h, v5.8h // Same as xtn, xtn2
3197 uzp1 v5.8h, v6.8h, v7.8h // Ditto
3198 .else
3199 rshrn v4.4h, v4.4s, #4
3200 rshrn2 v4.8h, v5.4s, #4
3201 rshrn v5.4h, v6.4s, #4
3202 rshrn2 v5.8h, v7.4s, #4
3203 sub v4.8h, v4.8h, v29.8h
3204 sub v5.8h, v5.8h, v29.8h
3205 .endif
3206 subs \h, \h, #2
3207 st1 {v4.8h}, [\dst], \d_strd
3208 st1 {v5.8h}, [\ds2], \d_strd
3209 b.le 9f
3210 mov v16.16b, v18.16b
3211 b 2b
3212 9:
3213 subs \w, \w, #8
3214 b.le 0f
3215 asr \s_strd, \s_strd, #1
3216 asr \d_strd, \d_strd, #1
3217 msub \src, \s_strd, \xmy, \src
3218 msub \dst, \d_strd, \xmy, \dst
3219 sub \src, \src, \s_strd, lsl #1
3220 mov \h, \my
3221 add \src, \src, #16
3222 add \dst, \dst, #16
3223 b 1b
3224 0:
3225 ret
3226
3227 L(\type\()_bilin_hv_tbl):
3228 .hword L(\type\()_bilin_hv_tbl) - 1280b
3229 .hword L(\type\()_bilin_hv_tbl) - 640b
3230 .hword L(\type\()_bilin_hv_tbl) - 320b
3231 .hword L(\type\()_bilin_hv_tbl) - 160b
3232 .hword L(\type\()_bilin_hv_tbl) - 80b
3233 .hword L(\type\()_bilin_hv_tbl) - 40b
3234 .hword L(\type\()_bilin_hv_tbl) - 20b
3235 .hword 0
3236 endfunc
3237 .endm
3238
3239 filter_fn put, x0, x1, x2, x3, w4, w5, w6, x6, w7, x7, w8, x9, x10
3240 filter_fn prep, x0, x8, x1, x2, w3, w4, w5, x5, w6, x6, w7, x9, x10
3241
3242 .macro load_filter_row dst, src, inc
3243 asr w13, \src, #10
3244 add \src, \src, \inc
3245 ldr \dst, [x11, w13, sxtw #3]
3246 .endm
3247
3248 function warp_filter_horz_neon
3249 add w12, w5, #512
3250
3251 ld1 {v16.8h, v17.8h}, [x2], x3
3252
3253 load_filter_row d0, w12, w7
3254 load_filter_row d1, w12, w7
3255 load_filter_row d2, w12, w7
3256 sxtl v0.8h, v0.8b
3257 load_filter_row d3, w12, w7
3258 sxtl v1.8h, v1.8b
3259 load_filter_row d4, w12, w7
3260 sxtl v2.8h, v2.8b
3261 load_filter_row d5, w12, w7
3262 sxtl v3.8h, v3.8b
3263 load_filter_row d6, w12, w7
3264 sxtl v4.8h, v4.8b
3265 load_filter_row d7, w12, w7
3266 sxtl v5.8h, v5.8b
3267 ext v18.16b, v16.16b, v17.16b, #2*1
3268 smull v8.4s, v16.4h, v0.4h
3269 smull2 v9.4s, v16.8h, v0.8h
3270 sxtl v6.8h, v6.8b
3271 ext v19.16b, v16.16b, v17.16b, #2*2
3272 smull v10.4s, v18.4h, v1.4h
3273 smull2 v11.4s, v18.8h, v1.8h
3274 sxtl v7.8h, v7.8b
3275 ext v20.16b, v16.16b, v17.16b, #2*3
3276 smull v0.4s, v19.4h, v2.4h
3277 smull2 v1.4s, v19.8h, v2.8h
3278 ext v21.16b, v16.16b, v17.16b, #2*4
3279 addp v8.4s, v8.4s, v9.4s
3280 smull v2.4s, v20.4h, v3.4h
3281 smull2 v3.4s, v20.8h, v3.8h
3282 ext v22.16b, v16.16b, v17.16b, #2*5
3283 addp v9.4s, v10.4s, v11.4s
3284 smull v10.4s, v21.4h, v4.4h
3285 smull2 v11.4s, v21.8h, v4.8h
3286 ext v23.16b, v16.16b, v17.16b, #2*6
3287 addp v0.4s, v0.4s, v1.4s
3288 smull v18.4s, v22.4h, v5.4h
3289 smull2 v19.4s, v22.8h, v5.8h
3290 ext v16.16b, v16.16b, v17.16b, #2*7
3291 addp v1.4s, v2.4s, v3.4s
3292 addp v2.4s, v10.4s, v11.4s
3293 smull v20.4s, v23.4h, v6.4h
3294 smull2 v21.4s, v23.8h, v6.8h
3295 addp v3.4s, v18.4s, v19.4s
3296 smull v22.4s, v16.4h, v7.4h
3297 smull2 v23.4s, v16.8h, v7.8h
3298 addp v4.4s, v20.4s, v21.4s
3299 addp v5.4s, v22.4s, v23.4s
3300
3301 addp v8.4s, v8.4s, v9.4s
3302 addp v0.4s, v0.4s, v1.4s
3303 addp v2.4s, v2.4s, v3.4s
3304 addp v4.4s, v4.4s, v5.4s
3305
3306 addp v16.4s, v8.4s, v0.4s
3307 addp v17.4s, v2.4s, v4.4s
3308
3309 add w5, w5, w8
3310
3311 srshl v16.4s, v16.4s, v14.4s // -(7 - intermediate_bits)
3312 srshl v17.4s, v17.4s, v14.4s // -(7 - intermediate_bits)
3313
3314 ret
3315 endfunc
3316
3317 // void dav1d_warp_affine_8x8_16bpc_neon(
3318 // pixel *dst, const ptrdiff_t dst_stride,
3319 // const pixel *src, const ptrdiff_t src_stride,
3320 // const int16_t *const abcd, int mx, int my,
3321 // const int bitdepth_max)
3322 .macro warp t
3323 function warp_affine_8x8\t\()_16bpc_neon, export=1
3324 stp d8, d9, [sp, #-0x40]!
3325 stp d10, d11, [sp, #0x10]
3326 stp d12, d13, [sp, #0x20]
3327 stp d14, d15, [sp, #0x30]
3328
3329 .ifb \t
3330 dup v15.8h, w7 // bitdepth_max
3331 .else
3332 movi v15.8h, #(PREP_BIAS >> 8), lsl #8
3333 .endif
3334 clz w7, w7
3335 // intermediate_bits = clz(bitdepth_max) - 18
3336 .ifb \t
3337 sub w8, w7, #11 // 7 + intermediate_bits = clz(bitdepth_max) - 18 + 7
3338 .endif
3339 sub w7, w7, #25 // -(7 - intermediate_bits)
3340 .ifb \t
3341 neg w8, w8 // -(7 + intermediate_bits)
3342 .endif
3343 dup v14.4s, w7 // -(7 - intermediate_bits)
3344 .ifb \t
3345 dup v13.4s, w8 // -(7 + intermediate_bits)
3346 .endif
3347
3348 ldr x4, [x4]
3349 sbfx x7, x4, #0, #16
3350 sbfx x8, x4, #16, #16
3351 sbfx x9, x4, #32, #16
3352 sbfx x4, x4, #48, #16
3353 mov w10, #8
3354 sub x2, x2, x3, lsl #1
3355 sub x2, x2, x3
3356 sub x2, x2, #6
3357 movrel x11, X(mc_warp_filter), 64*8
3358 mov x15, x30
3359 .ifnb \t
3360 lsl x1, x1, #1
3361 .endif
3362
3363 bl warp_filter_horz_neon
3364 uzp1 v24.8h, v16.8h, v17.8h // Same as xtn, xtn2
3365 bl warp_filter_horz_neon
3366 uzp1 v25.8h, v16.8h, v17.8h // Ditto
3367 bl warp_filter_horz_neon
3368 uzp1 v26.8h, v16.8h, v17.8h // Ditto
3369 bl warp_filter_horz_neon
3370 uzp1 v27.8h, v16.8h, v17.8h // Ditto
3371 bl warp_filter_horz_neon
3372 uzp1 v28.8h, v16.8h, v17.8h // Ditto
3373 bl warp_filter_horz_neon
3374 uzp1 v29.8h, v16.8h, v17.8h // Ditto
3375 bl warp_filter_horz_neon
3376 uzp1 v30.8h, v16.8h, v17.8h // Ditto
3377
3378 1:
3379 add w14, w6, #512
3380 bl warp_filter_horz_neon
3381 uzp1 v31.8h, v16.8h, v17.8h // Same as xtn, xtn2
3382
3383 load_filter_row d0, w14, w9
3384 load_filter_row d1, w14, w9
3385 load_filter_row d2, w14, w9
3386 load_filter_row d3, w14, w9
3387 load_filter_row d4, w14, w9
3388 load_filter_row d5, w14, w9
3389 load_filter_row d6, w14, w9
3390 load_filter_row d7, w14, w9
3391 transpose_8x8b_xtl v0, v1, v2, v3, v4, v5, v6, v7, sxtl
3392
3393 // This ordering of smull/smlal/smull2/smlal2 is highly
3394 // beneficial for Cortex A53 here.
3395 smull v16.4s, v24.4h, v0.4h
3396 smlal v16.4s, v25.4h, v1.4h
3397 smlal v16.4s, v26.4h, v2.4h
3398 smlal v16.4s, v27.4h, v3.4h
3399 smlal v16.4s, v28.4h, v4.4h
3400 smlal v16.4s, v29.4h, v5.4h
3401 smlal v16.4s, v30.4h, v6.4h
3402 smlal v16.4s, v31.4h, v7.4h
3403 smull2 v17.4s, v24.8h, v0.8h
3404 smlal2 v17.4s, v25.8h, v1.8h
3405 smlal2 v17.4s, v26.8h, v2.8h
3406 smlal2 v17.4s, v27.8h, v3.8h
3407 smlal2 v17.4s, v28.8h, v4.8h
3408 smlal2 v17.4s, v29.8h, v5.8h
3409 smlal2 v17.4s, v30.8h, v6.8h
3410 smlal2 v17.4s, v31.8h, v7.8h
3411
3412 mov v24.16b, v25.16b
3413 mov v25.16b, v26.16b
3414 .ifb \t
3415 srshl v16.4s, v16.4s, v13.4s // -(7 + intermediate_bits)
3416 srshl v17.4s, v17.4s, v13.4s // -(7 + intermediate_bits)
3417 .else
3418 rshrn v16.4h, v16.4s, #7
3419 rshrn2 v16.8h, v17.4s, #7
3420 .endif
3421 mov v26.16b, v27.16b
3422 .ifb \t
3423 sqxtun v16.4h, v16.4s
3424 sqxtun2 v16.8h, v17.4s
3425 .else
3426 sub v16.8h, v16.8h, v15.8h // PREP_BIAS
3427 .endif
3428 mov v27.16b, v28.16b
3429 mov v28.16b, v29.16b
3430 .ifb \t
3431 umin v16.8h, v16.8h, v15.8h // bitdepth_max
3432 .endif
3433 mov v29.16b, v30.16b
3434 mov v30.16b, v31.16b
3435 subs w10, w10, #1
3436 st1 {v16.8h}, [x0], x1
3437
3438 add w6, w6, w4
3439 b.gt 1b
3440
3441 ldp d14, d15, [sp, #0x30]
3442 ldp d12, d13, [sp, #0x20]
3443 ldp d10, d11, [sp, #0x10]
3444 ldp d8, d9, [sp], 0x40
3445
3446 ret x15
3447 endfunc
3448 .endm
3449
3450 warp
3451 warp t
3452
3453 // void dav1d_emu_edge_16bpc_neon(
3454 // const intptr_t bw, const intptr_t bh,
3455 // const intptr_t iw, const intptr_t ih,
3456 // const intptr_t x, const intptr_t y,
3457 // pixel *dst, const ptrdiff_t dst_stride,
3458 // const pixel *ref, const ptrdiff_t ref_stride)
3459 function emu_edge_16bpc_neon, export=1
3460 ldp x8, x9, [sp]
3461
3462 // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride)
3463 // ref += iclip(x, 0, iw - 1)
3464 sub x12, x3, #1 // ih - 1
3465 cmp x5, x3
3466 sub x13, x2, #1 // iw - 1
3467 csel x12, x12, x5, ge // min(y, ih - 1)
3468 cmp x4, x2
3469 bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0)
3470 csel x13, x13, x4, ge // min(x, iw - 1)
3471 bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0)
3472 madd x8, x12, x9, x8 // ref += iclip() * stride
3473 add x8, x8, x13, lsl #1 // ref += iclip()
3474
3475 // bottom_ext = iclip(y + bh - ih, 0, bh - 1)
3476 // top_ext = iclip(-y, 0, bh - 1)
3477 add x10, x5, x1 // y + bh
3478 neg x5, x5 // -y
3479 sub x10, x10, x3 // y + bh - ih
3480 sub x12, x1, #1 // bh - 1
3481 cmp x10, x1
3482 bic x5, x5, x5, asr #63 // max(-y, 0)
3483 csel x10, x10, x12, lt // min(y + bh - ih, bh-1)
3484 cmp x5, x1
3485 bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0)
3486 csel x5, x5, x12, lt // min(max(-y, 0), bh-1)
3487
3488 // right_ext = iclip(x + bw - iw, 0, bw - 1)
3489 // left_ext = iclip(-x, 0, bw - 1)
3490 add x11, x4, x0 // x + bw
3491 neg x4, x4 // -x
3492 sub x11, x11, x2 // x + bw - iw
3493 sub x13, x0, #1 // bw - 1
3494 cmp x11, x0
3495 bic x4, x4, x4, asr #63 // max(-x, 0)
3496 csel x11, x11, x13, lt // min(x + bw - iw, bw-1)
3497 cmp x4, x0
3498 bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0)
3499 csel x4, x4, x13, lt // min(max(-x, 0), bw - 1)
3500
3501 // center_h = bh - top_ext - bottom_ext
3502 // dst += top_ext * PXSTRIDE(dst_stride)
3503 // center_w = bw - left_ext - right_ext
3504 sub x1, x1, x5 // bh - top_ext
3505 madd x6, x5, x7, x6
3506 sub x2, x0, x4 // bw - left_ext
3507 sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext
3508 sub x2, x2, x11 // center_w = bw - left_ext - right_ext
3509
3510 mov x14, x6 // backup of dst
3511
3512 .macro v_loop need_left, need_right
3513 0:
3514 .if \need_left
3515 ld1r {v0.8h}, [x8]
3516 mov x12, x6 // out = dst
3517 mov x3, x4
3518 mov v1.16b, v0.16b
3519 1:
3520 subs x3, x3, #16
3521 st1 {v0.8h, v1.8h}, [x12], #32
3522 b.gt 1b
3523 .endif
3524 mov x13, x8
3525 add x12, x6, x4, lsl #1 // out = dst + left_ext
3526 mov x3, x2
3527 1:
3528 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x13], #64
3529 subs x3, x3, #32
3530 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64
3531 b.gt 1b
3532 .if \need_right
3533 add x3, x8, x2, lsl #1 // in + center_w
3534 sub x3, x3, #2 // in + center_w - 1
3535 add x12, x6, x4, lsl #1 // dst + left_ext
3536 ld1r {v0.8h}, [x3]
3537 add x12, x12, x2, lsl #1 // out = dst + left_ext + center_w
3538 mov x3, x11
3539 mov v1.16b, v0.16b
3540 1:
3541 subs x3, x3, #16
3542 st1 {v0.8h, v1.8h}, [x12], #32
3543 b.gt 1b
3544 .endif
3545
3546 subs x1, x1, #1 // center_h--
3547 add x6, x6, x7
3548 add x8, x8, x9
3549 b.gt 0b
3550 .endm
3551
3552 cbz x4, 2f
3553 // need_left
3554 cbz x11, 3f
3555 // need_left + need_right
3556 v_loop 1, 1
3557 b 5f
3558
3559 2:
3560 // !need_left
3561 cbz x11, 4f
3562 // !need_left + need_right
3563 v_loop 0, 1
3564 b 5f
3565
3566 3:
3567 // need_left + !need_right
3568 v_loop 1, 0
3569 b 5f
3570
3571 4:
3572 // !need_left + !need_right
3573 v_loop 0, 0
3574
3575 5:
3576
3577 cbz x10, 3f
3578 // need_bottom
3579 sub x8, x6, x7 // ref = dst - stride
3580 mov x4, x0
3581 1:
3582 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], #64
3583 mov x3, x10
3584 2:
3585 subs x3, x3, #1
3586 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7
3587 b.gt 2b
3588 msub x6, x7, x10, x6 // dst -= bottom_ext * stride
3589 subs x4, x4, #32 // bw -= 32
3590 add x6, x6, #64 // dst += 32
3591 b.gt 1b
3592
3593 3:
3594 cbz x5, 3f
3595 // need_top
3596 msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride
3597 1:
3598 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x14], #64
3599 mov x3, x5
3600 2:
3601 subs x3, x3, #1
3602 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7
3603 b.gt 2b
3604 msub x6, x7, x5, x6 // dst -= top_ext * stride
3605 subs x0, x0, #32 // bw -= 32
3606 add x6, x6, #64 // dst += 32
3607 b.gt 1b
3608
3609 3:
3610 ret
3611 endfunc