Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/looprestoration_tmpl.S

raw
1 /*
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2018, Martin Storsjo
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "src/arm/asm.S"
29
30 #define FILTER_OUT_STRIDE 384
31
32 .macro sgr_funcs bpc
33 // void dav1d_sgr_finish_filter1_Xbpc_neon(int16_t *tmp,
34 // const pixel *src, const ptrdiff_t stride,
35 // const int32_t *a, const int16_t *b,
36 // const int w, const int h);
37 function sgr_finish_filter1_\bpc\()bpc_neon, export=1
38 sub x7, x3, #(4*SUM_STRIDE)
39 add x8, x3, #(4*SUM_STRIDE)
40 sub x9, x4, #(2*SUM_STRIDE)
41 add x10, x4, #(2*SUM_STRIDE)
42 mov x11, #SUM_STRIDE
43 mov x12, #FILTER_OUT_STRIDE
44 add x13, x5, #7
45 bic x13, x13, #7 // Aligned width
46 .if \bpc == 8
47 sub x2, x2, x13
48 .else
49 sub x2, x2, x13, lsl #1
50 .endif
51 sub x12, x12, x13
52 sub x11, x11, x13
53 sub x11, x11, #4 // We read 4 extra elements from a
54 sub x14, x11, #4 // We read 8 extra elements from b
55 mov x13, x5
56 movi v6.8h, #3
57 movi v7.4s, #3
58 1:
59 ld1 {v0.8h, v1.8h}, [x9], #32
60 ld1 {v2.8h, v3.8h}, [x4], #32
61 ld1 {v4.8h, v5.8h}, [x10], #32
62 ld1 {v16.4s, v17.4s, v18.4s}, [x7], #48
63 ld1 {v19.4s, v20.4s, v21.4s}, [x3], #48
64 ld1 {v22.4s, v23.4s, v24.4s}, [x8], #48
65
66 2:
67 subs x5, x5, #8
68 ext v25.16b, v0.16b, v1.16b, #2 // -stride
69 ext v26.16b, v2.16b, v3.16b, #2 // 0
70 ext v27.16b, v4.16b, v5.16b, #2 // +stride
71 ext v28.16b, v0.16b, v1.16b, #4 // +1-stride
72 ext v29.16b, v2.16b, v3.16b, #4 // +1
73 ext v30.16b, v4.16b, v5.16b, #4 // +1+stride
74 add v2.8h, v2.8h, v25.8h // -1, -stride
75 add v26.8h, v26.8h, v27.8h // 0, +stride
76 add v0.8h, v0.8h, v28.8h // -1-stride, +1-stride
77 add v2.8h, v2.8h, v26.8h
78 add v4.8h, v4.8h, v30.8h // -1+stride, +1+stride
79 add v2.8h, v2.8h, v29.8h // +1
80 add v0.8h, v0.8h, v4.8h
81
82 ext v25.16b, v16.16b, v17.16b, #4 // -stride
83 ext v26.16b, v17.16b, v18.16b, #4
84 shl v2.8h, v2.8h, #2
85 ext v27.16b, v16.16b, v17.16b, #8 // +1-stride
86 ext v28.16b, v17.16b, v18.16b, #8
87 ext v29.16b, v19.16b, v20.16b, #4 // 0
88 ext v30.16b, v20.16b, v21.16b, #4
89 mla v2.8h, v0.8h, v6.8h // * 3 -> a
90 add v25.4s, v25.4s, v19.4s // -stride, -1
91 add v26.4s, v26.4s, v20.4s
92 add v16.4s, v16.4s, v27.4s // -1-stride, +1-stride
93 add v17.4s, v17.4s, v28.4s
94 ext v27.16b, v19.16b, v20.16b, #8 // +1
95 ext v28.16b, v20.16b, v21.16b, #8
96 add v16.4s, v16.4s, v22.4s // -1+stride
97 add v17.4s, v17.4s, v23.4s
98 add v29.4s, v29.4s, v27.4s // 0, +1
99 add v30.4s, v30.4s, v28.4s
100 add v25.4s, v25.4s, v29.4s
101 add v26.4s, v26.4s, v30.4s
102 ext v27.16b, v22.16b, v23.16b, #4 // +stride
103 ext v28.16b, v23.16b, v24.16b, #4
104 ext v29.16b, v22.16b, v23.16b, #8 // +1+stride
105 ext v30.16b, v23.16b, v24.16b, #8
106 .if \bpc == 8
107 ld1 {v19.8b}, [x1], #8 // src
108 .else
109 ld1 {v19.8h}, [x1], #16 // src
110 .endif
111 add v25.4s, v25.4s, v27.4s // +stride
112 add v26.4s, v26.4s, v28.4s
113 add v16.4s, v16.4s, v29.4s // +1+stride
114 add v17.4s, v17.4s, v30.4s
115 shl v25.4s, v25.4s, #2
116 shl v26.4s, v26.4s, #2
117 mla v25.4s, v16.4s, v7.4s // * 3 -> b
118 mla v26.4s, v17.4s, v7.4s
119 .if \bpc == 8
120 uxtl v19.8h, v19.8b // src
121 .endif
122 mov v0.16b, v1.16b
123 umlal v25.4s, v2.4h, v19.4h // b + a * src
124 umlal2 v26.4s, v2.8h, v19.8h
125 mov v2.16b, v3.16b
126 rshrn v25.4h, v25.4s, #9
127 rshrn2 v25.8h, v26.4s, #9
128 mov v4.16b, v5.16b
129 st1 {v25.8h}, [x0], #16
130
131 b.le 3f
132 mov v16.16b, v18.16b
133 mov v19.16b, v21.16b
134 mov v22.16b, v24.16b
135 ld1 {v1.8h}, [x9], #16
136 ld1 {v3.8h}, [x4], #16
137 ld1 {v5.8h}, [x10], #16
138 ld1 {v17.4s, v18.4s}, [x7], #32
139 ld1 {v20.4s, v21.4s}, [x3], #32
140 ld1 {v23.4s, v24.4s}, [x8], #32
141 b 2b
142
143 3:
144 subs x6, x6, #1
145 b.le 0f
146 mov x5, x13
147 add x0, x0, x12, lsl #1
148 add x1, x1, x2
149 add x3, x3, x11, lsl #2
150 add x7, x7, x11, lsl #2
151 add x8, x8, x11, lsl #2
152 add x4, x4, x14, lsl #1
153 add x9, x9, x14, lsl #1
154 add x10, x10, x14, lsl #1
155 b 1b
156 0:
157 ret
158 endfunc
159
160 // void dav1d_sgr_finish_filter2_Xbpc_neon(int16_t *tmp,
161 // const pixel *src, const ptrdiff_t stride,
162 // const int32_t *a, const int16_t *b,
163 // const int w, const int h);
164 function sgr_finish_filter2_\bpc\()bpc_neon, export=1
165 add x7, x3, #(4*(SUM_STRIDE))
166 sub x3, x3, #(4*(SUM_STRIDE))
167 add x8, x4, #(2*(SUM_STRIDE))
168 sub x4, x4, #(2*(SUM_STRIDE))
169 mov x9, #(2*SUM_STRIDE)
170 mov x10, #FILTER_OUT_STRIDE
171 add x11, x5, #7
172 bic x11, x11, #7 // Aligned width
173 .if \bpc == 8
174 sub x2, x2, x11
175 .else
176 sub x2, x2, x11, lsl #1
177 .endif
178 sub x10, x10, x11
179 sub x9, x9, x11
180 sub x9, x9, #4 // We read 4 extra elements from a
181 sub x12, x9, #4 // We read 8 extra elements from b
182 mov x11, x5
183 movi v4.8h, #5
184 movi v5.4s, #5
185 movi v6.8h, #6
186 movi v7.4s, #6
187 1:
188 ld1 {v0.8h, v1.8h}, [x4], #32
189 ld1 {v2.8h, v3.8h}, [x8], #32
190 ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48
191 ld1 {v19.4s, v20.4s, v21.4s}, [x7], #48
192
193 2:
194 subs x5, x5, #8
195 ext v24.16b, v0.16b, v1.16b, #4 // +1-stride
196 ext v25.16b, v2.16b, v3.16b, #4 // +1+stride
197 ext v22.16b, v0.16b, v1.16b, #2 // -stride
198 ext v23.16b, v2.16b, v3.16b, #2 // +stride
199 add v0.8h, v0.8h, v24.8h // -1-stride, +1-stride
200 add v25.8h, v2.8h, v25.8h // -1+stride, +1+stride
201 add v2.8h, v22.8h, v23.8h // -stride, +stride
202 add v0.8h, v0.8h, v25.8h
203
204 ext v22.16b, v16.16b, v17.16b, #4 // -stride
205 ext v23.16b, v17.16b, v18.16b, #4
206 ext v24.16b, v19.16b, v20.16b, #4 // +stride
207 ext v25.16b, v20.16b, v21.16b, #4
208 ext v26.16b, v16.16b, v17.16b, #8 // +1-stride
209 ext v27.16b, v17.16b, v18.16b, #8
210 ext v28.16b, v19.16b, v20.16b, #8 // +1+stride
211 ext v29.16b, v20.16b, v21.16b, #8
212 mul v0.8h, v0.8h, v4.8h // * 5
213 mla v0.8h, v2.8h, v6.8h // * 6
214 .if \bpc == 8
215 ld1 {v31.8b}, [x1], #8
216 .else
217 ld1 {v31.8h}, [x1], #16
218 .endif
219 add v16.4s, v16.4s, v26.4s // -1-stride, +1-stride
220 add v17.4s, v17.4s, v27.4s
221 add v19.4s, v19.4s, v28.4s // -1+stride, +1+stride
222 add v20.4s, v20.4s, v29.4s
223 add v16.4s, v16.4s, v19.4s
224 add v17.4s, v17.4s, v20.4s
225
226 add v22.4s, v22.4s, v24.4s // -stride, +stride
227 add v23.4s, v23.4s, v25.4s
228 // This is, surprisingly, faster than other variants where the
229 // mul+mla pairs are further apart, on Cortex A53.
230 mul v16.4s, v16.4s, v5.4s // * 5
231 mla v16.4s, v22.4s, v7.4s // * 6
232 mul v17.4s, v17.4s, v5.4s // * 5
233 mla v17.4s, v23.4s, v7.4s // * 6
234
235 .if \bpc == 8
236 uxtl v31.8h, v31.8b
237 .endif
238 umlal v16.4s, v0.4h, v31.4h // b + a * src
239 umlal2 v17.4s, v0.8h, v31.8h
240 mov v0.16b, v1.16b
241 rshrn v16.4h, v16.4s, #9
242 rshrn2 v16.8h, v17.4s, #9
243 mov v2.16b, v3.16b
244 st1 {v16.8h}, [x0], #16
245
246 b.le 3f
247 mov v16.16b, v18.16b
248 mov v19.16b, v21.16b
249 ld1 {v1.8h}, [x4], #16
250 ld1 {v3.8h}, [x8], #16
251 ld1 {v17.4s, v18.4s}, [x3], #32
252 ld1 {v20.4s, v21.4s}, [x7], #32
253 b 2b
254
255 3:
256 subs x6, x6, #1
257 b.le 0f
258 mov x5, x11
259 add x0, x0, x10, lsl #1
260 add x1, x1, x2
261 add x3, x3, x9, lsl #2
262 add x7, x7, x9, lsl #2
263 add x4, x4, x12, lsl #1
264 add x8, x8, x12, lsl #1
265 mov x13, x3
266 mov x14, x4
267
268 ld1 {v0.8h, v1.8h}, [x4], #32
269 ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48
270
271 4:
272 subs x5, x5, #8
273 ext v23.16b, v0.16b, v1.16b, #4 // +1
274 ext v22.16b, v0.16b, v1.16b, #2 // 0
275 add v0.8h, v0.8h, v23.8h // -1, +1
276
277 ext v24.16b, v16.16b, v17.16b, #4 // 0
278 ext v25.16b, v17.16b, v18.16b, #4
279 ext v26.16b, v16.16b, v17.16b, #8 // +1
280 ext v27.16b, v17.16b, v18.16b, #8
281 mul v2.8h, v22.8h, v6.8h // * 6
282 mla v2.8h, v0.8h, v4.8h // * 5 -> a
283 .if \bpc == 8
284 ld1 {v31.8b}, [x1], #8
285 .else
286 ld1 {v31.8h}, [x1], #16
287 .endif
288 add v16.4s, v16.4s, v26.4s // -1, +1
289 add v17.4s, v17.4s, v27.4s
290 .if \bpc == 8
291 uxtl v31.8h, v31.8b
292 .endif
293 // This is, surprisingly, faster than other variants where the
294 // mul+mla pairs are further apart, on Cortex A53.
295 mul v24.4s, v24.4s, v7.4s // * 6
296 mla v24.4s, v16.4s, v5.4s // * 5 -> b
297 mul v25.4s, v25.4s, v7.4s // * 6
298 mla v25.4s, v17.4s, v5.4s // * 5 -> b
299
300 umlal v24.4s, v2.4h, v31.4h // b + a * src
301 umlal2 v25.4s, v2.8h, v31.8h
302 mov v0.16b, v1.16b
303 rshrn v24.4h, v24.4s, #8
304 rshrn2 v24.8h, v25.4s, #8
305 mov v16.16b, v18.16b
306 st1 {v24.8h}, [x0], #16
307
308 b.le 5f
309 ld1 {v1.8h}, [x4], #16
310 ld1 {v17.4s, v18.4s}, [x3], #32
311 b 4b
312
313 5:
314 subs x6, x6, #1
315 b.le 0f
316 mov x5, x11
317 add x0, x0, x10, lsl #1
318 add x1, x1, x2
319 mov x3, x13 // Rewind x3/x4 to where they started
320 mov x4, x14
321 b 1b
322 0:
323 ret
324 endfunc
325
326 // void dav1d_sgr_weighted1_Xbpc_neon(pixel *dst, const ptrdiff_t dst_stride,
327 // const pixel *src, const ptrdiff_t src_stride,
328 // const int16_t *t1, const int w, const int h,
329 // const int wt, const int bitdepth_max);
330 function sgr_weighted1_\bpc\()bpc_neon, export=1
331 .if \bpc == 16
332 ldr w8, [sp]
333 .endif
334 dup v31.8h, w7
335 cmp x6, #2
336 .if \bpc == 16
337 dup v30.8h, w8
338 .endif
339 add x9, x0, x1
340 add x10, x2, x3
341 add x11, x4, #2*FILTER_OUT_STRIDE
342 mov x7, #(4*FILTER_OUT_STRIDE)
343 lsl x1, x1, #1
344 lsl x3, x3, #1
345 add x8, x5, #7
346 bic x8, x8, #7 // Aligned width
347 .if \bpc == 8
348 sub x1, x1, x8
349 sub x3, x3, x8
350 .else
351 sub x1, x1, x8, lsl #1
352 sub x3, x3, x8, lsl #1
353 .endif
354 sub x7, x7, x8, lsl #1
355 mov x8, x5
356 b.lt 2f
357 1:
358 .if \bpc == 8
359 ld1 {v0.8b}, [x2], #8
360 ld1 {v4.8b}, [x10], #8
361 .else
362 ld1 {v0.8h}, [x2], #16
363 ld1 {v4.8h}, [x10], #16
364 .endif
365 ld1 {v1.8h}, [x4], #16
366 ld1 {v5.8h}, [x11], #16
367 subs x5, x5, #8
368 .if \bpc == 8
369 ushll v0.8h, v0.8b, #4 // u
370 ushll v4.8h, v4.8b, #4 // u
371 .else
372 shl v0.8h, v0.8h, #4 // u
373 shl v4.8h, v4.8h, #4 // u
374 .endif
375 sub v1.8h, v1.8h, v0.8h // t1 - u
376 sub v5.8h, v5.8h, v4.8h // t1 - u
377 ushll v2.4s, v0.4h, #7 // u << 7
378 ushll2 v3.4s, v0.8h, #7 // u << 7
379 ushll v6.4s, v4.4h, #7 // u << 7
380 ushll2 v7.4s, v4.8h, #7 // u << 7
381 smlal v2.4s, v1.4h, v31.4h // v
382 smlal2 v3.4s, v1.8h, v31.8h // v
383 smlal v6.4s, v5.4h, v31.4h // v
384 smlal2 v7.4s, v5.8h, v31.8h // v
385 .if \bpc == 8
386 rshrn v2.4h, v2.4s, #11
387 rshrn2 v2.8h, v3.4s, #11
388 rshrn v6.4h, v6.4s, #11
389 rshrn2 v6.8h, v7.4s, #11
390 sqxtun v2.8b, v2.8h
391 sqxtun v6.8b, v6.8h
392 st1 {v2.8b}, [x0], #8
393 st1 {v6.8b}, [x9], #8
394 .else
395 sqrshrun v2.4h, v2.4s, #11
396 sqrshrun2 v2.8h, v3.4s, #11
397 sqrshrun v6.4h, v6.4s, #11
398 sqrshrun2 v6.8h, v7.4s, #11
399 umin v2.8h, v2.8h, v30.8h
400 umin v6.8h, v6.8h, v30.8h
401 st1 {v2.8h}, [x0], #16
402 st1 {v6.8h}, [x9], #16
403 .endif
404 b.gt 1b
405
406 sub x6, x6, #2
407 cmp x6, #1
408 b.lt 0f
409 mov x5, x8
410 add x0, x0, x1
411 add x9, x9, x1
412 add x2, x2, x3
413 add x10, x10, x3
414 add x4, x4, x7
415 add x11, x11, x7
416 b.eq 2f
417 b 1b
418
419 2:
420 .if \bpc == 8
421 ld1 {v0.8b}, [x2], #8
422 .else
423 ld1 {v0.8h}, [x2], #16
424 .endif
425 ld1 {v1.8h}, [x4], #16
426 subs x5, x5, #8
427 .if \bpc == 8
428 ushll v0.8h, v0.8b, #4 // u
429 .else
430 shl v0.8h, v0.8h, #4 // u
431 .endif
432 sub v1.8h, v1.8h, v0.8h // t1 - u
433 ushll v2.4s, v0.4h, #7 // u << 7
434 ushll2 v3.4s, v0.8h, #7 // u << 7
435 smlal v2.4s, v1.4h, v31.4h // v
436 smlal2 v3.4s, v1.8h, v31.8h // v
437 .if \bpc == 8
438 rshrn v2.4h, v2.4s, #11
439 rshrn2 v2.8h, v3.4s, #11
440 sqxtun v2.8b, v2.8h
441 st1 {v2.8b}, [x0], #8
442 .else
443 sqrshrun v2.4h, v2.4s, #11
444 sqrshrun2 v2.8h, v3.4s, #11
445 umin v2.8h, v2.8h, v30.8h
446 st1 {v2.8h}, [x0], #16
447 .endif
448 b.gt 2b
449 0:
450 ret
451 endfunc
452
453 // void dav1d_sgr_weighted2_Xbpc_neon(pixel *dst, const ptrdiff_t stride,
454 // const pixel *src, const ptrdiff_t src_stride,
455 // const int16_t *t1, const int16_t *t2,
456 // const int w, const int h,
457 // const int16_t wt[2], const int bitdepth_max);
458 function sgr_weighted2_\bpc\()bpc_neon, export=1
459 .if \bpc == 8
460 ldr x8, [sp]
461 .else
462 ldp x8, x9, [sp]
463 .endif
464 cmp x7, #2
465 add x10, x0, x1
466 add x11, x2, x3
467 add x12, x4, #2*FILTER_OUT_STRIDE
468 add x13, x5, #2*FILTER_OUT_STRIDE
469 ld2r {v30.8h, v31.8h}, [x8] // wt[0], wt[1]
470 .if \bpc == 16
471 dup v29.8h, w9
472 .endif
473 mov x8, #4*FILTER_OUT_STRIDE
474 lsl x1, x1, #1
475 lsl x3, x3, #1
476 add x9, x6, #7
477 bic x9, x9, #7 // Aligned width
478 .if \bpc == 8
479 sub x1, x1, x9
480 sub x3, x3, x9
481 .else
482 sub x1, x1, x9, lsl #1
483 sub x3, x3, x9, lsl #1
484 .endif
485 sub x8, x8, x9, lsl #1
486 mov x9, x6
487 b.lt 2f
488 1:
489 .if \bpc == 8
490 ld1 {v0.8b}, [x2], #8
491 ld1 {v16.8b}, [x11], #8
492 .else
493 ld1 {v0.8h}, [x2], #16
494 ld1 {v16.8h}, [x11], #16
495 .endif
496 ld1 {v1.8h}, [x4], #16
497 ld1 {v17.8h}, [x12], #16
498 ld1 {v2.8h}, [x5], #16
499 ld1 {v18.8h}, [x13], #16
500 subs x6, x6, #8
501 .if \bpc == 8
502 ushll v0.8h, v0.8b, #4 // u
503 ushll v16.8h, v16.8b, #4 // u
504 .else
505 shl v0.8h, v0.8h, #4 // u
506 shl v16.8h, v16.8h, #4 // u
507 .endif
508 sub v1.8h, v1.8h, v0.8h // t1 - u
509 sub v2.8h, v2.8h, v0.8h // t2 - u
510 sub v17.8h, v17.8h, v16.8h // t1 - u
511 sub v18.8h, v18.8h, v16.8h // t2 - u
512 ushll v3.4s, v0.4h, #7 // u << 7
513 ushll2 v4.4s, v0.8h, #7 // u << 7
514 ushll v19.4s, v16.4h, #7 // u << 7
515 ushll2 v20.4s, v16.8h, #7 // u << 7
516 smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u)
517 smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u)
518 smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u)
519 smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u)
520 smlal v19.4s, v17.4h, v30.4h // wt[0] * (t1 - u)
521 smlal v19.4s, v18.4h, v31.4h // wt[1] * (t2 - u)
522 smlal2 v20.4s, v17.8h, v30.8h // wt[0] * (t1 - u)
523 smlal2 v20.4s, v18.8h, v31.8h // wt[1] * (t2 - u)
524 .if \bpc == 8
525 rshrn v3.4h, v3.4s, #11
526 rshrn2 v3.8h, v4.4s, #11
527 rshrn v19.4h, v19.4s, #11
528 rshrn2 v19.8h, v20.4s, #11
529 sqxtun v3.8b, v3.8h
530 sqxtun v19.8b, v19.8h
531 st1 {v3.8b}, [x0], #8
532 st1 {v19.8b}, [x10], #8
533 .else
534 sqrshrun v3.4h, v3.4s, #11
535 sqrshrun2 v3.8h, v4.4s, #11
536 sqrshrun v19.4h, v19.4s, #11
537 sqrshrun2 v19.8h, v20.4s, #11
538 umin v3.8h, v3.8h, v29.8h
539 umin v19.8h, v19.8h, v29.8h
540 st1 {v3.8h}, [x0], #16
541 st1 {v19.8h}, [x10], #16
542 .endif
543 b.gt 1b
544
545 subs x7, x7, #2
546 cmp x7, #1
547 b.lt 0f
548 mov x6, x9
549 add x0, x0, x1
550 add x10, x10, x1
551 add x2, x2, x3
552 add x11, x11, x3
553 add x4, x4, x8
554 add x12, x12, x8
555 add x5, x5, x8
556 add x13, x13, x8
557 b.eq 2f
558 b 1b
559
560 2:
561 .if \bpc == 8
562 ld1 {v0.8b}, [x2], #8
563 .else
564 ld1 {v0.8h}, [x2], #16
565 .endif
566 ld1 {v1.8h}, [x4], #16
567 ld1 {v2.8h}, [x5], #16
568 subs x6, x6, #8
569 .if \bpc == 8
570 ushll v0.8h, v0.8b, #4 // u
571 .else
572 shl v0.8h, v0.8h, #4 // u
573 .endif
574 sub v1.8h, v1.8h, v0.8h // t1 - u
575 sub v2.8h, v2.8h, v0.8h // t2 - u
576 ushll v3.4s, v0.4h, #7 // u << 7
577 ushll2 v4.4s, v0.8h, #7 // u << 7
578 smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u)
579 smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u)
580 smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u)
581 smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u)
582 .if \bpc == 8
583 rshrn v3.4h, v3.4s, #11
584 rshrn2 v3.8h, v4.4s, #11
585 sqxtun v3.8b, v3.8h
586 st1 {v3.8b}, [x0], #8
587 .else
588 sqrshrun v3.4h, v3.4s, #11
589 sqrshrun2 v3.8h, v4.4s, #11
590 umin v3.8h, v3.8h, v29.8h
591 st1 {v3.8h}, [x0], #16
592 .endif
593 b.gt 1b
594 0:
595 ret
596 endfunc
597 .endm