Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/looprestoration16.S

raw
1 /*
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2020, Martin Storsjo
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "src/arm/asm.S"
29 #include "util.S"
30
31 const right_ext_mask_buf
32 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
33 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
34 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
35 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
36 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
37 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
38 right_ext_mask:
39 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
40 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
41 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
42 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
43 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
44 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
45 endconst
46
47 // void dav1d_wiener_filter7_16bpc_neon(pixel *p, const ptrdiff_t p_stride,
48 // const pixel (*left)[4], const pixel *lpf,
49 // const int w, int h,
50 // const int16_t filter[2][8],
51 // const enum LrEdgeFlags edges,
52 // const int bitdepth_max);
53 function wiener_filter7_16bpc_neon, export=1
54 ldr w8, [sp]
55 AARCH64_SIGN_LINK_REGISTER
56 stp x29, x30, [sp, #-32]!
57 stp d8, d9, [sp, #16]
58 mov x29, sp
59 ld1 {v0.8h, v1.8h}, [x6]
60 tst w7, #4 // LR_HAVE_TOP
61 sub_sp 384*2*6
62
63 dup v28.8h, w8 // bitdepth_max
64 clz w8, w8
65 movi v30.4s, #1
66 sub w10, w8, #38 // -(bitdepth + 6)
67 sub w11, w8, #11 // round_bits_v
68 sub w8, w8, #25 // -round_bits_h
69 neg w10, w10 // bitdepth + 6
70 neg w11, w11 // -round_bits_v
71 dup v2.4s, w10
72 dup v29.4s, w8 // -round_bits_h
73 dup v27.4s, w11 // -round_bits_v
74 movi v31.8h, #0x20, lsl #8 // 1 << 13 = 8192
75 ushl v30.4s, v30.4s, v2.4s // 1 << (bitdepth + 6)
76
77 zip1 v0.2d, v0.2d, v1.2d // move vertical coeffs to v0.h[4-7], freeing up v1
78
79 // x9 - t6
80 // x10 - t5
81 // x11 - t4
82 // x12 - t3
83 // x13 - t2
84 // x14 - t1
85 // x15 - t0
86 mov x14, sp // t1
87 b.eq L(no_top_7)
88
89 mov x16, x2 // backup left
90 mov x2, #0
91 bl wiener_filter7_h_16bpc_neon
92 add x3, x3, x1 // lpf += stride
93 mov x9, x14 // t6
94 mov x10, x14 // t5
95 add x14, x14, #384*2 // t1 += 384*2
96 bl wiener_filter7_h_16bpc_neon
97 add x3, x3, x1, lsl #2
98 add x3, x3, x1 // lpf += stride*5
99 mov x11, x14 // t4
100 add x14, x14, #384*2 // t1 += 384*2
101 mov x2, x16 // left
102 mov x16, x3 // backup lpf
103 mov x3, x0 // lpf = p
104 bl wiener_filter7_h_16bpc_neon
105 subs w5, w5, #1 // h--
106 mov x12, x14 // t3
107 mov x13, x14 // t2
108 b.eq L(v1_7)
109 add x3, x3, x1 // src += stride
110 add x14, x14, #384*2 // t1 += 384*2
111 bl wiener_filter7_h_16bpc_neon
112 mov x13, x14 // t2
113 subs w5, w5, #1 // h--
114 b.eq L(v2_7)
115 add x3, x3, x1 // src += stride
116 add x14, x14, #384*2 // t1 += 384*2
117 bl wiener_filter7_h_16bpc_neon
118 subs w5, w5, #1 // h--
119 b.eq L(v3_7)
120 add x3, x3, x1 // src += stride
121
122 L(main_7):
123 add x15, x14, #384*2 // t0 = t1 + 384*2
124 L(main_loop_7):
125 bl wiener_filter7_hv_16bpc_neon
126 subs w5, w5, #1 // h--
127 b.ne L(main_loop_7)
128 tst w7, #8 // LR_HAVE_BOTTOM
129 b.eq L(v3_7)
130
131 mov x3, x16 // restore lpf
132 mov x2, #0 // left = NULL
133 bl wiener_filter7_hv_16bpc_neon
134 bl wiener_filter7_hv_16bpc_neon
135 L(v1_7):
136 bl wiener_filter7_v_16bpc_neon
137
138 mov sp, x29
139 ldp d8, d9, [sp, #16]
140 ldp x29, x30, [sp], #32
141 AARCH64_VALIDATE_LINK_REGISTER
142 ret
143
144 L(no_top_7):
145 add x3, x3, x1, lsl #2
146 add x16, x3, x1, lsl #1 // lpf += stride*6, backup
147 mov x3, x0 // lpf = p
148
149 bl wiener_filter7_h_16bpc_neon
150 subs w5, w5, #1 // h--
151 mov x9, x14 // t6
152 mov x10, x14 // t5
153 mov x11, x14 // t4
154 mov x12, x14 // t3
155 mov x13, x14 // t2
156 b.eq L(v1_7)
157 add x3, x3, x1 // src += p_stride
158 add x14, x14, #384*2 // t1 += 384*2
159 bl wiener_filter7_h_16bpc_neon
160 subs w5, w5, #1 // h--
161 mov x13, x14 // t2
162 b.eq L(v2_7)
163 add x3, x3, x1 // src += p_stride
164 add x14, x14, #384*2 // t1 += 384*2
165 bl wiener_filter7_h_16bpc_neon
166 subs w5, w5, #1 // h--
167 b.eq L(v3_7)
168 add x3, x3, x1 // src += p_stride
169 add x15, x14, #384*2 // t0 = t1 + 384*2
170 bl wiener_filter7_hv_16bpc_neon
171 subs w5, w5, #1 // h--
172 b.eq L(v3_7)
173 add x15, x15, #384*2*4 // t0 += 384*2*4
174 bl wiener_filter7_hv_16bpc_neon
175 subs w5, w5, #1 // h--
176 b.ne L(main_7)
177 L(v3_7):
178 bl wiener_filter7_v_16bpc_neon
179 L(v2_7):
180 bl wiener_filter7_v_16bpc_neon
181 b L(v1_7)
182 endfunc
183
184
185 function wiener_filter7_h_16bpc_neon
186 stp x3, x4, [sp, #-32]!
187 str x14, [sp, #16]
188
189 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
190 tst w7, #1 // LR_HAVE_LEFT
191 b.eq 1f
192 // LR_HAVE_LEFT
193 cbnz x2, 0f
194 // left == NULL
195 sub x3, x3, #6
196 ld1 {v2.8h, v3.8h}, [x3], #32
197 b 2f
198
199 0:
200 // LR_HAVE_LEFT, left != NULL
201 ld1 {v2.8h, v3.8h}, [x3], #32
202 ld1 {v4.d}[1], [x2], #8
203 // Move x3 back to account for the last 3 pixels we loaded earlier,
204 // which we'll shift out.
205 sub x3, x3, #6
206 ext v3.16b, v2.16b, v3.16b, #10
207 ext v2.16b, v4.16b, v2.16b, #10
208 b 2f
209
210 1:
211 ld1 {v2.8h, v3.8h}, [x3], #32
212 // !LR_HAVE_LEFT, fill v4 with the leftmost pixel
213 // and shift v3 to have 3x the first pixel at the front.
214 dup v4.8h, v2.h[0]
215 // Move x3 back to account for the last 3 pixels we loaded before,
216 // which we shifted out.
217 sub x3, x3, #6
218 ext v3.16b, v2.16b, v3.16b, #10
219 ext v2.16b, v4.16b, v2.16b, #10
220
221 2:
222 ld1 {v4.8h}, [x3], #16
223
224 tst w7, #2 // LR_HAVE_RIGHT
225 b.ne 4f
226
227 3: // !LR_HAVE_RIGHT
228
229 // Check whether we need to pad the right edge
230 cmp w4, #19
231 b.ge 4f // If w >= 19, all used input pixels are valid
232
233 // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9,
234 // this ends up called again; it's not strictly needed in those
235 // cases (we pad enough here), but keeping the code as simple as possible.
236
237 // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
238 // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel.
239 sub w17, w4, #22
240 // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the
241 // buffer pointer.
242 movrel x6, right_ext_mask, -6
243 ldr h26, [x3, w17, sxtw #1]
244 sub x6, x6, w4, uxtw #1
245 dup v26.8h, v26.h[0]
246 ld1 {v23.16b, v24.16b, v25.16b}, [x6]
247
248 bit v2.16b, v26.16b, v23.16b
249 bit v3.16b, v26.16b, v24.16b
250 bit v4.16b, v26.16b, v25.16b
251
252 4: // Loop horizontally
253 // Interleaving the mul/mla chains actually hurts performance
254 // significantly on Cortex A53, thus keeping mul/mla tightly
255 // chained like this.
256 ext v17.16b, v2.16b, v3.16b, #4
257 ext v19.16b, v2.16b, v3.16b, #8
258 ext v16.16b, v2.16b, v3.16b, #2
259 ext v20.16b, v2.16b, v3.16b, #10
260 ext v21.16b, v2.16b, v3.16b, #12
261 ext v18.16b, v2.16b, v3.16b, #6
262 add v19.8h, v19.8h, v17.8h
263 add v20.8h, v20.8h, v16.8h
264 add v21.8h, v21.8h, v2.8h
265 smull v6.4s, v18.4h, v0.h[3]
266 smlal v6.4s, v19.4h, v0.h[2]
267 smlal v6.4s, v20.4h, v0.h[1]
268 smlal v6.4s, v21.4h, v0.h[0]
269 smull2 v7.4s, v18.8h, v0.h[3]
270 smlal2 v7.4s, v19.8h, v0.h[2]
271 smlal2 v7.4s, v20.8h, v0.h[1]
272 smlal2 v7.4s, v21.8h, v0.h[0]
273
274 ext v17.16b, v3.16b, v4.16b, #4
275 ext v19.16b, v3.16b, v4.16b, #8
276 ext v16.16b, v3.16b, v4.16b, #2
277 ext v20.16b, v3.16b, v4.16b, #10
278 ext v21.16b, v3.16b, v4.16b, #12
279 ext v18.16b, v3.16b, v4.16b, #6
280
281 add v19.8h, v19.8h, v17.8h
282 add v20.8h, v20.8h, v16.8h
283 add v21.8h, v21.8h, v3.8h
284 smull v16.4s, v18.4h, v0.h[3]
285 smlal v16.4s, v19.4h, v0.h[2]
286 smlal v16.4s, v20.4h, v0.h[1]
287 smlal v16.4s, v21.4h, v0.h[0]
288 smull2 v17.4s, v18.8h, v0.h[3]
289 smlal2 v17.4s, v19.8h, v0.h[2]
290 smlal2 v17.4s, v20.8h, v0.h[1]
291 smlal2 v17.4s, v21.8h, v0.h[0]
292
293 mvni v24.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
294 add v6.4s, v6.4s, v30.4s
295 add v7.4s, v7.4s, v30.4s
296 add v16.4s, v16.4s, v30.4s
297 add v17.4s, v17.4s, v30.4s
298 srshl v6.4s, v6.4s, v29.4s
299 srshl v7.4s, v7.4s, v29.4s
300 srshl v16.4s, v16.4s, v29.4s
301 srshl v17.4s, v17.4s, v29.4s
302 sqxtun v6.4h, v6.4s
303 sqxtun2 v6.8h, v7.4s
304 sqxtun v7.4h, v16.4s
305 sqxtun2 v7.8h, v17.4s
306 umin v6.8h, v6.8h, v24.8h
307 umin v7.8h, v7.8h, v24.8h
308 sub v6.8h, v6.8h, v31.8h
309 sub v7.8h, v7.8h, v31.8h
310
311 subs w4, w4, #16
312
313 st1 {v6.8h, v7.8h}, [x14], #32
314
315 b.le 0f
316 mov v2.16b, v4.16b
317 tst w7, #2 // LR_HAVE_RIGHT
318 ld1 {v3.8h, v4.8h}, [x3], #32
319 b.ne 4b // If we don't need to pad, just keep filtering.
320 b 3b // If we need to pad, check how many pixels we have left.
321
322 0:
323 ldr x14, [sp, #16]
324 ldp x3, x4, [sp], #32
325 ret
326 endfunc
327
328 function wiener_filter7_v_16bpc_neon
329 // Backing up/restoring registers shifted, so that x9 gets the value
330 // of x10, etc, afterwards.
331 stp x10, x11, [sp, #-64]!
332 stp x12, x13, [sp, #16]
333 stp x14, x14, [sp, #32]
334 stp x0, x4, [sp, #48]
335 1:
336 ld1 {v16.8h, v17.8h}, [x9], #32
337 ld1 {v18.8h, v19.8h}, [x10], #32
338 ld1 {v20.8h, v21.8h}, [x11], #32
339 ld1 {v22.8h, v23.8h}, [x12], #32
340 ld1 {v24.8h, v25.8h}, [x13], #32
341 ld1 {v6.8h, v7.8h}, [x14], #32
342
343 smull v2.4s, v16.4h, v0.h[4]
344 smlal v2.4s, v18.4h, v0.h[5]
345 smlal v2.4s, v20.4h, v0.h[6]
346 smlal v2.4s, v22.4h, v0.h[7]
347 smlal v2.4s, v24.4h, v0.h[6]
348 smlal v2.4s, v6.4h, v0.h[5]
349 smlal v2.4s, v6.4h, v0.h[4]
350 smull2 v3.4s, v16.8h, v0.h[4]
351 smlal2 v3.4s, v18.8h, v0.h[5]
352 smlal2 v3.4s, v20.8h, v0.h[6]
353 smlal2 v3.4s, v22.8h, v0.h[7]
354 smlal2 v3.4s, v24.8h, v0.h[6]
355 smlal2 v3.4s, v6.8h, v0.h[5]
356 smlal2 v3.4s, v6.8h, v0.h[4]
357 smull v4.4s, v17.4h, v0.h[4]
358 smlal v4.4s, v19.4h, v0.h[5]
359 smlal v4.4s, v21.4h, v0.h[6]
360 smlal v4.4s, v23.4h, v0.h[7]
361 smlal v4.4s, v25.4h, v0.h[6]
362 smlal v4.4s, v7.4h, v0.h[5]
363 smlal v4.4s, v7.4h, v0.h[4]
364 smull2 v5.4s, v17.8h, v0.h[4]
365 smlal2 v5.4s, v19.8h, v0.h[5]
366 smlal2 v5.4s, v21.8h, v0.h[6]
367 smlal2 v5.4s, v23.8h, v0.h[7]
368 smlal2 v5.4s, v25.8h, v0.h[6]
369 smlal2 v5.4s, v7.8h, v0.h[5]
370 smlal2 v5.4s, v7.8h, v0.h[4]
371 srshl v2.4s, v2.4s, v27.4s // -round_bits_v
372 srshl v3.4s, v3.4s, v27.4s
373 srshl v4.4s, v4.4s, v27.4s
374 srshl v5.4s, v5.4s, v27.4s
375 sqxtun v2.4h, v2.4s
376 sqxtun2 v2.8h, v3.4s
377 sqxtun v3.4h, v4.4s
378 sqxtun2 v3.8h, v5.4s
379 umin v2.8h, v2.8h, v28.8h // bitdepth_max
380 umin v3.8h, v3.8h, v28.8h
381 subs w4, w4, #16
382 st1 {v2.8h, v3.8h}, [x0], #32
383 b.gt 1b
384
385 ldp x0, x4, [sp, #48]
386 ldp x13, x14, [sp, #32]
387 ldp x11, x12, [sp, #16]
388 ldp x9, x10, [sp], #64
389
390 add x0, x0, x1
391 ret
392 endfunc
393
394 function wiener_filter7_hv_16bpc_neon
395 // Backing up/restoring registers shifted, so that x9 gets the value
396 // of x10, etc, and x15==x9, afterwards.
397 stp x10, x11, [sp, #-80]!
398 stp x12, x13, [sp, #16]
399 stp x14, x15, [sp, #32]
400 stp x10, x0, [sp, #48]
401 stp x3, x4, [sp, #64]
402
403 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
404 tst w7, #1 // LR_HAVE_LEFT
405 b.eq 1f
406 // LR_HAVE_LEFT
407 cbnz x2, 0f
408 // left == NULL
409 sub x3, x3, #6
410 ld1 {v2.8h, v3.8h}, [x3], #32
411 b 2f
412
413 0:
414 // LR_HAVE_LEFT, left != NULL
415 ld1 {v2.8h, v3.8h}, [x3], #32
416 ld1 {v4.d}[1], [x2], #8
417 // Move x3 back to account for the last 3 pixels we loaded earlier,
418 // which we'll shift out.
419 sub x3, x3, #6
420 ext v3.16b, v2.16b, v3.16b, #10
421 ext v2.16b, v4.16b, v2.16b, #10
422 b 2f
423 1:
424 ld1 {v2.8h, v3.8h}, [x3], #32
425 // !LR_HAVE_LEFT, fill v4 with the leftmost pixel
426 // and shift v3 to have 3x the first pixel at the front.
427 dup v4.8h, v2.h[0]
428 // Move x3 back to account for the last 3 pixels we loaded before,
429 // which we shifted out.
430 sub x3, x3, #6
431 ext v3.16b, v2.16b, v3.16b, #10
432 ext v2.16b, v4.16b, v2.16b, #10
433
434 2:
435 ld1 {v4.8h}, [x3], #16
436
437 tst w7, #2 // LR_HAVE_RIGHT
438 b.ne 4f
439
440 3: // !LR_HAVE_RIGHT
441
442 // Check whether we need to pad the right edge
443 cmp w4, #19
444 b.ge 4f // If w >= 19, all used input pixels are valid
445
446 // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9,
447 // this ends up called again; it's not strictly needed in those
448 // cases (we pad enough here), but keeping the code as simple as possible.
449
450 // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
451 // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel.
452 sub w17, w4, #22
453 // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the
454 // buffer pointer.
455 movrel x6, right_ext_mask, -6
456 ldr h26, [x3, w17, sxtw #1]
457 sub x6, x6, w4, uxtw #1
458 dup v26.8h, v26.h[0]
459 ld1 {v23.16b, v24.16b, v25.16b}, [x6]
460
461 bit v2.16b, v26.16b, v23.16b
462 bit v3.16b, v26.16b, v24.16b
463 bit v4.16b, v26.16b, v25.16b
464
465 4: // Loop horizontally
466 ext v17.16b, v2.16b, v3.16b, #4
467 ext v19.16b, v2.16b, v3.16b, #8
468 ext v16.16b, v2.16b, v3.16b, #2
469 ext v20.16b, v2.16b, v3.16b, #10
470 ext v21.16b, v2.16b, v3.16b, #12
471 ext v18.16b, v2.16b, v3.16b, #6
472 add v19.8h, v19.8h, v17.8h
473 add v20.8h, v20.8h, v16.8h
474 add v21.8h, v21.8h, v2.8h
475 smull v6.4s, v18.4h, v0.h[3]
476 smlal v6.4s, v19.4h, v0.h[2]
477 smlal v6.4s, v20.4h, v0.h[1]
478 smlal v6.4s, v21.4h, v0.h[0]
479 smull2 v7.4s, v18.8h, v0.h[3]
480 smlal2 v7.4s, v19.8h, v0.h[2]
481 smlal2 v7.4s, v20.8h, v0.h[1]
482 smlal2 v7.4s, v21.8h, v0.h[0]
483
484 ext v17.16b, v3.16b, v4.16b, #4
485 ext v19.16b, v3.16b, v4.16b, #8
486 ext v16.16b, v3.16b, v4.16b, #2
487 ext v20.16b, v3.16b, v4.16b, #10
488 ext v21.16b, v3.16b, v4.16b, #12
489 ext v18.16b, v3.16b, v4.16b, #6
490
491 add v19.8h, v19.8h, v17.8h
492 add v20.8h, v20.8h, v16.8h
493 add v21.8h, v21.8h, v3.8h
494 smull v24.4s, v18.4h, v0.h[3]
495 smlal v24.4s, v19.4h, v0.h[2]
496 smlal v24.4s, v20.4h, v0.h[1]
497 smlal v24.4s, v21.4h, v0.h[0]
498 smull2 v25.4s, v18.8h, v0.h[3]
499 smlal2 v25.4s, v19.8h, v0.h[2]
500 smlal2 v25.4s, v20.8h, v0.h[1]
501 smlal2 v25.4s, v21.8h, v0.h[0]
502
503 ld1 {v16.8h, v17.8h}, [x9], #32
504
505 mvni v26.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
506 add v6.4s, v6.4s, v30.4s
507 add v7.4s, v7.4s, v30.4s
508 add v24.4s, v24.4s, v30.4s
509 add v25.4s, v25.4s, v30.4s
510 ld1 {v18.8h, v19.8h}, [x10], #32
511 srshl v6.4s, v6.4s, v29.4s
512 srshl v7.4s, v7.4s, v29.4s
513 srshl v24.4s, v24.4s, v29.4s
514 srshl v25.4s, v25.4s, v29.4s
515 ld1 {v20.8h, v21.8h}, [x11], #32
516 sqxtun v6.4h, v6.4s
517 sqxtun2 v6.8h, v7.4s
518 sqxtun v7.4h, v24.4s
519 sqxtun2 v7.8h, v25.4s
520 ld1 {v22.8h, v23.8h}, [x12], #32
521 umin v6.8h, v6.8h, v26.8h
522 umin v7.8h, v7.8h, v26.8h
523 ld1 {v24.8h, v25.8h}, [x13], #32
524 sub v6.8h, v6.8h, v31.8h
525 sub v7.8h, v7.8h, v31.8h
526
527 ld1 {v8.8h, v9.8h}, [x14], #32
528
529 smull v1.4s, v16.4h, v0.h[4]
530 smlal v1.4s, v18.4h, v0.h[5]
531 smlal v1.4s, v20.4h, v0.h[6]
532 smlal v1.4s, v22.4h, v0.h[7]
533 smlal v1.4s, v24.4h, v0.h[6]
534 smlal v1.4s, v8.4h, v0.h[5]
535 smlal v1.4s, v6.4h, v0.h[4]
536 smull2 v5.4s, v16.8h, v0.h[4]
537 smlal2 v5.4s, v18.8h, v0.h[5]
538 smlal2 v5.4s, v20.8h, v0.h[6]
539 smlal2 v5.4s, v22.8h, v0.h[7]
540 smlal2 v5.4s, v24.8h, v0.h[6]
541 smlal2 v5.4s, v8.8h, v0.h[5]
542 smlal2 v5.4s, v6.8h, v0.h[4]
543 smull v26.4s, v17.4h, v0.h[4]
544 smlal v26.4s, v19.4h, v0.h[5]
545 smlal v26.4s, v21.4h, v0.h[6]
546 smlal v26.4s, v23.4h, v0.h[7]
547 smlal v26.4s, v25.4h, v0.h[6]
548 smlal v26.4s, v9.4h, v0.h[5]
549 smlal v26.4s, v7.4h, v0.h[4]
550 smull2 v16.4s, v17.8h, v0.h[4]
551 smlal2 v16.4s, v19.8h, v0.h[5]
552 smlal2 v16.4s, v21.8h, v0.h[6]
553 smlal2 v16.4s, v23.8h, v0.h[7]
554 smlal2 v16.4s, v25.8h, v0.h[6]
555 smlal2 v16.4s, v9.8h, v0.h[5]
556 smlal2 v16.4s, v7.8h, v0.h[4]
557 srshl v1.4s, v1.4s, v27.4s // -round_bits_v
558 srshl v5.4s, v5.4s, v27.4s
559 srshl v26.4s, v26.4s, v27.4s
560 srshl v16.4s, v16.4s, v27.4s
561 sqxtun v18.4h, v1.4s
562 sqxtun2 v18.8h, v5.4s
563 sqxtun v19.4h, v26.4s
564 sqxtun2 v19.8h, v16.4s
565 st1 {v6.8h, v7.8h}, [x15], #32
566 umin v18.8h, v18.8h, v28.8h // bitdepth_max
567 umin v19.8h, v19.8h, v28.8h
568 subs w4, w4, #16
569
570 st1 {v18.8h, v19.8h}, [x0], #32
571
572 b.le 0f
573 mov v2.16b, v4.16b
574 tst w7, #2 // LR_HAVE_RIGHT
575 ld1 {v3.8h, v4.8h}, [x3], #32
576 b.ne 4b // If we don't need to pad, just keep filtering.
577 b 3b // If we need to pad, check how many pixels we have left.
578
579 0:
580 ldp x3, x4, [sp, #64]
581 ldp x15, x0, [sp, #48]
582 ldp x13, x14, [sp, #32]
583 ldp x11, x12, [sp, #16]
584 ldp x9, x10, [sp], #80
585
586 add x3, x3, x1
587 add x0, x0, x1
588
589 ret
590 endfunc
591
592 // void dav1d_wiener_filter5_16bpc_neon(pixel *p, const ptrdiff_t p_stride,
593 // const pixel (*left)[4], const pixel *lpf,
594 // const int w, int h,
595 // const int16_t filter[2][8],
596 // const enum LrEdgeFlags edges,
597 // const int bitdepth_max);
598 function wiener_filter5_16bpc_neon, export=1
599 ldr w8, [sp]
600 AARCH64_SIGN_LINK_REGISTER
601 stp x29, x30, [sp, #-32]!
602 stp d8, d9, [sp, #16]
603 mov x29, sp
604 ld1 {v0.8h, v1.8h}, [x6]
605 tst w7, #4 // LR_HAVE_TOP
606 sub_sp 384*2*4
607
608 dup v28.8h, w8 // bitdepth_max
609 clz w8, w8
610 movi v30.4s, #1
611 sub w10, w8, #38 // -(bitdepth + 6)
612 sub w11, w8, #11 // round_bits_v
613 sub w8, w8, #25 // -round_bits_h
614 neg w10, w10 // bitdepth + 6
615 neg w11, w11 // -round_bits_v
616 dup v2.4s, w10
617 dup v29.4s, w8 // -round_bits_h
618 dup v27.4s, w11 // -round_bits_v
619 movi v31.8h, #0x20, lsl #8 // 1 << 13 = 8192
620 ushl v30.4s, v30.4s, v2.4s // 1 << (bitdepth + 6)
621
622 zip1 v0.2d, v0.2d, v1.2d // move vertical coeffs to v0.h[4-7], freeing up v1
623
624 // x11 - t4
625 // x12 - t3
626 // x13 - t2
627 // x14 - t1
628 // x15 - t0
629 mov x14, sp // t1
630 b.eq L(no_top_5)
631
632 mov x16, x2 // backup left
633 mov x2, #0
634 bl wiener_filter5_h_16bpc_neon
635 add x3, x3, x1 // lpf += stride
636 mov x11, x14 // t4
637 add x14, x14, #384*2 // t1 += 384*2
638 bl wiener_filter5_h_16bpc_neon
639 add x3, x3, x1, lsl #2
640 add x3, x3, x1 // lpf += stride*5
641 mov x12, x14 // t3
642 add x14, x14, #384*2 // t1 += 384*2
643 mov x2, x16 // left
644 mov x16, x3 // backup lpf
645 mov x3, x0 // lpf = p
646 bl wiener_filter5_h_16bpc_neon
647 subs w5, w5, #1 // h--
648 mov x13, x14 // t2
649 b.eq L(v1_5)
650 add x3, x3, x1 // src += stride
651 add x14, x14, #384*2 // t1 += 384*2
652 bl wiener_filter5_h_16bpc_neon
653 subs w5, w5, #1 // h--
654 b.eq L(v2_5)
655 add x3, x3, x1 // src += stride
656
657 L(main_5):
658 mov x15, x11 // t0 = t4
659 L(main_loop_5):
660 bl wiener_filter5_hv_16bpc_neon
661 subs w5, w5, #1 // h--
662 b.ne L(main_loop_5)
663 tst w7, #8 // LR_HAVE_BOTTOM
664 b.eq L(v2_5)
665
666 mov x3, x16 // restore lpf
667 mov x2, #0 // left = NULL
668 bl wiener_filter5_hv_16bpc_neon
669 bl wiener_filter5_hv_16bpc_neon
670 L(end_5):
671
672 mov sp, x29
673 ldp d8, d9, [sp, #16]
674 ldp x29, x30, [sp], #32
675 AARCH64_VALIDATE_LINK_REGISTER
676 ret
677
678 L(no_top_5):
679 add x3, x3, x1, lsl #2
680 add x16, x3, x1, lsl #1 // lpf += stride*6, backup
681 mov x3, x0 // lpf = p
682
683 bl wiener_filter5_h_16bpc_neon
684 subs w5, w5, #1 // h--
685 mov x11, x14 // t4
686 mov x12, x14 // t3
687 mov x13, x14 // t2
688 b.eq L(v1_5)
689 add x3, x3, x1 // src += stride
690 add x14, x14, #384*2 // t1 += 384*2
691 bl wiener_filter5_h_16bpc_neon
692 subs w5, w5, #1 // h--
693 b.eq L(v2_5)
694 add x3, x3, x1 // src += stride
695 add x15, x14, #384*2 // t0 = t1 + 384*2
696 bl wiener_filter5_hv_16bpc_neon
697 subs w5, w5, #1 // h--
698 b.eq L(v2_5)
699 add x15, x15, #384*2*3 // t0 += 384*2*3
700 bl wiener_filter5_hv_16bpc_neon
701 subs w5, w5, #1 // h--
702 b.ne L(main_5)
703 L(v2_5):
704 bl wiener_filter5_v_16bpc_neon
705 add x0, x0, x1
706 mov x11, x12
707 mov x12, x13
708 mov x13, x14
709 L(v1_5):
710 bl wiener_filter5_v_16bpc_neon
711 b L(end_5)
712 endfunc
713
714
715 function wiener_filter5_h_16bpc_neon
716 stp x3, x4, [sp, #-32]!
717 str x14, [sp, #16]
718
719 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
720 tst w7, #1 // LR_HAVE_LEFT
721 b.eq 1f
722 // LR_HAVE_LEFT
723 cbnz x2, 0f
724 // left == NULL
725 sub x3, x3, #4
726 ld1 {v2.8h, v3.8h}, [x3], #32
727 b 2f
728
729 0:
730 // LR_HAVE_LEFT, left != NULL
731 ld1 {v2.8h, v3.8h}, [x3], #32
732 ld1 {v4.d}[1], [x2], #8
733 // Move x3 back to account for the last 2 pixels we loaded earlier,
734 // which we'll shift out.
735 sub x3, x3, #4
736 ext v3.16b, v2.16b, v3.16b, #12
737 ext v2.16b, v4.16b, v2.16b, #12
738 b 2f
739
740 1:
741 ld1 {v2.8h, v3.8h}, [x3], #32
742 // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
743 // and shift v3 to have 3x the first pixel at the front.
744 dup v4.8h, v2.h[0]
745 // Move x3 back to account for the last 2 pixels we loaded before,
746 // which we shifted out.
747 sub x3, x3, #4
748 ext v3.16b, v2.16b, v3.16b, #12
749 ext v2.16b, v4.16b, v2.16b, #12
750
751 2:
752 ld1 {v4.8h}, [x3], #16
753
754 tst w7, #2 // LR_HAVE_RIGHT
755 b.ne 4f
756
757 3: // !LR_HAVE_RIGHT
758
759 // Check whether we need to pad the right edge
760 cmp w4, #18
761 b.ge 4f // If w >= 18, all used input pixels are valid
762
763 // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9,
764 // this ends up called again; it's not strictly needed in those
765 // cases (we pad enough here), but keeping the code as simple as possible.
766
767 // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
768 // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel.
769 sub w17, w4, #23
770 // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the
771 // buffer pointer.
772 movrel x6, right_ext_mask, -4
773 ldr h26, [x3, w17, sxtw #1]
774 sub x6, x6, w4, uxtw #1
775 dup v26.8h, v26.h[0]
776 ld1 {v23.16b, v24.16b, v25.16b}, [x6]
777
778 bit v2.16b, v26.16b, v23.16b
779 bit v3.16b, v26.16b, v24.16b
780 bit v4.16b, v26.16b, v25.16b
781
782 4: // Loop horizontally
783 // Interleaving the mul/mla chains actually hurts performance
784 // significantly on Cortex A53, thus keeping mul/mla tightly
785 // chained like this.
786 ext v16.16b, v2.16b, v3.16b, #2
787 ext v18.16b, v2.16b, v3.16b, #6
788 ext v19.16b, v2.16b, v3.16b, #8
789 ext v17.16b, v2.16b, v3.16b, #4
790 add v18.8h, v18.8h, v16.8h
791 add v19.8h, v19.8h, v2.8h
792 smull v6.4s, v17.4h, v0.h[3]
793 smlal v6.4s, v18.4h, v0.h[2]
794 smlal v6.4s, v19.4h, v0.h[1]
795 smull2 v7.4s, v17.8h, v0.h[3]
796 smlal2 v7.4s, v18.8h, v0.h[2]
797 smlal2 v7.4s, v19.8h, v0.h[1]
798
799 ext v16.16b, v3.16b, v4.16b, #2
800 ext v18.16b, v3.16b, v4.16b, #6
801 ext v19.16b, v3.16b, v4.16b, #8
802 ext v17.16b, v3.16b, v4.16b, #4
803 add v18.8h, v18.8h, v16.8h
804 add v19.8h, v19.8h, v3.8h
805 smull v16.4s, v17.4h, v0.h[3]
806 smlal v16.4s, v18.4h, v0.h[2]
807 smlal v16.4s, v19.4h, v0.h[1]
808 smull2 v17.4s, v17.8h, v0.h[3]
809 smlal2 v17.4s, v18.8h, v0.h[2]
810 smlal2 v17.4s, v19.8h, v0.h[1]
811
812 mvni v24.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
813 add v6.4s, v6.4s, v30.4s
814 add v7.4s, v7.4s, v30.4s
815 add v16.4s, v16.4s, v30.4s
816 add v17.4s, v17.4s, v30.4s
817 srshl v6.4s, v6.4s, v29.4s
818 srshl v7.4s, v7.4s, v29.4s
819 srshl v16.4s, v16.4s, v29.4s
820 srshl v17.4s, v17.4s, v29.4s
821 sqxtun v6.4h, v6.4s
822 sqxtun2 v6.8h, v7.4s
823 sqxtun v7.4h, v16.4s
824 sqxtun2 v7.8h, v17.4s
825 umin v6.8h, v6.8h, v24.8h
826 umin v7.8h, v7.8h, v24.8h
827 sub v6.8h, v6.8h, v31.8h
828 sub v7.8h, v7.8h, v31.8h
829
830 subs w4, w4, #16
831
832 st1 {v6.8h, v7.8h}, [x14], #32
833
834 b.le 0f
835 mov v2.16b, v4.16b
836 tst w7, #2 // LR_HAVE_RIGHT
837 ld1 {v3.8h, v4.8h}, [x3], #32
838 b.ne 4b // If we don't need to pad, just keep filtering.
839 b 3b // If we need to pad, check how many pixels we have left.
840
841 0:
842 ldr x14, [sp, #16]
843 ldp x3, x4, [sp], #32
844 ret
845 endfunc
846
847 function wiener_filter5_v_16bpc_neon
848 stp x11, x12, [sp, #-48]!
849 stp x13, x14, [sp, #16]
850 stp x0, x4, [sp, #32]
851 1:
852 ld1 {v16.8h, v17.8h}, [x11], #32
853 ld1 {v18.8h, v19.8h}, [x12], #32
854 ld1 {v20.8h, v21.8h}, [x13], #32
855 ld1 {v22.8h, v23.8h}, [x14], #32
856
857 smull v2.4s, v16.4h, v0.h[5]
858 smlal v2.4s, v18.4h, v0.h[6]
859 smlal v2.4s, v20.4h, v0.h[7]
860 smlal v2.4s, v22.4h, v0.h[6]
861 smlal v2.4s, v22.4h, v0.h[5]
862 smull2 v3.4s, v16.8h, v0.h[5]
863 smlal2 v3.4s, v18.8h, v0.h[6]
864 smlal2 v3.4s, v20.8h, v0.h[7]
865 smlal2 v3.4s, v22.8h, v0.h[6]
866 smlal2 v3.4s, v22.8h, v0.h[5]
867 smull v4.4s, v17.4h, v0.h[5]
868 smlal v4.4s, v19.4h, v0.h[6]
869 smlal v4.4s, v21.4h, v0.h[7]
870 smlal v4.4s, v23.4h, v0.h[6]
871 smlal v4.4s, v23.4h, v0.h[5]
872 smull2 v5.4s, v17.8h, v0.h[5]
873 smlal2 v5.4s, v19.8h, v0.h[6]
874 smlal2 v5.4s, v21.8h, v0.h[7]
875 smlal2 v5.4s, v23.8h, v0.h[6]
876 smlal2 v5.4s, v23.8h, v0.h[5]
877 srshl v2.4s, v2.4s, v27.4s // -round_bits_v
878 srshl v3.4s, v3.4s, v27.4s
879 srshl v4.4s, v4.4s, v27.4s
880 srshl v5.4s, v5.4s, v27.4s
881 sqxtun v2.4h, v2.4s
882 sqxtun2 v2.8h, v3.4s
883 sqxtun v3.4h, v4.4s
884 sqxtun2 v3.8h, v5.4s
885 umin v2.8h, v2.8h, v28.8h // bitdepth_max
886 umin v3.8h, v3.8h, v28.8h
887
888 subs w4, w4, #16
889 st1 {v2.8h, v3.8h}, [x0], #32
890 b.gt 1b
891
892 ldp x0, x4, [sp, #32]
893 ldp x13, x14, [sp, #16]
894 ldp x11, x12, [sp], #48
895
896 ret
897 endfunc
898
899 function wiener_filter5_hv_16bpc_neon
900 // Backing up/restoring registers shifted, so that x11 gets the value
901 // of x12, etc, and x15==x11, afterwards.
902 stp x12, x13, [sp, #-64]!
903 stp x14, x15, [sp, #16]
904 stp x12, x0, [sp, #32]
905 stp x3, x4, [sp, #48]
906
907 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
908 tst w7, #1 // LR_HAVE_LEFT
909 b.eq 1f
910 // LR_HAVE_LEFT
911 cbnz x2, 0f
912 // left == NULL
913 sub x3, x3, #4
914 ld1 {v2.8h, v3.8h}, [x3], #32
915 b 2f
916
917 0:
918 // LR_HAVE_LEFT, left != NULL
919 ld1 {v2.8h, v3.8h}, [x3], #32
920 ld1 {v4.d}[1], [x2], #8
921 // Move x3 back to account for the last 2 pixels we loaded earlier,
922 // which we'll shift out.
923 sub x3, x3, #4
924 ext v3.16b, v2.16b, v3.16b, #12
925 ext v2.16b, v4.16b, v2.16b, #12
926 b 2f
927 1:
928 ld1 {v2.8h, v3.8h}, [x3], #32
929 // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
930 // and shift v3 to have 2x the first pixel at the front.
931 dup v4.8h, v2.h[0]
932 // Move x3 back to account for the last 2 pixels we loaded before,
933 // which we shifted out.
934 sub x3, x3, #4
935 ext v3.16b, v2.16b, v3.16b, #12
936 ext v2.16b, v4.16b, v2.16b, #12
937
938 2:
939 ld1 {v4.8h}, [x3], #16
940
941 tst w7, #2 // LR_HAVE_RIGHT
942 b.ne 4f
943
944 3: // !LR_HAVE_RIGHT
945
946 // Check whether we need to pad the right edge
947 cmp w4, #18
948 b.ge 4f // If w >= 18, all used input pixels are valid
949
950 // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9,
951 // this ends up called again; it's not strictly needed in those
952 // cases (we pad enough here), but keeping the code as simple as possible.
953
954 // The padding pixel is v2/3/4.h[w+1]. x3 points at the next input, ie
955 // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel.
956 sub w17, w4, #23
957 // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the
958 // buffer pointer.
959 movrel x6, right_ext_mask, -4
960 ldr h26, [x3, w17, sxtw #1]
961 sub x6, x6, w4, uxtw #1
962 dup v26.8h, v26.h[0]
963 ld1 {v23.16b, v24.16b, v25.16b}, [x6]
964
965 bit v2.16b, v26.16b, v23.16b
966 bit v3.16b, v26.16b, v24.16b
967 bit v4.16b, v26.16b, v25.16b
968
969 4: // Loop horizontally
970 ext v16.16b, v2.16b, v3.16b, #2
971 ext v18.16b, v2.16b, v3.16b, #6
972 ext v19.16b, v2.16b, v3.16b, #8
973 ext v17.16b, v2.16b, v3.16b, #4
974 add v18.8h, v18.8h, v16.8h
975 add v19.8h, v19.8h, v2.8h
976 smull v6.4s, v17.4h, v0.h[3]
977 smlal v6.4s, v18.4h, v0.h[2]
978 smlal v6.4s, v19.4h, v0.h[1]
979 smull2 v7.4s, v17.8h, v0.h[3]
980 smlal2 v7.4s, v18.8h, v0.h[2]
981 smlal2 v7.4s, v19.8h, v0.h[1]
982
983 ext v16.16b, v3.16b, v4.16b, #2
984 ext v18.16b, v3.16b, v4.16b, #6
985 ext v19.16b, v3.16b, v4.16b, #8
986 ext v17.16b, v3.16b, v4.16b, #4
987 add v18.8h, v18.8h, v16.8h
988 add v19.8h, v19.8h, v3.8h
989 smull v24.4s, v17.4h, v0.h[3]
990 smlal v24.4s, v18.4h, v0.h[2]
991 smlal v24.4s, v19.4h, v0.h[1]
992 smull2 v25.4s, v17.8h, v0.h[3]
993 smlal2 v25.4s, v18.8h, v0.h[2]
994 smlal2 v25.4s, v19.8h, v0.h[1]
995
996 ld1 {v16.8h, v17.8h}, [x11], #32
997 mvni v26.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1
998 add v6.4s, v6.4s, v30.4s
999 add v7.4s, v7.4s, v30.4s
1000 add v24.4s, v24.4s, v30.4s
1001 add v25.4s, v25.4s, v30.4s
1002 ld1 {v18.8h, v19.8h}, [x12], #32
1003 srshl v6.4s, v6.4s, v29.4s
1004 srshl v7.4s, v7.4s, v29.4s
1005 srshl v24.4s, v24.4s, v29.4s
1006 srshl v25.4s, v25.4s, v29.4s
1007 ld1 {v20.8h, v21.8h}, [x13], #32
1008 sqxtun v6.4h, v6.4s
1009 sqxtun2 v6.8h, v7.4s
1010 sqxtun v7.4h, v24.4s
1011 sqxtun2 v7.8h, v25.4s
1012 ld1 {v22.8h, v23.8h}, [x14], #32
1013 umin v6.8h, v6.8h, v26.8h
1014 umin v7.8h, v7.8h, v26.8h
1015 sub v6.8h, v6.8h, v31.8h
1016 sub v7.8h, v7.8h, v31.8h
1017
1018 smull v8.4s, v16.4h, v0.h[5]
1019 smlal v8.4s, v18.4h, v0.h[6]
1020 smlal v8.4s, v20.4h, v0.h[7]
1021 smlal v8.4s, v22.4h, v0.h[6]
1022 smlal v8.4s, v6.4h, v0.h[5]
1023 smull2 v9.4s, v16.8h, v0.h[5]
1024 smlal2 v9.4s, v18.8h, v0.h[6]
1025 smlal2 v9.4s, v20.8h, v0.h[7]
1026 smlal2 v9.4s, v22.8h, v0.h[6]
1027 smlal2 v9.4s, v6.8h, v0.h[5]
1028 smull v1.4s, v17.4h, v0.h[5]
1029 smlal v1.4s, v19.4h, v0.h[6]
1030 smlal v1.4s, v21.4h, v0.h[7]
1031 smlal v1.4s, v23.4h, v0.h[6]
1032 smlal v1.4s, v7.4h, v0.h[5]
1033 smull2 v5.4s, v17.8h, v0.h[5]
1034 smlal2 v5.4s, v19.8h, v0.h[6]
1035 smlal2 v5.4s, v21.8h, v0.h[7]
1036 smlal2 v5.4s, v23.8h, v0.h[6]
1037 smlal2 v5.4s, v7.8h, v0.h[5]
1038 srshl v8.4s, v8.4s, v27.4s // -round_bits_v
1039 srshl v9.4s, v9.4s, v27.4s
1040 srshl v1.4s, v1.4s, v27.4s
1041 srshl v5.4s, v5.4s, v27.4s
1042 sqxtun v8.4h, v8.4s
1043 sqxtun2 v8.8h, v9.4s
1044 sqxtun v9.4h, v1.4s
1045 sqxtun2 v9.8h, v5.4s
1046 st1 {v6.8h, v7.8h}, [x15], #32
1047 umin v8.8h, v8.8h, v28.8h // bitdepth_max
1048 umin v9.8h, v9.8h, v28.8h
1049
1050 subs w4, w4, #16
1051
1052 st1 {v8.8h, v9.8h}, [x0], #32
1053
1054 b.le 0f
1055 mov v2.16b, v4.16b
1056 tst w7, #2 // LR_HAVE_RIGHT
1057 ld1 {v3.8h, v4.8h}, [x3], #32
1058 b.ne 4b // If we don't need to pad, just keep filtering.
1059 b 3b // If we need to pad, check how many pixels we have left.
1060
1061 0:
1062 ldp x3, x4, [sp, #48]
1063 ldp x15, x0, [sp, #32]
1064 ldp x13, x14, [sp, #16]
1065 ldp x11, x12, [sp], #64
1066
1067 add x3, x3, x1
1068 add x0, x0, x1
1069
1070 ret
1071 endfunc
1072
1073 #define SUM_STRIDE (384+16)
1074
1075 #include "looprestoration_tmpl.S"
1076
1077 // void dav1d_sgr_box3_h_16bpc_neon(int32_t *sumsq, int16_t *sum,
1078 // const pixel (*left)[4],
1079 // const pixel *src, const ptrdiff_t stride,
1080 // const int w, const int h,
1081 // const enum LrEdgeFlags edges);
1082 function sgr_box3_h_16bpc_neon, export=1
1083 add w5, w5, #2 // w += 2
1084
1085 // Set up pointers for reading/writing alternate rows
1086 add x10, x0, #(4*SUM_STRIDE) // sumsq
1087 add x11, x1, #(2*SUM_STRIDE) // sum
1088 add x12, x3, x4 // src
1089 lsl x4, x4, #1
1090 mov x9, #(2*2*SUM_STRIDE) // double sum stride
1091
1092 // Subtract the aligned width from the output stride.
1093 add w13, w5, #7
1094 bic w13, w13, #7
1095 sub x9, x9, w13, uxtw #1
1096
1097 // Store the width for the vertical loop
1098 mov w8, w5
1099
1100 // Subtract the number of pixels read from the input from the stride
1101 add w13, w13, #8
1102 sub x4, x4, w13, uxtw #1
1103
1104 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
1105 tst w7, #1 // LR_HAVE_LEFT
1106 b.eq 2f
1107 // LR_HAVE_LEFT
1108 cbnz x2, 0f
1109 // left == NULL
1110 sub x3, x3, #4
1111 sub x12, x12, #4
1112 b 1f
1113 0: // LR_HAVE_LEFT, left != NULL
1114 2: // !LR_HAVE_LEFT, increase the stride.
1115 // For this case we don't read the left 2 pixels from the src pointer,
1116 // but shift it as if we had done that.
1117 add x4, x4, #4
1118
1119
1120 1: // Loop vertically
1121 ld1 {v0.8h, v1.8h}, [x3], #32
1122 ld1 {v16.8h, v17.8h}, [x12], #32
1123
1124 tst w7, #1 // LR_HAVE_LEFT
1125 b.eq 0f
1126 cbz x2, 2f
1127 // LR_HAVE_LEFT, left != NULL
1128 ld1 {v2.d}[1], [x2], #8
1129 // Move x3/x12 back to account for the last 2 pixels we loaded earlier,
1130 // which we'll shift out.
1131 sub x3, x3, #4
1132 sub x12, x12, #4
1133 ld1 {v18.d}[1], [x2], #8
1134 ext v1.16b, v0.16b, v1.16b, #12
1135 ext v0.16b, v2.16b, v0.16b, #12
1136 ext v17.16b, v16.16b, v17.16b, #12
1137 ext v16.16b, v18.16b, v16.16b, #12
1138 b 2f
1139 0:
1140 // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
1141 // and shift v0/v1 to have 2x the first pixel at the front.
1142 dup v2.8h, v0.h[0]
1143 dup v18.8h, v16.h[0]
1144 // Move x3 back to account for the last 2 pixels we loaded before,
1145 // which we shifted out.
1146 sub x3, x3, #4
1147 sub x12, x12, #4
1148 ext v1.16b, v0.16b, v1.16b, #12
1149 ext v0.16b, v2.16b, v0.16b, #12
1150 ext v17.16b, v16.16b, v17.16b, #12
1151 ext v16.16b, v18.16b, v16.16b, #12
1152
1153 2:
1154 tst w7, #2 // LR_HAVE_RIGHT
1155 b.ne 4f
1156 // If we'll need to pad the right edge, load that pixel to pad with
1157 // here since we can find it pretty easily from here.
1158 sub w13, w5, #(2 + 16 - 2 + 1)
1159 ldr h30, [x3, w13, sxtw #1]
1160 ldr h31, [x12, w13, sxtw #1]
1161 // Fill v30/v31 with the right padding pixel
1162 dup v30.8h, v30.h[0]
1163 dup v31.8h, v31.h[0]
1164 3: // !LR_HAVE_RIGHT
1165
1166 // Check whether we need to pad the right edge
1167 cmp w5, #10
1168 b.ge 4f // If w >= 10, all used input pixels are valid
1169
1170 // 1 <= w < 10, w pixels valid in v0-v1. For w=9, this ends up called
1171 // again; it's not strictly needed in those cases (we pad enough here),
1172 // but keeping the code as simple as possible.
1173
1174 // Insert padding in v0/1.h[w] onwards
1175 movrel x13, right_ext_mask
1176 sub x13, x13, w5, uxtw #1
1177 ld1 {v28.16b, v29.16b}, [x13]
1178
1179 bit v0.16b, v30.16b, v28.16b
1180 bit v1.16b, v30.16b, v29.16b
1181 bit v16.16b, v31.16b, v28.16b
1182 bit v17.16b, v31.16b, v29.16b
1183
1184 4: // Loop horizontally
1185 ext v26.16b, v0.16b, v1.16b, #2
1186 ext v28.16b, v16.16b, v17.16b, #2
1187 ext v27.16b, v0.16b, v1.16b, #4
1188 ext v29.16b, v16.16b, v17.16b, #4
1189
1190 add v6.8h, v0.8h, v26.8h
1191 umull v22.4s, v0.4h, v0.4h
1192 umlal v22.4s, v26.4h, v26.4h
1193 umlal v22.4s, v27.4h, v27.4h
1194 add v7.8h, v16.8h, v28.8h
1195 umull v24.4s, v16.4h, v16.4h
1196 umlal v24.4s, v28.4h, v28.4h
1197 umlal v24.4s, v29.4h, v29.4h
1198 add v6.8h, v6.8h, v27.8h
1199 umull2 v23.4s, v0.8h, v0.8h
1200 umlal2 v23.4s, v26.8h, v26.8h
1201 umlal2 v23.4s, v27.8h, v27.8h
1202 add v7.8h, v7.8h, v29.8h
1203 umull2 v25.4s, v16.8h, v16.8h
1204 umlal2 v25.4s, v28.8h, v28.8h
1205 umlal2 v25.4s, v29.8h, v29.8h
1206
1207 subs w5, w5, #8
1208
1209 st1 {v6.8h}, [x1], #16
1210 st1 {v7.8h}, [x11], #16
1211 st1 {v22.4s,v23.4s}, [x0], #32
1212 st1 {v24.4s,v25.4s}, [x10], #32
1213
1214 b.le 9f
1215 tst w7, #2 // LR_HAVE_RIGHT
1216 mov v0.16b, v1.16b
1217 mov v16.16b, v17.16b
1218 ld1 {v1.8h}, [x3], #16
1219 ld1 {v17.8h}, [x12], #16
1220
1221 b.ne 4b // If we don't need to pad, just keep summing.
1222 b 3b // If we need to pad, check how many pixels we have left.
1223
1224 9:
1225 subs w6, w6, #2
1226 b.le 0f
1227 // Jump to the next row and loop horizontally
1228 add x0, x0, x9, lsl #1
1229 add x10, x10, x9, lsl #1
1230 add x1, x1, x9
1231 add x11, x11, x9
1232 add x3, x3, x4
1233 add x12, x12, x4
1234 mov w5, w8
1235 b 1b
1236 0:
1237 ret
1238 endfunc
1239
1240 // void dav1d_sgr_box5_h_16bpc_neon(int32_t *sumsq, int16_t *sum,
1241 // const pixel (*left)[4],
1242 // const pixel *src, const ptrdiff_t stride,
1243 // const int w, const int h,
1244 // const enum LrEdgeFlags edges);
1245 function sgr_box5_h_16bpc_neon, export=1
1246 add w5, w5, #2 // w += 2
1247
1248 // Set up pointers for reading/writing alternate rows
1249 add x10, x0, #(4*SUM_STRIDE) // sumsq
1250 add x11, x1, #(2*SUM_STRIDE) // sum
1251 add x12, x3, x4 // src
1252 lsl x4, x4, #1
1253 mov x9, #(2*2*SUM_STRIDE) // double sum stride
1254
1255 // Subtract the aligned width from the output stride.
1256 add w13, w5, #7
1257 bic w13, w13, #7
1258 sub x9, x9, w13, uxtw #1
1259 add w13, w13, #8
1260 sub x4, x4, w13, uxtw #1
1261
1262 // Store the width for the vertical loop
1263 mov w8, w5
1264
1265 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
1266 tst w7, #1 // LR_HAVE_LEFT
1267 b.eq 2f
1268 // LR_HAVE_LEFT
1269 cbnz x2, 0f
1270 // left == NULL
1271 sub x3, x3, #6
1272 sub x12, x12, #6
1273 b 1f
1274 0: // LR_HAVE_LEFT, left != NULL
1275 2: // !LR_HAVE_LEFT, increase the stride.
1276 // For this case we don't read the left 3 pixels from the src pointer,
1277 // but shift it as if we had done that.
1278 add x4, x4, #6
1279
1280 1: // Loop vertically
1281 ld1 {v0.8h, v1.8h}, [x3], #32
1282 ld1 {v16.8h, v17.8h}, [x12], #32
1283
1284 tst w7, #1 // LR_HAVE_LEFT
1285 b.eq 0f
1286 cbz x2, 2f
1287 // LR_HAVE_LEFT, left != NULL
1288 ld1 {v2.d}[1], [x2], #8
1289 // Move x3/x12 back to account for the last 3 pixels we loaded earlier,
1290 // which we'll shift out.
1291 sub x3, x3, #6
1292 sub x12, x12, #6
1293 ld1 {v18.d}[1], [x2], #8
1294 ext v1.16b, v0.16b, v1.16b, #10
1295 ext v0.16b, v2.16b, v0.16b, #10
1296 ext v17.16b, v16.16b, v17.16b, #10
1297 ext v16.16b, v18.16b, v16.16b, #10
1298 b 2f
1299 0:
1300 // !LR_HAVE_LEFT, fill v2 with the leftmost pixel
1301 // and shift v0/v1 to have 3x the first pixel at the front.
1302 dup v2.8h, v0.h[0]
1303 dup v18.8h, v16.h[0]
1304 // Move x3 back to account for the last 3 pixels we loaded before,
1305 // which we shifted out.
1306 sub x3, x3, #6
1307 sub x12, x12, #6
1308 ext v1.16b, v0.16b, v1.16b, #10
1309 ext v0.16b, v2.16b, v0.16b, #10
1310 ext v17.16b, v16.16b, v17.16b, #10
1311 ext v16.16b, v18.16b, v16.16b, #10
1312
1313 2:
1314 tst w7, #2 // LR_HAVE_RIGHT
1315 b.ne 4f
1316 // If we'll need to pad the right edge, load that pixel to pad with
1317 // here since we can find it pretty easily from here.
1318 sub w13, w5, #(2 + 16 - 3 + 1)
1319 ldr h30, [x3, w13, sxtw #1]
1320 ldr h31, [x12, w13, sxtw #1]
1321 // Fill v30/v31 with the right padding pixel
1322 dup v30.8h, v30.h[0]
1323 dup v31.8h, v31.h[0]
1324 3: // !LR_HAVE_RIGHT
1325
1326 // Check whether we need to pad the right edge
1327 cmp w5, #11
1328 b.ge 4f // If w >= 11, all used input pixels are valid
1329
1330 // 1 <= w < 11, w+1 pixels valid in v0-v1. For w=9 or w=10,
1331 // this ends up called again; it's not strictly needed in those
1332 // cases (we pad enough here), but keeping the code as simple as possible.
1333
1334 // Insert padding in v0/1.h[w+1] onwards; fuse the +1 into the
1335 // buffer pointer.
1336 movrel x13, right_ext_mask, -2
1337 sub x13, x13, w5, uxtw #1
1338 ld1 {v28.16b, v29.16b}, [x13]
1339
1340 bit v0.16b, v30.16b, v28.16b
1341 bit v1.16b, v30.16b, v29.16b
1342 bit v16.16b, v31.16b, v28.16b
1343 bit v17.16b, v31.16b, v29.16b
1344
1345 4: // Loop horizontally
1346 ext v26.16b, v0.16b, v1.16b, #2
1347 ext v28.16b, v16.16b, v17.16b, #2
1348 ext v27.16b, v0.16b, v1.16b, #4
1349 ext v29.16b, v16.16b, v17.16b, #4
1350
1351 add v6.8h, v0.8h, v26.8h
1352 umull v22.4s, v0.4h, v0.4h
1353 umlal v22.4s, v26.4h, v26.4h
1354 umlal v22.4s, v27.4h, v27.4h
1355 add v7.8h, v16.8h, v28.8h
1356 umull v24.4s, v16.4h, v16.4h
1357 umlal v24.4s, v28.4h, v28.4h
1358 umlal v24.4s, v29.4h, v29.4h
1359 add v6.8h, v6.8h, v27.8h
1360 umull2 v23.4s, v0.8h, v0.8h
1361 umlal2 v23.4s, v26.8h, v26.8h
1362 umlal2 v23.4s, v27.8h, v27.8h
1363 add v7.8h, v7.8h, v29.8h
1364 umull2 v25.4s, v16.8h, v16.8h
1365 umlal2 v25.4s, v28.8h, v28.8h
1366 umlal2 v25.4s, v29.8h, v29.8h
1367
1368 ext v26.16b, v0.16b, v1.16b, #6
1369 ext v28.16b, v16.16b, v17.16b, #6
1370 ext v27.16b, v0.16b, v1.16b, #8
1371 ext v29.16b, v16.16b, v17.16b, #8
1372
1373 add v6.8h, v6.8h, v26.8h
1374 umlal v22.4s, v26.4h, v26.4h
1375 umlal v22.4s, v27.4h, v27.4h
1376 add v7.8h, v7.8h, v28.8h
1377 umlal v24.4s, v28.4h, v28.4h
1378 umlal v24.4s, v29.4h, v29.4h
1379 add v6.8h, v6.8h, v27.8h
1380 umlal2 v23.4s, v26.8h, v26.8h
1381 umlal2 v23.4s, v27.8h, v27.8h
1382 add v7.8h, v7.8h, v29.8h
1383 umlal2 v25.4s, v28.8h, v28.8h
1384 umlal2 v25.4s, v29.8h, v29.8h
1385
1386 subs w5, w5, #8
1387
1388 st1 {v6.8h}, [x1], #16
1389 st1 {v7.8h}, [x11], #16
1390 st1 {v22.4s,v23.4s}, [x0], #32
1391 st1 {v24.4s,v25.4s}, [x10], #32
1392
1393 b.le 9f
1394 tst w7, #2 // LR_HAVE_RIGHT
1395 mov v0.16b, v1.16b
1396 mov v16.16b, v17.16b
1397 ld1 {v1.8h}, [x3], #16
1398 ld1 {v17.8h}, [x12], #16
1399
1400 b.ne 4b // If we don't need to pad, just keep summing.
1401 b 3b // If we need to pad, check how many pixels we have left.
1402
1403 9:
1404 subs w6, w6, #2
1405 b.le 0f
1406 // Jump to the next row and loop horizontally
1407 add x0, x0, x9, lsl #1
1408 add x10, x10, x9, lsl #1
1409 add x1, x1, x9
1410 add x11, x11, x9
1411 add x3, x3, x4
1412 add x12, x12, x4
1413 mov w5, w8
1414 b 1b
1415 0:
1416 ret
1417 endfunc
1418
1419 sgr_funcs 16