Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/looprestoration.S

raw
1 /*
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2018, Martin Storsjo
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "src/arm/asm.S"
29 #include "util.S"
30
31 const right_ext_mask_buf
32 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
33 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
34 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
35 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
36 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
37 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
38 right_ext_mask:
39 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
40 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
41 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
42 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
43 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
44 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
45 endconst
46
47 // void dav1d_wiener_filter7_8bpc_neon(pixel *p, const ptrdiff_t stride,
48 // const pixel (*left)[4], const pixel *lpf,
49 // const int w, int h,
50 // const int16_t filter[2][8],
51 // const enum LrEdgeFlags edges);
52 function wiener_filter7_8bpc_neon, export=1
53 AARCH64_SIGN_LINK_REGISTER
54 stp x29, x30, [sp, #-16]!
55 mov x29, sp
56 ld1 {v0.8h, v1.8h}, [x6]
57 tst w7, #4 // LR_HAVE_TOP
58 sub_sp 384*2*6
59
60 mov w17, #(1 << 14) - (1 << 2)
61 dup v30.8h, w17
62 movi v31.8h, #8, lsl #8
63
64 // x9 - t6
65 // x10 - t5
66 // x11 - t4
67 // x12 - t3
68 // x13 - t2
69 // x14 - t1
70 // x15 - t0
71 mov x14, sp // t1
72 b.eq L(no_top_7)
73
74 mov x16, x2 // backup left
75 mov x2, #0
76 bl wiener_filter7_h_8bpc_neon
77 add x3, x3, x1 // lpf += stride
78 mov x9, x14 // t6
79 mov x10, x14 // t5
80 add x14, x14, #384*2 // t1 += 384*2
81 bl wiener_filter7_h_8bpc_neon
82 add x3, x3, x1, lsl #2
83 add x3, x3, x1 // lpf += stride*5
84 mov x11, x14 // t4
85 add x14, x14, #384*2 // t1 += 384*2
86 mov x2, x16 // left
87 mov x16, x3 // backup lpf
88 mov x3, x0 // lpf = p
89 bl wiener_filter7_h_8bpc_neon
90 subs w5, w5, #1 // h--
91 mov x12, x14 // t3
92 mov x13, x14 // t2
93 b.eq L(v1_7)
94 add x3, x3, x1 // src += stride
95 add x14, x14, #384*2 // t1 += 384*2
96 bl wiener_filter7_h_8bpc_neon
97 mov x13, x14 // t2
98 subs w5, w5, #1 // h--
99 b.eq L(v2_7)
100 add x3, x3, x1 // src += stride
101 add x14, x14, #384*2 // t1 += 384*2
102 bl wiener_filter7_h_8bpc_neon
103 subs w5, w5, #1 // h--
104 b.eq L(v3_7)
105 add x3, x3, x1 // src += stride
106
107 L(main_7):
108 add x15, x14, #384*2 // t0 = t1 + 384*2
109 L(main_loop_7):
110 bl wiener_filter7_hv_8bpc_neon
111 subs w5, w5, #1 // h--
112 b.ne L(main_loop_7)
113 tst w7, #8 // LR_HAVE_BOTTOM
114 b.eq L(v3_7)
115
116 mov x3, x16 // restore lpf
117 mov x2, #0 // left = NULL
118 bl wiener_filter7_hv_8bpc_neon
119 bl wiener_filter7_hv_8bpc_neon
120 L(v1_7):
121 bl wiener_filter7_v_8bpc_neon
122
123 mov sp, x29
124 ldp x29, x30, [sp], #16
125 AARCH64_VALIDATE_LINK_REGISTER
126 ret
127
128 L(no_top_7):
129 add x3, x3, x1, lsl #2
130 add x16, x3, x1, lsl #1 // lpf += stride*6, backup
131 mov x3, x0 // lpf = p
132
133 bl wiener_filter7_h_8bpc_neon
134 subs w5, w5, #1 // h--
135 mov x9, x14 // t6
136 mov x10, x14 // t5
137 mov x11, x14 // t4
138 mov x12, x14 // t3
139 mov x13, x14 // t2
140 b.eq L(v1_7)
141 add x3, x3, x1 // src += stride
142 add x14, x14, #384*2 // t1 += 384*2
143 bl wiener_filter7_h_8bpc_neon
144 subs w5, w5, #1 // h--
145 mov x13, x14 // t2
146 b.eq L(v2_7)
147 add x3, x3, x1 // src += stride
148 add x14, x14, #384*2 // t1 += 384*2
149 bl wiener_filter7_h_8bpc_neon
150 subs w5, w5, #1 // h--
151 b.eq L(v3_7)
152 add x3, x3, x1 // src += stride
153 add x15, x14, #384*2 // t0 = t1 + 384*2
154 bl wiener_filter7_hv_8bpc_neon
155 subs w5, w5, #1 // h--
156 b.eq L(v3_7)
157 add x15, x15, #384*2*4 // t0 += 384*2*4
158 bl wiener_filter7_hv_8bpc_neon
159 subs w5, w5, #1 // h--
160 b.ne L(main_7)
161 L(v3_7):
162 bl wiener_filter7_v_8bpc_neon
163 L(v2_7):
164 bl wiener_filter7_v_8bpc_neon
165 b L(v1_7)
166 endfunc
167
168
169 function wiener_filter7_h_8bpc_neon
170 stp x3, x4, [sp, #-32]!
171 str x14, [sp, #16]
172
173 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
174 tst w7, #1 // LR_HAVE_LEFT
175 b.eq 1f
176 // LR_HAVE_LEFT
177 cbnz x2, 0f
178 // left == NULL
179 sub x3, x3, #3
180 ld1 {v3.16b}, [x3], #16
181 b 2f
182
183 0:
184 // LR_HAVE_LEFT, left != NULL
185 ld1 {v3.16b}, [x3], #16
186 ld1 {v2.s}[3], [x2], #4
187 // Move x3 back to account for the last 3 bytes we loaded earlier,
188 // which we'll shift out.
189 sub x3, x3, #3
190 ext v3.16b, v2.16b, v3.16b, #13
191 b 2f
192
193 1:
194 ld1 {v3.16b}, [x3], #16
195 // !LR_HAVE_LEFT, fill v2 with the leftmost byte
196 // and shift v3 to have 3x the first byte at the front.
197 dup v2.16b, v3.b[0]
198 // Move x3 back to account for the last 3 bytes we loaded before,
199 // which we shifted out.
200 sub x3, x3, #3
201 ext v3.16b, v2.16b, v3.16b, #13
202
203 2:
204 ld1 {v4.8b}, [x3], #8
205 uxtl v2.8h, v3.8b
206 uxtl2 v3.8h, v3.16b
207 uxtl v4.8h, v4.8b
208
209 tst w7, #2 // LR_HAVE_RIGHT
210 b.ne 4f
211
212 3: // !LR_HAVE_RIGHT
213
214 // Check whether we need to pad the right edge
215 cmp w4, #19
216 b.ge 4f // If w >= 19, all used input pixels are valid
217
218 // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9,
219 // this ends up called again; it's not strictly needed in those
220 // cases (we pad enough here), but keeping the code as simple as possible.
221
222 // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
223 // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel.
224 sub w17, w4, #22
225 // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the
226 // buffer pointer.
227 movrel x6, right_ext_mask, -6
228 ldr b28, [x3, w17, sxtw]
229 sub x6, x6, w4, uxtw #1
230 dup v28.8h, v28.h[0]
231 ld1 {v25.16b, v26.16b, v27.16b}, [x6]
232
233 bit v2.16b, v28.16b, v25.16b
234 bit v3.16b, v28.16b, v26.16b
235 bit v4.16b, v28.16b, v27.16b
236
237 4: // Loop horizontally
238 // Interleaving the mul/mla chains actually hurts performance
239 // significantly on Cortex A53, thus keeping mul/mla tightly
240 // chained like this.
241 ext v17.16b, v2.16b, v3.16b, #4
242 ext v19.16b, v2.16b, v3.16b, #8
243 ext v16.16b, v2.16b, v3.16b, #2
244 ext v20.16b, v2.16b, v3.16b, #10
245 ext v21.16b, v2.16b, v3.16b, #12
246 ext v18.16b, v2.16b, v3.16b, #6
247 add v19.8h, v19.8h, v17.8h
248 add v20.8h, v20.8h, v16.8h
249 add v21.8h, v21.8h, v2.8h
250 shl v22.8h, v18.8h, #7
251 mul v6.8h, v18.8h, v0.h[3]
252 mla v6.8h, v19.8h, v0.h[4]
253 mla v6.8h, v20.8h, v0.h[5]
254 mla v6.8h, v21.8h, v0.h[6]
255
256 ext v17.16b, v3.16b, v4.16b, #4
257 ext v19.16b, v3.16b, v4.16b, #8
258 ext v16.16b, v3.16b, v4.16b, #2
259 ext v20.16b, v3.16b, v4.16b, #10
260 ext v21.16b, v3.16b, v4.16b, #12
261 ext v18.16b, v3.16b, v4.16b, #6
262
263 add v19.8h, v19.8h, v17.8h
264 add v20.8h, v20.8h, v16.8h
265 add v21.8h, v21.8h, v3.8h
266 shl v23.8h, v18.8h, #7
267 mul v7.8h, v18.8h, v0.h[3]
268 mla v7.8h, v19.8h, v0.h[4]
269 mla v7.8h, v20.8h, v0.h[5]
270 mla v7.8h, v21.8h, v0.h[6]
271
272 sub v22.8h, v22.8h, v30.8h
273 sub v23.8h, v23.8h, v30.8h
274 sqadd v6.8h, v6.8h, v22.8h
275 sqadd v7.8h, v7.8h, v23.8h
276 sshr v6.8h, v6.8h, #3
277 sshr v7.8h, v7.8h, #3
278 add v6.8h, v6.8h, v31.8h
279 add v7.8h, v7.8h, v31.8h
280
281 subs w4, w4, #16
282
283 st1 {v6.8h, v7.8h}, [x14], #32
284
285 b.le 0f
286 mov v2.16b, v4.16b
287 ld1 {v4.16b}, [x3], #16
288 tst w7, #2 // LR_HAVE_RIGHT
289 uxtl v3.8h, v4.8b
290 uxtl2 v4.8h, v4.16b
291 b.ne 4b // If we don't need to pad, just keep filtering.
292 b 3b // If we need to pad, check how many pixels we have left.
293
294 0:
295 ldr x14, [sp, #16]
296 ldp x3, x4, [sp], #32
297 ret
298 endfunc
299
300 function wiener_filter7_v_8bpc_neon
301 // Backing up/restoring registers shifted, so that x9 gets the value
302 // of x10, etc, afterwards.
303 stp x10, x11, [sp, #-64]!
304 stp x12, x13, [sp, #16]
305 stp x14, x14, [sp, #32]
306 stp x0, x4, [sp, #48]
307 1:
308 ld1 {v20.8h, v21.8h}, [x11], #32
309 ld1 {v24.8h, v25.8h}, [x13], #32
310
311 ld1 {v18.8h, v19.8h}, [x10], #32
312 add v24.8h, v24.8h, v20.8h
313 ld1 {v26.8h, v27.8h}, [x14], #32
314
315 ld1 {v16.8h, v17.8h}, [x9], #32
316 add v28.8h, v26.8h, v18.8h
317 ld1 {v22.8h, v23.8h}, [x12], #32
318
319 add v16.8h, v26.8h, v16.8h
320 add v25.8h, v25.8h, v21.8h
321
322 smull v2.4s, v22.4h, v1.h[3]
323 smlal v2.4s, v24.4h, v1.h[4]
324 smlal v2.4s, v28.4h, v1.h[5]
325 smlal v2.4s, v16.4h, v1.h[6]
326 add v29.8h, v27.8h, v19.8h
327 smull2 v3.4s, v22.8h, v1.h[3]
328 smlal2 v3.4s, v24.8h, v1.h[4]
329 smlal2 v3.4s, v28.8h, v1.h[5]
330 smlal2 v3.4s, v16.8h, v1.h[6]
331 add v17.8h, v27.8h, v17.8h
332 smull v4.4s, v23.4h, v1.h[3]
333 smlal v4.4s, v25.4h, v1.h[4]
334 smlal v4.4s, v29.4h, v1.h[5]
335 smlal v4.4s, v17.4h, v1.h[6]
336 smull2 v5.4s, v23.8h, v1.h[3]
337 smlal2 v5.4s, v25.8h, v1.h[4]
338 smlal2 v5.4s, v29.8h, v1.h[5]
339 smlal2 v5.4s, v17.8h, v1.h[6]
340 sqrshrun v2.4h, v2.4s, #11
341 sqrshrun2 v2.8h, v3.4s, #11
342 sqrshrun v3.4h, v4.4s, #11
343 sqrshrun2 v3.8h, v5.4s, #11
344 sqxtun v2.8b, v2.8h
345 sqxtun2 v2.16b, v3.8h
346 subs w4, w4, #16
347 st1 {v2.16b}, [x0], #16
348 b.gt 1b
349
350 ldp x0, x4, [sp, #48]
351 ldp x13, x14, [sp, #32]
352 ldp x11, x12, [sp, #16]
353 ldp x9, x10, [sp], #64
354
355 add x0, x0, x1
356 ret
357 endfunc
358
359 function wiener_filter7_hv_8bpc_neon
360 // Backing up/restoring registers shifted, so that x9 gets the value
361 // of x10, etc, and x15==x9, afterwards.
362 stp x10, x11, [sp, #-80]!
363 stp x12, x13, [sp, #16]
364 stp x14, x15, [sp, #32]
365 stp x10, x0, [sp, #48]
366 stp x3, x4, [sp, #64]
367
368 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
369 tst w7, #1 // LR_HAVE_LEFT
370 b.eq 1f
371 // LR_HAVE_LEFT
372 cbnz x2, 0f
373 // left == NULL
374 sub x3, x3, #3
375 ld1 {v3.16b}, [x3], #16
376 b 2f
377
378 0:
379 // LR_HAVE_LEFT, left != NULL
380 ld1 {v3.16b}, [x3], #16
381 ld1 {v2.s}[3], [x2], #4
382 // Move x3 back to account for the last 3 bytes we loaded earlier,
383 // which we'll shift out.
384 sub x3, x3, #3
385 ext v3.16b, v2.16b, v3.16b, #13
386 b 2f
387 1:
388 ld1 {v3.16b}, [x3], #16
389 // !LR_HAVE_LEFT, fill v2 with the leftmost byte
390 // and shift v3 to have 3x the first byte at the front.
391 dup v2.16b, v3.b[0]
392 // Move x3 back to account for the last 3 bytes we loaded before,
393 // which we shifted out.
394 sub x3, x3, #3
395 ext v3.16b, v2.16b, v3.16b, #13
396
397 2:
398 ld1 {v4.8b}, [x3], #8
399 uxtl v2.8h, v3.8b
400 uxtl2 v3.8h, v3.16b
401 uxtl v4.8h, v4.8b
402
403 tst w7, #2 // LR_HAVE_RIGHT
404 b.ne 4f
405
406 3: // !LR_HAVE_RIGHT
407
408 // Check whether we need to pad the right edge
409 cmp w4, #19
410 b.ge 4f // If w >= 19, all used input pixels are valid
411
412 // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9,
413 // this ends up called again; it's not strictly needed in those
414 // cases (we pad enough here), but keeping the code as simple as possible.
415
416 // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
417 // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel.
418 sub w17, w4, #22
419 // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the
420 // buffer pointer.
421 movrel x6, right_ext_mask, -6
422 ldr b28, [x3, w17, sxtw]
423 sub x6, x6, w4, uxtw #1
424 dup v28.8h, v28.h[0]
425 ld1 {v25.16b, v26.16b, v27.16b}, [x6]
426
427 bit v2.16b, v28.16b, v25.16b
428 bit v3.16b, v28.16b, v26.16b
429 bit v4.16b, v28.16b, v27.16b
430
431 4: // Loop horizontally
432 ext v17.16b, v2.16b, v3.16b, #4
433 ext v19.16b, v2.16b, v3.16b, #8
434 ext v16.16b, v2.16b, v3.16b, #2
435 ext v20.16b, v2.16b, v3.16b, #10
436 ext v21.16b, v2.16b, v3.16b, #12
437 ext v18.16b, v2.16b, v3.16b, #6
438 add v19.8h, v19.8h, v17.8h
439 add v20.8h, v20.8h, v16.8h
440 add v21.8h, v21.8h, v2.8h
441 shl v22.8h, v18.8h, #7
442 mul v6.8h, v18.8h, v0.h[3]
443 mla v6.8h, v19.8h, v0.h[4]
444 mla v6.8h, v20.8h, v0.h[5]
445 mla v6.8h, v21.8h, v0.h[6]
446
447 ext v17.16b, v3.16b, v4.16b, #4
448 ext v19.16b, v3.16b, v4.16b, #8
449 ext v16.16b, v3.16b, v4.16b, #2
450 ext v20.16b, v3.16b, v4.16b, #10
451 ext v21.16b, v3.16b, v4.16b, #12
452 ext v18.16b, v3.16b, v4.16b, #6
453
454 add v19.8h, v19.8h, v17.8h
455 add v20.8h, v20.8h, v16.8h
456 add v21.8h, v21.8h, v3.8h
457 shl v23.8h, v18.8h, #7
458 mul v7.8h, v18.8h, v0.h[3]
459 mla v7.8h, v19.8h, v0.h[4]
460 mla v7.8h, v20.8h, v0.h[5]
461 mla v7.8h, v21.8h, v0.h[6]
462
463 ld1 {v20.8h, v21.8h}, [x11], #32
464
465 sub v22.8h, v22.8h, v30.8h
466 sub v23.8h, v23.8h, v30.8h
467 ld1 {v26.8h, v27.8h}, [x13], #32
468 sqadd v6.8h, v6.8h, v22.8h
469 sqadd v7.8h, v7.8h, v23.8h
470 ld1 {v18.8h, v19.8h}, [x10], #32
471 sshr v6.8h, v6.8h, #3
472 sshr v7.8h, v7.8h, #3
473 ld1 {v28.8h, v29.8h}, [x14], #32
474 add v6.8h, v6.8h, v31.8h
475 add v7.8h, v7.8h, v31.8h
476
477 ld1 {v16.8h, v17.8h}, [x9], #32
478 add v26.8h, v20.8h, v26.8h
479
480 ld1 {v24.8h, v25.8h}, [x12], #32
481 add v28.8h, v18.8h, v28.8h
482
483 add v16.8h, v16.8h, v6.8h
484 add v27.8h, v21.8h, v27.8h
485
486 smull v18.4s, v24.4h, v1.h[3]
487 smlal v18.4s, v26.4h, v1.h[4]
488 smlal v18.4s, v28.4h, v1.h[5]
489 smlal v18.4s, v16.4h, v1.h[6]
490 add v29.8h, v19.8h, v29.8h
491 smull2 v19.4s, v24.8h, v1.h[3]
492 smlal2 v19.4s, v26.8h, v1.h[4]
493 smlal2 v19.4s, v28.8h, v1.h[5]
494 smlal2 v19.4s, v16.8h, v1.h[6]
495 add v17.8h, v17.8h, v7.8h
496 smull v20.4s, v25.4h, v1.h[3]
497 smlal v20.4s, v27.4h, v1.h[4]
498 smlal v20.4s, v29.4h, v1.h[5]
499 smlal v20.4s, v17.4h, v1.h[6]
500 smull2 v21.4s, v25.8h, v1.h[3]
501 smlal2 v21.4s, v27.8h, v1.h[4]
502 smlal2 v21.4s, v29.8h, v1.h[5]
503 smlal2 v21.4s, v17.8h, v1.h[6]
504 sqrshrun v18.4h, v18.4s, #11
505 sqrshrun2 v18.8h, v19.4s, #11
506 sqrshrun v19.4h, v20.4s, #11
507 sqrshrun2 v19.8h, v21.4s, #11
508 st1 {v6.8h, v7.8h}, [x15], #32
509 sqxtun v18.8b, v18.8h
510 sqxtun2 v18.16b, v19.8h
511 subs w4, w4, #16
512
513 st1 {v18.16b}, [x0], #16
514
515 b.le 0f
516 mov v2.16b, v4.16b
517 ld1 {v4.16b}, [x3], #16
518 tst w7, #2 // LR_HAVE_RIGHT
519 uxtl v3.8h, v4.8b
520 uxtl2 v4.8h, v4.16b
521 b.ne 4b // If we don't need to pad, just keep filtering.
522 b 3b // If we need to pad, check how many pixels we have left.
523
524 0:
525 ldp x3, x4, [sp, #64]
526 ldp x15, x0, [sp, #48]
527 ldp x13, x14, [sp, #32]
528 ldp x11, x12, [sp, #16]
529 ldp x9, x10, [sp], #80
530
531 add x3, x3, x1
532 add x0, x0, x1
533
534 ret
535 endfunc
536
537 // void dav1d_wiener_filter5_8bpc_neon(pixel *p, const ptrdiff_t stride,
538 // const pixel (*left)[4], const pixel *lpf,
539 // const int w, int h,
540 // const int16_t filter[2][8],
541 // const enum LrEdgeFlags edges);
542 function wiener_filter5_8bpc_neon, export=1
543 AARCH64_SIGN_LINK_REGISTER
544 stp x29, x30, [sp, #-16]!
545 mov x29, sp
546 ld1 {v0.8h, v1.8h}, [x6]
547 tst w7, #4 // LR_HAVE_TOP
548 sub_sp 384*2*4
549
550 mov w17, #(1 << 14) - (1 << 2)
551 dup v30.8h, w17
552 movi v31.8h, #8, lsl #8
553
554 // x11 - t4
555 // x12 - t3
556 // x13 - t2
557 // x14 - t1
558 // x15 - t0
559 mov x14, sp // t1
560 b.eq L(no_top_5)
561
562 mov x16, x2 // backup left
563 mov x2, #0
564 bl wiener_filter5_h_8bpc_neon
565 add x3, x3, x1 // lpf += stride
566 mov x11, x14 // t4
567 add x14, x14, #384*2 // t1 += 384*2
568 bl wiener_filter5_h_8bpc_neon
569 add x3, x3, x1, lsl #2
570 add x3, x3, x1 // lpf += stride*5
571 mov x12, x14 // t3
572 add x14, x14, #384*2 // t1 += 384*2
573 mov x2, x16 // left
574 mov x16, x3 // backup lpf
575 mov x3, x0 // lpf = p
576 bl wiener_filter5_h_8bpc_neon
577 subs w5, w5, #1 // h--
578 mov x13, x14 // t2
579 b.eq L(v1_5)
580 add x3, x3, x1 // src += stride
581 add x14, x14, #384*2 // t1 += 384*2
582 bl wiener_filter5_h_8bpc_neon
583 subs w5, w5, #1 // h--
584 b.eq L(v2_5)
585 add x3, x3, x1 // src += stride
586
587 L(main_5):
588 mov x15, x11 // t0 = t4
589 L(main_loop_5):
590 bl wiener_filter5_hv_8bpc_neon
591 subs w5, w5, #1 // h--
592 b.ne L(main_loop_5)
593 tst w7, #8 // LR_HAVE_BOTTOM
594 b.eq L(v2_5)
595
596 mov x3, x16 // restore lpf
597 mov x2, #0 // left = NULL
598 bl wiener_filter5_hv_8bpc_neon
599 bl wiener_filter5_hv_8bpc_neon
600 L(end_5):
601
602 mov sp, x29
603 ldp x29, x30, [sp], #16
604 AARCH64_VALIDATE_LINK_REGISTER
605 ret
606
607 L(no_top_5):
608 add x3, x3, x1, lsl #2
609 add x16, x3, x1, lsl #1 // lpf += stride*6, backup
610 mov x3, x0 // lpf = p
611
612 bl wiener_filter5_h_8bpc_neon
613 subs w5, w5, #1 // h--
614 mov x11, x14 // t4
615 mov x12, x14 // t3
616 mov x13, x14 // t2
617 b.eq L(v1_5)
618 add x3, x3, x1 // src += stride
619 add x14, x14, #384*2 // t1 += 384*2
620 bl wiener_filter5_h_8bpc_neon
621 subs w5, w5, #1 // h--
622 b.eq L(v2_5)
623 add x3, x3, x1 // src += stride
624 add x15, x14, #384*2 // t0 = t1 + 384*2
625 bl wiener_filter5_hv_8bpc_neon
626 subs w5, w5, #1 // h--
627 b.eq L(v2_5)
628 add x15, x15, #384*2*3 // t0 += 384*2*3
629 bl wiener_filter5_hv_8bpc_neon
630 subs w5, w5, #1 // h--
631 b.ne L(main_5)
632 L(v2_5):
633 bl wiener_filter5_v_8bpc_neon
634 add x0, x0, x1
635 mov x11, x12
636 mov x12, x13
637 mov x13, x14
638 L(v1_5):
639 bl wiener_filter5_v_8bpc_neon
640 b L(end_5)
641 endfunc
642
643
644 function wiener_filter5_h_8bpc_neon
645 stp x3, x4, [sp, #-32]!
646 str x14, [sp, #16]
647
648 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
649 tst w7, #1 // LR_HAVE_LEFT
650 b.eq 1f
651 // LR_HAVE_LEFT
652 cbnz x2, 0f
653 // left == NULL
654 sub x3, x3, #2
655 ld1 {v3.16b}, [x3], #16
656 b 2f
657
658 0:
659 // LR_HAVE_LEFT, left != NULL
660 ld1 {v3.16b}, [x3], #16
661 ld1 {v2.s}[3], [x2], #4
662 // Move x3 back to account for the last 2 bytes we loaded earlier,
663 // which we'll shift out.
664 sub x3, x3, #2
665 ext v3.16b, v2.16b, v3.16b, #14
666 b 2f
667
668 1:
669 ld1 {v3.16b}, [x3], #16
670 // !LR_HAVE_LEFT, fill v2 with the leftmost byte
671 // and shift v3 to have 3x the first byte at the front.
672 dup v2.16b, v3.b[0]
673 // Move x3 back to account for the last 2 bytes we loaded before,
674 // which we shifted out.
675 sub x3, x3, #2
676 ext v3.16b, v2.16b, v3.16b, #14
677
678 2:
679 ld1 {v4.8b}, [x3], #8
680 uxtl v2.8h, v3.8b
681 uxtl2 v3.8h, v3.16b
682 uxtl v4.8h, v4.8b
683
684 tst w7, #2 // LR_HAVE_RIGHT
685 b.ne 4f
686
687 3: // !LR_HAVE_RIGHT
688
689 // Check whether we need to pad the right edge
690 cmp w4, #18
691 b.ge 4f // If w >= 18, all used input pixels are valid
692
693 // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9,
694 // this ends up called again; it's not strictly needed in those
695 // cases (we pad enough here), but keeping the code as simple as possible.
696
697 // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie
698 // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel.
699 sub w17, w4, #23
700 // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the
701 // buffer pointer.
702 movrel x6, right_ext_mask, -4
703 ldr b28, [x3, w17, sxtw]
704 sub x6, x6, w4, uxtw #1
705 dup v28.8h, v28.h[0]
706 ld1 {v25.16b, v26.16b, v27.16b}, [x6]
707
708 bit v2.16b, v28.16b, v25.16b
709 bit v3.16b, v28.16b, v26.16b
710 bit v4.16b, v28.16b, v27.16b
711
712 4: // Loop horizontally
713 // Interleaving the mul/mla chains actually hurts performance
714 // significantly on Cortex A53, thus keeping mul/mla tightly
715 // chained like this.
716 ext v16.16b, v2.16b, v3.16b, #2
717 ext v18.16b, v2.16b, v3.16b, #6
718 ext v19.16b, v2.16b, v3.16b, #8
719 ext v17.16b, v2.16b, v3.16b, #4
720 add v18.8h, v18.8h, v16.8h
721 add v19.8h, v19.8h, v2.8h
722 shl v22.8h, v17.8h, #7
723 mul v6.8h, v17.8h, v0.h[3]
724 mla v6.8h, v18.8h, v0.h[4]
725 mla v6.8h, v19.8h, v0.h[5]
726
727 ext v16.16b, v3.16b, v4.16b, #2
728 ext v18.16b, v3.16b, v4.16b, #6
729 ext v19.16b, v3.16b, v4.16b, #8
730 ext v17.16b, v3.16b, v4.16b, #4
731 add v18.8h, v18.8h, v16.8h
732 add v19.8h, v19.8h, v3.8h
733 shl v23.8h, v17.8h, #7
734 mul v7.8h, v17.8h, v0.h[3]
735 mla v7.8h, v18.8h, v0.h[4]
736 mla v7.8h, v19.8h, v0.h[5]
737
738 sub v22.8h, v22.8h, v30.8h
739 sub v23.8h, v23.8h, v30.8h
740 sqadd v6.8h, v6.8h, v22.8h
741 sqadd v7.8h, v7.8h, v23.8h
742 sshr v6.8h, v6.8h, #3
743 sshr v7.8h, v7.8h, #3
744 add v6.8h, v6.8h, v31.8h
745 add v7.8h, v7.8h, v31.8h
746
747 subs w4, w4, #16
748
749 st1 {v6.8h, v7.8h}, [x14], #32
750
751 b.le 0f
752 mov v2.16b, v4.16b
753 ld1 {v4.16b}, [x3], #16
754 tst w7, #2 // LR_HAVE_RIGHT
755 uxtl v3.8h, v4.8b
756 uxtl2 v4.8h, v4.16b
757 b.ne 4b // If we don't need to pad, just keep filtering.
758 b 3b // If we need to pad, check how many pixels we have left.
759
760 0:
761 ldr x14, [sp, #16]
762 ldp x3, x4, [sp], #32
763 ret
764 endfunc
765
766 function wiener_filter5_v_8bpc_neon
767 stp x11, x12, [sp, #-48]!
768 stp x13, x14, [sp, #16]
769 stp x0, x4, [sp, #32]
770 1:
771 ld1 {v18.8h, v19.8h}, [x12], #32
772 ld1 {v22.8h, v23.8h}, [x14], #32
773 ld1 {v16.8h, v17.8h}, [x11], #32
774
775 add v24.8h, v22.8h, v18.8h
776 ld1 {v20.8h, v21.8h}, [x13], #32
777 add v16.8h, v22.8h, v16.8h
778 add v25.8h, v23.8h, v19.8h
779
780 smull v2.4s, v20.4h, v1.h[3]
781 smlal v2.4s, v24.4h, v1.h[4]
782 smlal v2.4s, v16.4h, v1.h[5]
783 add v17.8h, v23.8h, v17.8h
784 smull2 v3.4s, v20.8h, v1.h[3]
785 smlal2 v3.4s, v24.8h, v1.h[4]
786 smlal2 v3.4s, v16.8h, v1.h[5]
787 smull v4.4s, v21.4h, v1.h[3]
788 smlal v4.4s, v25.4h, v1.h[4]
789 smlal v4.4s, v17.4h, v1.h[5]
790 smull2 v5.4s, v21.8h, v1.h[3]
791 smlal2 v5.4s, v25.8h, v1.h[4]
792 smlal2 v5.4s, v17.8h, v1.h[5]
793 sqrshrun v2.4h, v2.4s, #11
794 sqrshrun2 v2.8h, v3.4s, #11
795 sqrshrun v3.4h, v4.4s, #11
796 sqrshrun2 v3.8h, v5.4s, #11
797 sqxtun v2.8b, v2.8h
798 sqxtun2 v2.16b, v3.8h
799 subs w4, w4, #16
800 st1 {v2.16b}, [x0], #16
801 b.gt 1b
802
803 ldp x0, x4, [sp, #32]
804 ldp x13, x14, [sp, #16]
805 ldp x11, x12, [sp], #48
806
807 ret
808 endfunc
809
810 function wiener_filter5_hv_8bpc_neon
811 // Backing up/restoring registers shifted, so that x11 gets the value
812 // of x12, etc, and x15==x11, afterwards.
813 stp x12, x13, [sp, #-64]!
814 stp x14, x15, [sp, #16]
815 stp x12, x0, [sp, #32]
816 stp x3, x4, [sp, #48]
817
818 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
819 tst w7, #1 // LR_HAVE_LEFT
820 b.eq 1f
821 // LR_HAVE_LEFT
822 cbnz x2, 0f
823 // left == NULL
824 sub x3, x3, #2
825 ld1 {v3.16b}, [x3], #16
826 b 2f
827
828 0:
829 // LR_HAVE_LEFT, left != NULL
830 ld1 {v3.16b}, [x3], #16
831 ld1 {v2.s}[3], [x2], #4
832 // Move x3 back to account for the last 2 bytes we loaded earlier,
833 // which we'll shift out.
834 sub x3, x3, #2
835 ext v3.16b, v2.16b, v3.16b, #14
836 b 2f
837 1:
838 ld1 {v3.16b}, [x3], #16
839 // !LR_HAVE_LEFT, fill v2 with the leftmost byte
840 // and shift v3 to have 2x the first byte at the front.
841 dup v2.16b, v3.b[0]
842 // Move x3 back to account for the last 2 bytes we loaded before,
843 // which we shifted out.
844 sub x3, x3, #2
845 ext v3.16b, v2.16b, v3.16b, #14
846
847 2:
848 ld1 {v4.8b}, [x3], #8
849 uxtl v2.8h, v3.8b
850 uxtl2 v3.8h, v3.16b
851 uxtl v4.8h, v4.8b
852
853 tst w7, #2 // LR_HAVE_RIGHT
854 b.ne 4f
855
856 3: // !LR_HAVE_RIGHT
857
858 // Check whether we need to pad the right edge
859 cmp w4, #18
860 b.ge 4f // If w >= 18, all used input pixels are valid
861
862 // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9,
863 // this ends up called again; it's not strictly needed in those
864 // cases (we pad enough here), but keeping the code as simple as possible.
865
866 // The padding pixel is v2/3/4.h[w+1]. x3 points at the next input, ie
867 // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel.
868 sub w17, w4, #23
869 // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the
870 // buffer pointer.
871 movrel x6, right_ext_mask, -4
872 ldr b28, [x3, w17, sxtw]
873 sub x6, x6, w4, uxtw #1
874 dup v28.8h, v28.h[0]
875 ld1 {v25.16b, v26.16b, v27.16b}, [x6]
876
877 bit v2.16b, v28.16b, v25.16b
878 bit v3.16b, v28.16b, v26.16b
879 bit v4.16b, v28.16b, v27.16b
880
881 4: // Loop horizontally
882
883 ext v16.16b, v2.16b, v3.16b, #2
884 ext v18.16b, v2.16b, v3.16b, #6
885 ext v19.16b, v2.16b, v3.16b, #8
886 ext v17.16b, v2.16b, v3.16b, #4
887 add v18.8h, v18.8h, v16.8h
888 add v19.8h, v19.8h, v2.8h
889 shl v22.8h, v17.8h, #7
890 mul v6.8h, v17.8h, v0.h[3]
891 mla v6.8h, v18.8h, v0.h[4]
892 mla v6.8h, v19.8h, v0.h[5]
893
894 ext v16.16b, v3.16b, v4.16b, #2
895 ext v18.16b, v3.16b, v4.16b, #6
896 ext v19.16b, v3.16b, v4.16b, #8
897 ext v17.16b, v3.16b, v4.16b, #4
898 add v18.8h, v18.8h, v16.8h
899 add v19.8h, v19.8h, v3.8h
900 shl v23.8h, v17.8h, #7
901 mul v7.8h, v17.8h, v0.h[3]
902 mla v7.8h, v18.8h, v0.h[4]
903 mla v7.8h, v19.8h, v0.h[5]
904
905 ld1 {v18.8h, v19.8h}, [x12], #32
906
907 sub v22.8h, v22.8h, v30.8h
908 sub v23.8h, v23.8h, v30.8h
909 ld1 {v24.8h, v25.8h}, [x14], #32
910 sqadd v6.8h, v6.8h, v22.8h
911 sqadd v7.8h, v7.8h, v23.8h
912 ld1 {v16.8h, v17.8h}, [x11], #32
913 sshr v6.8h, v6.8h, #3
914 sshr v7.8h, v7.8h, #3
915 ld1 {v20.8h, v21.8h}, [x13], #32
916 add v6.8h, v6.8h, v31.8h
917 add v7.8h, v7.8h, v31.8h
918
919 add v24.8h, v24.8h, v18.8h
920 add v16.8h, v16.8h, v6.8h
921
922 smull v18.4s, v20.4h, v1.h[3]
923 smlal v18.4s, v24.4h, v1.h[4]
924 smlal v18.4s, v16.4h, v1.h[5]
925 add v25.8h, v25.8h, v19.8h
926 smull2 v19.4s, v20.8h, v1.h[3]
927 smlal2 v19.4s, v24.8h, v1.h[4]
928 smlal2 v19.4s, v16.8h, v1.h[5]
929 add v17.8h, v17.8h, v7.8h
930 smull v20.4s, v21.4h, v1.h[3]
931 smlal v20.4s, v25.4h, v1.h[4]
932 smlal v20.4s, v17.4h, v1.h[5]
933 smull2 v21.4s, v21.8h, v1.h[3]
934 smlal2 v21.4s, v25.8h, v1.h[4]
935 smlal2 v21.4s, v17.8h, v1.h[5]
936 sqrshrun v18.4h, v18.4s, #11
937 sqrshrun2 v18.8h, v19.4s, #11
938 sqrshrun v19.4h, v20.4s, #11
939 sqrshrun2 v19.8h, v21.4s, #11
940 st1 {v6.8h, v7.8h}, [x15], #32
941 sqxtun v18.8b, v18.8h
942 sqxtun2 v18.16b, v19.8h
943 subs w4, w4, #16
944
945 st1 {v18.16b}, [x0], #16
946
947 b.le 0f
948 mov v2.16b, v4.16b
949 ld1 {v4.16b}, [x3], #16
950 tst w7, #2 // LR_HAVE_RIGHT
951 uxtl v3.8h, v4.8b
952 uxtl2 v4.8h, v4.16b
953 b.ne 4b // If we don't need to pad, just keep filtering.
954 b 3b // If we need to pad, check how many pixels we have left.
955
956 0:
957 ldp x3, x4, [sp, #48]
958 ldp x15, x0, [sp, #32]
959 ldp x13, x14, [sp, #16]
960 ldp x11, x12, [sp], #64
961
962 add x3, x3, x1
963 add x0, x0, x1
964
965 ret
966 endfunc
967
968 #define SUM_STRIDE (384+16)
969
970 #include "looprestoration_tmpl.S"
971
972 // void dav1d_sgr_box3_h_8bpc_neon(int32_t *sumsq, int16_t *sum,
973 // const pixel (*left)[4],
974 // const pixel *src, const ptrdiff_t stride,
975 // const int w, const int h,
976 // const enum LrEdgeFlags edges);
977 function sgr_box3_h_8bpc_neon, export=1
978 add w5, w5, #2 // w += 2
979
980 // Set up pointers for reading/writing alternate rows
981 add x10, x0, #(4*SUM_STRIDE) // sumsq
982 add x11, x1, #(2*SUM_STRIDE) // sum
983 add x12, x3, x4 // src
984 lsl x4, x4, #1
985 mov x9, #(2*2*SUM_STRIDE) // double sum stride
986
987 // Subtract the aligned width from the output stride.
988 add w13, w5, #7
989 bic w13, w13, #7
990 sub x9, x9, w13, uxtw #1
991
992 // Store the width for the vertical loop
993 mov w8, w5
994
995 // Subtract the number of pixels read from the input from the stride
996 add w13, w13, #8
997 sub x4, x4, w13, uxtw
998
999 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
1000 tst w7, #1 // LR_HAVE_LEFT
1001 b.eq 2f
1002 // LR_HAVE_LEFT
1003 cbnz x2, 0f
1004 // left == NULL
1005 sub x3, x3, #2
1006 sub x12, x12, #2
1007 b 1f
1008 0: // LR_HAVE_LEFT, left != NULL
1009 2: // !LR_HAVE_LEFT, increase the stride.
1010 // For this case we don't read the left 2 pixels from the src pointer,
1011 // but shift it as if we had done that.
1012 add x4, x4, #2
1013
1014
1015 1: // Loop vertically
1016 ld1 {v0.16b}, [x3], #16
1017 ld1 {v4.16b}, [x12], #16
1018
1019 tst w7, #1 // LR_HAVE_LEFT
1020 b.eq 0f
1021 cbz x2, 2f
1022 // LR_HAVE_LEFT, left != NULL
1023 ld1 {v1.s}[3], [x2], #4
1024 // Move x3/x12 back to account for the last 2 bytes we loaded earlier,
1025 // which we'll shift out.
1026 sub x3, x3, #2
1027 sub x12, x12, #2
1028 ld1 {v5.s}[3], [x2], #4
1029 ext v0.16b, v1.16b, v0.16b, #14
1030 ext v4.16b, v5.16b, v4.16b, #14
1031 b 2f
1032 0:
1033 // !LR_HAVE_LEFT, fill v1 with the leftmost byte
1034 // and shift v0 to have 2x the first byte at the front.
1035 dup v1.16b, v0.b[0]
1036 dup v5.16b, v4.b[0]
1037 // Move x3 back to account for the last 2 bytes we loaded before,
1038 // which we shifted out.
1039 sub x3, x3, #2
1040 sub x12, x12, #2
1041 ext v0.16b, v1.16b, v0.16b, #14
1042 ext v4.16b, v5.16b, v4.16b, #14
1043
1044 2:
1045 umull v1.8h, v0.8b, v0.8b
1046 umull2 v2.8h, v0.16b, v0.16b
1047 umull v5.8h, v4.8b, v4.8b
1048 umull2 v6.8h, v4.16b, v4.16b
1049
1050 tst w7, #2 // LR_HAVE_RIGHT
1051 b.ne 4f
1052 // If we'll need to pad the right edge, load that byte to pad with
1053 // here since we can find it pretty easily from here.
1054 sub w13, w5, #(2 + 16 - 2 + 1)
1055 ldr b30, [x3, w13, sxtw]
1056 ldr b31, [x12, w13, sxtw]
1057 // Fill v30/v31 with the right padding pixel
1058 dup v30.16b, v30.b[0]
1059 dup v31.16b, v31.b[0]
1060 3: // !LR_HAVE_RIGHT
1061
1062 // Check whether we need to pad the right edge
1063 cmp w5, #10
1064 b.ge 4f // If w >= 10, all used input pixels are valid
1065
1066 // 1 <= w < 10, w pixels valid in v0. For w=9, this ends up called
1067 // again; it's not strictly needed in those cases (we pad enough here),
1068 // but keeping the code as simple as possible.
1069
1070 // Insert padding in v0/4.b[w] onwards
1071 movrel x13, right_ext_mask
1072 sub x13, x13, w5, uxtw
1073 ld1 {v29.16b}, [x13]
1074
1075 bit v0.16b, v30.16b, v29.16b
1076 bit v4.16b, v31.16b, v29.16b
1077
1078 // Update the precalculated squares
1079 umull v1.8h, v0.8b, v0.8b
1080 umull2 v2.8h, v0.16b, v0.16b
1081 umull v5.8h, v4.8b, v4.8b
1082 umull2 v6.8h, v4.16b, v4.16b
1083
1084 4: // Loop horizontally
1085 ext v16.16b, v0.16b, v0.16b, #1
1086 ext v17.16b, v0.16b, v0.16b, #2
1087 ext v18.16b, v4.16b, v4.16b, #1
1088 ext v19.16b, v4.16b, v4.16b, #2
1089 uaddl v3.8h, v0.8b, v16.8b
1090 uaddw v3.8h, v3.8h, v17.8b
1091 uaddl v7.8h, v4.8b, v18.8b
1092 uaddw v7.8h, v7.8h, v19.8b
1093
1094 ext v20.16b, v1.16b, v2.16b, #2
1095 ext v21.16b, v1.16b, v2.16b, #4
1096 ext v22.16b, v5.16b, v6.16b, #2
1097 ext v23.16b, v5.16b, v6.16b, #4
1098
1099 uaddl v26.4s, v1.4h, v20.4h
1100 uaddl2 v27.4s, v1.8h, v20.8h
1101 uaddw v26.4s, v26.4s, v21.4h
1102 uaddw2 v27.4s, v27.4s, v21.8h
1103
1104 uaddl v28.4s, v5.4h, v22.4h
1105 uaddl2 v29.4s, v5.8h, v22.8h
1106 uaddw v28.4s, v28.4s, v23.4h
1107 uaddw2 v29.4s, v29.4s, v23.8h
1108
1109 subs w5, w5, #8
1110
1111 st1 {v3.8h}, [x1], #16
1112 st1 {v7.8h}, [x11], #16
1113 st1 {v26.4s,v27.4s}, [x0], #32
1114 st1 {v28.4s,v29.4s}, [x10], #32
1115
1116 b.le 9f
1117 tst w7, #2 // LR_HAVE_RIGHT
1118 ld1 {v3.8b}, [x3], #8
1119 ld1 {v7.8b}, [x12], #8
1120 mov v1.16b, v2.16b
1121 mov v5.16b, v6.16b
1122 ext v0.16b, v0.16b, v3.16b, #8
1123 ext v4.16b, v4.16b, v7.16b, #8
1124 umull v2.8h, v3.8b, v3.8b
1125 umull v6.8h, v7.8b, v7.8b
1126
1127 b.ne 4b // If we don't need to pad, just keep summing.
1128 b 3b // If we need to pad, check how many pixels we have left.
1129
1130 9:
1131 subs w6, w6, #2
1132 b.le 0f
1133 // Jump to the next row and loop horizontally
1134 add x0, x0, x9, lsl #1
1135 add x10, x10, x9, lsl #1
1136 add x1, x1, x9
1137 add x11, x11, x9
1138 add x3, x3, x4
1139 add x12, x12, x4
1140 mov w5, w8
1141 b 1b
1142 0:
1143 ret
1144 endfunc
1145
1146 // void dav1d_sgr_box5_h_8bpc_neon(int32_t *sumsq, int16_t *sum,
1147 // const pixel (*left)[4],
1148 // const pixel *src, const ptrdiff_t stride,
1149 // const int w, const int h,
1150 // const enum LrEdgeFlags edges);
1151 function sgr_box5_h_8bpc_neon, export=1
1152 add w5, w5, #2 // w += 2
1153
1154 // Set up pointers for reading/writing alternate rows
1155 add x10, x0, #(4*SUM_STRIDE) // sumsq
1156 add x11, x1, #(2*SUM_STRIDE) // sum
1157 add x12, x3, x4 // src
1158 lsl x4, x4, #1
1159 mov x9, #(2*2*SUM_STRIDE) // double sum stride
1160
1161 // Subtract the aligned width from the output stride.
1162 add w13, w5, #7
1163 bic w13, w13, #7
1164 sub x9, x9, w13, uxtw #1
1165 add w13, w13, #8
1166 sub x4, x4, w13, uxtw
1167
1168 // Store the width for the vertical loop
1169 mov w8, w5
1170
1171 // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
1172 tst w7, #1 // LR_HAVE_LEFT
1173 b.eq 2f
1174 // LR_HAVE_LEFT
1175 cbnz x2, 0f
1176 // left == NULL
1177 sub x3, x3, #3
1178 sub x12, x12, #3
1179 b 1f
1180 0: // LR_HAVE_LEFT, left != NULL
1181 2: // !LR_HAVE_LEFT, increase the stride.
1182 // For this case we don't read the left 3 pixels from the src pointer,
1183 // but shift it as if we had done that.
1184 add x4, x4, #3
1185
1186 1: // Loop vertically
1187 ld1 {v0.16b}, [x3], #16
1188 ld1 {v4.16b}, [x12], #16
1189
1190 tst w7, #1 // LR_HAVE_LEFT
1191 b.eq 0f
1192 cbz x2, 2f
1193 // LR_HAVE_LEFT, left != NULL
1194 ld1 {v1.s}[3], [x2], #4
1195 // Move x3/x12 back to account for the last 3 bytes we loaded earlier,
1196 // which we'll shift out.
1197 sub x3, x3, #3
1198 sub x12, x12, #3
1199 ld1 {v5.s}[3], [x2], #4
1200 ext v0.16b, v1.16b, v0.16b, #13
1201 ext v4.16b, v5.16b, v4.16b, #13
1202 b 2f
1203 0:
1204 // !LR_HAVE_LEFT, fill v1 with the leftmost byte
1205 // and shift v0 to have 3x the first byte at the front.
1206 dup v1.16b, v0.b[0]
1207 dup v5.16b, v4.b[0]
1208 // Move x3 back to account for the last 3 bytes we loaded before,
1209 // which we shifted out.
1210 sub x3, x3, #3
1211 sub x12, x12, #3
1212 ext v0.16b, v1.16b, v0.16b, #13
1213 ext v4.16b, v5.16b, v4.16b, #13
1214
1215 2:
1216 umull v1.8h, v0.8b, v0.8b
1217 umull2 v2.8h, v0.16b, v0.16b
1218 umull v5.8h, v4.8b, v4.8b
1219 umull2 v6.8h, v4.16b, v4.16b
1220
1221 tst w7, #2 // LR_HAVE_RIGHT
1222 b.ne 4f
1223 // If we'll need to pad the right edge, load that byte to pad with
1224 // here since we can find it pretty easily from here.
1225 sub w13, w5, #(2 + 16 - 3 + 1)
1226 ldr b30, [x3, w13, sxtw]
1227 ldr b31, [x12, w13, sxtw]
1228 // Fill v30/v31 with the right padding pixel
1229 dup v30.16b, v30.b[0]
1230 dup v31.16b, v31.b[0]
1231 3: // !LR_HAVE_RIGHT
1232
1233 // Check whether we need to pad the right edge
1234 cmp w5, #11
1235 b.ge 4f // If w >= 11, all used input pixels are valid
1236
1237 // 1 <= w < 11, w+1 pixels valid in v0. For w=9 or w=10,
1238 // this ends up called again; it's not strictly needed in those
1239 // cases (we pad enough here), but keeping the code as simple as possible.
1240
1241 // Insert padding in v0/4.b[w+1] onwards; fuse the +1 into the
1242 // buffer pointer.
1243 movrel x13, right_ext_mask, -1
1244 sub x13, x13, w5, uxtw
1245 ld1 {v29.16b}, [x13]
1246
1247 bit v0.16b, v30.16b, v29.16b
1248 bit v4.16b, v31.16b, v29.16b
1249
1250 // Update the precalculated squares
1251 umull v1.8h, v0.8b, v0.8b
1252 umull2 v2.8h, v0.16b, v0.16b
1253 umull v5.8h, v4.8b, v4.8b
1254 umull2 v6.8h, v4.16b, v4.16b
1255
1256 4: // Loop horizontally
1257 ext v16.16b, v0.16b, v0.16b, #1
1258 ext v17.16b, v0.16b, v0.16b, #2
1259 ext v18.16b, v0.16b, v0.16b, #3
1260 ext v19.16b, v0.16b, v0.16b, #4
1261 ext v20.16b, v4.16b, v4.16b, #1
1262 ext v21.16b, v4.16b, v4.16b, #2
1263 ext v22.16b, v4.16b, v4.16b, #3
1264 ext v23.16b, v4.16b, v4.16b, #4
1265 uaddl v3.8h, v0.8b, v16.8b
1266 uaddl v24.8h, v17.8b, v18.8b
1267 uaddl v7.8h, v4.8b, v20.8b
1268 uaddw v3.8h, v3.8h, v19.8b
1269 uaddl v25.8h, v21.8b, v22.8b
1270 uaddw v7.8h, v7.8h, v23.8b
1271 add v3.8h, v3.8h, v24.8h
1272 add v7.8h, v7.8h, v25.8h
1273
1274 ext v16.16b, v1.16b, v2.16b, #2
1275 ext v17.16b, v1.16b, v2.16b, #4
1276 ext v18.16b, v1.16b, v2.16b, #6
1277 ext v19.16b, v1.16b, v2.16b, #8
1278 ext v20.16b, v5.16b, v6.16b, #2
1279 ext v21.16b, v5.16b, v6.16b, #4
1280 ext v22.16b, v5.16b, v6.16b, #6
1281 ext v23.16b, v5.16b, v6.16b, #8
1282
1283 uaddl v26.4s, v1.4h, v16.4h
1284 uaddl2 v27.4s, v1.8h, v16.8h
1285 uaddl v16.4s, v17.4h, v18.4h
1286 uaddl2 v17.4s, v17.8h, v18.8h
1287 uaddl v28.4s, v5.4h, v20.4h
1288 uaddl2 v29.4s, v5.8h, v20.8h
1289 uaddw v26.4s, v26.4s, v19.4h
1290 uaddw2 v27.4s, v27.4s, v19.8h
1291 uaddl v20.4s, v21.4h, v22.4h
1292 uaddl2 v21.4s, v21.8h, v22.8h
1293 uaddw v28.4s, v28.4s, v23.4h
1294 uaddw2 v29.4s, v29.4s, v23.8h
1295 add v26.4s, v26.4s, v16.4s
1296 add v27.4s, v27.4s, v17.4s
1297 add v28.4s, v28.4s, v20.4s
1298 add v29.4s, v29.4s, v21.4s
1299
1300 subs w5, w5, #8
1301
1302 st1 {v3.8h}, [x1], #16
1303 st1 {v7.8h}, [x11], #16
1304 st1 {v26.4s,v27.4s}, [x0], #32
1305 st1 {v28.4s,v29.4s}, [x10], #32
1306
1307 b.le 9f
1308 tst w7, #2 // LR_HAVE_RIGHT
1309 ld1 {v3.8b}, [x3], #8
1310 ld1 {v7.8b}, [x12], #8
1311 mov v1.16b, v2.16b
1312 mov v5.16b, v6.16b
1313 ext v0.16b, v0.16b, v3.16b, #8
1314 ext v4.16b, v4.16b, v7.16b, #8
1315 umull v2.8h, v3.8b, v3.8b
1316 umull v6.8h, v7.8b, v7.8b
1317 b.ne 4b // If we don't need to pad, just keep summing.
1318 b 3b // If we need to pad, check how many pixels we have left.
1319
1320 9:
1321 subs w6, w6, #2
1322 b.le 0f
1323 // Jump to the next row and loop horizontally
1324 add x0, x0, x9, lsl #1
1325 add x10, x10, x9, lsl #1
1326 add x1, x1, x9
1327 add x11, x11, x9
1328 add x3, x3, x4
1329 add x12, x12, x4
1330 mov w5, w8
1331 b 1b
1332 0:
1333 ret
1334 endfunc
1335
1336 sgr_funcs 8