Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/looprestoration_common.S

raw
1 /*
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2018, Martin Storsjo
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "src/arm/asm.S"
29 #include "util.S"
30
31 #define SUM_STRIDE (384+16)
32
33 // void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum,
34 // const int w, const int h,
35 // const enum LrEdgeFlags edges);
36 function sgr_box3_v_neon, export=1
37 add w10, w3, #2 // Number of output rows to move back
38 mov w11, w3 // Number of input rows to move back
39 add w2, w2, #2 // Actual summed width
40 mov x7, #(4*SUM_STRIDE) // sumsq stride
41 mov x8, #(2*SUM_STRIDE) // sum stride
42 sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride
43 sub x1, x1, #(2*SUM_STRIDE) // sum -= stride
44
45 tst w4, #4 // LR_HAVE_TOP
46 b.eq 0f
47 // If have top, read from row -2.
48 sub x5, x0, #(4*SUM_STRIDE)
49 sub x6, x1, #(2*SUM_STRIDE)
50 add w11, w11, #2
51 b 1f
52 0:
53 // !LR_HAVE_TOP
54 // If we don't have top, read from row 0 even if
55 // we start writing to row -1.
56 add x5, x0, #(4*SUM_STRIDE)
57 add x6, x1, #(2*SUM_STRIDE)
58 1:
59
60 tst w4, #8 // LR_HAVE_BOTTOM
61 b.eq 1f
62 // LR_HAVE_BOTTOM
63 add w3, w3, #2 // Sum all h+2 lines with the main loop
64 add w11, w11, #2
65 1:
66 mov w9, w3 // Backup of h for next loops
67
68 1:
69 // Start of horizontal loop; start one vertical filter slice.
70 // Start loading rows into v16-v21 and v24-v26 taking top
71 // padding into consideration.
72 tst w4, #4 // LR_HAVE_TOP
73 ld1 {v16.4s, v17.4s}, [x5], x7
74 ld1 {v24.8h}, [x6], x8
75 b.eq 2f
76 // LR_HAVE_TOP
77 ld1 {v18.4s, v19.4s}, [x5], x7
78 ld1 {v25.8h}, [x6], x8
79 ld1 {v20.4s, v21.4s}, [x5], x7
80 ld1 {v26.8h}, [x6], x8
81 b 3f
82 2: // !LR_HAVE_TOP
83 mov v18.16b, v16.16b
84 mov v19.16b, v17.16b
85 mov v25.16b, v24.16b
86 mov v20.16b, v16.16b
87 mov v21.16b, v17.16b
88 mov v26.16b, v24.16b
89
90 3:
91 subs w3, w3, #1
92 .macro add3
93 add v16.4s, v16.4s, v18.4s
94 add v17.4s, v17.4s, v19.4s
95 add v24.8h, v24.8h, v25.8h
96 add v16.4s, v16.4s, v20.4s
97 add v17.4s, v17.4s, v21.4s
98 add v24.8h, v24.8h, v26.8h
99 st1 {v16.4s, v17.4s}, [x0], x7
100 st1 {v24.8h}, [x1], x8
101 .endm
102 add3
103 mov v16.16b, v18.16b
104 mov v17.16b, v19.16b
105 mov v24.16b, v25.16b
106 mov v18.16b, v20.16b
107 mov v19.16b, v21.16b
108 mov v25.16b, v26.16b
109 b.le 4f
110 ld1 {v20.4s, v21.4s}, [x5], x7
111 ld1 {v26.8h}, [x6], x8
112 b 3b
113
114 4:
115 tst w4, #8 // LR_HAVE_BOTTOM
116 b.ne 5f
117 // !LR_HAVE_BOTTOM
118 // Produce two more rows, extending the already loaded rows.
119 add3
120 mov v16.16b, v18.16b
121 mov v17.16b, v19.16b
122 mov v24.16b, v25.16b
123 add3
124
125 5: // End of one vertical slice.
126 subs w2, w2, #8
127 b.le 0f
128 // Move pointers back up to the top and loop horizontally.
129 // Input pointers
130 msub x5, x7, x11, x5
131 msub x6, x8, x11, x6
132 // Output pointers
133 msub x0, x7, x10, x0
134 msub x1, x8, x10, x1
135 add x0, x0, #32
136 add x1, x1, #16
137 add x5, x5, #32
138 add x6, x6, #16
139 mov w3, w9
140 b 1b
141
142 0:
143 ret
144 .purgem add3
145 endfunc
146
147 // void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum,
148 // const int w, const int h,
149 // const enum LrEdgeFlags edges);
150 function sgr_box5_v_neon, export=1
151 add w10, w3, #2 // Number of output rows to move back
152 mov w11, w3 // Number of input rows to move back
153 add w2, w2, #8 // Actual summed width
154 mov x7, #(4*SUM_STRIDE) // sumsq stride
155 mov x8, #(2*SUM_STRIDE) // sum stride
156 sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride
157 sub x1, x1, #(2*SUM_STRIDE) // sum -= stride
158
159 tst w4, #4 // LR_HAVE_TOP
160 b.eq 0f
161 // If have top, read from row -2.
162 sub x5, x0, #(4*SUM_STRIDE)
163 sub x6, x1, #(2*SUM_STRIDE)
164 add w11, w11, #2
165 b 1f
166 0:
167 // !LR_HAVE_TOP
168 // If we don't have top, read from row 0 even if
169 // we start writing to row -1.
170 add x5, x0, #(4*SUM_STRIDE)
171 add x6, x1, #(2*SUM_STRIDE)
172 1:
173
174 tst w4, #8 // LR_HAVE_BOTTOM
175 b.eq 0f
176 // LR_HAVE_BOTTOM
177 add w3, w3, #2 // Handle h+2 lines with the main loop
178 add w11, w11, #2
179 b 1f
180 0:
181 // !LR_HAVE_BOTTOM
182 sub w3, w3, #1 // Handle h-1 lines with the main loop
183 1:
184 mov w9, w3 // Backup of h for next loops
185
186 1:
187 // Start of horizontal loop; start one vertical filter slice.
188 // Start loading rows into v16-v25 and v26-v30 taking top
189 // padding into consideration.
190 tst w4, #4 // LR_HAVE_TOP
191 ld1 {v16.4s, v17.4s}, [x5], x7
192 ld1 {v26.8h}, [x6], x8
193 b.eq 2f
194 // LR_HAVE_TOP
195 ld1 {v20.4s, v21.4s}, [x5], x7
196 ld1 {v28.8h}, [x6], x8
197 mov v18.16b, v16.16b
198 mov v19.16b, v17.16b
199 mov v27.16b, v26.16b
200 ld1 {v22.4s, v23.4s}, [x5], x7
201 ld1 {v29.8h}, [x6], x8
202 b 3f
203 2: // !LR_HAVE_TOP
204 mov v18.16b, v16.16b
205 mov v19.16b, v17.16b
206 mov v27.16b, v26.16b
207 mov v20.16b, v16.16b
208 mov v21.16b, v17.16b
209 mov v28.16b, v26.16b
210 mov v22.16b, v16.16b
211 mov v23.16b, v17.16b
212 mov v29.16b, v26.16b
213
214 3:
215 cbz w3, 4f
216 ld1 {v24.4s, v25.4s}, [x5], x7
217 ld1 {v30.8h}, [x6], x8
218
219 3:
220 // Start of vertical loop
221 subs w3, w3, #2
222 .macro add5
223 add v16.4s, v16.4s, v18.4s
224 add v17.4s, v17.4s, v19.4s
225 add v26.8h, v26.8h, v27.8h
226 add v0.4s, v20.4s, v22.4s
227 add v1.4s, v21.4s, v23.4s
228 add v2.8h, v28.8h, v29.8h
229 add v16.4s, v16.4s, v24.4s
230 add v17.4s, v17.4s, v25.4s
231 add v26.8h, v26.8h, v30.8h
232 add v16.4s, v16.4s, v0.4s
233 add v17.4s, v17.4s, v1.4s
234 add v26.8h, v26.8h, v2.8h
235 st1 {v16.4s, v17.4s}, [x0], x7
236 st1 {v26.8h}, [x1], x8
237 .endm
238 add5
239 .macro shift2
240 mov v16.16b, v20.16b
241 mov v17.16b, v21.16b
242 mov v26.16b, v28.16b
243 mov v18.16b, v22.16b
244 mov v19.16b, v23.16b
245 mov v27.16b, v29.16b
246 mov v20.16b, v24.16b
247 mov v21.16b, v25.16b
248 mov v28.16b, v30.16b
249 .endm
250 shift2
251 add x0, x0, x7
252 add x1, x1, x8
253 b.le 5f
254 ld1 {v22.4s, v23.4s}, [x5], x7
255 ld1 {v29.8h}, [x6], x8
256 ld1 {v24.4s, v25.4s}, [x5], x7
257 ld1 {v30.8h}, [x6], x8
258 b 3b
259
260 4:
261 // h == 1, !LR_HAVE_BOTTOM.
262 // Pad the last row with the only content row, and add.
263 mov v24.16b, v22.16b
264 mov v25.16b, v23.16b
265 mov v30.16b, v29.16b
266 add5
267 shift2
268 add x0, x0, x7
269 add x1, x1, x8
270 add5
271 b 6f
272
273 5:
274 tst w4, #8 // LR_HAVE_BOTTOM
275 b.ne 6f
276 // !LR_HAVE_BOTTOM
277 cbnz w3, 5f
278 // The intended three edge rows left; output the one at h-2 and
279 // the past edge one at h.
280 ld1 {v22.4s, v23.4s}, [x5], x7
281 ld1 {v29.8h}, [x6], x8
282 // Pad the past-edge row from the last content row.
283 mov v24.16b, v22.16b
284 mov v25.16b, v23.16b
285 mov v30.16b, v29.16b
286 add5
287 shift2
288 add x0, x0, x7
289 add x1, x1, x8
290 // The last two rows are already padded properly here.
291 add5
292 b 6f
293
294 5:
295 // w3 == -1, two rows left, output one.
296 // Pad the last two rows from the mid one.
297 mov v22.16b, v20.16b
298 mov v23.16b, v21.16b
299 mov v29.16b, v28.16b
300 mov v24.16b, v20.16b
301 mov v25.16b, v21.16b
302 mov v30.16b, v28.16b
303 add5
304 add x0, x0, x7
305 add x1, x1, x8
306 b 6f
307
308 6: // End of one vertical slice.
309 subs w2, w2, #8
310 b.le 0f
311 // Move pointers back up to the top and loop horizontally.
312 // Input pointers
313 msub x5, x7, x11, x5
314 msub x6, x8, x11, x6
315 // Output pointers
316 msub x0, x7, x10, x0
317 msub x1, x8, x10, x1
318 add x0, x0, #32
319 add x1, x1, #16
320 add x5, x5, #32
321 add x6, x6, #16
322 mov w3, w9
323 b 1b
324
325 0:
326 ret
327 .purgem add5
328 endfunc
329
330 // void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b,
331 // const int w, const int h, const int strength,
332 // const int bitdepth_max);
333 // void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b,
334 // const int w, const int h, const int strength,
335 // const int bitdepth_max);
336 function sgr_calc_ab1_neon, export=1
337 clz w9, w5
338 add x3, x3, #2 // h += 2
339 movi v31.4s, #9 // n
340 mov x5, #455
341 mov x8, #SUM_STRIDE
342 b sgr_calc_ab_neon
343 endfunc
344
345 function sgr_calc_ab2_neon, export=1
346 clz w9, w5
347 add x3, x3, #3 // h += 3
348 asr x3, x3, #1 // h /= 2
349 movi v31.4s, #25 // n
350 mov x5, #164
351 mov x8, #(2*SUM_STRIDE)
352 endfunc
353
354 function sgr_calc_ab_neon
355 sub w9, w9, #24 // -bitdepth_min_8
356 movrel x12, X(sgr_x_by_x)
357 ld1 {v16.16b, v17.16b, v18.16b}, [x12]
358 dup v6.8h, w9 // -bitdepth_min_8
359 movi v19.16b, #5
360 movi v20.8b, #55 // idx of last 5
361 movi v21.8b, #72 // idx of last 4
362 movi v22.8b, #101 // idx of last 3
363 movi v23.8b, #169 // idx of last 2
364 movi v24.8b, #254 // idx of last 1
365 saddl v7.4s, v6.4h, v6.4h // -2*bitdepth_min_8
366 add x2, x2, #2 // w += 2
367 add x7, x2, #7
368 bic x7, x7, #7 // aligned w
369 sub x7, x8, x7 // increment between rows
370 movi v29.8h, #1, lsl #8
371 dup v28.4s, w4
372 dup v30.4s, w5 // one_by_x
373 sub x0, x0, #(4*(SUM_STRIDE))
374 sub x1, x1, #(2*(SUM_STRIDE))
375 mov x6, x2 // backup of w
376 sub v16.16b, v16.16b, v19.16b
377 sub v17.16b, v17.16b, v19.16b
378 sub v18.16b, v18.16b, v19.16b
379 1:
380 subs x2, x2, #8
381 ld1 {v0.4s, v1.4s}, [x0] // a
382 ld1 {v2.8h}, [x1] // b
383 srshl v0.4s, v0.4s, v7.4s
384 srshl v1.4s, v1.4s, v7.4s
385 srshl v4.8h, v2.8h, v6.8h
386 mul v0.4s, v0.4s, v31.4s // a * n
387 mul v1.4s, v1.4s, v31.4s // a * n
388 umull v3.4s, v4.4h, v4.4h // b * b
389 umull2 v4.4s, v4.8h, v4.8h // b * b
390 uqsub v0.4s, v0.4s, v3.4s // imax(a * n - b * b, 0)
391 uqsub v1.4s, v1.4s, v4.4s // imax(a * n - b * b, 0)
392 mul v0.4s, v0.4s, v28.4s // p * s
393 mul v1.4s, v1.4s, v28.4s // p * s
394 uqshrn v0.4h, v0.4s, #16
395 uqshrn2 v0.8h, v1.4s, #16
396 uqrshrn v0.8b, v0.8h, #4 // imin(z, 255)
397
398 cmhi v25.8b, v0.8b, v20.8b // = -1 if sgr_x_by_x[v0] < 5
399 cmhi v26.8b, v0.8b, v21.8b // = -1 if sgr_x_by_x[v0] < 4
400 tbl v1.8b, {v16.16b, v17.16b, v18.16b}, v0.8b
401 cmhi v27.8b, v0.8b, v22.8b // = -1 if sgr_x_by_x[v0] < 3
402 cmhi v4.8b, v0.8b, v23.8b // = -1 if sgr_x_by_x[v0] < 2
403 add v25.8b, v25.8b, v26.8b
404 cmhi v5.8b, v0.8b, v24.8b // = -1 if sgr_x_by_x[v0] < 1
405 add v27.8b, v27.8b, v4.8b
406 add v5.8b, v5.8b, v19.8b
407 add v25.8b, v25.8b, v27.8b
408 add v1.8b, v1.8b, v5.8b
409 add v1.8b, v1.8b, v25.8b
410 uxtl v1.8h, v1.8b // x
411
412 umull v3.4s, v1.4h, v2.4h // x * BB[i]
413 umull2 v4.4s, v1.8h, v2.8h // x * BB[i]
414 mul v3.4s, v3.4s, v30.4s // x * BB[i] * sgr_one_by_x
415 mul v4.4s, v4.4s, v30.4s // x * BB[i] * sgr_one_by_x
416 srshr v3.4s, v3.4s, #12 // AA[i]
417 srshr v4.4s, v4.4s, #12 // AA[i]
418 sub v2.8h, v29.8h, v1.8h // 256 - x
419
420 st1 {v3.4s, v4.4s}, [x0], #32
421 st1 {v2.8h}, [x1], #16
422 b.gt 1b
423
424 subs x3, x3, #1
425 b.le 0f
426 add x0, x0, x7, lsl #2
427 add x1, x1, x7, lsl #1
428 mov x2, x6
429 b 1b
430 0:
431 ret
432 endfunc