Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/sad.S

raw
1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 * Copyright (c) 2020-2023, The rav1e contributors. All rights reserved
4 *
5 * This source code is subject to the terms of the BSD 2 Clause License and
6 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
7 * was not distributed with this source code in the LICENSE file, you can
8 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
9 * Media Patent License 1.0 was not distributed with this source code in the
10 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
11 */
12
13 #include "src/arm/asm.S"
14 #include "util.S"
15
16 .macro sad_rect width, height
17 function sad\width\()x\height\()_neon, export=1
18 movi v0.4s, #0
19 sxtw x1, w1
20 sxtw x3, w3
21 mov w4, \height
22 .if \width >= 16
23 mov v1.16b, v0.16b
24 .endif
25 b L(sad_w\width\())
26 endfunc
27 .endm
28
29 function sad4x4_neon, export=1
30 movi v0.4s, #0
31 sxtw x1, w1
32 sxtw x3, w3
33 mov w4, #4
34 L(sad_w4):
35 ldr s2, [x0]
36 ldr s3, [x2]
37 add x0, x0, x1
38 add x2, x2, x3
39 subs w4, w4, #1
40 uabal v0.8h, v2.8b, v3.8b
41 bne L(sad_w4)
42 uaddlp v0.2s, v0.4h
43 uaddlp v0.1d, v0.2s
44 fmov w0, s0
45 ret
46 endfunc
47
48 sad_rect 4, 8
49 sad_rect 4, 16
50
51 .macro horizontal_long_add_16x8
52 ushll v2.4s, v1.4h, #0
53 uaddw2 v1.4s, v2.4s, v1.8h
54 uaddw v1.4s, v1.4s, v0.4h
55 uaddw2 v0.4s, v1.4s, v0.8h
56 uaddlp v0.2d, v0.4s
57 ext v1.16b, v0.16b, v0.16b, #8
58 add v0.2s, v1.2s, v0.2s
59 fmov w0, s0
60 ret
61 .endm
62
63 .macro horizontal_add_16x8
64 uaddlp v0.4s, v0.8h
65 uaddlp v0.2d, v0.4s
66 ext v1.16b, v0.16b, v0.16b, #8
67 add v0.2s, v1.2s, v0.2s
68 fmov w0, s0
69 ret
70 .endm
71
72 function sad64x64_neon, export=1
73 movi v0.4s, #0
74 sxtw x1, w1
75 sxtw x3, w3
76 mov w4, #64
77 mov v1.16b, v0.16b
78 L(sad_w64):
79 ldp q2, q4, [x0]
80 ldp q3, q5, [x2]
81 ldp q6, q16, [x0, #32]
82 ldp q7, q17, [x2, #32]
83 add x0, x0, x1
84 add x2, x2, x3
85 subs w4, w4, #1
86 uabal v0.8h, v2.8b, v3.8b
87 uabal2 v1.8h, v2.16b, v3.16b
88 uabal v0.8h, v4.8b, v5.8b
89 uabal2 v1.8h, v4.16b, v5.16b
90 uabal v0.8h, v6.8b, v7.8b
91 uabal2 v1.8h, v6.16b, v7.16b
92 uabal v0.8h, v16.8b, v17.8b
93 uabal2 v1.8h, v16.16b, v17.16b
94 bne L(sad_w64)
95 horizontal_long_add_16x8
96 endfunc
97
98 sad_rect 64, 16
99 sad_rect 64, 32
100 sad_rect 64, 128
101
102 function sad128x128_neon, export=1
103 movi v0.4s, #0
104 sxtw x1, w1
105 sxtw x3, w3
106 mov w4, #128
107 mov v1.16b, v0.16b
108 L(sad_w128):
109 ldp q2, q4, [x0]
110 ldp q3, q5, [x2]
111 ldp q6, q16, [x0, #32]
112 ldp q7, q17, [x2, #32]
113 ldp q18, q20, [x0, #64]
114 ldp q19, q21, [x2, #64]
115 ldp q22, q24, [x0, #96]
116 ldp q23, q25, [x2, #96]
117 add x0, x0, x1
118 add x2, x2, x3
119 subs w4, w4, #1
120 uabdl v26.8h, v2.8b, v3.8b
121 uabal2 v26.8h, v2.16b, v3.16b
122 uabal v26.8h, v4.8b, v5.8b
123 uabal2 v26.8h, v4.16b, v5.16b
124 uabal v26.8h, v6.8b, v7.8b
125 uabal2 v26.8h, v6.16b, v7.16b
126 uabal v26.8h, v16.8b, v17.8b
127 uabal2 v26.8h, v16.16b, v17.16b
128 uabal v26.8h, v18.8b, v19.8b
129 uabal2 v26.8h, v18.16b, v19.16b
130 uabal v26.8h, v20.8b, v21.8b
131 uabal2 v26.8h, v20.16b, v21.16b
132 uabal v26.8h, v22.8b, v23.8b
133 uabal2 v26.8h, v22.16b, v23.16b
134 uabal v26.8h, v24.8b, v25.8b
135 uabal2 v26.8h, v24.16b, v25.16b
136 uaddw v1.4s, v1.4s, v26.4h
137 uaddw2 v0.4s, v0.4s, v26.8h
138 bne L(sad_w128)
139 add v0.4s, v0.4s, v1.4s
140 uaddlp v0.2d, v0.4s
141 dup d3, v0.d[1]
142 add v0.2s, v0.2s, v3.2s
143 umov w0, v0.s[0]
144 ret
145 endfunc
146
147 sad_rect 128, 64
148
149 function sad32x32_neon, export=1
150 movi v0.4s, #0
151 sxtw x1, w1
152 sxtw x3, w3
153 mov w4, #32
154 mov v1.16b, v0.16b
155 L(sad_w32):
156 ldp q2, q4, [x0]
157 ldp q3, q5, [x2]
158 add x0, x0, x1
159 add x2, x2, x3
160 subs w4, w4, #1
161 uabal v1.8h, v2.8b, v3.8b
162 uabal2 v0.8h, v2.16b, v3.16b
163 uabal v1.8h, v4.8b, v5.8b
164 uabal2 v0.8h, v4.16b, v5.16b
165 bne L(sad_w32)
166 add v0.8h, v0.8h, v1.8h
167 horizontal_add_16x8
168 endfunc
169
170 sad_rect 32, 8
171 sad_rect 32, 16
172 sad_rect 32, 64
173
174 function sad16x16_neon, export=1
175 movi v0.4s, #0
176 sxtw x1, w1
177 sxtw x3, w3
178 mov w4, #16
179 mov v1.16b, v0.16b
180 L(sad_w16):
181 ldr q2, [x0]
182 ldr q3, [x2]
183 add x0, x0, x1
184 add x2, x2, x3
185 subs w4, w4, #1
186 uabal v0.8h, v2.8b, v3.8b
187 uabal2 v1.8h, v2.16b, v3.16b
188 bne L(sad_w16)
189 add v0.8h, v0.8h, v1.8h
190 horizontal_add_16x8
191 endfunc
192
193 sad_rect 16, 4
194 sad_rect 16, 8
195 sad_rect 16, 32
196 sad_rect 16, 64
197
198 function sad8x8_neon, export=1
199 movi v0.4s, #0
200 sxtw x1, w1
201 sxtw x3, w3
202 mov w4, #8
203 L(sad_w8):
204 ldr d2, [x0]
205 ldr d3, [x2]
206 add x0, x0, x1
207 add x2, x2, x3
208 subs w4, w4, #1
209 uabal v0.8h, v2.8b, v3.8b
210 bne L(sad_w8)
211 horizontal_add_16x8
212 endfunc
213
214 sad_rect 8, 4
215 sad_rect 8, 16
216 sad_rect 8, 32
217
218 .macro sad_hbd_rect width, height
219 function sad\width\()x\height\()_hbd_neon, export=1
220 movi v0.4s, #0
221 sxtw x1, w1
222 sxtw x3, w3
223 mov w4, \height
224 .if \width >= 8
225 mov v1.16b, v0.16b
226 .endif
227 .if \width >= 64 && \height >= 64
228 b L(sad_hbd_large_w\width\())
229 .else
230 b L(sad_hbd_w\width\())
231 .endif
232 endfunc
233 .endm
234
235 function sad4x4_hbd_neon, export=1
236 movi v0.4s, #0
237 sxtw x1, w1
238 sxtw x3, w3
239 mov w4, #4
240 L(sad_hbd_w4):
241 ldr d2, [x0]
242 ldr d3, [x2]
243 add x0, x0, x1
244 add x2, x2, x3
245 subs w4, w4, #1
246 uabal v0.4s, v2.4h, v3.4h
247 bne L(sad_hbd_w4)
248 addv s0, v0.4s
249 fmov w0, s0
250 ret
251 endfunc
252
253 sad_hbd_rect 4, 8
254 sad_hbd_rect 4, 16
255
256 .macro horizontal_add_32x4
257 uaddlp v0.2d, v0.4s
258 ext v1.16b, v0.16b, v0.16b, #8
259 add v0.2s, v1.2s, v0.2s
260 fmov w0, s0
261 ret
262 .endm
263
264 function sad64x32_hbd_neon, export=1
265 movi v0.4s, #0
266 sxtw x1, w1
267 sxtw x3, w3
268 mov w4, #32
269 mov v1.16b, v0.16b
270 L(sad_hbd_w64):
271 ldp q2, q4, [x0]
272 ldp q3, q5, [x2]
273 ldp q6, q16, [x0, #32]
274 ldp q7, q17, [x2, #32]
275 ldp q18, q20, [x0, #64]
276 ldp q19, q21, [x2, #64]
277 ldp q22, q24, [x0, #96]
278 ldp q23, q25, [x2, #96]
279 add x0, x0, x1
280 add x2, x2, x3
281 subs w4, w4, #1
282 uabal v0.4s, v2.4h, v3.4h
283 uabal2 v1.4s, v2.8h, v3.8h
284 uabal v0.4s, v4.4h, v5.4h
285 uabal2 v1.4s, v4.8h, v5.8h
286 uabal v0.4s, v6.4h, v7.4h
287 uabal2 v1.4s, v6.8h, v7.8h
288 uabal v0.4s, v16.4h, v17.4h
289 uabal2 v1.4s, v16.8h, v17.8h
290 uabal v0.4s, v18.4h, v19.4h
291 uabal2 v1.4s, v18.8h, v19.8h
292 uabal v0.4s, v20.4h, v21.4h
293 uabal2 v1.4s, v20.8h, v21.8h
294 uabal v0.4s, v22.4h, v23.4h
295 uabal2 v1.4s, v22.8h, v23.8h
296 uabal v0.4s, v24.4h, v25.4h
297 uabal2 v1.4s, v24.8h, v25.8h
298 bne L(sad_hbd_w64)
299 horizontal_long_add_16x8
300 endfunc
301
302 function sad64x64_hbd_neon, export=1
303 movi v0.4s, #0
304 sxtw x1, w1
305 sxtw x3, w3
306 mov w4, #64
307 mov v1.16b, v0.16b
308 L(sad_hbd_large_w64):
309 ldp q2, q4, [x0]
310 ldp q3, q5, [x2]
311 ldp q6, q16, [x0, #32]
312 ldp q7, q17, [x2, #32]
313 ldp q18, q20, [x0, #64]
314 ldp q19, q21, [x2, #64]
315 ldp q22, q24, [x0, #96]
316 ldp q23, q25, [x2, #96]
317 add x0, x0, x1
318 add x2, x2, x3
319 subs w4, w4, #1
320 uabdl v26.4s, v2.4h, v3.4h
321 uabal2 v26.4s, v2.8h, v3.8h
322 uabal v26.4s, v4.4h, v5.4h
323 uabal2 v26.4s, v4.8h, v5.8h
324 uabal v26.4s, v6.4h, v7.4h
325 uabal2 v26.4s, v6.8h, v7.8h
326 uabal v26.4s, v16.4h, v17.4h
327 uabal2 v26.4s, v16.8h, v17.8h
328 uabal v26.4s, v18.4h, v19.4h
329 uabal2 v26.4s, v18.8h, v19.8h
330 uabal v26.4s, v20.4h, v21.4h
331 uabal2 v26.4s, v20.8h, v21.8h
332 uabal v26.4s, v22.4h, v23.4h
333 uabal2 v26.4s, v22.8h, v23.8h
334 uabal v26.4s, v24.4h, v25.4h
335 uabal2 v26.4s, v24.8h, v25.8h
336 uaddw v1.2d, v1.2d, v26.2s
337 uaddw2 v0.2d, v0.2d, v26.4s
338 bne L(sad_hbd_large_w64)
339 add v0.2d, v0.2d, v1.2d
340 dup d3, v0.d[1]
341 add v0.2d, v0.2d, v3.2d
342 umov x0, v0.d[0]
343 ret
344 endfunc
345
346 sad_hbd_rect 64, 16
347 sad_hbd_rect 64, 128
348
349 function sad128x128_hbd_neon, export=1
350 movi v0.4s, #0
351 sxtw x1, w1
352 sxtw x3, w3
353 mov w4, #128
354 mov v1.16b, v0.16b
355 L(sad_hbd_large_w128):
356 ldp q2, q4, [x0]
357 ldp q3, q5, [x2]
358 ldp q6, q16, [x0, #32]
359 ldp q7, q17, [x2, #32]
360 ldp q18, q20, [x0, #64]
361 ldp q19, q21, [x2, #64]
362 ldp q22, q24, [x0, #96]
363 ldp q23, q25, [x2, #96]
364 uabdl v26.4s, v2.4h, v3.4h
365 uabal2 v26.4s, v2.8h, v3.8h
366 uabal v26.4s, v4.4h, v5.4h
367 uabal2 v26.4s, v4.8h, v5.8h
368 uabal v26.4s, v6.4h, v7.4h
369 uabal2 v26.4s, v6.8h, v7.8h
370 uabal v26.4s, v16.4h, v17.4h
371 uabal2 v26.4s, v16.8h, v17.8h
372 uabal v26.4s, v18.4h, v19.4h
373 uabal2 v26.4s, v18.8h, v19.8h
374 uabal v26.4s, v20.4h, v21.4h
375 uabal2 v26.4s, v20.8h, v21.8h
376 uabal v26.4s, v22.4h, v23.4h
377 uabal2 v26.4s, v22.8h, v23.8h
378 uabal v26.4s, v24.4h, v25.4h
379 uabal2 v26.4s, v24.8h, v25.8h
380 uaddw v1.2d, v1.2d, v26.2s
381 uaddw2 v0.2d, v0.2d, v26.4s
382 ldp q2, q4, [x0, #128]
383 ldp q3, q5, [x2, #128]
384 ldp q6, q16, [x0, #160]
385 ldp q7, q17, [x2, #160]
386 ldp q18, q20, [x0, #192]
387 ldp q19, q21, [x2, #192]
388 ldp q22, q24, [x0, #224]
389 ldp q23, q25, [x2, #224]
390 add x0, x0, x1
391 add x2, x2, x3
392 subs w4, w4, #1
393 uabdl v26.4s, v2.4h, v3.4h
394 uabal2 v26.4s, v2.8h, v3.8h
395 uabal v26.4s, v4.4h, v5.4h
396 uabal2 v26.4s, v4.8h, v5.8h
397 uabal v26.4s, v6.4h, v7.4h
398 uabal2 v26.4s, v6.8h, v7.8h
399 uabal v26.4s, v16.4h, v17.4h
400 uabal2 v26.4s, v16.8h, v17.8h
401 uabal v26.4s, v18.4h, v19.4h
402 uabal2 v26.4s, v18.8h, v19.8h
403 uabal v26.4s, v20.4h, v21.4h
404 uabal2 v26.4s, v20.8h, v21.8h
405 uabal v26.4s, v22.4h, v23.4h
406 uabal2 v26.4s, v22.8h, v23.8h
407 uabal v26.4s, v24.4h, v25.4h
408 uabal2 v26.4s, v24.8h, v25.8h
409 uaddw v1.2d, v1.2d, v26.2s
410 uaddw2 v0.2d, v0.2d, v26.4s
411 bne L(sad_hbd_large_w128)
412 add v0.2d, v0.2d, v1.2d
413 dup d3, v0.d[1]
414 add v0.2d, v0.2d, v3.2d
415 umov x0, v0.d[0]
416 ret
417 endfunc
418
419 sad_hbd_rect 128, 64
420
421 function sad32x32_hbd_neon, export=1
422 movi v0.4s, #0
423 sxtw x1, w1
424 sxtw x3, w3
425 mov w4, #32
426 mov v1.16b, v0.16b
427 L(sad_hbd_w32):
428 ldp q2, q4, [x0]
429 ldp q3, q5, [x2]
430 ldp q6, q16, [x0, #32]
431 ldp q7, q17, [x2, #32]
432 add x0, x0, x1
433 add x2, x2, x3
434 subs w4, w4, #1
435 uabal v0.4s, v2.4h, v3.4h
436 uabal2 v1.4s, v2.8h, v3.8h
437 uabal v0.4s, v4.4h, v5.4h
438 uabal2 v1.4s, v4.8h, v5.8h
439 uabal v0.4s, v6.4h, v7.4h
440 uabal2 v1.4s, v6.8h, v7.8h
441 uabal v0.4s, v16.4h, v17.4h
442 uabal2 v1.4s, v16.8h, v17.8h
443 bne L(sad_hbd_w32)
444 add v0.4s, v0.4s, v1.4s
445 horizontal_add_32x4
446 endfunc
447
448 sad_hbd_rect 32, 8
449 sad_hbd_rect 32, 16
450 sad_hbd_rect 32, 64
451
452 function sad16x16_hbd_neon, export=1
453 movi v0.4s, #0
454 sxtw x1, w1
455 sxtw x3, w3
456 mov w4, #16
457 mov v1.16b, v0.16b
458 L(sad_hbd_w16):
459 ldp q2, q4, [x0]
460 ldp q3, q5, [x2]
461 add x0, x0, x1
462 add x2, x2, x3
463 subs w4, w4, #1
464 uabal v0.4s, v2.4h, v3.4h
465 uabal2 v1.4s, v2.8h, v3.8h
466 uabal v0.4s, v4.4h, v5.4h
467 uabal2 v1.4s, v4.8h, v5.8h
468 bne L(sad_hbd_w16)
469 add v0.4s, v0.4s, v1.4s
470 horizontal_add_32x4
471 endfunc
472
473 sad_hbd_rect 16, 4
474 sad_hbd_rect 16, 8
475 sad_hbd_rect 16, 32
476 sad_hbd_rect 16, 64
477
478 function sad8x8_hbd_neon, export=1
479 movi v0.4s, #0
480 sxtw x1, w1
481 sxtw x3, w3
482 mov w4, #8
483 mov v1.16b, v0.16b
484 L(sad_hbd_w8):
485 ldr q2, [x0]
486 ldr q3, [x2]
487 add x0, x0, x1
488 add x2, x2, x3
489 subs w4, w4, #1
490 uabal v0.4s, v2.4h, v3.4h
491 uabal2 v1.4s, v2.8h, v3.8h
492 bne L(sad_hbd_w8)
493 add v0.4s, v0.4s, v1.4s
494 horizontal_add_32x4
495 endfunc
496
497 sad_hbd_rect 8, 4
498 sad_hbd_rect 8, 16
499 sad_hbd_rect 8, 32