Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/sse.S

raw
1 /* Copyright (c) 2023, The rav1e contributors. All rights reserved
2 *
3 * This source code is subject to the terms of the BSD 2 Clause License and
4 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 * was not distributed with this source code in the LICENSE file, you can
6 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 * Media Patent License 1.0 was not distributed with this source code in the
8 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9 */
10
11 #include "src/arm/asm.S"
12 #include "util.S"
13
14 .macro LOAD_X4 t=q
15 ldr \t\()0, [x0]
16 ldr \t\()4, [x2]
17 ldr \t\()1, [x0, x1]
18 ldr \t\()5, [x2, x3]
19 ldr \t\()2, [x0, x11]
20 ldr \t\()6, [x2, x12]
21 ldr \t\()3, [x0, x8]
22 ldr \t\()7, [x2, x9]
23 ldr \t\()16, [x4]
24 add x0, x0, x1, lsl 2
25 add x2, x2, x3, lsl 2
26 add x4, x4, x5
27 subs w10, w10, #4
28 .ifc \t,q
29 mov v18.d[0], v16.d[1]
30 .else
31 mov v0.\t[1], v1.\t[0]
32 mov v4.\t[1], v5.\t[0]
33 mov v2.\t[1], v3.\t[0]
34 mov v6.\t[1], v7.\t[0]
35 .endif
36 .endm
37
38 .macro INIT width, height
39 .if \width <= 16
40 add x11, x1, x1
41 add x12, x3, x3
42 add x8, x1, x1, lsl 1
43 add x9, x3, x3, lsl 1
44 .elseif \width >= 64
45 mov w8, #(\width)
46 sxtw x9, w8
47 .endif
48 movi v17.4s, #0
49 mov w10, #(\height)
50 .endm
51
52 // x0: src: *const u8,
53 // x1: src_stride: isize,
54 // x2: dst: *const u8,
55 // x3: dst_stride: isize,
56 // x4: scale: *const u32,
57 // x5: scale_stride: isize,
58 function weighted_sse_4x4_neon, export=1
59 INIT 4, 4
60 L(wsse_w4):
61 LOAD_X4 t=s
62 uabd v0.8b, v0.8b, v4.8b // diff pixel values
63 uabd v1.8b, v2.8b, v6.8b
64 umull v0.8h, v0.8b, v0.8b // square
65 umull v1.8h, v1.8b, v1.8b
66 uaddl v2.4s, v0.4h, v1.4h // accumulate
67 uaddl2 v3.4s, v0.8h, v1.8h
68 add v0.4s, v2.4s, v3.4s
69 addv s0, v0.4s
70 umull v0.2d, v0.2s, v16.2s // apply scale
71 urshr d0, d0, #8
72 add v17.2d, v17.2d, v0.2d
73 bne L(wsse_w4)
74 fmov x0, d17
75 ret
76 endfunc
77
78 .macro RET_SUM
79 mov v0.d[0], v17.d[1]
80 add d0, d0, d17
81 fmov x0, d0
82 ret
83 .endm
84
85 function weighted_sse_8x8_neon, export=1
86 INIT 8, 8
87 L(wsse_w8):
88 LOAD_X4 t=d
89 uabd v0.16b, v0.16b, v4.16b // diff pixel values
90 uabd v1.16b, v2.16b, v6.16b
91 umull2 v2.8h, v0.16b, v0.16b // square
92 umull2 v3.8h, v1.16b, v1.16b
93 umull v0.8h, v0.8b, v0.8b
94 umull v1.8h, v1.8b, v1.8b
95 uaddlp v2.4s, v2.8h // accumulate
96 uaddlp v3.4s, v3.8h
97 uaddlp v0.4s, v0.8h
98 uaddlp v1.4s, v1.8h
99 uaddlp v2.2d, v2.4s
100 uadalp v2.2d, v3.4s
101 uadalp v2.2d, v0.4s
102 uadalp v2.2d, v1.4s
103 xtn v0.2s, v2.2d
104 umull v0.2d, v0.2s, v16.2s // apply scale
105 urshr v0.2d, v0.2d, #8
106 add v17.2d, v17.2d, v0.2d
107 bne L(wsse_w8)
108 RET_SUM
109 endfunc
110
111 function weighted_sse_16x16_neon, export=1
112 INIT 16, 16
113 L(wsse_w16):
114 LOAD_X4
115 uabd v0.16b, v0.16b, v4.16b // diff pixel values
116 uabd v1.16b, v1.16b, v5.16b
117 uabd v2.16b, v2.16b, v6.16b
118 uabd v3.16b, v3.16b, v7.16b
119 umull2 v4.8h, v0.16b, v0.16b // square
120 umull2 v5.8h, v1.16b, v1.16b
121 umull2 v6.8h, v2.16b, v2.16b
122 umull2 v7.8h, v3.16b, v3.16b
123 umull v0.8h, v0.8b, v0.8b
124 umull v1.8h, v1.8b, v1.8b
125 umull v2.8h, v2.8b, v2.8b
126 umull v3.8h, v3.8b, v3.8b
127 uaddlp v4.4s, v4.8h // accumulate
128 uaddlp v5.4s, v5.8h
129 uaddlp v6.4s, v6.8h
130 uaddlp v7.4s, v7.8h
131 uaddlp v0.4s, v0.8h
132 uaddlp v1.4s, v1.8h
133 uaddlp v2.4s, v2.8h
134 uaddlp v3.4s, v3.8h
135 uaddlp v4.2d, v4.4s
136 uadalp v4.2d, v5.4s
137 uadalp v4.2d, v6.4s
138 uadalp v4.2d, v7.4s
139 xtn v4.2s, v4.2d
140 uaddlp v0.2d, v0.4s
141 uadalp v0.2d, v1.4s
142 uadalp v0.2d, v2.4s
143 uadalp v0.2d, v3.4s
144 xtn v0.2s, v0.2d
145 umull v4.2d, v4.2s, v18.2s // apply scale
146 umull v0.2d, v0.2s, v16.2s
147 urshr v4.2d, v4.2d, #8
148 urshr v0.2d, v0.2d, #8
149 add v17.2d, v17.2d, v4.2d
150 add v17.2d, v17.2d, v0.2d
151 bne L(wsse_w16)
152 RET_SUM
153 endfunc
154
155 .macro LOAD_32X4 vert=1, hbd=0
156 ldp q0, q22, [x0]
157 ldp q4, q26, [x2]
158 add x0, x0, x1
159 add x2, x2, x3
160 ldp q1, q23, [x0]
161 ldp q5, q27, [x2]
162 add x0, x0, x1
163 add x2, x2, x3
164 ldp q2, q24, [x0]
165 ldp q6, q28, [x2]
166 add x0, x0, x1
167 add x2, x2, x3
168 ldp q3, q25, [x0]
169 ldp q7, q29, [x2]
170 add x0, x0, x1
171 add x2, x2, x3
172 .if \vert == 1
173 .if \hbd == 0
174 ldp q16, q19, [x4]
175 .else
176 ldr q16, [x4]
177 .endif
178 add x4, x4, x5
179 subs w10, w10, #4
180 .else
181 sub x0, x0, x1, lsl 2
182 sub x2, x2, x3, lsl 2
183 add x0, x0, #32
184 add x2, x2, #32
185 .if \hbd == 0
186 ldp q16, q19, [x4]
187 add x4, x4, #32
188 .else
189 ldr q16, [x4]
190 add x4, x4, #16
191 .endif
192 .endif
193 mov v18.d[0], v16.d[1]
194 .if \hbd == 0
195 mov v20.d[0], v19.d[1]
196 .endif
197 .endm
198
199 .macro WEIGHTED_SSE_32X4
200 uabd v0.16b, v0.16b, v4.16b // diff pixel values
201 uabd v1.16b, v1.16b, v5.16b
202 uabd v2.16b, v2.16b, v6.16b
203 uabd v3.16b, v3.16b, v7.16b
204 uabd v22.16b, v22.16b, v26.16b
205 uabd v23.16b, v23.16b, v27.16b
206 uabd v24.16b, v24.16b, v28.16b
207 uabd v25.16b, v25.16b, v29.16b
208 umull2 v4.8h, v0.16b, v0.16b // square
209 umull2 v5.8h, v1.16b, v1.16b
210 umull2 v6.8h, v2.16b, v2.16b
211 umull2 v7.8h, v3.16b, v3.16b
212 umull2 v26.8h, v22.16b, v22.16b
213 umull2 v27.8h, v23.16b, v23.16b
214 umull2 v28.8h, v24.16b, v24.16b
215 umull2 v29.8h, v25.16b, v25.16b
216 umull v0.8h, v0.8b, v0.8b
217 umull v1.8h, v1.8b, v1.8b
218 umull v2.8h, v2.8b, v2.8b
219 umull v3.8h, v3.8b, v3.8b
220 umull v22.8h, v22.8b, v22.8b
221 umull v23.8h, v23.8b, v23.8b
222 umull v24.8h, v24.8b, v24.8b
223 umull v25.8h, v25.8b, v25.8b
224 uaddlp v4.4s, v4.8h // accumulate
225 uadalp v4.4s, v5.8h
226 uadalp v4.4s, v6.8h
227 uadalp v4.4s, v7.8h
228 uaddlp v26.4s, v26.8h
229 uadalp v26.4s, v27.8h
230 uadalp v26.4s, v28.8h
231 uadalp v26.4s, v29.8h
232 uaddlp v0.4s, v0.8h
233 uadalp v0.4s, v1.8h
234 uadalp v0.4s, v2.8h
235 uadalp v0.4s, v3.8h
236 uaddlp v22.4s, v22.8h
237 uadalp v22.4s, v23.8h
238 uadalp v22.4s, v24.8h
239 uadalp v22.4s, v25.8h
240 uaddlp v4.2d, v4.4s
241 uaddlp v26.2d, v26.4s
242 uaddlp v0.2d, v0.4s
243 uaddlp v22.2d, v22.4s
244 xtn v4.2s, v4.2d
245 xtn v26.2s, v26.2d
246 xtn v0.2s, v0.2d
247 xtn v22.2s, v22.2d
248 umull v4.2d, v4.2s, v18.2s // apply scale
249 umull v26.2d, v26.2s, v20.2s
250 umull v0.2d, v0.2s, v16.2s
251 umull v22.2d, v22.2s, v19.2s
252 urshr v4.2d, v4.2d, #8
253 urshr v26.2d, v26.2d, #8
254 urshr v0.2d, v0.2d, #8
255 urshr v22.2d, v22.2d, #8
256 add v4.2d, v4.2d, v26.2d
257 add v0.2d, v0.2d, v22.2d
258 add v17.2d, v17.2d, v4.2d
259 add v17.2d, v17.2d, v0.2d
260 .endm
261
262 function weighted_sse_32x32_neon, export=1
263 INIT 32, 32
264 L(wsse_w32):
265 LOAD_32X4
266 WEIGHTED_SSE_32X4
267 bne L(wsse_w32)
268 RET_SUM
269 endfunc
270
271 function weighted_sse_64x64_neon, export=1
272 INIT 64, 64
273 L(wsse_w32up):
274 LOAD_32X4 vert=0
275 WEIGHTED_SSE_32X4
276 subs w8, w8, #32
277 bne L(wsse_w32up)
278 mov w8, w9
279 sub x0, x0, x9
280 sub x2, x2, x9
281 add x0, x0, x1, lsl 2
282 add x2, x2, x3, lsl 2
283 sub x4, x4, x9
284 add x4, x4, x5
285 subs w10, w10, #4
286 bne L(wsse_w32up)
287 RET_SUM
288 endfunc
289
290 .macro weighted_sse width, height
291 function weighted_sse_\width\()x\height\()_neon, export=1
292 INIT \width, \height
293 .if \width <= 32
294 b L(wsse_w\width)
295 .else
296 b L(wsse_w32up)
297 .endif
298 endfunc
299 .endm
300
301 weighted_sse 4, 8
302 weighted_sse 4, 16
303 weighted_sse 8, 4
304 weighted_sse 8, 16
305 weighted_sse 8, 32
306 weighted_sse 16, 4
307 weighted_sse 16, 8
308 weighted_sse 16, 32
309 weighted_sse 16, 64
310 weighted_sse 32, 8
311 weighted_sse 32, 16
312 weighted_sse 32, 64
313 weighted_sse 64, 16
314 weighted_sse 64, 32
315 weighted_sse 64, 128
316 weighted_sse 128, 64
317 weighted_sse 128, 128
318
319 .macro LOAD_X4_HBD t=q
320 ldr \t\()0, [x0]
321 ldr \t\()4, [x2]
322 ldr \t\()1, [x0, x1]
323 ldr \t\()5, [x2, x3]
324 ldr \t\()2, [x0, x11]
325 ldr \t\()6, [x2, x12]
326 ldr \t\()3, [x0, x8]
327 ldr \t\()7, [x2, x9]
328 .ifc \t,q
329 ldr d16, [x4]
330 .else
331 ldr s16, [x4]
332 .endif
333 add x0, x0, x1, lsl 2
334 add x2, x2, x3, lsl 2
335 add x4, x4, x5
336 subs w10, w10, #4
337 .endm
338
339 .macro INIT_HBD width, height
340 .if \width <= 8
341 add x11, x1, x1
342 add x12, x3, x3
343 add x8, x1, x1, lsl 1
344 add x9, x3, x3, lsl 1
345 .elseif \width >= 32
346 mov w8, #(\width)
347 sxtw x9, w8
348 .endif
349 movi v17.4s, #0
350 mov w10, #(\height)
351 .endm
352
353 // x0: src: *const u16,
354 // x1: src_stride: isize,
355 // x2: dst: *const u16,
356 // x3: dst_stride: isize,
357 // x4: scale: *const u32,
358 // x5: scale_stride: isize,
359 function weighted_sse_4x4_hbd_neon, export=1
360 INIT_HBD 4, 4
361 L(wsse_hbd_w4):
362 LOAD_X4_HBD t=d
363 uabd v0.8h, v0.8h, v4.8h // diff pixel values
364 uabd v1.8h, v1.8h, v5.8h
365 uabd v2.8h, v2.8h, v6.8h
366 uabd v3.8h, v3.8h, v7.8h
367 umull v0.4s, v0.4h, v0.4h // square
368 umull v1.4s, v1.4h, v1.4h
369 umull v2.4s, v2.4h, v2.4h
370 umull v3.4s, v3.4h, v3.4h
371 add v0.4s, v0.4s, v1.4s // accumulate
372 add v2.4s, v2.4s, v3.4s
373 add v0.4s, v0.4s, v2.4s
374 addv s0, v0.4s
375 umull v0.2d, v0.2s, v16.2s // apply scale
376 urshr d0, d0, #8
377 add v17.2d, v17.2d, v0.2d
378 bne L(wsse_hbd_w4)
379 fmov x0, d17
380 ret
381 endfunc
382
383 function weighted_sse_8x8_hbd_neon, export=1
384 INIT_HBD 8, 8
385 L(wsse_hbd_w8):
386 LOAD_X4_HBD
387 uabd v4.8h, v0.8h, v4.8h // diff pixel values
388 uabd v5.8h, v1.8h, v5.8h
389 uabd v6.8h, v2.8h, v6.8h
390 uabd v7.8h, v3.8h, v7.8h
391 umull v0.4s, v4.4h, v4.4h // square
392 umull v1.4s, v5.4h, v5.4h
393 umull v2.4s, v6.4h, v6.4h
394 umull v3.4s, v7.4h, v7.4h
395 umull2 v4.4s, v4.8h, v4.8h
396 umull2 v5.4s, v5.8h, v5.8h
397 umull2 v6.4s, v6.8h, v6.8h
398 umull2 v7.4s, v7.8h, v7.8h
399 add v0.4s, v0.4s, v1.4s // accumulate
400 add v2.4s, v2.4s, v3.4s
401 add v4.4s, v4.4s, v5.4s
402 add v6.4s, v6.4s, v7.4s
403 add v0.4s, v0.4s, v2.4s
404 add v4.4s, v4.4s, v6.4s
405 addv s0, v0.4s
406 addv s4, v4.4s
407 mov v0.s[1], v4.s[0]
408 umull v0.2d, v0.2s, v16.2s // apply scale
409 urshr v0.2d, v0.2d, #8
410 add v17.2d, v17.2d, v0.2d
411 bne L(wsse_hbd_w8)
412 RET_SUM
413 endfunc
414
415 .macro WEIGHTED_SSE_16X4_HBD
416 uabd v0.8h, v0.8h, v4.8h // diff pixel values
417 uabd v1.8h, v1.8h, v5.8h
418 uabd v2.8h, v2.8h, v6.8h
419 uabd v3.8h, v3.8h, v7.8h
420 uabd v22.8h, v22.8h, v26.8h
421 uabd v23.8h, v23.8h, v27.8h
422 uabd v24.8h, v24.8h, v28.8h
423 uabd v25.8h, v25.8h, v29.8h
424 umull2 v4.4s, v0.8h, v0.8h // square
425 umull2 v5.4s, v1.8h, v1.8h
426 umull2 v6.4s, v2.8h, v2.8h
427 umull2 v7.4s, v3.8h, v3.8h
428 umull2 v26.4s, v22.8h, v22.8h
429 umull2 v27.4s, v23.8h, v23.8h
430 umull2 v28.4s, v24.8h, v24.8h
431 umull2 v29.4s, v25.8h, v25.8h
432 umull v0.4s, v0.4h, v0.4h
433 umull v1.4s, v1.4h, v1.4h
434 umull v2.4s, v2.4h, v2.4h
435 umull v3.4s, v3.4h, v3.4h
436 umull v22.4s, v22.4h, v22.4h
437 umull v23.4s, v23.4h, v23.4h
438 umull v24.4s, v24.4h, v24.4h
439 umull v25.4s, v25.4h, v25.4h
440 add v0.4s, v0.4s, v1.4s // accumulate
441 add v2.4s, v2.4s, v3.4s
442 add v4.4s, v4.4s, v5.4s
443 add v6.4s, v6.4s, v7.4s
444 add v22.4s, v22.4s, v23.4s
445 add v24.4s, v24.4s, v25.4s
446 add v26.4s, v26.4s, v27.4s
447 add v28.4s, v28.4s, v29.4s
448 add v0.4s, v0.4s, v2.4s
449 add v4.4s, v4.4s, v6.4s
450 add v22.4s, v22.4s, v24.4s
451 add v26.4s, v26.4s, v28.4s
452 addv s0, v0.4s
453 addv s4, v4.4s
454 addv s22, v22.4s
455 addv s26, v26.4s
456 mov v0.s[1], v4.s[0]
457 mov v22.s[1], v26.s[0]
458 umull v0.2d, v0.2s, v16.2s // apply scale
459 umull v22.2d, v22.2s, v18.2s
460 urshr v0.2d, v0.2d, #8
461 urshr v22.2d, v22.2d, #8
462 add v0.2d, v0.2d, v22.2d
463 add v17.2d, v17.2d, v0.2d
464 .endm
465
466 function weighted_sse_16x16_hbd_neon, export=1
467 INIT_HBD 16, 16
468 L(wsse_hbd_w16):
469 LOAD_32X4 vert=1, hbd=1
470 WEIGHTED_SSE_16X4_HBD
471 bne L(wsse_hbd_w16)
472 RET_SUM
473 endfunc
474
475 function weighted_sse_32x32_hbd_neon, export=1
476 INIT_HBD 32, 32
477 L(wsse_hbd_w32up):
478 LOAD_32X4 vert=0, hbd=1
479 WEIGHTED_SSE_16X4_HBD
480 subs w8, w8, #16
481 bne L(wsse_hbd_w32up)
482 mov w8, w9
483 sub x0, x0, x9, lsl 1
484 sub x2, x2, x9, lsl 1
485 add x0, x0, x1, lsl 2
486 add x2, x2, x3, lsl 2
487 sub x4, x4, x9
488 add x4, x4, x5
489 subs w10, w10, #4
490 bne L(wsse_hbd_w32up)
491 RET_SUM
492 endfunc
493
494 .macro weighted_sse_hbd width, height
495 function weighted_sse_\width\()x\height\()_hbd_neon, export=1
496 INIT_HBD \width, \height
497 .if \width <= 16
498 b L(wsse_hbd_w\width)
499 .else
500 b L(wsse_hbd_w32up)
501 .endif
502 endfunc
503 .endm
504
505 weighted_sse_hbd 4, 8
506 weighted_sse_hbd 4, 16
507 weighted_sse_hbd 8, 4
508 weighted_sse_hbd 8, 16
509 weighted_sse_hbd 8, 32
510 weighted_sse_hbd 16, 4
511 weighted_sse_hbd 16, 8
512 weighted_sse_hbd 16, 32
513 weighted_sse_hbd 16, 64
514 weighted_sse_hbd 32, 8
515 weighted_sse_hbd 32, 16
516 weighted_sse_hbd 32, 64
517 weighted_sse_hbd 64, 16
518 weighted_sse_hbd 64, 32
519 weighted_sse_hbd 64, 64
520 weighted_sse_hbd 64, 128
521 weighted_sse_hbd 128, 64
522 weighted_sse_hbd 128, 128